mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge c920180bac into 5114e1c515
This commit is contained in:
commit
2a086a7d09
6 changed files with 193 additions and 17 deletions
|
|
@ -18,7 +18,7 @@ import datetime
|
|||
import os
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime as datetimeObj
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
|
|
@ -287,7 +287,7 @@ class DataDogLogger(
|
|||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
request_data: Mapping[str, object],
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
traceback_str: str | None = None,
|
||||
|
|
@ -303,11 +303,15 @@ class DataDogLogger(
|
|||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
StandardLoggingPayloadSetup,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import scrubbed_error_information_for_request
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
||||
error_information: Final = StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=original_exception,
|
||||
traceback_str=traceback_str,
|
||||
error_information: Final = scrubbed_error_information_for_request(
|
||||
StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=original_exception,
|
||||
traceback_str=traceback_str,
|
||||
),
|
||||
request_data,
|
||||
)
|
||||
_code: Final = error_information.get("error_code") or ""
|
||||
status_code: int | None = None
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
"""``CustomLogger`` adapter on the OpenTelemetry span engine."""
|
||||
|
||||
import sys
|
||||
import weakref
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from collections.abc import Callable, Collection, Iterator, Mapping, Sequence
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import replace
|
||||
from datetime import datetime
|
||||
|
|
@ -106,18 +107,22 @@ def _span_error_from_exception(
|
|||
*,
|
||||
status_code: int | None = None,
|
||||
traceback_str: str | None = None,
|
||||
scrub_texts: Collection[str] | None = None,
|
||||
) -> SpanError:
|
||||
"""A ``SpanError`` for a proxy-level failure that never produced a
|
||||
``StandardLoggingPayload`` (auth / validation / malformed-body rejections),
|
||||
mirroring ``_parse_error``'s field mapping so it stamps the same v2 keys a
|
||||
failed LLM call does. ``status_code`` pins ``error.code`` to the real response
|
||||
status, matching v1's SERVER-span behavior."""
|
||||
status, matching v1's SERVER-span behavior. ``scrub_texts`` is the request's
|
||||
own text to scrub out of the error, set when its message logging is redacted."""
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.redact_messages import scrubbed_error_information
|
||||
|
||||
info: Final = StandardLoggingPayloadSetup.get_error_information(
|
||||
raw_info: Final = StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=exception,
|
||||
traceback_str=traceback_str,
|
||||
)
|
||||
info: Final = raw_info if scrub_texts is None else scrubbed_error_information(raw_info, scrub_texts)
|
||||
return SpanError(
|
||||
error_type=info.get("error_class") or info.get("error_code") or None,
|
||||
message=info.get("error_message") or None,
|
||||
|
|
@ -127,6 +132,10 @@ def _span_error_from_exception(
|
|||
)
|
||||
|
||||
|
||||
# The request text each failure hook scrubbed out of its span's error, or None when the request isn't redacted.
|
||||
_scrub_texts_by_span: Final[weakref.WeakKeyDictionary[Span, frozenset[str] | None]] = weakref.WeakKeyDictionary()
|
||||
|
||||
|
||||
# Any callback whose class belongs to one of these modules is "the OTel
|
||||
# callback" for proxy-global-registration purposes.
|
||||
_OTEL_MODULES: Final = (
|
||||
|
|
@ -843,19 +852,20 @@ class OpenTelemetryV2(CustomLogger):
|
|||
this span already — ``async_post_call_failure_hook`` and the ``auth`` phase
|
||||
span record their own, and a second event would duplicate it — while the
|
||||
attributes are always restamped so ``error.code`` stays pinned to the real
|
||||
response status."""
|
||||
response status. The restamp scrubs the text of the request the failure hook
|
||||
stamped this span for, as the hook did."""
|
||||
if span is None or not is_recordable_span(span):
|
||||
return
|
||||
already_stamped: Final = Error.TYPE in (getattr(span, "attributes", None) or ())
|
||||
stamp_error(
|
||||
span,
|
||||
_span_error_from_exception(exception, status_code=status_code),
|
||||
_span_error_from_exception(exception, status_code=status_code, scrub_texts=_scrub_texts_by_span.get(span)),
|
||||
record_event=not already_stamped,
|
||||
)
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
request_data: Mapping[str, object],
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
traceback_str: "str | None" = None,
|
||||
|
|
@ -875,7 +885,14 @@ class OpenTelemetryV2(CustomLogger):
|
|||
span: Final = mcp_message_transport_span() or request_root_span() or user_api_key_dict.parent_otel_span
|
||||
if span is None or not is_recordable_span(span):
|
||||
return
|
||||
stamp_error(span, _span_error_from_exception(original_exception, traceback_str=traceback_str))
|
||||
from litellm.litellm_core_utils.redact_messages import failure_request_texts, should_redact_failed_request
|
||||
|
||||
scrub_texts: Final = failure_request_texts(request_data) if should_redact_failed_request(request_data) else None
|
||||
_scrub_texts_by_span[span] = scrub_texts
|
||||
stamp_error(
|
||||
span,
|
||||
_span_error_from_exception(original_exception, traceback_str=traceback_str, scrub_texts=scrub_texts),
|
||||
)
|
||||
return
|
||||
|
||||
def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None:
|
||||
|
|
|
|||
|
|
@ -22,7 +22,10 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
|
||||
from litellm.litellm_core_utils.redact_messages import MESSAGE_REDACTION_ENABLE_HEADERS
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
MESSAGE_REDACTION_ENABLE_HEADERS,
|
||||
scrubbed_error_information_for_request,
|
||||
)
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
|
|
@ -442,9 +445,13 @@ class _ProxyDBLogger(CustomLogger):
|
|||
)
|
||||
_metadata["user_api_key"] = LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict)
|
||||
_metadata["status"] = "failure"
|
||||
_error_information = StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=original_exception,
|
||||
traceback_str=traceback_str,
|
||||
request_fields: Final = _str_keyed_fields(request_data)
|
||||
_error_information = scrubbed_error_information_for_request(
|
||||
StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=original_exception,
|
||||
traceback_str=traceback_str,
|
||||
),
|
||||
request_fields,
|
||||
)
|
||||
if should_suppress_spend_log_tracebacks():
|
||||
# Drop the traceback key entirely so the per-row Metadata pane in
|
||||
|
|
@ -466,7 +473,6 @@ class _ProxyDBLogger(CustomLogger):
|
|||
metadata=_metadata, original_exception=original_exception
|
||||
)
|
||||
|
||||
request_fields: Final = _str_keyed_fields(request_data)
|
||||
_metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route)
|
||||
|
||||
existing_metadata: Final[dict] = request_data.get("metadata", None) or {}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import json
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -6,6 +7,7 @@ import pytest
|
|||
from httpx import Request, Response
|
||||
from pydantic import BaseModel, computed_field
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.datadog.datadog import DataDogLogger
|
||||
from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError
|
||||
from litellm.types.integrations.datadog import (
|
||||
|
|
@ -558,3 +560,26 @@ async def test_raised_intake_error_preserves_datadog_requeue_behavior(datadog_en
|
|||
await logger.async_send_batch()
|
||||
|
||||
assert [event["message"] for event in logger.log_queue] == ['{"event": 0}', '{"event": 1}']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"])
|
||||
async def test_failure_hook_scrubs_the_prompt_out_of_the_logged_exception(datadog_env, monkeypatch, redact):
|
||||
secret = "my ssn is 123-45-6789"
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", redact)
|
||||
with patch("asyncio.create_task"):
|
||||
logger = DataDogLogger()
|
||||
|
||||
await logger.async_post_call_failure_hook(
|
||||
request_data={"messages": [{"role": "user", "content": secret}], "metadata": {}},
|
||||
original_exception=litellm.BadRequestError(
|
||||
message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai"
|
||||
),
|
||||
user_api_key_dict=type("UserKey", (), {})(),
|
||||
traceback_str=f"Traceback: {secret}",
|
||||
)
|
||||
|
||||
message = json.loads(logger.log_queue[0]["message"])
|
||||
assert (secret in json.dumps(message)) is not redact
|
||||
assert "Unsupported content: " in message["exception"]
|
||||
assert message["error_class"] == "BadRequestError"
|
||||
|
|
|
|||
|
|
@ -10,8 +10,10 @@ import ast
|
|||
import asyncio
|
||||
import contextlib
|
||||
import contextvars
|
||||
import gc
|
||||
import os
|
||||
import re
|
||||
import weakref
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
|
@ -1548,6 +1550,101 @@ def test_record_error_attributes_on_span_decorates_without_ending():
|
|||
assert [e.name for e in span.events] == ["exception"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"])
|
||||
def test_failure_hook_scrubs_the_prompt_out_of_the_server_span_and_the_restamp_keeps_it(monkeypatch, redact):
|
||||
import json
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
secret = "my ssn is 123-45-6789"
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", redact)
|
||||
logger, exporter = _logger()
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
set_request_root_span(server)
|
||||
exc = _proxy_exc(f"blocked: {secret}", 400)
|
||||
asyncio.run(
|
||||
logger.async_post_call_failure_hook(
|
||||
request_data={"messages": [{"role": "user", "content": secret}]},
|
||||
original_exception=exc,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
)
|
||||
)
|
||||
logger.record_error_attributes_on_span(server, exc, 400)
|
||||
server.end()
|
||||
(span,) = exporter.get_finished_spans()
|
||||
logged = json.dumps(
|
||||
[dict(span.attributes), [dict(event.attributes or {}) for event in span.events], span.status.description]
|
||||
)
|
||||
assert (secret in logged) is not redact
|
||||
assert span.attributes["error.message"].startswith("blocked: ")
|
||||
assert span.attributes["error.type"] == "ProxyException"
|
||||
|
||||
|
||||
def _raised(exc):
|
||||
try:
|
||||
raise exc
|
||||
except Exception as raised:
|
||||
return raised
|
||||
|
||||
|
||||
@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"])
|
||||
def test_restamp_shows_the_handlers_own_error_with_the_prompt_scrubbed(monkeypatch, redact):
|
||||
"""The proxy's failure hook and its exception handler stamp the same SERVER span from different
|
||||
exceptions. The handler's restamp keeps its own wording and stack trace, scrubbed like the hook's."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
secret = "my ssn is 123-45-6789"
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", redact)
|
||||
logger, exporter = _logger()
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
set_request_root_span(server)
|
||||
asyncio.run(
|
||||
logger.async_post_call_failure_hook(
|
||||
request_data={"messages": [{"role": "user", "content": secret}]},
|
||||
original_exception=HTTPException(status_code=400, detail={"error": f"blocked: {secret}", "mode": "pre"}),
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
traceback_str=f' File "/srv/litellm/router.py", line 7, in route\nHTTPException: blocked: {secret}\n',
|
||||
)
|
||||
)
|
||||
logger.record_error_attributes_on_span(server, _raised(_proxy_exc(f"blocked: {secret}", 500)), 500)
|
||||
server.end()
|
||||
(span,) = exporter.get_finished_spans()
|
||||
|
||||
assert span.attributes["error.message"] == f"blocked: {'redacted-by-litellm' if redact else secret}"
|
||||
assert "_raised" in span.attributes["litellm.provider.error.stack_trace"]
|
||||
assert "router.py" not in span.attributes["litellm.provider.error.stack_trace"]
|
||||
|
||||
|
||||
def test_failure_hook_lets_the_failed_requests_span_be_freed(monkeypatch):
|
||||
"""The proxy keeps the request's root span in the request's own metadata. What the failure hook
|
||||
keeps for the handler's restamp must not hold that span alive once the request is done."""
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
logger, _ = _logger()
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
server_ref: Final = weakref.ref(server)
|
||||
asyncio.run(
|
||||
logger.async_post_call_failure_hook(
|
||||
request_data={
|
||||
"messages": [{"role": "user", "content": "my ssn is 123-45-6789"}],
|
||||
"metadata": {"litellm_parent_otel_span": server},
|
||||
},
|
||||
original_exception=_proxy_exc("blocked", 400),
|
||||
user_api_key_dict=UserAPIKeyAuth(parent_otel_span=server),
|
||||
)
|
||||
)
|
||||
del server
|
||||
gc.collect()
|
||||
|
||||
assert server_ref() is None
|
||||
|
||||
|
||||
def test_record_error_attributes_on_span_does_not_duplicate_an_already_stamped_error():
|
||||
"""A failure that already went through ``async_post_call_failure_hook`` reaches
|
||||
the exception handler too; the second stamp must keep one exception event while
|
||||
|
|
|
|||
|
|
@ -3724,3 +3724,30 @@ async def test_async_post_call_failure_hook_disable_pre_routing_error_logs(
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
assert len(writer.rows) == rows_written
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"])
|
||||
async def test_async_post_call_failure_hook_scrubs_the_prompt_out_of_the_persisted_error(
|
||||
monkeypatch: pytest.MonkeyPatch, redact: bool
|
||||
):
|
||||
secret: Final = "my ssn is 123-45-6789"
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", redact)
|
||||
spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True)
|
||||
|
||||
await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook(
|
||||
request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": secret}], "metadata": {}},
|
||||
original_exception=litellm.BadRequestError(
|
||||
message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai"
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
|
||||
traceback_str=f"Traceback: {secret}",
|
||||
)
|
||||
|
||||
persisted: Final = spend_writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][
|
||||
"error_information"
|
||||
]
|
||||
assert (secret in json.dumps(persisted)) is not redact
|
||||
assert "Unsupported content: " in persisted["error_message"]
|
||||
assert persisted["error_class"] == "BadRequestError"
|
||||
assert persisted["error_code"] == "400"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue