mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
fix(sentry): scrub PII and secrets inside object reprs and nested locals, add SENTRY_SEND_DEFAULT_PII opt-in (#43123)
* fix(sentry): scrub PII and secrets inside object reprs and nested locals, add SENTRY_SEND_DEFAULT_PII opt-in * fix(sentry): keep the SDK denylist and filter the request headers a virtual key arrives in * fix(sentry): leave source context lines unscrubbed * fix(sentry): filter bracketed secret values and cap the JSON walk depth * ci(deps): install sentry-sdk in the proxy-dev group so the unit shards import it * fix(sentry): scrub source-context names outside real stack frames * fix(sentry): tie the key pattern floor to the custom key minimum * fix(sentry): keep the key pattern floor at or below a generated key's length --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
1fd04abb92
commit
a09f8b84a4
8 changed files with 481 additions and 133 deletions
|
|
@ -1952,6 +1952,15 @@ SENTRY_DENYLIST: Final = [
|
|||
"auth_token",
|
||||
"jwt_token",
|
||||
"private_key",
|
||||
"authorization",
|
||||
"api-key",
|
||||
"x-api-key",
|
||||
"x-goog-api-key",
|
||||
"ocp-apim-subscription-key",
|
||||
"x-litellm-api-key",
|
||||
"x-mcp-auth",
|
||||
"cookie",
|
||||
"set-cookie",
|
||||
"SLACK_WEBHOOK_URL",
|
||||
"ALERTING_WEBHOOK_URL",
|
||||
"webhook_url",
|
||||
|
|
@ -1974,6 +1983,12 @@ SENTRY_DENYLIST: Final = [
|
|||
]
|
||||
SENTRY_PII_DENYLIST: Final = [
|
||||
"user_id",
|
||||
"user_email",
|
||||
"end_user_id",
|
||||
"user_api_key_hash",
|
||||
"user_api_key_user_id",
|
||||
"user_api_key_user_email",
|
||||
"user_api_key_end_user_id",
|
||||
"email",
|
||||
"phone",
|
||||
"address",
|
||||
|
|
|
|||
|
|
@ -43,8 +43,6 @@ from litellm.constants import (
|
|||
DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
|
||||
EMPTY_MAPPING,
|
||||
PROVIDER_REQUEST_ID_HEADERS,
|
||||
SENTRY_DENYLIST,
|
||||
SENTRY_PII_DENYLIST,
|
||||
)
|
||||
from litellm.cost_calculator import (
|
||||
RealtimeAPITokenUsageProcessor,
|
||||
|
|
@ -4423,21 +4421,10 @@ def set_callbacks(callback_list, function_id=None):
|
|||
print_verbose("Package 'sentry_sdk' is missing. Installing it...")
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "sentry_sdk"])
|
||||
import sentry_sdk
|
||||
from sentry_sdk.scrubber import EventScrubber
|
||||
from litellm.litellm_core_utils.sentry_scrubbing import build_sentry_init_options
|
||||
|
||||
sentry_sdk_instance = sentry_sdk
|
||||
sentry_trace_rate = os.environ.get("SENTRY_API_TRACE_RATE", "1.0")
|
||||
sentry_sample_rate = (
|
||||
os.environ.get("SENTRY_API_SAMPLE_RATE") if "SENTRY_API_SAMPLE_RATE" in os.environ else "1.0"
|
||||
)
|
||||
sentry_sdk_instance.init(
|
||||
dsn=os.environ.get("SENTRY_DSN"),
|
||||
traces_sample_rate=float(sentry_trace_rate),
|
||||
sample_rate=float(sentry_sample_rate if sentry_sample_rate else 1.0),
|
||||
send_default_pii=False, # Prevent sending Personal Identifiable Information
|
||||
event_scrubber=EventScrubber(denylist=SENTRY_DENYLIST, pii_denylist=SENTRY_PII_DENYLIST),
|
||||
environment=os.environ.get("SENTRY_ENVIRONMENT", "production"),
|
||||
)
|
||||
sentry_sdk_instance.init(**build_sentry_init_options(os.environ))
|
||||
capture_exception = sentry_sdk_instance.capture_exception
|
||||
add_breadcrumb = sentry_sdk_instance.add_breadcrumb
|
||||
elif callback == "slack":
|
||||
|
|
|
|||
152
litellm/litellm_core_utils/sentry_scrubbing.py
Normal file
152
litellm/litellm_core_utils/sentry_scrubbing.py
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from functools import reduce
|
||||
from typing import TYPE_CHECKING, Final, TypeAlias, cast
|
||||
|
||||
from pydantic import JsonValue
|
||||
from sentry_sdk.scrubber import DEFAULT_DENYLIST, DEFAULT_PII_DENYLIST, EventScrubber
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.constants import (
|
||||
LENGTH_OF_LITELLM_GENERATED_KEY,
|
||||
MINIMUM_CUSTOM_KEY_LENGTH,
|
||||
SENTRY_DENYLIST,
|
||||
SENTRY_PII_DENYLIST,
|
||||
)
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sentry_sdk.types import Event, Hint
|
||||
|
||||
EventScrubFn: TypeAlias = "Callable[[Event, Hint], Event]"
|
||||
JsonPath: TypeAlias = tuple[str, ...]
|
||||
|
||||
FILTERED: Final = "[Filtered]"
|
||||
SEND_DEFAULT_PII_ENV: Final = "SENTRY_SEND_DEFAULT_PII"
|
||||
SECRET_FIELD_NAMES: Final = tuple(DEFAULT_DENYLIST) + tuple(SENTRY_DENYLIST)
|
||||
PII_FIELD_NAMES: Final = tuple(DEFAULT_PII_DENYLIST) + tuple(SENTRY_PII_DENYLIST)
|
||||
|
||||
KEY_PREFIX: Final = "sk-"
|
||||
|
||||
|
||||
def build_key_pattern(custom_key_minimum: int, generated_key_bytes: int) -> re.Pattern[str]:
|
||||
generated_suffix_length: Final = (generated_key_bytes * 4 + 2) // 3
|
||||
floor: Final = min(custom_key_minimum - len(KEY_PREFIX), generated_suffix_length)
|
||||
return re.compile(rf"{KEY_PREFIX}[A-Za-z0-9_-]{{{floor},}}")
|
||||
|
||||
|
||||
LITELLM_KEY_PATTERN: Final = build_key_pattern(MINIMUM_CUSTOM_KEY_LENGTH, LENGTH_OF_LITELLM_GENERATED_KEY)
|
||||
SOURCE_CONTEXT_KEYS: Final = frozenset({"pre_context", "context_line", "post_context"})
|
||||
STACK_FRAME_PATHS: Final = frozenset(
|
||||
{
|
||||
("exception", "values", "*", "stacktrace", "frames", "*"),
|
||||
("threads", "values", "*", "stacktrace", "frames", "*"),
|
||||
("stacktrace", "frames", "*"),
|
||||
}
|
||||
)
|
||||
MAX_SCRUB_DEPTH: Final = 64
|
||||
EMAIL_PATTERN: Final = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}")
|
||||
SHA256_HEX_PATTERN: Final = re.compile(r"(?<![0-9A-Za-z])[0-9a-f]{64}(?![0-9A-Za-z])")
|
||||
QUOTED_VALUE: Final = r"'(?:[^'\\]|\\.)*'|\"(?:[^\"\\]|\\.)*\""
|
||||
BRACKET_ATOM: Final = rf"(?:{QUOTED_VALUE})|[^\[\]{{}}()'\"]"
|
||||
NESTED_BRACKET_LEVELS: Final = 3
|
||||
BRACKETED_VALUE: Final = reduce(
|
||||
lambda inner, _: rf"[\[{{(](?:{BRACKET_ATOM}|{inner})*[\]}})]",
|
||||
range(NESTED_BRACKET_LEVELS),
|
||||
rf"[\[{{(](?:{BRACKET_ATOM})*[\]}})]",
|
||||
)
|
||||
BARE_VALUE: Final = r"(?!None(?![0-9A-Za-z_]))[^,)\]}\s]+"
|
||||
|
||||
|
||||
class SentryInitOptions(TypedDict):
|
||||
dsn: ReadOnly[str | None]
|
||||
traces_sample_rate: ReadOnly[float]
|
||||
sample_rate: ReadOnly[float]
|
||||
send_default_pii: ReadOnly[bool]
|
||||
event_scrubber: ReadOnly[EventScrubber]
|
||||
before_send: ReadOnly[EventScrubFn]
|
||||
before_send_transaction: ReadOnly[EventScrubFn]
|
||||
environment: ReadOnly[str]
|
||||
|
||||
|
||||
def build_repr_field_pattern(field_names: Sequence[str]) -> re.Pattern[str]:
|
||||
names: Final = "|".join(re.escape(name) for name in field_names)
|
||||
return re.compile(
|
||||
rf"(?P<field>(?<![0-9A-Za-z_])(?:{names})=|['\"](?:{names})['\"]:\s*)(?P<value>{QUOTED_VALUE}|{BRACKETED_VALUE}|{BARE_VALUE})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def build_string_scrubber(send_default_pii: bool) -> Callable[[str], str]:
|
||||
field_names: Final = SECRET_FIELD_NAMES if send_default_pii else SECRET_FIELD_NAMES + PII_FIELD_NAMES
|
||||
field_pattern: Final = build_repr_field_pattern(field_names)
|
||||
value_patterns: Final = (
|
||||
(LITELLM_KEY_PATTERN,) if send_default_pii else (LITELLM_KEY_PATTERN, EMAIL_PATTERN, SHA256_HEX_PATTERN)
|
||||
)
|
||||
|
||||
def scrub(text: str) -> str:
|
||||
fields_scrubbed: Final = field_pattern.sub(_filtered_field, text)
|
||||
return _substitute_all(value_patterns, fields_scrubbed)
|
||||
|
||||
return scrub
|
||||
|
||||
|
||||
def _filtered_field(match: re.Match[str]) -> str:
|
||||
quote: Final = '"' if match.group("value").startswith('"') else "'"
|
||||
return f"{match.group('field')}{quote}{FILTERED}{quote}"
|
||||
|
||||
|
||||
def _substitute_all(patterns: Sequence[re.Pattern[str]], text: str) -> str:
|
||||
return reduce(lambda scrubbed, pattern: pattern.sub(FILTERED, scrubbed), patterns, text)
|
||||
|
||||
|
||||
def scrub_json_strings(value: JsonValue, scrub: Callable[[str], str], path: JsonPath = ()) -> JsonValue:
|
||||
if len(path) > MAX_SCRUB_DEPTH:
|
||||
return FILTERED
|
||||
if isinstance(value, str):
|
||||
return scrub(value)
|
||||
if isinstance(value, dict):
|
||||
unscrubbed_keys: Final = SOURCE_CONTEXT_KEYS if path in STACK_FRAME_PATHS else frozenset[str]()
|
||||
return { # mutable-ok: JSON object
|
||||
key: item if key in unscrubbed_keys else scrub_json_strings(item, scrub, (*path, key))
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [scrub_json_strings(item, scrub, (*path, "*")) for item in value] # mutable-ok: JSON array
|
||||
return value
|
||||
|
||||
|
||||
def build_event_scrubber(send_default_pii: bool) -> EventScrubFn:
|
||||
scrub: Final = build_string_scrubber(send_default_pii)
|
||||
|
||||
def scrub_event(event: Event, _hint: Hint) -> Event:
|
||||
json_event: Final = cast("JsonValue", event) # cast-ok: [LIT006] the SDK serialized the event to JSON already
|
||||
return cast("Event", scrub_json_strings(json_event, scrub)) # cast-ok: [LIT006] same JSON shape going back
|
||||
|
||||
return scrub_event
|
||||
|
||||
|
||||
def send_default_pii_from_env(env: Mapping[str, str]) -> bool:
|
||||
return str_to_bool(env.get(SEND_DEFAULT_PII_ENV)) is True
|
||||
|
||||
|
||||
def build_sentry_init_options(env: Mapping[str, str]) -> SentryInitOptions:
|
||||
send_default_pii: Final = send_default_pii_from_env(env)
|
||||
scrub_event: Final = build_event_scrubber(send_default_pii)
|
||||
return SentryInitOptions(
|
||||
dsn=env.get("SENTRY_DSN"),
|
||||
traces_sample_rate=float(env.get("SENTRY_API_TRACE_RATE") or "1.0"),
|
||||
sample_rate=float(env.get("SENTRY_API_SAMPLE_RATE") or "1.0"),
|
||||
send_default_pii=send_default_pii,
|
||||
event_scrubber=EventScrubber(
|
||||
denylist=list(SECRET_FIELD_NAMES), # mutable-ok: EventScrubber appends pii_denylist onto denylist in place
|
||||
pii_denylist=list(PII_FIELD_NAMES), # mutable-ok: EventScrubber takes List[str]
|
||||
recursive=True,
|
||||
send_default_pii=send_default_pii,
|
||||
),
|
||||
before_send=scrub_event,
|
||||
before_send_transaction=scrub_event,
|
||||
environment=env.get("SENTRY_ENVIRONMENT", "production"),
|
||||
)
|
||||
|
|
@ -249,6 +249,7 @@ proxy-dev = [
|
|||
"prisma==0.11.0",
|
||||
"hypercorn==0.17.3",
|
||||
"prometheus-client==0.20.0",
|
||||
"sentry-sdk==2.21.0",
|
||||
"opentelemetry-api==1.33.1",
|
||||
"opentelemetry-sdk==1.33.1",
|
||||
"opentelemetry-exporter-otlp==1.33.1",
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ IGNORE_FUNCTIONS = [
|
|||
"_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
|
||||
"_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
|
||||
"_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
|
||||
"scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap.
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from openai._legacy_response import HttpxBinaryResponseContent
|
|||
|
||||
import litellm
|
||||
from litellm._logging import session_id_var, trace_id_var
|
||||
from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST
|
||||
from litellm.constants import SENTRY_PII_DENYLIST
|
||||
from litellm.cost_calculator import ocr_batch_cost
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
|
||||
|
|
@ -357,108 +357,23 @@ def test_post_call_serializes_dict_with_datetime(logging_obj):
|
|||
assert "2026-05-11" in serialized
|
||||
|
||||
|
||||
def test_sentry_sample_rate(monkeypatch):
|
||||
existing_sample_rate = os.getenv("SENTRY_API_SAMPLE_RATE")
|
||||
try:
|
||||
# test with default value by removing the environment variable
|
||||
if existing_sample_rate:
|
||||
del os.environ["SENTRY_API_SAMPLE_RATE"]
|
||||
|
||||
set_callbacks(["sentry"])
|
||||
# Check if the default sample rate is set to 1.0
|
||||
assert os.environ.get("SENTRY_API_SAMPLE_RATE") == "1.0"
|
||||
|
||||
# test with custom value
|
||||
monkeypatch.setenv("SENTRY_API_SAMPLE_RATE", "0.5")
|
||||
|
||||
set_callbacks(["sentry"])
|
||||
# Check if the custom sample rate is set correctly
|
||||
assert os.environ.get("SENTRY_API_SAMPLE_RATE") == "0.5"
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
finally:
|
||||
# Restore the original environment variable
|
||||
if existing_sample_rate:
|
||||
monkeypatch.setenv("SENTRY_API_SAMPLE_RATE", existing_sample_rate)
|
||||
else:
|
||||
if "SENTRY_API_SAMPLE_RATE" in os.environ:
|
||||
del os.environ["SENTRY_API_SAMPLE_RATE"]
|
||||
|
||||
|
||||
def test_sentry_environment(monkeypatch):
|
||||
"""Test that SENTRY_ENVIRONMENT is properly handled during Sentry initialization"""
|
||||
existing_environment = os.getenv("SENTRY_ENVIRONMENT")
|
||||
existing_dsn = os.getenv("SENTRY_DSN")
|
||||
import sentry_sdk
|
||||
|
||||
# Create mock sentry_sdk module
|
||||
mock_event_scrubber_instance = MagicMock()
|
||||
mock_event_scrubber_cls = MagicMock(return_value=mock_event_scrubber_instance)
|
||||
|
||||
mock_scrubber_module = MagicMock()
|
||||
mock_scrubber_module.EventScrubber = mock_event_scrubber_cls
|
||||
|
||||
mock_sentry_sdk = MagicMock()
|
||||
mock_sentry_sdk.scrubber = mock_scrubber_module
|
||||
mock_init = MagicMock()
|
||||
mock_sentry_sdk.init = mock_init
|
||||
monkeypatch.setattr(sentry_sdk, "init", mock_init)
|
||||
monkeypatch.setenv("SENTRY_DSN", "https://test@sentry.io/123456")
|
||||
monkeypatch.delenv("SENTRY_ENVIRONMENT", raising=False)
|
||||
|
||||
# Inject mocks into sys.modules
|
||||
sys.modules["sentry_sdk"] = mock_sentry_sdk
|
||||
sys.modules["sentry_sdk.scrubber"] = mock_scrubber_module
|
||||
|
||||
try:
|
||||
# Set a mock DSN to allow Sentry initialization
|
||||
monkeypatch.setenv("SENTRY_DSN", "https://test@sentry.io/123456")
|
||||
|
||||
# Test with default value (no environment set)
|
||||
if existing_environment:
|
||||
del os.environ["SENTRY_ENVIRONMENT"]
|
||||
set_callbacks(["sentry"])
|
||||
assert mock_init.call_args[1]["environment"] == "production"
|
||||
|
||||
for environment in ("development", "staging"):
|
||||
monkeypatch.setenv("SENTRY_ENVIRONMENT", environment)
|
||||
mock_init.reset_mock()
|
||||
set_callbacks(["sentry"])
|
||||
# Check that init was called with default environment "production"
|
||||
mock_init.assert_called_once()
|
||||
call_kwargs = mock_init.call_args[1]
|
||||
assert call_kwargs["environment"] == "production"
|
||||
|
||||
# Test with custom environment value
|
||||
monkeypatch.setenv("SENTRY_ENVIRONMENT", "development")
|
||||
|
||||
mock_init.reset_mock()
|
||||
set_callbacks(["sentry"])
|
||||
# Check that init was called with custom environment "development"
|
||||
mock_init.assert_called_once()
|
||||
call_kwargs = mock_init.call_args[1]
|
||||
assert call_kwargs["environment"] == "development"
|
||||
|
||||
# Test with staging environment
|
||||
monkeypatch.setenv("SENTRY_ENVIRONMENT", "staging")
|
||||
|
||||
mock_init.reset_mock()
|
||||
set_callbacks(["sentry"])
|
||||
# Check that init was called with custom environment "staging"
|
||||
mock_init.assert_called_once()
|
||||
call_kwargs = mock_init.call_args[1]
|
||||
assert call_kwargs["environment"] == "staging"
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
raise
|
||||
finally:
|
||||
# Restore the original environment variables
|
||||
if existing_environment:
|
||||
monkeypatch.setenv("SENTRY_ENVIRONMENT", existing_environment)
|
||||
else:
|
||||
if "SENTRY_ENVIRONMENT" in os.environ:
|
||||
del os.environ["SENTRY_ENVIRONMENT"]
|
||||
|
||||
if existing_dsn:
|
||||
monkeypatch.setenv("SENTRY_DSN", existing_dsn)
|
||||
else:
|
||||
if "SENTRY_DSN" in os.environ:
|
||||
del os.environ["SENTRY_DSN"]
|
||||
|
||||
|
||||
assert mock_init.call_args[1]["environment"] == environment
|
||||
def test_use_custom_pricing_for_model():
|
||||
from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
|
||||
|
||||
|
|
@ -3100,37 +3015,34 @@ def test_speech_call_is_still_priced_from_input_characters(call_type):
|
|||
|
||||
|
||||
def test_sentry_event_scrubber_initialization(monkeypatch):
|
||||
# Step 1: Create a fake sentry_sdk.scrubber module
|
||||
mock_event_scrubber_instance = MagicMock()
|
||||
mock_event_scrubber_cls = MagicMock(return_value=mock_event_scrubber_instance)
|
||||
import sentry_sdk
|
||||
|
||||
mock_scrubber_module = MagicMock()
|
||||
mock_scrubber_module.EventScrubber = mock_event_scrubber_cls
|
||||
|
||||
# Step 2: Create a fake sentry_sdk module and insert into sys.modules
|
||||
mock_sentry_sdk = MagicMock()
|
||||
mock_sentry_sdk.scrubber = mock_scrubber_module
|
||||
mock_init = MagicMock()
|
||||
mock_sentry_sdk.init = mock_init
|
||||
monkeypatch.setattr(sentry_sdk, "init", mock_init)
|
||||
monkeypatch.delenv("SENTRY_SEND_DEFAULT_PII", raising=False)
|
||||
|
||||
# Step 3: Inject both into sys.modules BEFORE import occurs
|
||||
sys.modules["sentry_sdk"] = mock_sentry_sdk
|
||||
sys.modules["sentry_sdk.scrubber"] = mock_scrubber_module
|
||||
|
||||
# Step 4: Run the actual sentry setup code
|
||||
set_callbacks(["sentry"])
|
||||
|
||||
# Step 5: Assert the EventScrubber was constructed correctly
|
||||
mock_event_scrubber_cls.assert_called_once_with(
|
||||
denylist=SENTRY_DENYLIST,
|
||||
pii_denylist=SENTRY_PII_DENYLIST,
|
||||
)
|
||||
|
||||
# Step 6: Assert the event_scrubber and PII args were passed
|
||||
mock_init.assert_called_once()
|
||||
call_args = mock_init.call_args[1]
|
||||
assert call_args["event_scrubber"] == mock_event_scrubber_instance
|
||||
assert call_args["send_default_pii"] is False
|
||||
assert call_args["event_scrubber"].recursive is True
|
||||
assert {name.lower() for name in SENTRY_PII_DENYLIST} <= {name.lower() for name in call_args["event_scrubber"].denylist}
|
||||
assert call_args["before_send"] is call_args["before_send_transaction"]
|
||||
|
||||
|
||||
def test_sentry_send_default_pii_opt_in(monkeypatch):
|
||||
import sentry_sdk
|
||||
|
||||
mock_init = MagicMock()
|
||||
monkeypatch.setattr(sentry_sdk, "init", mock_init)
|
||||
monkeypatch.setenv("SENTRY_SEND_DEFAULT_PII", "true")
|
||||
|
||||
set_callbacks(["sentry"])
|
||||
|
||||
call_args = mock_init.call_args[1]
|
||||
assert call_args["send_default_pii"] is True
|
||||
assert not {name.lower() for name in SENTRY_PII_DENYLIST} & {name.lower() for name in call_args["event_scrubber"].denylist}
|
||||
|
||||
|
||||
def test_get_masked_values():
|
||||
|
|
|
|||
278
tests/test_litellm/litellm_core_utils/test_sentry_scrubbing.py
Normal file
278
tests/test_litellm/litellm_core_utils/test_sentry_scrubbing.py
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from collections.abc import Callable, Mapping
|
||||
from functools import reduce
|
||||
from typing import Final, cast
|
||||
|
||||
import pytest
|
||||
import sentry_sdk
|
||||
from pydantic import JsonValue
|
||||
from sentry_sdk.envelope import Envelope
|
||||
from sentry_sdk.transport import Transport
|
||||
from sentry_sdk.utils import event_from_exception
|
||||
|
||||
from litellm.constants import LENGTH_OF_LITELLM_GENERATED_KEY, MINIMUM_CUSTOM_KEY_LENGTH
|
||||
from litellm.litellm_core_utils.sentry_scrubbing import (
|
||||
FILTERED,
|
||||
MAX_SCRUB_DEPTH,
|
||||
build_key_pattern,
|
||||
build_sentry_init_options,
|
||||
build_string_scrubber,
|
||||
scrub_json_strings,
|
||||
)
|
||||
from litellm.proxy._types import LiteLLM_UserTable, UserAPIKeyAuth
|
||||
|
||||
EMAIL: Final = "qa.user@example.com"
|
||||
VIRTUAL_KEY: Final = "sk-virtual-key-under-test"
|
||||
KEY_HASH: Final = hashlib.sha256(VIRTUAL_KEY.encode()).hexdigest()
|
||||
MASTER_KEY: Final = "sk-master-key-under-test"
|
||||
DATABASE_URL: Final = "postgresql://litellm:db-password-under-test@db.internal:5432/litellm"
|
||||
PII_ON: Final = {"SENTRY_DSN": "https://key@sentry.example/1", "SENTRY_SEND_DEFAULT_PII": "true"}
|
||||
PII_OFF: Final = {"SENTRY_DSN": "https://key@sentry.example/1"}
|
||||
|
||||
|
||||
class RecordingTransport(Transport):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.last_envelope: Envelope | None = None
|
||||
|
||||
def capture_envelope(self, envelope: Envelope) -> None:
|
||||
self.last_envelope = envelope
|
||||
|
||||
|
||||
def reject_request(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
user_obj: LiteLLM_UserTable,
|
||||
general_settings: Mapping[str, str],
|
||||
data: Mapping[str, Mapping[str, str]],
|
||||
raw_headers: Mapping[str, str],
|
||||
) -> None:
|
||||
raise RuntimeError(f"key {valid_token.token} owned by {user_obj.user_email} was rejected")
|
||||
|
||||
|
||||
def raise_with_identity_locals() -> None:
|
||||
reject_request(
|
||||
valid_token=UserAPIKeyAuth(token=KEY_HASH, key_name="sk-...test", user_id=EMAIL, user_email=EMAIL),
|
||||
user_obj=LiteLLM_UserTable(user_id=EMAIL, user_email=EMAIL, user_role="internal_user"),
|
||||
general_settings={"master_key": MASTER_KEY, "database_url": DATABASE_URL},
|
||||
data={"metadata": {"user_api_key_hash": KEY_HASH, "user_api_key_user_email": EMAIL}},
|
||||
raw_headers={"authorization": f"Bearer {VIRTUAL_KEY}", "x-api-key": VIRTUAL_KEY, "content-type": "application/json"},
|
||||
)
|
||||
|
||||
|
||||
def raise_with_source_context_named_locals() -> None:
|
||||
metadata: Final = {"context_line": f"Bearer {VIRTUAL_KEY}", "pre_context": [EMAIL], "post_context": [KEY_HASH]}
|
||||
stacktrace: Final = {"frames": [{"context_line": MASTER_KEY, "pre_context": [EMAIL]}]}
|
||||
raise RuntimeError(f"rejected with {len(metadata)} metadata fields and {len(stacktrace)} stack fields")
|
||||
|
||||
|
||||
def capture_serialized_event(env: Mapping[str, str], raiser: Callable[[], None] = raise_with_identity_locals) -> str:
|
||||
transport: Final = RecordingTransport()
|
||||
client: Final = sentry_sdk.Client(transport=transport, **build_sentry_init_options(env))
|
||||
try:
|
||||
raiser()
|
||||
except RuntimeError as error:
|
||||
event, hint = event_from_exception(error, client_options=client.options)
|
||||
client.capture_event(event, hint=hint)
|
||||
assert transport.last_envelope is not None
|
||||
return json.dumps(transport.last_envelope.items[0].payload.json)
|
||||
|
||||
|
||||
def innermost_frame_vars(serialized: str) -> dict[str, JsonValue]:
|
||||
event: Final = json.loads(serialized)
|
||||
frames: Final = event["exception"]["values"][0]["stacktrace"]["frames"]
|
||||
return frames[-1]["vars"]
|
||||
|
||||
|
||||
def test_default_event_carries_no_email_hash_or_secret_anywhere() -> None:
|
||||
serialized: Final = capture_serialized_event(PII_OFF)
|
||||
assert EMAIL not in serialized
|
||||
assert KEY_HASH not in serialized
|
||||
assert MASTER_KEY not in serialized
|
||||
assert VIRTUAL_KEY not in serialized
|
||||
assert "db-password-under-test" not in serialized
|
||||
frame_vars: Final = innermost_frame_vars(serialized)
|
||||
assert frame_vars["raw_headers"] == {"authorization": FILTERED, "x-api-key": FILTERED, "content-type": "'application/json'"}
|
||||
assert f"token='{FILTERED}'" in frame_vars["valid_token"]
|
||||
assert f"user_id='{FILTERED}'" in frame_vars["valid_token"]
|
||||
assert f"user_email='{FILTERED}'" in frame_vars["user_obj"]
|
||||
assert frame_vars["general_settings"] == {"master_key": FILTERED, "database_url": FILTERED}
|
||||
assert frame_vars["data"] == {"metadata": {"user_api_key_hash": FILTERED, "user_api_key_user_email": FILTERED}}
|
||||
assert "key_name='sk-...test'" in frame_vars["valid_token"]
|
||||
assert "user_role='internal_user'" in frame_vars["user_obj"]
|
||||
|
||||
|
||||
def test_source_context_lines_are_left_readable() -> None:
|
||||
frames: Final = json.loads(capture_serialized_event(PII_OFF))["exception"]["values"][0]["stacktrace"]["frames"]
|
||||
source_lines: Final = tuple(
|
||||
line
|
||||
for frame in frames
|
||||
for line in (*frame.get("pre_context", []), frame.get("context_line", ""), *frame.get("post_context", []))
|
||||
)
|
||||
assert any("token=KEY_HASH" in line for line in source_lines)
|
||||
assert not any(FILTERED in line for line in source_lines)
|
||||
|
||||
|
||||
def test_source_context_names_outside_stack_frames_are_scrubbed() -> None:
|
||||
serialized: Final = capture_serialized_event(PII_OFF, raise_with_source_context_named_locals)
|
||||
assert VIRTUAL_KEY not in serialized
|
||||
assert MASTER_KEY not in serialized
|
||||
assert EMAIL not in serialized
|
||||
assert KEY_HASH not in serialized
|
||||
frame_vars: Final = innermost_frame_vars(serialized)
|
||||
assert frame_vars["metadata"] == {
|
||||
"context_line": f"'Bearer {FILTERED}'",
|
||||
"pre_context": [f"'{FILTERED}'"],
|
||||
"post_context": [f"'{FILTERED}'"],
|
||||
}
|
||||
assert frame_vars["stacktrace"] == {"frames": [{"context_line": f"'{FILTERED}'", "pre_context": [f"'{FILTERED}'"]}]}
|
||||
innermost_frame: Final = json.loads(serialized)["exception"]["values"][0]["stacktrace"]["frames"][-1]
|
||||
assert "raise RuntimeError" in innermost_frame["context_line"]
|
||||
assert FILTERED not in json.dumps(innermost_frame["pre_context"])
|
||||
|
||||
|
||||
def test_default_event_keeps_the_exception_message_shape() -> None:
|
||||
serialized: Final = capture_serialized_event(PII_OFF)
|
||||
message: Final = json.loads(serialized)["exception"]["values"][0]["value"]
|
||||
assert message == f"key {FILTERED} owned by {FILTERED} was rejected"
|
||||
|
||||
|
||||
def test_pii_opt_in_keeps_identifiers_and_still_scrubs_secrets() -> None:
|
||||
serialized: Final = capture_serialized_event(PII_ON)
|
||||
frame_vars: Final = innermost_frame_vars(serialized)
|
||||
assert f"user_id='{EMAIL}'" in frame_vars["valid_token"]
|
||||
assert f"user_email='{EMAIL}'" in frame_vars["user_obj"]
|
||||
assert frame_vars["data"] == {
|
||||
"metadata": {"user_api_key_hash": f"'{KEY_HASH}'", "user_api_key_user_email": f"'{EMAIL}'"}
|
||||
}
|
||||
assert f"token='{FILTERED}'" in frame_vars["valid_token"]
|
||||
assert frame_vars["general_settings"] == {"master_key": FILTERED, "database_url": FILTERED}
|
||||
assert frame_vars["raw_headers"] == {"authorization": FILTERED, "x-api-key": FILTERED, "content-type": "'application/json'"}
|
||||
assert MASTER_KEY not in serialized
|
||||
assert VIRTUAL_KEY not in serialized
|
||||
assert "db-password-under-test" not in serialized
|
||||
|
||||
|
||||
def test_transaction_events_are_scrubbed_too() -> None:
|
||||
transport: Final = RecordingTransport()
|
||||
client: Final = sentry_sdk.Client(transport=transport, **build_sentry_init_options(PII_OFF))
|
||||
client.capture_event(
|
||||
{
|
||||
"type": "transaction",
|
||||
"transaction": "/user/info",
|
||||
"contexts": {"trace": {"trace_id": "a" * 32, "span_id": "b" * 16}},
|
||||
"spans": [{"description": f"lookup {EMAIL} by {KEY_HASH}", "span_id": "c" * 16, "trace_id": "a" * 32}],
|
||||
}
|
||||
)
|
||||
assert transport.last_envelope is not None
|
||||
serialized: Final = json.dumps(transport.last_envelope.items[0].payload.json)
|
||||
assert EMAIL not in serialized
|
||||
assert KEY_HASH not in serialized
|
||||
assert f"lookup {FILTERED} by {FILTERED}" in serialized
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
(
|
||||
"UserAPIKeyAuth(token='abc', key_alias='team-a', user_id=None)",
|
||||
f"UserAPIKeyAuth(token='{FILTERED}', key_alias='team-a', user_id=None)",
|
||||
),
|
||||
('{"api_key": "sk-1", "model": "gpt-5"}', f'{{"api_key": "{FILTERED}", "model": "gpt-5"}}'),
|
||||
("{'user_id': 'u-1', 'max_budget': 5}", f"{{'user_id': '{FILTERED}', 'max_budget': 5}}"),
|
||||
("Config(OPENAI_API_KEY=sk-live, timeout=10)", f"Config(OPENAI_API_KEY='{FILTERED}', timeout=10)"),
|
||||
("lookup for somebody@example.com failed", f"lookup for {FILTERED} failed"),
|
||||
(f"hashed key {KEY_HASH} not found", f"hashed key {FILTERED} not found"),
|
||||
("request id 0123456789abcdef0123456789abcdef stays", "request id 0123456789abcdef0123456789abcdef stays"),
|
||||
("monkey=banana", "monkey=banana"),
|
||||
(
|
||||
"{'x-api-key': 'k-1', 'cookie': 'session=abc', 'content-type': 'application/json'}",
|
||||
f"{{'x-api-key': '{FILTERED}', 'cookie': '{FILTERED}', 'content-type': 'application/json'}}",
|
||||
),
|
||||
(
|
||||
"headers={'x-tenant-key': 'sk-custom-header-key-0123456789'} key_name='sk-...6789'",
|
||||
f"headers={{'x-tenant-key': '{FILTERED}'}} key_name='sk-...6789'",
|
||||
),
|
||||
(
|
||||
"master_key={'value': 'not-a-litellm-key'} timeout=10",
|
||||
f"master_key='{FILTERED}' timeout=10",
|
||||
),
|
||||
(
|
||||
"credentials=[{'value': ('deep', 'secret')}], model='gpt-5'",
|
||||
f"credentials='{FILTERED}', model='gpt-5'",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_string_scrubber_rewrites_field_and_value_forms(text: str, expected: str) -> None:
|
||||
assert build_string_scrubber(send_default_pii=False)(text) == expected
|
||||
|
||||
|
||||
def test_bare_key_floor_follows_the_custom_key_minimum() -> None:
|
||||
scrub: Final = build_string_scrubber(send_default_pii=False)
|
||||
shortest_key: Final = "sk-" + "a" * (MINIMUM_CUSTOM_KEY_LENGTH - len("sk-"))
|
||||
assert scrub(f"label={shortest_key} model=gpt-5") == f"label={FILTERED} model=gpt-5"
|
||||
assert scrub(f"label={shortest_key[:-1]} model=gpt-5") == f"label={shortest_key[:-1]} model=gpt-5"
|
||||
|
||||
|
||||
def test_key_pattern_floor_never_exceeds_a_generated_key() -> None:
|
||||
generated_key: Final = "sk-" + secrets.token_urlsafe(LENGTH_OF_LITELLM_GENERATED_KEY)
|
||||
stricter_custom_minimum: Final = len(generated_key) + 10
|
||||
assert build_key_pattern(stricter_custom_minimum, LENGTH_OF_LITELLM_GENERATED_KEY).fullmatch(generated_key)
|
||||
assert build_key_pattern(stricter_custom_minimum, LENGTH_OF_LITELLM_GENERATED_KEY).fullmatch(generated_key[:-1]) is None
|
||||
|
||||
|
||||
def test_json_walk_fails_closed_past_the_depth_cap() -> None:
|
||||
scrub: Final = build_string_scrubber(send_default_pii=False)
|
||||
nested: Final = reduce(lambda inner, _: [inner], range(MAX_SCRUB_DEPTH + 1), cast("JsonValue", "api_key=sk-1"))
|
||||
assert FILTERED in json.dumps(scrub_json_strings(nested, scrub))
|
||||
assert "sk-1" not in json.dumps(scrub_json_strings(nested, scrub))
|
||||
assert scrub_json_strings([["api_key=sk-1"]], scrub) == [[f"api_key='{FILTERED}'"]]
|
||||
|
||||
|
||||
def test_string_scrubber_with_pii_on_only_scrubs_secrets() -> None:
|
||||
scrub: Final = build_string_scrubber(send_default_pii=True)
|
||||
assert scrub(f"user_id='{EMAIL}', token='{KEY_HASH}', email {EMAIL} hash {KEY_HASH}") == (
|
||||
f"user_id='{EMAIL}', token='{FILTERED}', email {EMAIL} hash {KEY_HASH}"
|
||||
)
|
||||
assert scrub(f"headers={{'authorization': 'Bearer {VIRTUAL_KEY}'}} sent {VIRTUAL_KEY}") == (
|
||||
f"headers={{'authorization': '{FILTERED}'}} sent {FILTERED}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("env", "expected"),
|
||||
[
|
||||
({}, False),
|
||||
({"SENTRY_SEND_DEFAULT_PII": "true"}, True),
|
||||
({"SENTRY_SEND_DEFAULT_PII": "True"}, True),
|
||||
({"SENTRY_SEND_DEFAULT_PII": "false"}, False),
|
||||
({"SENTRY_SEND_DEFAULT_PII": "yes please"}, False),
|
||||
],
|
||||
)
|
||||
def test_send_default_pii_comes_from_the_environment(env: Mapping[str, str], expected: bool) -> None:
|
||||
assert build_sentry_init_options(env)["send_default_pii"] is expected
|
||||
|
||||
|
||||
def test_init_options_read_dsn_rates_and_environment() -> None:
|
||||
options: Final = build_sentry_init_options(
|
||||
{
|
||||
"SENTRY_DSN": "https://key@sentry.example/7",
|
||||
"SENTRY_API_TRACE_RATE": "0.25",
|
||||
"SENTRY_API_SAMPLE_RATE": "0.5",
|
||||
"SENTRY_ENVIRONMENT": "staging",
|
||||
}
|
||||
)
|
||||
assert options["dsn"] == "https://key@sentry.example/7"
|
||||
assert options["traces_sample_rate"] == 0.25
|
||||
assert options["sample_rate"] == 0.5
|
||||
assert options["environment"] == "staging"
|
||||
assert options["event_scrubber"].recursive is True
|
||||
|
||||
|
||||
def test_init_options_defaults() -> None:
|
||||
options: Final = build_sentry_init_options({})
|
||||
assert options["dsn"] is None
|
||||
assert options["traces_sample_rate"] == 1.0
|
||||
assert options["sample_rate"] == 1.0
|
||||
assert options["environment"] == "production"
|
||||
2
uv.lock
generated
2
uv.lock
generated
|
|
@ -4743,6 +4743,7 @@ proxy-dev = [
|
|||
{ name = "opentelemetry-sdk" },
|
||||
{ name = "prisma" },
|
||||
{ name = "prometheus-client" },
|
||||
{ name = "sentry-sdk" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
|
|
@ -4956,6 +4957,7 @@ proxy-dev = [
|
|||
{ name = "opentelemetry-sdk", specifier = "==1.33.1" },
|
||||
{ name = "prisma", specifier = "==0.11.0" },
|
||||
{ name = "prometheus-client", specifier = "==0.20.0" },
|
||||
{ name = "sentry-sdk", specifier = "==2.21.0" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue