diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json index 996e440d77d..cedbbd0f07b 100644 --- a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json +++ b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json @@ -6563,6 +6563,87 @@ ], "title": "litellm_project_model_rate_limit_used_metric", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 447 + }, + "id": 115, + "panels": [], + "title": "Agent tracing export (needs LITELLM_LENS_URL)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Request records the Lens exporter delivered to the Lens service against the records it dropped before the service accepted them", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 448 + }, + "id": 116, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(litellm_lens_export_rows_written_total[$__rate_interval]))", + "legendFormat": "written", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(litellm_lens_export_rows_dropped_total[$__rate_interval]))", + "legendFormat": "dropped", + "range": true, + "refId": "B" + } + ], + "title": "litellm_lens_export_rows_written and litellm_lens_export_rows_dropped rate", + "type": "timeseries" } ], "preload": false, diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md index 5d3d2c159c1..dcac5db6dc0 100644 --- a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md +++ b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md @@ -1,10 +1,10 @@ # LiteLLM All Prometheus Metrics dashboard -Every `litellm_*` metric family the proxy can expose on `/metrics` (141 families across 97 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about +Every `litellm_*` metric family the proxy can expose on `/metrics` (143 families across 98 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes), project model rate limits, and the agent tracing export to Lens. Panel titles are the metric names so you can grep the JSON for the metric you care about Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/unit/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard -The first eleven rows need only `callbacks: ["prometheus"]`. The last three rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected +The first eleven rows and the project model rate limits row need only `callbacks: ["prometheus"]`. The other rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, the agent tracing export row needs `LITELLM_LENS_URL`, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected ## Pre-requisites diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0cda24c853b..7038cc531f5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4507,6 +4507,32 @@ def get_masked_values( _get_masked_values = get_masked_values +@lru_cache(maxsize=1) +def start_sentry(dsn: str) -> None: + try: + import sentry_sdk + except ImportError: + print_verbose("Package 'sentry_sdk' is missing. Installing it...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "sentry_sdk"]) + import sentry_sdk + from litellm.litellm_core_utils.sentry_scrubbing import build_sentry_init_options + + sentry_sdk.init(**build_sentry_init_options({**os.environ, "SENTRY_DSN": dsn})) + + +def initialize_sentry() -> None: + global sentry_sdk_instance, capture_exception, add_breadcrumb + dsn: Final = os.environ.get("SENTRY_DSN") + if not dsn: + return + start_sentry(dsn) + import sentry_sdk + + sentry_sdk_instance = sentry_sdk # rebind-ok: process-wide Sentry SDK handle read by every logging path + capture_exception = sentry_sdk.capture_exception # rebind-ok: process-wide Sentry SDK handle + add_breadcrumb = sentry_sdk.add_breadcrumb # rebind-ok: process-wide Sentry SDK handle + + def set_callbacks( callback_list: Iterable[str | Callable[..., object] | CustomLogger], function_id: str | None = None, @@ -4515,9 +4541,6 @@ def set_callbacks( Globally sets the callback client """ global \ - sentry_sdk_instance, \ - capture_exception, \ - add_breadcrumb, \ slack_app, \ alerts_channel, \ traceloopLogger, \ @@ -4541,18 +4564,7 @@ def set_callbacks( try: for callback in callback_list: if callback == "sentry": - try: - import sentry_sdk - except ImportError: - print_verbose("Package 'sentry_sdk' is missing. Installing it...") - subprocess.check_call([sys.executable, "-m", "pip", "install", "sentry_sdk"]) - import sentry_sdk - from litellm.litellm_core_utils.sentry_scrubbing import build_sentry_init_options - - sentry_sdk_instance = sentry_sdk - sentry_sdk_instance.init(**build_sentry_init_options(os.environ)) - capture_exception = sentry_sdk_instance.capture_exception - add_breadcrumb = sentry_sdk_instance.add_breadcrumb + initialize_sentry() elif callback == "slack": try: from slack_bolt import App diff --git a/litellm/litellm_core_utils/sentry_scrubbing.py b/litellm/litellm_core_utils/sentry_scrubbing.py index 7147f792411..b2c5b1f5712 100644 --- a/litellm/litellm_core_utils/sentry_scrubbing.py +++ b/litellm/litellm_core_utils/sentry_scrubbing.py @@ -3,7 +3,7 @@ from __future__ import annotations import re from collections.abc import Callable, Mapping, Sequence from functools import reduce -from typing import TYPE_CHECKING, Final, TypeAlias, cast +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast from pydantic import JsonValue from sentry_sdk.scrubber import DEFAULT_DENYLIST, DEFAULT_PII_DENYLIST, EventScrubber @@ -65,6 +65,8 @@ class SentryInitOptions(TypedDict): traces_sample_rate: ReadOnly[float] sample_rate: ReadOnly[float] send_default_pii: ReadOnly[bool] + max_request_body_size: ReadOnly[Literal["never"]] + include_local_variables: ReadOnly[bool] event_scrubber: ReadOnly[EventScrubber] before_send: ReadOnly[EventScrubFn] before_send_transaction: ReadOnly[EventScrubFn] @@ -140,6 +142,8 @@ def build_sentry_init_options(env: Mapping[str, str]) -> SentryInitOptions: traces_sample_rate=float(env.get("SENTRY_API_TRACE_RATE") or "1.0"), sample_rate=float(env.get("SENTRY_API_SAMPLE_RATE") or "1.0"), send_default_pii=send_default_pii, + max_request_body_size="never", + include_local_variables=send_default_pii, event_scrubber=EventScrubber( denylist=list(SECRET_FIELD_NAMES), pii_denylist=list(PII_FIELD_NAMES), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 60842648d16..fb6a3a8a88d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -90,6 +90,7 @@ from litellm.litellm_core_utils.bug_report import ( ) from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, + initialize_sentry, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -4962,6 +4963,16 @@ def _is_remote_module_url(value: object) -> bool: return isinstance(value, str) and (value.startswith("s3://") or value.startswith("gcs://")) +def sentry_is_configured(general_settings: Mapping[str, object]) -> bool: + alerting: Final = general_settings.get("alerting") + return ( + "sentry" in litellm.input_callback + or "sentry" in litellm.success_callback + or "sentry" in litellm.failure_callback + or (isinstance(alerting, list) and "sentry" in alerting) + ) + + def _scrub_guardrail_inner(inner: dict[str, JsonValue]) -> None: """Strip remote-URL entries from a guardrail's ``callbacks`` list and ``guardrail`` (v2 module-path) field. Mutates in place.""" @@ -6911,6 +6922,9 @@ class ProxyConfig: if general_settings is None: general_settings = {} + if sentry_is_configured(general_settings): + initialize_sentry() + typed_general_settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings) ConfigGeneralSettings.model_validate( MappingProxyType( diff --git a/litellm/proxy/tracing_runtime.py b/litellm/proxy/tracing_runtime.py index 8de151421f2..0ac05216a30 100644 --- a/litellm/proxy/tracing_runtime.py +++ b/litellm/proxy/tracing_runtime.py @@ -46,10 +46,8 @@ async def manage_tracing( return try: connection: Final = LensConnection.from_env() - except ValueError: - verbose_proxy_logger.warning( - "Agent tracing unavailable: configure LITELLM_LENS_URL and LITELLM_LENS_SERVICE_TOKEN" - ) + except ValueError as error: + verbose_proxy_logger.error("Agent tracing store lens failed to start, tracing is disabled: %s", error) yield None return async with client_factory(connection) as client: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 699fde28d4d..9788c4635c9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3250,14 +3250,14 @@ class ProxyLogging: alerting_metadata=alerting_metadata, **extra_kwargs, ) - for client in self.alerting: - if client == "sentry": - if litellm.utils.sentry_sdk_instance is not None: - litellm.utils.sentry_sdk_instance.capture_message(formatted_message) - else: - raise Exception("Missing SENTRY_DSN from environment") + if "sentry" in self.alerting: + litellm.litellm_core_utils.litellm_logging.initialize_sentry() + sentry: Final = litellm.litellm_core_utils.litellm_logging.sentry_sdk_instance + if sentry is None: + raise Exception("Missing SENTRY_DSN from environment") + sentry.capture_message(formatted_message) - async def failure_handler(self, original_exception, duration: float, call_type: str, traceback_str=""): + async def failure_handler(self, original_exception: Exception, duration: float, call_type: str, traceback_str=""): """ Log failed db read/writes @@ -3298,8 +3298,9 @@ class ProxyLogging: call_type=call_type, ) - if litellm.utils.capture_exception: - litellm.utils.capture_exception(error=original_exception) + capture: Final = litellm.litellm_core_utils.litellm_logging.capture_exception + if capture is not None: + capture(error=original_exception) async def post_call_failure_hook( self, diff --git a/litellm/tracing/exporter.py b/litellm/tracing/exporter.py index 5025b5eae2a..68916558443 100644 --- a/litellm/tracing/exporter.py +++ b/litellm/tracing/exporter.py @@ -3,9 +3,11 @@ import json from collections import deque from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from contextlib import suppress +from dataclasses import dataclass from enum import Enum +from functools import cache from io import BytesIO -from typing import Final +from typing import TYPE_CHECKING, Final import httpx from pydantic import TypeAdapter, ValidationError @@ -16,6 +18,9 @@ from litellm.integrations.clickhouse.clickhouse_spend_logger import spend_log_ro from litellm.integrations.custom_logger import CustomLogger from litellm.tracing.types import SpendLogPayload +if TYPE_CHECKING: + from prometheus_client import Counter as PrometheusCounter + MAX_EVENT_BYTES: Final = 1024 * 1024 MAX_BUFFER_BYTES: Final = 32 * 1024 * 1024 MAX_BUFFER_EVENTS: Final = 1000 @@ -89,11 +94,49 @@ def encode_record(value: Mapping[str, object]) -> bytes | ExportFailure: return ExportFailure.INVALID +@dataclass(frozen=True, slots=True) +class ExportCounters: + written: "PrometheusCounter | None" + dropped: "PrometheusCounter | None" + + def add_written(self, count: int) -> None: + if self.written is not None: + self.written.inc(count) + + def add_dropped(self, count: int) -> None: + if self.dropped is not None: + self.dropped.inc(count) + + +@cache +def lens_export_counters() -> ExportCounters: + try: + from prometheus_client import Counter + except ImportError: + return ExportCounters(written=None, dropped=None) + return ExportCounters( + written=Counter( + "litellm_lens_export_rows_written_total", + "Request records the Lens exporter delivered to the Lens service", + ), + dropped=Counter( + "litellm_lens_export_rows_dropped_total", + "Request records the Lens exporter dropped before the Lens service accepted them", + ), + ) + + class LensExporter(CustomLogger): - def __init__(self, client: httpx.AsyncClient, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep) -> None: + def __init__( + self, + client: httpx.AsyncClient, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + counters: ExportCounters | None = None, + ) -> None: super().__init__() self.client: Final = client self.sleep: Final = sleep + self.counters: Final = counters if counters is not None else lens_export_counters() self.queue: Final[deque[bytes]] = deque() # mutable-ok: bounded producer-consumer queue self.wake: Final = asyncio.Event() self.closed = False @@ -108,6 +151,14 @@ class LensExporter(CustomLogger): if self.task is None: self.task = asyncio.create_task(self._run()) + def _wrote(self, count: int) -> None: + self.rows_written += count + self.counters.add_written(count) + + def _drop(self, count: int) -> None: + self.rows_dropped += count + self.counters.add_dropped(count) + def enqueue(self, record: bytes) -> bool: if ( self.closed @@ -115,7 +166,7 @@ class LensExporter(CustomLogger): or self.buffered_events >= MAX_BUFFER_EVENTS or self.buffered_bytes + len(record) > MAX_BUFFER_BYTES ): - self.rows_dropped += 1 + self._drop(1) return False self.queue.append(record) self.buffered_events += 1 @@ -138,13 +189,13 @@ class LensExporter(CustomLogger): if raw is None or self.closed: return if self.buffered_events >= MAX_BUFFER_EVENTS or self.buffered_bytes >= MAX_BUFFER_BYTES: - self.rows_dropped += 1 + self._drop(1) return try: checked: Final = _check_size(raw, MAX_EVENT_BYTES) if isinstance(checked, ExportFailure): - self.rows_dropped += 1 - self._warn(checked.value) + self._drop(1) + self._log_failure(checked.value) return payload: Final = _PAYLOAD.validate_python(raw) if str(payload.get("call_type", "")).startswith(("/v1/traces", "/v1/logs")): @@ -152,24 +203,26 @@ class LensExporter(CustomLogger): row: Final = spend_log_row_from_payload(payload, kwargs) record: Final = encode_record(row) if isinstance(record, ExportFailure): - self.rows_dropped += 1 - self._warn(record.value) + self._drop(1) + self._log_failure(record.value) return self.enqueue(record) except ValidationError as error: - self.rows_dropped += 1 + self._drop(1) fields: Final = tuple( str(issue["loc"][0]) if issue["loc"] else "$" for issue in error.errors(include_input=False, include_context=False, include_url=False)[:5] ) - self._warn("invalid request record fields: " + ", ".join(fields)) + self._log_failure("invalid request record fields: " + ", ".join(fields)) except (ValueError, TypeError, OverflowError, RecursionError) as error: - self.rows_dropped += 1 - self._warn(type(error).__name__) + self._drop(1) + self._log_failure(type(error).__name__) - def _warn(self, reason: str) -> None: + def _log_failure(self, reason: str) -> None: if reason != self.last_error: - verbose_proxy_logger.warning("Lens request export failed (%s); model requests continue", reason) + verbose_proxy_logger.error( + "Agent tracing store lens dropped request records (%s); model requests continue", reason + ) self.last_error = reason def _batch(self) -> tuple[bytes, ...]: @@ -196,13 +249,13 @@ class LensExporter(CustomLogger): self.last_error = "" return True if response.status_code not in (429, 502, 503, 504): - self._warn(f"HTTP {response.status_code}") + self._log_failure(f"HTTP {response.status_code}") return False except httpx.HTTPError: pass if attempt < 2: await self.sleep(float(1 << attempt)) - self._warn("retry limit reached") + self._log_failure("retry limit reached") return False async def _run(self) -> None: @@ -217,11 +270,11 @@ class LensExporter(CustomLogger): batch: Final = self._batch() try: if await self._send(batch): - self.rows_written += len(batch) + self._wrote(len(batch)) else: - self.rows_dropped += len(batch) + self._drop(len(batch)) except asyncio.CancelledError: - self.rows_dropped += len(batch) + self._drop(len(batch)) raise finally: self.buffered_events -= len(batch) @@ -237,7 +290,7 @@ class LensExporter(CustomLogger): self.task.cancel() with suppress(asyncio.CancelledError): await self.task - self.rows_dropped += len(self.queue) + self._drop(len(self.queue)) self.queue.clear() self.buffered_bytes = 0 self.buffered_events = 0 diff --git a/litellm/types/router.py b/litellm/types/router.py index 8b53b7b6e12..0ca39e2400d 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1084,6 +1084,8 @@ class RouterRateLimitErrorBasic(ValueError): Raise a basic error inside helper functions. """ + status_code: Final = 429 + def __init__( self, model: str, @@ -1099,6 +1101,8 @@ class RouterErrorTypes(str, enum.Enum): class RouterRateLimitError(ValueError): + status_code: Final = 429 + def __init__( self, model: str, diff --git a/tests/unit/integrations/test_prometheus_metric_name_consistency.py b/tests/unit/integrations/test_prometheus_metric_name_consistency.py index 4db34274e82..2c1def4c613 100644 --- a/tests/unit/integrations/test_prometheus_metric_name_consistency.py +++ b/tests/unit/integrations/test_prometheus_metric_name_consistency.py @@ -26,6 +26,7 @@ from litellm.integrations.prometheus_services import PrometheusServicesLogger from litellm.proxy.db.db_transaction_queue.spend_log_cleanup_metrics import SpendLogCleanupMetrics from litellm.proxy.middleware.admission_control_middleware import create_prometheus_admission_metrics from litellm.proxy.middleware.in_flight_requests_middleware import InFlightRequestsMiddleware +from litellm.tracing.exporter import lens_export_counters _GRAFANA_DIR: Final = Path(__file__).parents[3] / "cookbook" / "litellm_proxy_server" / "grafana_dashboard" _ALL_METRICS_DASHBOARD: Final = _GRAFANA_DIR / "dashboard_all_metrics" / "grafana_dashboard.json" @@ -62,6 +63,9 @@ def _lazy_owner_collectors() -> tuple[Collector, ...]: in_flight: Final = InFlightRequestsMiddleware._get_gauge() assert in_flight is not None services: Final = PrometheusServicesLogger() + lens: Final = lens_export_counters() + assert lens.written is not None + assert lens.dropped is not None return ( SpendLogCleanupMetrics.rows_deleted, SpendLogCleanupMetrics.batch_duration, @@ -72,6 +76,8 @@ def _lazy_owner_collectors() -> tuple[Collector, ...]: services.redis_breaker_state, services.redis_breaker_transitions, services.redis_breaker_failures, + lens.written, + lens.dropped, ) diff --git a/tests/unit/litellm_core_utils/test_bug_report.py b/tests/unit/litellm_core_utils/test_bug_report.py index 62d7090b960..03d6c800b60 100644 --- a/tests/unit/litellm_core_utils/test_bug_report.py +++ b/tests/unit/litellm_core_utils/test_bug_report.py @@ -26,6 +26,7 @@ from litellm.litellm_core_utils.bug_report import ( strip_bug_report_notice, ) from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic def test_build_bug_report_keeps_only_litellm_frames(): @@ -138,6 +139,8 @@ def test_bug_report_can_be_disabled(monkeypatch: pytest.MonkeyPatch): request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), ), BadRequestError(message="bad input", llm_provider="openai", model="gpt-4"), + RouterRateLimitError(model="gpt-4", cooldown_time=5.0, enable_pre_call_checks=False, cooldown_list=[]), + RouterRateLimitErrorBasic(model="gpt-4"), "not an exception", ], ) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 12cc57ada4c..14334690624 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -32,6 +32,7 @@ from litellm.caching.caching import DualCache from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE, REDACTED_BY_LITELLM, SENTRY_PII_DENYLIST from litellm.cost_calculator import ocr_batch_cost from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.litellm_logging import ( Logging, @@ -422,6 +423,13 @@ def test_post_call_serializes_dict_with_datetime(logging_obj): assert "2026-05-11" in serialized +def _reset_sentry_globals(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm_logging, "sentry_sdk_instance", None) + litellm_logging.start_sentry.cache_clear() + monkeypatch.setattr(litellm_logging, "capture_exception", None) + monkeypatch.setattr(litellm_logging, "add_breadcrumb", None) + + def test_sentry_environment(monkeypatch): import sentry_sdk @@ -430,15 +438,59 @@ def test_sentry_environment(monkeypatch): monkeypatch.setenv("SENTRY_DSN", "https://test@sentry.io/123456") monkeypatch.delenv("SENTRY_ENVIRONMENT", raising=False) + _reset_sentry_globals(monkeypatch) set_callbacks(["sentry"]) assert mock_init.call_args[1]["environment"] == "production" for environment in ("development", "staging"): monkeypatch.setenv("SENTRY_ENVIRONMENT", environment) mock_init.reset_mock() + _reset_sentry_globals(monkeypatch) set_callbacks(["sentry"]) mock_init.assert_called_once() assert mock_init.call_args[1]["environment"] == environment + + +def test_set_callbacks_initializes_sentry_once_per_dsn(monkeypatch): + import sentry_sdk + + mock_init: Final = MagicMock() + monkeypatch.setattr(sentry_sdk, "init", mock_init) + monkeypatch.setenv("SENTRY_DSN", "https://first@sentry.example/1") + _reset_sentry_globals(monkeypatch) + + set_callbacks(["sentry"]) + set_callbacks(["sentry"]) + monkeypatch.setenv("SENTRY_DSN", "https://second@sentry.example/2") + set_callbacks(["sentry"]) + + assert [call.kwargs["dsn"] for call in mock_init.call_args_list] == [ + "https://first@sentry.example/1", + "https://second@sentry.example/2", + ] + assert litellm_logging.sentry_sdk_instance is sentry_sdk + assert litellm_logging.capture_exception is sentry_sdk.capture_exception + + +def test_sentry_started_before_its_dsn_is_loaded_picks_the_dsn_up_later(monkeypatch): + import sentry_sdk + + mock_init: Final = MagicMock() + monkeypatch.setattr(sentry_sdk, "init", mock_init) + monkeypatch.delenv("SENTRY_DSN", raising=False) + _reset_sentry_globals(monkeypatch) + + litellm_logging.initialize_sentry() + assert litellm_logging.sentry_sdk_instance is None + + monkeypatch.setenv("SENTRY_DSN", "https://from-db@sentry.example/3") + set_callbacks(["sentry"]) + + mock_init.assert_called_once() + assert mock_init.call_args.kwargs["dsn"] == "https://from-db@sentry.example/3" + assert litellm_logging.sentry_sdk_instance is sentry_sdk + + def test_use_custom_pricing_for_model(): from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model @@ -3375,7 +3427,9 @@ def test_sentry_event_scrubber_initialization(monkeypatch): mock_init = MagicMock() monkeypatch.setattr(sentry_sdk, "init", mock_init) + monkeypatch.setenv("SENTRY_DSN", "https://key@sentry.example/1") monkeypatch.delenv("SENTRY_SEND_DEFAULT_PII", raising=False) + _reset_sentry_globals(monkeypatch) set_callbacks(["sentry"]) @@ -3392,7 +3446,9 @@ def test_sentry_send_default_pii_opt_in(monkeypatch): mock_init = MagicMock() monkeypatch.setattr(sentry_sdk, "init", mock_init) + monkeypatch.setenv("SENTRY_DSN", "https://key@sentry.example/1") monkeypatch.setenv("SENTRY_SEND_DEFAULT_PII", "true") + _reset_sentry_globals(monkeypatch) set_callbacks(["sentry"]) diff --git a/tests/unit/litellm_core_utils/test_sentry_scrubbing.py b/tests/unit/litellm_core_utils/test_sentry_scrubbing.py index 9aae3999129..c5bf8334b91 100644 --- a/tests/unit/litellm_core_utils/test_sentry_scrubbing.py +++ b/tests/unit/litellm_core_utils/test_sentry_scrubbing.py @@ -30,6 +30,7 @@ MASTER_KEY: Final = "sk-master-key-under-test" DATABASE_URL: Final = "postgresql://litellm:db-password-under-test@db.internal:5432/litellm" PII_ON: Final = {"SENTRY_DSN": "https://key@sentry.example/1", "SENTRY_SEND_DEFAULT_PII": "true"} PII_OFF: Final = {"SENTRY_DSN": "https://key@sentry.example/1"} +PROMPT: Final = "confidential prompt text under test" class RecordingTransport(Transport): @@ -62,8 +63,12 @@ def raise_with_identity_locals() -> None: def raise_with_source_context_named_locals() -> None: - metadata: Final = {"context_line": f"Bearer {VIRTUAL_KEY}", "pre_context": [EMAIL], "post_context": [KEY_HASH]} - stacktrace: Final = {"frames": [{"context_line": MASTER_KEY, "pre_context": [EMAIL]}]} + metadata: Final = { + "context_line": f"Bearer {VIRTUAL_KEY}", + "pre_context": [MASTER_KEY], + "post_context": [VIRTUAL_KEY], + } + stacktrace: Final = {"frames": [{"context_line": MASTER_KEY, "pre_context": [VIRTUAL_KEY]}]} raise RuntimeError(f"rejected with {len(metadata)} metadata fields and {len(stacktrace)} stack fields") @@ -92,15 +97,21 @@ def test_default_event_carries_no_email_hash_or_secret_anywhere() -> None: assert MASTER_KEY not in serialized assert VIRTUAL_KEY not in serialized assert "db-password-under-test" not in serialized - frame_vars: Final = innermost_frame_vars(serialized) - assert frame_vars["raw_headers"] == {"authorization": FILTERED, "x-api-key": FILTERED, "content-type": "'application/json'"} - assert f"token='{FILTERED}'" in frame_vars["valid_token"] - assert f"user_id='{FILTERED}'" in frame_vars["valid_token"] - assert f"user_email='{FILTERED}'" in frame_vars["user_obj"] - assert frame_vars["general_settings"] == {"master_key": FILTERED, "database_url": FILTERED} - assert frame_vars["data"] == {"metadata": {"user_api_key_hash": FILTERED, "user_api_key_user_email": FILTERED}} - assert "key_name='sk-...test'" in frame_vars["valid_token"] - assert "user_role='internal_user'" in frame_vars["user_obj"] + + +def raise_with_prompt_locals() -> None: + messages: Final = [{"role": "user", "content": PROMPT}] + raise RuntimeError(f"provider rejected {len(messages)} messages") + + +@pytest.mark.parametrize(("env", "locals_sent"), ((PII_OFF, False), (PII_ON, True)), ids=("default", "pii-opt-in")) +def test_frame_locals_carrying_prompts_are_sent_only_with_the_pii_opt_in( + env: Mapping[str, str], locals_sent: bool +) -> None: + serialized: Final = capture_serialized_event(env, raise_with_prompt_locals) + frames: Final = json.loads(serialized)["exception"]["values"][0]["stacktrace"]["frames"] + assert (PROMPT in serialized) is locals_sent + assert any("vars" in frame for frame in frames) is locals_sent def test_source_context_lines_are_left_readable() -> None: @@ -115,11 +126,9 @@ def test_source_context_lines_are_left_readable() -> None: def test_source_context_names_outside_stack_frames_are_scrubbed() -> None: - serialized: Final = capture_serialized_event(PII_OFF, raise_with_source_context_named_locals) + serialized: Final = capture_serialized_event(PII_ON, raise_with_source_context_named_locals) assert VIRTUAL_KEY not in serialized assert MASTER_KEY not in serialized - assert EMAIL not in serialized - assert KEY_HASH not in serialized frame_vars: Final = innermost_frame_vars(serialized) assert frame_vars["metadata"] == { "context_line": f"'Bearer {FILTERED}'", @@ -276,3 +285,8 @@ def test_init_options_defaults() -> None: assert options["traces_sample_rate"] == 1.0 assert options["sample_rate"] == 1.0 assert options["environment"] == "production" + + +@pytest.mark.parametrize("env", ({}, {"SENTRY_SEND_DEFAULT_PII": "true"}), ids=("default", "pii-opt-in")) +def test_init_options_never_attach_request_bodies(env: Mapping[str, str]) -> None: + assert build_sentry_init_options(env)["max_request_body_size"] == "never" diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index a1e22780d57..5fd088acd01 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -186,6 +186,68 @@ async def test_tracing_config_automatically_exports_spend_without_a_storage_depe assert logger.task is not None and logger.task.done() and not logger.task.cancelled() +@pytest.mark.asyncio +async def test_tracing_logs_an_error_naming_the_store_when_lens_is_misconfigured( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + from litellm.proxy.tracing_runtime import manage_tracing + + monkeypatch.delenv("LITELLM_LENS_URL", raising=False) + monkeypatch.delenv("LITELLM_LENS_SERVICE_TOKEN", raising=False) + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + async with manage_tracing(enabled=True) as receiver: + assert receiver is None + + failures: Final = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.ERROR and "lens" in record.getMessage() + ] + assert len(failures) == 1 + assert "LITELLM_LENS_URL" in failures[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "config_text", + ( + "model_list: []\nlitellm_settings:\n failure_callback:\n - sentry\n", + "model_list: []\nlitellm_settings:\n success_callback:\n - sentry\n", + "model_list: []\nlitellm_settings:\n input_callback:\n - sentry\n", + "model_list: []\ngeneral_settings:\n alerting:\n - sentry\n", + ), + ids=("failure_callback", "success_callback", "input_callback", "alerting"), +) +async def test_proxy_config_starts_sentry_at_boot_without_request_bodies( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config_text: str +) -> None: + import sentry_sdk + + from litellm.litellm_core_utils import litellm_logging + + init: Final = MagicMock() + monkeypatch.setattr(sentry_sdk, "init", init) + monkeypatch.setenv("SENTRY_DSN", "https://key@sentry.example/1") + monkeypatch.setattr(litellm_logging, "sentry_sdk_instance", None) + litellm_logging.start_sentry.cache_clear() + monkeypatch.setattr(litellm_logging, "capture_exception", None) + monkeypatch.setattr(litellm_logging, "add_breadcrumb", None) + monkeypatch.setattr(litellm, "input_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + config_file: Final = tmp_path / "sentry.yaml" + config_file.write_text(config_text) + + await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + + init.assert_called_once() + assert init.call_args.kwargs["max_request_body_size"] == "never" + assert litellm_logging.sentry_sdk_instance is sentry_sdk + # --------------------------------------------------------------------------- # _is_remote_module_url diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index ec8e2e0438a..1d197602543 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.bug_report import ( bug_report_notice, build_bug_report, ) +from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic from litellm.constants import ( CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_LITELLM_CALL_ID_LENGTH, @@ -4674,6 +4675,21 @@ class TestHandleLLMApiExceptionRetryAfter: proxy_exc = await self._invoke(ValueError("some other failure")) assert "retry-after" not in proxy_exc.headers + @pytest.mark.parametrize( + "exc", + ( + RouterRateLimitError(model="gpt-4", cooldown_time=42.3, enable_pre_call_checks=False, cooldown_list=[]), + RouterRateLimitErrorBasic(model="gpt-4"), + ), + ids=("cooldown", "basic"), + ) + async def test_handle_llm_api_exception_answers_a_cooldown_with_429_and_no_bug_notice(self, exc, caplog): + with caplog.at_level("ERROR"): + proxy_exc = await self._invoke(exc) + + assert (proxy_exc.code, proxy_exc.type) == ("429", "rate_limit_error") + assert ISSUE_URL_BASE not in caplog.text + async def test_handle_llm_api_exception_strips_bug_report_notice_from_client_message(self, caplog): report = build_bug_report(RuntimeError("boom"), surface="sdk") notice = bug_report_notice(report) diff --git a/tests/unit/proxy/utils/proxy_logging/test_alerting.py b/tests/unit/proxy/utils/proxy_logging/test_alerting.py index 43ee1094330..91c77fdbe4d 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_alerting.py +++ b/tests/unit/proxy/utils/proxy_logging/test_alerting.py @@ -7,15 +7,15 @@ Covers ``failed_tracking_alert``, ``budget_alerts``, ``alerting_handler``, from __future__ import annotations from datetime import datetime -from typing import Any, Dict +from typing import Any, Dict, Final from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException from prisma.errors import PrismaError -import litellm from litellm._service_logger import ServiceTypes +from litellm.litellm_core_utils import litellm_logging from litellm.proxy._types import AlertType, CallInfo @@ -210,11 +210,45 @@ async def test_alerting_handler_sends_to_slack(proxy_logging): @pytest.mark.asyncio async def test_alerting_handler_sentry_without_sdk_error_raises(proxy_logging, monkeypatch): proxy_logging.alerting = ["sentry"] - monkeypatch.setattr(litellm.utils, "sentry_sdk_instance", None) + monkeypatch.delenv("SENTRY_DSN", raising=False) + monkeypatch.setattr(litellm_logging, "sentry_sdk_instance", None) + litellm_logging.start_sentry.cache_clear() with pytest.raises(Exception, match="SENTRY_DSN"): await proxy_logging.alerting_handler(message="x", level="Low", alert_type=AlertType.db_exceptions) +@pytest.mark.asyncio +async def test_alerting_handler_sentry_sends_through_the_initialized_sdk(proxy_logging, monkeypatch): + proxy_logging.alerting = ["sentry"] + sentry = MagicMock() + monkeypatch.setattr(litellm_logging, "sentry_sdk_instance", sentry) + await proxy_logging.alerting_handler(message="db down", level="High", alert_type=AlertType.db_exceptions) + sentry.capture_message.assert_called_once() + assert "db down" in sentry.capture_message.call_args.args[0] + + +@pytest.mark.asyncio +async def test_alerting_handler_starts_sentry_when_the_alert_was_not_configured_at_boot(proxy_logging, monkeypatch): + import sentry_sdk + + monkeypatch.setattr(proxy_logging, "alerting", ["sentry"]) + init: Final = MagicMock() + capture_message: Final = MagicMock() + monkeypatch.setattr(sentry_sdk, "init", init) + monkeypatch.setattr(sentry_sdk, "capture_message", capture_message) + monkeypatch.setenv("SENTRY_DSN", "https://key@sentry.example/1") + monkeypatch.setattr(litellm_logging, "sentry_sdk_instance", None) + litellm_logging.start_sentry.cache_clear() + monkeypatch.setattr(litellm_logging, "capture_exception", None) + monkeypatch.setattr(litellm_logging, "add_breadcrumb", None) + + await proxy_logging.alerting_handler(message="db down", level="High", alert_type=AlertType.db_exceptions) + + assert init.call_args.kwargs["dsn"] == "https://key@sentry.example/1" + capture_message.assert_called_once() + assert "db down" in capture_message.call_args.args[0] + + # --------------------------------------------------------------------------- # failure_handler # --------------------------------------------------------------------------- @@ -235,7 +269,7 @@ async def test_failure_handler_logs_db_error_and_calls_service_logging(proxy_log proxy_logging.alert_types = [AlertType.db_exceptions] proxy_logging.alerting_handler = AsyncMock() proxy_logging.service_logging_obj = MagicMock(async_service_failure_hook=AsyncMock()) - monkeypatch.setattr(litellm.utils, "capture_exception", None) + monkeypatch.setattr(litellm_logging, "capture_exception", None) await proxy_logging.failure_handler( original_exception=HTTPException(status_code=500, detail="boom"), duration=1.5, @@ -262,7 +296,7 @@ async def test_failure_handler_alerts_but_leaves_prisma_error_event_to_log_db_me proxy_logging.alert_types = [AlertType.db_exceptions] proxy_logging.alerting_handler = AsyncMock() proxy_logging.service_logging_obj = MagicMock(async_service_failure_hook=AsyncMock()) - monkeypatch.setattr(litellm.utils, "capture_exception", None) + monkeypatch.setattr(litellm_logging, "capture_exception", None) await proxy_logging.failure_handler( original_exception=PrismaError("connection reset"), duration=1.0, call_type=call_type ) @@ -278,7 +312,7 @@ async def test_failure_handler_still_emits_db_event_for_wrapped_insert_error(pro proxy_logging.alert_types = [AlertType.db_exceptions] proxy_logging.alerting_handler = AsyncMock() proxy_logging.service_logging_obj = MagicMock(async_service_failure_hook=AsyncMock()) - monkeypatch.setattr(litellm.utils, "capture_exception", None) + monkeypatch.setattr(litellm_logging, "capture_exception", None) await proxy_logging.failure_handler( original_exception=HTTPException(status_code=400, detail={"error": "Foreign Key Constraint failed"}), duration=1.0, @@ -298,7 +332,7 @@ async def test_failure_handler_with_capture_exception_invoked(proxy_logging, mon def fake_capture(error): captured["error"] = error - monkeypatch.setattr(litellm.utils, "capture_exception", fake_capture) + monkeypatch.setattr(litellm_logging, "capture_exception", fake_capture) err = RuntimeError("real") await proxy_logging.failure_handler(original_exception=err, duration=1.0, call_type="db_read") snapshot = { @@ -320,7 +354,7 @@ async def test_failure_handler_propagates_service_logging_error_raises(proxy_log proxy_logging.service_logging_obj = MagicMock( async_service_failure_hook=AsyncMock(side_effect=RuntimeError("svc")) ) - monkeypatch.setattr(litellm.utils, "capture_exception", None) + monkeypatch.setattr(litellm_logging, "capture_exception", None) with pytest.raises(RuntimeError): await proxy_logging.failure_handler( original_exception=Exception("x"), duration=0.0, call_type="db_read" diff --git a/tests/unit/tracing/test_exporter.py b/tests/unit/tracing/test_exporter.py index 5f67fbbb25b..225c96f2c57 100644 --- a/tests/unit/tracing/test_exporter.py +++ b/tests/unit/tracing/test_exporter.py @@ -1,10 +1,12 @@ import asyncio import json +import logging from itertools import chain from typing import Final import httpx import pytest +from prometheus_client import REGISTRY from litellm.tracing.exporter import MAX_BUFFER_EVENTS, MAX_EVENT_BYTES, ExportFailure, LensExporter, encode_record @@ -306,3 +308,51 @@ def test_serialization_rejects_recursive_payloads() -> None: @pytest.mark.parametrize("value", (float("nan"), object(), "\ud800"), ids=("nan", "unsupported", "surrogate")) def test_serialization_returns_a_failure_for_invalid_payloads(value: object) -> None: assert encode_record({"value": value}) is ExportFailure.INVALID + + +def _counter_value(name: str) -> float: + return REGISTRY.get_sample_value(name) or 0.0 + + +async def _drained(exporter: LensExporter) -> None: + while exporter.buffered_events: + await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_written_and_dropped_rows_are_exported_as_prometheus_counters() -> None: + statuses: Final = iter((204, 500)) + + def answer(request: httpx.Request) -> httpx.Response: + return httpx.Response(next(statuses)) + + written_before: Final = _counter_value("litellm_lens_export_rows_written_total") + dropped_before: Final = _counter_value("litellm_lens_export_rows_dropped_total") + async with httpx.AsyncClient(base_url="http://lens", transport=httpx.MockTransport(answer)) as client: + exporter: Final = LensExporter(client) + exporter.start() + for request_id in ("accepted", "rejected"): + await exporter.async_log_success_event( + {"standard_logging_object": {"id": request_id, "response_cost": 0.1}}, None, None, None + ) + await _drained(exporter) + await exporter.aclose() + assert (exporter.rows_written, exporter.rows_dropped) == (1, 1) + assert _counter_value("litellm_lens_export_rows_written_total") - written_before == exporter.rows_written + assert _counter_value("litellm_lens_export_rows_dropped_total") - dropped_before == exporter.rows_dropped + + +@pytest.mark.asyncio +async def test_a_rejecting_store_is_logged_as_an_error_naming_lens(caplog: pytest.LogCaptureFixture) -> None: + def reject(request: httpx.Request) -> httpx.Response: + return httpx.Response(500) + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + async with httpx.AsyncClient(base_url="http://lens", transport=httpx.MockTransport(reject)) as client: + exporter: Final = LensExporter(client) + assert exporter.enqueue(b"{}") + exporter.start() + await exporter.aclose() + failures: Final = [record.getMessage() for record in caplog.records if record.levelno == logging.ERROR] + assert len(failures) == 1 + assert "lens" in failures[0] and "HTTP 500" in failures[0]