mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
fix(proxy): cooldown errors carry 429, Sentry starts at boot without request bodies, tracing failures log at ERROR, exporter counters exported (#46344)
* fix(router): carry a 429 status on cooldown and no-deployment errors RouterRateLimitError and RouterRateLimitErrorBasic had no status_code, so the proxy answered an all-deployments-cooled-down request as a generic 500 class error and logged the bug report notice for every cooldown. Both now carry status_code 429, which the proxy maps to a rate_limit_error and which the retry policy already treats as retryable * fix(proxy): start Sentry at config load and never attach request bodies Sentry only came up inside set_callbacks on the first LLM call, so boot and background errors were never reported, and general_settings.alerting: [sentry] always raised Missing SENTRY_DSN because the alerting path read a dead litellm.utils pin instead of the live SDK. The proxy now initializes Sentry once while loading the config whenever sentry is a configured callback or alerting target, the alerting and failure handlers read the live instance, and max_request_body_size is pinned to never so prompts and completions stay out of Sentry events * fix(tracing): log Lens failures at ERROR and export row counters to Prometheus A tracing store that fails to start, or a Lens service that rejects a batch, was only logged at WARNING, so operators running at the default ERROR level never saw tracing silently go dark. Both now log at ERROR and name the lens store. The exporter's rows_written and rows_dropped tallies were never exported, so the two counts are now Prometheus counters litellm_lens_export_rows_written_total and litellm_lens_export_rows_dropped_total, registered lazily on the default registry like the other proxy-level counters and charted on the all-metrics Grafana dashboard * test(proxy): drop the litellm import the alerting tests no longer use * fix(proxy): start Sentry once a DSN is set, from any config source Boot-time init used to lock Sentry in even with no SENTRY_DSN, so a DSN loaded later from DB environment_variables was never picked up. initialize_sentry now skips when no DSN is set and re-inits when the DSN changes. alerting_handler calls it too, so alerting: [sentry] set through the DB works. Drops the litellm_settings.callbacks check since that config fails to load with sentry in it * fix(sentry): keep stack frame locals out of events unless PII is opted in Frame locals carried prompts (messages, request bodies, provider options) into every error event. include_local_variables now follows SENTRY_SEND_DEFAULT_PII * test(router): drop the test that only pinned the 429 class attribute * fix(proxy): start Sentry at boot when it is only an input callback * fix(proxy): keep Sentry init and alerting within the strict ruff and basedpyright ceilings --------- Co-authored-by: yuneng-jiang <yuneng@berri.ai>
This commit is contained in:
ryan-crabbe-berri
GitHub
parent
b06f4345e4
commit
38f3f8a1a9
17 changed files with 481 additions and 73 deletions
|
|
@ -6563,6 +6563,87 @@
|
|||
],
|
||||
"title": "litellm_project_model_rate_limit_used_metric",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 447
|
||||
},
|
||||
"id": 115,
|
||||
"panels": [],
|
||||
"title": "Agent tracing export (needs LITELLM_LENS_URL)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Request records the Lens exporter delivered to the Lens service against the records it dropped before the service accepted them",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 448
|
||||
},
|
||||
"id": 116,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_lens_export_rows_written_total[$__rate_interval]))",
|
||||
"legendFormat": "written",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_lens_export_rows_dropped_total[$__rate_interval]))",
|
||||
"legendFormat": "dropped",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "litellm_lens_export_rows_written and litellm_lens_export_rows_dropped rate",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"preload": false,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# LiteLLM All Prometheus Metrics dashboard
|
||||
|
||||
Every `litellm_*` metric family the proxy can expose on `/metrics` (141 families across 97 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
|
||||
Every `litellm_*` metric family the proxy can expose on `/metrics` (143 families across 98 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes), project model rate limits, and the agent tracing export to Lens. Panel titles are the metric names so you can grep the JSON for the metric you care about
|
||||
|
||||
Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/unit/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
|
||||
|
||||
The first eleven rows need only `callbacks: ["prometheus"]`. The last three rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected
|
||||
The first eleven rows and the project model rate limits row need only `callbacks: ["prometheus"]`. The other rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, the agent tracing export row needs `LITELLM_LENS_URL`, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected
|
||||
|
||||
## Pre-requisites
|
||||
|
||||
|
|
|
|||
|
|
@ -4507,6 +4507,32 @@ def get_masked_values(
|
|||
_get_masked_values = get_masked_values
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def start_sentry(dsn: str) -> None:
|
||||
try:
|
||||
import sentry_sdk
|
||||
except ImportError:
|
||||
print_verbose("Package 'sentry_sdk' is missing. Installing it...")
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "sentry_sdk"])
|
||||
import sentry_sdk
|
||||
from litellm.litellm_core_utils.sentry_scrubbing import build_sentry_init_options
|
||||
|
||||
sentry_sdk.init(**build_sentry_init_options({**os.environ, "SENTRY_DSN": dsn}))
|
||||
|
||||
|
||||
def initialize_sentry() -> None:
|
||||
global sentry_sdk_instance, capture_exception, add_breadcrumb
|
||||
dsn: Final = os.environ.get("SENTRY_DSN")
|
||||
if not dsn:
|
||||
return
|
||||
start_sentry(dsn)
|
||||
import sentry_sdk
|
||||
|
||||
sentry_sdk_instance = sentry_sdk # rebind-ok: process-wide Sentry SDK handle read by every logging path
|
||||
capture_exception = sentry_sdk.capture_exception # rebind-ok: process-wide Sentry SDK handle
|
||||
add_breadcrumb = sentry_sdk.add_breadcrumb # rebind-ok: process-wide Sentry SDK handle
|
||||
|
||||
|
||||
def set_callbacks(
|
||||
callback_list: Iterable[str | Callable[..., object] | CustomLogger],
|
||||
function_id: str | None = None,
|
||||
|
|
@ -4515,9 +4541,6 @@ def set_callbacks(
|
|||
Globally sets the callback client
|
||||
"""
|
||||
global \
|
||||
sentry_sdk_instance, \
|
||||
capture_exception, \
|
||||
add_breadcrumb, \
|
||||
slack_app, \
|
||||
alerts_channel, \
|
||||
traceloopLogger, \
|
||||
|
|
@ -4541,18 +4564,7 @@ def set_callbacks(
|
|||
try:
|
||||
for callback in callback_list:
|
||||
if callback == "sentry":
|
||||
try:
|
||||
import sentry_sdk
|
||||
except ImportError:
|
||||
print_verbose("Package 'sentry_sdk' is missing. Installing it...")
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "sentry_sdk"])
|
||||
import sentry_sdk
|
||||
from litellm.litellm_core_utils.sentry_scrubbing import build_sentry_init_options
|
||||
|
||||
sentry_sdk_instance = sentry_sdk
|
||||
sentry_sdk_instance.init(**build_sentry_init_options(os.environ))
|
||||
capture_exception = sentry_sdk_instance.capture_exception
|
||||
add_breadcrumb = sentry_sdk_instance.add_breadcrumb
|
||||
initialize_sentry()
|
||||
elif callback == "slack":
|
||||
try:
|
||||
from slack_bolt import App
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
|||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from functools import reduce
|
||||
from typing import TYPE_CHECKING, Final, TypeAlias, cast
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast
|
||||
|
||||
from pydantic import JsonValue
|
||||
from sentry_sdk.scrubber import DEFAULT_DENYLIST, DEFAULT_PII_DENYLIST, EventScrubber
|
||||
|
|
@ -65,6 +65,8 @@ class SentryInitOptions(TypedDict):
|
|||
traces_sample_rate: ReadOnly[float]
|
||||
sample_rate: ReadOnly[float]
|
||||
send_default_pii: ReadOnly[bool]
|
||||
max_request_body_size: ReadOnly[Literal["never"]]
|
||||
include_local_variables: ReadOnly[bool]
|
||||
event_scrubber: ReadOnly[EventScrubber]
|
||||
before_send: ReadOnly[EventScrubFn]
|
||||
before_send_transaction: ReadOnly[EventScrubFn]
|
||||
|
|
@ -140,6 +142,8 @@ def build_sentry_init_options(env: Mapping[str, str]) -> SentryInitOptions:
|
|||
traces_sample_rate=float(env.get("SENTRY_API_TRACE_RATE") or "1.0"),
|
||||
sample_rate=float(env.get("SENTRY_API_SAMPLE_RATE") or "1.0"),
|
||||
send_default_pii=send_default_pii,
|
||||
max_request_body_size="never",
|
||||
include_local_variables=send_default_pii,
|
||||
event_scrubber=EventScrubber(
|
||||
denylist=list(SECRET_FIELD_NAMES),
|
||||
pii_denylist=list(PII_FIELD_NAMES),
|
||||
|
|
|
|||
|
|
@ -90,6 +90,7 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
_init_custom_logger_compatible_class,
|
||||
initialize_sentry,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
|
@ -4962,6 +4963,16 @@ def _is_remote_module_url(value: object) -> bool:
|
|||
return isinstance(value, str) and (value.startswith("s3://") or value.startswith("gcs://"))
|
||||
|
||||
|
||||
def sentry_is_configured(general_settings: Mapping[str, object]) -> bool:
|
||||
alerting: Final = general_settings.get("alerting")
|
||||
return (
|
||||
"sentry" in litellm.input_callback
|
||||
or "sentry" in litellm.success_callback
|
||||
or "sentry" in litellm.failure_callback
|
||||
or (isinstance(alerting, list) and "sentry" in alerting)
|
||||
)
|
||||
|
||||
|
||||
def _scrub_guardrail_inner(inner: dict[str, JsonValue]) -> None:
|
||||
"""Strip remote-URL entries from a guardrail's ``callbacks`` list
|
||||
and ``guardrail`` (v2 module-path) field. Mutates in place."""
|
||||
|
|
@ -6911,6 +6922,9 @@ class ProxyConfig:
|
|||
if general_settings is None:
|
||||
general_settings = {}
|
||||
|
||||
if sentry_is_configured(general_settings):
|
||||
initialize_sentry()
|
||||
|
||||
typed_general_settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings)
|
||||
ConfigGeneralSettings.model_validate(
|
||||
MappingProxyType(
|
||||
|
|
|
|||
|
|
@ -46,10 +46,8 @@ async def manage_tracing(
|
|||
return
|
||||
try:
|
||||
connection: Final = LensConnection.from_env()
|
||||
except ValueError:
|
||||
verbose_proxy_logger.warning(
|
||||
"Agent tracing unavailable: configure LITELLM_LENS_URL and LITELLM_LENS_SERVICE_TOKEN"
|
||||
)
|
||||
except ValueError as error:
|
||||
verbose_proxy_logger.error("Agent tracing store lens failed to start, tracing is disabled: %s", error)
|
||||
yield None
|
||||
return
|
||||
async with client_factory(connection) as client:
|
||||
|
|
|
|||
|
|
@ -3250,14 +3250,14 @@ class ProxyLogging:
|
|||
alerting_metadata=alerting_metadata,
|
||||
**extra_kwargs,
|
||||
)
|
||||
for client in self.alerting:
|
||||
if client == "sentry":
|
||||
if litellm.utils.sentry_sdk_instance is not None:
|
||||
litellm.utils.sentry_sdk_instance.capture_message(formatted_message)
|
||||
else:
|
||||
raise Exception("Missing SENTRY_DSN from environment")
|
||||
if "sentry" in self.alerting:
|
||||
litellm.litellm_core_utils.litellm_logging.initialize_sentry()
|
||||
sentry: Final = litellm.litellm_core_utils.litellm_logging.sentry_sdk_instance
|
||||
if sentry is None:
|
||||
raise Exception("Missing SENTRY_DSN from environment")
|
||||
sentry.capture_message(formatted_message)
|
||||
|
||||
async def failure_handler(self, original_exception, duration: float, call_type: str, traceback_str=""):
|
||||
async def failure_handler(self, original_exception: Exception, duration: float, call_type: str, traceback_str=""):
|
||||
"""
|
||||
Log failed db read/writes
|
||||
|
||||
|
|
@ -3298,8 +3298,9 @@ class ProxyLogging:
|
|||
call_type=call_type,
|
||||
)
|
||||
|
||||
if litellm.utils.capture_exception:
|
||||
litellm.utils.capture_exception(error=original_exception)
|
||||
capture: Final = litellm.litellm_core_utils.litellm_logging.capture_exception
|
||||
if capture is not None:
|
||||
capture(error=original_exception)
|
||||
|
||||
async def post_call_failure_hook(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@ import json
|
|||
from collections import deque
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from functools import cache
|
||||
from io import BytesIO
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
|
@ -16,6 +18,9 @@ from litellm.integrations.clickhouse.clickhouse_spend_logger import spend_log_ro
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.tracing.types import SpendLogPayload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prometheus_client import Counter as PrometheusCounter
|
||||
|
||||
MAX_EVENT_BYTES: Final = 1024 * 1024
|
||||
MAX_BUFFER_BYTES: Final = 32 * 1024 * 1024
|
||||
MAX_BUFFER_EVENTS: Final = 1000
|
||||
|
|
@ -89,11 +94,49 @@ def encode_record(value: Mapping[str, object]) -> bytes | ExportFailure:
|
|||
return ExportFailure.INVALID
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExportCounters:
|
||||
written: "PrometheusCounter | None"
|
||||
dropped: "PrometheusCounter | None"
|
||||
|
||||
def add_written(self, count: int) -> None:
|
||||
if self.written is not None:
|
||||
self.written.inc(count)
|
||||
|
||||
def add_dropped(self, count: int) -> None:
|
||||
if self.dropped is not None:
|
||||
self.dropped.inc(count)
|
||||
|
||||
|
||||
@cache
|
||||
def lens_export_counters() -> ExportCounters:
|
||||
try:
|
||||
from prometheus_client import Counter
|
||||
except ImportError:
|
||||
return ExportCounters(written=None, dropped=None)
|
||||
return ExportCounters(
|
||||
written=Counter(
|
||||
"litellm_lens_export_rows_written_total",
|
||||
"Request records the Lens exporter delivered to the Lens service",
|
||||
),
|
||||
dropped=Counter(
|
||||
"litellm_lens_export_rows_dropped_total",
|
||||
"Request records the Lens exporter dropped before the Lens service accepted them",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class LensExporter(CustomLogger):
|
||||
def __init__(self, client: httpx.AsyncClient, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
counters: ExportCounters | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.client: Final = client
|
||||
self.sleep: Final = sleep
|
||||
self.counters: Final = counters if counters is not None else lens_export_counters()
|
||||
self.queue: Final[deque[bytes]] = deque() # mutable-ok: bounded producer-consumer queue
|
||||
self.wake: Final = asyncio.Event()
|
||||
self.closed = False
|
||||
|
|
@ -108,6 +151,14 @@ class LensExporter(CustomLogger):
|
|||
if self.task is None:
|
||||
self.task = asyncio.create_task(self._run())
|
||||
|
||||
def _wrote(self, count: int) -> None:
|
||||
self.rows_written += count
|
||||
self.counters.add_written(count)
|
||||
|
||||
def _drop(self, count: int) -> None:
|
||||
self.rows_dropped += count
|
||||
self.counters.add_dropped(count)
|
||||
|
||||
def enqueue(self, record: bytes) -> bool:
|
||||
if (
|
||||
self.closed
|
||||
|
|
@ -115,7 +166,7 @@ class LensExporter(CustomLogger):
|
|||
or self.buffered_events >= MAX_BUFFER_EVENTS
|
||||
or self.buffered_bytes + len(record) > MAX_BUFFER_BYTES
|
||||
):
|
||||
self.rows_dropped += 1
|
||||
self._drop(1)
|
||||
return False
|
||||
self.queue.append(record)
|
||||
self.buffered_events += 1
|
||||
|
|
@ -138,13 +189,13 @@ class LensExporter(CustomLogger):
|
|||
if raw is None or self.closed:
|
||||
return
|
||||
if self.buffered_events >= MAX_BUFFER_EVENTS or self.buffered_bytes >= MAX_BUFFER_BYTES:
|
||||
self.rows_dropped += 1
|
||||
self._drop(1)
|
||||
return
|
||||
try:
|
||||
checked: Final = _check_size(raw, MAX_EVENT_BYTES)
|
||||
if isinstance(checked, ExportFailure):
|
||||
self.rows_dropped += 1
|
||||
self._warn(checked.value)
|
||||
self._drop(1)
|
||||
self._log_failure(checked.value)
|
||||
return
|
||||
payload: Final = _PAYLOAD.validate_python(raw)
|
||||
if str(payload.get("call_type", "")).startswith(("/v1/traces", "/v1/logs")):
|
||||
|
|
@ -152,24 +203,26 @@ class LensExporter(CustomLogger):
|
|||
row: Final = spend_log_row_from_payload(payload, kwargs)
|
||||
record: Final = encode_record(row)
|
||||
if isinstance(record, ExportFailure):
|
||||
self.rows_dropped += 1
|
||||
self._warn(record.value)
|
||||
self._drop(1)
|
||||
self._log_failure(record.value)
|
||||
return
|
||||
self.enqueue(record)
|
||||
except ValidationError as error:
|
||||
self.rows_dropped += 1
|
||||
self._drop(1)
|
||||
fields: Final = tuple(
|
||||
str(issue["loc"][0]) if issue["loc"] else "$"
|
||||
for issue in error.errors(include_input=False, include_context=False, include_url=False)[:5]
|
||||
)
|
||||
self._warn("invalid request record fields: " + ", ".join(fields))
|
||||
self._log_failure("invalid request record fields: " + ", ".join(fields))
|
||||
except (ValueError, TypeError, OverflowError, RecursionError) as error:
|
||||
self.rows_dropped += 1
|
||||
self._warn(type(error).__name__)
|
||||
self._drop(1)
|
||||
self._log_failure(type(error).__name__)
|
||||
|
||||
def _warn(self, reason: str) -> None:
|
||||
def _log_failure(self, reason: str) -> None:
|
||||
if reason != self.last_error:
|
||||
verbose_proxy_logger.warning("Lens request export failed (%s); model requests continue", reason)
|
||||
verbose_proxy_logger.error(
|
||||
"Agent tracing store lens dropped request records (%s); model requests continue", reason
|
||||
)
|
||||
self.last_error = reason
|
||||
|
||||
def _batch(self) -> tuple[bytes, ...]:
|
||||
|
|
@ -196,13 +249,13 @@ class LensExporter(CustomLogger):
|
|||
self.last_error = ""
|
||||
return True
|
||||
if response.status_code not in (429, 502, 503, 504):
|
||||
self._warn(f"HTTP {response.status_code}")
|
||||
self._log_failure(f"HTTP {response.status_code}")
|
||||
return False
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
if attempt < 2:
|
||||
await self.sleep(float(1 << attempt))
|
||||
self._warn("retry limit reached")
|
||||
self._log_failure("retry limit reached")
|
||||
return False
|
||||
|
||||
async def _run(self) -> None:
|
||||
|
|
@ -217,11 +270,11 @@ class LensExporter(CustomLogger):
|
|||
batch: Final = self._batch()
|
||||
try:
|
||||
if await self._send(batch):
|
||||
self.rows_written += len(batch)
|
||||
self._wrote(len(batch))
|
||||
else:
|
||||
self.rows_dropped += len(batch)
|
||||
self._drop(len(batch))
|
||||
except asyncio.CancelledError:
|
||||
self.rows_dropped += len(batch)
|
||||
self._drop(len(batch))
|
||||
raise
|
||||
finally:
|
||||
self.buffered_events -= len(batch)
|
||||
|
|
@ -237,7 +290,7 @@ class LensExporter(CustomLogger):
|
|||
self.task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await self.task
|
||||
self.rows_dropped += len(self.queue)
|
||||
self._drop(len(self.queue))
|
||||
self.queue.clear()
|
||||
self.buffered_bytes = 0
|
||||
self.buffered_events = 0
|
||||
|
|
|
|||
|
|
@ -1084,6 +1084,8 @@ class RouterRateLimitErrorBasic(ValueError):
|
|||
Raise a basic error inside helper functions.
|
||||
"""
|
||||
|
||||
status_code: Final = 429
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -1099,6 +1101,8 @@ class RouterErrorTypes(str, enum.Enum):
|
|||
|
||||
|
||||
class RouterRateLimitError(ValueError):
|
||||
status_code: Final = 429
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from litellm.integrations.prometheus_services import PrometheusServicesLogger
|
|||
from litellm.proxy.db.db_transaction_queue.spend_log_cleanup_metrics import SpendLogCleanupMetrics
|
||||
from litellm.proxy.middleware.admission_control_middleware import create_prometheus_admission_metrics
|
||||
from litellm.proxy.middleware.in_flight_requests_middleware import InFlightRequestsMiddleware
|
||||
from litellm.tracing.exporter import lens_export_counters
|
||||
|
||||
_GRAFANA_DIR: Final = Path(__file__).parents[3] / "cookbook" / "litellm_proxy_server" / "grafana_dashboard"
|
||||
_ALL_METRICS_DASHBOARD: Final = _GRAFANA_DIR / "dashboard_all_metrics" / "grafana_dashboard.json"
|
||||
|
|
@ -62,6 +63,9 @@ def _lazy_owner_collectors() -> tuple[Collector, ...]:
|
|||
in_flight: Final = InFlightRequestsMiddleware._get_gauge()
|
||||
assert in_flight is not None
|
||||
services: Final = PrometheusServicesLogger()
|
||||
lens: Final = lens_export_counters()
|
||||
assert lens.written is not None
|
||||
assert lens.dropped is not None
|
||||
return (
|
||||
SpendLogCleanupMetrics.rows_deleted,
|
||||
SpendLogCleanupMetrics.batch_duration,
|
||||
|
|
@ -72,6 +76,8 @@ def _lazy_owner_collectors() -> tuple[Collector, ...]:
|
|||
services.redis_breaker_state,
|
||||
services.redis_breaker_transitions,
|
||||
services.redis_breaker_failures,
|
||||
lens.written,
|
||||
lens.dropped,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
strip_bug_report_notice,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic
|
||||
|
||||
|
||||
def test_build_bug_report_keeps_only_litellm_frames():
|
||||
|
|
@ -138,6 +139,8 @@ def test_bug_report_can_be_disabled(monkeypatch: pytest.MonkeyPatch):
|
|||
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"),
|
||||
),
|
||||
BadRequestError(message="bad input", llm_provider="openai", model="gpt-4"),
|
||||
RouterRateLimitError(model="gpt-4", cooldown_time=5.0, enable_pre_call_checks=False, cooldown_list=[]),
|
||||
RouterRateLimitErrorBasic(model="gpt-4"),
|
||||
"not an exception",
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from litellm.caching.caching import DualCache
|
|||
from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE, REDACTED_BY_LITELLM, SENTRY_PII_DENYLIST
|
||||
from litellm.cost_calculator import ocr_batch_cost
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
Logging,
|
||||
|
|
@ -422,6 +423,13 @@ def test_post_call_serializes_dict_with_datetime(logging_obj):
|
|||
assert "2026-05-11" in serialized
|
||||
|
||||
|
||||
def _reset_sentry_globals(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm_logging, "sentry_sdk_instance", None)
|
||||
litellm_logging.start_sentry.cache_clear()
|
||||
monkeypatch.setattr(litellm_logging, "capture_exception", None)
|
||||
monkeypatch.setattr(litellm_logging, "add_breadcrumb", None)
|
||||
|
||||
|
||||
def test_sentry_environment(monkeypatch):
|
||||
import sentry_sdk
|
||||
|
||||
|
|
@ -430,15 +438,59 @@ def test_sentry_environment(monkeypatch):
|
|||
monkeypatch.setenv("SENTRY_DSN", "https://test@sentry.io/123456")
|
||||
monkeypatch.delenv("SENTRY_ENVIRONMENT", raising=False)
|
||||
|
||||
_reset_sentry_globals(monkeypatch)
|
||||
set_callbacks(["sentry"])
|
||||
assert mock_init.call_args[1]["environment"] == "production"
|
||||
|
||||
for environment in ("development", "staging"):
|
||||
monkeypatch.setenv("SENTRY_ENVIRONMENT", environment)
|
||||
mock_init.reset_mock()
|
||||
_reset_sentry_globals(monkeypatch)
|
||||
set_callbacks(["sentry"])
|
||||
mock_init.assert_called_once()
|
||||
assert mock_init.call_args[1]["environment"] == environment
|
||||
|
||||
|
||||
def test_set_callbacks_initializes_sentry_once_per_dsn(monkeypatch):
|
||||
import sentry_sdk
|
||||
|
||||
mock_init: Final = MagicMock()
|
||||
monkeypatch.setattr(sentry_sdk, "init", mock_init)
|
||||
monkeypatch.setenv("SENTRY_DSN", "https://first@sentry.example/1")
|
||||
_reset_sentry_globals(monkeypatch)
|
||||
|
||||
set_callbacks(["sentry"])
|
||||
set_callbacks(["sentry"])
|
||||
monkeypatch.setenv("SENTRY_DSN", "https://second@sentry.example/2")
|
||||
set_callbacks(["sentry"])
|
||||
|
||||
assert [call.kwargs["dsn"] for call in mock_init.call_args_list] == [
|
||||
"https://first@sentry.example/1",
|
||||
"https://second@sentry.example/2",
|
||||
]
|
||||
assert litellm_logging.sentry_sdk_instance is sentry_sdk
|
||||
assert litellm_logging.capture_exception is sentry_sdk.capture_exception
|
||||
|
||||
|
||||
def test_sentry_started_before_its_dsn_is_loaded_picks_the_dsn_up_later(monkeypatch):
|
||||
import sentry_sdk
|
||||
|
||||
mock_init: Final = MagicMock()
|
||||
monkeypatch.setattr(sentry_sdk, "init", mock_init)
|
||||
monkeypatch.delenv("SENTRY_DSN", raising=False)
|
||||
_reset_sentry_globals(monkeypatch)
|
||||
|
||||
litellm_logging.initialize_sentry()
|
||||
assert litellm_logging.sentry_sdk_instance is None
|
||||
|
||||
monkeypatch.setenv("SENTRY_DSN", "https://from-db@sentry.example/3")
|
||||
set_callbacks(["sentry"])
|
||||
|
||||
mock_init.assert_called_once()
|
||||
assert mock_init.call_args.kwargs["dsn"] == "https://from-db@sentry.example/3"
|
||||
assert litellm_logging.sentry_sdk_instance is sentry_sdk
|
||||
|
||||
|
||||
def test_use_custom_pricing_for_model():
|
||||
from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
|
||||
|
||||
|
|
@ -3375,7 +3427,9 @@ def test_sentry_event_scrubber_initialization(monkeypatch):
|
|||
|
||||
mock_init = MagicMock()
|
||||
monkeypatch.setattr(sentry_sdk, "init", mock_init)
|
||||
monkeypatch.setenv("SENTRY_DSN", "https://key@sentry.example/1")
|
||||
monkeypatch.delenv("SENTRY_SEND_DEFAULT_PII", raising=False)
|
||||
_reset_sentry_globals(monkeypatch)
|
||||
|
||||
set_callbacks(["sentry"])
|
||||
|
||||
|
|
@ -3392,7 +3446,9 @@ def test_sentry_send_default_pii_opt_in(monkeypatch):
|
|||
|
||||
mock_init = MagicMock()
|
||||
monkeypatch.setattr(sentry_sdk, "init", mock_init)
|
||||
monkeypatch.setenv("SENTRY_DSN", "https://key@sentry.example/1")
|
||||
monkeypatch.setenv("SENTRY_SEND_DEFAULT_PII", "true")
|
||||
_reset_sentry_globals(monkeypatch)
|
||||
|
||||
set_callbacks(["sentry"])
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ MASTER_KEY: Final = "sk-master-key-under-test"
|
|||
DATABASE_URL: Final = "postgresql://litellm:db-password-under-test@db.internal:5432/litellm"
|
||||
PII_ON: Final = {"SENTRY_DSN": "https://key@sentry.example/1", "SENTRY_SEND_DEFAULT_PII": "true"}
|
||||
PII_OFF: Final = {"SENTRY_DSN": "https://key@sentry.example/1"}
|
||||
PROMPT: Final = "confidential prompt text under test"
|
||||
|
||||
|
||||
class RecordingTransport(Transport):
|
||||
|
|
@ -62,8 +63,12 @@ def raise_with_identity_locals() -> None:
|
|||
|
||||
|
||||
def raise_with_source_context_named_locals() -> None:
|
||||
metadata: Final = {"context_line": f"Bearer {VIRTUAL_KEY}", "pre_context": [EMAIL], "post_context": [KEY_HASH]}
|
||||
stacktrace: Final = {"frames": [{"context_line": MASTER_KEY, "pre_context": [EMAIL]}]}
|
||||
metadata: Final = {
|
||||
"context_line": f"Bearer {VIRTUAL_KEY}",
|
||||
"pre_context": [MASTER_KEY],
|
||||
"post_context": [VIRTUAL_KEY],
|
||||
}
|
||||
stacktrace: Final = {"frames": [{"context_line": MASTER_KEY, "pre_context": [VIRTUAL_KEY]}]}
|
||||
raise RuntimeError(f"rejected with {len(metadata)} metadata fields and {len(stacktrace)} stack fields")
|
||||
|
||||
|
||||
|
|
@ -92,15 +97,21 @@ def test_default_event_carries_no_email_hash_or_secret_anywhere() -> None:
|
|||
assert MASTER_KEY not in serialized
|
||||
assert VIRTUAL_KEY not in serialized
|
||||
assert "db-password-under-test" not in serialized
|
||||
frame_vars: Final = innermost_frame_vars(serialized)
|
||||
assert frame_vars["raw_headers"] == {"authorization": FILTERED, "x-api-key": FILTERED, "content-type": "'application/json'"}
|
||||
assert f"token='{FILTERED}'" in frame_vars["valid_token"]
|
||||
assert f"user_id='{FILTERED}'" in frame_vars["valid_token"]
|
||||
assert f"user_email='{FILTERED}'" in frame_vars["user_obj"]
|
||||
assert frame_vars["general_settings"] == {"master_key": FILTERED, "database_url": FILTERED}
|
||||
assert frame_vars["data"] == {"metadata": {"user_api_key_hash": FILTERED, "user_api_key_user_email": FILTERED}}
|
||||
assert "key_name='sk-...test'" in frame_vars["valid_token"]
|
||||
assert "user_role='internal_user'" in frame_vars["user_obj"]
|
||||
|
||||
|
||||
def raise_with_prompt_locals() -> None:
|
||||
messages: Final = [{"role": "user", "content": PROMPT}]
|
||||
raise RuntimeError(f"provider rejected {len(messages)} messages")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("env", "locals_sent"), ((PII_OFF, False), (PII_ON, True)), ids=("default", "pii-opt-in"))
|
||||
def test_frame_locals_carrying_prompts_are_sent_only_with_the_pii_opt_in(
|
||||
env: Mapping[str, str], locals_sent: bool
|
||||
) -> None:
|
||||
serialized: Final = capture_serialized_event(env, raise_with_prompt_locals)
|
||||
frames: Final = json.loads(serialized)["exception"]["values"][0]["stacktrace"]["frames"]
|
||||
assert (PROMPT in serialized) is locals_sent
|
||||
assert any("vars" in frame for frame in frames) is locals_sent
|
||||
|
||||
|
||||
def test_source_context_lines_are_left_readable() -> None:
|
||||
|
|
@ -115,11 +126,9 @@ def test_source_context_lines_are_left_readable() -> None:
|
|||
|
||||
|
||||
def test_source_context_names_outside_stack_frames_are_scrubbed() -> None:
|
||||
serialized: Final = capture_serialized_event(PII_OFF, raise_with_source_context_named_locals)
|
||||
serialized: Final = capture_serialized_event(PII_ON, raise_with_source_context_named_locals)
|
||||
assert VIRTUAL_KEY not in serialized
|
||||
assert MASTER_KEY not in serialized
|
||||
assert EMAIL not in serialized
|
||||
assert KEY_HASH not in serialized
|
||||
frame_vars: Final = innermost_frame_vars(serialized)
|
||||
assert frame_vars["metadata"] == {
|
||||
"context_line": f"'Bearer {FILTERED}'",
|
||||
|
|
@ -276,3 +285,8 @@ def test_init_options_defaults() -> None:
|
|||
assert options["traces_sample_rate"] == 1.0
|
||||
assert options["sample_rate"] == 1.0
|
||||
assert options["environment"] == "production"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("env", ({}, {"SENTRY_SEND_DEFAULT_PII": "true"}), ids=("default", "pii-opt-in"))
|
||||
def test_init_options_never_attach_request_bodies(env: Mapping[str, str]) -> None:
|
||||
assert build_sentry_init_options(env)["max_request_body_size"] == "never"
|
||||
|
|
|
|||
|
|
@ -186,6 +186,68 @@ async def test_tracing_config_automatically_exports_spend_without_a_storage_depe
|
|||
assert logger.task is not None and logger.task.done() and not logger.task.cancelled()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tracing_logs_an_error_naming_the_store_when_lens_is_misconfigured(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
from litellm.proxy.tracing_runtime import manage_tracing
|
||||
|
||||
monkeypatch.delenv("LITELLM_LENS_URL", raising=False)
|
||||
monkeypatch.delenv("LITELLM_LENS_SERVICE_TOKEN", raising=False)
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
|
||||
async with manage_tracing(enabled=True) as receiver:
|
||||
assert receiver is None
|
||||
|
||||
failures: Final = [
|
||||
record.getMessage()
|
||||
for record in caplog.records
|
||||
if record.levelno == logging.ERROR and "lens" in record.getMessage()
|
||||
]
|
||||
assert len(failures) == 1
|
||||
assert "LITELLM_LENS_URL" in failures[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"config_text",
|
||||
(
|
||||
"model_list: []\nlitellm_settings:\n failure_callback:\n - sentry\n",
|
||||
"model_list: []\nlitellm_settings:\n success_callback:\n - sentry\n",
|
||||
"model_list: []\nlitellm_settings:\n input_callback:\n - sentry\n",
|
||||
"model_list: []\ngeneral_settings:\n alerting:\n - sentry\n",
|
||||
),
|
||||
ids=("failure_callback", "success_callback", "input_callback", "alerting"),
|
||||
)
|
||||
async def test_proxy_config_starts_sentry_at_boot_without_request_bodies(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config_text: str
|
||||
) -> None:
|
||||
import sentry_sdk
|
||||
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
init: Final = MagicMock()
|
||||
monkeypatch.setattr(sentry_sdk, "init", init)
|
||||
monkeypatch.setenv("SENTRY_DSN", "https://key@sentry.example/1")
|
||||
monkeypatch.setattr(litellm_logging, "sentry_sdk_instance", None)
|
||||
litellm_logging.start_sentry.cache_clear()
|
||||
monkeypatch.setattr(litellm_logging, "capture_exception", None)
|
||||
monkeypatch.setattr(litellm_logging, "add_breadcrumb", None)
|
||||
monkeypatch.setattr(litellm, "input_callback", [])
|
||||
monkeypatch.setattr(litellm, "success_callback", [])
|
||||
monkeypatch.setattr(litellm, "failure_callback", [])
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False)
|
||||
monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
|
||||
config_file: Final = tmp_path / "sentry.yaml"
|
||||
config_file.write_text(config_text)
|
||||
|
||||
await ProxyConfig().load_config(router=None, config_file_path=str(config_file))
|
||||
|
||||
init.assert_called_once()
|
||||
assert init.call_args.kwargs["max_request_body_size"] == "never"
|
||||
assert litellm_logging.sentry_sdk_instance is sentry_sdk
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _is_remote_module_url
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
bug_report_notice,
|
||||
build_bug_report,
|
||||
)
|
||||
from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic
|
||||
from litellm.constants import (
|
||||
CLIENT_REQUESTED_MODEL_SCOPE_KEY,
|
||||
MAX_LITELLM_CALL_ID_LENGTH,
|
||||
|
|
@ -4674,6 +4675,21 @@ class TestHandleLLMApiExceptionRetryAfter:
|
|||
proxy_exc = await self._invoke(ValueError("some other failure"))
|
||||
assert "retry-after" not in proxy_exc.headers
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
(
|
||||
RouterRateLimitError(model="gpt-4", cooldown_time=42.3, enable_pre_call_checks=False, cooldown_list=[]),
|
||||
RouterRateLimitErrorBasic(model="gpt-4"),
|
||||
),
|
||||
ids=("cooldown", "basic"),
|
||||
)
|
||||
async def test_handle_llm_api_exception_answers_a_cooldown_with_429_and_no_bug_notice(self, exc, caplog):
|
||||
with caplog.at_level("ERROR"):
|
||||
proxy_exc = await self._invoke(exc)
|
||||
|
||||
assert (proxy_exc.code, proxy_exc.type) == ("429", "rate_limit_error")
|
||||
assert ISSUE_URL_BASE not in caplog.text
|
||||
|
||||
async def test_handle_llm_api_exception_strips_bug_report_notice_from_client_message(self, caplog):
|
||||
report = build_bug_report(RuntimeError("boom"), surface="sdk")
|
||||
notice = bug_report_notice(report)
|
||||
|
|
|
|||
|
|
@ -7,15 +7,15 @@ Covers ``failed_tracking_alert``, ``budget_alerts``, ``alerting_handler``,
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict
|
||||
from typing import Any, Dict, Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from prisma.errors import PrismaError
|
||||
|
||||
import litellm
|
||||
from litellm._service_logger import ServiceTypes
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
from litellm.proxy._types import AlertType, CallInfo
|
||||
|
||||
|
||||
|
|
@ -210,11 +210,45 @@ async def test_alerting_handler_sends_to_slack(proxy_logging):
|
|||
@pytest.mark.asyncio
|
||||
async def test_alerting_handler_sentry_without_sdk_error_raises(proxy_logging, monkeypatch):
|
||||
proxy_logging.alerting = ["sentry"]
|
||||
monkeypatch.setattr(litellm.utils, "sentry_sdk_instance", None)
|
||||
monkeypatch.delenv("SENTRY_DSN", raising=False)
|
||||
monkeypatch.setattr(litellm_logging, "sentry_sdk_instance", None)
|
||||
litellm_logging.start_sentry.cache_clear()
|
||||
with pytest.raises(Exception, match="SENTRY_DSN"):
|
||||
await proxy_logging.alerting_handler(message="x", level="Low", alert_type=AlertType.db_exceptions)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alerting_handler_sentry_sends_through_the_initialized_sdk(proxy_logging, monkeypatch):
|
||||
proxy_logging.alerting = ["sentry"]
|
||||
sentry = MagicMock()
|
||||
monkeypatch.setattr(litellm_logging, "sentry_sdk_instance", sentry)
|
||||
await proxy_logging.alerting_handler(message="db down", level="High", alert_type=AlertType.db_exceptions)
|
||||
sentry.capture_message.assert_called_once()
|
||||
assert "db down" in sentry.capture_message.call_args.args[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alerting_handler_starts_sentry_when_the_alert_was_not_configured_at_boot(proxy_logging, monkeypatch):
|
||||
import sentry_sdk
|
||||
|
||||
monkeypatch.setattr(proxy_logging, "alerting", ["sentry"])
|
||||
init: Final = MagicMock()
|
||||
capture_message: Final = MagicMock()
|
||||
monkeypatch.setattr(sentry_sdk, "init", init)
|
||||
monkeypatch.setattr(sentry_sdk, "capture_message", capture_message)
|
||||
monkeypatch.setenv("SENTRY_DSN", "https://key@sentry.example/1")
|
||||
monkeypatch.setattr(litellm_logging, "sentry_sdk_instance", None)
|
||||
litellm_logging.start_sentry.cache_clear()
|
||||
monkeypatch.setattr(litellm_logging, "capture_exception", None)
|
||||
monkeypatch.setattr(litellm_logging, "add_breadcrumb", None)
|
||||
|
||||
await proxy_logging.alerting_handler(message="db down", level="High", alert_type=AlertType.db_exceptions)
|
||||
|
||||
assert init.call_args.kwargs["dsn"] == "https://key@sentry.example/1"
|
||||
capture_message.assert_called_once()
|
||||
assert "db down" in capture_message.call_args.args[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# failure_handler
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -235,7 +269,7 @@ async def test_failure_handler_logs_db_error_and_calls_service_logging(proxy_log
|
|||
proxy_logging.alert_types = [AlertType.db_exceptions]
|
||||
proxy_logging.alerting_handler = AsyncMock()
|
||||
proxy_logging.service_logging_obj = MagicMock(async_service_failure_hook=AsyncMock())
|
||||
monkeypatch.setattr(litellm.utils, "capture_exception", None)
|
||||
monkeypatch.setattr(litellm_logging, "capture_exception", None)
|
||||
await proxy_logging.failure_handler(
|
||||
original_exception=HTTPException(status_code=500, detail="boom"),
|
||||
duration=1.5,
|
||||
|
|
@ -262,7 +296,7 @@ async def test_failure_handler_alerts_but_leaves_prisma_error_event_to_log_db_me
|
|||
proxy_logging.alert_types = [AlertType.db_exceptions]
|
||||
proxy_logging.alerting_handler = AsyncMock()
|
||||
proxy_logging.service_logging_obj = MagicMock(async_service_failure_hook=AsyncMock())
|
||||
monkeypatch.setattr(litellm.utils, "capture_exception", None)
|
||||
monkeypatch.setattr(litellm_logging, "capture_exception", None)
|
||||
await proxy_logging.failure_handler(
|
||||
original_exception=PrismaError("connection reset"), duration=1.0, call_type=call_type
|
||||
)
|
||||
|
|
@ -278,7 +312,7 @@ async def test_failure_handler_still_emits_db_event_for_wrapped_insert_error(pro
|
|||
proxy_logging.alert_types = [AlertType.db_exceptions]
|
||||
proxy_logging.alerting_handler = AsyncMock()
|
||||
proxy_logging.service_logging_obj = MagicMock(async_service_failure_hook=AsyncMock())
|
||||
monkeypatch.setattr(litellm.utils, "capture_exception", None)
|
||||
monkeypatch.setattr(litellm_logging, "capture_exception", None)
|
||||
await proxy_logging.failure_handler(
|
||||
original_exception=HTTPException(status_code=400, detail={"error": "Foreign Key Constraint failed"}),
|
||||
duration=1.0,
|
||||
|
|
@ -298,7 +332,7 @@ async def test_failure_handler_with_capture_exception_invoked(proxy_logging, mon
|
|||
def fake_capture(error):
|
||||
captured["error"] = error
|
||||
|
||||
monkeypatch.setattr(litellm.utils, "capture_exception", fake_capture)
|
||||
monkeypatch.setattr(litellm_logging, "capture_exception", fake_capture)
|
||||
err = RuntimeError("real")
|
||||
await proxy_logging.failure_handler(original_exception=err, duration=1.0, call_type="db_read")
|
||||
snapshot = {
|
||||
|
|
@ -320,7 +354,7 @@ async def test_failure_handler_propagates_service_logging_error_raises(proxy_log
|
|||
proxy_logging.service_logging_obj = MagicMock(
|
||||
async_service_failure_hook=AsyncMock(side_effect=RuntimeError("svc"))
|
||||
)
|
||||
monkeypatch.setattr(litellm.utils, "capture_exception", None)
|
||||
monkeypatch.setattr(litellm_logging, "capture_exception", None)
|
||||
with pytest.raises(RuntimeError):
|
||||
await proxy_logging.failure_handler(
|
||||
original_exception=Exception("x"), duration=0.0, call_type="db_read"
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from prometheus_client import REGISTRY
|
||||
|
||||
from litellm.tracing.exporter import MAX_BUFFER_EVENTS, MAX_EVENT_BYTES, ExportFailure, LensExporter, encode_record
|
||||
|
||||
|
|
@ -306,3 +308,51 @@ def test_serialization_rejects_recursive_payloads() -> None:
|
|||
@pytest.mark.parametrize("value", (float("nan"), object(), "\ud800"), ids=("nan", "unsupported", "surrogate"))
|
||||
def test_serialization_returns_a_failure_for_invalid_payloads(value: object) -> None:
|
||||
assert encode_record({"value": value}) is ExportFailure.INVALID
|
||||
|
||||
|
||||
def _counter_value(name: str) -> float:
|
||||
return REGISTRY.get_sample_value(name) or 0.0
|
||||
|
||||
|
||||
async def _drained(exporter: LensExporter) -> None:
|
||||
while exporter.buffered_events:
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_written_and_dropped_rows_are_exported_as_prometheus_counters() -> None:
|
||||
statuses: Final = iter((204, 500))
|
||||
|
||||
def answer(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(next(statuses))
|
||||
|
||||
written_before: Final = _counter_value("litellm_lens_export_rows_written_total")
|
||||
dropped_before: Final = _counter_value("litellm_lens_export_rows_dropped_total")
|
||||
async with httpx.AsyncClient(base_url="http://lens", transport=httpx.MockTransport(answer)) as client:
|
||||
exporter: Final = LensExporter(client)
|
||||
exporter.start()
|
||||
for request_id in ("accepted", "rejected"):
|
||||
await exporter.async_log_success_event(
|
||||
{"standard_logging_object": {"id": request_id, "response_cost": 0.1}}, None, None, None
|
||||
)
|
||||
await _drained(exporter)
|
||||
await exporter.aclose()
|
||||
assert (exporter.rows_written, exporter.rows_dropped) == (1, 1)
|
||||
assert _counter_value("litellm_lens_export_rows_written_total") - written_before == exporter.rows_written
|
||||
assert _counter_value("litellm_lens_export_rows_dropped_total") - dropped_before == exporter.rows_dropped
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_rejecting_store_is_logged_as_an_error_naming_lens(caplog: pytest.LogCaptureFixture) -> None:
|
||||
def reject(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(500)
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
|
||||
async with httpx.AsyncClient(base_url="http://lens", transport=httpx.MockTransport(reject)) as client:
|
||||
exporter: Final = LensExporter(client)
|
||||
assert exporter.enqueue(b"{}")
|
||||
exporter.start()
|
||||
await exporter.aclose()
|
||||
failures: Final = [record.getMessage() for record in caplog.records if record.levelno == logging.ERROR]
|
||||
assert len(failures) == 1
|
||||
assert "lens" in failures[0] and "HTTP 500" in failures[0]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue