mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-21 05:15:06 +00:00
38f3f8a1a9
* fix(router): carry a 429 status on cooldown and no-deployment errors RouterRateLimitError and RouterRateLimitErrorBasic had no status_code, so the proxy answered an all-deployments-cooled-down request as a generic 500 class error and logged the bug report notice for every cooldown. Both now carry status_code 429, which the proxy maps to a rate_limit_error and which the retry policy already treats as retryable * fix(proxy): start Sentry at config load and never attach request bodies Sentry only came up inside set_callbacks on the first LLM call, so boot and background errors were never reported, and general_settings.alerting: [sentry] always raised Missing SENTRY_DSN because the alerting path read a dead litellm.utils pin instead of the live SDK. The proxy now initializes Sentry once while loading the config whenever sentry is a configured callback or alerting target, the alerting and failure handlers read the live instance, and max_request_body_size is pinned to never so prompts and completions stay out of Sentry events * fix(tracing): log Lens failures at ERROR and export row counters to Prometheus A tracing store that fails to start, or a Lens service that rejects a batch, was only logged at WARNING, so operators running at the default ERROR level never saw tracing silently go dark. Both now log at ERROR and name the lens store. The exporter's rows_written and rows_dropped tallies were never exported, so the two counts are now Prometheus counters litellm_lens_export_rows_written_total and litellm_lens_export_rows_dropped_total, registered lazily on the default registry like the other proxy-level counters and charted on the all-metrics Grafana dashboard * test(proxy): drop the litellm import the alerting tests no longer use * fix(proxy): start Sentry once a DSN is set, from any config source Boot-time init used to lock Sentry in even with no SENTRY_DSN, so a DSN loaded later from DB environment_variables was never picked up. initialize_sentry now skips when no DSN is set and re-inits when the DSN changes. alerting_handler calls it too, so alerting: [sentry] set through the DB works. Drops the litellm_settings.callbacks check since that config fails to load with sentry in it * fix(sentry): keep stack frame locals out of events unless PII is opted in Frame locals carried prompts (messages, request bodies, provider options) into every error event. include_local_variables now follows SENTRY_SEND_DEFAULT_PII * test(router): drop the test that only pinned the 429 class attribute * fix(proxy): start Sentry at boot when it is only an input callback * fix(proxy): keep Sentry init and alerting within the strict ruff and basedpyright ceilings --------- Co-authored-by: yuneng-jiang <yuneng@berri.ai>
77 lines
2.9 KiB
Python
77 lines
2.9 KiB
Python
from collections.abc import AsyncGenerator, Callable, Mapping
|
|
from contextlib import asynccontextmanager
|
|
from typing import Final
|
|
|
|
import httpx
|
|
from fastapi import HTTPException, Request
|
|
from pydantic import ConfigDict, TypeAdapter
|
|
|
|
import litellm
|
|
from litellm._logging import verbose_proxy_logger
|
|
from litellm.tracing import TraceReceiver
|
|
from litellm.tracing.exporter import LensExporter
|
|
from litellm.tracing.remote import LensConnection, RemoteTraceStore
|
|
from litellm.tracing.storage import LensTraceStorage
|
|
|
|
_RECEIVER_ADAPTER: Final[TypeAdapter[TraceReceiver | None]] = TypeAdapter(
|
|
TraceReceiver | None, config=ConfigDict(arbitrary_types_allowed=True)
|
|
)
|
|
_UNAVAILABLE_DETAIL: Final = "Agent tracing is not enabled. Configure the Lens service and LITELLM_LENS_URL."
|
|
|
|
|
|
def require_receiver(tracing: TraceReceiver | None) -> TraceReceiver:
|
|
if tracing is None:
|
|
raise HTTPException(status_code=501, detail=_UNAVAILABLE_DETAIL)
|
|
return tracing
|
|
|
|
|
|
async def provide_receiver(request: Request) -> TraceReceiver | None:
|
|
return _RECEIVER_ADAPTER.validate_python(getattr(request.state, "tracing_receiver", None))
|
|
|
|
|
|
async def provide_storage(request: Request) -> LensTraceStorage | None:
|
|
tracing: Final = await provide_receiver(request)
|
|
return tracing.storage if tracing is not None else None
|
|
|
|
|
|
@asynccontextmanager
|
|
async def manage_tracing(
|
|
enabled: bool,
|
|
receiver_factory: Callable[[], TraceReceiver] | None = None,
|
|
settings: Mapping[str, object] | None = None,
|
|
client_factory: Callable[[LensConnection], httpx.AsyncClient] = LensConnection.lifespan_client,
|
|
) -> AsyncGenerator[TraceReceiver | None, None]:
|
|
if not enabled:
|
|
yield None
|
|
return
|
|
try:
|
|
connection: Final = LensConnection.from_env()
|
|
except ValueError as error:
|
|
verbose_proxy_logger.error("Agent tracing store lens failed to start, tracing is disabled: %s", error)
|
|
yield None
|
|
return
|
|
async with client_factory(connection) as client:
|
|
tracing: Final = (
|
|
receiver_factory()
|
|
if receiver_factory
|
|
else TraceReceiver(storage=LensTraceStorage(RemoteTraceStore(client)))
|
|
)
|
|
async with _export_requests(LensExporter(client)):
|
|
yield tracing
|
|
|
|
|
|
@asynccontextmanager
|
|
async def _export_requests(spend_logger: LensExporter) -> AsyncGenerator[None, None]:
|
|
spend_logger.start()
|
|
manager: Final = litellm.logging_callback_manager
|
|
manager.add_litellm_callback(spend_logger)
|
|
manager.add_litellm_success_callback(spend_logger)
|
|
manager.add_litellm_failure_callback(spend_logger)
|
|
manager.add_litellm_async_success_callback(spend_logger)
|
|
manager.add_litellm_async_failure_callback(spend_logger)
|
|
verbose_proxy_logger.info("Agent tracing enabled (store=lens)")
|
|
try:
|
|
yield None
|
|
finally:
|
|
manager.remove_callback_from_all_lists(spend_logger)
|
|
await spend_logger.aclose()
|