diff --git a/litellm/__init__.py b/litellm/__init__.py index 62477dd6264..0af2600b924 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -165,6 +165,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "posthog", "levo", "compression_interception", + "tickerr", "newrelic", ] cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index 7a2295a35ae..2def1ba4259 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -1,4 +1,19 @@ [ + { + "id": "tickerr", + "displayName": "Tickerr", + "logo": "tickerr.png", + "supports_key_team_logging": false, + "dynamic_params": { + "TICKERR_REGION": { + "type": "text", + "ui_name": "Region", + "description": "Optional. Your deployment region, e.g. us-east-1. Used for regional signal breakdown.", + "required": false + } + }, + "description": "Outage radar for AI agents. Reports LLM API failures anonymously to Tickerr and returns live signal from other agents — how many are hitting the same issue and which model to fall back to. No API key required." + }, { "id": "arize", "displayName": "Arize", diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py new file mode 100644 index 00000000000..b51dcd6f6fe --- /dev/null +++ b/litellm/integrations/tickerr.py @@ -0,0 +1,158 @@ +""" +Tickerr - crowd-sourced outage radar for AI agents. + +Reports LLM API failures to https://tickerr.ai so every agent +can see when a provider is down and which model to fall back to. + +Usage: + litellm.callbacks = ["tickerr"] + +No API key. No account. Failure-only by default. Success sampling is opt-in. +""" + +from __future__ import annotations + +import os +import random +import threading +from collections.abc import Mapping +from datetime import datetime + +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.custom_httpx.http_handler import ( + _get_httpx_client, # noqa: TID251 # internal API needed for non-blocking HTTP +) + +_REPORT_URL = "https://tickerr.ai/api/v1/report" +_UA = "litellm-tickerr/1.0" +_MAX_INFLIGHT = threading.Semaphore(10) + + +class TickerrLogger(CustomLogger): + """ + LiteLLM callback that reports LLM API failures to Tickerr. + + When explicitly enabled via ``litellm.callbacks = ["tickerr"]``, + anonymous failure metadata is reported. No prompts, responses, + API keys, or personal data are sent. + + Optional env vars: + TICKERR_DISABLED - set to "true" to disable all reporting + TICKERR_REGION - e.g. us-east-1 + TICKERR_SAMPLE_RATE - fraction of successes to report (0.0-1.0, default 0 = off) + """ + + def __init__( + self, **kwargs: object + ) -> None: # kwargs-ok: parent signature is untyped # pyright: ignore[reportAny] + super().__init__(**kwargs) + self.disabled: bool = os.environ.get("TICKERR_DISABLED", "").lower() in ( + "1", + "true", + "yes", + ) + self.region: str | None = os.environ.get("TICKERR_REGION") + try: + self.sample_rate: float = min(1.0, max(0.0, float(os.environ.get("TICKERR_SAMPLE_RATE", "0")))) + except (ValueError, TypeError): + self.sample_rate = 0.0 + + def log_failure_event( # pyright: ignore[reportAny] # parent is untyped + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime | float, + end_time: datetime | float, + ) -> None: + self._report(kwargs, start_time, end_time) + + async def async_log_failure_event( # pyright: ignore[reportAny] # parent is untyped + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime | float, + end_time: datetime | float, + ) -> None: + self._report(kwargs, start_time, end_time) + + def log_success_event( # pyright: ignore[reportAny] # parent is untyped + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime | float, + end_time: datetime | float, + ) -> None: + if self.sample_rate > 0 and random.random() < self.sample_rate: + self._report(kwargs, start_time, end_time, is_success=True) + + async def async_log_success_event( # pyright: ignore[reportAny] # parent is untyped + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime | float, + end_time: datetime | float, + ) -> None: + if self.sample_rate > 0 and random.random() < self.sample_rate: + self._report(kwargs, start_time, end_time, is_success=True) + + def _report( + self, + kwargs: Mapping[str, object], + start_time: datetime | float, + end_time: datetime | float, + is_success: bool = False, + ) -> None: + if self.disabled: + return + + model: str = str(kwargs.get("model", "") or "") + + if isinstance(start_time, datetime) and isinstance(end_time, datetime): + latency = round((end_time - start_time).total_seconds() * 1000) + else: + latency = round((float(end_time) - float(start_time)) * 1000) + + litellm_params = kwargs.get("litellm_params") + provider: str | None = None + if isinstance(litellm_params, Mapping): + raw = litellm_params.get("custom_llm_provider") + if isinstance(raw, str): + provider = raw + if provider is None: + raw_fallback = kwargs.get("custom_llm_provider") + if isinstance(raw_fallback, str): + provider = raw_fallback + + exception = kwargs.get("exception") + status_code: int | None = None + raw_code = getattr(exception, "status_code", None) + if isinstance(raw_code, int): + status_code = raw_code + + pairs: tuple[tuple[str, str | int], ...] = ( + *((("provider", provider),) if provider is not None else ()), + *((("model", model),) if model else ()), + ("latency_ms", latency), + ("event_type", "success" if is_success else "failure"), + *((("status_code", status_code),) if status_code is not None else ()), + *((("region", self.region),) if self.region is not None else ()), + ) + payload = dict(pairs) # mutable-ok: consumed once by httpx.post(json=...) + + def _send() -> None: + if not _MAX_INFLIGHT.acquire(blocking=False): + return + try: + client = _get_httpx_client() + client.post( + _REPORT_URL, + json=payload, + headers={"User-Agent": _UA}, # mutable-ok: consumed once by httpx + timeout=2, + ) + except (OSError, ValueError): # fire-and-forget; network errors are expected + pass + finally: + _MAX_INFLIGHT.release() + + threading.Thread(target=_send, daemon=True).start() diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index 6449aa4d46e..1f0b6f2fc4d 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -47,6 +47,7 @@ from litellm.integrations.posthog import PostHogLogger from litellm.integrations.prometheus import PrometheusLogger from litellm.integrations.s3_v2 import S3Logger from litellm.integrations.sqs import SQSLogger +from litellm.integrations.tickerr import TickerrLogger from litellm.integrations.vantage.vantage_logger import VantageLogger from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( VectorStorePreCallHook, @@ -107,6 +108,7 @@ class CustomLoggerRegistry: "mavvrik": MavvrikFocusLogger, "vantage": VantageLogger, "posthog": PostHogLogger, + "tickerr": TickerrLogger, "newrelic": NewRelicLogger, } diff --git a/tests/test_litellm/integrations/test_tickerr_callback.py b/tests/test_litellm/integrations/test_tickerr_callback.py new file mode 100644 index 00000000000..e5e679ccefd --- /dev/null +++ b/tests/test_litellm/integrations/test_tickerr_callback.py @@ -0,0 +1,248 @@ +""" +Unit tests for the Tickerr LiteLLM callback. +""" + +import os +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.integrations.tickerr import TickerrLogger + + +# -- Config -------------------------------------------------------------------- + + +def test_default_init(): + logger = TickerrLogger() + assert logger.disabled is False + assert logger.sample_rate == 0.0 + assert logger.region is None + + +def test_reads_env_vars(): + with patch.dict(os.environ, {"TICKERR_REGION": "us-east-1", "TICKERR_SAMPLE_RATE": "0.1"}): + logger = TickerrLogger() + assert logger.region == "us-east-1" + assert logger.sample_rate == 0.1 + + +def test_disabled_flag(): + with patch.dict(os.environ, {"TICKERR_DISABLED": "true"}): + logger = TickerrLogger() + assert logger.disabled is True + + +def test_invalid_sample_rate_does_not_crash(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "notanumber"}): + logger = TickerrLogger() + assert logger.sample_rate == 0.0 + + +def test_sample_rate_clamped_to_one(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "5.0"}): + logger = TickerrLogger() + assert logger.sample_rate == 1.0 + + +# -- Disabled ------------------------------------------------------------------ + + +def test_disabled_skips_report(): + with patch.dict(os.environ, {"TICKERR_DISABLED": "1"}): + logger = TickerrLogger() + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + logger._report({"model": "gpt-4o-mini", "exception": None}, start, end) + mock_thread.assert_not_called() + + +# -- Payload ------------------------------------------------------------------- + + +def _run_report(logger, kwargs, start, end, is_success=False): + """Helper: run _report synchronously and return the payload sent via httpx.""" + mock_client = MagicMock() + with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report(kwargs, start, end, is_success=is_success) + return mock_client.post.call_args + + +def test_failure_payload(): + logger = TickerrLogger() + + exc = MagicMock() + exc.status_code = 429 + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=500) + kwargs = { + "model": "claude-haiku-4-5", + "exception": exc, + "litellm_params": {"custom_llm_provider": "anthropic"}, + } + + call = _run_report(logger, kwargs, start, end) + payload = call.kwargs["json"] + assert payload["provider"] == "anthropic" + assert payload["model"] == "claude-haiku-4-5" + assert payload["status_code"] == 429 + assert payload["event_type"] == "failure" + assert payload["latency_ms"] == 500 + + +def test_model_passed_as_is(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + call = _run_report(logger, {"model": "openai/gpt-4o-mini", "exception": None}, start, end) + payload = call.kwargs["json"] + assert payload["model"] == "openai/gpt-4o-mini" + + +def test_no_exception_omits_status_code(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) + + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) + payload = call.kwargs["json"] + assert "status_code" not in payload + + +def test_region_included(): + with patch.dict(os.environ, {"TICKERR_REGION": "eu-west-1"}): + logger = TickerrLogger() + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) + payload = call.kwargs["json"] + assert payload["region"] == "eu-west-1" + + +def test_latency_from_floats(): + logger = TickerrLogger() + + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, 1000.0, 1001.5) + payload = call.kwargs["json"] + assert payload["latency_ms"] == 1500 + + +def test_provider_from_top_level_kwarg(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + call = _run_report(logger, {"model": "gpt-4o", "exception": None, "custom_llm_provider": "openai"}, start, end) + payload = call.kwargs["json"] + assert payload["provider"] == "openai" + + +def test_no_provider_omits_field(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) + payload = call.kwargs["json"] + assert "provider" not in payload + + +# -- Hooks --------------------------------------------------------------------- + + +def test_sync_failure_delegates(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=300) + + with patch.object(logger, "_report") as mock: + logger.log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end) + mock.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_failure_delegates(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) + + with patch.object(logger, "_report") as mock: + await logger.async_log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end) + mock.assert_called_once() + + +# -- Success sampling ---------------------------------------------------------- + + +def test_success_not_reported_at_rate_zero(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch.object(logger, "_report") as mock: + logger.log_success_event({"model": "gpt-4o", "exception": None}, None, start, end) + mock.assert_not_called() + + +def test_success_reported_when_sampled(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}): + logger = TickerrLogger() + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) + + call = _run_report( + logger, + {"model": "gpt-4o", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, + start, end, is_success=True, + ) + payload = call.kwargs["json"] + assert payload["event_type"] == "success" + assert payload["provider"] == "openai" + + +@pytest.mark.asyncio +async def test_async_success_reported_when_sampled(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}): + logger = TickerrLogger() + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=150) + + mock_client = MagicMock() + with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + await logger.async_log_success_event( + {"model": "gpt-4o-mini", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, + None, start, end, + ) + + payload = mock_client.post.call_args.kwargs["json"] + assert payload["event_type"] == "success" + + +# -- Network failure does not crash -------------------------------------------- + + +def test_silent_on_network_error(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + mock_client = MagicMock() + mock_client.post.side_effect = OSError("refused") + with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, start, end) + # No exception raised — test passes diff --git a/ui/litellm-dashboard/public/assets/logos/tickerr.png b/ui/litellm-dashboard/public/assets/logos/tickerr.png new file mode 100644 index 00000000000..c33d829d4d8 Binary files /dev/null and b/ui/litellm-dashboard/public/assets/logos/tickerr.png differ