Merge 1ed552cebf into ee1a6407cb

litellm/__init__.py

@@ -165,6 +165,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
     "posthog",
     "levo",
     "compression_interception",
+    "tickerr",
     "newrelic",
 ]
 cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None

litellm/integrations/callback_configs.json

@@ -1,4 +1,19 @@
 [
+  {
+    "id": "tickerr",
+    "displayName": "Tickerr",
+    "logo": "tickerr.png",
+    "supports_key_team_logging": false,
+    "dynamic_params": {
+      "TICKERR_REGION": {
+        "type": "text",
+        "ui_name": "Region",
+        "description": "Optional. Your deployment region, e.g. us-east-1. Used for regional signal breakdown.",
+        "required": false
+      }
+    },
+    "description": "Outage radar for AI agents. Reports LLM API failures anonymously to Tickerr and returns live signal from other agents — how many are hitting the same issue and which model to fall back to. No API key required."
+  },
   {
     "id": "arize",
     "displayName": "Arize",

litellm/integrations/tickerr.py

@@ -0,0 +1,158 @@
+"""
+Tickerr - crowd-sourced outage radar for AI agents.
+
+Reports LLM API failures to https://tickerr.ai so every agent
+can see when a provider is down and which model to fall back to.
+
+Usage:
+    litellm.callbacks = ["tickerr"]
+
+No API key. No account. Failure-only by default. Success sampling is opt-in.
+"""
+
+from __future__ import annotations
+
+import os
+import random
+import threading
+from collections.abc import Mapping
+from datetime import datetime
+
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.llms.custom_httpx.http_handler import (
+    _get_httpx_client,  # noqa: TID251  # internal API needed for non-blocking HTTP
+)
+
+_REPORT_URL = "https://tickerr.ai/api/v1/report"
+_UA = "litellm-tickerr/1.0"
+_MAX_INFLIGHT = threading.Semaphore(10)
+
+
+class TickerrLogger(CustomLogger):
+    """
+    LiteLLM callback that reports LLM API failures to Tickerr.
+
+    When explicitly enabled via ``litellm.callbacks = ["tickerr"]``,
+    anonymous failure metadata is reported. No prompts, responses,
+    API keys, or personal data are sent.
+
+    Optional env vars:
+        TICKERR_DISABLED      - set to "true" to disable all reporting
+        TICKERR_REGION        - e.g. us-east-1
+        TICKERR_SAMPLE_RATE   - fraction of successes to report (0.0-1.0, default 0 = off)
+    """
+
+    def __init__(
+        self, **kwargs: object
+    ) -> None:  # kwargs-ok: parent signature is untyped  # pyright: ignore[reportAny]
+        super().__init__(**kwargs)
+        self.disabled: bool = os.environ.get("TICKERR_DISABLED", "").lower() in (
+            "1",
+            "true",
+            "yes",
+        )
+        self.region: str | None = os.environ.get("TICKERR_REGION")
+        try:
+            self.sample_rate: float = min(1.0, max(0.0, float(os.environ.get("TICKERR_SAMPLE_RATE", "0"))))
+        except (ValueError, TypeError):
+            self.sample_rate = 0.0
+
+    def log_failure_event(  # pyright: ignore[reportAny]  # parent is untyped
+        self,
+        kwargs: Mapping[str, object],
+        response_obj: object,
+        start_time: datetime | float,
+        end_time: datetime | float,
+    ) -> None:
+        self._report(kwargs, start_time, end_time)
+
+    async def async_log_failure_event(  # pyright: ignore[reportAny]  # parent is untyped
+        self,
+        kwargs: Mapping[str, object],
+        response_obj: object,
+        start_time: datetime | float,
+        end_time: datetime | float,
+    ) -> None:
+        self._report(kwargs, start_time, end_time)
+
+    def log_success_event(  # pyright: ignore[reportAny]  # parent is untyped
+        self,
+        kwargs: Mapping[str, object],
+        response_obj: object,
+        start_time: datetime | float,
+        end_time: datetime | float,
+    ) -> None:
+        if self.sample_rate > 0 and random.random() < self.sample_rate:
+            self._report(kwargs, start_time, end_time, is_success=True)
+
+    async def async_log_success_event(  # pyright: ignore[reportAny]  # parent is untyped
+        self,
+        kwargs: Mapping[str, object],
+        response_obj: object,
+        start_time: datetime | float,
+        end_time: datetime | float,
+    ) -> None:
+        if self.sample_rate > 0 and random.random() < self.sample_rate:
+            self._report(kwargs, start_time, end_time, is_success=True)
+
+    def _report(
+        self,
+        kwargs: Mapping[str, object],
+        start_time: datetime | float,
+        end_time: datetime | float,
+        is_success: bool = False,
+    ) -> None:
+        if self.disabled:
+            return
+
+        model: str = str(kwargs.get("model", "") or "")
+
+        if isinstance(start_time, datetime) and isinstance(end_time, datetime):
+            latency = round((end_time - start_time).total_seconds() * 1000)
+        else:
+            latency = round((float(end_time) - float(start_time)) * 1000)
+
+        litellm_params = kwargs.get("litellm_params")
+        provider: str | None = None
+        if isinstance(litellm_params, Mapping):
+            raw = litellm_params.get("custom_llm_provider")
+            if isinstance(raw, str):
+                provider = raw
+        if provider is None:
+            raw_fallback = kwargs.get("custom_llm_provider")
+            if isinstance(raw_fallback, str):
+                provider = raw_fallback
+
+        exception = kwargs.get("exception")
+        status_code: int | None = None
+        raw_code = getattr(exception, "status_code", None)
+        if isinstance(raw_code, int):
+            status_code = raw_code
+
+        pairs: tuple[tuple[str, str | int], ...] = (
+            *((("provider", provider),) if provider is not None else ()),
+            *((("model", model),) if model else ()),
+            ("latency_ms", latency),
+            ("event_type", "success" if is_success else "failure"),
+            *((("status_code", status_code),) if status_code is not None else ()),
+            *((("region", self.region),) if self.region is not None else ()),
+        )
+        payload = dict(pairs)  # mutable-ok: consumed once by httpx.post(json=...)
+
+        def _send() -> None:
+            if not _MAX_INFLIGHT.acquire(blocking=False):
+                return
+            try:
+                client = _get_httpx_client()
+                client.post(
+                    _REPORT_URL,
+                    json=payload,
+                    headers={"User-Agent": _UA},  # mutable-ok: consumed once by httpx
+                    timeout=2,
+                )
+            except (OSError, ValueError):  # fire-and-forget; network errors are expected
+                pass
+            finally:
+                _MAX_INFLIGHT.release()
+
+        threading.Thread(target=_send, daemon=True).start()

litellm/litellm_core_utils/custom_logger_registry.py

@@ -47,6 +47,7 @@ from litellm.integrations.posthog import PostHogLogger
 from litellm.integrations.prometheus import PrometheusLogger
 from litellm.integrations.s3_v2 import S3Logger
 from litellm.integrations.sqs import SQSLogger
+from litellm.integrations.tickerr import TickerrLogger
 from litellm.integrations.vantage.vantage_logger import VantageLogger
 from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import (
     VectorStorePreCallHook,
@@ -107,6 +108,7 @@ class CustomLoggerRegistry:
         "mavvrik": MavvrikFocusLogger,
         "vantage": VantageLogger,
         "posthog": PostHogLogger,
+        "tickerr": TickerrLogger,
         "newrelic": NewRelicLogger,
     }
 

tests/test_litellm/integrations/test_tickerr_callback.py

@@ -0,0 +1,248 @@
+"""
+Unit tests for the Tickerr LiteLLM callback.
+"""
+
+import os
+from datetime import datetime, timedelta
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from litellm.integrations.tickerr import TickerrLogger
+
+
+# -- Config --------------------------------------------------------------------
+
+
+def test_default_init():
+    logger = TickerrLogger()
+    assert logger.disabled is False
+    assert logger.sample_rate == 0.0
+    assert logger.region is None
+
+
+def test_reads_env_vars():
+    with patch.dict(os.environ, {"TICKERR_REGION": "us-east-1", "TICKERR_SAMPLE_RATE": "0.1"}):
+        logger = TickerrLogger()
+    assert logger.region == "us-east-1"
+    assert logger.sample_rate == 0.1
+
+
+def test_disabled_flag():
+    with patch.dict(os.environ, {"TICKERR_DISABLED": "true"}):
+        logger = TickerrLogger()
+    assert logger.disabled is True
+
+
+def test_invalid_sample_rate_does_not_crash():
+    with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "notanumber"}):
+        logger = TickerrLogger()
+    assert logger.sample_rate == 0.0
+
+
+def test_sample_rate_clamped_to_one():
+    with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "5.0"}):
+        logger = TickerrLogger()
+    assert logger.sample_rate == 1.0
+
+
+# -- Disabled ------------------------------------------------------------------
+
+
+def test_disabled_skips_report():
+    with patch.dict(os.environ, {"TICKERR_DISABLED": "1"}):
+        logger = TickerrLogger()
+
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=100)
+
+    with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread:
+        logger._report({"model": "gpt-4o-mini", "exception": None}, start, end)
+        mock_thread.assert_not_called()
+
+
+# -- Payload -------------------------------------------------------------------
+
+
+def _run_report(logger, kwargs, start, end, is_success=False):
+    """Helper: run _report synchronously and return the payload sent via httpx."""
+    mock_client = MagicMock()
+    with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client):
+        with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread:
+            mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]()
+            logger._report(kwargs, start, end, is_success=is_success)
+    return mock_client.post.call_args
+
+
+def test_failure_payload():
+    logger = TickerrLogger()
+
+    exc = MagicMock()
+    exc.status_code = 429
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=500)
+    kwargs = {
+        "model": "claude-haiku-4-5",
+        "exception": exc,
+        "litellm_params": {"custom_llm_provider": "anthropic"},
+    }
+
+    call = _run_report(logger, kwargs, start, end)
+    payload = call.kwargs["json"]
+    assert payload["provider"] == "anthropic"
+    assert payload["model"] == "claude-haiku-4-5"
+    assert payload["status_code"] == 429
+    assert payload["event_type"] == "failure"
+    assert payload["latency_ms"] == 500
+
+
+def test_model_passed_as_is():
+    logger = TickerrLogger()
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=100)
+
+    call = _run_report(logger, {"model": "openai/gpt-4o-mini", "exception": None}, start, end)
+    payload = call.kwargs["json"]
+    assert payload["model"] == "openai/gpt-4o-mini"
+
+
+def test_no_exception_omits_status_code():
+    logger = TickerrLogger()
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=200)
+
+    call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end)
+    payload = call.kwargs["json"]
+    assert "status_code" not in payload
+
+
+def test_region_included():
+    with patch.dict(os.environ, {"TICKERR_REGION": "eu-west-1"}):
+        logger = TickerrLogger()
+
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=100)
+
+    call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end)
+    payload = call.kwargs["json"]
+    assert payload["region"] == "eu-west-1"
+
+
+def test_latency_from_floats():
+    logger = TickerrLogger()
+
+    call = _run_report(logger, {"model": "gpt-4o", "exception": None}, 1000.0, 1001.5)
+    payload = call.kwargs["json"]
+    assert payload["latency_ms"] == 1500
+
+
+def test_provider_from_top_level_kwarg():
+    logger = TickerrLogger()
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=100)
+
+    call = _run_report(logger, {"model": "gpt-4o", "exception": None, "custom_llm_provider": "openai"}, start, end)
+    payload = call.kwargs["json"]
+    assert payload["provider"] == "openai"
+
+
+def test_no_provider_omits_field():
+    logger = TickerrLogger()
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=100)
+
+    call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end)
+    payload = call.kwargs["json"]
+    assert "provider" not in payload
+
+
+# -- Hooks ---------------------------------------------------------------------
+
+
+def test_sync_failure_delegates():
+    logger = TickerrLogger()
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=300)
+
+    with patch.object(logger, "_report") as mock:
+        logger.log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end)
+        mock.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_async_failure_delegates():
+    logger = TickerrLogger()
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=200)
+
+    with patch.object(logger, "_report") as mock:
+        await logger.async_log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end)
+        mock.assert_called_once()
+
+
+# -- Success sampling ----------------------------------------------------------
+
+
+def test_success_not_reported_at_rate_zero():
+    logger = TickerrLogger()
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=100)
+
+    with patch.object(logger, "_report") as mock:
+        logger.log_success_event({"model": "gpt-4o", "exception": None}, None, start, end)
+        mock.assert_not_called()
+
+
+def test_success_reported_when_sampled():
+    with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}):
+        logger = TickerrLogger()
+
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=200)
+
+    call = _run_report(
+        logger,
+        {"model": "gpt-4o", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}},
+        start, end, is_success=True,
+    )
+    payload = call.kwargs["json"]
+    assert payload["event_type"] == "success"
+    assert payload["provider"] == "openai"
+
+
+@pytest.mark.asyncio
+async def test_async_success_reported_when_sampled():
+    with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}):
+        logger = TickerrLogger()
+
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=150)
+
+    mock_client = MagicMock()
+    with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client):
+        with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread:
+            mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]()
+            await logger.async_log_success_event(
+                {"model": "gpt-4o-mini", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}},
+                None, start, end,
+            )
+
+    payload = mock_client.post.call_args.kwargs["json"]
+    assert payload["event_type"] == "success"
+
+
+# -- Network failure does not crash --------------------------------------------
+
+
+def test_silent_on_network_error():
+    logger = TickerrLogger()
+    start = datetime(2024, 1, 1)
+    end = start + timedelta(milliseconds=100)
+
+    mock_client = MagicMock()
+    mock_client.post.side_effect = OSError("refused")
+    with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client):
+        with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread:
+            mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]()
+            logger._report({"model": "gpt-4o", "exception": None}, start, end)
+    # No exception raised — test passes