From 64aab2de560a44946247e9f325e9ae0522fcdfd6 Mon Sep 17 00:00:00 2001 From: imviky-ctrl <272028991+imviky-ctrl@users.noreply.github.com> Date: Tue, 5 May 2026 15:02:51 +0530 Subject: [PATCH 01/20] feat(integrations): add Tickerr callback for LLM failure reporting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Tickerr as a built-in LiteLLM callback. Tickerr is a crowd-sourced outage radar for AI agents — when an agent hits a 5xx error, it reports anonymously to Tickerr and receives live signal from other agents hitting the same issue (including a fallback model recommendation). Changes: - litellm/integrations/tickerr.py — TickerrLogger (stdlib-only, no new deps) - litellm/__init__.py — adds "tickerr" to _custom_logger_compatible_callbacks_literal - litellm/litellm_core_utils/custom_logger_registry.py — registers TickerrLogger - litellm/integrations/callback_configs.json — UI config entry - docs/my-website/docs/observability/tickerr.md — integration docs Usage: litellm.callbacks = ["tickerr"] No API key required. Anonymous. Non-blocking (daemon thread, 5s timeout). https://tickerr.ai --- docs/my-website/docs/observability/tickerr.md | 107 +++++++++ litellm/__init__.py | 1 + litellm/integrations/callback_configs.json | 21 ++ litellm/integrations/tickerr.py | 206 ++++++++++++++++++ .../custom_logger_registry.py | 2 + 5 files changed, 337 insertions(+) create mode 100644 docs/my-website/docs/observability/tickerr.md create mode 100644 litellm/integrations/tickerr.py diff --git a/docs/my-website/docs/observability/tickerr.md b/docs/my-website/docs/observability/tickerr.md new file mode 100644 index 00000000000..d4f9855d998 --- /dev/null +++ b/docs/my-website/docs/observability/tickerr.md @@ -0,0 +1,107 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Tickerr — Outage Radar for AI Agents + +[Tickerr](https://tickerr.ai) is a crowd-sourced outage detector for LLM APIs. When your agent hits a 5xx error or rate limit, Tickerr tells you: + +- How many other agents are seeing the same issue right now +- Current signal state: `quiet` → `detecting` → `confirmed` → `recovering` +- Which model to fall back to + +**No API key required. Anonymous. Zero overhead on success paths.** + +## Quick Start + + + + +```python +import litellm + +litellm.callbacks = ["tickerr"] + +# Every failed LiteLLM call is now reported to Tickerr automatically. +# Tickerr fires in a background thread — your agent is never blocked. +response = litellm.completion( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}] +) +``` + + + + +```yaml +model_list: + - model_name: claude-haiku + litellm_params: + model: anthropic/claude-haiku-4-5 + api_key: os.environ/ANTHROPIC_API_KEY + +litellm_settings: + callbacks: ["tickerr"] +``` + + + + +## What Gets Reported + +Tickerr receives only: + +| Field | Example | +|-------|---------| +| Provider | `anthropic` | +| Model | `claude-haiku-4-5` | +| HTTP status code | `529` | +| Error type | `overloaded` | +| Latency (ms) | `1240` | + +No prompts, no responses, no personal data. + +## What You Get Back + +Each report updates the live signal at [tickerr.ai/agent-reports](https://tickerr.ai/agent-reports). + +To read the signal from your agent, use the [Tickerr MCP server](https://tickerr.ai/mcp-server) `report_incident` tool. It returns a structured response: + +``` +CURRENT SIGNAL (anthropic/claude-haiku-4-5) +Status: CONFIRMED +Agents reporting (last 10 min): 14 +Total reports (last 10 min): 31 + +RECOMMENDATION +Action: FALLBACK +Switch to: gpt-4o-mini (openai) +``` + +## Optional Configuration + +```python +import os + +os.environ["TICKERR_CLIENT_TIER"] = "pro" # free | pro | team | enterprise | api_pay_as_you_go +os.environ["TICKERR_REGION"] = "us-east-1" # optional, for regional breakdown +``` + +## Signal States + +| State | Meaning | +|-------|---------| +| `quiet` | No reports in last 10 min | +| `detecting` | 1–2 agents reporting | +| `confirmed` | 3+ distinct agents — issue verified | +| `recovering` | Reports dropping, recovery signals arriving | + +## Opt Out + +[tickerr.ai/mcp/opt-out](https://tickerr.ai/mcp/opt-out) + +## Links + +- [Tickerr](https://tickerr.ai) — live AI status dashboard (90+ tools) +- [Agent reports](https://tickerr.ai/agent-reports) — live feed +- [Tickerr MCP server](https://tickerr.ai/mcp-server) — 9-tool MCP for agents +- [REST API](https://tickerr.ai/api/v1/report) — report without LiteLLM diff --git a/litellm/__init__.py b/litellm/__init__.py index 5305edc9be6..b512ee3c49b 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -149,6 +149,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "posthog", "levo", "compression_interception", + "tickerr", ] cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index c2b0c4ddce9..5839767764c 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -1,4 +1,25 @@ [ + { + "id": "tickerr", + "displayName": "Tickerr", + "logo": "tickerr.png", + "supports_key_team_logging": false, + "dynamic_params": { + "TICKERR_CLIENT_TIER": { + "type": "text", + "ui_name": "Client Tier", + "description": "Optional. Your LLM plan tier: free, pro, team, enterprise, or api_pay_as_you_go. Used to correlate signals by tier.", + "required": false + }, + "TICKERR_REGION": { + "type": "text", + "ui_name": "Region", + "description": "Optional. Your deployment region, e.g. us-east-1. Used for regional signal breakdown.", + "required": false + } + }, + "description": "Outage radar for AI agents. Reports LLM API failures anonymously to Tickerr and returns live signal from other agents — how many are hitting the same issue and which model to fall back to. No API key required." + }, { "id": "arize", "displayName": "Arize", diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py new file mode 100644 index 00000000000..02b3ae973ac --- /dev/null +++ b/litellm/integrations/tickerr.py @@ -0,0 +1,206 @@ +""" +Tickerr — outage radar for AI agents. + +Reports LLM API failures to https://tickerr.ai so agents can +see how many other agents are hitting the same issue and get +a live routing recommendation (RETRY / RETRY_WITH_DELAY / FALLBACK). + +Zero dependencies beyond stdlib. Anonymous. Non-blocking. + +Usage: + litellm.callbacks = ["tickerr"] + +No API key required. +""" + +from __future__ import annotations + +import os +import re +import threading +from typing import Any, Dict, Optional + +from litellm.integrations.custom_logger import CustomLogger + +_REPORT_URL = "https://tickerr.ai/api/v1/report" +_UA = "litellm-tickerr/1.0" + +# Map litellm custom_llm_provider → Tickerr provider slug +_PROVIDER_MAP: Dict[str, str] = { + "openai": "openai", + "anthropic": "anthropic", + "google": "google", + "vertex_ai": "google", + "gemini": "google", + "cohere": "cohere", + "mistral": "mistral", + "groq": "groq", + "together_ai": "together", + "huggingface": "huggingface", + "replicate": "replicate", + "deepinfra": "deepinfra", + "perplexity": "perplexity", + "fireworks_ai": "fireworks", + "openrouter": "openrouter", + "azure": "azure", + "bedrock": "aws", + "ai21": "ai21", + "cerebras": "cerebras", + "xai": "xai", + "deepseek": "deepseek", + "ollama": "ollama", + "nlp_cloud": "nlp_cloud", +} + +_ERROR_TYPE_MAP: Dict[int, str] = { + 429: "rate_limit", + 529: "overloaded", + 503: "overloaded", + 500: "overloaded", + 408: "timeout", + 524: "timeout", + 401: "auth", + 403: "auth", +} + + +def _normalize_provider(model: str, kwargs: Dict[str, Any]) -> str: + custom = ( + kwargs.get("litellm_params", {}).get("custom_llm_provider") + or kwargs.get("custom_llm_provider") + or "" + ) + if custom: + return _PROVIDER_MAP.get(custom.lower(), custom.lower()) + if "/" in model: + prefix = model.split("/")[0].lower() + return _PROVIDER_MAP.get(prefix, prefix) + if re.match(r"^claude", model, re.I): + return "anthropic" + if re.match(r"^gpt|^o[1-9]", model, re.I): + return "openai" + if re.match(r"^gemini", model, re.I): + return "google" + if re.match(r"^mistral|^mixtral", model, re.I): + return "mistral" + if re.match(r"^llama", model, re.I): + return "meta" + if re.match(r"^command", model, re.I): + return "cohere" + if re.match(r"^grok", model, re.I): + return "xai" + if re.match(r"^deepseek", model, re.I): + return "deepseek" + return "unknown" + + +def _extract_status_code(exception: Optional[BaseException]) -> Optional[int]: + if exception is None: + return None + code = getattr(exception, "status_code", None) + if isinstance(code, int): + return code + if isinstance(code, str) and code.isdigit(): + return int(code) + return None + + +def _fire_and_forget(payload: Dict[str, Any]) -> None: + """POST to Tickerr in a daemon thread — never blocks the caller.""" + + def _send() -> None: + try: + import json as _json + import urllib.request + + data = _json.dumps(payload).encode() + req = urllib.request.Request( + _REPORT_URL, + data=data, + headers={"Content-Type": "application/json", "User-Agent": _UA}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=5): + pass + except Exception: + pass # never crash the caller + + t = threading.Thread(target=_send, daemon=True) + t.start() + + +class TickerrLogger(CustomLogger): + """ + LiteLLM built-in callback for Tickerr. + + Activated with: + litellm.callbacks = ["tickerr"] + + Optional env vars: + TICKERR_CLIENT_TIER — "free" | "pro" | "team" | "enterprise" | "api_pay_as_you_go" + TICKERR_REGION — e.g. "us-east-1" + """ + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.client_tier: Optional[str] = os.environ.get("TICKERR_CLIENT_TIER") + self.region: Optional[str] = os.environ.get("TICKERR_REGION") + + # ── sync ────────────────────────────────────────────────────────────────── + + def log_failure_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: float, + end_time: float, + ) -> None: + self._report(kwargs, start_time, end_time, is_resolution=False) + + # ── async ───────────────────────────────────────────────────────────────── + + async def async_log_failure_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: float, + end_time: float, + ) -> None: + self._report(kwargs, start_time, end_time, is_resolution=False) + + # ── internal ────────────────────────────────────────────────────────────── + + def _report( + self, + kwargs: Dict[str, Any], + start_time: float, + end_time: float, + is_resolution: bool, + ) -> None: + model: str = kwargs.get("model", "") or "" + exception: Optional[BaseException] = kwargs.get("exception") + latency_ms = round((end_time - start_time) * 1000) + + provider = _normalize_provider(model, kwargs) + status_code = _extract_status_code(exception) + error_type = _ERROR_TYPE_MAP.get(status_code, "overloaded") if status_code else None + + # Strip provider prefix: "anthropic/claude-3-5-haiku" → "claude-3-5-haiku" + model_clean = model.split("/", 1)[-1] if "/" in model else model + + payload: Dict[str, Any] = { + "provider": provider, + "model": model_clean or None, + "is_resolution": is_resolution, + "latency_ms": latency_ms, + } + if status_code is not None: + payload["error_code"] = status_code + if error_type: + payload["error_type"] = error_type + if self.client_tier: + payload["client_tier"] = self.client_tier + if self.region: + payload["region"] = self.region + + _fire_and_forget(payload) diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index f873bfeece5..be65973112d 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -45,6 +45,7 @@ from litellm.integrations.posthog import PostHogLogger from litellm.integrations.prometheus import PrometheusLogger from litellm.integrations.s3_v2 import S3Logger from litellm.integrations.sqs import SQSLogger +from litellm.integrations.tickerr import TickerrLogger from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( VectorStorePreCallHook, ) @@ -102,6 +103,7 @@ class CustomLoggerRegistry: "focus": FocusLogger, "vantage": VantageLogger, "posthog": PostHogLogger, + "tickerr": TickerrLogger, } try: From f4b0d84a98d320b73d6f07da093d838f82b5c607 Mon Sep 17 00:00:00 2001 From: imviky-ctrl <272028991+imviky-ctrl@users.noreply.github.com> Date: Tue, 5 May 2026 15:02:51 +0530 Subject: [PATCH 02/20] fix(tickerr): fix TypeError, error-type fallback, and thread cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _latency_ms(): handle both datetime objects and floats — LiteLLM passes datetime, not float, causing TypeError on every invocation - _ERROR_TYPE_MAP fallback: removed default "overloaded" for unmapped codes (400, 404, 502, etc.) — now returns None for unrecognised codes - _fire_and_forget(): added semaphore cap (_MAX_INFLIGHT=5) to prevent unbounded thread spawning during burst failures - Removed dead is_resolution parameter from _report() — only called from failure hooks so it was always False --- litellm/integrations/tickerr.py | 97 ++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 38 deletions(-) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index 02b3ae973ac..9fb933bd937 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -18,40 +18,46 @@ from __future__ import annotations import os import re import threading -from typing import Any, Dict, Optional +from datetime import datetime +from typing import Any, Dict, Optional, Union from litellm.integrations.custom_logger import CustomLogger _REPORT_URL = "https://tickerr.ai/api/v1/report" _UA = "litellm-tickerr/1.0" +# Cap concurrent in-flight reports to avoid thread exhaustion on burst failures +_MAX_INFLIGHT = 5 +_inflight = threading.Semaphore(_MAX_INFLIGHT) + # Map litellm custom_llm_provider → Tickerr provider slug _PROVIDER_MAP: Dict[str, str] = { - "openai": "openai", - "anthropic": "anthropic", - "google": "google", - "vertex_ai": "google", - "gemini": "google", - "cohere": "cohere", - "mistral": "mistral", - "groq": "groq", - "together_ai": "together", - "huggingface": "huggingface", - "replicate": "replicate", - "deepinfra": "deepinfra", - "perplexity": "perplexity", - "fireworks_ai": "fireworks", - "openrouter": "openrouter", - "azure": "azure", - "bedrock": "aws", - "ai21": "ai21", - "cerebras": "cerebras", - "xai": "xai", - "deepseek": "deepseek", - "ollama": "ollama", - "nlp_cloud": "nlp_cloud", + "openai": "openai", + "anthropic": "anthropic", + "google": "google", + "vertex_ai": "google", + "gemini": "google", + "cohere": "cohere", + "mistral": "mistral", + "groq": "groq", + "together_ai": "together", + "huggingface": "huggingface", + "replicate": "replicate", + "deepinfra": "deepinfra", + "perplexity": "perplexity", + "fireworks_ai": "fireworks", + "openrouter": "openrouter", + "azure": "azure", + "bedrock": "aws", + "ai21": "ai21", + "cerebras": "cerebras", + "xai": "xai", + "deepseek": "deepseek", + "ollama": "ollama", + "nlp_cloud": "nlp_cloud", } +# Only map codes that unambiguously indicate the specific error type _ERROR_TYPE_MAP: Dict[int, str] = { 429: "rate_limit", 529: "overloaded", @@ -105,8 +111,23 @@ def _extract_status_code(exception: Optional[BaseException]) -> Optional[int]: return None +def _latency_ms( + start_time: Union[datetime, float], end_time: Union[datetime, float] +) -> int: + """Compute elapsed ms whether LiteLLM passes datetime objects or floats.""" + if isinstance(start_time, datetime) and isinstance(end_time, datetime): + return round((end_time - start_time).total_seconds() * 1000) + return round((float(end_time) - float(start_time)) * 1000) # type: ignore[arg-type] + + def _fire_and_forget(payload: Dict[str, Any]) -> None: - """POST to Tickerr in a daemon thread — never blocks the caller.""" + """POST to Tickerr in a daemon thread — never blocks the caller. + + A semaphore caps concurrent in-flight threads so a burst of failures + (e.g. 100 errors/s) cannot exhaust the thread pool. + """ + if not _inflight.acquire(blocking=False): + return # already at max concurrent reports — drop silently def _send() -> None: try: @@ -124,6 +145,8 @@ def _fire_and_forget(payload: Dict[str, Any]) -> None: pass except Exception: pass # never crash the caller + finally: + _inflight.release() t = threading.Thread(target=_send, daemon=True) t.start() @@ -152,10 +175,10 @@ class TickerrLogger(CustomLogger): self, kwargs: Dict[str, Any], response_obj: Any, - start_time: float, - end_time: float, + start_time: Union[datetime, float], + end_time: Union[datetime, float], ) -> None: - self._report(kwargs, start_time, end_time, is_resolution=False) + self._report(kwargs, start_time, end_time) # ── async ───────────────────────────────────────────────────────────────── @@ -163,27 +186,26 @@ class TickerrLogger(CustomLogger): self, kwargs: Dict[str, Any], response_obj: Any, - start_time: float, - end_time: float, + start_time: Union[datetime, float], + end_time: Union[datetime, float], ) -> None: - self._report(kwargs, start_time, end_time, is_resolution=False) + self._report(kwargs, start_time, end_time) # ── internal ────────────────────────────────────────────────────────────── def _report( self, kwargs: Dict[str, Any], - start_time: float, - end_time: float, - is_resolution: bool, + start_time: Union[datetime, float], + end_time: Union[datetime, float], ) -> None: model: str = kwargs.get("model", "") or "" exception: Optional[BaseException] = kwargs.get("exception") - latency_ms = round((end_time - start_time) * 1000) provider = _normalize_provider(model, kwargs) status_code = _extract_status_code(exception) - error_type = _ERROR_TYPE_MAP.get(status_code, "overloaded") if status_code else None + # Only set error_type for codes we can classify with confidence + error_type: Optional[str] = _ERROR_TYPE_MAP.get(status_code) if status_code is not None else None # Strip provider prefix: "anthropic/claude-3-5-haiku" → "claude-3-5-haiku" model_clean = model.split("/", 1)[-1] if "/" in model else model @@ -191,8 +213,7 @@ class TickerrLogger(CustomLogger): payload: Dict[str, Any] = { "provider": provider, "model": model_clean or None, - "is_resolution": is_resolution, - "latency_ms": latency_ms, + "latency_ms": _latency_ms(start_time, end_time), } if status_code is not None: payload["error_code"] = status_code From 72d1a93ea10b0a13931afbddb46e8d32f930cf43 Mon Sep 17 00:00:00 2001 From: imviky-ctrl <272028991+imviky-ctrl@users.noreply.github.com> Date: Tue, 5 May 2026 15:02:51 +0530 Subject: [PATCH 03/20] test(tickerr): add unit tests for TickerrLogger callback Covers: provider normalization, status code extraction, latency calculation with both datetime and float types, error type mapping, payload construction, semaphore cap, silent network failure, and async hook wiring. --- .../test_tickerr_callback.py | 266 ++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 tests/logging_callback_tests/test_tickerr_callback.py diff --git a/tests/logging_callback_tests/test_tickerr_callback.py b/tests/logging_callback_tests/test_tickerr_callback.py new file mode 100644 index 00000000000..1444a16c3d5 --- /dev/null +++ b/tests/logging_callback_tests/test_tickerr_callback.py @@ -0,0 +1,266 @@ +""" +Unit tests for the Tickerr LiteLLM callback. + +Tests cover: +- TickerrLogger instantiation and env var config +- Provider normalization from model names and litellm_params +- Status code extraction from exceptions +- Latency calculation for both datetime and float timestamps +- Error type mapping (only known codes, no fallback default) +- Payload construction +- Thread cap (semaphore) under burst conditions +- Fire-and-forget does not block or raise on network failure +""" + +import os +import sys +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from litellm.integrations.tickerr import ( + TickerrLogger, + _ERROR_TYPE_MAP, + _extract_status_code, + _fire_and_forget, + _inflight, + _latency_ms, + _normalize_provider, + _MAX_INFLIGHT, +) + + +# ── Provider normalization ──────────────────────────────────────────────────── + + +def test_normalize_provider_from_litellm_params(): + kwargs = {"litellm_params": {"custom_llm_provider": "anthropic"}} + assert _normalize_provider("some-model", kwargs) == "anthropic" + + +def test_normalize_provider_from_custom_llm_provider(): + kwargs = {"custom_llm_provider": "openai"} + assert _normalize_provider("gpt-4o", kwargs) == "openai" + + +def test_normalize_provider_from_model_prefix(): + assert _normalize_provider("anthropic/claude-3-5-haiku", {}) == "anthropic" + assert _normalize_provider("openai/gpt-4o", {}) == "openai" + + +def test_normalize_provider_from_model_name_pattern(): + assert _normalize_provider("claude-haiku-4-5", {}) == "anthropic" + assert _normalize_provider("gpt-4o-mini", {}) == "openai" + assert _normalize_provider("gemini-2.5-flash", {}) == "google" + assert _normalize_provider("mistral-small-latest", {}) == "mistral" + assert _normalize_provider("llama-3.3-70b", {}) == "meta" + assert _normalize_provider("grok-3-mini", {}) == "xai" + assert _normalize_provider("deepseek-v3", {}) == "deepseek" + + +def test_normalize_provider_unknown(): + assert _normalize_provider("some-unknown-model-xyz", {}) == "unknown" + + +# ── Status code extraction ──────────────────────────────────────────────────── + + +def test_extract_status_code_int(): + exc = MagicMock() + exc.status_code = 429 + assert _extract_status_code(exc) == 429 + + +def test_extract_status_code_string(): + exc = MagicMock() + exc.status_code = "503" + assert _extract_status_code(exc) == 503 + + +def test_extract_status_code_none_exception(): + assert _extract_status_code(None) is None + + +def test_extract_status_code_no_attribute(): + assert _extract_status_code(ValueError("oops")) is None + + +# ── Latency calculation ─────────────────────────────────────────────────────── + + +def test_latency_ms_with_datetime(): + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=1240) + assert _latency_ms(start, end) == 1240 + + +def test_latency_ms_with_floats(): + assert _latency_ms(1000.0, 1001.5) == 1500 + + +def test_latency_ms_mixed_types_float(): + # Both floats — should not raise + result = _latency_ms(0.0, 0.5) + assert result == 500 + + +# ── Error type mapping ──────────────────────────────────────────────────────── + + +def test_error_type_known_codes(): + assert _ERROR_TYPE_MAP[429] == "rate_limit" + assert _ERROR_TYPE_MAP[529] == "overloaded" + assert _ERROR_TYPE_MAP[503] == "overloaded" + assert _ERROR_TYPE_MAP[408] == "timeout" + assert _ERROR_TYPE_MAP[401] == "auth" + + +def test_error_type_no_default_for_unknown_codes(): + # Unknown codes (400, 404, 502) must NOT map to "overloaded" or any value + for code in (400, 404, 502, 422, 301): + assert code not in _ERROR_TYPE_MAP, f"code {code} should not be in _ERROR_TYPE_MAP" + + +# ── TickerrLogger instantiation ─────────────────────────────────────────────── + + +def test_tickerr_logger_default_init(): + logger = TickerrLogger() + assert logger.client_tier is None + assert logger.region is None + + +def test_tickerr_logger_reads_env_vars(): + with patch.dict(os.environ, {"TICKERR_CLIENT_TIER": "pro", "TICKERR_REGION": "us-east-1"}): + logger = TickerrLogger() + assert logger.client_tier == "pro" + assert logger.region == "us-east-1" + + +# ── Payload construction via _report ───────────────────────────────────────── + + +def test_report_builds_correct_payload(): + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + exc = MagicMock() + exc.status_code = 429 + + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=500) + + kwargs = { + "model": "claude-haiku-4-5", + "exception": exc, + "litellm_params": {"custom_llm_provider": "anthropic"}, + } + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["provider"] == "anthropic" + assert captured["model"] == "claude-haiku-4-5" + assert captured["error_code"] == 429 + assert captured["error_type"] == "rate_limit" + assert captured["latency_ms"] == 500 + + +def test_report_strips_provider_prefix_from_model(): + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=100) + + kwargs = {"model": "openai/gpt-4o-mini", "exception": None} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["model"] == "gpt-4o-mini" + + +def test_report_omits_error_type_for_unknown_code(): + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + exc = MagicMock() + exc.status_code = 400 # not in _ERROR_TYPE_MAP + + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=100) + kwargs = {"model": "gpt-4o", "exception": exc} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert "error_type" not in captured + assert captured["error_code"] == 400 + + +# ── Thread cap ──────────────────────────────────────────────────────────────── + + +def test_fire_and_forget_respects_semaphore_cap(): + """Reports beyond _MAX_INFLIGHT are dropped silently.""" + sent = [] + + def slow_send(payload): + sent.append(payload) + # simulate slow network + import time + time.sleep(0.1) + + # Exhaust the semaphore + for _ in range(_MAX_INFLIGHT): + _inflight.acquire() + + try: + # This call should be dropped (semaphore exhausted) + with patch("litellm.integrations.tickerr._fire_and_forget"): + _fire_and_forget({"provider": "openai"}) + # Since semaphore is exhausted, the thread should not be started + finally: + # Restore semaphore + for _ in range(_MAX_INFLIGHT): + _inflight.release() + + +# ── Network failure is silent ───────────────────────────────────────────────── + + +def test_fire_and_forget_silent_on_network_error(): + """A network error in the background thread must not propagate.""" + with patch("urllib.request.urlopen", side_effect=OSError("connection refused")): + # Should complete without raising + _fire_and_forget({"provider": "anthropic", "model": "claude-haiku-4-5"}) + import time + time.sleep(0.2) # let daemon thread run + + +# ── Async hooks ─────────────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_async_log_failure_event_calls_report(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) + kwargs = {"model": "gpt-4o-mini", "exception": None} + + with patch.object(logger, "_report") as mock_report: + await logger.async_log_failure_event(kwargs, None, start, end) + mock_report.assert_called_once_with(kwargs, start, end) From 94a163863075b08bbf97f4313bd7896282a89d3f Mon Sep 17 00:00:00 2001 From: imviky-ctrl <272028991+imviky-ctrl@users.noreply.github.com> Date: Tue, 5 May 2026 15:20:18 +0530 Subject: [PATCH 04/20] fix(tickerr): fix semaphore leak, remove 500 from error map, fix semaphore test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Semaphore leak: wrap t.start() in try/except and release _inflight on RuntimeError so the slot is not permanently lost if the OS thread limit is hit - Remove 500 from _ERROR_TYPE_MAP: 500 is a generic internal server error (crash/bug/misconfiguration), not a capacity/overload condition — sending 'overloaded' for 500s would corrupt crowd-sourced signal - Rewrite semaphore cap test to actually invoke real _fire_and_forget with exhausted semaphore and assert threading.Thread is never called - Add test for semaphore release on thread start failure - Update error-type test to explicitly assert 500 is excluded --- litellm/integrations/tickerr.py | 12 ++++-- .../test_tickerr_callback.py | 40 +++++++++++-------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index 9fb933bd937..810cea13aa1 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -57,12 +57,13 @@ _PROVIDER_MAP: Dict[str, str] = { "nlp_cloud": "nlp_cloud", } -# Only map codes that unambiguously indicate the specific error type +# Only map codes that unambiguously indicate the specific error type. +# 500 is intentionally excluded: it is a generic "Internal Server Error" that +# indicates a crash or bug, not a capacity/overload condition. _ERROR_TYPE_MAP: Dict[int, str] = { 429: "rate_limit", 529: "overloaded", 503: "overloaded", - 500: "overloaded", 408: "timeout", 524: "timeout", 401: "auth", @@ -149,7 +150,12 @@ def _fire_and_forget(payload: Dict[str, Any]) -> None: _inflight.release() t = threading.Thread(target=_send, daemon=True) - t.start() + try: + t.start() + except Exception: + # Thread could not be started (e.g. OS thread limit). + # Release the slot so future reports are not permanently blocked. + _inflight.release() class TickerrLogger(CustomLogger): diff --git a/tests/logging_callback_tests/test_tickerr_callback.py b/tests/logging_callback_tests/test_tickerr_callback.py index 1444a16c3d5..58877af1bb1 100644 --- a/tests/logging_callback_tests/test_tickerr_callback.py +++ b/tests/logging_callback_tests/test_tickerr_callback.py @@ -119,8 +119,9 @@ def test_error_type_known_codes(): def test_error_type_no_default_for_unknown_codes(): - # Unknown codes (400, 404, 502) must NOT map to "overloaded" or any value - for code in (400, 404, 502, 422, 301): + # 500 is a generic server error (crash/bug), not definitively "overloaded". + # Unknown codes must NOT map to any value. + for code in (400, 404, 500, 502, 422, 301): assert code not in _ERROR_TYPE_MAP, f"code {code} should not be in _ERROR_TYPE_MAP" @@ -215,30 +216,35 @@ def test_report_omits_error_type_for_unknown_code(): def test_fire_and_forget_respects_semaphore_cap(): - """Reports beyond _MAX_INFLIGHT are dropped silently.""" - sent = [] - - def slow_send(payload): - sent.append(payload) - # simulate slow network - import time - time.sleep(0.1) - - # Exhaust the semaphore + """Reports beyond _MAX_INFLIGHT are dropped silently without blocking.""" + # Exhaust the semaphore by acquiring all slots directly for _ in range(_MAX_INFLIGHT): - _inflight.acquire() + acquired = _inflight.acquire(blocking=False) + assert acquired, "semaphore should have slots available at test start" try: - # This call should be dropped (semaphore exhausted) - with patch("litellm.integrations.tickerr._fire_and_forget"): + # With semaphore exhausted, _fire_and_forget must return immediately + # without starting a thread (non-blocking acquire fails → early return) + with patch("threading.Thread") as mock_thread: _fire_and_forget({"provider": "openai"}) - # Since semaphore is exhausted, the thread should not be started + mock_thread.assert_not_called() finally: - # Restore semaphore for _ in range(_MAX_INFLIGHT): _inflight.release() +def test_semaphore_released_on_thread_start_failure(): + """If t.start() raises, the semaphore slot must be released so future reports work.""" + before = _inflight._value + + with patch("threading.Thread") as mock_thread: + mock_thread.return_value.start.side_effect = RuntimeError("OS thread limit") + _fire_and_forget({"provider": "openai"}) + + # Slot must be back to its original value after the exception + assert _inflight._value == before + + # ── Network failure is silent ───────────────────────────────────────────────── From a39ba5c52f300dfdc2b8c1525aadb0cedb62e45a Mon Sep 17 00:00:00 2001 From: imviky-ctrl <272028991+imviky-ctrl@users.noreply.github.com> Date: Tue, 5 May 2026 15:28:54 +0530 Subject: [PATCH 05/20] test(tickerr): fix race condition and private API usage in tests - test_fire_and_forget_silent_on_network_error: mock threading.Thread entirely so no real thread or network call can escape the test boundary - test_semaphore_released_on_thread_start_failure: replace _inflight._value (private CPython attr) with a non-blocking acquire/release assertion --- .../test_tickerr_callback.py | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/tests/logging_callback_tests/test_tickerr_callback.py b/tests/logging_callback_tests/test_tickerr_callback.py index 58877af1bb1..b8f949c0f4d 100644 --- a/tests/logging_callback_tests/test_tickerr_callback.py +++ b/tests/logging_callback_tests/test_tickerr_callback.py @@ -235,26 +235,41 @@ def test_fire_and_forget_respects_semaphore_cap(): def test_semaphore_released_on_thread_start_failure(): """If t.start() raises, the semaphore slot must be released so future reports work.""" - before = _inflight._value - + # Verify the semaphore can be re-acquired after a thread-start failure, + # which proves the slot was released (without reading private CPython internals). with patch("threading.Thread") as mock_thread: mock_thread.return_value.start.side_effect = RuntimeError("OS thread limit") _fire_and_forget({"provider": "openai"}) - # Slot must be back to its original value after the exception - assert _inflight._value == before + # If the semaphore was not released, this acquire would block forever. + # Use non-blocking to fail fast in case of a bug. + acquired = _inflight.acquire(blocking=False) + assert acquired, "semaphore slot was not released after thread start failure" + _inflight.release() # restore # ── Network failure is silent ───────────────────────────────────────────────── def test_fire_and_forget_silent_on_network_error(): - """A network error in the background thread must not propagate.""" - with patch("urllib.request.urlopen", side_effect=OSError("connection refused")): - # Should complete without raising - _fire_and_forget({"provider": "anthropic", "model": "claude-haiku-4-5"}) - import time - time.sleep(0.2) # let daemon thread run + """A network error in the send function must not propagate to the caller.""" + # Mock threading.Thread so no real thread is created and no real network + # call can escape the test boundary (repo rule: no real network calls). + with patch("threading.Thread") as mock_thread_cls: + mock_thread = MagicMock() + mock_thread_cls.return_value = mock_thread + + # Simulate the send function raising an OSError inside the thread + def run_target(*args, **kwargs): + target = mock_thread_cls.call_args[1].get("target") or mock_thread_cls.call_args[0][0] + try: + target() + except Exception: + pass # errors in thread body must not surface + + mock_thread.start.side_effect = run_target + with patch("urllib.request.urlopen", side_effect=OSError("connection refused")): + _fire_and_forget({"provider": "anthropic", "model": "claude-haiku-4-5"}) # ── Async hooks ─────────────────────────────────────────────────────────────── From c2c2f8285de72fb818056490f60253ad2910e41d Mon Sep 17 00:00:00 2001 From: imviky-ctrl <272028991+imviky-ctrl@users.noreply.github.com> Date: Tue, 5 May 2026 15:34:15 +0530 Subject: [PATCH 06/20] style: apply Black formatting to tickerr.py --- litellm/integrations/tickerr.py | 48 +++++++++++++++++---------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index 810cea13aa1..72538580d7e 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -32,29 +32,29 @@ _inflight = threading.Semaphore(_MAX_INFLIGHT) # Map litellm custom_llm_provider → Tickerr provider slug _PROVIDER_MAP: Dict[str, str] = { - "openai": "openai", - "anthropic": "anthropic", - "google": "google", - "vertex_ai": "google", - "gemini": "google", - "cohere": "cohere", - "mistral": "mistral", - "groq": "groq", - "together_ai": "together", - "huggingface": "huggingface", - "replicate": "replicate", - "deepinfra": "deepinfra", - "perplexity": "perplexity", + "openai": "openai", + "anthropic": "anthropic", + "google": "google", + "vertex_ai": "google", + "gemini": "google", + "cohere": "cohere", + "mistral": "mistral", + "groq": "groq", + "together_ai": "together", + "huggingface": "huggingface", + "replicate": "replicate", + "deepinfra": "deepinfra", + "perplexity": "perplexity", "fireworks_ai": "fireworks", - "openrouter": "openrouter", - "azure": "azure", - "bedrock": "aws", - "ai21": "ai21", - "cerebras": "cerebras", - "xai": "xai", - "deepseek": "deepseek", - "ollama": "ollama", - "nlp_cloud": "nlp_cloud", + "openrouter": "openrouter", + "azure": "azure", + "bedrock": "aws", + "ai21": "ai21", + "cerebras": "cerebras", + "xai": "xai", + "deepseek": "deepseek", + "ollama": "ollama", + "nlp_cloud": "nlp_cloud", } # Only map codes that unambiguously indicate the specific error type. @@ -211,7 +211,9 @@ class TickerrLogger(CustomLogger): provider = _normalize_provider(model, kwargs) status_code = _extract_status_code(exception) # Only set error_type for codes we can classify with confidence - error_type: Optional[str] = _ERROR_TYPE_MAP.get(status_code) if status_code is not None else None + error_type: Optional[str] = ( + _ERROR_TYPE_MAP.get(status_code) if status_code is not None else None + ) # Strip provider prefix: "anthropic/claude-3-5-haiku" → "claude-3-5-haiku" model_clean = model.split("/", 1)[-1] if "/" in model else model From 8047392b2161b97ab88e4c8de7fd5d95279826a1 Mon Sep 17 00:00:00 2001 From: imviky-ctrl <272028991+imviky-ctrl@users.noreply.github.com> Date: Tue, 5 May 2026 15:48:28 +0530 Subject: [PATCH 07/20] fix: add missing provider aliases, fix test semaphore accounting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add vertex_ai_anthropic → anthropic, bedrock_converse → aws, azure_ai → azure to _PROVIDER_MAP so composite providers report the canonical slug instead of their raw internal key - Fix test_fire_and_forget_respects_semaphore_cap to release only as many slots as were actually acquired, preventing semaphore count from exceeding _MAX_INFLIGHT if an assert fires mid-loop --- litellm/integrations/tickerr.py | 3 +++ .../test_tickerr_callback.py | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index 72538580d7e..75732d8e515 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -48,7 +48,10 @@ _PROVIDER_MAP: Dict[str, str] = { "fireworks_ai": "fireworks", "openrouter": "openrouter", "azure": "azure", + "azure_ai": "azure", "bedrock": "aws", + "bedrock_converse": "aws", + "vertex_ai_anthropic": "anthropic", "ai21": "ai21", "cerebras": "cerebras", "xai": "xai", diff --git a/tests/logging_callback_tests/test_tickerr_callback.py b/tests/logging_callback_tests/test_tickerr_callback.py index b8f949c0f4d..ca37d82bb6e 100644 --- a/tests/logging_callback_tests/test_tickerr_callback.py +++ b/tests/logging_callback_tests/test_tickerr_callback.py @@ -217,10 +217,21 @@ def test_report_omits_error_type_for_unknown_code(): def test_fire_and_forget_respects_semaphore_cap(): """Reports beyond _MAX_INFLIGHT are dropped silently without blocking.""" - # Exhaust the semaphore by acquiring all slots directly + # Exhaust the semaphore by acquiring all slots directly. + # Track how many we actually acquired so the finally block releases exactly + # that many — releasing more than acquired would push the count above its + # initial maximum and corrupt later tests. + acquired_count = 0 for _ in range(_MAX_INFLIGHT): - acquired = _inflight.acquire(blocking=False) - assert acquired, "semaphore should have slots available at test start" + if _inflight.acquire(blocking=False): + acquired_count += 1 + else: + break + + assert acquired_count == _MAX_INFLIGHT, ( + f"semaphore should have {_MAX_INFLIGHT} slots available at test start, " + f"got {acquired_count}" + ) try: # With semaphore exhausted, _fire_and_forget must return immediately @@ -229,7 +240,7 @@ def test_fire_and_forget_respects_semaphore_cap(): _fire_and_forget({"provider": "openai"}) mock_thread.assert_not_called() finally: - for _ in range(_MAX_INFLIGHT): + for _ in range(acquired_count): _inflight.release() From 6123020b528c725a5ae4fda94d29767cfaf0f833 Mon Sep 17 00:00:00 2001 From: imviky-ctrl <272028991+imviky-ctrl@users.noreply.github.com> Date: Wed, 6 May 2026 16:51:32 +0530 Subject: [PATCH 08/20] test(tickerr): add 23 tests to reach 100% coverage on tickerr.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers: - log_failure_event sync hook - _normalize_provider: vertex_ai, vertex_ai_anthropic, azure, bedrock, bedrock_converse, command-*, mixtral-*, deepseek, openrouter, fireworks_ai, cerebras, xai - _report: no exception, client_tier/region payload fields, empty model → None, timeout/auth error_type mapping - _fire_and_forget: success path + semaphore release via finally - _extract_status_code: non-digit string, 529 - _latency_ms: zero, large value --- .../test_tickerr_callback.py | 247 ++++++++++++++++++ 1 file changed, 247 insertions(+) diff --git a/tests/logging_callback_tests/test_tickerr_callback.py b/tests/logging_callback_tests/test_tickerr_callback.py index ca37d82bb6e..8690690873c 100644 --- a/tests/logging_callback_tests/test_tickerr_callback.py +++ b/tests/logging_callback_tests/test_tickerr_callback.py @@ -296,3 +296,250 @@ async def test_async_log_failure_event_calls_report(): with patch.object(logger, "_report") as mock_report: await logger.async_log_failure_event(kwargs, None, start, end) mock_report.assert_called_once_with(kwargs, start, end) + + +# ── Additional tests added to hit 67 % patch coverage ──────────────────────── + + +# ── Sync log_failure_event hook ─────────────────────────────────────────────── + + +def test_log_failure_event_sync_calls_report(): + """log_failure_event (sync) must delegate to _report.""" + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=300) + kwargs = {"model": "gpt-4o-mini", "exception": None} + + with patch.object(logger, "_report") as mock_report: + logger.log_failure_event(kwargs, None, start, end) + mock_report.assert_called_once_with(kwargs, start, end) + + +# ── _normalize_provider: extra provider map entries ─────────────────────────── + + +def test_normalize_provider_vertex_ai_maps_to_google(): + kwargs = {"custom_llm_provider": "vertex_ai"} + assert _normalize_provider("gemini-2.5-pro", kwargs) == "google" + + +def test_normalize_provider_vertex_ai_anthropic_maps_to_anthropic(): + kwargs = {"custom_llm_provider": "vertex_ai_anthropic"} + assert _normalize_provider("claude-3-5-sonnet", kwargs) == "anthropic" + + +def test_normalize_provider_azure_maps_to_azure(): + kwargs = {"litellm_params": {"custom_llm_provider": "azure"}} + assert _normalize_provider("gpt-4o", kwargs) == "azure" + + +def test_normalize_provider_bedrock_maps_to_aws(): + kwargs = {"custom_llm_provider": "bedrock"} + assert _normalize_provider("claude-3-haiku", kwargs) == "aws" + + +def test_normalize_provider_bedrock_converse_maps_to_aws(): + kwargs = {"custom_llm_provider": "bedrock_converse"} + assert _normalize_provider("claude-3-haiku", kwargs) == "aws" + + +def test_normalize_provider_command_model_pattern(): + """command-* model names should map to cohere via regex.""" + assert _normalize_provider("command-r-plus", {}) == "cohere" + assert _normalize_provider("command-r", {}) == "cohere" + + +def test_normalize_provider_mixtral_model_pattern(): + """mixtral-* model names should map to mistral via regex.""" + assert _normalize_provider("mixtral-8x7b-instruct", {}) == "mistral" + + +def test_normalize_provider_deepseek_from_param(): + kwargs = {"custom_llm_provider": "deepseek"} + assert _normalize_provider("deepseek-v3", kwargs) == "deepseek" + + +def test_normalize_provider_openrouter_from_param(): + kwargs = {"litellm_params": {"custom_llm_provider": "openrouter"}} + assert _normalize_provider("meta-llama/llama-3.3-70b-instruct", kwargs) == "openrouter" + + +def test_normalize_provider_fireworks_ai_from_param(): + kwargs = {"custom_llm_provider": "fireworks_ai"} + assert _normalize_provider("llama-v3-70b-instruct", kwargs) == "fireworks" + + +def test_normalize_provider_cerebras_from_param(): + kwargs = {"custom_llm_provider": "cerebras"} + assert _normalize_provider("llama3.1-8b", kwargs) == "cerebras" + + +def test_normalize_provider_xai_from_param(): + kwargs = {"custom_llm_provider": "xai"} + assert _normalize_provider("grok-3-mini", kwargs) == "xai" + + +# ── _report: edge cases ─────────────────────────────────────────────────────── + + +def test_report_no_exception_omits_error_fields(): + """When exception is None, payload must not include error_code or error_type.""" + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) + kwargs = {"model": "gemini-2.5-flash", "exception": None} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert "error_code" not in captured + assert "error_type" not in captured + assert captured["provider"] == "google" + + +def test_report_includes_client_tier_and_region_in_payload(): + """client_tier and region must appear in the payload when set via env.""" + with patch.dict(os.environ, {"TICKERR_CLIENT_TIER": "enterprise", "TICKERR_REGION": "eu-west-1"}): + logger = TickerrLogger() + + captured = {} + + def fake_fire(payload): + captured.update(payload) + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + kwargs = {"model": "gpt-4o", "exception": None} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["client_tier"] == "enterprise" + assert captured["region"] == "eu-west-1" + + +def test_report_empty_model_sets_model_none(): + """An empty model string must result in model=None in the payload.""" + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=50) + kwargs = {"model": "", "exception": None} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["model"] is None + + +def test_report_timeout_error_type(): + """408 and 524 both map to error_type='timeout'.""" + logger = TickerrLogger() + + for code in (408, 524): + captured = {} + + def fake_fire(payload, _c=code): + captured.update(payload) + + exc = MagicMock() + exc.status_code = code + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=5000) + kwargs = {"model": "claude-haiku-4-5", "exception": exc} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["error_type"] == "timeout", f"Expected timeout for {code}" + assert captured["error_code"] == code + + +def test_report_auth_error_type(): + """401 and 403 both map to error_type='auth'.""" + logger = TickerrLogger() + + for code in (401, 403): + captured = {} + + def fake_fire(payload): + captured.update(payload) + + exc = MagicMock() + exc.status_code = code + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) + kwargs = {"model": "gpt-4o", "exception": exc} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["error_type"] == "auth", f"Expected auth for {code}" + + +# ── _fire_and_forget: normal success path ───────────────────────────────────── + + +def test_fire_and_forget_success_path_releases_semaphore(): + """On a successful HTTP call, the semaphore slot must be released.""" + mock_response = MagicMock() + mock_response.__enter__ = MagicMock(return_value=mock_response) + mock_response.__exit__ = MagicMock(return_value=False) + + with patch("threading.Thread") as mock_thread_cls: + mock_thread = MagicMock() + mock_thread_cls.return_value = mock_thread + + def run_target_inline(*args, **kwargs): + # Extract and call the real target so _send runs synchronously + target = mock_thread_cls.call_args[1].get("target") or mock_thread_cls.call_args[0][0] + target() + + mock_thread.start.side_effect = run_target_inline + + with patch("urllib.request.urlopen", return_value=mock_response): + _fire_and_forget({"provider": "openai", "model": "gpt-4o-mini"}) + + # Semaphore must be acquirable — proves the finally block released the slot + acquired = _inflight.acquire(blocking=False) + assert acquired, "semaphore slot was not released after successful send" + _inflight.release() + + +# ── _latency_ms edge cases ──────────────────────────────────────────────────── + + +def test_latency_ms_zero(): + assert _latency_ms(1000.0, 1000.0) == 0 + + +def test_latency_ms_large_value(): + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(seconds=30) + assert _latency_ms(start, end) == 30_000 + + +# ── _extract_status_code edge cases ────────────────────────────────────────── + + +def test_extract_status_code_non_digit_string(): + """A non-numeric string status_code must return None.""" + exc = MagicMock() + exc.status_code = "N/A" + assert _extract_status_code(exc) is None + + +def test_extract_status_code_529(): + exc = MagicMock() + exc.status_code = 529 + assert _extract_status_code(exc) == 529 From e2790761932827f5ebfc83b72c5ee5df8cc8da26 Mon Sep 17 00:00:00 2001 From: imviky-ctrl <272028991+imviky-ctrl@users.noreply.github.com> Date: Wed, 6 May 2026 17:19:15 +0530 Subject: [PATCH 09/20] test(tickerr): move tests to integrations folder so CI picks them up for coverage --- .../integrations/test_tickerr_callback.py | 542 ++++++++++++++++++ 1 file changed, 542 insertions(+) create mode 100644 tests/test_litellm/integrations/test_tickerr_callback.py diff --git a/tests/test_litellm/integrations/test_tickerr_callback.py b/tests/test_litellm/integrations/test_tickerr_callback.py new file mode 100644 index 00000000000..37a1df9d560 --- /dev/null +++ b/tests/test_litellm/integrations/test_tickerr_callback.py @@ -0,0 +1,542 @@ +""" +Unit tests for the Tickerr LiteLLM callback. + +Tests cover: +- TickerrLogger instantiation and env var config +- Provider normalization from model names and litellm_params +- Status code extraction from exceptions +- Latency calculation for both datetime and float timestamps +- Error type mapping (only known codes, no fallback default) +- Payload construction +- Thread cap (semaphore) under burst conditions +- Fire-and-forget does not block or raise on network failure +""" + +import os +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.integrations.tickerr import ( + TickerrLogger, + _ERROR_TYPE_MAP, + _extract_status_code, + _fire_and_forget, + _inflight, + _latency_ms, + _normalize_provider, + _MAX_INFLIGHT, +) + + +# ── Provider normalization ──────────────────────────────────────────────────── + + +def test_normalize_provider_from_litellm_params(): + kwargs = {"litellm_params": {"custom_llm_provider": "anthropic"}} + assert _normalize_provider("some-model", kwargs) == "anthropic" + + +def test_normalize_provider_from_custom_llm_provider(): + kwargs = {"custom_llm_provider": "openai"} + assert _normalize_provider("gpt-4o", kwargs) == "openai" + + +def test_normalize_provider_from_model_prefix(): + assert _normalize_provider("anthropic/claude-3-5-haiku", {}) == "anthropic" + assert _normalize_provider("openai/gpt-4o", {}) == "openai" + + +def test_normalize_provider_from_model_name_pattern(): + assert _normalize_provider("claude-haiku-4-5", {}) == "anthropic" + assert _normalize_provider("gpt-4o-mini", {}) == "openai" + assert _normalize_provider("gemini-2.5-flash", {}) == "google" + assert _normalize_provider("mistral-small-latest", {}) == "mistral" + assert _normalize_provider("llama-3.3-70b", {}) == "meta" + assert _normalize_provider("grok-3-mini", {}) == "xai" + assert _normalize_provider("deepseek-v3", {}) == "deepseek" + + +def test_normalize_provider_unknown(): + assert _normalize_provider("some-unknown-model-xyz", {}) == "unknown" + + +# ── Status code extraction ──────────────────────────────────────────────────── + + +def test_extract_status_code_int(): + exc = MagicMock() + exc.status_code = 429 + assert _extract_status_code(exc) == 429 + + +def test_extract_status_code_string(): + exc = MagicMock() + exc.status_code = "503" + assert _extract_status_code(exc) == 503 + + +def test_extract_status_code_none_exception(): + assert _extract_status_code(None) is None + + +def test_extract_status_code_no_attribute(): + assert _extract_status_code(ValueError("oops")) is None + + +# ── Latency calculation ─────────────────────────────────────────────────────── + + +def test_latency_ms_with_datetime(): + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=1240) + assert _latency_ms(start, end) == 1240 + + +def test_latency_ms_with_floats(): + assert _latency_ms(1000.0, 1001.5) == 1500 + + +def test_latency_ms_mixed_types_float(): + # Both floats — should not raise + result = _latency_ms(0.0, 0.5) + assert result == 500 + + +# ── Error type mapping ──────────────────────────────────────────────────────── + + +def test_error_type_known_codes(): + assert _ERROR_TYPE_MAP[429] == "rate_limit" + assert _ERROR_TYPE_MAP[529] == "overloaded" + assert _ERROR_TYPE_MAP[503] == "overloaded" + assert _ERROR_TYPE_MAP[408] == "timeout" + assert _ERROR_TYPE_MAP[401] == "auth" + + +def test_error_type_no_default_for_unknown_codes(): + # 500 is a generic server error (crash/bug), not definitively "overloaded". + # Unknown codes must NOT map to any value. + for code in (400, 404, 500, 502, 422, 301): + assert code not in _ERROR_TYPE_MAP, f"code {code} should not be in _ERROR_TYPE_MAP" + + +# ── TickerrLogger instantiation ─────────────────────────────────────────────── + + +def test_tickerr_logger_default_init(): + logger = TickerrLogger() + assert logger.client_tier is None + assert logger.region is None + + +def test_tickerr_logger_reads_env_vars(): + with patch.dict(os.environ, {"TICKERR_CLIENT_TIER": "pro", "TICKERR_REGION": "us-east-1"}): + logger = TickerrLogger() + assert logger.client_tier == "pro" + assert logger.region == "us-east-1" + + +# ── Payload construction via _report ───────────────────────────────────────── + + +def test_report_builds_correct_payload(): + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + exc = MagicMock() + exc.status_code = 429 + + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=500) + + kwargs = { + "model": "claude-haiku-4-5", + "exception": exc, + "litellm_params": {"custom_llm_provider": "anthropic"}, + } + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["provider"] == "anthropic" + assert captured["model"] == "claude-haiku-4-5" + assert captured["error_code"] == 429 + assert captured["error_type"] == "rate_limit" + assert captured["latency_ms"] == 500 + + +def test_report_strips_provider_prefix_from_model(): + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=100) + + kwargs = {"model": "openai/gpt-4o-mini", "exception": None} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["model"] == "gpt-4o-mini" + + +def test_report_omits_error_type_for_unknown_code(): + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + exc = MagicMock() + exc.status_code = 400 # not in _ERROR_TYPE_MAP + + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=100) + kwargs = {"model": "gpt-4o", "exception": exc} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert "error_type" not in captured + assert captured["error_code"] == 400 + + +# ── Thread cap ──────────────────────────────────────────────────────────────── + + +def test_fire_and_forget_respects_semaphore_cap(): + """Reports beyond _MAX_INFLIGHT are dropped silently without blocking.""" + # Exhaust the semaphore by acquiring all slots directly. + # Track how many we actually acquired so the finally block releases exactly + # that many — releasing more than acquired would push the count above its + # initial maximum and corrupt later tests. + acquired_count = 0 + for _ in range(_MAX_INFLIGHT): + if _inflight.acquire(blocking=False): + acquired_count += 1 + else: + break + + assert acquired_count == _MAX_INFLIGHT, ( + f"semaphore should have {_MAX_INFLIGHT} slots available at test start, " + f"got {acquired_count}" + ) + + try: + # With semaphore exhausted, _fire_and_forget must return immediately + # without starting a thread (non-blocking acquire fails → early return) + with patch("threading.Thread") as mock_thread: + _fire_and_forget({"provider": "openai"}) + mock_thread.assert_not_called() + finally: + for _ in range(acquired_count): + _inflight.release() + + +def test_semaphore_released_on_thread_start_failure(): + """If t.start() raises, the semaphore slot must be released so future reports work.""" + # Verify the semaphore can be re-acquired after a thread-start failure, + # which proves the slot was released (without reading private CPython internals). + with patch("threading.Thread") as mock_thread: + mock_thread.return_value.start.side_effect = RuntimeError("OS thread limit") + _fire_and_forget({"provider": "openai"}) + + # If the semaphore was not released, this acquire would block forever. + # Use non-blocking to fail fast in case of a bug. + acquired = _inflight.acquire(blocking=False) + assert acquired, "semaphore slot was not released after thread start failure" + _inflight.release() # restore + + +# ── Network failure is silent ───────────────────────────────────────────────── + + +def test_fire_and_forget_silent_on_network_error(): + """A network error in the send function must not propagate to the caller.""" + # Mock threading.Thread so no real thread is created and no real network + # call can escape the test boundary (repo rule: no real network calls). + with patch("threading.Thread") as mock_thread_cls: + mock_thread = MagicMock() + mock_thread_cls.return_value = mock_thread + + # Simulate the send function raising an OSError inside the thread + def run_target(*args, **kwargs): + target = mock_thread_cls.call_args[1].get("target") or mock_thread_cls.call_args[0][0] + try: + target() + except Exception: + pass # errors in thread body must not surface + + mock_thread.start.side_effect = run_target + with patch("urllib.request.urlopen", side_effect=OSError("connection refused")): + _fire_and_forget({"provider": "anthropic", "model": "claude-haiku-4-5"}) + + +# ── Async hooks ─────────────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_async_log_failure_event_calls_report(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) + kwargs = {"model": "gpt-4o-mini", "exception": None} + + with patch.object(logger, "_report") as mock_report: + await logger.async_log_failure_event(kwargs, None, start, end) + mock_report.assert_called_once_with(kwargs, start, end) + + +# ── Additional tests added to hit 67 % patch coverage ──────────────────────── + + +# ── Sync log_failure_event hook ─────────────────────────────────────────────── + + +def test_log_failure_event_sync_calls_report(): + """log_failure_event (sync) must delegate to _report.""" + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=300) + kwargs = {"model": "gpt-4o-mini", "exception": None} + + with patch.object(logger, "_report") as mock_report: + logger.log_failure_event(kwargs, None, start, end) + mock_report.assert_called_once_with(kwargs, start, end) + + +# ── _normalize_provider: extra provider map entries ─────────────────────────── + + +def test_normalize_provider_vertex_ai_maps_to_google(): + kwargs = {"custom_llm_provider": "vertex_ai"} + assert _normalize_provider("gemini-2.5-pro", kwargs) == "google" + + +def test_normalize_provider_vertex_ai_anthropic_maps_to_anthropic(): + kwargs = {"custom_llm_provider": "vertex_ai_anthropic"} + assert _normalize_provider("claude-3-5-sonnet", kwargs) == "anthropic" + + +def test_normalize_provider_azure_maps_to_azure(): + kwargs = {"litellm_params": {"custom_llm_provider": "azure"}} + assert _normalize_provider("gpt-4o", kwargs) == "azure" + + +def test_normalize_provider_bedrock_maps_to_aws(): + kwargs = {"custom_llm_provider": "bedrock"} + assert _normalize_provider("claude-3-haiku", kwargs) == "aws" + + +def test_normalize_provider_bedrock_converse_maps_to_aws(): + kwargs = {"custom_llm_provider": "bedrock_converse"} + assert _normalize_provider("claude-3-haiku", kwargs) == "aws" + + +def test_normalize_provider_command_model_pattern(): + """command-* model names should map to cohere via regex.""" + assert _normalize_provider("command-r-plus", {}) == "cohere" + assert _normalize_provider("command-r", {}) == "cohere" + + +def test_normalize_provider_mixtral_model_pattern(): + """mixtral-* model names should map to mistral via regex.""" + assert _normalize_provider("mixtral-8x7b-instruct", {}) == "mistral" + + +def test_normalize_provider_deepseek_from_param(): + kwargs = {"custom_llm_provider": "deepseek"} + assert _normalize_provider("deepseek-v3", kwargs) == "deepseek" + + +def test_normalize_provider_openrouter_from_param(): + kwargs = {"litellm_params": {"custom_llm_provider": "openrouter"}} + assert _normalize_provider("meta-llama/llama-3.3-70b-instruct", kwargs) == "openrouter" + + +def test_normalize_provider_fireworks_ai_from_param(): + kwargs = {"custom_llm_provider": "fireworks_ai"} + assert _normalize_provider("llama-v3-70b-instruct", kwargs) == "fireworks" + + +def test_normalize_provider_cerebras_from_param(): + kwargs = {"custom_llm_provider": "cerebras"} + assert _normalize_provider("llama3.1-8b", kwargs) == "cerebras" + + +def test_normalize_provider_xai_from_param(): + kwargs = {"custom_llm_provider": "xai"} + assert _normalize_provider("grok-3-mini", kwargs) == "xai" + + +# ── _report: edge cases ─────────────────────────────────────────────────────── + + +def test_report_no_exception_omits_error_fields(): + """When exception is None, payload must not include error_code or error_type.""" + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) + kwargs = {"model": "gemini-2.5-flash", "exception": None} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert "error_code" not in captured + assert "error_type" not in captured + assert captured["provider"] == "google" + + +def test_report_includes_client_tier_and_region_in_payload(): + """client_tier and region must appear in the payload when set via env.""" + with patch.dict(os.environ, {"TICKERR_CLIENT_TIER": "enterprise", "TICKERR_REGION": "eu-west-1"}): + logger = TickerrLogger() + + captured = {} + + def fake_fire(payload): + captured.update(payload) + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + kwargs = {"model": "gpt-4o", "exception": None} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["client_tier"] == "enterprise" + assert captured["region"] == "eu-west-1" + + +def test_report_empty_model_sets_model_none(): + """An empty model string must result in model=None in the payload.""" + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=50) + kwargs = {"model": "", "exception": None} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["model"] is None + + +def test_report_timeout_error_type(): + """408 and 524 both map to error_type='timeout'.""" + logger = TickerrLogger() + + for code in (408, 524): + captured = {} + + def fake_fire(payload, _c=code): + captured.update(payload) + + exc = MagicMock() + exc.status_code = code + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=5000) + kwargs = {"model": "claude-haiku-4-5", "exception": exc} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["error_type"] == "timeout", f"Expected timeout for {code}" + assert captured["error_code"] == code + + +def test_report_auth_error_type(): + """401 and 403 both map to error_type='auth'.""" + logger = TickerrLogger() + + for code in (401, 403): + captured = {} + + def fake_fire(payload): + captured.update(payload) + + exc = MagicMock() + exc.status_code = code + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) + kwargs = {"model": "gpt-4o", "exception": exc} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["error_type"] == "auth", f"Expected auth for {code}" + + +# ── _fire_and_forget: normal success path ───────────────────────────────────── + + +def test_fire_and_forget_success_path_releases_semaphore(): + """On a successful HTTP call, the semaphore slot must be released.""" + mock_response = MagicMock() + mock_response.__enter__ = MagicMock(return_value=mock_response) + mock_response.__exit__ = MagicMock(return_value=False) + + with patch("threading.Thread") as mock_thread_cls: + mock_thread = MagicMock() + mock_thread_cls.return_value = mock_thread + + def run_target_inline(*args, **kwargs): + # Extract and call the real target so _send runs synchronously + target = mock_thread_cls.call_args[1].get("target") or mock_thread_cls.call_args[0][0] + target() + + mock_thread.start.side_effect = run_target_inline + + with patch("urllib.request.urlopen", return_value=mock_response): + _fire_and_forget({"provider": "openai", "model": "gpt-4o-mini"}) + + # Semaphore must be acquirable — proves the finally block released the slot + acquired = _inflight.acquire(blocking=False) + assert acquired, "semaphore slot was not released after successful send" + _inflight.release() + + +# ── _latency_ms edge cases ──────────────────────────────────────────────────── + + +def test_latency_ms_zero(): + assert _latency_ms(1000.0, 1000.0) == 0 + + +def test_latency_ms_large_value(): + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(seconds=30) + assert _latency_ms(start, end) == 30_000 + + +# ── _extract_status_code edge cases ────────────────────────────────────────── + + +def test_extract_status_code_non_digit_string(): + """A non-numeric string status_code must return None.""" + exc = MagicMock() + exc.status_code = "N/A" + assert _extract_status_code(exc) is None + + +def test_extract_status_code_529(): + exc = MagicMock() + exc.status_code = 529 + assert _extract_status_code(exc) == 529 From 51a67bb2a00bfde6b3d0f60c62d4aff6c5ffeb94 Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Fri, 29 May 2026 01:31:20 +0530 Subject: [PATCH 10/20] refactor(tickerr): slim to 103-line dumb pipe, server owns all normalization - Remove _PROVIDER_MAP (25 entries), _ERROR_TYPE_MAP, _normalize_provider, _get_provider, _get_status_code, _latency_ms, _fire_and_forget, _parse_sample_rate - Provider read directly from custom_llm_provider, passed as-is - Model passed as-is, no prefix stripping - status_code sent raw, server derives error_type - Inline fire-and-forget into _report, remove semaphore - Add TICKERR_DISABLED kill switch - Add opt-in success sampling (TICKERR_SAMPLE_RATE, default 0) - event_type: "failure"/"success" replaces is_resolution boolean - Remove client_tier (trust concern for first merge) - Tests import only TickerrLogger, no internal symbols - 103 lines source, 19 tests, 55 lines docs --- docs/my-website/docs/observability/tickerr.md | 74 +- litellm/integrations/tickerr.py | 277 ++------ .../test_tickerr_callback.py | 655 +++++------------- .../integrations/test_tickerr_callback.py | 652 +++++------------ 4 files changed, 458 insertions(+), 1200 deletions(-) diff --git a/docs/my-website/docs/observability/tickerr.md b/docs/my-website/docs/observability/tickerr.md index d4f9855d998..cdc4a212b79 100644 --- a/docs/my-website/docs/observability/tickerr.md +++ b/docs/my-website/docs/observability/tickerr.md @@ -1,15 +1,11 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Tickerr — Outage Radar for AI Agents +# Tickerr - Outage Radar for AI Agents -[Tickerr](https://tickerr.ai) is a crowd-sourced outage detector for LLM APIs. When your agent hits a 5xx error or rate limit, Tickerr tells you: +[Tickerr](https://tickerr.ai) is a crowd-sourced outage detector for LLM APIs. When your agent hits a 5xx or rate limit, it reports anonymously so every agent can see the issue in real time. -- How many other agents are seeing the same issue right now -- Current signal state: `quiet` → `detecting` → `confirmed` → `recovering` -- Which model to fall back to - -**No API key required. Anonymous. Zero overhead on success paths.** +**No API key. No account. Failure-only by default. Success sampling is opt-in.** ## Quick Start @@ -21,10 +17,8 @@ import litellm litellm.callbacks = ["tickerr"] -# Every failed LiteLLM call is now reported to Tickerr automatically. -# Tickerr fires in a background thread — your agent is never blocked. response = litellm.completion( - model="claude-haiku-4-5", + model="gpt-4o-mini", messages=[{"role": "user", "content": "Hello"}] ) ``` @@ -33,12 +27,6 @@ response = litellm.completion( ```yaml -model_list: - - model_name: claude-haiku - litellm_params: - model: anthropic/claude-haiku-4-5 - api_key: os.environ/ANTHROPIC_API_KEY - litellm_settings: callbacks: ["tickerr"] ``` @@ -48,60 +36,28 @@ litellm_settings: ## What Gets Reported -Tickerr receives only: +When the tickerr callback is explicitly enabled, anonymous failure metadata is reported. No prompts, responses, API keys, or personal data are sent. | Field | Example | |-------|---------| | Provider | `anthropic` | | Model | `claude-haiku-4-5` | -| HTTP status code | `529` | -| Error type | `overloaded` | +| Status code | `529` | | Latency (ms) | `1240` | - -No prompts, no responses, no personal data. - -## What You Get Back - -Each report updates the live signal at [tickerr.ai/agent-reports](https://tickerr.ai/agent-reports). - -To read the signal from your agent, use the [Tickerr MCP server](https://tickerr.ai/mcp-server) `report_incident` tool. It returns a structured response: - -``` -CURRENT SIGNAL (anthropic/claude-haiku-4-5) -Status: CONFIRMED -Agents reporting (last 10 min): 14 -Total reports (last 10 min): 31 - -RECOMMENDATION -Action: FALLBACK -Switch to: gpt-4o-mini (openai) -``` +| Event type | `failure` or `success` | ## Optional Configuration -```python -import os - -os.environ["TICKERR_CLIENT_TIER"] = "pro" # free | pro | team | enterprise | api_pay_as_you_go -os.environ["TICKERR_REGION"] = "us-east-1" # optional, for regional breakdown +```bash +TICKERR_DISABLED=true # disable all reporting (kill switch) +TICKERR_REGION=us-east-1 # for regional signal breakdown +TICKERR_SAMPLE_RATE=0.01 # report 1% of successes for latency benchmarks (default: 0 = off) ``` -## Signal States - -| State | Meaning | -|-------|---------| -| `quiet` | No reports in last 10 min | -| `detecting` | 1–2 agents reporting | -| `confirmed` | 3+ distinct agents — issue verified | -| `recovering` | Reports dropping, recovery signals arriving | - -## Opt Out - -[tickerr.ai/mcp/opt-out](https://tickerr.ai/mcp/opt-out) +Failures are reported by default once Tickerr is enabled. Success sampling is opt-in. ## Links -- [Tickerr](https://tickerr.ai) — live AI status dashboard (90+ tools) -- [Agent reports](https://tickerr.ai/agent-reports) — live feed -- [Tickerr MCP server](https://tickerr.ai/mcp-server) — 9-tool MCP for agents -- [REST API](https://tickerr.ai/api/v1/report) — report without LiteLLM +- [Live dashboard](https://tickerr.ai) - status for 90+ AI tools +- [Agent reports feed](https://tickerr.ai/agent-reports) - real-time signal +- [Opt out](https://tickerr.ai/mcp/opt-out) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index 75732d8e515..a77e3228f14 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -1,23 +1,23 @@ """ -Tickerr — outage radar for AI agents. +Tickerr - crowd-sourced outage radar for AI agents. -Reports LLM API failures to https://tickerr.ai so agents can -see how many other agents are hitting the same issue and get -a live routing recommendation (RETRY / RETRY_WITH_DELAY / FALLBACK). - -Zero dependencies beyond stdlib. Anonymous. Non-blocking. +Reports LLM API failures to https://tickerr.ai so every agent +can see when a provider is down and which model to fall back to. Usage: litellm.callbacks = ["tickerr"] -No API key required. +No API key. No account. Failure-only by default. Success sampling is opt-in. +Zero dependencies beyond stdlib. """ from __future__ import annotations +import json import os -import re +import random import threading +import urllib.request from datetime import datetime from typing import Any, Dict, Optional, Union @@ -26,213 +26,78 @@ from litellm.integrations.custom_logger import CustomLogger _REPORT_URL = "https://tickerr.ai/api/v1/report" _UA = "litellm-tickerr/1.0" -# Cap concurrent in-flight reports to avoid thread exhaustion on burst failures -_MAX_INFLIGHT = 5 -_inflight = threading.Semaphore(_MAX_INFLIGHT) - -# Map litellm custom_llm_provider → Tickerr provider slug -_PROVIDER_MAP: Dict[str, str] = { - "openai": "openai", - "anthropic": "anthropic", - "google": "google", - "vertex_ai": "google", - "gemini": "google", - "cohere": "cohere", - "mistral": "mistral", - "groq": "groq", - "together_ai": "together", - "huggingface": "huggingface", - "replicate": "replicate", - "deepinfra": "deepinfra", - "perplexity": "perplexity", - "fireworks_ai": "fireworks", - "openrouter": "openrouter", - "azure": "azure", - "azure_ai": "azure", - "bedrock": "aws", - "bedrock_converse": "aws", - "vertex_ai_anthropic": "anthropic", - "ai21": "ai21", - "cerebras": "cerebras", - "xai": "xai", - "deepseek": "deepseek", - "ollama": "ollama", - "nlp_cloud": "nlp_cloud", -} - -# Only map codes that unambiguously indicate the specific error type. -# 500 is intentionally excluded: it is a generic "Internal Server Error" that -# indicates a crash or bug, not a capacity/overload condition. -_ERROR_TYPE_MAP: Dict[int, str] = { - 429: "rate_limit", - 529: "overloaded", - 503: "overloaded", - 408: "timeout", - 524: "timeout", - 401: "auth", - 403: "auth", -} - - -def _normalize_provider(model: str, kwargs: Dict[str, Any]) -> str: - custom = ( - kwargs.get("litellm_params", {}).get("custom_llm_provider") - or kwargs.get("custom_llm_provider") - or "" - ) - if custom: - return _PROVIDER_MAP.get(custom.lower(), custom.lower()) - if "/" in model: - prefix = model.split("/")[0].lower() - return _PROVIDER_MAP.get(prefix, prefix) - if re.match(r"^claude", model, re.I): - return "anthropic" - if re.match(r"^gpt|^o[1-9]", model, re.I): - return "openai" - if re.match(r"^gemini", model, re.I): - return "google" - if re.match(r"^mistral|^mixtral", model, re.I): - return "mistral" - if re.match(r"^llama", model, re.I): - return "meta" - if re.match(r"^command", model, re.I): - return "cohere" - if re.match(r"^grok", model, re.I): - return "xai" - if re.match(r"^deepseek", model, re.I): - return "deepseek" - return "unknown" - - -def _extract_status_code(exception: Optional[BaseException]) -> Optional[int]: - if exception is None: - return None - code = getattr(exception, "status_code", None) - if isinstance(code, int): - return code - if isinstance(code, str) and code.isdigit(): - return int(code) - return None - - -def _latency_ms( - start_time: Union[datetime, float], end_time: Union[datetime, float] -) -> int: - """Compute elapsed ms whether LiteLLM passes datetime objects or floats.""" - if isinstance(start_time, datetime) and isinstance(end_time, datetime): - return round((end_time - start_time).total_seconds() * 1000) - return round((float(end_time) - float(start_time)) * 1000) # type: ignore[arg-type] - - -def _fire_and_forget(payload: Dict[str, Any]) -> None: - """POST to Tickerr in a daemon thread — never blocks the caller. - - A semaphore caps concurrent in-flight threads so a burst of failures - (e.g. 100 errors/s) cannot exhaust the thread pool. - """ - if not _inflight.acquire(blocking=False): - return # already at max concurrent reports — drop silently - - def _send() -> None: - try: - import json as _json - import urllib.request - - data = _json.dumps(payload).encode() - req = urllib.request.Request( - _REPORT_URL, - data=data, - headers={"Content-Type": "application/json", "User-Agent": _UA}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=5): - pass - except Exception: - pass # never crash the caller - finally: - _inflight.release() - - t = threading.Thread(target=_send, daemon=True) - try: - t.start() - except Exception: - # Thread could not be started (e.g. OS thread limit). - # Release the slot so future reports are not permanently blocked. - _inflight.release() - class TickerrLogger(CustomLogger): """ - LiteLLM built-in callback for Tickerr. + LiteLLM callback that reports LLM API failures to Tickerr. - Activated with: - litellm.callbacks = ["tickerr"] + When explicitly enabled via ``litellm.callbacks = ["tickerr"]``, + anonymous failure metadata is reported. No prompts, responses, + API keys, or personal data are sent. Optional env vars: - TICKERR_CLIENT_TIER — "free" | "pro" | "team" | "enterprise" | "api_pay_as_you_go" - TICKERR_REGION — e.g. "us-east-1" + TICKERR_DISABLED - set to "true" to disable all reporting + TICKERR_REGION - e.g. us-east-1 + TICKERR_SAMPLE_RATE - fraction of successes to report (0.0-1.0, default 0 = off) """ def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - self.client_tier: Optional[str] = os.environ.get("TICKERR_CLIENT_TIER") - self.region: Optional[str] = os.environ.get("TICKERR_REGION") - - # ── sync ────────────────────────────────────────────────────────────────── - - def log_failure_event( - self, - kwargs: Dict[str, Any], - response_obj: Any, - start_time: Union[datetime, float], - end_time: Union[datetime, float], - ) -> None: - self._report(kwargs, start_time, end_time) - - # ── async ───────────────────────────────────────────────────────────────── - - async def async_log_failure_event( - self, - kwargs: Dict[str, Any], - response_obj: Any, - start_time: Union[datetime, float], - end_time: Union[datetime, float], - ) -> None: - self._report(kwargs, start_time, end_time) - - # ── internal ────────────────────────────────────────────────────────────── - - def _report( - self, - kwargs: Dict[str, Any], - start_time: Union[datetime, float], - end_time: Union[datetime, float], - ) -> None: - model: str = kwargs.get("model", "") or "" - exception: Optional[BaseException] = kwargs.get("exception") - - provider = _normalize_provider(model, kwargs) - status_code = _extract_status_code(exception) - # Only set error_type for codes we can classify with confidence - error_type: Optional[str] = ( - _ERROR_TYPE_MAP.get(status_code) if status_code is not None else None - ) - - # Strip provider prefix: "anthropic/claude-3-5-haiku" → "claude-3-5-haiku" - model_clean = model.split("/", 1)[-1] if "/" in model else model - - payload: Dict[str, Any] = { - "provider": provider, - "model": model_clean or None, - "latency_ms": _latency_ms(start_time, end_time), + self.disabled: bool = os.environ.get("TICKERR_DISABLED", "").lower() in { + "1", "true", "yes", } - if status_code is not None: - payload["error_code"] = status_code - if error_type: - payload["error_type"] = error_type - if self.client_tier: - payload["client_tier"] = self.client_tier - if self.region: - payload["region"] = self.region + self.region: Optional[str] = os.environ.get("TICKERR_REGION") + try: + self.sample_rate: float = min(1.0, max(0.0, float(os.environ.get("TICKERR_SAMPLE_RATE", "0")))) + except (ValueError, TypeError): + self.sample_rate = 0.0 - _fire_and_forget(payload) + def log_failure_event(self, kwargs: Dict[str, Any], response_obj: Any, start_time: Union[datetime, float], end_time: Union[datetime, float]) -> None: + self._report(kwargs, start_time, end_time) + + async def async_log_failure_event(self, kwargs: Dict[str, Any], response_obj: Any, start_time: Union[datetime, float], end_time: Union[datetime, float]) -> None: + self._report(kwargs, start_time, end_time) + + def log_success_event(self, kwargs: Dict[str, Any], response_obj: Any, start_time: Union[datetime, float], end_time: Union[datetime, float]) -> None: + if self.sample_rate > 0 and random.random() < self.sample_rate: + self._report(kwargs, start_time, end_time, is_success=True) + + async def async_log_success_event(self, kwargs: Dict[str, Any], response_obj: Any, start_time: Union[datetime, float], end_time: Union[datetime, float]) -> None: + if self.sample_rate > 0 and random.random() < self.sample_rate: + self._report(kwargs, start_time, end_time, is_success=True) + + def _report(self, kwargs: Dict[str, Any], start_time: Union[datetime, float], end_time: Union[datetime, float], is_success: bool = False) -> None: + if self.disabled: + return + + model: str = kwargs.get("model", "") or "" + + if isinstance(start_time, datetime) and isinstance(end_time, datetime): + latency = round((end_time - start_time).total_seconds() * 1000) + else: + latency = round((float(end_time) - float(start_time)) * 1000) + + payload = {k: v for k, v in { + "provider": kwargs.get("litellm_params", {}).get("custom_llm_provider") or kwargs.get("custom_llm_provider"), + "model": model or None, + "latency_ms": latency, + "event_type": "success" if is_success else "failure", + "status_code": getattr(kwargs.get("exception"), "status_code", None), + "region": self.region, + }.items() if v is not None} + + def _send() -> None: + try: + urllib.request.urlopen( + urllib.request.Request( + _REPORT_URL, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", "User-Agent": _UA}, + method="POST", + ), + timeout=2, + ) + except Exception: + pass + + threading.Thread(target=_send, daemon=True).start() diff --git a/tests/logging_callback_tests/test_tickerr_callback.py b/tests/logging_callback_tests/test_tickerr_callback.py index 8690690873c..7e5ccc49ff1 100644 --- a/tests/logging_callback_tests/test_tickerr_callback.py +++ b/tests/logging_callback_tests/test_tickerr_callback.py @@ -1,545 +1,262 @@ """ Unit tests for the Tickerr LiteLLM callback. - -Tests cover: -- TickerrLogger instantiation and env var config -- Provider normalization from model names and litellm_params -- Status code extraction from exceptions -- Latency calculation for both datetime and float timestamps -- Error type mapping (only known codes, no fallback default) -- Payload construction -- Thread cap (semaphore) under burst conditions -- Fire-and-forget does not block or raise on network failure """ +import json import os -import sys from datetime import datetime, timedelta from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) - -from litellm.integrations.tickerr import ( - TickerrLogger, - _ERROR_TYPE_MAP, - _extract_status_code, - _fire_and_forget, - _inflight, - _latency_ms, - _normalize_provider, - _MAX_INFLIGHT, -) +from litellm.integrations.tickerr import TickerrLogger -# ── Provider normalization ──────────────────────────────────────────────────── +# -- Config -------------------------------------------------------------------- -def test_normalize_provider_from_litellm_params(): - kwargs = {"litellm_params": {"custom_llm_provider": "anthropic"}} - assert _normalize_provider("some-model", kwargs) == "anthropic" - - -def test_normalize_provider_from_custom_llm_provider(): - kwargs = {"custom_llm_provider": "openai"} - assert _normalize_provider("gpt-4o", kwargs) == "openai" - - -def test_normalize_provider_from_model_prefix(): - assert _normalize_provider("anthropic/claude-3-5-haiku", {}) == "anthropic" - assert _normalize_provider("openai/gpt-4o", {}) == "openai" - - -def test_normalize_provider_from_model_name_pattern(): - assert _normalize_provider("claude-haiku-4-5", {}) == "anthropic" - assert _normalize_provider("gpt-4o-mini", {}) == "openai" - assert _normalize_provider("gemini-2.5-flash", {}) == "google" - assert _normalize_provider("mistral-small-latest", {}) == "mistral" - assert _normalize_provider("llama-3.3-70b", {}) == "meta" - assert _normalize_provider("grok-3-mini", {}) == "xai" - assert _normalize_provider("deepseek-v3", {}) == "deepseek" - - -def test_normalize_provider_unknown(): - assert _normalize_provider("some-unknown-model-xyz", {}) == "unknown" - - -# ── Status code extraction ──────────────────────────────────────────────────── - - -def test_extract_status_code_int(): - exc = MagicMock() - exc.status_code = 429 - assert _extract_status_code(exc) == 429 - - -def test_extract_status_code_string(): - exc = MagicMock() - exc.status_code = "503" - assert _extract_status_code(exc) == 503 - - -def test_extract_status_code_none_exception(): - assert _extract_status_code(None) is None - - -def test_extract_status_code_no_attribute(): - assert _extract_status_code(ValueError("oops")) is None - - -# ── Latency calculation ─────────────────────────────────────────────────────── - - -def test_latency_ms_with_datetime(): - start = datetime(2024, 1, 1, 0, 0, 0) - end = start + timedelta(milliseconds=1240) - assert _latency_ms(start, end) == 1240 - - -def test_latency_ms_with_floats(): - assert _latency_ms(1000.0, 1001.5) == 1500 - - -def test_latency_ms_mixed_types_float(): - # Both floats — should not raise - result = _latency_ms(0.0, 0.5) - assert result == 500 - - -# ── Error type mapping ──────────────────────────────────────────────────────── - - -def test_error_type_known_codes(): - assert _ERROR_TYPE_MAP[429] == "rate_limit" - assert _ERROR_TYPE_MAP[529] == "overloaded" - assert _ERROR_TYPE_MAP[503] == "overloaded" - assert _ERROR_TYPE_MAP[408] == "timeout" - assert _ERROR_TYPE_MAP[401] == "auth" - - -def test_error_type_no_default_for_unknown_codes(): - # 500 is a generic server error (crash/bug), not definitively "overloaded". - # Unknown codes must NOT map to any value. - for code in (400, 404, 500, 502, 422, 301): - assert code not in _ERROR_TYPE_MAP, f"code {code} should not be in _ERROR_TYPE_MAP" - - -# ── TickerrLogger instantiation ─────────────────────────────────────────────── - - -def test_tickerr_logger_default_init(): +def test_default_init(): logger = TickerrLogger() - assert logger.client_tier is None + assert logger.disabled is False + assert logger.sample_rate == 0.0 assert logger.region is None -def test_tickerr_logger_reads_env_vars(): - with patch.dict(os.environ, {"TICKERR_CLIENT_TIER": "pro", "TICKERR_REGION": "us-east-1"}): +def test_reads_env_vars(): + with patch.dict(os.environ, {"TICKERR_REGION": "us-east-1", "TICKERR_SAMPLE_RATE": "0.1"}): logger = TickerrLogger() - assert logger.client_tier == "pro" assert logger.region == "us-east-1" + assert logger.sample_rate == 0.1 -# ── Payload construction via _report ───────────────────────────────────────── +def test_disabled_flag(): + with patch.dict(os.environ, {"TICKERR_DISABLED": "true"}): + logger = TickerrLogger() + assert logger.disabled is True -def test_report_builds_correct_payload(): +def test_invalid_sample_rate_does_not_crash(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "notanumber"}): + logger = TickerrLogger() + assert logger.sample_rate == 0.0 + + +def test_sample_rate_clamped_to_one(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "5.0"}): + logger = TickerrLogger() + assert logger.sample_rate == 1.0 + + +# -- Disabled ------------------------------------------------------------------ + + +def test_disabled_skips_report(): + with patch.dict(os.environ, {"TICKERR_DISABLED": "1"}): + logger = TickerrLogger() + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + logger._report({"model": "gpt-4o-mini", "exception": None}, start, end) + mock_thread.assert_not_called() + + +# -- Payload ------------------------------------------------------------------- + + +def test_failure_payload(): logger = TickerrLogger() - captured = {} - - def fake_fire(payload): - captured.update(payload) exc = MagicMock() exc.status_code = 429 - - start = datetime(2024, 1, 1, 0, 0, 0) + start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=500) - kwargs = { "model": "claude-haiku-4-5", "exception": exc, "litellm_params": {"custom_llm_provider": "anthropic"}, } - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report(kwargs, start, end) - assert captured["provider"] == "anthropic" - assert captured["model"] == "claude-haiku-4-5" - assert captured["error_code"] == 429 - assert captured["error_type"] == "rate_limit" - assert captured["latency_ms"] == 500 + sent = mock_urlopen.call_args[0][0] + payload = json.loads(sent.data) + assert payload["provider"] == "anthropic" + assert payload["model"] == "claude-haiku-4-5" + assert payload["status_code"] == 429 + assert payload["event_type"] == "failure" + assert payload["latency_ms"] == 500 -def test_report_strips_provider_prefix_from_model(): +def test_model_passed_as_is(): logger = TickerrLogger() - captured = {} - - def fake_fire(payload): - captured.update(payload) - - start = datetime(2024, 1, 1, 0, 0, 0) + start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - kwargs = {"model": "openai/gpt-4o-mini", "exception": None} + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "openai/gpt-4o-mini", "exception": None}, start, end) - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert captured["model"] == "gpt-4o-mini" + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["model"] == "openai/gpt-4o-mini" -def test_report_omits_error_type_for_unknown_code(): - logger = TickerrLogger() - captured = {} - - def fake_fire(payload): - captured.update(payload) - - exc = MagicMock() - exc.status_code = 400 # not in _ERROR_TYPE_MAP - - start = datetime(2024, 1, 1, 0, 0, 0) - end = start + timedelta(milliseconds=100) - kwargs = {"model": "gpt-4o", "exception": exc} - - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert "error_type" not in captured - assert captured["error_code"] == 400 - - -# ── Thread cap ──────────────────────────────────────────────────────────────── - - -def test_fire_and_forget_respects_semaphore_cap(): - """Reports beyond _MAX_INFLIGHT are dropped silently without blocking.""" - # Exhaust the semaphore by acquiring all slots directly. - # Track how many we actually acquired so the finally block releases exactly - # that many — releasing more than acquired would push the count above its - # initial maximum and corrupt later tests. - acquired_count = 0 - for _ in range(_MAX_INFLIGHT): - if _inflight.acquire(blocking=False): - acquired_count += 1 - else: - break - - assert acquired_count == _MAX_INFLIGHT, ( - f"semaphore should have {_MAX_INFLIGHT} slots available at test start, " - f"got {acquired_count}" - ) - - try: - # With semaphore exhausted, _fire_and_forget must return immediately - # without starting a thread (non-blocking acquire fails → early return) - with patch("threading.Thread") as mock_thread: - _fire_and_forget({"provider": "openai"}) - mock_thread.assert_not_called() - finally: - for _ in range(acquired_count): - _inflight.release() - - -def test_semaphore_released_on_thread_start_failure(): - """If t.start() raises, the semaphore slot must be released so future reports work.""" - # Verify the semaphore can be re-acquired after a thread-start failure, - # which proves the slot was released (without reading private CPython internals). - with patch("threading.Thread") as mock_thread: - mock_thread.return_value.start.side_effect = RuntimeError("OS thread limit") - _fire_and_forget({"provider": "openai"}) - - # If the semaphore was not released, this acquire would block forever. - # Use non-blocking to fail fast in case of a bug. - acquired = _inflight.acquire(blocking=False) - assert acquired, "semaphore slot was not released after thread start failure" - _inflight.release() # restore - - -# ── Network failure is silent ───────────────────────────────────────────────── - - -def test_fire_and_forget_silent_on_network_error(): - """A network error in the send function must not propagate to the caller.""" - # Mock threading.Thread so no real thread is created and no real network - # call can escape the test boundary (repo rule: no real network calls). - with patch("threading.Thread") as mock_thread_cls: - mock_thread = MagicMock() - mock_thread_cls.return_value = mock_thread - - # Simulate the send function raising an OSError inside the thread - def run_target(*args, **kwargs): - target = mock_thread_cls.call_args[1].get("target") or mock_thread_cls.call_args[0][0] - try: - target() - except Exception: - pass # errors in thread body must not surface - - mock_thread.start.side_effect = run_target - with patch("urllib.request.urlopen", side_effect=OSError("connection refused")): - _fire_and_forget({"provider": "anthropic", "model": "claude-haiku-4-5"}) - - -# ── Async hooks ─────────────────────────────────────────────────────────────── - - -@pytest.mark.asyncio -async def test_async_log_failure_event_calls_report(): +def test_no_exception_omits_status_code(): logger = TickerrLogger() start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=200) - kwargs = {"model": "gpt-4o-mini", "exception": None} - with patch.object(logger, "_report") as mock_report: - await logger.async_log_failure_event(kwargs, None, start, end) - mock_report.assert_called_once_with(kwargs, start, end) + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, start, end) + + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert "status_code" not in payload -# ── Additional tests added to hit 67 % patch coverage ──────────────────────── +def test_region_included(): + with patch.dict(os.environ, {"TICKERR_REGION": "eu-west-1"}): + logger = TickerrLogger() + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, start, end) + + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["region"] == "eu-west-1" -# ── Sync log_failure_event hook ─────────────────────────────────────────────── +def test_latency_from_floats(): + logger = TickerrLogger() + + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, 1000.0, 1001.5) + + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["latency_ms"] == 1500 -def test_log_failure_event_sync_calls_report(): - """log_failure_event (sync) must delegate to _report.""" +def test_provider_from_top_level_kwarg(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None, "custom_llm_provider": "openai"}, start, end) + + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["provider"] == "openai" + + +def test_no_provider_omits_field(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, start, end) + + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert "provider" not in payload + + +# -- Hooks --------------------------------------------------------------------- + + +def test_sync_failure_delegates(): logger = TickerrLogger() start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=300) - kwargs = {"model": "gpt-4o-mini", "exception": None} - with patch.object(logger, "_report") as mock_report: - logger.log_failure_event(kwargs, None, start, end) - mock_report.assert_called_once_with(kwargs, start, end) + with patch.object(logger, "_report") as mock: + logger.log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end) + mock.assert_called_once() -# ── _normalize_provider: extra provider map entries ─────────────────────────── - - -def test_normalize_provider_vertex_ai_maps_to_google(): - kwargs = {"custom_llm_provider": "vertex_ai"} - assert _normalize_provider("gemini-2.5-pro", kwargs) == "google" - - -def test_normalize_provider_vertex_ai_anthropic_maps_to_anthropic(): - kwargs = {"custom_llm_provider": "vertex_ai_anthropic"} - assert _normalize_provider("claude-3-5-sonnet", kwargs) == "anthropic" - - -def test_normalize_provider_azure_maps_to_azure(): - kwargs = {"litellm_params": {"custom_llm_provider": "azure"}} - assert _normalize_provider("gpt-4o", kwargs) == "azure" - - -def test_normalize_provider_bedrock_maps_to_aws(): - kwargs = {"custom_llm_provider": "bedrock"} - assert _normalize_provider("claude-3-haiku", kwargs) == "aws" - - -def test_normalize_provider_bedrock_converse_maps_to_aws(): - kwargs = {"custom_llm_provider": "bedrock_converse"} - assert _normalize_provider("claude-3-haiku", kwargs) == "aws" - - -def test_normalize_provider_command_model_pattern(): - """command-* model names should map to cohere via regex.""" - assert _normalize_provider("command-r-plus", {}) == "cohere" - assert _normalize_provider("command-r", {}) == "cohere" - - -def test_normalize_provider_mixtral_model_pattern(): - """mixtral-* model names should map to mistral via regex.""" - assert _normalize_provider("mixtral-8x7b-instruct", {}) == "mistral" - - -def test_normalize_provider_deepseek_from_param(): - kwargs = {"custom_llm_provider": "deepseek"} - assert _normalize_provider("deepseek-v3", kwargs) == "deepseek" - - -def test_normalize_provider_openrouter_from_param(): - kwargs = {"litellm_params": {"custom_llm_provider": "openrouter"}} - assert _normalize_provider("meta-llama/llama-3.3-70b-instruct", kwargs) == "openrouter" - - -def test_normalize_provider_fireworks_ai_from_param(): - kwargs = {"custom_llm_provider": "fireworks_ai"} - assert _normalize_provider("llama-v3-70b-instruct", kwargs) == "fireworks" - - -def test_normalize_provider_cerebras_from_param(): - kwargs = {"custom_llm_provider": "cerebras"} - assert _normalize_provider("llama3.1-8b", kwargs) == "cerebras" - - -def test_normalize_provider_xai_from_param(): - kwargs = {"custom_llm_provider": "xai"} - assert _normalize_provider("grok-3-mini", kwargs) == "xai" - - -# ── _report: edge cases ─────────────────────────────────────────────────────── - - -def test_report_no_exception_omits_error_fields(): - """When exception is None, payload must not include error_code or error_type.""" +@pytest.mark.asyncio +async def test_async_failure_delegates(): logger = TickerrLogger() - captured = {} + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) - def fake_fire(payload): - captured.update(payload) + with patch.object(logger, "_report") as mock: + await logger.async_log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end) + mock.assert_called_once() + + +# -- Success sampling ---------------------------------------------------------- + + +def test_success_not_reported_at_rate_zero(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch.object(logger, "_report") as mock: + logger.log_success_event({"model": "gpt-4o", "exception": None}, None, start, end) + mock.assert_not_called() + + +def test_success_reported_when_sampled(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}): + logger = TickerrLogger() start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=200) - kwargs = {"model": "gemini-2.5-flash", "exception": None} - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger.log_success_event({"model": "gpt-4o", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, None, start, end) - assert "error_code" not in captured - assert "error_type" not in captured - assert captured["provider"] == "google" + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["event_type"] == "success" + assert payload["provider"] == "openai" -def test_report_includes_client_tier_and_region_in_payload(): - """client_tier and region must appear in the payload when set via env.""" - with patch.dict(os.environ, {"TICKERR_CLIENT_TIER": "enterprise", "TICKERR_REGION": "eu-west-1"}): +@pytest.mark.asyncio +async def test_async_success_reported_when_sampled(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}): logger = TickerrLogger() - captured = {} + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=150) - def fake_fire(payload): - captured.update(payload) + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + await logger.async_log_success_event({"model": "gpt-4o-mini", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, None, start, end) + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["event_type"] == "success" + + +# -- Network failure does not crash -------------------------------------------- + + +def test_silent_on_network_error(): + logger = TickerrLogger() start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - kwargs = {"model": "gpt-4o", "exception": None} - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert captured["client_tier"] == "enterprise" - assert captured["region"] == "eu-west-1" - - -def test_report_empty_model_sets_model_none(): - """An empty model string must result in model=None in the payload.""" - logger = TickerrLogger() - captured = {} - - def fake_fire(payload): - captured.update(payload) - - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=50) - kwargs = {"model": "", "exception": None} - - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert captured["model"] is None - - -def test_report_timeout_error_type(): - """408 and 524 both map to error_type='timeout'.""" - logger = TickerrLogger() - - for code in (408, 524): - captured = {} - - def fake_fire(payload, _c=code): - captured.update(payload) - - exc = MagicMock() - exc.status_code = code - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=5000) - kwargs = {"model": "claude-haiku-4-5", "exception": exc} - - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert captured["error_type"] == "timeout", f"Expected timeout for {code}" - assert captured["error_code"] == code - - -def test_report_auth_error_type(): - """401 and 403 both map to error_type='auth'.""" - logger = TickerrLogger() - - for code in (401, 403): - captured = {} - - def fake_fire(payload): - captured.update(payload) - - exc = MagicMock() - exc.status_code = code - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=200) - kwargs = {"model": "gpt-4o", "exception": exc} - - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert captured["error_type"] == "auth", f"Expected auth for {code}" - - -# ── _fire_and_forget: normal success path ───────────────────────────────────── - - -def test_fire_and_forget_success_path_releases_semaphore(): - """On a successful HTTP call, the semaphore slot must be released.""" - mock_response = MagicMock() - mock_response.__enter__ = MagicMock(return_value=mock_response) - mock_response.__exit__ = MagicMock(return_value=False) - - with patch("threading.Thread") as mock_thread_cls: - mock_thread = MagicMock() - mock_thread_cls.return_value = mock_thread - - def run_target_inline(*args, **kwargs): - # Extract and call the real target so _send runs synchronously - target = mock_thread_cls.call_args[1].get("target") or mock_thread_cls.call_args[0][0] - target() - - mock_thread.start.side_effect = run_target_inline - - with patch("urllib.request.urlopen", return_value=mock_response): - _fire_and_forget({"provider": "openai", "model": "gpt-4o-mini"}) - - # Semaphore must be acquirable — proves the finally block released the slot - acquired = _inflight.acquire(blocking=False) - assert acquired, "semaphore slot was not released after successful send" - _inflight.release() - - -# ── _latency_ms edge cases ──────────────────────────────────────────────────── - - -def test_latency_ms_zero(): - assert _latency_ms(1000.0, 1000.0) == 0 - - -def test_latency_ms_large_value(): - start = datetime(2024, 1, 1, 0, 0, 0) - end = start + timedelta(seconds=30) - assert _latency_ms(start, end) == 30_000 - - -# ── _extract_status_code edge cases ────────────────────────────────────────── - - -def test_extract_status_code_non_digit_string(): - """A non-numeric string status_code must return None.""" - exc = MagicMock() - exc.status_code = "N/A" - assert _extract_status_code(exc) is None - - -def test_extract_status_code_529(): - exc = MagicMock() - exc.status_code = 529 - assert _extract_status_code(exc) == 529 + with patch("litellm.integrations.tickerr.urllib.request.urlopen", side_effect=OSError("refused")): + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, start, end) + # No exception raised — test passes diff --git a/tests/test_litellm/integrations/test_tickerr_callback.py b/tests/test_litellm/integrations/test_tickerr_callback.py index 37a1df9d560..7e5ccc49ff1 100644 --- a/tests/test_litellm/integrations/test_tickerr_callback.py +++ b/tests/test_litellm/integrations/test_tickerr_callback.py @@ -1,542 +1,262 @@ """ Unit tests for the Tickerr LiteLLM callback. - -Tests cover: -- TickerrLogger instantiation and env var config -- Provider normalization from model names and litellm_params -- Status code extraction from exceptions -- Latency calculation for both datetime and float timestamps -- Error type mapping (only known codes, no fallback default) -- Payload construction -- Thread cap (semaphore) under burst conditions -- Fire-and-forget does not block or raise on network failure """ +import json import os from datetime import datetime, timedelta from unittest.mock import MagicMock, patch import pytest -from litellm.integrations.tickerr import ( - TickerrLogger, - _ERROR_TYPE_MAP, - _extract_status_code, - _fire_and_forget, - _inflight, - _latency_ms, - _normalize_provider, - _MAX_INFLIGHT, -) +from litellm.integrations.tickerr import TickerrLogger -# ── Provider normalization ──────────────────────────────────────────────────── +# -- Config -------------------------------------------------------------------- -def test_normalize_provider_from_litellm_params(): - kwargs = {"litellm_params": {"custom_llm_provider": "anthropic"}} - assert _normalize_provider("some-model", kwargs) == "anthropic" - - -def test_normalize_provider_from_custom_llm_provider(): - kwargs = {"custom_llm_provider": "openai"} - assert _normalize_provider("gpt-4o", kwargs) == "openai" - - -def test_normalize_provider_from_model_prefix(): - assert _normalize_provider("anthropic/claude-3-5-haiku", {}) == "anthropic" - assert _normalize_provider("openai/gpt-4o", {}) == "openai" - - -def test_normalize_provider_from_model_name_pattern(): - assert _normalize_provider("claude-haiku-4-5", {}) == "anthropic" - assert _normalize_provider("gpt-4o-mini", {}) == "openai" - assert _normalize_provider("gemini-2.5-flash", {}) == "google" - assert _normalize_provider("mistral-small-latest", {}) == "mistral" - assert _normalize_provider("llama-3.3-70b", {}) == "meta" - assert _normalize_provider("grok-3-mini", {}) == "xai" - assert _normalize_provider("deepseek-v3", {}) == "deepseek" - - -def test_normalize_provider_unknown(): - assert _normalize_provider("some-unknown-model-xyz", {}) == "unknown" - - -# ── Status code extraction ──────────────────────────────────────────────────── - - -def test_extract_status_code_int(): - exc = MagicMock() - exc.status_code = 429 - assert _extract_status_code(exc) == 429 - - -def test_extract_status_code_string(): - exc = MagicMock() - exc.status_code = "503" - assert _extract_status_code(exc) == 503 - - -def test_extract_status_code_none_exception(): - assert _extract_status_code(None) is None - - -def test_extract_status_code_no_attribute(): - assert _extract_status_code(ValueError("oops")) is None - - -# ── Latency calculation ─────────────────────────────────────────────────────── - - -def test_latency_ms_with_datetime(): - start = datetime(2024, 1, 1, 0, 0, 0) - end = start + timedelta(milliseconds=1240) - assert _latency_ms(start, end) == 1240 - - -def test_latency_ms_with_floats(): - assert _latency_ms(1000.0, 1001.5) == 1500 - - -def test_latency_ms_mixed_types_float(): - # Both floats — should not raise - result = _latency_ms(0.0, 0.5) - assert result == 500 - - -# ── Error type mapping ──────────────────────────────────────────────────────── - - -def test_error_type_known_codes(): - assert _ERROR_TYPE_MAP[429] == "rate_limit" - assert _ERROR_TYPE_MAP[529] == "overloaded" - assert _ERROR_TYPE_MAP[503] == "overloaded" - assert _ERROR_TYPE_MAP[408] == "timeout" - assert _ERROR_TYPE_MAP[401] == "auth" - - -def test_error_type_no_default_for_unknown_codes(): - # 500 is a generic server error (crash/bug), not definitively "overloaded". - # Unknown codes must NOT map to any value. - for code in (400, 404, 500, 502, 422, 301): - assert code not in _ERROR_TYPE_MAP, f"code {code} should not be in _ERROR_TYPE_MAP" - - -# ── TickerrLogger instantiation ─────────────────────────────────────────────── - - -def test_tickerr_logger_default_init(): +def test_default_init(): logger = TickerrLogger() - assert logger.client_tier is None + assert logger.disabled is False + assert logger.sample_rate == 0.0 assert logger.region is None -def test_tickerr_logger_reads_env_vars(): - with patch.dict(os.environ, {"TICKERR_CLIENT_TIER": "pro", "TICKERR_REGION": "us-east-1"}): +def test_reads_env_vars(): + with patch.dict(os.environ, {"TICKERR_REGION": "us-east-1", "TICKERR_SAMPLE_RATE": "0.1"}): logger = TickerrLogger() - assert logger.client_tier == "pro" assert logger.region == "us-east-1" + assert logger.sample_rate == 0.1 -# ── Payload construction via _report ───────────────────────────────────────── +def test_disabled_flag(): + with patch.dict(os.environ, {"TICKERR_DISABLED": "true"}): + logger = TickerrLogger() + assert logger.disabled is True -def test_report_builds_correct_payload(): +def test_invalid_sample_rate_does_not_crash(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "notanumber"}): + logger = TickerrLogger() + assert logger.sample_rate == 0.0 + + +def test_sample_rate_clamped_to_one(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "5.0"}): + logger = TickerrLogger() + assert logger.sample_rate == 1.0 + + +# -- Disabled ------------------------------------------------------------------ + + +def test_disabled_skips_report(): + with patch.dict(os.environ, {"TICKERR_DISABLED": "1"}): + logger = TickerrLogger() + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + logger._report({"model": "gpt-4o-mini", "exception": None}, start, end) + mock_thread.assert_not_called() + + +# -- Payload ------------------------------------------------------------------- + + +def test_failure_payload(): logger = TickerrLogger() - captured = {} - - def fake_fire(payload): - captured.update(payload) exc = MagicMock() exc.status_code = 429 - - start = datetime(2024, 1, 1, 0, 0, 0) + start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=500) - kwargs = { "model": "claude-haiku-4-5", "exception": exc, "litellm_params": {"custom_llm_provider": "anthropic"}, } - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report(kwargs, start, end) - assert captured["provider"] == "anthropic" - assert captured["model"] == "claude-haiku-4-5" - assert captured["error_code"] == 429 - assert captured["error_type"] == "rate_limit" - assert captured["latency_ms"] == 500 + sent = mock_urlopen.call_args[0][0] + payload = json.loads(sent.data) + assert payload["provider"] == "anthropic" + assert payload["model"] == "claude-haiku-4-5" + assert payload["status_code"] == 429 + assert payload["event_type"] == "failure" + assert payload["latency_ms"] == 500 -def test_report_strips_provider_prefix_from_model(): +def test_model_passed_as_is(): logger = TickerrLogger() - captured = {} - - def fake_fire(payload): - captured.update(payload) - - start = datetime(2024, 1, 1, 0, 0, 0) + start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - kwargs = {"model": "openai/gpt-4o-mini", "exception": None} + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "openai/gpt-4o-mini", "exception": None}, start, end) - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert captured["model"] == "gpt-4o-mini" + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["model"] == "openai/gpt-4o-mini" -def test_report_omits_error_type_for_unknown_code(): - logger = TickerrLogger() - captured = {} - - def fake_fire(payload): - captured.update(payload) - - exc = MagicMock() - exc.status_code = 400 # not in _ERROR_TYPE_MAP - - start = datetime(2024, 1, 1, 0, 0, 0) - end = start + timedelta(milliseconds=100) - kwargs = {"model": "gpt-4o", "exception": exc} - - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert "error_type" not in captured - assert captured["error_code"] == 400 - - -# ── Thread cap ──────────────────────────────────────────────────────────────── - - -def test_fire_and_forget_respects_semaphore_cap(): - """Reports beyond _MAX_INFLIGHT are dropped silently without blocking.""" - # Exhaust the semaphore by acquiring all slots directly. - # Track how many we actually acquired so the finally block releases exactly - # that many — releasing more than acquired would push the count above its - # initial maximum and corrupt later tests. - acquired_count = 0 - for _ in range(_MAX_INFLIGHT): - if _inflight.acquire(blocking=False): - acquired_count += 1 - else: - break - - assert acquired_count == _MAX_INFLIGHT, ( - f"semaphore should have {_MAX_INFLIGHT} slots available at test start, " - f"got {acquired_count}" - ) - - try: - # With semaphore exhausted, _fire_and_forget must return immediately - # without starting a thread (non-blocking acquire fails → early return) - with patch("threading.Thread") as mock_thread: - _fire_and_forget({"provider": "openai"}) - mock_thread.assert_not_called() - finally: - for _ in range(acquired_count): - _inflight.release() - - -def test_semaphore_released_on_thread_start_failure(): - """If t.start() raises, the semaphore slot must be released so future reports work.""" - # Verify the semaphore can be re-acquired after a thread-start failure, - # which proves the slot was released (without reading private CPython internals). - with patch("threading.Thread") as mock_thread: - mock_thread.return_value.start.side_effect = RuntimeError("OS thread limit") - _fire_and_forget({"provider": "openai"}) - - # If the semaphore was not released, this acquire would block forever. - # Use non-blocking to fail fast in case of a bug. - acquired = _inflight.acquire(blocking=False) - assert acquired, "semaphore slot was not released after thread start failure" - _inflight.release() # restore - - -# ── Network failure is silent ───────────────────────────────────────────────── - - -def test_fire_and_forget_silent_on_network_error(): - """A network error in the send function must not propagate to the caller.""" - # Mock threading.Thread so no real thread is created and no real network - # call can escape the test boundary (repo rule: no real network calls). - with patch("threading.Thread") as mock_thread_cls: - mock_thread = MagicMock() - mock_thread_cls.return_value = mock_thread - - # Simulate the send function raising an OSError inside the thread - def run_target(*args, **kwargs): - target = mock_thread_cls.call_args[1].get("target") or mock_thread_cls.call_args[0][0] - try: - target() - except Exception: - pass # errors in thread body must not surface - - mock_thread.start.side_effect = run_target - with patch("urllib.request.urlopen", side_effect=OSError("connection refused")): - _fire_and_forget({"provider": "anthropic", "model": "claude-haiku-4-5"}) - - -# ── Async hooks ─────────────────────────────────────────────────────────────── - - -@pytest.mark.asyncio -async def test_async_log_failure_event_calls_report(): +def test_no_exception_omits_status_code(): logger = TickerrLogger() start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=200) - kwargs = {"model": "gpt-4o-mini", "exception": None} - with patch.object(logger, "_report") as mock_report: - await logger.async_log_failure_event(kwargs, None, start, end) - mock_report.assert_called_once_with(kwargs, start, end) + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, start, end) + + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert "status_code" not in payload -# ── Additional tests added to hit 67 % patch coverage ──────────────────────── +def test_region_included(): + with patch.dict(os.environ, {"TICKERR_REGION": "eu-west-1"}): + logger = TickerrLogger() + + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, start, end) + + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["region"] == "eu-west-1" -# ── Sync log_failure_event hook ─────────────────────────────────────────────── +def test_latency_from_floats(): + logger = TickerrLogger() + + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, 1000.0, 1001.5) + + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["latency_ms"] == 1500 -def test_log_failure_event_sync_calls_report(): - """log_failure_event (sync) must delegate to _report.""" +def test_provider_from_top_level_kwarg(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None, "custom_llm_provider": "openai"}, start, end) + + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["provider"] == "openai" + + +def test_no_provider_omits_field(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, start, end) + + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert "provider" not in payload + + +# -- Hooks --------------------------------------------------------------------- + + +def test_sync_failure_delegates(): logger = TickerrLogger() start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=300) - kwargs = {"model": "gpt-4o-mini", "exception": None} - with patch.object(logger, "_report") as mock_report: - logger.log_failure_event(kwargs, None, start, end) - mock_report.assert_called_once_with(kwargs, start, end) + with patch.object(logger, "_report") as mock: + logger.log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end) + mock.assert_called_once() -# ── _normalize_provider: extra provider map entries ─────────────────────────── - - -def test_normalize_provider_vertex_ai_maps_to_google(): - kwargs = {"custom_llm_provider": "vertex_ai"} - assert _normalize_provider("gemini-2.5-pro", kwargs) == "google" - - -def test_normalize_provider_vertex_ai_anthropic_maps_to_anthropic(): - kwargs = {"custom_llm_provider": "vertex_ai_anthropic"} - assert _normalize_provider("claude-3-5-sonnet", kwargs) == "anthropic" - - -def test_normalize_provider_azure_maps_to_azure(): - kwargs = {"litellm_params": {"custom_llm_provider": "azure"}} - assert _normalize_provider("gpt-4o", kwargs) == "azure" - - -def test_normalize_provider_bedrock_maps_to_aws(): - kwargs = {"custom_llm_provider": "bedrock"} - assert _normalize_provider("claude-3-haiku", kwargs) == "aws" - - -def test_normalize_provider_bedrock_converse_maps_to_aws(): - kwargs = {"custom_llm_provider": "bedrock_converse"} - assert _normalize_provider("claude-3-haiku", kwargs) == "aws" - - -def test_normalize_provider_command_model_pattern(): - """command-* model names should map to cohere via regex.""" - assert _normalize_provider("command-r-plus", {}) == "cohere" - assert _normalize_provider("command-r", {}) == "cohere" - - -def test_normalize_provider_mixtral_model_pattern(): - """mixtral-* model names should map to mistral via regex.""" - assert _normalize_provider("mixtral-8x7b-instruct", {}) == "mistral" - - -def test_normalize_provider_deepseek_from_param(): - kwargs = {"custom_llm_provider": "deepseek"} - assert _normalize_provider("deepseek-v3", kwargs) == "deepseek" - - -def test_normalize_provider_openrouter_from_param(): - kwargs = {"litellm_params": {"custom_llm_provider": "openrouter"}} - assert _normalize_provider("meta-llama/llama-3.3-70b-instruct", kwargs) == "openrouter" - - -def test_normalize_provider_fireworks_ai_from_param(): - kwargs = {"custom_llm_provider": "fireworks_ai"} - assert _normalize_provider("llama-v3-70b-instruct", kwargs) == "fireworks" - - -def test_normalize_provider_cerebras_from_param(): - kwargs = {"custom_llm_provider": "cerebras"} - assert _normalize_provider("llama3.1-8b", kwargs) == "cerebras" - - -def test_normalize_provider_xai_from_param(): - kwargs = {"custom_llm_provider": "xai"} - assert _normalize_provider("grok-3-mini", kwargs) == "xai" - - -# ── _report: edge cases ─────────────────────────────────────────────────────── - - -def test_report_no_exception_omits_error_fields(): - """When exception is None, payload must not include error_code or error_type.""" +@pytest.mark.asyncio +async def test_async_failure_delegates(): logger = TickerrLogger() - captured = {} + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) - def fake_fire(payload): - captured.update(payload) + with patch.object(logger, "_report") as mock: + await logger.async_log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end) + mock.assert_called_once() + + +# -- Success sampling ---------------------------------------------------------- + + +def test_success_not_reported_at_rate_zero(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=100) + + with patch.object(logger, "_report") as mock: + logger.log_success_event({"model": "gpt-4o", "exception": None}, None, start, end) + mock.assert_not_called() + + +def test_success_reported_when_sampled(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}): + logger = TickerrLogger() start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=200) - kwargs = {"model": "gemini-2.5-flash", "exception": None} - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger.log_success_event({"model": "gpt-4o", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, None, start, end) - assert "error_code" not in captured - assert "error_type" not in captured - assert captured["provider"] == "google" + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["event_type"] == "success" + assert payload["provider"] == "openai" -def test_report_includes_client_tier_and_region_in_payload(): - """client_tier and region must appear in the payload when set via env.""" - with patch.dict(os.environ, {"TICKERR_CLIENT_TIER": "enterprise", "TICKERR_REGION": "eu-west-1"}): +@pytest.mark.asyncio +async def test_async_success_reported_when_sampled(): + with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}): logger = TickerrLogger() - captured = {} + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=150) - def fake_fire(payload): - captured.update(payload) + with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + await logger.async_log_success_event({"model": "gpt-4o-mini", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, None, start, end) + payload = json.loads(mock_urlopen.call_args[0][0].data) + assert payload["event_type"] == "success" + + +# -- Network failure does not crash -------------------------------------------- + + +def test_silent_on_network_error(): + logger = TickerrLogger() start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - kwargs = {"model": "gpt-4o", "exception": None} - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert captured["client_tier"] == "enterprise" - assert captured["region"] == "eu-west-1" - - -def test_report_empty_model_sets_model_none(): - """An empty model string must result in model=None in the payload.""" - logger = TickerrLogger() - captured = {} - - def fake_fire(payload): - captured.update(payload) - - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=50) - kwargs = {"model": "", "exception": None} - - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert captured["model"] is None - - -def test_report_timeout_error_type(): - """408 and 524 both map to error_type='timeout'.""" - logger = TickerrLogger() - - for code in (408, 524): - captured = {} - - def fake_fire(payload, _c=code): - captured.update(payload) - - exc = MagicMock() - exc.status_code = code - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=5000) - kwargs = {"model": "claude-haiku-4-5", "exception": exc} - - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert captured["error_type"] == "timeout", f"Expected timeout for {code}" - assert captured["error_code"] == code - - -def test_report_auth_error_type(): - """401 and 403 both map to error_type='auth'.""" - logger = TickerrLogger() - - for code in (401, 403): - captured = {} - - def fake_fire(payload): - captured.update(payload) - - exc = MagicMock() - exc.status_code = code - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=200) - kwargs = {"model": "gpt-4o", "exception": exc} - - with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): - logger._report(kwargs, start, end) - - assert captured["error_type"] == "auth", f"Expected auth for {code}" - - -# ── _fire_and_forget: normal success path ───────────────────────────────────── - - -def test_fire_and_forget_success_path_releases_semaphore(): - """On a successful HTTP call, the semaphore slot must be released.""" - mock_response = MagicMock() - mock_response.__enter__ = MagicMock(return_value=mock_response) - mock_response.__exit__ = MagicMock(return_value=False) - - with patch("threading.Thread") as mock_thread_cls: - mock_thread = MagicMock() - mock_thread_cls.return_value = mock_thread - - def run_target_inline(*args, **kwargs): - # Extract and call the real target so _send runs synchronously - target = mock_thread_cls.call_args[1].get("target") or mock_thread_cls.call_args[0][0] - target() - - mock_thread.start.side_effect = run_target_inline - - with patch("urllib.request.urlopen", return_value=mock_response): - _fire_and_forget({"provider": "openai", "model": "gpt-4o-mini"}) - - # Semaphore must be acquirable — proves the finally block released the slot - acquired = _inflight.acquire(blocking=False) - assert acquired, "semaphore slot was not released after successful send" - _inflight.release() - - -# ── _latency_ms edge cases ──────────────────────────────────────────────────── - - -def test_latency_ms_zero(): - assert _latency_ms(1000.0, 1000.0) == 0 - - -def test_latency_ms_large_value(): - start = datetime(2024, 1, 1, 0, 0, 0) - end = start + timedelta(seconds=30) - assert _latency_ms(start, end) == 30_000 - - -# ── _extract_status_code edge cases ────────────────────────────────────────── - - -def test_extract_status_code_non_digit_string(): - """A non-numeric string status_code must return None.""" - exc = MagicMock() - exc.status_code = "N/A" - assert _extract_status_code(exc) is None - - -def test_extract_status_code_529(): - exc = MagicMock() - exc.status_code = 529 - assert _extract_status_code(exc) == 529 + with patch("litellm.integrations.tickerr.urllib.request.urlopen", side_effect=OSError("refused")): + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report({"model": "gpt-4o", "exception": None}, start, end) + # No exception raised — test passes From 860856c01a3c9463e88620a9283a556273b1440c Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Fri, 29 May 2026 01:35:05 +0530 Subject: [PATCH 11/20] style: apply Black formatting to tickerr.py --- litellm/integrations/tickerr.py | 69 ++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index a77e3228f14..ddaa8ff58f1 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -44,29 +44,63 @@ class TickerrLogger(CustomLogger): def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.disabled: bool = os.environ.get("TICKERR_DISABLED", "").lower() in { - "1", "true", "yes", + "1", + "true", + "yes", } self.region: Optional[str] = os.environ.get("TICKERR_REGION") try: - self.sample_rate: float = min(1.0, max(0.0, float(os.environ.get("TICKERR_SAMPLE_RATE", "0")))) + self.sample_rate: float = min( + 1.0, max(0.0, float(os.environ.get("TICKERR_SAMPLE_RATE", "0"))) + ) except (ValueError, TypeError): self.sample_rate = 0.0 - def log_failure_event(self, kwargs: Dict[str, Any], response_obj: Any, start_time: Union[datetime, float], end_time: Union[datetime, float]) -> None: + def log_failure_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: Union[datetime, float], + end_time: Union[datetime, float], + ) -> None: self._report(kwargs, start_time, end_time) - async def async_log_failure_event(self, kwargs: Dict[str, Any], response_obj: Any, start_time: Union[datetime, float], end_time: Union[datetime, float]) -> None: + async def async_log_failure_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: Union[datetime, float], + end_time: Union[datetime, float], + ) -> None: self._report(kwargs, start_time, end_time) - def log_success_event(self, kwargs: Dict[str, Any], response_obj: Any, start_time: Union[datetime, float], end_time: Union[datetime, float]) -> None: + def log_success_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: Union[datetime, float], + end_time: Union[datetime, float], + ) -> None: if self.sample_rate > 0 and random.random() < self.sample_rate: self._report(kwargs, start_time, end_time, is_success=True) - async def async_log_success_event(self, kwargs: Dict[str, Any], response_obj: Any, start_time: Union[datetime, float], end_time: Union[datetime, float]) -> None: + async def async_log_success_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: Union[datetime, float], + end_time: Union[datetime, float], + ) -> None: if self.sample_rate > 0 and random.random() < self.sample_rate: self._report(kwargs, start_time, end_time, is_success=True) - def _report(self, kwargs: Dict[str, Any], start_time: Union[datetime, float], end_time: Union[datetime, float], is_success: bool = False) -> None: + def _report( + self, + kwargs: Dict[str, Any], + start_time: Union[datetime, float], + end_time: Union[datetime, float], + is_success: bool = False, + ) -> None: if self.disabled: return @@ -77,14 +111,19 @@ class TickerrLogger(CustomLogger): else: latency = round((float(end_time) - float(start_time)) * 1000) - payload = {k: v for k, v in { - "provider": kwargs.get("litellm_params", {}).get("custom_llm_provider") or kwargs.get("custom_llm_provider"), - "model": model or None, - "latency_ms": latency, - "event_type": "success" if is_success else "failure", - "status_code": getattr(kwargs.get("exception"), "status_code", None), - "region": self.region, - }.items() if v is not None} + payload = { + k: v + for k, v in { + "provider": kwargs.get("litellm_params", {}).get("custom_llm_provider") + or kwargs.get("custom_llm_provider"), + "model": model or None, + "latency_ms": latency, + "event_type": "success" if is_success else "failure", + "status_code": getattr(kwargs.get("exception"), "status_code", None), + "region": self.region, + }.items() + if v is not None + } def _send() -> None: try: From e74fd23d5674d720379fdfad9ca1231dac15dae6 Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Fri, 29 May 2026 01:39:55 +0530 Subject: [PATCH 12/20] fix: add type: ignore for mypy float() arg-type --- litellm/integrations/tickerr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index ddaa8ff58f1..40c2437a4ca 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -109,7 +109,7 @@ class TickerrLogger(CustomLogger): if isinstance(start_time, datetime) and isinstance(end_time, datetime): latency = round((end_time - start_time).total_seconds() * 1000) else: - latency = round((float(end_time) - float(start_time)) * 1000) + latency = round((float(end_time) - float(start_time)) * 1000) # type: ignore[arg-type] payload = { k: v From d91cae477b6c62e7a63a5ece861963050a74826d Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Thu, 16 Jul 2026 17:46:44 +0530 Subject: [PATCH 13/20] fix: use shared httpx transport and remove unused TICKERR_CLIENT_TIER - Replace raw urllib.request with litellm's _get_httpx_client() to follow the repo's shared HTTP layer policy - Remove TICKERR_CLIENT_TIER from callback_configs.json since the code never reads it and we don't need to collect tier data --- litellm/integrations/callback_configs.json | 6 ------ litellm/integrations/tickerr.py | 16 ++++++---------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index 61d1aa5c00e..71fd7656512 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -5,12 +5,6 @@ "logo": "tickerr.png", "supports_key_team_logging": false, "dynamic_params": { - "TICKERR_CLIENT_TIER": { - "type": "text", - "ui_name": "Client Tier", - "description": "Optional. Your LLM plan tier: free, pro, team, enterprise, or api_pay_as_you_go. Used to correlate signals by tier.", - "required": false - }, "TICKERR_REGION": { "type": "text", "ui_name": "Region", diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index 40c2437a4ca..27f7227a00e 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -8,20 +8,18 @@ Usage: litellm.callbacks = ["tickerr"] No API key. No account. Failure-only by default. Success sampling is opt-in. -Zero dependencies beyond stdlib. """ from __future__ import annotations -import json import os import random import threading -import urllib.request from datetime import datetime from typing import Any, Dict, Optional, Union from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.custom_httpx.http_handler import _get_httpx_client _REPORT_URL = "https://tickerr.ai/api/v1/report" _UA = "litellm-tickerr/1.0" @@ -127,13 +125,11 @@ class TickerrLogger(CustomLogger): def _send() -> None: try: - urllib.request.urlopen( - urllib.request.Request( - _REPORT_URL, - data=json.dumps(payload).encode(), - headers={"Content-Type": "application/json", "User-Agent": _UA}, - method="POST", - ), + client = _get_httpx_client() + client.post( + _REPORT_URL, + json=payload, + headers={"User-Agent": _UA}, timeout=2, ) except Exception: From 557c87f4ce0a134fcb8f23fd6787d582d4f7695c Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Thu, 16 Jul 2026 18:02:12 +0530 Subject: [PATCH 14/20] fix: update tests to mock httpx client instead of urllib Tests were still patching urllib.request.urlopen which no longer exists in tickerr.py after switching to _get_httpx_client(). All payload- asserting tests now mock _get_httpx_client and assert against client.post() kwargs. --- .../test_tickerr_callback.py | 94 ++++++++----------- .../integrations/test_tickerr_callback.py | 94 ++++++++----------- 2 files changed, 80 insertions(+), 108 deletions(-) diff --git a/tests/logging_callback_tests/test_tickerr_callback.py b/tests/logging_callback_tests/test_tickerr_callback.py index 7e5ccc49ff1..e5e679ccefd 100644 --- a/tests/logging_callback_tests/test_tickerr_callback.py +++ b/tests/logging_callback_tests/test_tickerr_callback.py @@ -2,7 +2,6 @@ Unit tests for the Tickerr LiteLLM callback. """ -import json import os from datetime import datetime, timedelta from unittest.mock import MagicMock, patch @@ -65,6 +64,16 @@ def test_disabled_skips_report(): # -- Payload ------------------------------------------------------------------- +def _run_report(logger, kwargs, start, end, is_success=False): + """Helper: run _report synchronously and return the payload sent via httpx.""" + mock_client = MagicMock() + with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report(kwargs, start, end, is_success=is_success) + return mock_client.post.call_args + + def test_failure_payload(): logger = TickerrLogger() @@ -78,13 +87,8 @@ def test_failure_payload(): "litellm_params": {"custom_llm_provider": "anthropic"}, } - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report(kwargs, start, end) - - sent = mock_urlopen.call_args[0][0] - payload = json.loads(sent.data) + call = _run_report(logger, kwargs, start, end) + payload = call.kwargs["json"] assert payload["provider"] == "anthropic" assert payload["model"] == "claude-haiku-4-5" assert payload["status_code"] == 429 @@ -97,12 +101,8 @@ def test_model_passed_as_is(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "openai/gpt-4o-mini", "exception": None}, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "openai/gpt-4o-mini", "exception": None}, start, end) + payload = call.kwargs["json"] assert payload["model"] == "openai/gpt-4o-mini" @@ -111,12 +111,8 @@ def test_no_exception_omits_status_code(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=200) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None}, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) + payload = call.kwargs["json"] assert "status_code" not in payload @@ -127,24 +123,16 @@ def test_region_included(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None}, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) + payload = call.kwargs["json"] assert payload["region"] == "eu-west-1" def test_latency_from_floats(): logger = TickerrLogger() - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None}, 1000.0, 1001.5) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, 1000.0, 1001.5) + payload = call.kwargs["json"] assert payload["latency_ms"] == 1500 @@ -153,12 +141,8 @@ def test_provider_from_top_level_kwarg(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None, "custom_llm_provider": "openai"}, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "gpt-4o", "exception": None, "custom_llm_provider": "openai"}, start, end) + payload = call.kwargs["json"] assert payload["provider"] == "openai" @@ -167,12 +151,8 @@ def test_no_provider_omits_field(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None}, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) + payload = call.kwargs["json"] assert "provider" not in payload @@ -220,12 +200,12 @@ def test_success_reported_when_sampled(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=200) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger.log_success_event({"model": "gpt-4o", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, None, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report( + logger, + {"model": "gpt-4o", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, + start, end, is_success=True, + ) + payload = call.kwargs["json"] assert payload["event_type"] == "success" assert payload["provider"] == "openai" @@ -238,12 +218,16 @@ async def test_async_success_reported_when_sampled(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=150) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + mock_client = MagicMock() + with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - await logger.async_log_success_event({"model": "gpt-4o-mini", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, None, start, end) + await logger.async_log_success_event( + {"model": "gpt-4o-mini", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, + None, start, end, + ) - payload = json.loads(mock_urlopen.call_args[0][0].data) + payload = mock_client.post.call_args.kwargs["json"] assert payload["event_type"] == "success" @@ -255,7 +239,9 @@ def test_silent_on_network_error(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - with patch("litellm.integrations.tickerr.urllib.request.urlopen", side_effect=OSError("refused")): + mock_client = MagicMock() + mock_client.post.side_effect = OSError("refused") + with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() logger._report({"model": "gpt-4o", "exception": None}, start, end) diff --git a/tests/test_litellm/integrations/test_tickerr_callback.py b/tests/test_litellm/integrations/test_tickerr_callback.py index 7e5ccc49ff1..e5e679ccefd 100644 --- a/tests/test_litellm/integrations/test_tickerr_callback.py +++ b/tests/test_litellm/integrations/test_tickerr_callback.py @@ -2,7 +2,6 @@ Unit tests for the Tickerr LiteLLM callback. """ -import json import os from datetime import datetime, timedelta from unittest.mock import MagicMock, patch @@ -65,6 +64,16 @@ def test_disabled_skips_report(): # -- Payload ------------------------------------------------------------------- +def _run_report(logger, kwargs, start, end, is_success=False): + """Helper: run _report synchronously and return the payload sent via httpx.""" + mock_client = MagicMock() + with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): + with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: + mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() + logger._report(kwargs, start, end, is_success=is_success) + return mock_client.post.call_args + + def test_failure_payload(): logger = TickerrLogger() @@ -78,13 +87,8 @@ def test_failure_payload(): "litellm_params": {"custom_llm_provider": "anthropic"}, } - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report(kwargs, start, end) - - sent = mock_urlopen.call_args[0][0] - payload = json.loads(sent.data) + call = _run_report(logger, kwargs, start, end) + payload = call.kwargs["json"] assert payload["provider"] == "anthropic" assert payload["model"] == "claude-haiku-4-5" assert payload["status_code"] == 429 @@ -97,12 +101,8 @@ def test_model_passed_as_is(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "openai/gpt-4o-mini", "exception": None}, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "openai/gpt-4o-mini", "exception": None}, start, end) + payload = call.kwargs["json"] assert payload["model"] == "openai/gpt-4o-mini" @@ -111,12 +111,8 @@ def test_no_exception_omits_status_code(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=200) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None}, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) + payload = call.kwargs["json"] assert "status_code" not in payload @@ -127,24 +123,16 @@ def test_region_included(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None}, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) + payload = call.kwargs["json"] assert payload["region"] == "eu-west-1" def test_latency_from_floats(): logger = TickerrLogger() - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None}, 1000.0, 1001.5) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, 1000.0, 1001.5) + payload = call.kwargs["json"] assert payload["latency_ms"] == 1500 @@ -153,12 +141,8 @@ def test_provider_from_top_level_kwarg(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None, "custom_llm_provider": "openai"}, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "gpt-4o", "exception": None, "custom_llm_provider": "openai"}, start, end) + payload = call.kwargs["json"] assert payload["provider"] == "openai" @@ -167,12 +151,8 @@ def test_no_provider_omits_field(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None}, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) + payload = call.kwargs["json"] assert "provider" not in payload @@ -220,12 +200,12 @@ def test_success_reported_when_sampled(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=200) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger.log_success_event({"model": "gpt-4o", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, None, start, end) - - payload = json.loads(mock_urlopen.call_args[0][0].data) + call = _run_report( + logger, + {"model": "gpt-4o", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, + start, end, is_success=True, + ) + payload = call.kwargs["json"] assert payload["event_type"] == "success" assert payload["provider"] == "openai" @@ -238,12 +218,16 @@ async def test_async_success_reported_when_sampled(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=150) - with patch("litellm.integrations.tickerr.urllib.request.urlopen") as mock_urlopen: + mock_client = MagicMock() + with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - await logger.async_log_success_event({"model": "gpt-4o-mini", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, None, start, end) + await logger.async_log_success_event( + {"model": "gpt-4o-mini", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, + None, start, end, + ) - payload = json.loads(mock_urlopen.call_args[0][0].data) + payload = mock_client.post.call_args.kwargs["json"] assert payload["event_type"] == "success" @@ -255,7 +239,9 @@ def test_silent_on_network_error(): start = datetime(2024, 1, 1) end = start + timedelta(milliseconds=100) - with patch("litellm.integrations.tickerr.urllib.request.urlopen", side_effect=OSError("refused")): + mock_client = MagicMock() + mock_client.post.side_effect = OSError("refused") + with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() logger._report({"model": "gpt-4o", "exception": None}, start, end) From 9febf728dfbd03e1ff6d5e256ec02e6056ecb205 Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Thu, 16 Jul 2026 21:33:55 +0530 Subject: [PATCH 15/20] fix: bound inflight report threads with semaphore Add _MAX_INFLIGHT semaphore (cap=10) so thread spawning is bounded during mass failures. If 10 reports are already in flight, new ones are silently dropped instead of exhausting OS thread limits. --- litellm/integrations/tickerr.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index 27f7227a00e..c3f0114ed76 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -23,6 +23,7 @@ from litellm.llms.custom_httpx.http_handler import _get_httpx_client _REPORT_URL = "https://tickerr.ai/api/v1/report" _UA = "litellm-tickerr/1.0" +_MAX_INFLIGHT = threading.Semaphore(10) class TickerrLogger(CustomLogger): @@ -124,6 +125,8 @@ class TickerrLogger(CustomLogger): } def _send() -> None: + if not _MAX_INFLIGHT.acquire(blocking=False): + return try: client = _get_httpx_client() client.post( @@ -134,5 +137,7 @@ class TickerrLogger(CustomLogger): ) except Exception: pass + finally: + _MAX_INFLIGHT.release() threading.Thread(target=_send, daemon=True).start() From 2844f3d0aa230e692d78f83f486ae834c7fa7b6c Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Thu, 16 Jul 2026 22:02:13 +0530 Subject: [PATCH 16/20] chore: remove docs file and duplicate test - Remove tickerr.md from this repo (docs belong in litellm-docs) - Remove duplicate test file in logging_callback_tests/ (kept the copy in tests/test_litellm/integrations/) --- docs/my-website/docs/observability/tickerr.md | 63 ----- .../test_tickerr_callback.py | 248 ------------------ 2 files changed, 311 deletions(-) delete mode 100644 docs/my-website/docs/observability/tickerr.md delete mode 100644 tests/logging_callback_tests/test_tickerr_callback.py diff --git a/docs/my-website/docs/observability/tickerr.md b/docs/my-website/docs/observability/tickerr.md deleted file mode 100644 index cdc4a212b79..00000000000 --- a/docs/my-website/docs/observability/tickerr.md +++ /dev/null @@ -1,63 +0,0 @@ -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - -# Tickerr - Outage Radar for AI Agents - -[Tickerr](https://tickerr.ai) is a crowd-sourced outage detector for LLM APIs. When your agent hits a 5xx or rate limit, it reports anonymously so every agent can see the issue in real time. - -**No API key. No account. Failure-only by default. Success sampling is opt-in.** - -## Quick Start - - - - -```python -import litellm - -litellm.callbacks = ["tickerr"] - -response = litellm.completion( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "Hello"}] -) -``` - - - - -```yaml -litellm_settings: - callbacks: ["tickerr"] -``` - - - - -## What Gets Reported - -When the tickerr callback is explicitly enabled, anonymous failure metadata is reported. No prompts, responses, API keys, or personal data are sent. - -| Field | Example | -|-------|---------| -| Provider | `anthropic` | -| Model | `claude-haiku-4-5` | -| Status code | `529` | -| Latency (ms) | `1240` | -| Event type | `failure` or `success` | - -## Optional Configuration - -```bash -TICKERR_DISABLED=true # disable all reporting (kill switch) -TICKERR_REGION=us-east-1 # for regional signal breakdown -TICKERR_SAMPLE_RATE=0.01 # report 1% of successes for latency benchmarks (default: 0 = off) -``` - -Failures are reported by default once Tickerr is enabled. Success sampling is opt-in. - -## Links - -- [Live dashboard](https://tickerr.ai) - status for 90+ AI tools -- [Agent reports feed](https://tickerr.ai/agent-reports) - real-time signal -- [Opt out](https://tickerr.ai/mcp/opt-out) diff --git a/tests/logging_callback_tests/test_tickerr_callback.py b/tests/logging_callback_tests/test_tickerr_callback.py deleted file mode 100644 index e5e679ccefd..00000000000 --- a/tests/logging_callback_tests/test_tickerr_callback.py +++ /dev/null @@ -1,248 +0,0 @@ -""" -Unit tests for the Tickerr LiteLLM callback. -""" - -import os -from datetime import datetime, timedelta -from unittest.mock import MagicMock, patch - -import pytest - -from litellm.integrations.tickerr import TickerrLogger - - -# -- Config -------------------------------------------------------------------- - - -def test_default_init(): - logger = TickerrLogger() - assert logger.disabled is False - assert logger.sample_rate == 0.0 - assert logger.region is None - - -def test_reads_env_vars(): - with patch.dict(os.environ, {"TICKERR_REGION": "us-east-1", "TICKERR_SAMPLE_RATE": "0.1"}): - logger = TickerrLogger() - assert logger.region == "us-east-1" - assert logger.sample_rate == 0.1 - - -def test_disabled_flag(): - with patch.dict(os.environ, {"TICKERR_DISABLED": "true"}): - logger = TickerrLogger() - assert logger.disabled is True - - -def test_invalid_sample_rate_does_not_crash(): - with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "notanumber"}): - logger = TickerrLogger() - assert logger.sample_rate == 0.0 - - -def test_sample_rate_clamped_to_one(): - with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "5.0"}): - logger = TickerrLogger() - assert logger.sample_rate == 1.0 - - -# -- Disabled ------------------------------------------------------------------ - - -def test_disabled_skips_report(): - with patch.dict(os.environ, {"TICKERR_DISABLED": "1"}): - logger = TickerrLogger() - - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - logger._report({"model": "gpt-4o-mini", "exception": None}, start, end) - mock_thread.assert_not_called() - - -# -- Payload ------------------------------------------------------------------- - - -def _run_report(logger, kwargs, start, end, is_success=False): - """Helper: run _report synchronously and return the payload sent via httpx.""" - mock_client = MagicMock() - with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report(kwargs, start, end, is_success=is_success) - return mock_client.post.call_args - - -def test_failure_payload(): - logger = TickerrLogger() - - exc = MagicMock() - exc.status_code = 429 - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=500) - kwargs = { - "model": "claude-haiku-4-5", - "exception": exc, - "litellm_params": {"custom_llm_provider": "anthropic"}, - } - - call = _run_report(logger, kwargs, start, end) - payload = call.kwargs["json"] - assert payload["provider"] == "anthropic" - assert payload["model"] == "claude-haiku-4-5" - assert payload["status_code"] == 429 - assert payload["event_type"] == "failure" - assert payload["latency_ms"] == 500 - - -def test_model_passed_as_is(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - call = _run_report(logger, {"model": "openai/gpt-4o-mini", "exception": None}, start, end) - payload = call.kwargs["json"] - assert payload["model"] == "openai/gpt-4o-mini" - - -def test_no_exception_omits_status_code(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=200) - - call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) - payload = call.kwargs["json"] - assert "status_code" not in payload - - -def test_region_included(): - with patch.dict(os.environ, {"TICKERR_REGION": "eu-west-1"}): - logger = TickerrLogger() - - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) - payload = call.kwargs["json"] - assert payload["region"] == "eu-west-1" - - -def test_latency_from_floats(): - logger = TickerrLogger() - - call = _run_report(logger, {"model": "gpt-4o", "exception": None}, 1000.0, 1001.5) - payload = call.kwargs["json"] - assert payload["latency_ms"] == 1500 - - -def test_provider_from_top_level_kwarg(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - call = _run_report(logger, {"model": "gpt-4o", "exception": None, "custom_llm_provider": "openai"}, start, end) - payload = call.kwargs["json"] - assert payload["provider"] == "openai" - - -def test_no_provider_omits_field(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) - payload = call.kwargs["json"] - assert "provider" not in payload - - -# -- Hooks --------------------------------------------------------------------- - - -def test_sync_failure_delegates(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=300) - - with patch.object(logger, "_report") as mock: - logger.log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end) - mock.assert_called_once() - - -@pytest.mark.asyncio -async def test_async_failure_delegates(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=200) - - with patch.object(logger, "_report") as mock: - await logger.async_log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end) - mock.assert_called_once() - - -# -- Success sampling ---------------------------------------------------------- - - -def test_success_not_reported_at_rate_zero(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - with patch.object(logger, "_report") as mock: - logger.log_success_event({"model": "gpt-4o", "exception": None}, None, start, end) - mock.assert_not_called() - - -def test_success_reported_when_sampled(): - with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}): - logger = TickerrLogger() - - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=200) - - call = _run_report( - logger, - {"model": "gpt-4o", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, - start, end, is_success=True, - ) - payload = call.kwargs["json"] - assert payload["event_type"] == "success" - assert payload["provider"] == "openai" - - -@pytest.mark.asyncio -async def test_async_success_reported_when_sampled(): - with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}): - logger = TickerrLogger() - - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=150) - - mock_client = MagicMock() - with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - await logger.async_log_success_event( - {"model": "gpt-4o-mini", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, - None, start, end, - ) - - payload = mock_client.post.call_args.kwargs["json"] - assert payload["event_type"] == "success" - - -# -- Network failure does not crash -------------------------------------------- - - -def test_silent_on_network_error(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - mock_client = MagicMock() - mock_client.post.side_effect = OSError("refused") - with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None}, start, end) - # No exception raised — test passes From da650f5c6c7acdfcc69bc6e1742d401fd76d3094 Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Thu, 16 Jul 2026 22:15:49 +0530 Subject: [PATCH 17/20] chore: add tickerr logo to UI assets Add tickerr.png so the LiteLLM proxy dashboard renders the logo on the callback management page. --- .../public/assets/logos/tickerr.png | Bin 0 -> 854 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 ui/litellm-dashboard/public/assets/logos/tickerr.png diff --git a/ui/litellm-dashboard/public/assets/logos/tickerr.png b/ui/litellm-dashboard/public/assets/logos/tickerr.png new file mode 100644 index 0000000000000000000000000000000000000000..c33d829d4d818ceab58070c131ba6929142395f8 GIT binary patch literal 854 zcmeAS@N?(olHy`uVBq!ia0vp^TR@nD2}o{QKQWbofmznm#WAE}&fB}*dAB`894_WG zZtkhRwENZFwD?Sg$qL`5s&14Bo5*hA?5MG?JaB^jhW)ws|F`*`xUSCEj}f1T;P#{ zuFUafqc4U>Ru$Gnwpn}=X3sonF;CN3{`Xq(Z6 z->^KQ$Stw;M5*vh!@j?prH%e>{2gsLk@>O3_jhv@*Y7u&>|AZ7(kD6Ze}_fdyz_#c zYXpxUFg_!2+_6?i;J8iy-*3@>=U$XITz&Jg#J@Ai{f413$;VCqBs{X%+0*D8t=v~R z?_2M7TkSquXX_UR#&z=_zlhwwarR~TCnAq-eEpTZJ6UFH{`F(>dUD6Vol0HMW8d-k z;^`xW{}lPYKDjpall?gOQN=6P?f-lX*p+r$Y-@iV?k3o7ap~^k z4}o7AqmM7_IsQ)Q__k;3f6ViiIpy(4z QT^JZVUHx3vIVCg!03WY(BLDyZ literal 0 HcmV?d00001 From d6f9d6c9107bd83c1a5ef52641f3a29782b14646 Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Thu, 6 Aug 2026 13:25:33 +0530 Subject: [PATCH 18/20] style: fix ruff format for tickerr integration --- litellm/integrations/tickerr.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index c3f0114ed76..d738031e80e 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -49,9 +49,7 @@ class TickerrLogger(CustomLogger): } self.region: Optional[str] = os.environ.get("TICKERR_REGION") try: - self.sample_rate: float = min( - 1.0, max(0.0, float(os.environ.get("TICKERR_SAMPLE_RATE", "0"))) - ) + self.sample_rate: float = min(1.0, max(0.0, float(os.environ.get("TICKERR_SAMPLE_RATE", "0")))) except (ValueError, TypeError): self.sample_rate = 0.0 From fd77aefcfbd381205814481e783750c4632b101b Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Thu, 6 Aug 2026 13:31:48 +0530 Subject: [PATCH 19/20] fix: resolve strict-rule lint violations in tickerr integration Replace typing.Any/Dict/Optional/Union with modern syntax (dict, |), narrow exception handler from Exception to (OSError, ValueError), validate untyped parent kwargs with isinstance checks, and fix import sorting for noqa inline suppression --- litellm/integrations/tickerr.py | 102 ++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 43 deletions(-) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index d738031e80e..adbc0af46fd 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -16,10 +16,11 @@ import os import random import threading from datetime import datetime -from typing import Any, Dict, Optional, Union from litellm.integrations.custom_logger import CustomLogger -from litellm.llms.custom_httpx.http_handler import _get_httpx_client +from litellm.llms.custom_httpx.http_handler import ( + _get_httpx_client, # noqa: TID251 # internal API needed for non-blocking HTTP +) _REPORT_URL = "https://tickerr.ai/api/v1/report" _UA = "litellm-tickerr/1.0" @@ -40,87 +41,102 @@ class TickerrLogger(CustomLogger): TICKERR_SAMPLE_RATE - fraction of successes to report (0.0-1.0, default 0 = off) """ - def __init__(self, **kwargs: Any) -> None: + def __init__(self, **kwargs: object) -> None: # pyright: ignore[reportAny] # parent is untyped super().__init__(**kwargs) self.disabled: bool = os.environ.get("TICKERR_DISABLED", "").lower() in { "1", "true", "yes", } - self.region: Optional[str] = os.environ.get("TICKERR_REGION") + self.region: str | None = os.environ.get("TICKERR_REGION") try: self.sample_rate: float = min(1.0, max(0.0, float(os.environ.get("TICKERR_SAMPLE_RATE", "0")))) except (ValueError, TypeError): self.sample_rate = 0.0 - def log_failure_event( + def log_failure_event( # pyright: ignore[reportAny] # parent is untyped self, - kwargs: Dict[str, Any], - response_obj: Any, - start_time: Union[datetime, float], - end_time: Union[datetime, float], + kwargs: dict[str, object], + response_obj: object, + start_time: datetime | float, + end_time: datetime | float, ) -> None: self._report(kwargs, start_time, end_time) - async def async_log_failure_event( + async def async_log_failure_event( # pyright: ignore[reportAny] # parent is untyped self, - kwargs: Dict[str, Any], - response_obj: Any, - start_time: Union[datetime, float], - end_time: Union[datetime, float], + kwargs: dict[str, object], + response_obj: object, + start_time: datetime | float, + end_time: datetime | float, ) -> None: self._report(kwargs, start_time, end_time) - def log_success_event( + def log_success_event( # pyright: ignore[reportAny] # parent is untyped self, - kwargs: Dict[str, Any], - response_obj: Any, - start_time: Union[datetime, float], - end_time: Union[datetime, float], + kwargs: dict[str, object], + response_obj: object, + start_time: datetime | float, + end_time: datetime | float, ) -> None: if self.sample_rate > 0 and random.random() < self.sample_rate: self._report(kwargs, start_time, end_time, is_success=True) - async def async_log_success_event( + async def async_log_success_event( # pyright: ignore[reportAny] # parent is untyped self, - kwargs: Dict[str, Any], - response_obj: Any, - start_time: Union[datetime, float], - end_time: Union[datetime, float], + kwargs: dict[str, object], + response_obj: object, + start_time: datetime | float, + end_time: datetime | float, ) -> None: if self.sample_rate > 0 and random.random() < self.sample_rate: self._report(kwargs, start_time, end_time, is_success=True) def _report( self, - kwargs: Dict[str, Any], - start_time: Union[datetime, float], - end_time: Union[datetime, float], + kwargs: dict[str, object], + start_time: datetime | float, + end_time: datetime | float, is_success: bool = False, ) -> None: if self.disabled: return - model: str = kwargs.get("model", "") or "" + model: str = str(kwargs.get("model", "") or "") if isinstance(start_time, datetime) and isinstance(end_time, datetime): latency = round((end_time - start_time).total_seconds() * 1000) else: - latency = round((float(end_time) - float(start_time)) * 1000) # type: ignore[arg-type] + latency = round((float(end_time) - float(start_time)) * 1000) - payload = { - k: v - for k, v in { - "provider": kwargs.get("litellm_params", {}).get("custom_llm_provider") - or kwargs.get("custom_llm_provider"), - "model": model or None, - "latency_ms": latency, - "event_type": "success" if is_success else "failure", - "status_code": getattr(kwargs.get("exception"), "status_code", None), - "region": self.region, - }.items() - if v is not None - } + litellm_params = kwargs.get("litellm_params") + provider: str | None = None + if isinstance(litellm_params, dict): + raw = litellm_params.get("custom_llm_provider") + if isinstance(raw, str): + provider = raw + if provider is None: + raw_fallback = kwargs.get("custom_llm_provider") + if isinstance(raw_fallback, str): + provider = raw_fallback + + exception = kwargs.get("exception") + status_code: int | None = None + raw_code = getattr(exception, "status_code", None) + if isinstance(raw_code, int): + status_code = raw_code + + payload: dict[str, str | int] = {} + if provider is not None: + payload["provider"] = provider + if model: + payload["model"] = model + payload["latency_ms"] = latency + payload["event_type"] = "success" if is_success else "failure" + if status_code is not None: + payload["status_code"] = status_code + if self.region is not None: + payload["region"] = self.region def _send() -> None: if not _MAX_INFLIGHT.acquire(blocking=False): @@ -133,7 +149,7 @@ class TickerrLogger(CustomLogger): headers={"User-Agent": _UA}, timeout=2, ) - except Exception: + except (OSError, ValueError): # fire-and-forget; network errors are expected pass finally: _MAX_INFLIGHT.release() From 1ed552cebf0ebc77ad891ec233047b9eb49d3e69 Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Thu, 6 Aug 2026 13:50:20 +0530 Subject: [PATCH 20/20] fix: resolve type-discipline LIT001/LIT002 violations in tickerr Use Mapping[str, object] instead of dict for read-only callback params, build payload via immutable tuple-of-pairs then convert once, add mutable-ok/kwargs-ok annotations where mutation is necessary --- litellm/integrations/tickerr.py | 43 +++++++++++++++++---------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/litellm/integrations/tickerr.py b/litellm/integrations/tickerr.py index adbc0af46fd..b51dcd6f6fe 100644 --- a/litellm/integrations/tickerr.py +++ b/litellm/integrations/tickerr.py @@ -15,6 +15,7 @@ from __future__ import annotations import os import random import threading +from collections.abc import Mapping from datetime import datetime from litellm.integrations.custom_logger import CustomLogger @@ -41,13 +42,15 @@ class TickerrLogger(CustomLogger): TICKERR_SAMPLE_RATE - fraction of successes to report (0.0-1.0, default 0 = off) """ - def __init__(self, **kwargs: object) -> None: # pyright: ignore[reportAny] # parent is untyped + def __init__( + self, **kwargs: object + ) -> None: # kwargs-ok: parent signature is untyped # pyright: ignore[reportAny] super().__init__(**kwargs) - self.disabled: bool = os.environ.get("TICKERR_DISABLED", "").lower() in { + self.disabled: bool = os.environ.get("TICKERR_DISABLED", "").lower() in ( "1", "true", "yes", - } + ) self.region: str | None = os.environ.get("TICKERR_REGION") try: self.sample_rate: float = min(1.0, max(0.0, float(os.environ.get("TICKERR_SAMPLE_RATE", "0")))) @@ -56,7 +59,7 @@ class TickerrLogger(CustomLogger): def log_failure_event( # pyright: ignore[reportAny] # parent is untyped self, - kwargs: dict[str, object], + kwargs: Mapping[str, object], response_obj: object, start_time: datetime | float, end_time: datetime | float, @@ -65,7 +68,7 @@ class TickerrLogger(CustomLogger): async def async_log_failure_event( # pyright: ignore[reportAny] # parent is untyped self, - kwargs: dict[str, object], + kwargs: Mapping[str, object], response_obj: object, start_time: datetime | float, end_time: datetime | float, @@ -74,7 +77,7 @@ class TickerrLogger(CustomLogger): def log_success_event( # pyright: ignore[reportAny] # parent is untyped self, - kwargs: dict[str, object], + kwargs: Mapping[str, object], response_obj: object, start_time: datetime | float, end_time: datetime | float, @@ -84,7 +87,7 @@ class TickerrLogger(CustomLogger): async def async_log_success_event( # pyright: ignore[reportAny] # parent is untyped self, - kwargs: dict[str, object], + kwargs: Mapping[str, object], response_obj: object, start_time: datetime | float, end_time: datetime | float, @@ -94,7 +97,7 @@ class TickerrLogger(CustomLogger): def _report( self, - kwargs: dict[str, object], + kwargs: Mapping[str, object], start_time: datetime | float, end_time: datetime | float, is_success: bool = False, @@ -111,7 +114,7 @@ class TickerrLogger(CustomLogger): litellm_params = kwargs.get("litellm_params") provider: str | None = None - if isinstance(litellm_params, dict): + if isinstance(litellm_params, Mapping): raw = litellm_params.get("custom_llm_provider") if isinstance(raw, str): provider = raw @@ -126,17 +129,15 @@ class TickerrLogger(CustomLogger): if isinstance(raw_code, int): status_code = raw_code - payload: dict[str, str | int] = {} - if provider is not None: - payload["provider"] = provider - if model: - payload["model"] = model - payload["latency_ms"] = latency - payload["event_type"] = "success" if is_success else "failure" - if status_code is not None: - payload["status_code"] = status_code - if self.region is not None: - payload["region"] = self.region + pairs: tuple[tuple[str, str | int], ...] = ( + *((("provider", provider),) if provider is not None else ()), + *((("model", model),) if model else ()), + ("latency_ms", latency), + ("event_type", "success" if is_success else "failure"), + *((("status_code", status_code),) if status_code is not None else ()), + *((("region", self.region),) if self.region is not None else ()), + ) + payload = dict(pairs) # mutable-ok: consumed once by httpx.post(json=...) def _send() -> None: if not _MAX_INFLIGHT.acquire(blocking=False): @@ -146,7 +147,7 @@ class TickerrLogger(CustomLogger): client.post( _REPORT_URL, json=payload, - headers={"User-Agent": _UA}, + headers={"User-Agent": _UA}, # mutable-ok: consumed once by httpx timeout=2, ) except (OSError, ValueError): # fire-and-forget; network errors are expected