From 2844f3d0aa230e692d78f83f486ae834c7fa7b6c Mon Sep 17 00:00:00 2001 From: Vikas Sharma Date: Thu, 16 Jul 2026 22:02:13 +0530 Subject: [PATCH] chore: remove docs file and duplicate test - Remove tickerr.md from this repo (docs belong in litellm-docs) - Remove duplicate test file in logging_callback_tests/ (kept the copy in tests/test_litellm/integrations/) --- docs/my-website/docs/observability/tickerr.md | 63 ----- .../test_tickerr_callback.py | 248 ------------------ 2 files changed, 311 deletions(-) delete mode 100644 docs/my-website/docs/observability/tickerr.md delete mode 100644 tests/logging_callback_tests/test_tickerr_callback.py diff --git a/docs/my-website/docs/observability/tickerr.md b/docs/my-website/docs/observability/tickerr.md deleted file mode 100644 index cdc4a212b79..00000000000 --- a/docs/my-website/docs/observability/tickerr.md +++ /dev/null @@ -1,63 +0,0 @@ -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - -# Tickerr - Outage Radar for AI Agents - -[Tickerr](https://tickerr.ai) is a crowd-sourced outage detector for LLM APIs. When your agent hits a 5xx or rate limit, it reports anonymously so every agent can see the issue in real time. - -**No API key. No account. Failure-only by default. Success sampling is opt-in.** - -## Quick Start - - - - -```python -import litellm - -litellm.callbacks = ["tickerr"] - -response = litellm.completion( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "Hello"}] -) -``` - - - - -```yaml -litellm_settings: - callbacks: ["tickerr"] -``` - - - - -## What Gets Reported - -When the tickerr callback is explicitly enabled, anonymous failure metadata is reported. No prompts, responses, API keys, or personal data are sent. - -| Field | Example | -|-------|---------| -| Provider | `anthropic` | -| Model | `claude-haiku-4-5` | -| Status code | `529` | -| Latency (ms) | `1240` | -| Event type | `failure` or `success` | - -## Optional Configuration - -```bash -TICKERR_DISABLED=true # disable all reporting (kill switch) -TICKERR_REGION=us-east-1 # for regional signal breakdown -TICKERR_SAMPLE_RATE=0.01 # report 1% of successes for latency benchmarks (default: 0 = off) -``` - -Failures are reported by default once Tickerr is enabled. Success sampling is opt-in. - -## Links - -- [Live dashboard](https://tickerr.ai) - status for 90+ AI tools -- [Agent reports feed](https://tickerr.ai/agent-reports) - real-time signal -- [Opt out](https://tickerr.ai/mcp/opt-out) diff --git a/tests/logging_callback_tests/test_tickerr_callback.py b/tests/logging_callback_tests/test_tickerr_callback.py deleted file mode 100644 index e5e679ccefd..00000000000 --- a/tests/logging_callback_tests/test_tickerr_callback.py +++ /dev/null @@ -1,248 +0,0 @@ -""" -Unit tests for the Tickerr LiteLLM callback. -""" - -import os -from datetime import datetime, timedelta -from unittest.mock import MagicMock, patch - -import pytest - -from litellm.integrations.tickerr import TickerrLogger - - -# -- Config -------------------------------------------------------------------- - - -def test_default_init(): - logger = TickerrLogger() - assert logger.disabled is False - assert logger.sample_rate == 0.0 - assert logger.region is None - - -def test_reads_env_vars(): - with patch.dict(os.environ, {"TICKERR_REGION": "us-east-1", "TICKERR_SAMPLE_RATE": "0.1"}): - logger = TickerrLogger() - assert logger.region == "us-east-1" - assert logger.sample_rate == 0.1 - - -def test_disabled_flag(): - with patch.dict(os.environ, {"TICKERR_DISABLED": "true"}): - logger = TickerrLogger() - assert logger.disabled is True - - -def test_invalid_sample_rate_does_not_crash(): - with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "notanumber"}): - logger = TickerrLogger() - assert logger.sample_rate == 0.0 - - -def test_sample_rate_clamped_to_one(): - with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "5.0"}): - logger = TickerrLogger() - assert logger.sample_rate == 1.0 - - -# -- Disabled ------------------------------------------------------------------ - - -def test_disabled_skips_report(): - with patch.dict(os.environ, {"TICKERR_DISABLED": "1"}): - logger = TickerrLogger() - - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - logger._report({"model": "gpt-4o-mini", "exception": None}, start, end) - mock_thread.assert_not_called() - - -# -- Payload ------------------------------------------------------------------- - - -def _run_report(logger, kwargs, start, end, is_success=False): - """Helper: run _report synchronously and return the payload sent via httpx.""" - mock_client = MagicMock() - with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report(kwargs, start, end, is_success=is_success) - return mock_client.post.call_args - - -def test_failure_payload(): - logger = TickerrLogger() - - exc = MagicMock() - exc.status_code = 429 - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=500) - kwargs = { - "model": "claude-haiku-4-5", - "exception": exc, - "litellm_params": {"custom_llm_provider": "anthropic"}, - } - - call = _run_report(logger, kwargs, start, end) - payload = call.kwargs["json"] - assert payload["provider"] == "anthropic" - assert payload["model"] == "claude-haiku-4-5" - assert payload["status_code"] == 429 - assert payload["event_type"] == "failure" - assert payload["latency_ms"] == 500 - - -def test_model_passed_as_is(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - call = _run_report(logger, {"model": "openai/gpt-4o-mini", "exception": None}, start, end) - payload = call.kwargs["json"] - assert payload["model"] == "openai/gpt-4o-mini" - - -def test_no_exception_omits_status_code(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=200) - - call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) - payload = call.kwargs["json"] - assert "status_code" not in payload - - -def test_region_included(): - with patch.dict(os.environ, {"TICKERR_REGION": "eu-west-1"}): - logger = TickerrLogger() - - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) - payload = call.kwargs["json"] - assert payload["region"] == "eu-west-1" - - -def test_latency_from_floats(): - logger = TickerrLogger() - - call = _run_report(logger, {"model": "gpt-4o", "exception": None}, 1000.0, 1001.5) - payload = call.kwargs["json"] - assert payload["latency_ms"] == 1500 - - -def test_provider_from_top_level_kwarg(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - call = _run_report(logger, {"model": "gpt-4o", "exception": None, "custom_llm_provider": "openai"}, start, end) - payload = call.kwargs["json"] - assert payload["provider"] == "openai" - - -def test_no_provider_omits_field(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - call = _run_report(logger, {"model": "gpt-4o", "exception": None}, start, end) - payload = call.kwargs["json"] - assert "provider" not in payload - - -# -- Hooks --------------------------------------------------------------------- - - -def test_sync_failure_delegates(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=300) - - with patch.object(logger, "_report") as mock: - logger.log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end) - mock.assert_called_once() - - -@pytest.mark.asyncio -async def test_async_failure_delegates(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=200) - - with patch.object(logger, "_report") as mock: - await logger.async_log_failure_event({"model": "gpt-4o-mini", "exception": None}, None, start, end) - mock.assert_called_once() - - -# -- Success sampling ---------------------------------------------------------- - - -def test_success_not_reported_at_rate_zero(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - with patch.object(logger, "_report") as mock: - logger.log_success_event({"model": "gpt-4o", "exception": None}, None, start, end) - mock.assert_not_called() - - -def test_success_reported_when_sampled(): - with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}): - logger = TickerrLogger() - - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=200) - - call = _run_report( - logger, - {"model": "gpt-4o", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, - start, end, is_success=True, - ) - payload = call.kwargs["json"] - assert payload["event_type"] == "success" - assert payload["provider"] == "openai" - - -@pytest.mark.asyncio -async def test_async_success_reported_when_sampled(): - with patch.dict(os.environ, {"TICKERR_SAMPLE_RATE": "1.0"}): - logger = TickerrLogger() - - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=150) - - mock_client = MagicMock() - with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - await logger.async_log_success_event( - {"model": "gpt-4o-mini", "exception": None, "litellm_params": {"custom_llm_provider": "openai"}}, - None, start, end, - ) - - payload = mock_client.post.call_args.kwargs["json"] - assert payload["event_type"] == "success" - - -# -- Network failure does not crash -------------------------------------------- - - -def test_silent_on_network_error(): - logger = TickerrLogger() - start = datetime(2024, 1, 1) - end = start + timedelta(milliseconds=100) - - mock_client = MagicMock() - mock_client.post.side_effect = OSError("refused") - with patch("litellm.integrations.tickerr._get_httpx_client", return_value=mock_client): - with patch("litellm.integrations.tickerr.threading.Thread") as mock_thread: - mock_thread.return_value.start = lambda: mock_thread.call_args[1]["target"]() - logger._report({"model": "gpt-4o", "exception": None}, start, end) - # No exception raised — test passes