From 72d1a93ea10b0a13931afbddb46e8d32f930cf43 Mon Sep 17 00:00:00 2001 From: imviky-ctrl <272028991+imviky-ctrl@users.noreply.github.com> Date: Tue, 5 May 2026 15:02:51 +0530 Subject: [PATCH] test(tickerr): add unit tests for TickerrLogger callback Covers: provider normalization, status code extraction, latency calculation with both datetime and float types, error type mapping, payload construction, semaphore cap, silent network failure, and async hook wiring. --- .../test_tickerr_callback.py | 266 ++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 tests/logging_callback_tests/test_tickerr_callback.py diff --git a/tests/logging_callback_tests/test_tickerr_callback.py b/tests/logging_callback_tests/test_tickerr_callback.py new file mode 100644 index 00000000000..1444a16c3d5 --- /dev/null +++ b/tests/logging_callback_tests/test_tickerr_callback.py @@ -0,0 +1,266 @@ +""" +Unit tests for the Tickerr LiteLLM callback. + +Tests cover: +- TickerrLogger instantiation and env var config +- Provider normalization from model names and litellm_params +- Status code extraction from exceptions +- Latency calculation for both datetime and float timestamps +- Error type mapping (only known codes, no fallback default) +- Payload construction +- Thread cap (semaphore) under burst conditions +- Fire-and-forget does not block or raise on network failure +""" + +import os +import sys +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from litellm.integrations.tickerr import ( + TickerrLogger, + _ERROR_TYPE_MAP, + _extract_status_code, + _fire_and_forget, + _inflight, + _latency_ms, + _normalize_provider, + _MAX_INFLIGHT, +) + + +# ── Provider normalization ──────────────────────────────────────────────────── + + +def test_normalize_provider_from_litellm_params(): + kwargs = {"litellm_params": {"custom_llm_provider": "anthropic"}} + assert _normalize_provider("some-model", kwargs) == "anthropic" + + +def test_normalize_provider_from_custom_llm_provider(): + kwargs = {"custom_llm_provider": "openai"} + assert _normalize_provider("gpt-4o", kwargs) == "openai" + + +def test_normalize_provider_from_model_prefix(): + assert _normalize_provider("anthropic/claude-3-5-haiku", {}) == "anthropic" + assert _normalize_provider("openai/gpt-4o", {}) == "openai" + + +def test_normalize_provider_from_model_name_pattern(): + assert _normalize_provider("claude-haiku-4-5", {}) == "anthropic" + assert _normalize_provider("gpt-4o-mini", {}) == "openai" + assert _normalize_provider("gemini-2.5-flash", {}) == "google" + assert _normalize_provider("mistral-small-latest", {}) == "mistral" + assert _normalize_provider("llama-3.3-70b", {}) == "meta" + assert _normalize_provider("grok-3-mini", {}) == "xai" + assert _normalize_provider("deepseek-v3", {}) == "deepseek" + + +def test_normalize_provider_unknown(): + assert _normalize_provider("some-unknown-model-xyz", {}) == "unknown" + + +# ── Status code extraction ──────────────────────────────────────────────────── + + +def test_extract_status_code_int(): + exc = MagicMock() + exc.status_code = 429 + assert _extract_status_code(exc) == 429 + + +def test_extract_status_code_string(): + exc = MagicMock() + exc.status_code = "503" + assert _extract_status_code(exc) == 503 + + +def test_extract_status_code_none_exception(): + assert _extract_status_code(None) is None + + +def test_extract_status_code_no_attribute(): + assert _extract_status_code(ValueError("oops")) is None + + +# ── Latency calculation ─────────────────────────────────────────────────────── + + +def test_latency_ms_with_datetime(): + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=1240) + assert _latency_ms(start, end) == 1240 + + +def test_latency_ms_with_floats(): + assert _latency_ms(1000.0, 1001.5) == 1500 + + +def test_latency_ms_mixed_types_float(): + # Both floats — should not raise + result = _latency_ms(0.0, 0.5) + assert result == 500 + + +# ── Error type mapping ──────────────────────────────────────────────────────── + + +def test_error_type_known_codes(): + assert _ERROR_TYPE_MAP[429] == "rate_limit" + assert _ERROR_TYPE_MAP[529] == "overloaded" + assert _ERROR_TYPE_MAP[503] == "overloaded" + assert _ERROR_TYPE_MAP[408] == "timeout" + assert _ERROR_TYPE_MAP[401] == "auth" + + +def test_error_type_no_default_for_unknown_codes(): + # Unknown codes (400, 404, 502) must NOT map to "overloaded" or any value + for code in (400, 404, 502, 422, 301): + assert code not in _ERROR_TYPE_MAP, f"code {code} should not be in _ERROR_TYPE_MAP" + + +# ── TickerrLogger instantiation ─────────────────────────────────────────────── + + +def test_tickerr_logger_default_init(): + logger = TickerrLogger() + assert logger.client_tier is None + assert logger.region is None + + +def test_tickerr_logger_reads_env_vars(): + with patch.dict(os.environ, {"TICKERR_CLIENT_TIER": "pro", "TICKERR_REGION": "us-east-1"}): + logger = TickerrLogger() + assert logger.client_tier == "pro" + assert logger.region == "us-east-1" + + +# ── Payload construction via _report ───────────────────────────────────────── + + +def test_report_builds_correct_payload(): + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + exc = MagicMock() + exc.status_code = 429 + + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=500) + + kwargs = { + "model": "claude-haiku-4-5", + "exception": exc, + "litellm_params": {"custom_llm_provider": "anthropic"}, + } + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["provider"] == "anthropic" + assert captured["model"] == "claude-haiku-4-5" + assert captured["error_code"] == 429 + assert captured["error_type"] == "rate_limit" + assert captured["latency_ms"] == 500 + + +def test_report_strips_provider_prefix_from_model(): + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=100) + + kwargs = {"model": "openai/gpt-4o-mini", "exception": None} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert captured["model"] == "gpt-4o-mini" + + +def test_report_omits_error_type_for_unknown_code(): + logger = TickerrLogger() + captured = {} + + def fake_fire(payload): + captured.update(payload) + + exc = MagicMock() + exc.status_code = 400 # not in _ERROR_TYPE_MAP + + start = datetime(2024, 1, 1, 0, 0, 0) + end = start + timedelta(milliseconds=100) + kwargs = {"model": "gpt-4o", "exception": exc} + + with patch("litellm.integrations.tickerr._fire_and_forget", side_effect=fake_fire): + logger._report(kwargs, start, end) + + assert "error_type" not in captured + assert captured["error_code"] == 400 + + +# ── Thread cap ──────────────────────────────────────────────────────────────── + + +def test_fire_and_forget_respects_semaphore_cap(): + """Reports beyond _MAX_INFLIGHT are dropped silently.""" + sent = [] + + def slow_send(payload): + sent.append(payload) + # simulate slow network + import time + time.sleep(0.1) + + # Exhaust the semaphore + for _ in range(_MAX_INFLIGHT): + _inflight.acquire() + + try: + # This call should be dropped (semaphore exhausted) + with patch("litellm.integrations.tickerr._fire_and_forget"): + _fire_and_forget({"provider": "openai"}) + # Since semaphore is exhausted, the thread should not be started + finally: + # Restore semaphore + for _ in range(_MAX_INFLIGHT): + _inflight.release() + + +# ── Network failure is silent ───────────────────────────────────────────────── + + +def test_fire_and_forget_silent_on_network_error(): + """A network error in the background thread must not propagate.""" + with patch("urllib.request.urlopen", side_effect=OSError("connection refused")): + # Should complete without raising + _fire_and_forget({"provider": "anthropic", "model": "claude-haiku-4-5"}) + import time + time.sleep(0.2) # let daemon thread run + + +# ── Async hooks ─────────────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_async_log_failure_event_calls_report(): + logger = TickerrLogger() + start = datetime(2024, 1, 1) + end = start + timedelta(milliseconds=200) + kwargs = {"model": "gpt-4o-mini", "exception": None} + + with patch.object(logger, "_report") as mock_report: + await logger.async_log_failure_event(kwargs, None, start, end) + mock_report.assert_called_once_with(kwargs, start, end)