fix(proxy): treat an open Redis breaker as a dropped spend counter update, not a cost tracking failure

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/proxy/proxy_server.py

@@ -240,6 +240,7 @@ import litellm._redis
 from litellm import Router
 from litellm._logging import _redact_string, verbose_proxy_logger, verbose_router_logger
 from litellm.caching.caching import DualCache, RedisCache
+from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
 from litellm.caching.redis_cluster_cache import RedisClusterCache
 from litellm.constants import (
     _REALTIME_BODY_CACHE_SIZE,
@@ -3369,9 +3370,11 @@ async def _apply_spend_counter_increments(pending: Sequence[_PendingSpendIncreme
     ]
     try:
         results: Final = await redis_cache.async_increment_pipeline(increment_list=increment_list)
-    except Exception:
+    except Exception as e:
         await asyncio.gather(*(_invalidate_spend_counter(counter_key=item.counter_key) for item in pending))
-        raise
+        if not isinstance(e, RedisCircuitBreakerOpenError):
+            raise
+        return
     for item, current_value in zip(pending, results or ()):
         spend_counter_cache.in_memory_cache.set_cache(key=item.counter_key, value=current_value)
 

tests/test_litellm/proxy/proxy_server/test_spend_counters.py

@@ -1348,6 +1348,48 @@ async def test_is_spend_counter_cache_warm_redis_error_falls_back_to_in_memory(
     assert result is False
 
 
+# ---------------------------------------------------------------------------
+# _apply_spend_counter_increments
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_apply_spend_counter_increments_open_breaker_invalidates_and_returns(monkeypatch):
+    """An open Redis breaker fast-fails the pipeline on every request, so the callback
+    must not turn each one into a tracking-cost failure: drop the stale local copies
+    and return like a miss, without raising."""
+    from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
+
+    fake_cache = _make_spend_counter_cache()
+    fake_cache.redis_cache.async_increment_pipeline = AsyncMock(
+        side_effect=RedisCircuitBreakerOpenError("Redis circuit breaker is open, skipping async_increment_pipeline")
+    )
+    monkeypatch.setattr(ps, "spend_counter_cache", fake_cache)
+    pending: Final = (
+        ps._PendingSpendIncrement(counter_key="spend:key:k", increment=1.0),
+        ps._PendingSpendIncrement(counter_key="spend:team:t", increment=1.0),
+    )
+
+    await ps._apply_spend_counter_increments(pending=pending)
+
+    deleted: Final = sorted(call.kwargs["key"] for call in fake_cache.in_memory_cache.delete_cache.call_args_list)
+    assert deleted == ["spend:key:k", "spend:team:t"]
+    assert fake_cache.in_memory_cache.set_cache.called is False
+
+
+@pytest.mark.asyncio
+async def test_apply_spend_counter_increments_other_redis_error_invalidates_and_raises(monkeypatch):
+    fake_cache = _make_spend_counter_cache()
+    fake_cache.redis_cache.async_increment_pipeline = AsyncMock(side_effect=RuntimeError("incr fail"))
+    monkeypatch.setattr(ps, "spend_counter_cache", fake_cache)
+    pending: Final = (ps._PendingSpendIncrement(counter_key="spend:key:k", increment=1.0),)
+
+    with pytest.raises(RuntimeError):
+        await ps._apply_spend_counter_increments(pending=pending)
+
+    assert fake_cache.in_memory_cache.delete_cache.called is True
+
+
 # ---------------------------------------------------------------------------
 # _increment_spend_counter_cache
 # ---------------------------------------------------------------------------