From 01903cb3ae39092badfb89a6772ca9b8b42ee94a Mon Sep 17 00:00:00 2001 From: gabriele Date: Mon, 28 Sep 2026 11:28:00 +0000 Subject: [PATCH] fix(spend-tracking): stop caching failed spend-log metadata lookups as confirmed misses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../spend_tracking/key_metadata_recovery.py | 13 ++++++-- .../test_key_metadata_recovery.py | 32 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index ce96dc62780..120ff0a680f 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -370,10 +370,17 @@ async def _spend_log_metadata_one_query_at_a_time( fresh: Final = ( await _query_spend_log_metadata(prisma_client, pending, window) if pending else _EMPTY_KEY_METADATA ) - found: Final = fresh if fresh is not None else _EMPTY_KEY_METADATA + if fresh is None: + for digest in pending: + cache.set_cache( + _spend_log_cache_key(digest, window), + KeyMetadataDict(), + ttl=SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL, + ) + return settled for digest in pending: - _remember_spend_log_metadata(cache, digest, window, found.get(digest)) - return MappingProxyType({**settled, **found}) + _remember_spend_log_metadata(cache, digest, window, fresh.get(digest)) + return MappingProxyType({**settled, **fresh}) async def recover_key_metadata_from_spend_logs( diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index acd03964bf3..d96156a9d10 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -3,6 +3,7 @@ import time from collections.abc import Sequence from datetime import datetime, timedelta from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -439,6 +440,37 @@ async def test_recover_key_metadata_from_spend_logs_retries_a_failed_query_only_ assert result[digest]["key_alias"] == "back-online" +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_never_caches_repeated_query_failures_as_long_as_a_hit(): + digest: Final = hash_token("cli-session-repeated-failure") + window: Final = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + cache: Final = InMemoryCache(default_ttl=SPEND_LOG_KEY_METADATA_CACHE_TTL) + mock_prisma: Final = MagicMock() + _spend_log_transaction( + mock_prisma, + AsyncMock( + side_effect=[ + PrismaError("statement timeout"), + PrismaError("statement timeout"), + [_spend_log_row(digest, "back-online", None, None)], + ] + ), + ) + await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) + miss_key: Final = next(key for key in cache.ttl_dict if digest in key and not key.endswith(":missed-before")) + cache.ttl_dict[miss_key] = time.time() - 1 + second_query_started: Final = time.time() + + await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) + + assert cache.ttl_dict[miss_key] - second_query_started <= SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL + 1 + cache.ttl_dict[miss_key] = time.time() - 1 + + result: Final = await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) + + assert result[digest]["key_alias"] == "back-online" + + @pytest.mark.asyncio async def test_recover_key_metadata_from_spend_logs_drops_the_owner_of_a_digest_shared_by_several_users(): shared_ui_digest = hash_token("ui-token")