scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

fix(spend-tracking): stop caching failed spend-log metadata lookups as confirmed misses

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
gabriele 2026-09-28 11:28:00 +00:00
parent 90e4962c81
commit 01903cb3ae
2 changed files with 42 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
litellm/proxy/spend_tracking/key_metadata_recovery.py
View file

@ -370,10 +370,17 @@ async def _spend_log_metadata_one_query_at_a_time(
fresh: Final = (
await _query_spend_log_metadata(prisma_client, pending, window) if pending else _EMPTY_KEY_METADATA
)
found: Final = fresh if fresh is not None else _EMPTY_KEY_METADATA
if fresh is None:
for digest in pending:
cache.set_cache(
_spend_log_cache_key(digest, window),
KeyMetadataDict(),
ttl=SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL,
)
return settled
for digest in pending:
_remember_spend_log_metadata(cache, digest, window, found.get(digest))
return MappingProxyType({**settled, **found})
_remember_spend_log_metadata(cache, digest, window, fresh.get(digest))
return MappingProxyType({**settled, **fresh})
async def recover_key_metadata_from_spend_logs(

32
tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py
View file

@ -3,6 +3,7 @@ import time
from collections.abc import Sequence
from datetime import datetime, timedelta
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -439,6 +440,37 @@ async def test_recover_key_metadata_from_spend_logs_retries_a_failed_query_only_
assert result[digest]["key_alias"] == "back-online"
@pytest.mark.asyncio
async def test_recover_key_metadata_from_spend_logs_never_caches_repeated_query_failures_as_long_as_a_hit():
digest: Final = hash_token("cli-session-repeated-failure")
window: Final = (datetime(2026, 9, 7), datetime(2026, 9, 10))
cache: Final = InMemoryCache(default_ttl=SPEND_LOG_KEY_METADATA_CACHE_TTL)
mock_prisma: Final = MagicMock()
_spend_log_transaction(
mock_prisma,
AsyncMock(
side_effect=[
PrismaError("statement timeout"),
PrismaError("statement timeout"),
[_spend_log_row(digest, "back-online", None, None)],
]
),
)
await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache)
miss_key: Final = next(key for key in cache.ttl_dict if digest in key and not key.endswith(":missed-before"))
cache.ttl_dict[miss_key] = time.time() - 1
second_query_started: Final = time.time()
await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache)
assert cache.ttl_dict[miss_key] - second_query_started <= SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL + 1
cache.ttl_dict[miss_key] = time.time() - 1
result: Final = await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache)
assert result[digest]["key_alias"] == "back-online"
@pytest.mark.asyncio
async def test_recover_key_metadata_from_spend_logs_drops_the_owner_of_a_digest_shared_by_several_users():
shared_ui_digest = hash_token("ui-token")