scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-12 03:48:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 57

fix(router): preserve healthy legacy cache checkpoints
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
Details
LiteLLM Rust / rust-test (push) Has been cancelled
Details
LiteLLM Rust / rust-wheel (push) Has been cancelled
Details
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Details
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Details

Browse source
This commit is contained in:
Tin 2026-10-09 23:13:35 -07:00
parent a3f36b571b
commit d1b31d0743
2 changed files with 28 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
litellm/proxy/spend_tracking/baseline_accounting.py
View file

@ -384,7 +384,14 @@ def advance_baseline_history(
)
legacy_lifetime: Final = (
MAX_ESTIMATED_CACHE_TTL
if history.version == 1 and (history.uncertain_before > 0 or any(entry.uncertain for entry in history.entries))
if history.version == 1
and (
history.uncertain_before > (history.first_at or 0)
or history.uncertain_before > 0
and not history.entries
or history.blocked_until > 0
or any(entry.uncertain for entry in history.entries)
)
else 0
)
legacy_until: Final = (

23
tests/unit/proxy/spend_tracking/test_baseline_accounting.py
View file

@ -119,11 +119,13 @@ def test_skipped_long_cache_lifetime_survives_shorter_followups(ttl: int | None,
assert warm[0].usage is not None and warm[0].usage.prompt_tokens_details.cached_tokens == 6000
@pytest.mark.parametrize("ambiguous_entry", (False, True))
def test_legacy_checkpoint_accounts_for_refreshes_after_a_skipped_request(ambiguous_entry: bool) -> None:
@pytest.mark.parametrize("uncertain_before,ambiguous_entry", ((1000.0, False), (0.0, True), (1.0, False)))
def test_legacy_checkpoint_accounts_for_refreshes_after_a_skipped_request(
uncertain_before: float, ambiguous_entry: bool,
) -> None:
restored: Final = TypeAdapter(BaselineHistory).validate_python({
"first_at": 1.0, "last_at": 80000.0, "equivalent": False,
"uncertain_before": 0.0 if ambiguous_entry else 1000.0,
"uncertain_before": uncertain_before,
"entries": (CacheEntry("p:300", "p", 6000, 300, 80000.0, 83600.0, uncertain=True),)
if ambiguous_entry else (),
})
@ -134,6 +136,21 @@ def test_legacy_checkpoint_accounts_for_refreshes_after_a_skipped_request(ambigu
assert expired[0].usage is not None and expired[0].usage.prompt_tokens_details.cached_tokens == 0
@pytest.mark.parametrize("policy", ("anthropic", "estimated"))
def test_healthy_legacy_checkpoint_preserves_known_cache_hits(policy: str) -> None:
history, _ = advance_baseline_history(BaselineHistory(), (_observation("first"),))
adapter: Final = TypeAdapter(BaselineHistory)
restored: Final = adapter.validate_python(adapter.dump_python(
history, exclude={"version", "uncertain_until", "uncertain_ttl_seconds"},
))
updated, estimates = advance_baseline_history(
restored, (_observation("after_upgrade", 10002.0, cache_policy=policy),),
)
assert estimates[0].reason == "cache_prefix_available"
assert estimates[0].usage is not None and estimates[0].usage.prompt_tokens_details.cached_tokens == 6000
assert updated.uncertain_until == 0 and updated.uncertain_ttl_seconds == 0
def test_skipped_turn_preserves_lifetime_of_an_earlier_ambiguous_write() -> None:
first: Final = _observation("long", 1.0, cache_policy="estimated", cache_ttl_seconds=3600)
short: Final = _observation(