mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
fix(caching): read s-max-age first on the sync cache read like the async read
The previous commit gave s-maxage priority on the sync read, while the async
read, the batch Redis hook and the Rust cache give s-max-age priority. A
request that sent both keys could then hit the cache on one path and miss on
the other, and `{"s-maxage": None, "s-max-age": 1}` served stale audio on
this line's speech path. The sync read now uses the async read's order
This commit is contained in:
parent
c2ba33609a
commit
d94be668d2
2 changed files with 25 additions and 1 deletions
|
|
@ -663,7 +663,7 @@ class Cache:
|
||||||
if cache_key is not None:
|
if cache_key is not None:
|
||||||
cache_control_args: Final[DynamicCacheControl] = kwargs.get("cache", {})
|
cache_control_args: Final[DynamicCacheControl] = kwargs.get("cache", {})
|
||||||
max_age: Final = cache_control_args.get(
|
max_age: Final = cache_control_args.get(
|
||||||
"s-maxage", cache_control_args.get("s-max-age", float("inf"))
|
"s-max-age", cache_control_args.get("s-maxage", float("inf"))
|
||||||
)
|
)
|
||||||
cache_lookup_kwargs: Final = self._get_safe_cache_lookup_kwargs(kwargs)
|
cache_lookup_kwargs: Final = self._get_safe_cache_lookup_kwargs(kwargs)
|
||||||
if dynamic_cache_object is not None:
|
if dynamic_cache_object is not None:
|
||||||
|
|
|
||||||
|
|
@ -1455,6 +1455,30 @@ def test_sync_completion_with_zero_s_maxage_skips_the_cached_response(
|
||||||
assert open_max_age.id == zero_max_age.id
|
assert open_max_age.id == zero_max_age.id
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"cache_controls",
|
||||||
|
[{"s-maxage": 0, "s-max-age": 60}, {"s-maxage": 60, "s-max-age": 0}, {"s-maxage": None, "s-max-age": 0}],
|
||||||
|
)
|
||||||
|
async def test_sync_and_async_completion_agree_on_the_cache_when_both_max_age_keys_are_sent(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, cache_controls: dict[str, int | None]
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL))
|
||||||
|
sync_messages: Final = [{"role": "user", "content": f"sync both max-age keys {uuid.uuid4().hex}"}]
|
||||||
|
sync_first: Final = completion(model="gpt-4o-mini", messages=sync_messages, mock_response="first")
|
||||||
|
sync_second: Final = completion(
|
||||||
|
model="gpt-4o-mini", messages=sync_messages, cache=cache_controls, mock_response="second"
|
||||||
|
)
|
||||||
|
async_messages: Final = [{"role": "user", "content": f"async both max-age keys {uuid.uuid4().hex}"}]
|
||||||
|
async_first: Final = await acompletion(model="gpt-4o-mini", messages=async_messages, mock_response="first")
|
||||||
|
await asyncio.gather(*_PENDING_CACHE_WRITES)
|
||||||
|
async_second: Final = await acompletion(
|
||||||
|
model="gpt-4o-mini", messages=async_messages, cache=cache_controls, mock_response="second"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (sync_second.id == sync_first.id) == (async_second.id == async_first.id)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_embedding_caching_redis_ttl(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None:
|
async def test_embedding_caching_redis_ttl(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None:
|
||||||
mock_pipeline: Final = AsyncMock()
|
mock_pipeline: Final = AsyncMock()
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue