From b84caced7cb72297c9401ded9671a995fff690d1 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 23 Sep 2026 16:43:06 -0700 Subject: [PATCH] test(cache): add backend contract suite for Python and native cache implementations Co-Authored-By: Claude Opus 5.5 --- .../test_cache_backend_contract.py | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 tests/test_litellm_rust/test_cache_backend_contract.py diff --git a/tests/test_litellm_rust/test_cache_backend_contract.py b/tests/test_litellm_rust/test_cache_backend_contract.py new file mode 100644 index 00000000000..b24b0d8337b --- /dev/null +++ b/tests/test_litellm_rust/test_cache_backend_contract.py @@ -0,0 +1,194 @@ +import itertools +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest + +from litellm.caching.base_cache import BaseCache +from litellm.caching.caching import Cache +from litellm.caching.disk_cache import DiskCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.rust_bridge import catalog +from litellm.rust_bridge.catalog import CacheRule +from litellm.rust_bridge.configuration import Rollout +from litellm.rust_bridge.response_cache import resolve_response_cache +from litellm.types.caching import LiteLLMCacheType + +pytestmark: Final = pytest.mark.requires_rust_extension + +Writer = Callable[[str, object], Awaitable[None]] +Reader = Callable[[str], Awaitable[object]] +OVERRIDDEN: Final = {"source": "subclass override"} + + +@dataclass(frozen=True, slots=True) +class Backend: + type: LiteLLMCacheType + cls: type[BaseCache] + config: Callable[[Path], Mapping[str, str]] + facade_config: Callable[[Path], Mapping[str, str]] + + +BACKENDS: Final = ( + pytest.param(Backend(LiteLLMCacheType.LOCAL, InMemoryCache, lambda _: {}, lambda _: {}), id="local"), + pytest.param( + Backend( + LiteLLMCacheType.DISK, + DiskCache, + lambda root: {"disk_cache_dir": str(root)}, + lambda root: {"disk_cache_dir": str(root)}, + ), + id="disk", + ), +) + + +@pytest.fixture(params=(Rollout.PYTHON_ONLY, Rollout.RUST_REQUIRED), ids=("python", "native")) +def rollout(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> Rollout: + selected: Final[Rollout] = request.param + monkeypatch.setattr(catalog, "RULES", (CacheRule(selected),)) + return selected + + +@dataclass(frozen=True, slots=True) +class Paths: + writers: Mapping[str, Writer] + readers: Mapping[str, Reader] + + +def paths(facade: Cache) -> Paths: + async def facade_add(key: str, value: object) -> None: + facade.add_cache(value, cache_key=key) + + async def facade_async_add(key: str, value: object) -> None: + await facade.async_add_cache(value, cache_key=key) + + async def facade_get(key: str) -> object: + return facade.get_cache(cache_key=key) + + async def facade_async_get(key: str) -> object: + return await facade.async_get_cache(cache_key=key) + + facade_writers: Final = {"facade.add_cache": facade_add, "facade.async_add_cache": facade_async_add} + facade_readers: Final = {"facade.get_cache": facade_get, "facade.async_get_cache": facade_async_get} + runtime: Final = resolve_response_cache(facade) + if runtime is None: + return Paths(facade_writers, facade_readers) + + async def native_store(key: str, value: object) -> None: + runtime.store(runtime.request(facade, {"cache_key": key}), value) + + async def native_async_store(key: str, value: object) -> None: + await runtime.async_store(runtime.request(facade, {"cache_key": key}), value) + + async def native_lookup(key: str) -> object: + return runtime.lookup(runtime.request(facade, {"cache_key": key})) + + async def native_async_lookup(key: str) -> object: + return await runtime.async_lookup(runtime.request(facade, {"cache_key": key})) + + return Paths( + {**facade_writers, "runtime.store": native_store, "runtime.async_store": native_async_store}, + {**facade_readers, "runtime.lookup": native_lookup, "runtime.async_lookup": native_async_lookup}, + ) + + +async def disagreements(paths: Paths, expected: Callable[[object], object]) -> tuple[str, ...]: + async def check(index: int, writer: str, reader: str) -> str | None: + key: Final = f"{writer}->{reader}" + value: Final = {"written_by": writer, "index": index} + await paths.writers[writer](key, value) + seen: Final = await paths.readers[reader](key) + return None if seen == expected(value) else f"{key}: saw {seen!r}" + + results: Final = [ + await check(index, writer, reader) + for index, (writer, reader) in enumerate(itertools.product(paths.writers, paths.readers)) + ] + return tuple(result for result in results if result is not None) + + +@pytest.mark.usefixtures("rollout") +@pytest.mark.parametrize("backend", BACKENDS) +async def test_every_writer_is_visible_to_every_reader(backend: Backend, tmp_path: Path) -> None: + facade: Final = Cache(type=backend.type, **backend.facade_config(tmp_path)) + assert await disagreements(paths(facade), lambda value: value) == () + + +@pytest.mark.usefixtures("rollout") +@pytest.mark.parametrize("backend", BACKENDS) +async def test_backend_subclass_overrides_win_on_every_reader(backend: Backend, tmp_path: Path) -> None: + class Overriding(backend.cls): + def get_cache(self, key: str, **kwargs: object) -> object: + return OVERRIDDEN + + async def async_get_cache(self, key: str, **kwargs: object) -> object: + return OVERRIDDEN + + facade: Final = Cache(type=backend.type, **backend.facade_config(tmp_path)) + facade.cache = Overriding(**backend.config(tmp_path)) + assert await disagreements(paths(facade), lambda _value: OVERRIDDEN) == () + + +def is_wrapped(response: object) -> bool: + return isinstance(response, dict) and response.keys() == {"wrapped"} + + +def wrap_response(entry: object) -> object: + if not isinstance(entry, dict) or "response" not in entry or is_wrapped(entry["response"]): + return entry + return {**entry, "response": {"wrapped": entry["response"]}} + + +@pytest.mark.usefixtures("rollout") +@pytest.mark.parametrize("backend", BACKENDS) +async def test_backend_subclass_super_calls_reach_the_shared_store(backend: Backend, tmp_path: Path) -> None: + class Wrapping(backend.cls): + def set_cache(self, key: str, value: object, **kwargs: object) -> None: + super().set_cache(key, wrap_response(value), **kwargs) + + async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None: + await super().async_set_cache(key, wrap_response(value), **kwargs) + + facade: Final = Cache(type=backend.type, **backend.facade_config(tmp_path)) + facade.cache = Wrapping(**backend.config(tmp_path)) + assert await disagreements(paths(facade), lambda value: {"wrapped": value}) == () + + +@pytest.mark.usefixtures("rollout") +async def test_custom_cache_objects_serve_every_path() -> None: + class DictCache(BaseCache): + def __init__(self) -> None: + super().__init__() + self.entries: dict[str, object] = {} # mutable-ok: a user-defined cache is a mutable store by design + + def set_cache(self, key: str, value: object, **kwargs: object) -> None: + self.entries[key] = value + + async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None: + self.set_cache(key, value) + + def get_cache(self, key: str, **kwargs: object) -> object: + return self.entries.get(key) + + async def async_get_cache(self, key: str, **kwargs: object) -> object: + return self.get_cache(key) + + async def async_set_cache_pipeline(self, cache_list: object, **kwargs: object) -> None: + raise NotImplementedError + + facade: Final = Cache(type=LiteLLMCacheType.LOCAL) + facade.cache = DictCache() + assert await disagreements(paths(facade), lambda value: value) == () + + +@pytest.mark.usefixtures("rollout") +async def test_memory_backend_returns_the_stored_object_itself() -> None: + facade: Final = Cache(type=LiteLLMCacheType.LOCAL) + stored: Final = object() + facade.cache.set_cache("identity", stored) + await facade.cache.async_set_cache("async identity", stored) + assert facade.cache.get_cache("identity") is stored + assert await facade.cache.async_get_cache("async identity") is stored