test(cache): encode the cache compatibility spec as a Python and native contract suite
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled

This commit is contained in:
Yujong Lee 2026-09-23 19:46:44 -07:00
parent b84caced7c
commit 7f3dffe3aa

View file

@ -1,48 +1,138 @@
import itertools
from collections.abc import Awaitable, Callable, Mapping
import os
import subprocess
import sys
import threading
from collections.abc import Awaitable, Callable, Generator, Mapping
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final
from unittest.mock import patch
from uuid import uuid4
import fakeredis
import pytest
import litellm
from litellm.caching.base_cache import BaseCache
from litellm.caching.caching import Cache
from litellm.caching.disk_cache import DiskCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cache import RedisCache
from litellm.caching.s3_cache import S3Cache
from litellm.rust_bridge import catalog
from litellm.rust_bridge.catalog import CacheRule
from litellm.rust_bridge.configuration import Rollout
from litellm.rust_bridge.response_cache import resolve_response_cache
from litellm.types.caching import LiteLLMCacheType
from litellm.types.utils import Choices, ModelResponse
from tests.test_litellm_rust.support.s3_stub import S3Stub
pytestmark: Final = pytest.mark.requires_rust_extension
Writer = Callable[[str, object], Awaitable[None]]
Reader = Callable[[str], Awaitable[object]]
OVERRIDDEN: Final = {"source": "subclass override"}
Settings = Callable[[pytest.FixtureRequest], Mapping[str, object]]
OVERRIDDEN: Final = {"source": "override"}
@dataclass(frozen=True, slots=True)
class Backend:
type: LiteLLMCacheType
cls: type[BaseCache]
config: Callable[[Path], Mapping[str, str]]
facade_config: Callable[[Path], Mapping[str, str]]
settings: Settings
backend_settings: Settings
def redis_settings(request: pytest.FixtureRequest) -> Mapping[str, object]:
host, port = request.getfixturevalue("redis_address")
return {"host": host, "port": port}
def s3_settings(request: pytest.FixtureRequest) -> Mapping[str, object]:
stub: Final[S3Stub] = request.getfixturevalue("s3_stub")
return {
"s3_bucket_name": "cache-bucket",
"s3_region_name": "us-east-1",
"s3_endpoint_url": stub.url,
"s3_aws_access_key_id": "key",
"s3_aws_secret_access_key": "secret",
"s3_path": "team",
}
def disk_settings(request: pytest.FixtureRequest) -> Mapping[str, object]:
return {"disk_cache_dir": str(request.getfixturevalue("tmp_path"))}
BACKENDS: Final = (
pytest.param(Backend(LiteLLMCacheType.LOCAL, InMemoryCache, lambda _: {}, lambda _: {}), id="local"),
pytest.param(
Backend(
LiteLLMCacheType.DISK,
DiskCache,
lambda root: {"disk_cache_dir": str(root)},
lambda root: {"disk_cache_dir": str(root)},
),
id="disk",
),
Backend(LiteLLMCacheType.LOCAL, InMemoryCache, lambda _: {}, lambda _: {}),
Backend(LiteLLMCacheType.DISK, DiskCache, disk_settings, disk_settings),
Backend(LiteLLMCacheType.REDIS, RedisCache, redis_settings, redis_settings),
Backend(LiteLLMCacheType.S3, S3Cache, s3_settings, s3_settings),
)
EXCLUDED_BACKENDS: Final = MappingProxyType(
{
LiteLLMCacheType.GCS: "Python GCSCache has no configurable endpoint, so it cannot reach an in-process fake",
LiteLLMCacheType.AZURE_BLOB: "no in-process Azure Blob fake yet",
LiteLLMCacheType.REDIS_SEMANTIC: "fakeredis lacks the vector search commands the backend sends",
LiteLLMCacheType.VALKEY_SEMANTIC: "fakeredis lacks the vector search commands the backend sends",
LiteLLMCacheType.QDRANT_SEMANTIC: "no in-process Qdrant fake yet",
}
)
BACKEND_PARAMS: Final = tuple(pytest.param(backend, id=backend.type.value) for backend in BACKENDS)
SEPARATE_STORES: Final = "Cache and the response-cache runtime keep separate local stores"
OVERRIDES_BYPASSED: Final = (
"the response-cache runtime reads storage directly instead of calling overridden Cache methods"
)
STORAGE_OBJECT_IGNORED: Final = "native storage ignores an object assigned to Cache.cache"
PER_CALL_OBJECT_IGNORED: Final = "native writes ignore the per-call storage object"
FORKED_CHILD_LOSES_ENTRIES: Final = "a forked child loses entries written through the native cache"
BACKEND_IDS: Final = tuple(backend.type.value for backend in BACKENDS)
NATIVE_GAPS: Final = MappingProxyType(
{
("test_every_writer_is_visible_to_every_reader", "local"): SEPARATE_STORES,
**{
("test_cache_subclass_overrides_run_on_every_read_and_super_reaches_the_store", backend): OVERRIDES_BYPASSED
for backend in BACKEND_IDS
},
**{
("test_backend_subclass_overrides_win_on_every_reader", backend): STORAGE_OBJECT_IGNORED
for backend in BACKEND_IDS
},
**{
("test_backend_subclass_super_calls_reach_the_shared_store", backend): STORAGE_OBJECT_IGNORED
for backend in BACKEND_IDS
},
("test_custom_storage_backend_serves_every_path", None): STORAGE_OBJECT_IGNORED,
("test_patched_public_methods_win_on_every_reader", None): OVERRIDES_BYPASSED,
("test_per_call_storage_object_replaces_the_configured_backend", None): PER_CALL_OBJECT_IGNORED,
("test_a_forked_child_reads_inherited_entries_and_writes_its_own", None): FORKED_CHILD_LOSES_ENTRIES,
}
)
@pytest.fixture
def redis_address() -> Generator[tuple[str, int]]:
server: Final = fakeredis.TcpFakeServer(("127.0.0.1", 0), server_type="redis")
worker: Final = threading.Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
yield ("127.0.0.1", server.server_address[1])
finally:
server.shutdown()
server.server_close()
worker.join(timeout=5)
@pytest.fixture
def s3_stub() -> Generator[S3Stub]:
stub: Final = S3Stub()
try:
yield stub
finally:
stub.close()
@pytest.fixture(params=(Rollout.PYTHON_ONLY, Rollout.RUST_REQUIRED), ids=("python", "native"))
@ -52,6 +142,18 @@ def rollout(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) ->
return selected
@pytest.fixture(autouse=True)
def expected_native_gap(request: pytest.FixtureRequest) -> None:
params: Final[Mapping[str, object]] = getattr(getattr(request.node, "callspec", None), "params", {})
if params.get("rollout") is not Rollout.RUST_REQUIRED:
return
backend: Final = params.get("backend")
backend_id: Final = backend.type.value if isinstance(backend, Backend) else None
reason: Final = NATIVE_GAPS.get((request.node.originalname, backend_id))
if reason is not None:
request.applymarker(pytest.mark.xfail(strict=True, reason=f"native: {reason}"))
@dataclass(frozen=True, slots=True)
class Paths:
writers: Mapping[str, Writer]
@ -110,26 +212,12 @@ async def disagreements(paths: Paths, expected: Callable[[object], object]) -> t
return tuple(result for result in results if result is not None)
@pytest.mark.usefixtures("rollout")
@pytest.mark.parametrize("backend", BACKENDS)
async def test_every_writer_is_visible_to_every_reader(backend: Backend, tmp_path: Path) -> None:
facade: Final = Cache(type=backend.type, **backend.facade_config(tmp_path))
assert await disagreements(paths(facade), lambda value: value) == ()
def serve_override(*_args: object, **_kwargs: object) -> object:
return OVERRIDDEN
@pytest.mark.usefixtures("rollout")
@pytest.mark.parametrize("backend", BACKENDS)
async def test_backend_subclass_overrides_win_on_every_reader(backend: Backend, tmp_path: Path) -> None:
class Overriding(backend.cls):
def get_cache(self, key: str, **kwargs: object) -> object:
return OVERRIDDEN
async def async_get_cache(self, key: str, **kwargs: object) -> object:
return OVERRIDDEN
facade: Final = Cache(type=backend.type, **backend.facade_config(tmp_path))
facade.cache = Overriding(**backend.config(tmp_path))
assert await disagreements(paths(facade), lambda _value: OVERRIDDEN) == ()
async def async_serve_override(*_args: object, **_kwargs: object) -> object:
return OVERRIDDEN
def is_wrapped(response: object) -> bool:
@ -142,9 +230,61 @@ def wrap_response(entry: object) -> object:
return {**entry, "response": {"wrapped": entry["response"]}}
def test_every_cache_type_has_a_contract_row_or_a_recorded_exclusion() -> None:
covered: Final = {backend.type for backend in BACKENDS}
assert covered.isdisjoint(EXCLUDED_BACKENDS)
assert covered | EXCLUDED_BACKENDS.keys() == set(LiteLLMCacheType)
@pytest.mark.usefixtures("rollout")
@pytest.mark.parametrize("backend", BACKENDS)
async def test_backend_subclass_super_calls_reach_the_shared_store(backend: Backend, tmp_path: Path) -> None:
@pytest.mark.parametrize("backend", BACKEND_PARAMS)
async def test_every_writer_is_visible_to_every_reader(backend: Backend, request: pytest.FixtureRequest) -> None:
facade: Final = Cache(type=backend.type, **backend.settings(request))
assert await disagreements(paths(facade), lambda value: value) == ()
@pytest.mark.usefixtures("rollout")
@pytest.mark.parametrize("backend", BACKEND_PARAMS)
async def test_cache_subclass_overrides_run_on_every_read_and_super_reaches_the_store(
backend: Backend, request: pytest.FixtureRequest
) -> None:
reads: Final[list[str]] = [] # mutable-ok: the override records each read it serves
class AuditedCache(Cache):
def get_cache(self, *args: object, **kwargs: object) -> object:
reads.append("sync")
return super().get_cache(*args, **kwargs)
async def async_get_cache(self, *args: object, **kwargs: object) -> object:
reads.append("async")
return await super().async_get_cache(*args, **kwargs)
facade: Final = AuditedCache(type=backend.type, **backend.settings(request))
contract: Final = paths(facade)
assert await disagreements(contract, lambda value: value) == ()
assert len(reads) == len(contract.writers) * len(contract.readers)
@pytest.mark.usefixtures("rollout")
@pytest.mark.parametrize("backend", BACKEND_PARAMS)
async def test_backend_subclass_overrides_win_on_every_reader(backend: Backend, request: pytest.FixtureRequest) -> None:
class Overriding(backend.cls):
def get_cache(self, key: str, **kwargs: object) -> object:
return {"timestamp": 0, "response": OVERRIDDEN}
async def async_get_cache(self, key: str, **kwargs: object) -> object:
return {"timestamp": 0, "response": OVERRIDDEN}
facade: Final = Cache(type=backend.type, **backend.settings(request))
facade.cache = Overriding(**backend.backend_settings(request))
assert await disagreements(paths(facade), lambda _value: OVERRIDDEN) == ()
@pytest.mark.usefixtures("rollout")
@pytest.mark.parametrize("backend", BACKEND_PARAMS)
async def test_backend_subclass_super_calls_reach_the_shared_store(
backend: Backend, request: pytest.FixtureRequest
) -> None:
class Wrapping(backend.cls):
def set_cache(self, key: str, value: object, **kwargs: object) -> None:
super().set_cache(key, wrap_response(value), **kwargs)
@ -152,13 +292,13 @@ async def test_backend_subclass_super_calls_reach_the_shared_store(backend: Back
async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None:
await super().async_set_cache(key, wrap_response(value), **kwargs)
facade: Final = Cache(type=backend.type, **backend.facade_config(tmp_path))
facade.cache = Wrapping(**backend.config(tmp_path))
facade: Final = Cache(type=backend.type, **backend.settings(request))
facade.cache = Wrapping(**backend.backend_settings(request))
assert await disagreements(paths(facade), lambda value: {"wrapped": value}) == ()
@pytest.mark.usefixtures("rollout")
async def test_custom_cache_objects_serve_every_path() -> None:
async def test_custom_storage_backend_serves_every_path() -> None:
class DictCache(BaseCache):
def __init__(self) -> None:
super().__init__()
@ -179,9 +319,104 @@ async def test_custom_cache_objects_serve_every_path() -> None:
async def async_set_cache_pipeline(self, cache_list: object, **kwargs: object) -> None:
raise NotImplementedError
storage: Final = DictCache()
facade: Final = Cache(type=LiteLLMCacheType.LOCAL)
facade.cache = DictCache()
assert await disagreements(paths(facade), lambda value: value) == ()
facade.cache = storage
contract: Final = paths(facade)
assert await disagreements(contract, lambda value: value) == ()
assert len(storage.entries) == len(contract.writers) * len(contract.readers)
@pytest.mark.usefixtures("rollout")
async def test_patched_public_methods_win_on_every_reader() -> None:
facade: Final = Cache(type=LiteLLMCacheType.LOCAL)
with (
patch.object(Cache, "get_cache", serve_override),
patch.object(Cache, "async_get_cache", async_serve_override),
):
assert await disagreements(paths(facade), lambda _value: OVERRIDDEN) == ()
@pytest.mark.usefixtures("rollout")
async def test_per_call_storage_object_replaces_the_configured_backend() -> None:
facade: Final = Cache(type=LiteLLMCacheType.LOCAL)
override: Final = InMemoryCache()
await facade.async_add_cache({"source": "override"}, cache_key="key", dynamic_cache_object=override)
assert facade.get_cache(cache_key="key", dynamic_cache_object=override) == {"source": "override"}
assert await facade.async_get_cache(cache_key="key", dynamic_cache_object=override) == {"source": "override"}
assert facade.get_cache(cache_key="key") is None
assert await facade.async_get_cache(cache_key="key") is None
def reply(response: object) -> object:
assert isinstance(response, ModelResponse)
choice: Final = response.choices[0]
assert isinstance(choice, Choices)
return choice.message.content
@pytest.mark.usefixtures("rollout")
def test_reassigning_the_global_cache_takes_effect_on_the_next_call() -> None:
messages: Final = [{"role": "user", "content": f"contract {uuid4().hex}"}]
def call(mock: str) -> object:
return reply(litellm.completion(model="gpt-4o-mini", messages=messages, mock_response=mock, caching=True))
litellm.enable_cache(type=LiteLLMCacheType.LOCAL)
original: Final = litellm.cache
replacement: Final = Cache(type=LiteLLMCacheType.LOCAL)
stored: Final = (call("first"), call("second"))
litellm.cache = replacement # test-quality-ok: reassignment is the behavior under test; conftest restores it
replaced: Final = (call("third"), call("fourth"))
litellm.disable_cache()
disabled: Final = (call("fifth"), call("sixth"))
litellm.cache = original # test-quality-ok: reassignment is the behavior under test; conftest restores it
restored: Final = call("seventh")
assert stored == ("first", "first")
assert replaced == ("third", "third")
assert disabled == ("fifth", "sixth")
assert restored == "first"
FORK_SCRIPT: Final = """
import os
import sys
from litellm.caching.caching import Cache
from litellm.rust_bridge import catalog
from litellm.rust_bridge.catalog import CacheRule
from litellm.rust_bridge.configuration import Rollout
catalog.RULES = (CacheRule(Rollout[sys.argv[1]]),)
cache = Cache(type="local")
cache.add_cache({"answer": "parent"}, cache_key="parent-key")
read_fd, write_fd = os.pipe()
child = os.fork()
if child == 0:
os.close(read_fd)
cache.add_cache({"answer": "child"}, cache_key="child-key")
seen = (cache.get_cache(cache_key="parent-key"), cache.get_cache(cache_key="child-key"))
os.write(write_fd, repr(seen).encode())
os._exit(0)
os.close(write_fd)
print(os.read(read_fd, 4096).decode())
os.waitpid(child, 0)
"""
@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires fork")
def test_a_forked_child_reads_inherited_entries_and_writes_its_own(rollout: Rollout) -> None:
completed: Final = subprocess.run(
[sys.executable, "-P", "-c", FORK_SCRIPT, rollout.name],
capture_output=True,
text=True,
timeout=30,
check=True,
env={**os.environ, "LITELLM_RUST": "1", "LITELLM_LOCAL_MODEL_COST_MAP": "True"},
)
assert completed.stdout.strip() == "({'answer': 'parent'}, {'answer': 'child'})"
@pytest.mark.usefixtures("rollout")