diff --git a/tests/test_litellm_rust/test_cache_backend_contract.py b/tests/test_litellm_rust/test_cache_backend_contract.py index b24b0d8337b..d668f76a16a 100644 --- a/tests/test_litellm_rust/test_cache_backend_contract.py +++ b/tests/test_litellm_rust/test_cache_backend_contract.py @@ -1,48 +1,138 @@ import itertools -from collections.abc import Awaitable, Callable, Mapping +import os +import subprocess +import sys +import threading +from collections.abc import Awaitable, Callable, Generator, Mapping from dataclasses import dataclass -from pathlib import Path +from types import MappingProxyType from typing import Final +from unittest.mock import patch +from uuid import uuid4 +import fakeredis import pytest +import litellm from litellm.caching.base_cache import BaseCache from litellm.caching.caching import Cache from litellm.caching.disk_cache import DiskCache from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.redis_cache import RedisCache +from litellm.caching.s3_cache import S3Cache from litellm.rust_bridge import catalog from litellm.rust_bridge.catalog import CacheRule from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.response_cache import resolve_response_cache from litellm.types.caching import LiteLLMCacheType +from litellm.types.utils import Choices, ModelResponse +from tests.test_litellm_rust.support.s3_stub import S3Stub pytestmark: Final = pytest.mark.requires_rust_extension Writer = Callable[[str, object], Awaitable[None]] Reader = Callable[[str], Awaitable[object]] -OVERRIDDEN: Final = {"source": "subclass override"} +Settings = Callable[[pytest.FixtureRequest], Mapping[str, object]] +OVERRIDDEN: Final = {"source": "override"} @dataclass(frozen=True, slots=True) class Backend: type: LiteLLMCacheType cls: type[BaseCache] - config: Callable[[Path], Mapping[str, str]] - facade_config: Callable[[Path], Mapping[str, str]] + settings: Settings + backend_settings: Settings + + +def redis_settings(request: pytest.FixtureRequest) -> Mapping[str, object]: + host, port = request.getfixturevalue("redis_address") + return {"host": host, "port": port} + + +def s3_settings(request: pytest.FixtureRequest) -> Mapping[str, object]: + stub: Final[S3Stub] = request.getfixturevalue("s3_stub") + return { + "s3_bucket_name": "cache-bucket", + "s3_region_name": "us-east-1", + "s3_endpoint_url": stub.url, + "s3_aws_access_key_id": "key", + "s3_aws_secret_access_key": "secret", + "s3_path": "team", + } + + +def disk_settings(request: pytest.FixtureRequest) -> Mapping[str, object]: + return {"disk_cache_dir": str(request.getfixturevalue("tmp_path"))} BACKENDS: Final = ( - pytest.param(Backend(LiteLLMCacheType.LOCAL, InMemoryCache, lambda _: {}, lambda _: {}), id="local"), - pytest.param( - Backend( - LiteLLMCacheType.DISK, - DiskCache, - lambda root: {"disk_cache_dir": str(root)}, - lambda root: {"disk_cache_dir": str(root)}, - ), - id="disk", - ), + Backend(LiteLLMCacheType.LOCAL, InMemoryCache, lambda _: {}, lambda _: {}), + Backend(LiteLLMCacheType.DISK, DiskCache, disk_settings, disk_settings), + Backend(LiteLLMCacheType.REDIS, RedisCache, redis_settings, redis_settings), + Backend(LiteLLMCacheType.S3, S3Cache, s3_settings, s3_settings), ) +EXCLUDED_BACKENDS: Final = MappingProxyType( + { + LiteLLMCacheType.GCS: "Python GCSCache has no configurable endpoint, so it cannot reach an in-process fake", + LiteLLMCacheType.AZURE_BLOB: "no in-process Azure Blob fake yet", + LiteLLMCacheType.REDIS_SEMANTIC: "fakeredis lacks the vector search commands the backend sends", + LiteLLMCacheType.VALKEY_SEMANTIC: "fakeredis lacks the vector search commands the backend sends", + LiteLLMCacheType.QDRANT_SEMANTIC: "no in-process Qdrant fake yet", + } +) +BACKEND_PARAMS: Final = tuple(pytest.param(backend, id=backend.type.value) for backend in BACKENDS) + +SEPARATE_STORES: Final = "Cache and the response-cache runtime keep separate local stores" +OVERRIDES_BYPASSED: Final = ( + "the response-cache runtime reads storage directly instead of calling overridden Cache methods" +) +STORAGE_OBJECT_IGNORED: Final = "native storage ignores an object assigned to Cache.cache" +PER_CALL_OBJECT_IGNORED: Final = "native writes ignore the per-call storage object" +FORKED_CHILD_LOSES_ENTRIES: Final = "a forked child loses entries written through the native cache" +BACKEND_IDS: Final = tuple(backend.type.value for backend in BACKENDS) +NATIVE_GAPS: Final = MappingProxyType( + { + ("test_every_writer_is_visible_to_every_reader", "local"): SEPARATE_STORES, + **{ + ("test_cache_subclass_overrides_run_on_every_read_and_super_reaches_the_store", backend): OVERRIDES_BYPASSED + for backend in BACKEND_IDS + }, + **{ + ("test_backend_subclass_overrides_win_on_every_reader", backend): STORAGE_OBJECT_IGNORED + for backend in BACKEND_IDS + }, + **{ + ("test_backend_subclass_super_calls_reach_the_shared_store", backend): STORAGE_OBJECT_IGNORED + for backend in BACKEND_IDS + }, + ("test_custom_storage_backend_serves_every_path", None): STORAGE_OBJECT_IGNORED, + ("test_patched_public_methods_win_on_every_reader", None): OVERRIDES_BYPASSED, + ("test_per_call_storage_object_replaces_the_configured_backend", None): PER_CALL_OBJECT_IGNORED, + ("test_a_forked_child_reads_inherited_entries_and_writes_its_own", None): FORKED_CHILD_LOSES_ENTRIES, + } +) + + +@pytest.fixture +def redis_address() -> Generator[tuple[str, int]]: + server: Final = fakeredis.TcpFakeServer(("127.0.0.1", 0), server_type="redis") + worker: Final = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + yield ("127.0.0.1", server.server_address[1]) + finally: + server.shutdown() + server.server_close() + worker.join(timeout=5) + + +@pytest.fixture +def s3_stub() -> Generator[S3Stub]: + stub: Final = S3Stub() + try: + yield stub + finally: + stub.close() @pytest.fixture(params=(Rollout.PYTHON_ONLY, Rollout.RUST_REQUIRED), ids=("python", "native")) @@ -52,6 +142,18 @@ def rollout(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> return selected +@pytest.fixture(autouse=True) +def expected_native_gap(request: pytest.FixtureRequest) -> None: + params: Final[Mapping[str, object]] = getattr(getattr(request.node, "callspec", None), "params", {}) + if params.get("rollout") is not Rollout.RUST_REQUIRED: + return + backend: Final = params.get("backend") + backend_id: Final = backend.type.value if isinstance(backend, Backend) else None + reason: Final = NATIVE_GAPS.get((request.node.originalname, backend_id)) + if reason is not None: + request.applymarker(pytest.mark.xfail(strict=True, reason=f"native: {reason}")) + + @dataclass(frozen=True, slots=True) class Paths: writers: Mapping[str, Writer] @@ -110,26 +212,12 @@ async def disagreements(paths: Paths, expected: Callable[[object], object]) -> t return tuple(result for result in results if result is not None) -@pytest.mark.usefixtures("rollout") -@pytest.mark.parametrize("backend", BACKENDS) -async def test_every_writer_is_visible_to_every_reader(backend: Backend, tmp_path: Path) -> None: - facade: Final = Cache(type=backend.type, **backend.facade_config(tmp_path)) - assert await disagreements(paths(facade), lambda value: value) == () +def serve_override(*_args: object, **_kwargs: object) -> object: + return OVERRIDDEN -@pytest.mark.usefixtures("rollout") -@pytest.mark.parametrize("backend", BACKENDS) -async def test_backend_subclass_overrides_win_on_every_reader(backend: Backend, tmp_path: Path) -> None: - class Overriding(backend.cls): - def get_cache(self, key: str, **kwargs: object) -> object: - return OVERRIDDEN - - async def async_get_cache(self, key: str, **kwargs: object) -> object: - return OVERRIDDEN - - facade: Final = Cache(type=backend.type, **backend.facade_config(tmp_path)) - facade.cache = Overriding(**backend.config(tmp_path)) - assert await disagreements(paths(facade), lambda _value: OVERRIDDEN) == () +async def async_serve_override(*_args: object, **_kwargs: object) -> object: + return OVERRIDDEN def is_wrapped(response: object) -> bool: @@ -142,9 +230,61 @@ def wrap_response(entry: object) -> object: return {**entry, "response": {"wrapped": entry["response"]}} +def test_every_cache_type_has_a_contract_row_or_a_recorded_exclusion() -> None: + covered: Final = {backend.type for backend in BACKENDS} + assert covered.isdisjoint(EXCLUDED_BACKENDS) + assert covered | EXCLUDED_BACKENDS.keys() == set(LiteLLMCacheType) + + @pytest.mark.usefixtures("rollout") -@pytest.mark.parametrize("backend", BACKENDS) -async def test_backend_subclass_super_calls_reach_the_shared_store(backend: Backend, tmp_path: Path) -> None: +@pytest.mark.parametrize("backend", BACKEND_PARAMS) +async def test_every_writer_is_visible_to_every_reader(backend: Backend, request: pytest.FixtureRequest) -> None: + facade: Final = Cache(type=backend.type, **backend.settings(request)) + assert await disagreements(paths(facade), lambda value: value) == () + + +@pytest.mark.usefixtures("rollout") +@pytest.mark.parametrize("backend", BACKEND_PARAMS) +async def test_cache_subclass_overrides_run_on_every_read_and_super_reaches_the_store( + backend: Backend, request: pytest.FixtureRequest +) -> None: + reads: Final[list[str]] = [] # mutable-ok: the override records each read it serves + + class AuditedCache(Cache): + def get_cache(self, *args: object, **kwargs: object) -> object: + reads.append("sync") + return super().get_cache(*args, **kwargs) + + async def async_get_cache(self, *args: object, **kwargs: object) -> object: + reads.append("async") + return await super().async_get_cache(*args, **kwargs) + + facade: Final = AuditedCache(type=backend.type, **backend.settings(request)) + contract: Final = paths(facade) + assert await disagreements(contract, lambda value: value) == () + assert len(reads) == len(contract.writers) * len(contract.readers) + + +@pytest.mark.usefixtures("rollout") +@pytest.mark.parametrize("backend", BACKEND_PARAMS) +async def test_backend_subclass_overrides_win_on_every_reader(backend: Backend, request: pytest.FixtureRequest) -> None: + class Overriding(backend.cls): + def get_cache(self, key: str, **kwargs: object) -> object: + return {"timestamp": 0, "response": OVERRIDDEN} + + async def async_get_cache(self, key: str, **kwargs: object) -> object: + return {"timestamp": 0, "response": OVERRIDDEN} + + facade: Final = Cache(type=backend.type, **backend.settings(request)) + facade.cache = Overriding(**backend.backend_settings(request)) + assert await disagreements(paths(facade), lambda _value: OVERRIDDEN) == () + + +@pytest.mark.usefixtures("rollout") +@pytest.mark.parametrize("backend", BACKEND_PARAMS) +async def test_backend_subclass_super_calls_reach_the_shared_store( + backend: Backend, request: pytest.FixtureRequest +) -> None: class Wrapping(backend.cls): def set_cache(self, key: str, value: object, **kwargs: object) -> None: super().set_cache(key, wrap_response(value), **kwargs) @@ -152,13 +292,13 @@ async def test_backend_subclass_super_calls_reach_the_shared_store(backend: Back async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None: await super().async_set_cache(key, wrap_response(value), **kwargs) - facade: Final = Cache(type=backend.type, **backend.facade_config(tmp_path)) - facade.cache = Wrapping(**backend.config(tmp_path)) + facade: Final = Cache(type=backend.type, **backend.settings(request)) + facade.cache = Wrapping(**backend.backend_settings(request)) assert await disagreements(paths(facade), lambda value: {"wrapped": value}) == () @pytest.mark.usefixtures("rollout") -async def test_custom_cache_objects_serve_every_path() -> None: +async def test_custom_storage_backend_serves_every_path() -> None: class DictCache(BaseCache): def __init__(self) -> None: super().__init__() @@ -179,9 +319,104 @@ async def test_custom_cache_objects_serve_every_path() -> None: async def async_set_cache_pipeline(self, cache_list: object, **kwargs: object) -> None: raise NotImplementedError + storage: Final = DictCache() facade: Final = Cache(type=LiteLLMCacheType.LOCAL) - facade.cache = DictCache() - assert await disagreements(paths(facade), lambda value: value) == () + facade.cache = storage + contract: Final = paths(facade) + assert await disagreements(contract, lambda value: value) == () + assert len(storage.entries) == len(contract.writers) * len(contract.readers) + + +@pytest.mark.usefixtures("rollout") +async def test_patched_public_methods_win_on_every_reader() -> None: + facade: Final = Cache(type=LiteLLMCacheType.LOCAL) + with ( + patch.object(Cache, "get_cache", serve_override), + patch.object(Cache, "async_get_cache", async_serve_override), + ): + assert await disagreements(paths(facade), lambda _value: OVERRIDDEN) == () + + +@pytest.mark.usefixtures("rollout") +async def test_per_call_storage_object_replaces_the_configured_backend() -> None: + facade: Final = Cache(type=LiteLLMCacheType.LOCAL) + override: Final = InMemoryCache() + + await facade.async_add_cache({"source": "override"}, cache_key="key", dynamic_cache_object=override) + + assert facade.get_cache(cache_key="key", dynamic_cache_object=override) == {"source": "override"} + assert await facade.async_get_cache(cache_key="key", dynamic_cache_object=override) == {"source": "override"} + assert facade.get_cache(cache_key="key") is None + assert await facade.async_get_cache(cache_key="key") is None + + +def reply(response: object) -> object: + assert isinstance(response, ModelResponse) + choice: Final = response.choices[0] + assert isinstance(choice, Choices) + return choice.message.content + + +@pytest.mark.usefixtures("rollout") +def test_reassigning_the_global_cache_takes_effect_on_the_next_call() -> None: + messages: Final = [{"role": "user", "content": f"contract {uuid4().hex}"}] + + def call(mock: str) -> object: + return reply(litellm.completion(model="gpt-4o-mini", messages=messages, mock_response=mock, caching=True)) + + litellm.enable_cache(type=LiteLLMCacheType.LOCAL) + original: Final = litellm.cache + replacement: Final = Cache(type=LiteLLMCacheType.LOCAL) + stored: Final = (call("first"), call("second")) + litellm.cache = replacement # test-quality-ok: reassignment is the behavior under test; conftest restores it + replaced: Final = (call("third"), call("fourth")) + litellm.disable_cache() + disabled: Final = (call("fifth"), call("sixth")) + litellm.cache = original # test-quality-ok: reassignment is the behavior under test; conftest restores it + restored: Final = call("seventh") + + assert stored == ("first", "first") + assert replaced == ("third", "third") + assert disabled == ("fifth", "sixth") + assert restored == "first" + + +FORK_SCRIPT: Final = """ +import os +import sys +from litellm.caching.caching import Cache +from litellm.rust_bridge import catalog +from litellm.rust_bridge.catalog import CacheRule +from litellm.rust_bridge.configuration import Rollout + +catalog.RULES = (CacheRule(Rollout[sys.argv[1]]),) +cache = Cache(type="local") +cache.add_cache({"answer": "parent"}, cache_key="parent-key") +read_fd, write_fd = os.pipe() +child = os.fork() +if child == 0: + os.close(read_fd) + cache.add_cache({"answer": "child"}, cache_key="child-key") + seen = (cache.get_cache(cache_key="parent-key"), cache.get_cache(cache_key="child-key")) + os.write(write_fd, repr(seen).encode()) + os._exit(0) +os.close(write_fd) +print(os.read(read_fd, 4096).decode()) +os.waitpid(child, 0) +""" + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires fork") +def test_a_forked_child_reads_inherited_entries_and_writes_its_own(rollout: Rollout) -> None: + completed: Final = subprocess.run( + [sys.executable, "-P", "-c", FORK_SCRIPT, rollout.name], + capture_output=True, + text=True, + timeout=30, + check=True, + env={**os.environ, "LITELLM_RUST": "1", "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, + ) + assert completed.stdout.strip() == "({'answer': 'parent'}, {'answer': 'child'})" @pytest.mark.usefixtures("rollout")