import asyncio from collections.abc import Callable from pathlib import Path from types import SimpleNamespace from typing import Final, TypeAlias, cast from urllib.parse import urlparse from uuid import uuid4 import pytest from litellm.caching.caching import Cache from litellm.rust_bridge.response_cache import NativeResponseCacheRuntime, ResponseCacheRuntime, resolve_response_cache from litellm.types.caching import LiteLLMCacheType from litellm.types.utils import EmbeddingResponse from tests.test_litellm_rust.support.cache import assert_native_runtime, completion_kwargs, require_rust from tests.test_litellm_rust.support.s3_stub import S3Stub pytestmark: Final = pytest.mark.requires_rust_extension CacheFactory: TypeAlias = Callable[[], Cache] @pytest.fixture def cache_factory(request: pytest.FixtureRequest, tmp_path: Path) -> CacheFactory: backend: Final = cast(LiteLLMCacheType, request.param) match backend: case LiteLLMCacheType.LOCAL: return lambda: Cache(type=backend) case LiteLLMCacheType.DISK: return lambda: Cache(type=backend, disk_cache_dir=str(tmp_path)) case LiteLLMCacheType.REDIS: parsed: Final = urlparse(cast(str, request.getfixturevalue("redis_url"))) return lambda: Cache(type=backend, host=parsed.hostname, port=str(parsed.port)) case LiteLLMCacheType.S3: stub: Final = cast(S3Stub, request.getfixturevalue("s3_stub")) return lambda: Cache( type=backend, s3_bucket_name="cache-bucket", s3_region_name="us-east-1", s3_endpoint_url=stub.url, s3_aws_access_key_id="key", s3_aws_secret_access_key="secret", s3_path="team", ) case LiteLLMCacheType.GCS: return lambda: Cache(type=backend, gcs_bucket_name="bucket", gcs_path="cache/") case LiteLLMCacheType.REDIS_SEMANTIC: return lambda: Cache( type=backend, redis_url="redis://127.0.0.1:6379", similarity_threshold=0.8, redis_semantic_cache_embedding_model="text-embedding-3-small", ) case LiteLLMCacheType.VALKEY_SEMANTIC: return lambda: Cache(type=backend, redis_url="redis://127.0.0.1:6390/0", similarity_threshold=0.8) case _: raise AssertionError(f"no local factory for {backend}") ROUND_TRIP_BACKENDS: Final = ( LiteLLMCacheType.LOCAL, LiteLLMCacheType.DISK, LiteLLMCacheType.REDIS, LiteLLMCacheType.S3, ) SHARED_STORE_BACKENDS: Final = (LiteLLMCacheType.DISK, LiteLLMCacheType.REDIS, LiteLLMCacheType.S3) @pytest.mark.parametrize("backend", list(LiteLLMCacheType)) def test_shipped_rules_keep_every_backend_on_python(backend: LiteLLMCacheType) -> None: assert resolve_response_cache(cast(Cache, SimpleNamespace(type=backend))) is None @pytest.mark.parametrize( "cache_factory", [ LiteLLMCacheType.LOCAL, LiteLLMCacheType.DISK, LiteLLMCacheType.REDIS, LiteLLMCacheType.S3, LiteLLMCacheType.GCS, LiteLLMCacheType.REDIS_SEMANTIC, LiteLLMCacheType.VALKEY_SEMANTIC, ], indirect=True, ) def test_shipped_rules_construct_python_backed_facades(cache_factory: CacheFactory) -> None: assert cache_factory()._native_cache is None # pyright: ignore[reportPrivateUsage] # the activation under test has no public accessor @pytest.mark.parametrize( "cache_factory", [ LiteLLMCacheType.LOCAL, LiteLLMCacheType.DISK, LiteLLMCacheType.REDIS, LiteLLMCacheType.S3, LiteLLMCacheType.GCS, LiteLLMCacheType.REDIS_SEMANTIC, LiteLLMCacheType.VALKEY_SEMANTIC, ], indirect=True, ) def test_rust_required_rule_activates_the_native_backend( cache_factory: CacheFactory, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest ) -> None: require_rust(monkeypatch, cast(LiteLLMCacheType, request.node.callspec.params["cache_factory"])) assert_native_runtime(cache_factory()) @pytest.mark.parametrize("cache_factory", ROUND_TRIP_BACKENDS, indirect=True) async def test_facade_storage_calls_round_trip_through_the_native_backend( cache_factory: CacheFactory, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest ) -> None: require_rust(monkeypatch, cast(LiteLLMCacheType, request.node.callspec.params["cache_factory"])) facade: Final = cache_factory() assert_native_runtime(facade) sync_kwargs: Final = completion_kwargs("sync") facade.add_cache({"answer": 1}, **sync_kwargs) assert facade.get_cache(**sync_kwargs) == {"answer": 1} async_kwargs: Final = completion_kwargs("async") await facade.async_add_cache({"answer": 2}, **async_kwargs) assert await facade.async_get_cache(**async_kwargs) == {"answer": 2} assert facade.get_cache(**completion_kwargs("absent")) is None async def test_memory_facade_writes_bypass_the_python_backend(monkeypatch: pytest.MonkeyPatch) -> None: require_rust(monkeypatch, LiteLLMCacheType.LOCAL) facade: Final = Cache(type=LiteLLMCacheType.LOCAL) assert_native_runtime(facade) kwargs: Final = completion_kwargs("memory") facade.add_cache({"answer": 1}, **kwargs) assert facade.cache.get_cache(facade.get_cache_key(**kwargs)) is None assert facade.get_cache(**kwargs) == {"answer": 1} @pytest.mark.parametrize("cache_factory", SHARED_STORE_BACKENDS, indirect=True) async def test_native_and_python_facades_share_one_wire_format( cache_factory: CacheFactory, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest ) -> None: python_facade: Final = cache_factory() assert python_facade._native_cache is None # pyright: ignore[reportPrivateUsage] # the activation under test has no public accessor require_rust(monkeypatch, cast(LiteLLMCacheType, request.node.callspec.params["cache_factory"])) native_facade: Final = cache_factory() assert_native_runtime(native_facade) native_written: Final = completion_kwargs("native") native_facade.add_cache({"writer": "native"}, **native_written) assert python_facade.get_cache(**native_written) == {"writer": "native"} python_written: Final = completion_kwargs("python") python_facade.add_cache({"writer": "python"}, **python_written) assert native_facade.get_cache(**python_written) == {"writer": "python"} async_native: Final = completion_kwargs("async-native") await native_facade.async_add_cache({"writer": "async-native"}, **async_native) assert await python_facade.async_get_cache(**async_native) == {"writer": "async-native"} async_python: Final = completion_kwargs("async-python") await python_facade.async_add_cache({"writer": "async-python"}, **async_python) assert await native_facade.async_get_cache(**async_python) == {"writer": "async-python"} @pytest.mark.parametrize("cache_factory", ROUND_TRIP_BACKENDS, indirect=True) async def test_embedding_pipeline_stores_one_native_entry_per_input( cache_factory: CacheFactory, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest ) -> None: require_rust(monkeypatch, cast(LiteLLMCacheType, request.node.callspec.params["cache_factory"])) facade: Final = cache_factory() assert_native_runtime(facade) inputs: Final = [f"alpha {uuid4().hex}", f"beta {uuid4().hex}"] result: Final = EmbeddingResponse( model="text-embedding-3-small", data=[ {"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}, {"object": "embedding", "index": 1, "embedding": [0.3, 0.4]}, ], ) await facade.async_add_cache_pipeline(result, model="text-embedding-3-small", input=inputs) keys: Final = [facade.get_cache_key(model="text-embedding-3-small", input=text) for text in inputs] assert len(set(keys)) == len(inputs) for text, expected in zip(inputs, ([0.1, 0.2], [0.3, 0.4]), strict=True): cached = await facade.async_get_cache(model="text-embedding-3-small", input=text) assert isinstance(cached, dict) assert cached["embedding"] == expected assert await facade.async_get_cache(model="text-embedding-3-small", input=inputs) is None @pytest.mark.parametrize( ("backend", "settings", "message"), [ pytest.param( LiteLLMCacheType.VALKEY_SEMANTIC, {"redis_url": "rediss://127.0.0.1:6390/0", "similarity_threshold": 0.8}, "native Valkey semantic cache does not support TLS connections", id="valkey-tls", ), pytest.param( LiteLLMCacheType.VALKEY_SEMANTIC, {"redis_url": "redis://127.0.0.1:6390/0?socket_timeout=1", "similarity_threshold": 0.8}, "native Redis uses fixed socket timeouts; socket_timeout and socket_connect_timeout require Python", id="valkey-socket-timeout", ), pytest.param( LiteLLMCacheType.REDIS_SEMANTIC, {"redis_url": "rediss://127.0.0.1:6380", "similarity_threshold": 0.8}, "native Redis semantic cache does not support TLS or query options in redis_url", id="redis-semantic-tls", ), pytest.param( LiteLLMCacheType.REDIS_SEMANTIC, {"redis_url": "redis://127.0.0.1:6379?socket_timeout=1", "similarity_threshold": 0.8}, "native Redis semantic cache does not support TLS or query options in redis_url", id="redis-semantic-query", ), ], ) def test_semantic_settings_the_native_client_cannot_honor_decline( monkeypatch: pytest.MonkeyPatch, backend: LiteLLMCacheType, settings: dict[str, object], message: str ) -> None: require_rust(monkeypatch, backend) with pytest.raises(RuntimeError, match=f"declined the cache: {message}"): Cache(type=backend, **settings) class _SemanticHit: """A native semantic runtime that answers every lookup with one cached response.""" kind: Final = "native" def lookup_semantic(self, request: object) -> tuple[object, float | None]: return {"answer": 42}, 0.97 async def async_lookup_semantic(self, request: object) -> tuple[object, float | None]: return {"answer": 42}, 0.97 @pytest.mark.parametrize("semantic_type", [LiteLLMCacheType.QDRANT_SEMANTIC, LiteLLMCacheType.REDIS_SEMANTIC]) @pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) def test_native_semantic_hit_stamps_similarity_on_request_metadata( semantic_type: LiteLLMCacheType, use_async: bool ) -> None: """Python semantic backends write `metadata["semantic-similarity"]` on every lookup, and the facade copies it to the caller's metadata; the native path must report it the same way.""" facade: Final = Cache() facade.type = semantic_type facade._native_cache = ResponseCacheRuntime(cast(NativeResponseCacheRuntime, _SemanticHit())) # pyright: ignore[reportPrivateUsage] # the native path under test has no public setter metadata: Final[dict[str, object]] = {} kwargs: Final = { "cache_key": "semantic-key", "messages": [{"role": "user", "content": "hello"}], "metadata": metadata, } result: Final = asyncio.run(facade.async_get_cache(**kwargs)) if use_async else facade.get_cache(**kwargs) assert result == {"answer": 42} assert metadata["semantic-similarity"] == 0.97