mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
test: support clustered provider caches with live fallback
This commit is contained in:
parent
7ce358175c
commit
d55bcbde1f
3 changed files with 49 additions and 9 deletions
|
|
@ -377,7 +377,11 @@ def test_stream_completion_controls_publication(store: RedisResponseStore, provi
|
|||
assert len(provider.hits) == (2 if truncated else 1)
|
||||
|
||||
|
||||
def test_store_outage_preserves_provider_success(provider: Provider) -> None:
|
||||
@pytest.mark.parametrize("cluster_mode", ("0", "1"))
|
||||
def test_store_outage_preserves_provider_success(
|
||||
provider: Provider, monkeypatch: pytest.MonkeyPatch, cluster_mode: str,
|
||||
) -> None:
|
||||
monkeypatch.setenv("E2E_PROVIDER_CACHE_REDIS_CLUSTER", cluster_mode)
|
||||
with socket.socket() as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
port: Final = probe.getsockname()[1]
|
||||
|
|
@ -401,6 +405,20 @@ def test_old_lease_cannot_overwrite_new_owner(store: RedisResponseStore) -> None
|
|||
assert isinstance(hit, CacheHit) and hit.payload == b"new"
|
||||
|
||||
|
||||
def test_separate_namespaces_keep_their_own_response(store: RedisResponseStore) -> None:
|
||||
other: Final = replace(store, namespace=store.namespace + "-other")
|
||||
first_lease: Final = store.lookup("shared-key")
|
||||
assert isinstance(first_lease, CaptureLease)
|
||||
assert store.publish("shared-key", first_lease, b"first namespace")
|
||||
other_lease: Final = other.lookup("shared-key")
|
||||
assert isinstance(other_lease, CaptureLease)
|
||||
assert other.publish("shared-key", other_lease, b"other namespace")
|
||||
first_hit: Final = store.lookup("shared-key")
|
||||
other_hit: Final = other.lookup("shared-key")
|
||||
assert isinstance(first_hit, CacheHit) and first_hit.payload == b"first namespace"
|
||||
assert isinstance(other_hit, CacheHit) and other_hit.payload == b"other namespace"
|
||||
|
||||
|
||||
def test_identity_preserves_values_and_never_contains_credentials() -> None:
|
||||
variants: Final = (b'{}', b'{"a":null}', b'{"a":false}', b'{"a":0}', b'{"a":0.0}', b'{"a":"0"}', b' { }', None, b'')
|
||||
url: Final = "https://example.invalid/v1/chat/completions"
|
||||
|
|
|
|||
|
|
@ -326,3 +326,7 @@ other.<area>.<case>.<assertion>
|
|||
- do not use xfail markers, tests should be written in a form that the end user expects it to pass
|
||||
|
||||
- a cell addresses the stack through its front door (`PROXY_BASE_URL`) the way a customer does, never a gateway pod by address, and proves a cross-replica property with N independent calls through that door, naming the miss odds (2^-N at two pods) in its docstring. `PROXY_REPLICA_URLS` is for read-backs that are per pod by nature (polling until a management write has converged on every replica, `/metrics`, RSS), never for steering the scenario a cell asserts on at a chosen pod
|
||||
|
||||
## Provider response cache
|
||||
|
||||
The provider response cache can share the stage Valkey instance through a `rediss://` URL. Set `E2E_PROVIDER_CACHE_REDIS_CLUSTER=1` for a cluster-aware client, including ElastiCache Serverless. Keep the dedicated HMAC key and namespace, and restrict test cleanup to keys in the test's own namespace
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ from typing import Final, Protocol, cast
|
|||
|
||||
from provider_cache import LIFETIME_SECONDS, CacheBusy, CacheEdge, CacheHit, CacheLookup, CacheUnavailable, CaptureLease
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from redis import Redis
|
||||
from redis.exceptions import RedisError
|
||||
from redis import Redis, RedisCluster
|
||||
from redis.exceptions import RedisClusterException, RedisError
|
||||
|
||||
REDIS_ARRAY: Final[TypeAdapter[list[bytes]]] = TypeAdapter(list[bytes])
|
||||
|
||||
|
|
@ -83,7 +83,7 @@ class RedisResponseStore:
|
|||
started: Final = time.monotonic()
|
||||
try:
|
||||
result: Final = self.client.eval(LOOKUP, 2, *self.keys(key), token, self.lifetime_ms, self.lease_ms)
|
||||
except (RedisError, OSError):
|
||||
except (RedisError, RedisClusterException, OSError):
|
||||
return CacheUnavailable()
|
||||
try:
|
||||
parts: Final = tuple(REDIS_ARRAY.validate_python(result, strict=True))
|
||||
|
|
@ -102,28 +102,46 @@ class RedisResponseStore:
|
|||
result: Final = self.client.eval(
|
||||
PUBLISH, 2, *self.keys(key), lease.token, lease.captured_at_ms, lease.expires_at_ms, payload, self.lifetime_ms,
|
||||
)
|
||||
except (RedisError, OSError):
|
||||
except (RedisError, RedisClusterException, OSError):
|
||||
return False
|
||||
return result == 1
|
||||
|
||||
def release(self, key: str, lease: CaptureLease) -> bool:
|
||||
try:
|
||||
result: Final = self.client.eval(RELEASE, 1, self.keys(key)[1], lease.token)
|
||||
except (RedisError, OSError):
|
||||
except (RedisError, RedisClusterException, OSError):
|
||||
return False
|
||||
return result == 1
|
||||
|
||||
def discard(self, key: str, payload: bytes) -> bool:
|
||||
try:
|
||||
result: Final = self.client.eval(DISCARD, 1, self.keys(key)[0], payload)
|
||||
except (RedisError, OSError):
|
||||
except (RedisError, RedisClusterException, OSError):
|
||||
return False
|
||||
return result == 1
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=16)
|
||||
def _redis_commands(url: str, cluster: bool) -> RedisCommands:
|
||||
if cluster:
|
||||
return cast(
|
||||
RedisCommands, RedisCluster(url=url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)
|
||||
)
|
||||
return cast(RedisCommands, Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LazyRedisCommands:
|
||||
url: str
|
||||
cluster: bool
|
||||
|
||||
def eval(self, script: str, numkeys: int, *args: str | bytes | int) -> object:
|
||||
return _redis_commands(self.url, self.cluster).eval(script, numkeys, *args)
|
||||
|
||||
|
||||
def redis_store(url: str, namespace: str) -> RedisResponseStore:
|
||||
client: Final = Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)
|
||||
return RedisResponseStore(cast(RedisCommands, client), namespace)
|
||||
client: Final = LazyRedisCommands(url, os.environ.get("E2E_PROVIDER_CACHE_REDIS_CLUSTER") == "1")
|
||||
return RedisResponseStore(client, namespace)
|
||||
|
||||
|
||||
def write_metrics(cache: CacheEdge) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue