From 6e8aa3d21318c410b1f5559745f65529a2dd41c2 Mon Sep 17 00:00:00 2001 From: Chenglun Hu Date: Wed, 10 Jun 2026 07:30:22 +0800 Subject: [PATCH 1/2] fix(parallel_request_limiter_v3): opt-in hash-tag grouping for non-OSS-Cluster Redis (#30065) Azure Redis Enterprise / Azure Managed Redis with the EnterpriseCluster clustering policy enforce cross-slot restrictions but expose a single endpoint and don't use the OSS Cluster protocol, so _is_redis_cluster() returns False for them. Batched EVALSHA pipelines then fail with 'CROSSSLOT Keys in request don't hash to the same slot'. Users on those backends can now set litellm.force_redis_hash_tag_grouping = True to route through the slot-grouping path even though the cache isn't a RedisClusterCache. Default is False so the standalone-Redis path is unchanged. Fixes #30065 --- .../hooks/parallel_request_limiter_v3.py | 24 +++++++- .../hooks/test_parallel_request_limiter_v3.py | 57 +++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 22ea9fe176a..beb23be627a 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -565,6 +565,23 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self.internal_usage_cache.dual_cache.redis_cache, RedisClusterCache ) + @staticmethod + def _force_hash_tag_grouping_enabled() -> bool: + """ + Some non-OSS-Cluster Redis backends (e.g. Azure Redis Enterprise / + Azure Managed Redis with the EnterpriseCluster clustering policy) + enforce cross-slot restrictions but expose a single endpoint and + don't use the OSS Cluster protocol, so ``_is_redis_cluster()`` is + ``False`` for them. Without hash-tag grouping, batched EVALSHA + pipelines fail with ``CROSSSLOT Keys in request don't hash to the + same slot``. Users on those backends can set + ``litellm.force_redis_hash_tag_grouping = True`` to opt into the + slot-grouping path. + """ + import litellm + + return bool(getattr(litellm, "force_redis_hash_tag_grouping", False)) + async def in_memory_cache_sliding_window( self, keys: List[str], @@ -730,8 +747,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): """ groups: Dict[str, List[str]] = {} - # Use slot calculation for Redis clusters only - if self._is_redis_cluster(): + # Use slot calculation for Redis clusters, or for non-OSS-Cluster + # backends that still enforce cross-slot restrictions (Azure Redis + # Enterprise with EnterpriseCluster policy), opted into via + # ``litellm.force_redis_hash_tag_grouping = True``. + if self._is_redis_cluster() or self._force_hash_tag_grouping_enabled(): for key in keys: slot = self.keyslot_for_redis_cluster(key) slot_key = f"slot_{slot}" diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index c76e1a60afd..fe48d0985d3 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -1860,6 +1860,63 @@ def test_group_keys_by_hash_tag_redis_cluster(): ), "All keys should be present in groups" +def test_group_keys_by_hash_tag_force_grouping_flag(): + """ + Regression for #30065. Non-OSS-Cluster Redis backends (e.g. Azure Redis + Enterprise with EnterpriseCluster policy) enforce cross-slot + restrictions but ``_is_redis_cluster()`` returns False for them, so + batched EVALSHA pipelines fail with CROSSSLOT. Users opt into + slot-grouping via ``litellm.force_redis_hash_tag_grouping = True``. + """ + import litellm + + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + keys_two_slots = [ + "{REDACTED}:max_parallel_requests", + "{team:1f44b322-dead-41d9-9244-09515df45269}:tokens", + ] + + # Default: no flag, non-cluster cache -> single group (current behaviour). + groups_default = handler._group_keys_by_hash_tag(keys_two_slots) + assert len(groups_default) == 1 + assert "all_keys" in groups_default + + # With the opt-in flag, two different hash tags must land in two + # distinct slot groups even though _is_redis_cluster() is False. + setattr(litellm, "force_redis_hash_tag_grouping", True) + try: + groups_forced = handler._group_keys_by_hash_tag(keys_two_slots) + finally: + setattr(litellm, "force_redis_hash_tag_grouping", False) + + assert all(g.startswith("slot_") for g in groups_forced) + assert ( + len(groups_forced) == 2 + ), f"Two different hash tags must be in two slot groups, got {groups_forced}" + + +def test_force_hash_tag_grouping_enabled_default_false(): + """ + Without the opt-in flag set, ``_force_hash_tag_grouping_enabled`` is + False so the non-cluster code path is unchanged for users on plain + Redis / standalone deployments. + """ + import litellm + + # Defensive: clear any prior test leakage. + setattr(litellm, "force_redis_hash_tag_grouping", False) + + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()) + ) + + assert handler._force_hash_tag_grouping_enabled() is False + + def test_keyslot_for_redis_cluster(): """ Test the keyslot calculation for Redis cluster. From 12dce42b35e1659a37e4f126197bcc1c9ea7c210 Mon Sep 17 00:00:00 2001 From: Chenglun Hu Date: Wed, 10 Jun 2026 09:27:48 +0800 Subject: [PATCH 2/2] test(parallel_request_limiter_v3): clear force_redis_hash_tag_grouping flag before default-behaviour assertion Greptile flagged the default-behaviour assertion as fragile under test ordering: another test that sets litellm.force_redis_hash_tag_grouping and exits without clearing would carry the flag into this test's default-path check. Reset the flag explicitly at the top. --- .../proxy/hooks/test_parallel_request_limiter_v3.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index fe48d0985d3..09c80bd61bb 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -1880,6 +1880,9 @@ def test_group_keys_by_hash_tag_force_grouping_flag(): "{team:1f44b322-dead-41d9-9244-09515df45269}:tokens", ] + # Defensive: clear any prior test leakage before asserting default behaviour. + setattr(litellm, "force_redis_hash_tag_grouping", False) + # Default: no flag, non-cluster cache -> single group (current behaviour). groups_default = handler._group_keys_by_hash_tag(keys_two_slots) assert len(groups_default) == 1