fix(realtime): enforce multipart authorization and cluster call leases

This commit is contained in:
jibanez-staticduo 2026-09-11 10:20:26 +02:00
parent 34dcfbf833
commit 464c1eb2bc
No known key found for this signature in database
10 changed files with 662 additions and 111 deletions

View file

@ -1865,15 +1865,26 @@ def _extract_model_candidates_from_request(
uses_completion_model_sources: Final = _route_matches_any_marker(
route=route, markers=_MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS
)
session: Final[object] = (
request_data.get("session")
if _route_matches_any_marker(route=route, markers=_MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS)
else None
)
parsed_session: Final[object] = safe_json_loads(session) if isinstance(session, str) else session
session_model: Final[object] = parsed_session.get("model") if isinstance(parsed_session, dict) else None
if (
_route_matches_any_marker(route=route, markers=("/realtime/calls",))
and isinstance(session_model, str)
and session_model
):
return [session_model]
body_model: Final = request_data.get("model")
_append_model_candidates(candidates, body_model)
if uses_body_target_model_sources or not body_model:
_append_model_candidates(candidates, request_data.get("target_model_names"))
if _route_matches_any_marker(route=route, markers=_MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS):
session: Final = request_data.get("session")
if isinstance(session, dict):
_append_model_candidates(candidates, session.get("model"))
_append_model_candidates(candidates, session_model)
if uses_completion_model_sources and isinstance(request_data.get("completion"), dict):
_append_model_candidates(candidates, request_data["completion"].get("model"))

View file

@ -446,7 +446,7 @@ def _without_parallel_limit(descriptor: RateLimitDescriptor) -> RateLimitDescrip
class ParallelSlotAcquisition(TypedDict):
slot_id: str
counter_keys: list[str]
counter_keys: Sequence[str]
class RateLimitStatus(TypedDict):
@ -1038,7 +1038,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
async def in_memory_cache_sliding_window(
self,
keys: list[str],
keys: Sequence[str],
now_int: int,
window_size: int,
) -> CacheCounterValues:
@ -1192,7 +1192,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
crc: Final = binascii.crc_hqx(key.encode("utf-8"), 0)
return crc % REDIS_CLUSTER_SLOTS
def _group_keys_by_hash_tag(self, keys: list[str]) -> dict[str, list[str]]:
def _group_keys_by_hash_tag(self, keys: Sequence[str]) -> Mapping[str, Sequence[str]]:
"""
Group keys by their Redis hash tag to ensure cluster compatibility.
@ -1212,7 +1212,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
groups[slot_key].append(key)
else:
# For regular Redis, no grouping needed - process all keys together
groups[REDIS_NODE_HASHTAG_NAME] = keys
return MappingProxyType({REDIS_NODE_HASHTAG_NAME: keys})
return groups
@ -1508,12 +1508,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
"""
gauge_keys: Final = [gauge["counter_key"] for gauge in gauges]
if self._is_redis_cluster() and self.parallel_acquire_script is not None:
return await self._check_cluster_parallel_gauges(gauges, slot_id, parent_otel_span, read_only)
if read_only:
if self.parallel_count_script is not None:
try:
raw_counts: Final[list[CacheCounterValue]] = await self.parallel_count_script(
keys=gauge_keys,
args=[PARALLEL_REQUEST_SLOT_TTL_SECONDS for _ in gauges],
args=tuple(PARALLEL_REQUEST_SLOT_TTL_SECONDS for _ in gauges),
)
counts = [max(0, int(value)) for value in raw_counts]
except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to the local mirror, never a 500
@ -1571,6 +1574,133 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
async with self._check_and_increment_lock:
return await self._acquire_parallel_slots_in_memory(gauges, slot_id, parent_otel_span)
async def _check_cluster_parallel_gauges(
self,
gauges: Sequence[ParallelRequestGauge],
slot_id: str,
parent_otel_span: Span | None,
read_only: bool,
) -> RateLimitResponse:
by_key: Final = MappingProxyType(
{
gauge["counter_key"]: min(
(candidate for candidate in gauges if candidate["counter_key"] == gauge["counter_key"]),
key=lambda candidate: candidate["limit"],
)
for gauge in gauges
}
)
groups: Final = self._group_keys_by_hash_tag(tuple(by_key))
counts: Final[dict[str, int]] = {} # mutable-ok: gather independent Redis-slot results
attempted: Final[list[str]] = [] # mutable-ok: rollback includes requests whose responses were lost
try:
for keys in groups.values():
if read_only:
if self.parallel_count_script is None:
raise RuntimeError("Redis cluster parallel count script is unavailable")
counts.update(
(key, max(0, int(count)))
for key, count in zip(
keys,
await self.parallel_count_script(
keys=keys, args=tuple(PARALLEL_REQUEST_SLOT_TTL_SECONDS for _ in keys)
),
strict=True,
)
)
continue
if self.parallel_acquire_script is None:
raise RuntimeError("Redis cluster parallel acquire script is unavailable")
attempted.extend(keys)
(raw,) = (
await self.parallel_acquire_script(
keys=keys,
args=tuple(
arg
for key in keys
for arg in (by_key[key]["limit"], PARALLEL_REQUEST_SLOT_TTL_SECONDS, slot_id)
),
),
)
if int(raw[0]) == 1:
await self._rollback_cluster_parallel_slots(tuple(attempted), slot_id, parent_otel_span)
return RateLimitResponse(
overall_code="OVER_LIMIT",
statuses=[self._gauge_status(by_key[keys[int(raw[1]) - 1]], int(raw[2]), "OVER_LIMIT")],
)
counts.update((key, int(count)) for key, count in zip(keys, raw[1:], strict=True))
for key in keys:
await self.internal_usage_cache.async_set_cache(
key=key,
value=counts[key],
ttl=PARALLEL_REQUEST_SLOT_TTL_SECONDS,
litellm_parent_otel_span=parent_otel_span,
local_only=True,
)
except BaseException:
if attempted:
await self._rollback_cluster_parallel_slots(tuple(attempted), slot_id, parent_otel_span)
raise
statuses: Final = tuple(
self._gauge_status(
gauge,
counts[gauge["counter_key"]],
"OVER_LIMIT" if read_only and counts[gauge["counter_key"]] >= gauge["limit"] else "OK",
)
for gauge in gauges
)
return RateLimitResponse(
overall_code="OVER_LIMIT" if any(item["code"] == "OVER_LIMIT" for item in statuses) else "OK",
statuses=list(statuses),
)
async def _rollback_cluster_parallel_slots(
self, counter_keys: tuple[str, ...], slot_id: str, parent_otel_span: Span | None
) -> None:
rollback: Final = asyncio.create_task(
self._release_cluster_parallel_slots(counter_keys, slot_id, parent_otel_span)
)
cancelled = False # rebind-ok: defer repeated caller cancellation until compensation finishes
while not rollback.done():
try:
await asyncio.shield(rollback)
except asyncio.CancelledError:
cancelled = True
except Exception: # noqa: BLE001 # retrieve and report the completed task's exception below
break
try:
rollback.result()
except Exception: # noqa: BLE001 # preserve admission failure; unreachable Redis slots expire by TTL
verbose_proxy_logger.error("Could not roll back all Redis cluster parallel request slots")
if cancelled:
raise asyncio.CancelledError
async def _release_cluster_parallel_slots(
self, counter_keys: tuple[str, ...], slot_id: str, parent_otel_span: Span | None
) -> None:
first_error: Exception | None = None # rebind-ok: finish every shard before reporting the first failure
for keys in self._group_keys_by_hash_tag(counter_keys).values():
try:
if self.parallel_release_script is None:
raise RuntimeError("Redis cluster parallel release script is unavailable")
for key, count in zip(
keys,
await self.parallel_release_script(keys=keys, args=tuple(slot_id for _ in keys)),
strict=True,
):
await self.internal_usage_cache.async_set_cache(
key=key,
value=max(0, int(count)),
ttl=PARALLEL_REQUEST_SLOT_TTL_SECONDS,
litellm_parent_otel_span=parent_otel_span,
local_only=True,
)
except Exception as exc: # noqa: BLE001 # one unreachable shard must not strand the other shards
if first_error is None:
first_error = exc
if first_error is not None:
raise first_error
async def _read_local_gauge_counts(
self,
gauge_keys: list[str],
@ -1656,7 +1786,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
async def release() -> None:
await self._release_parallel_request_slots(
ParallelSlotAcquisition(slot_id=slot_id, counter_keys=list(counter_keys))
ParallelSlotAcquisition(slot_id=slot_id, counter_keys=counter_keys)
)
return RealtimeCallLease(renew=renew, release=release)
@ -1664,10 +1794,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
async def _renew_realtime_call_slot(self, slot_id: str, counter_keys: tuple[str, ...]) -> bool:
if self.parallel_renew_script is not None:
try:
result: Final = await self.parallel_renew_script(
keys=counter_keys, args=(slot_id, PARALLEL_REQUEST_SLOT_TTL_SECONDS)
)
return tuple(result) == (1,)
for keys in self._group_keys_by_hash_tag(counter_keys).values():
if tuple(
await self.parallel_renew_script(keys=keys, args=(slot_id, PARALLEL_REQUEST_SLOT_TTL_SECONDS))
) != (1,):
return False
return True
except Exception: # noqa: BLE001 # Redis ownership cannot be established by a local count mirror
return False
async with self._check_and_increment_lock:
@ -1717,11 +1849,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
slot_id: Final = acquisition["slot_id"]
if not counter_keys or not slot_id:
return
if self._is_redis_cluster() and self.parallel_release_script is not None:
await self._release_cluster_parallel_slots(tuple(counter_keys), slot_id, parent_otel_span)
return
if self.parallel_release_script is not None:
try:
raw: Final[list[CacheCounterValue]] = await self.parallel_release_script(
keys=counter_keys,
args=[slot_id for _ in counter_keys],
args=tuple(slot_id for _ in counter_keys),
)
for counter_key, remaining in zip(counter_keys, raw):
await self.internal_usage_cache.async_set_cache(

View file

@ -93,9 +93,16 @@ class CallSupervisor:
if not self._started or self._terminal or self._task.done():
raise RuntimeError("Call observer ended before session became available")
except BaseException:
await self.close()
await self._close_after_failed_start()
raise
async def _close_after_failed_start(self) -> None:
cleanup: Final = asyncio.create_task(self.close())
while not cleanup.done():
with suppress(asyncio.CancelledError):
await asyncio.shield(cleanup)
cleanup.result()
async def close(self) -> None:
self._stop.set()
await self.wait()

View file

@ -207,7 +207,7 @@ dev = [
"opentelemetry-instrumentation-fastapi==0.49b0",
"langfuse==2.59.7",
"fastapi-offline==1.7.6",
"fakeredis==2.34.1",
"fakeredis[lua]==2.34.1",
"pytest-rerunfailures==15.1",
"pytest-cov==5.0.0",
"parameterized==0.9.0",

View file

@ -947,7 +947,8 @@ def test_get_model_from_request_handles_managed_id_decoder_failures():
"/openai/v1/realtime/calls",
],
)
def test_get_model_from_request_extracts_realtime_session_model(route):
@pytest.mark.parametrize("encoded", [False, True])
def test_get_model_from_request_extracts_realtime_session_model(route, encoded):
"""The effective realtime model lives in ``session.model`` (not the
top-level ``model``). It must be surfaced so can_key_call_model() can
validate the model a restricted key is actually requesting.
@ -957,13 +958,31 @@ def test_get_model_from_request_extracts_realtime_session_model(route):
"""
assert (
get_model_from_request(
request_data={"session": {"type": "realtime", "model": "gpt-realtime"}},
request_data={"session": '{"model":"gpt-realtime"}' if encoded else {"model": "gpt-realtime"}},
route=route,
)
== "gpt-realtime"
)
@pytest.mark.parametrize("session", ['{"model":"actual-voice"}', {"model": "actual-voice"}])
def test_realtime_calls_auth_uses_executed_session_model_despite_decoys(session):
assert (
get_model_from_request(
request_data={"model": "body-decoy", "session": session},
route="/v1/realtime/calls",
request_query_params={"model": "query-decoy"},
request_headers={"x-litellm-model": "header-decoy"},
)
== "actual-voice"
)
@pytest.mark.parametrize("session", ["invalid", "null", "[]", "12", '"text"', "{}"])
def test_realtime_model_extraction_ignores_invalid_serialized_session(session):
assert get_model_from_request(request_data={"session": session}, route="/v1/realtime/calls") is None
def test_get_model_from_request_realtime_includes_top_level_and_session_model():
"""When both top-level and session model are present, both are returned so
neither path can smuggle a disallowed model past the model-access check."""

View file

@ -3,13 +3,12 @@ Unit Tests for the max parallel request limiter v1 for the proxy
"""
import asyncio
import shutil
import socket
import subprocess
from datetime import datetime
from unittest.mock import AsyncMock, MagicMock
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import pytest_asyncio
from fakeredis import FakeAsyncRedis, FakeRedis, FakeServer
from litellm.caching.caching import DualCache
from litellm.caching.redis_cache import RedisCache
@ -22,55 +21,23 @@ from litellm.proxy.utils import InternalUsageCache, hash_token
from litellm.types.utils import EmbeddingResponse, TextCompletionResponse, Usage
@pytest.fixture
def isolated_legacy_redis(tmp_path):
executable = shutil.which("redis-server")
if executable is None:
pytest.skip("redis-server is required to exercise atomic Lua updates")
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
port = listener.getsockname()[1]
process = subprocess.Popen(
[
executable,
"--bind",
"127.0.0.1",
"--port",
str(port),
"--save",
"",
"--appendonly",
"no",
"--dir",
str(tmp_path),
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
import redis
client = redis.Redis(host="127.0.0.1", port=port)
for _ in range(100):
try:
client.ping()
break
except redis.ConnectionError:
import time
time.sleep(0.01)
else:
pytest.fail("isolated Redis did not start")
yield port
client.close()
finally:
process.terminate()
process.wait(timeout=5)
@pytest_asyncio.fixture(loop_scope="function")
async def isolated_legacy_redis():
server = FakeServer()
client = FakeRedis(server=server)
async with FakeAsyncRedis(server=server) as async_client:
with (
patch("redis.Redis", autospec=True, return_value=client),
patch("redis.asyncio.BlockingConnectionPool", autospec=True, return_value=async_client.connection_pool),
patch("redis.asyncio.Redis", autospec=True, return_value=async_client),
):
yield RedisCache(host="fake-legacy-redis", namespace="legacy-test")
client.close()
@pytest.mark.asyncio
async def test_concurrent_realtime_releases_update_redis_without_lost_decrement(isolated_legacy_redis):
remote = RedisCache(host="127.0.0.1", port=isolated_legacy_redis, namespace="legacy-test")
remote = isolated_legacy_redis
first_cache, second_cache = DualCache(redis_cache=remote), DualCache(redis_cache=remote)
first, second = (_PROXY_MaxParallelRequestsHandler(InternalUsageCache(c)) for c in (first_cache, second_cache))
auth = UserAPIKeyAuth(api_key="concurrent-key", max_parallel_requests=2)

View file

@ -134,50 +134,24 @@ async def test_realtime_lease_renewal_preserves_quota_past_ttl_and_does_not_resu
@pytest.mark.asyncio
async def test_realtime_lease_redis_renewal_is_atomic_and_does_not_resurrect():
import shutil
import subprocess
import tempfile
from redis.asyncio import Redis
from redis.exceptions import ConnectionError as RedisConnectionError
from fakeredis import FakeAsyncRedis
from litellm.proxy.hooks.parallel_request_limiter_v3 import PARALLEL_RENEW_SCRIPT
executable = shutil.which("redis-server")
if executable is None:
pytest.skip("redis-server is required for the Lua regression")
with tempfile.TemporaryDirectory(prefix="rtc-") as temporary:
socket = f"{temporary}/redis.sock"
process = subprocess.Popen(
[executable, "--port", "0", "--unixsocket", socket, "--save", "", "--appendonly", "no"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
client = Redis(unix_socket_path=socket)
try:
for attempt in range(100):
try:
await client.ping()
break
except RedisConnectionError:
await asyncio.sleep(0.01)
else:
pytest.fail("isolated Redis did not start")
now = (await client.time())[0]
await client.zadd("first", {"owner": now - 10, "other": now})
await client.zadd("second", {"owner": now - PARALLEL_REQUEST_SLOT_TTL_SECONDS})
renew = client.register_script(PARALLEL_RENEW_SCRIPT)
assert await renew(keys=["first", "second"], args=["owner", PARALLEL_REQUEST_SLOT_TTL_SECONDS]) == [0]
assert await client.zscore("first", "owner") == now - 10
await client.zadd("second", {"owner": now - 10})
assert await renew(keys=["first", "second"], args=["owner", PARALLEL_REQUEST_SLOT_TTL_SECONDS]) == [1]
assert await client.zscore("first", "owner") >= now
assert await client.ttl("first") > PARALLEL_REQUEST_SLOT_TTL_SECONDS - 10
await client.zrem("second", "owner")
assert await renew(keys=["first", "second"], args=["owner", PARALLEL_REQUEST_SLOT_TTL_SECONDS]) == [0]
assert await client.zscore("second", "owner") is None
assert await client.zscore("first", "other") == now
finally:
await client.aclose()
process.terminate()
process.wait(timeout=5)
async with FakeAsyncRedis() as client:
now = (await client.time())[0]
await client.zadd("first", {"owner": now - 10, "other": now})
await client.zadd("second", {"owner": now - PARALLEL_REQUEST_SLOT_TTL_SECONDS})
renew = client.register_script(PARALLEL_RENEW_SCRIPT)
assert await renew(keys=["first", "second"], args=["owner", PARALLEL_REQUEST_SLOT_TTL_SECONDS]) == [0]
assert await client.zscore("first", "owner") == now - 10
await client.zadd("second", {"owner": now - 10})
assert await renew(keys=["first", "second"], args=["owner", PARALLEL_REQUEST_SLOT_TTL_SECONDS]) == [1]
assert await client.zscore("first", "owner") >= now
assert await client.ttl("first") > PARALLEL_REQUEST_SLOT_TTL_SECONDS - 10
await client.zrem("second", "owner")
assert await renew(keys=["first", "second"], args=["owner", PARALLEL_REQUEST_SLOT_TTL_SECONDS]) == [0]
assert await client.zscore("second", "owner") is None
assert await client.zscore("first", "other") == now
@pytest.fixture
@ -6365,3 +6339,197 @@ async def test_post_call_success_hook_leaves_raw_provider_dict_untouched():
)
assert response == {"id": "msg_123", "type": "message", "role": "assistant", "content": []}
class _ClusterParallelTransport:
def __init__(self, handler):
self.handler = handler
self.members = {}
self.now = 10000
self.calls = []
self.fail = None
self.lose_acquire_response = None
self.pause_acquire = None
self.entered = asyncio.Event()
def script(self, operation):
async def run(*, keys, args):
self.calls.append((operation, tuple(keys)))
if len({self.handler.keyslot_for_redis_cluster(key) for key in keys}) > 1:
raise RuntimeError("CROSSSLOT Keys in request do not hash to the same slot")
if self.fail is not None and (operation, keys[0]) == self.fail:
raise RuntimeError("Shard unavailable")
if operation in ("acquire", "count"):
for key in keys:
self.members[key] = {
slot: score for slot, score in self.members.get(key, {}).items()
if score > self.now - PARALLEL_REQUEST_SLOT_TTL_SECONDS
}
if operation == "acquire":
for index, key in enumerate(keys):
if len(self.members[key]) >= args[index * 3]:
return [1, index + 1, len(self.members[key])]
for index, key in enumerate(keys):
self.members[key][args[index * 3 + 2]] = self.now
if keys[0] == self.pause_acquire:
self.entered.set()
await asyncio.Event().wait()
if keys[0] == self.lose_acquire_response:
raise RuntimeError("Reply lost after Redis admitted slot")
return [0, *(len(self.members[key]) for key in keys)]
if operation == "count":
return [len(self.members.get(key, {})) for key in keys]
if operation == "renew":
if any(self.members.get(key, {}).get(args[0], 0) <= self.now - args[1] for key in keys):
return [0]
for key in keys:
self.members[key][args[0]] = self.now
return [1]
assert operation == "release"
for index, key in enumerate(keys):
self.members.get(key, {}).pop(args[index], None)
return [len(self.members.get(key, {})) for key in keys]
return run
def _cluster_parallel_fixture(monkeypatch):
handler = _PROXY_MaxParallelRequestsHandler(InternalUsageCache(DualCache()))
monkeypatch.setattr(handler, "_is_redis_cluster", lambda: True)
transport = _ClusterParallelTransport(handler)
for operation in ("acquire", "count", "renew", "release"):
monkeypatch.setattr(handler, f"parallel_{operation}_script", transport.script(operation))
gauges = [
{"counter_key": "{api_key:owner}:max_parallel_requests", "limit": 1, "descriptor_key": "api_key"},
{"counter_key": "{team:group}:max_parallel_requests", "limit": 2, "descriptor_key": "team"},
{"counter_key": "{api_key:owner}:another-parallel-scope", "limit": 1, "descriptor_key": "extra"},
]
return handler, transport, gauges
@pytest.mark.asyncio
async def test_cluster_parallel_slots_admit_count_renew_release_across_hash_slots(monkeypatch):
handler, transport, gauges = _cluster_parallel_fixture(monkeypatch)
keys = tuple(gauge["counter_key"] for gauge in gauges)
transport.members[keys[1]] = {"unrelated": transport.now}
result = await handler._check_parallel_request_gauges(gauges, "owner")
assert result["overall_code"] == "OK"
assert len([call for call in transport.calls if call[0] == "acquire"]) == 2
assert all("owner" in transport.members[key] for key in keys)
result = await handler._check_parallel_request_gauges(gauges, "reader", read_only=True)
assert result["overall_code"] == "OVER_LIMIT"
assert [status["descriptor_key"] for status in result["statuses"]] == ["api_key", "team", "extra"]
transport.now += PARALLEL_REQUEST_SLOT_TTL_SECONDS - 1
assert await handler._renew_realtime_call_slot("owner", keys)
transport.now += 2
result = await handler._check_parallel_request_gauges(gauges, "second")
assert result["overall_code"] == "OVER_LIMIT"
acquisition = ParallelSlotAcquisition(slot_id="owner", counter_keys=list(keys))
await handler._release_parallel_request_slots(acquisition)
await handler._release_parallel_request_slots(acquisition)
assert all("owner" not in transport.members[key] for key in keys)
assert not await handler._renew_realtime_call_slot("owner", keys)
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["limit", "unreachable", "lost_reply", "cancel"])
async def test_cluster_parallel_acquire_rolls_back_attempted_shards_without_releasing_others(monkeypatch, failure):
handler, transport, gauges = _cluster_parallel_fixture(monkeypatch)
keys = tuple(gauge["counter_key"] for gauge in gauges)
transport.members[keys[1]] = {"unrelated": transport.now}
if failure == "limit":
gauges[1]["limit"] = 1
elif failure == "unreachable":
transport.fail = ("acquire", keys[1])
elif failure == "lost_reply":
transport.lose_acquire_response = keys[1]
else:
transport.pause_acquire = keys[1]
task = asyncio.create_task(handler._check_parallel_request_gauges(gauges, "owner"))
if failure == "cancel":
await asyncio.wait_for(transport.entered.wait(), timeout=1)
task.cancel()
if failure == "limit":
assert (await task)["overall_code"] == "OVER_LIMIT"
else:
with pytest.raises(asyncio.CancelledError if failure == "cancel" else RuntimeError):
await task
assert all("owner" not in transport.members.get(key, {}) for key in keys)
assert transport.members[keys[1]] == {"unrelated": transport.now}
released_keys = {key for operation, group in transport.calls if operation == "release" for key in group}
assert released_keys == set(keys)
@pytest.mark.asyncio
async def test_cluster_parallel_release_continues_after_shard_failure_and_renewal_fails_closed(monkeypatch):
handler, transport, gauges = _cluster_parallel_fixture(monkeypatch)
keys = tuple(gauge["counter_key"] for gauge in gauges)
assert (await handler._check_parallel_request_gauges(gauges, "owner"))["overall_code"] == "OK"
transport.fail = ("renew", keys[1])
assert not await handler._renew_realtime_call_slot("owner", keys)
transport.fail = None
transport.members[keys[1]].pop("owner")
assert not await handler._renew_realtime_call_slot("owner", keys)
assert "owner" not in transport.members[keys[1]]
transport.fail = ("count", keys[1])
with pytest.raises(RuntimeError, match="Shard unavailable"):
await handler._check_parallel_request_gauges(gauges, "reader", read_only=True)
transport.fail = ("release", keys[0])
receipt = ParallelSlotAcquisition(slot_id="owner", counter_keys=list(keys))
with pytest.raises(RuntimeError, match="Shard unavailable"):
await handler._release_parallel_request_slots(receipt)
assert "owner" not in transport.members[keys[1]]
transport.fail = None
await handler._release_parallel_request_slots(receipt)
assert all("owner" not in transport.members.get(key, {}) for key in keys)
@pytest.mark.asyncio
async def test_cluster_parallel_duplicate_scope_keeps_strictest_limit(monkeypatch):
handler, transport, gauges = _cluster_parallel_fixture(monkeypatch)
gauges.append({**gauges[0], "limit": 100})
transport.members[gauges[0]["counter_key"]] = {"unrelated": transport.now}
assert (await handler._check_parallel_request_gauges(gauges, "owner"))["overall_code"] == "OVER_LIMIT"
assert transport.members[gauges[0]["counter_key"]] == {"unrelated": transport.now}
@pytest.mark.asyncio
async def test_cluster_rollback_waits_through_repeated_cancellation(monkeypatch):
handler, transport, gauges = _cluster_parallel_fixture(monkeypatch)
keys = tuple(gauge["counter_key"] for gauge in gauges)
transport.lose_acquire_response = keys[1]
release_entered, finish_release = asyncio.Event(), asyncio.Event()
release = handler.parallel_release_script
async def blocked_release(*, keys, args):
release_entered.set()
await finish_release.wait()
return await release(keys=keys, args=args)
monkeypatch.setattr(handler, "parallel_release_script", blocked_release)
task = asyncio.create_task(handler._check_parallel_request_gauges(gauges, "owner"))
await asyncio.wait_for(release_entered.wait(), timeout=1)
try:
for _ in range(3):
task.cancel()
await asyncio.sleep(0)
assert not task.done(), "admission returned while its Redis compensation was still running"
finally:
finish_release.set()
await asyncio.gather(task, return_exceptions=True)
await asyncio.sleep(0)
assert task.cancelled()
assert all("owner" not in transport.members.get(key, {}) for key in keys)
@pytest.mark.asyncio
async def test_standalone_realtime_renewal_keeps_single_atomic_batch(monkeypatch):
from unittest.mock import AsyncMock
handler = _PROXY_MaxParallelRequestsHandler(InternalUsageCache(DualCache()))
monkeypatch.setattr(handler, "_is_redis_cluster", lambda: False)
renew = AsyncMock(return_value=[1])
monkeypatch.setattr(handler, "parallel_renew_script", renew)
keys = ("{api_key:owner}:max_parallel_requests", "{team:group}:max_parallel_requests")
assert await handler._renew_realtime_call_slot("owner", keys)
renew.assert_awaited_once_with(keys=keys, args=("owner", PARALLEL_REQUEST_SLOT_TTL_SECONDS))

View file

@ -11,6 +11,102 @@ from litellm.llms.chatgpt.codex import CodexRealtimeCall
from litellm.proxy.realtime_endpoints.call_sessions import decode_call, encode_call
@pytest.mark.asyncio
@pytest.mark.parametrize("multipart", [False, True])
@pytest.mark.parametrize("policy", ["budget", "personal_models"])
async def test_offer_auth_enforces_session_model_policy_before_upstream(monkeypatch, multipart, policy):
import json
from unittest.mock import AsyncMock, MagicMock
import httpx
from fastapi import Request
import litellm
from litellm.exceptions import BudgetExceededError
from litellm.proxy import proxy_server as server
from litellm.proxy._types import LiteLLM_UserTable
from litellm.proxy.auth.auth_checks import common_checks
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
session = {"model": "forbidden-voice"}
payload = (
{"files": {"sdp": (None, "v=0"), "session": (None, json.dumps(session)), "model": (None, "body-decoy")}}
if multipart
else {"json": {"sdp": "v=0", "session": session, "model": "body-decoy"}}
)
outbound = httpx.Request("POST", "http://localhost/v1/realtime/calls", **payload)
body = outbound.read()
async def receive():
return {"type": "http.request", "body": body, "more_body": False}
request = Request(
{
"type": "http",
"method": "POST",
"path": "/v1/realtime/calls",
"query_string": b"model=query-decoy&policy=keep",
"client": ("127.0.0.7", 1234),
"headers": [
*((key.lower(), value) for key, value in outbound.headers.raw),
(b"x-policy-key", b"Bearer test-key"),
(b"x-custom-policy", b"preserved"),
(b"x-litellm-model", b"header-decoy"),
],
},
receive,
)
token = UserAPIKeyAuth(token="test-key", user_id="personal-user", model_max_budget={"forbidden-voice": 0})
budget = AsyncMock(side_effect=BudgetExceededError(current_cost=1, max_budget=0))
upstream = AsyncMock()
async def custom_auth(request: Request, api_key: str):
assert api_key == "test-key"
assert request.headers["x-custom-policy"] == "preserved"
assert request.query_params["policy"] == "keep"
assert request.client.host == "127.0.0.7"
parsed = await _read_request_body(request)
assert parsed["model"] == "body-decoy"
assert isinstance(parsed["session"], str) is multipart
if policy == "personal_models":
await common_checks(
request_body=parsed,
team_object=None,
user_object=LiteLLM_UserTable(
user_id="personal-user", models=["allowed-voice", "body-decoy", "query-decoy", "header-decoy"]
),
end_user_object=None,
global_proxy_spend=None,
general_settings={},
route="/v1/realtime/calls",
llm_router=None,
proxy_logging_obj=MagicMock(),
valid_token=token,
request=request,
skip_budget_checks=True,
)
return token
custom = AsyncMock(side_effect=custom_auth)
monkeypatch.setattr(server, "general_settings", {"litellm_key_header_name": "x-policy-key"})
monkeypatch.setattr(server, "user_custom_auth", custom)
monkeypatch.setattr(server, "llm_router", None)
monkeypatch.setattr(server, "llm_model_list", [])
monkeypatch.setattr(server, "model_max_budget_limiter", SimpleNamespace(is_key_within_model_budget=budget))
monkeypatch.setattr(server, "route_request", upstream)
monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False)
with pytest.raises(ProxyException) as denied:
await codex.create_codex_realtime_call(request)
if policy == "personal_models":
assert "user not allowed to access model" in str(denied.value)
assert "forbidden-voice" in str(denied.value)
custom.assert_awaited_once()
upstream.assert_not_awaited()
if policy == "budget":
budget.assert_awaited_once()
assert budget.await_args.kwargs["model"] == "forbidden-voice"
@pytest.mark.asyncio
@pytest.mark.parametrize("route_type", ["arealtime_calls", "_arealtime"])
@pytest.mark.parametrize("observer", [False, True])

View file

@ -276,6 +276,86 @@ async def test_cancelled_start_hangs_up_and_drains_terminal_usage():
assert sink.events[-1]["usage"]["total_tokens"] == 42
@pytest.mark.asyncio
@pytest.mark.parametrize("cancel_count", [1, 2, 3])
async def test_repeated_start_cancellation_keeps_lease_until_shutdown_finishes(cancel_count):
from litellm.proxy.hooks.realtime_call_lease import RealtimeCallLease
reading = asyncio.Event()
close_entered = asyncio.Event()
allow_close = asyncio.Event()
released = asyncio.Event()
class ObservedSocket(Socket):
async def __anext__(self):
reading.set()
return await super().__anext__()
socket = ObservedSocket()
logger = MagicMock(spec=Logging)
logger.model_call_details = {}
sink = Sink(logger)
async def close_call():
close_entered.set()
await allow_close.wait()
await socket.messages.put({"type": "session.closed", "usage": {"audio_duration_ms": 1000}})
async def release():
released.set()
lease = RealtimeCallLease(renew=AsyncMock(return_value=True), release=release)
lease.start()
supervisor = CallSupervisor(
socket, sink, logger, UserAPIKeyAuth(), close_call, lease=lease, ready_timeout=10, termination_timeout=10
)
registry = CallSupervisors()
async def signaling():
transferred = False
try:
await registry.start(supervisor)
transferred = True
finally:
# The signaling endpoint retains lease ownership until registry startup succeeds.
if not transferred:
await lease.close()
started = asyncio.create_task(signaling())
shutdown = None
try:
await asyncio.wait_for(reading.wait(), timeout=1)
started.cancel()
await asyncio.wait_for(close_entered.wait(), timeout=1)
for _ in range(cancel_count - 1):
started.cancel()
done, _ = await asyncio.wait({started}, timeout=0.02)
assert not done
assert not released.is_set()
shutdown = asyncio.create_task(registry.shutdown())
done, _ = await asyncio.wait({started, shutdown}, timeout=0.02)
assert not done
assert not released.is_set()
assert not socket.closed
assert sink.logs == 0
allow_close.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(started, timeout=1)
await asyncio.wait_for(shutdown, timeout=1)
assert socket.closed
assert sink.logs == 1
assert released.is_set()
assert sink.events[-1]["usage"]["audio_duration_ms"] == 1000
finally:
allow_close.set()
await asyncio.wait_for(supervisor.wait(), timeout=1)
await asyncio.gather(started, return_exceptions=True)
if shutdown is not None:
await shutdown
await registry.shutdown()
await lease.close()
@pytest.mark.asyncio
async def test_worker_shutdown_drains_all_calls():
registry = CallSupervisors()

72
uv.lock generated
View file

@ -1875,6 +1875,11 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/49/b5/82f89307d0d769cd9bf46a54fb9136be08e4e57c5570ae421db4c9a2ba62/fakeredis-2.34.1-py3-none-any.whl", hash = "sha256:0107ec99d48913e7eec2a5e3e2403d1bd5f8aa6489d1a634571b975289c48f12", size = 122160, upload-time = "2026-02-25T13:17:49.701Z" },
]
[package.optional-dependencies]
lua = [
{ name = "lupa" },
]
[[package]]
name = "fastapi"
version = "0.136.3"
@ -4520,7 +4525,7 @@ dev = [
{ name = "basedpyright" },
{ name = "botocore-stubs" },
{ name = "diff-cover" },
{ name = "fakeredis" },
{ name = "fakeredis", extra = ["lua"] },
{ name = "fastapi-offline" },
{ name = "hypothesis" },
{ name = "keyring" },
@ -4708,7 +4713,7 @@ dev = [
{ name = "basedpyright", specifier = "==1.39.7" },
{ name = "botocore-stubs", specifier = "==1.43.14" },
{ name = "diff-cover", specifier = "==9.7.2" },
{ name = "fakeredis", specifier = "==2.34.1" },
{ name = "fakeredis", extras = ["lua"], specifier = "==2.34.1" },
{ name = "fastapi-offline", specifier = "==1.7.6" },
{ name = "hypothesis", specifier = "==6.165.10" },
{ name = "keyring", specifier = "==25.7.0" },
@ -5003,6 +5008,69 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/db/a4/441aee36c6f6b249823d20fd91f9be9ab89d7c5a8ae542a4a4ca6d342d56/lxml-6.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:ed21202aec73cda4d55d1ce57b389aadb90ffb044e6cd1080b8347efe1b1ec84", size = 3508989, upload-time = "2026-05-18T19:18:38.158Z" },
]
[[package]]
name = "lupa"
version = "2.8"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/c3/a6/0f869fbb07c393f15473b1eefefb7b5bec162fb7481803d040ed4dc46002/lupa-2.8.tar.gz", hash = "sha256:d8022641b9ec8ecf2c5ecbe9f47e5a70e0b87c4b5ae921b92cb02a638e0acd08", size = 6156370, upload-time = "2026-04-15T20:08:30.534Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/09/21/9be4516ddd22f8eadba336d9ba065d17d79108465ae1b7f71424ab99b9d0/lupa-2.8-cp310-abi3-win32.whl", hash = "sha256:c2a5fd15dc62374e1661a55f01744c9ec1c56f291ba4a0749d3af2174556e78f", size = 1594887, upload-time = "2026-04-15T20:05:23.377Z" },
{ url = "https://files.pythonhosted.org/packages/2d/99/1557c9685d7034d9ce8dd2b54c40a26d6deb7c67c1fdb5c801abd1a02c3f/lupa-2.8-cp310-abi3-win_arm64.whl", hash = "sha256:9e304fb1c50cf23fd8882afbe1aa87525ef8a72667bcab3b37b2bbb2bc542269", size = 1371742, upload-time = "2026-04-15T20:05:27.417Z" },
{ url = "https://files.pythonhosted.org/packages/1c/34/05ce4745b191633f90ff1ab50f1a19a37da282bb0a41fb500d9157fc9b8f/lupa-2.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:97bd01e90b8031e56a5fd5bb70605aea09f1dba675c1140308a52780f93d06f1", size = 1202714, upload-time = "2026-04-15T20:05:31.088Z" },
{ url = "https://files.pythonhosted.org/packages/7d/d2/f70fdbeec2d4c69ee6a469e6cddde9635fff4af4e13fb652e6a1229eef51/lupa-2.8-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b5ebe1a13c45767919c86750b84fe2da9f6288b6f3cea4ce7660bb2abc9d921", size = 1857453, upload-time = "2026-04-15T20:05:34.611Z" },
{ url = "https://files.pythonhosted.org/packages/97/dc/6fcda0e36e75eb6cb98dc9190fa4737d727eeae29e58f892980b2c96b656/lupa-2.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:097e7d0f1719a88020b67c82e05d53d7973c166952393afcecfd8434c7e19a15", size = 2408890, upload-time = "2026-04-15T20:05:37.994Z" },
{ url = "https://files.pythonhosted.org/packages/58/29/7ea176eac3c1dac83d059762daa875ad1390decc0bf2c3b4c7bbfc1f1665/lupa-2.8-cp310-cp310-win_amd64.whl", hash = "sha256:7bb223ee8f72d0dc076b0d65296ee72f1c69450f9d2fed5315f7707d98c4a03d", size = 1910396, upload-time = "2026-04-15T20:05:41.163Z" },
{ url = "https://files.pythonhosted.org/packages/b7/0a/5a740717f27aa77481e6a61b97cf79d1e0c1ede729b1268caacded915326/lupa-2.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b12e43c1fb787189dfc28cd604aef0baa2cb95e27da19498d520361d0ace070a", size = 1202376, upload-time = "2026-04-15T20:05:44.049Z" },
{ url = "https://files.pythonhosted.org/packages/1b/75/6b64d0098c64275a801896cb7a6a30e7e653d25fa102c64e747292afcdbb/lupa-2.8-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6f603391dffb256e36a79fd2044084d5f4b8a0a4c0e5ad291cd3ab3aaf1fd0a", size = 1839271, upload-time = "2026-04-15T20:05:47.399Z" },
{ url = "https://files.pythonhosted.org/packages/7b/2f/0d4f00563046ff616ef6a421f8b776a5ffb327f7b32ed69e856d52b917a8/lupa-2.8-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f6f41c91366e7d0d474f87d81c1274af861f40812bf729c9f97ab4c8f3c7ac8", size = 2376251, upload-time = "2026-04-15T20:05:49.891Z" },
{ url = "https://files.pythonhosted.org/packages/4c/8e/caa83237f427d9e85b7f02c816e7270c9c9571dec1673e06b0180402f70e/lupa-2.8-cp311-cp311-win_amd64.whl", hash = "sha256:f5a6af145b0ea818f01d27bfe2583a4b538570bef61d22c8773e0eccf011234c", size = 1923488, upload-time = "2026-04-15T20:05:52.954Z" },
{ url = "https://files.pythonhosted.org/packages/ad/0b/368f2f0bc750b25c69d4563e44f677925ab5dd3d2887f9b0c15465d21a2a/lupa-2.8-cp312-abi3-macosx_10_13_x86_64.whl", hash = "sha256:f4342f4de76ae7ce2ab0672d36003bdb7e1a33252f293b569298ddd792e70e33", size = 1194056, upload-time = "2026-04-15T20:05:55.794Z" },
{ url = "https://files.pythonhosted.org/packages/5b/0f/c89eb8dd36fdea4e50ae3f7f5275bea3b0cc5d4057b8ee7b3bbc78010422/lupa-2.8-cp312-abi3-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:4203fa1659315e939a5304e75001b8cc14234fb3cbb3ed86c049b0cc5d90fcee", size = 1434278, upload-time = "2026-04-15T20:05:57.94Z" },
{ url = "https://files.pythonhosted.org/packages/47/30/c3b4d2cd8733621b404b8a4214e5f852955c4ba632546dc84123bea9ee89/lupa-2.8-cp312-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:81f2d843ce668b653146c007467570210ae44be51dac6926666c51d49536f307", size = 1150068, upload-time = "2026-04-15T20:06:01.04Z" },
{ url = "https://files.pythonhosted.org/packages/8d/d2/bac12c398519efafc6af84be1974edd0d7a4895fb4735b5c8d615d298595/lupa-2.8-cp312-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d3d0cde2c77588d1c60875a4f34f059513476c6e1775351897195b51e0f3df08", size = 1409532, upload-time = "2026-04-15T20:06:03.592Z" },
{ url = "https://files.pythonhosted.org/packages/9c/6a/18b52e11962014026e07813530b0b108ee8bc0a2a13ef0eaea5d41dce023/lupa-2.8-cp312-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e0d11b8f3a8dac6413f704fef7161d048bb10c58bdac6cbffa5e60efa56e9a3", size = 1242687, upload-time = "2026-04-15T20:06:06.863Z" },
{ url = "https://files.pythonhosted.org/packages/b3/8e/7fd4eb049875f61429b96780d2eae4700f0e78fe0a52db8edb231b1cd09f/lupa-2.8-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:54cff414f21f8cd8c6be4aae52541f3b9cd39602b59e3a3db9b5c9f9f674ff18", size = 1856038, upload-time = "2026-04-15T20:06:09.358Z" },
{ url = "https://files.pythonhosted.org/packages/e9/f9/37ad9d2773d30f2931890d310a4bdce28d45484206e6f48bc18b0325eabd/lupa-2.8-cp312-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:24b4d8af5558e549b70daf1547f5c1c1d664ecea9fc790f83efe5d75e9a93797", size = 1128982, upload-time = "2026-04-15T20:06:12.312Z" },
{ url = "https://files.pythonhosted.org/packages/57/31/c0fd7984c24844ea79caa45c0235f61a06b38fd69a839f6c62770f8d684a/lupa-2.8-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:ce86dff1ee7f7cf45f5622065ae991949dd7bb1703581cbc58a630137bb7ccf9", size = 1457594, upload-time = "2026-04-15T20:06:15.881Z" },
{ url = "https://files.pythonhosted.org/packages/11/f5/a28e411be30ec1bf0db1eb0c087eebc73be9e7a1adcfe6ac209861ccc446/lupa-2.8-cp312-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:f4d01b2a08c70bbb883a9e082b6b36b89121ed5910b710f1ba11c73295ff4fba", size = 1425721, upload-time = "2026-04-15T20:06:18.009Z" },
{ url = "https://files.pythonhosted.org/packages/ed/c1/359f767c4ae024be30d909fe8a9f0e9af266bad47ce2bd2ed248fb986fcf/lupa-2.8-cp312-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:7f210d5a8353e510ea1199c42cf3cbdd630553bf2bc8fb4c00fea06fdec7c798", size = 1253258, upload-time = "2026-04-15T20:06:21.17Z" },
{ url = "https://files.pythonhosted.org/packages/17/52/473f11790c261fd02bbf318a546fe040e9ec9f677181272fa78d3b4112a4/lupa-2.8-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4f81a02806e7c7ad26d8c6fa222c8bef1b0c1b124347c879be880b41339d41e4", size = 2395272, upload-time = "2026-04-15T20:06:24.137Z" },
{ url = "https://files.pythonhosted.org/packages/94/bf/75c8795655a8836eab6a11a630352c4b7c5dc5c54d075077bc9bffdeee45/lupa-2.8-cp312-abi3-win32.whl", hash = "sha256:360056453a7a4eaa4ac5a204c31a5a014b1eb2ee5490603234d2ba831684f1f2", size = 1606136, upload-time = "2026-04-15T20:06:27.815Z" },
{ url = "https://files.pythonhosted.org/packages/d8/29/11a2cdd612b6f55e506292dfb6ba343216e80a693e7fe3f876ef204ce9c6/lupa-2.8-cp312-abi3-win_arm64.whl", hash = "sha256:1628371c6592a6d5650497a9e31fb2bb3a7e9883c1f301d1111265e484045af9", size = 1364495, upload-time = "2026-04-15T20:06:30.254Z" },
{ url = "https://files.pythonhosted.org/packages/4d/17/fa834b6b09ad17e7df5d0f7715d64877a125a3776ada689751a1f9dc2959/lupa-2.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:450650f91c48c2415b0d59ab3abfcfda3b6efb5b858205f4d4bda8ad141fa529", size = 1190111, upload-time = "2026-04-15T20:06:32.84Z" },
{ url = "https://files.pythonhosted.org/packages/ab/43/45589901b7d1a0e3a9d91d19a311fb6a56924e8571536c3f2212160fd953/lupa-2.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27044f3363047f946b3d3aab9157cbd172b3538ada9ec1baef43432bf7d03a78", size = 1812999, upload-time = "2026-04-15T20:06:35.664Z" },
{ url = "https://files.pythonhosted.org/packages/a1/ac/4ade7d15ff5c61758d7943ac6f0a496bf1cc65b6c09f842b52a0702e664c/lupa-2.8-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8cf4f064a0e5531afce2d7d750120c10c10f9529139af6ca6150d13151034398", size = 2368731, upload-time = "2026-04-15T20:06:37.959Z" },
{ url = "https://files.pythonhosted.org/packages/0c/27/05f950d15b8ab120b39c43588b438ff3ace70c1b1b0225a960393a497483/lupa-2.8-cp312-cp312-win_amd64.whl", hash = "sha256:281bedc5deb92d31e649a3552edd662449365a635904fa4d5cb4509c7245e34e", size = 1941809, upload-time = "2026-04-15T20:06:40.302Z" },
{ url = "https://files.pythonhosted.org/packages/a6/3f/19f83c3a0c84dc8bea8a58e7416dca6a3ede662c33c8d1ec758e5afc754a/lupa-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45fc9da0145ecb0083ef5ff9975116cc784bd0258bdc2bd131ba15483ce18398", size = 1201203, upload-time = "2026-04-15T20:06:42.169Z" },
{ url = "https://files.pythonhosted.org/packages/89/0f/a14f0073f09610158038582e230618a48c14da6bd88185289461aa4cb854/lupa-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:58e18afed57955b41130e269c78f53d4123ab86e236b53816f4cbffa25cb5d30", size = 1806210, upload-time = "2026-04-15T20:06:45.486Z" },
{ url = "https://files.pythonhosted.org/packages/2f/14/48fff156c63a136001a7620878af7d31aa07e66b495ed621e3eddd73c294/lupa-2.8-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc47f536ac13a79cef47d29a2b205576a22841f042a2bcec1676b95806e7706a", size = 2359005, upload-time = "2026-04-15T20:06:47.819Z" },
{ url = "https://files.pythonhosted.org/packages/fe/18/3ac638ec90edf178242b8a2b2f00f8adae694248c03a26341ef941bb746e/lupa-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:ce9404c661dbac65cc9bed351ad45e797af93d30d70be309a3fa8209ac86d93b", size = 1936754, upload-time = "2026-04-15T20:06:50.448Z" },
{ url = "https://files.pythonhosted.org/packages/b0/ef/5ee5fed6ea7459a671196359ce04bfeeaf26be1dac8ff24bf28e5c7a6e81/lupa-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:348c3f8ecabb6324dcbc05c2740d762ef8fcec7b06c79e45262ab97a217684e3", size = 1209388, upload-time = "2026-04-15T20:06:53.022Z" },
{ url = "https://files.pythonhosted.org/packages/6e/b1/67a940d5542cb0384b443fe951b5a83ea9340d1333a733a258fdd1c619ba/lupa-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:951496471056061598a7d1729a6cdf48d662fec777a9f2d8aa5a1e62fd30e5a5", size = 1826821, upload-time = "2026-04-15T20:06:55.699Z" },
{ url = "https://files.pythonhosted.org/packages/a1/a2/b354e5ba3b911ec50686003dc8897e892b9e8c5c036b33219b03d54c4daf/lupa-2.8-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a591b9947ca347b41a63370e121d6e2b1458fe6dde9ae065029ec10a37f25ff4", size = 2366893, upload-time = "2026-04-15T20:06:58.9Z" },
{ url = "https://files.pythonhosted.org/packages/8e/52/d76066401f29539df5352f70ecded66576f32933b6045cd0bfc56cb770b9/lupa-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:3903c9cf628dae2f56405503247b77a61a3a61bd2dda470e336950c74776d55d", size = 1994716, upload-time = "2026-04-15T20:07:19.194Z" },
{ url = "https://files.pythonhosted.org/packages/c3/bd/3efc437a4361c16d25e66478c50357c9a8e8ecfb718fe749eb9ca3176ef6/lupa-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f711a8ab0486b9ac6fdda94a22ddcfbc9f0d4a27e3a8cf1bf79c6e48b33017c1", size = 1251217, upload-time = "2026-04-15T20:07:01.64Z" },
{ url = "https://files.pythonhosted.org/packages/ea/f4/2e9f8ecbaca854bfdf14af8a9b505ec0cbc640377b3b218921594b7563cd/lupa-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc51250e76367a3e27fcd01dc769b9bfcbbc34f48df48dde53d6af6e75b7eaa5", size = 1814701, upload-time = "2026-04-15T20:07:04.149Z" },
{ url = "https://files.pythonhosted.org/packages/ba/53/4000b1acaa8b1f3827fcff0cfcdff44d3befddda42cab7e685a49689b5a1/lupa-2.8-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f8a22088a552828958603323f0a5c4b3e11e03b75d0bf4c965ef879de9b60a8d", size = 2348414, upload-time = "2026-04-15T20:07:07.285Z" },
{ url = "https://files.pythonhosted.org/packages/d5/78/26ee48d3890cddf03cefb65f433e3492759c0b3c0582180755bddbaab7bd/lupa-2.8-cp314-cp314t-win32.whl", hash = "sha256:4f7c553c1d8cfffbe85d81daef730d12cae4b6002d457542914da0ac8a1145b3", size = 1831611, upload-time = "2026-04-15T20:07:09.752Z" },
{ url = "https://files.pythonhosted.org/packages/3c/d1/4a5cc64a3cad22821ae4c3f7a90456a08ca19457d8354f4abf46ad03c7e8/lupa-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:d8766aff03a78c80ad2d188a8bdb216de5ec838359cd87e05bbdfa56394a6105", size = 2209250, upload-time = "2026-04-15T20:07:11.906Z" },
{ url = "https://files.pythonhosted.org/packages/37/7c/cdcb654daf668192aaf36b0aeb94f2281dad092aaa5003688691131736ea/lupa-2.8-cp314-cp314t-win_arm64.whl", hash = "sha256:91d622777febda3ab1bed1d45295f2f32a4680c7b3d7caf8c669998ed5c44118", size = 1126735, upload-time = "2026-04-15T20:07:15.434Z" },
{ url = "https://files.pythonhosted.org/packages/1d/44/de1961ad38e17cd326a53c246c7e3b91178ed578f4cf22ffcd5e7e11b041/lupa-2.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b036738282a5acd2e71fdddb317c9df8b87c1673aa57f403d05fcc2be8abc4ba", size = 1186020, upload-time = "2026-04-15T20:07:35.017Z" },
{ url = "https://files.pythonhosted.org/packages/13/c2/276f0b9dc8bcc5a8a58af5316dfa0e6f56be3613dd6dbcc8d3d2cb6559ba/lupa-2.8-cp39-abi3-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:ac6b6e8d0e617e26a98cbb44880bcd75de5d32b3ad7b3b3793583909292b47ed", size = 1468944, upload-time = "2026-04-15T20:07:37.782Z" },
{ url = "https://files.pythonhosted.org/packages/63/38/52934e52a5180dc6425d20284d004fe4b27a4f9171a82dc99fb67af250bf/lupa-2.8-cp39-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ba3a7dd839f90c3d2e53bebe3c192b1f3f9fd720a6781256405123211fd0dce6", size = 1172998, upload-time = "2026-04-15T20:07:40.812Z" },
{ url = "https://files.pythonhosted.org/packages/c7/82/76b3809bd0839d9b3b4ec58d06591e08f17337b6d9576877cb9d48b34e94/lupa-2.8-cp39-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d7edb13a7a5250b5c6c22d1495d9e842b5c9fc5081c8fe6b5efe2112fe3e41f9", size = 1449975, upload-time = "2026-04-15T20:07:44.262Z" },
{ url = "https://files.pythonhosted.org/packages/16/07/2f89d54f747c67c23b4b9ae4aa8c8dd06bb409155dedcf406157f2736b66/lupa-2.8-cp39-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:891f72e0bffbed1e4175f975aeb2a083956586a100066525e1be485f617f7b25", size = 1281944, upload-time = "2026-04-15T20:07:46.458Z" },
{ url = "https://files.pythonhosted.org/packages/e7/bd/7375d2b0fcae79d806baf52a76f26c96964593f58e1372d13ae5ac09c676/lupa-2.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a295f87b5b7ebbfd5191932e8cb0e51df3c7769101ac6b6c7d7c9fb27bfd1307", size = 1910455, upload-time = "2026-04-15T20:07:49.75Z" },
{ url = "https://files.pythonhosted.org/packages/8b/0c/8abb3bc0e08b311fc01db05b6e9f9ff31a8f65e4fc3f0aeb05cfef75c8ac/lupa-2.8-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4fe5d7a810b64ea8511eb885fc8cdde042ee5ff7b7d08ae78f32449756acb177", size = 1155548, upload-time = "2026-04-15T20:07:52.657Z" },
{ url = "https://files.pythonhosted.org/packages/80/2e/9eeecd3f493099721c1d3f31beeca23a4237db1a54223684df4dc96aa1bd/lupa-2.8-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:bfc470012ef66ad064c7bd77416af03a3452ef630b04b9012595ea13f2e54518", size = 1489232, upload-time = "2026-04-15T20:07:54.92Z" },
{ url = "https://files.pythonhosted.org/packages/c3/13/731c99dc2e7652ae818a6de45bdf0142049f7cb566049061c898355f1891/lupa-2.8-cp39-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:250e035fdaffe8c87093e3ebc206ac29a26131b1568ea711d780c26001ce96e7", size = 1466321, upload-time = "2026-04-15T20:07:57.627Z" },
{ url = "https://files.pythonhosted.org/packages/de/71/3ad8cc4fc05a77dc0d3f7079348bd1cad4675a0d14c24f8e6a3ce5f008f7/lupa-2.8-cp39-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:b9bddb09acfffb4f828f790f444b11dc0cca591afea1a244d9329eea2d20c003", size = 1288577, upload-time = "2026-04-15T20:07:59.913Z" },
{ url = "https://files.pythonhosted.org/packages/d8/b2/1175f6d0aa7b68627fbe2f58bd1e8bea36a89d10dfd67671d2b024c96162/lupa-2.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2e64acbbd47e9b82a64405a39e0d2b36a5a7dad8ab41c0f3437f572f7d282ba3", size = 2444866, upload-time = "2026-04-15T20:08:02.753Z" },
{ url = "https://files.pythonhosted.org/packages/92/f7/e78df680c7a0ea452daac07467ca188d63c2c00ca1c884c0a50e27eb83b5/lupa-2.8-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32e4e5103bbddcdd2458fb2ccae6c8ba11c9997c711d7e379e0d45551d109c76", size = 1778509, upload-time = "2026-04-15T20:08:21.784Z" },
{ url = "https://files.pythonhosted.org/packages/e6/23/0e53cabb16b2a8aa9cf1fde499c097d8942c5dab709fc8e921f3b824b18b/lupa-2.8-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7667001804657496dee9feced2daae5000b4604a3218dd8e6b7b754982ba88b8", size = 2300480, upload-time = "2026-04-15T20:08:24.394Z" },
{ url = "https://files.pythonhosted.org/packages/7e/85/0271227eab939921a12ebba5d17aa4cd18346aa534ca7f5da09cd0b63dd4/lupa-2.8-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:86f6f668966965b15247dc32d064cfe7be67b71e584ccfacbe2f637575296878", size = 1847445, upload-time = "2026-04-15T20:08:27.031Z" },
]
[[package]]
name = "mako"
version = "1.3.12"