diff --git a/litellm/scheduler.py b/litellm/scheduler.py index 48be61283df..8daf08abe99 100644 --- a/litellm/scheduler.py +++ b/litellm/scheduler.py @@ -2,8 +2,10 @@ import asyncio import enum import heapq import logging as _logging +import random as _random import time from collections.abc import Awaitable, Callable, Sequence +from functools import partial as _partial from typing import Final, TypeAlias from typing import Protocol as _Protocol @@ -12,9 +14,17 @@ from pydantic import TypeAdapter from litellm import print_verbose from litellm._internal_context import with_service_target from litellm._logging import verbose_logger as _verbose_logger +from litellm._logging import verbose_router_logger as _verbose_router_logger from litellm.caching.caching import DualCache, RedisCache -from litellm.caching.redis_cache import RedisCircuitBreakerOpenError as _RedisCircuitBreakerOpenError -from litellm.caching.redis_cache import log_redis_failure as _log_redis_failure +from litellm.caching.redis_cache import ( + RedisCircuitBreakerOpenError as _RedisCircuitBreakerOpenError, +) +from litellm.caching.redis_cache import ( + _is_redis_health_failure, # pyright: ignore[reportPrivateUsage] # RedisCache's connectivity classifier has no public counterpart +) +from litellm.caching.redis_cache import ( + log_redis_failure as _log_redis_failure, +) from litellm.constants import DEFAULT_IN_MEMORY_TTL, DEFAULT_POLLING_INTERVAL from litellm.exceptions import Timeout from litellm.types.llms.base import LiteLLMBaseModel @@ -22,6 +32,25 @@ from litellm.types.llms.base import LiteLLMBaseModel SCHEDULER_QUEUE_TARGET: Final = "scheduler_queue" QueueEntry: TypeAlias = tuple[int, str] _QUEUE_ENTRIES: Final = TypeAdapter(list[QueueEntry]) +_Queue: TypeAlias = tuple[QueueEntry, ...] +_QueueResult: TypeAlias = tuple[bool, _Queue] +_QUEUE_SNAPSHOT: Final = TypeAdapter(_Queue) +_RAW_QUEUE: Final = TypeAdapter[str | bytes | None](str | bytes | None) +_CAS_RESULT: Final = TypeAdapter(bool) +_QUEUE_READ: Final = "return redis.call('GET', KEYS[1])" +_QUEUE_COMPARE_SET: Final = """ +local current = redis.call('GET', KEYS[1]) +if (ARGV[1] == '0' and current ~= false) or + (ARGV[1] == '1' and current ~= ARGV[2]) then + return 0 +end +if ARGV[4] == '' then + redis.call('SET', KEYS[1], ARGV[3]) +else + redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[4]) +end +return 1 +""" class SchedulerCacheKeys(enum.Enum): @@ -35,6 +64,98 @@ class FlowItem(LiteLLMBaseModel): model_name: str +class _LocalQueueCache(_Protocol): + @property + def default_in_memory_ttl(self) -> float | None: ... + + async def async_get_cache(self, key: str, *, local_only: bool) -> object: ... + + async def async_set_cache(self, key: str, value: Sequence[QueueEntry], *, local_only: bool) -> None: ... + + +async def _read_local_queue(cache: _LocalQueueCache, key: str) -> _Queue: + local: Final = await cache.async_get_cache(key=key, local_only=True) + return _QUEUE_SNAPSHOT.validate_python(local or ()) + + +async def _write_local_queue(cache: _LocalQueueCache, key: str, queue: _Queue) -> None: + await cache.async_set_cache(key=key, value=list(queue), local_only=True) + + +def _enqueue(entry: QueueEntry, queue: _Queue) -> _QueueResult: + return False, queue if entry in queue else tuple(sorted((*queue, entry))) + + +def _remove(request_id: str, queue: _Queue) -> _QueueResult: + return False, tuple(sorted(item for item in queue if item[1] != request_id)) + + +def _poll(entry: QueueEntry, queue: _Queue) -> _QueueResult: + pending: Final = tuple(sorted(queue if entry in queue else (*queue, entry))) + return (True, pending[1:]) if pending[0] == entry else (False, pending) + + +def _redis_unavailable(error: Exception) -> bool: + return isinstance(error, _RedisCircuitBreakerOpenError) or _is_redis_health_failure(error) + + +async def _read_redis_queue(cache: RedisCache, key: str) -> tuple[str | bytes | None, _Queue]: + raw: Final = _RAW_QUEUE.validate_python(await cache.async_register_script(_QUEUE_READ)(keys=(key,), args=())) + return raw, () if raw is None else _QUEUE_SNAPSHOT.validate_json(raw) + + +async def _mutate_redis_queue( + cache: RedisCache, key: str, model_name: str, ttl: int | None, change: Callable[[_Queue], _QueueResult] +) -> _QueueResult: + script: Final = cache.async_register_script(_QUEUE_COMPARE_SET) + for attempt in range(100): + raw, queue = await _read_redis_queue(cache, key) + admitted, updated = change(queue) + if updated == queue: + return admitted, updated + if _CAS_RESULT.validate_python( + await script( + keys=(key,), + args=( + "0" if raw is None else "1", + raw or "", + _QUEUE_SNAPSHOT.dump_json(updated), + "" if ttl is None else str(ttl), + ), + ) + ): + return admitted, updated + if attempt < 99: + await asyncio.sleep(_random.uniform(0, min(0.05, 0.001 * 2 ** min(attempt, 6)))) + raise Timeout( + message="Scheduler queue remained busy after concurrent updates", model=model_name, llm_provider="openai" + ) + + +async def _shared_queue_result( + cache: _LocalQueueCache, + redis_cache: RedisCache, + key: str, + model_name: str, + change: Callable[[_Queue], _QueueResult], +) -> _QueueResult: + try: + return await _mutate_redis_queue( + redis_cache, + key, + model_name, + redis_cache.default_ttl if cache.default_in_memory_ttl is None else int(cache.default_in_memory_ttl), + change, + ) + except Exception as error: + if not _redis_unavailable(error): + raise + _log_redis_failure( + _verbose_router_logger, _logging.WARNING, "Scheduler using local queue while Redis is unavailable", error + ) + return change(await _read_local_queue(cache, key)) + + class _QueueCache(_Protocol): async def async_get_cache(self, key: str, *, local_only: bool = False, raise_on_error: bool = False) -> object: ... @@ -74,7 +195,22 @@ class Scheduler: self.cache = DualCache(redis_cache=redis_cache, default_in_memory_ttl=default_in_memory_ttl) self.polling_interval = polling_interval or DEFAULT_POLLING_INTERVAL # default to 3ms + @with_service_target(SCHEDULER_QUEUE_TARGET) + async def _update_shared_queue( + self, redis_cache: RedisCache, model_name: str, change: Callable[[_Queue], _QueueResult] + ) -> bool: + key: Final = f"{SchedulerCacheKeys.queue.value}:{model_name}" + result: Final = await _shared_queue_result(self.cache, redis_cache, key, model_name, change) + await _write_local_queue(self.cache, key, result[1]) + return result[0] + async def add_request(self, request: FlowItem): + redis_cache: Final = self.cache.redis_cache + if isinstance(redis_cache, RedisCache): + await self._update_shared_queue( + redis_cache, request.model_name, _partial(_enqueue, (request.priority, request.request_id)) + ) + return # We use the priority directly, as lower values indicate higher priority # get the queue queue: Final = await self.get_queue(model_name=request.model_name) @@ -103,6 +239,11 @@ class Scheduler: if len(health_deployments) > 0: return True + redis_cache: Final = self.cache.redis_cache + if isinstance(redis_cache, RedisCache): + return await self._update_shared_queue( + redis_cache, request.model_name, _partial(_poll, (request.priority, request.request_id)) + ) queue: Final = await self.get_queue(model_name=request.model_name) entry: Final = (request.priority, request.request_id) print_verbose(f"queue: {queue}, seeking {entry}") @@ -134,14 +275,24 @@ class Scheduler: return await asyncio.sleep(self.polling_interval) finally: - await asyncio.shield(self.remove_request(request_id=request.request_id, model_name=request.model_name)) + await asyncio.shield(self._cleanup_request(request)) raise Timeout(message="Request timed out while polling queue", model=request.model_name, llm_provider="openai") + async def _cleanup_request(self, request: FlowItem) -> None: + try: + await self.remove_request(request_id=request.request_id, model_name=request.model_name) + except Exception as error: # noqa: BLE001 # Cleanup must preserve every request outcome, including cancellation + _log_redis_failure(_verbose_router_logger, _logging.WARNING, "Scheduler queue cleanup failed", error) + async def remove_request(self, request_id: str, model_name: str) -> None: """ Remove a specific request from the priority queue for a model. Used when a request times out while waiting in the queue. """ + redis_cache: Final = self.cache.redis_cache + if isinstance(redis_cache, RedisCache): + await self._update_shared_queue(redis_cache, model_name, _partial(_remove, request_id)) + return queue: Final = await self.get_queue(model_name=model_name) filtered_queue: Final = [item for item in queue if item[1] != request_id] heapq.heapify(filtered_queue) # restore heap invariant after filtering @@ -176,6 +327,17 @@ class Scheduler: Redis hands the queue back as JSON lists, so every entry is validated into the (priority, request_id) tuple the heap operations compare against. """ + redis_cache: Final = self.cache.redis_cache + if isinstance(redis_cache, RedisCache): + key: Final = f"{SchedulerCacheKeys.queue.value}:{model_name}" + try: + _, snapshot = await _read_redis_queue(redis_cache, key) + except Exception as error: + if not _redis_unavailable(error): + raise + return list(await _read_local_queue(self.cache, key)) + await _write_local_queue(self.cache, key, snapshot) + return list(snapshot) if self.cache is not None: _cache_key: Final = f"{SchedulerCacheKeys.queue.value}:{model_name}" response: Final = await _read_queue(self.cache, self.cache.redis_cache, _cache_key) diff --git a/pyproject.toml b/pyproject.toml index 02834f604d2..c6ee24a90ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -236,7 +236,7 @@ dev = [ "opentelemetry-instrumentation-fastapi==0.66b1", "langfuse>=4.7,<5.0", "fastapi-offline==1.7.6", - "fakeredis==2.34.1", + "fakeredis[lua]==2.34.1", "pytest-rerunfailures==15.1", "pytest-cov==5.0.0", "parameterized==0.9.0", diff --git a/tests/integration/sdk/test_dual_cache_redis.py b/tests/integration/sdk/test_dual_cache_redis.py index 7d05eb2795b..00395f0ce2c 100644 --- a/tests/integration/sdk/test_dual_cache_redis.py +++ b/tests/integration/sdk/test_dual_cache_redis.py @@ -1,15 +1,22 @@ import asyncio import os +import threading import uuid -from typing import Final +from collections.abc import AsyncIterator, Iterator +from typing import Final, Literal from unittest.mock import patch import pytest +import pytest_asyncio +from integration._support.async_server import LoopbackServer, close_writer +from integration._support.ports import allocate_port +from pydantic import ValidationError from litellm.caching.affinity_cache import claim_affinity_pin, read_affinity_pin from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache +from litellm.scheduler import FlowItem, Scheduler, SchedulerCacheKeys def _redis_cache() -> RedisCache: @@ -95,6 +102,212 @@ async def test_sync_and_async_batch_reads_share_one_redis_without_sync_reads_goi assert await dual_cache.async_batch_get_cache(keys=[survivor_key]) == [expected[survivor_key]] +@pytest_asyncio.fixture(loop_scope="function") +async def shared_scheduler_cache() -> AsyncIterator[RedisCache]: + cache: Final = RedisCache( + host=os.environ["REDIS_HOST"], + port=int(os.environ["REDIS_PORT"]), + namespace="scheduler-atomic-" + uuid.uuid4().hex, + ) + try: + yield cache + finally: + client: Final = cache.init_async_client() + await client.delete(cache.check_and_fix_namespace(f"{SchedulerCacheKeys.queue.value}:shared")) + await client.aclose() + cache.redis_client.close() + + +async def test_shared_scheduler_preserves_concurrent_admissions(shared_scheduler_cache: RedisCache) -> None: + first: Final = Scheduler(redis_cache=shared_scheduler_cache) + second: Final = Scheduler(redis_cache=shared_scheduler_cache) + await asyncio.gather(first.get_queue("shared"), second.get_queue("shared")) + await asyncio.gather( + first.add_request(FlowItem(priority=1, request_id="first", model_name="shared")), + second.add_request(FlowItem(priority=2, request_id="second", model_name="shared")), + ) + assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == [[1, "first"], [2, "second"]] + assert await first.get_queue("shared") == [(1, "first"), (2, "second")] + assert await second.get_queue("shared") == [(1, "first"), (2, "second")] + + +async def test_shared_scheduler_does_not_resurrect_peer_removal(shared_scheduler_cache: RedisCache) -> None: + await shared_scheduler_cache.async_set_cache("scheduler:queue:shared", [[1, "removed"], [2, "waiting"]]) + first: Final = Scheduler(redis_cache=shared_scheduler_cache) + second: Final = Scheduler(redis_cache=shared_scheduler_cache) + await asyncio.gather(first.get_queue("shared"), second.get_queue("shared")) + await first.remove_request("removed", "shared") + await second.add_request(FlowItem(priority=3, request_id="new", model_name="shared")) + assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == [[2, "waiting"], [3, "new"]] + + +async def test_shared_scheduler_observes_peer_pop(shared_scheduler_cache: RedisCache) -> None: + await shared_scheduler_cache.async_set_cache("scheduler:queue:shared", [[1, "first"], [2, "second"]]) + first: Final = Scheduler(redis_cache=shared_scheduler_cache) + second: Final = Scheduler(redis_cache=shared_scheduler_cache) + await asyncio.gather(first.get_queue("shared"), second.get_queue("shared")) + assert await first.poll(FlowItem(priority=1, request_id="first", model_name="shared"), []) + assert await second.poll(FlowItem(priority=2, request_id="second", model_name="shared"), []) + assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == [] + + +async def test_shared_scheduler_keeps_other_requests_during_concurrent_cleanup( + shared_scheduler_cache: RedisCache, +) -> None: + schedulers: Final = tuple(Scheduler(redis_cache=shared_scheduler_cache) for _ in range(16)) + await asyncio.gather( + *( + scheduler.add_request(FlowItem(priority=i, request_id=str(i), model_name="shared")) + for i, scheduler in enumerate(schedulers) + ) + ) + await asyncio.gather( + *(scheduler.remove_request(str(i), "shared") for i, scheduler in enumerate(schedulers) if i % 2 == 0) + ) + assert await schedulers[0].get_queue("shared") == [(i, str(i)) for i in range(16) if i % 2] + + +@pytest.fixture +def interrupted_scheduler_cache(shared_scheduler_cache: RedisCache) -> Iterator[tuple[RedisCache, threading.Event]]: + interrupted: Final = threading.Event() + + async def relay(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + upstream_reader, upstream_writer = await asyncio.open_connection( + os.environ["REDIS_HOST"], int(os.environ["REDIS_PORT"]) + ) + + async def forward(source: asyncio.StreamReader, destination: asyncio.StreamWriter) -> None: + try: + while chunk := await source.read(65536): + if interrupted.is_set(): + return + destination.write(chunk) + await destination.drain() + except ConnectionError: + return + finally: + await close_writer(destination) + + await asyncio.gather(forward(reader, upstream_writer), forward(upstream_reader, writer)) + + port: Final = allocate_port() + server: Final = LoopbackServer(port, relay) + server.start() + cache: Final = RedisCache(host="127.0.0.1", port=port, namespace=shared_scheduler_cache.namespace) + try: + yield cache, interrupted + finally: + cache.redis_client.close() + server.stop() + + +async def test_shared_scheduler_keeps_local_fallback_out_of_recovered_shared_queue( + shared_scheduler_cache: RedisCache, + interrupted_scheduler_cache: tuple[RedisCache, threading.Event], +) -> None: + cache, interrupted = interrupted_scheduler_cache + scheduler: Final = Scheduler(redis_cache=cache) + try: + await scheduler.add_request(FlowItem(priority=1, request_id="removed", model_name="shared")) + interrupted.set() + await scheduler.add_request(FlowItem(priority=2, request_id="local", model_name="shared")) + assert await scheduler.get_queue("shared") == [(1, "removed"), (2, "local")] + await Scheduler(redis_cache=shared_scheduler_cache).remove_request("removed", "shared") + interrupted.clear() + await scheduler.add_request(FlowItem(priority=3, request_id="new", model_name="shared")) + assert await scheduler.get_queue("shared") == [(3, "new")] + assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == [[3, "new"]] + finally: + interrupted.clear() + await cache.init_async_client().aclose() + + +async def test_shared_scheduler_preserves_invalid_queue_for_diagnosis(shared_scheduler_cache: RedisCache) -> None: + await shared_scheduler_cache.async_set_cache("scheduler:queue:shared", {"invalid": "queue"}) + scheduler: Final = Scheduler(redis_cache=shared_scheduler_cache) + with pytest.raises(ValidationError): + await scheduler.add_request(FlowItem(priority=1, request_id="first", model_name="shared")) + assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == {"invalid": "queue"} + + +@pytest.mark.parametrize("priority", [0, -1, 2**80]) +async def test_shared_scheduler_retains_priority_and_request_id( + shared_scheduler_cache: RedisCache, priority: int +) -> None: + scheduler: Final = Scheduler(redis_cache=shared_scheduler_cache) + scheduler.cache.update_cache_ttl(default_in_memory_ttl=17, default_redis_ttl=None) + await scheduler.add_request(FlowItem(priority=priority, request_id="请求-é", model_name="shared")) + assert await scheduler.get_queue("shared") == [(priority, "请求-é")] + assert ( + 0 + < shared_scheduler_cache.redis_client.ttl( + shared_scheduler_cache.check_and_fix_namespace("scheduler:queue:shared") + ) + <= 17 + ) + + +@pytest.mark.parametrize("outcome", ["admitted", "failed", "cancelled"]) +async def test_shared_scheduler_cleanup_failure_preserves_request_outcome( + shared_scheduler_cache: RedisCache, + outcome: Literal["admitted", "failed", "cancelled"], + caplog: pytest.LogCaptureFixture, +) -> None: + scheduler: Final = Scheduler(redis_cache=shared_scheduler_cache) + original: Final = RuntimeError("health lookup failed") + + async def health_after_corrupting_queue() -> tuple[object, ...]: + await shared_scheduler_cache.async_set_cache("scheduler:queue:shared", {"invalid": "queue"}) + match outcome: + case "failed": + raise original + case "cancelled": + raise asyncio.CancelledError + case "admitted": + return ("healthy",) + + async def wait() -> None: + await scheduler.wait_for_turn( + FlowItem(priority=1, request_id="cleanup", model_name="shared"), + timeout=5, + get_healthy_deployments=health_after_corrupting_queue, + ) + + match outcome: + case "admitted": + await wait() + case "failed": + with pytest.raises(RuntimeError) as caught: + await wait() + assert caught.value is original + case "cancelled": + with pytest.raises(asyncio.CancelledError): + await wait() + assert "Scheduler queue cleanup failed" in caplog.text + assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == {"invalid": "queue"} + + +async def test_shared_scheduler_handles_three_hundred_concurrent_enqueues_and_removals( + shared_scheduler_cache: RedisCache, +) -> None: + schedulers: Final = tuple(Scheduler(redis_cache=shared_scheduler_cache) for _ in range(300)) + for scheduler in schedulers: + scheduler.cache.update_cache_ttl(default_in_memory_ttl=60, default_redis_ttl=None) + requests: Final = tuple( + FlowItem(priority=i % 8, request_id=f"burst-{i:03}", model_name="shared") for i in range(300) + ) + + await asyncio.gather(*(scheduler.add_request(request) for scheduler, request in zip(schedulers, requests))) + + assert await schedulers[0].get_queue("shared") == sorted( + (request.priority, request.request_id) for request in requests + ) + await asyncio.gather( + *(scheduler.remove_request(request.request_id, "shared") for scheduler, request in zip(schedulers, requests)) + ) + assert await schedulers[0].get_queue("shared") == [] + + @pytest.mark.asyncio async def test_concurrent_session_repin_claims_keep_the_shared_winner_after_recovery() -> None: redis: Final = _redis_cache() diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index fbc3c0e47c3..e700b6e4acc 100644 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -5,11 +5,16 @@ import asyncio import importlib import json import os -import time -from collections.abc import Sequence -from typing import Final +from collections.abc import Iterator, Sequence +from typing import Final, Literal +from unittest.mock import AsyncMock, patch import pytest +from pydantic import ValidationError +from redis.exceptions import ConnectionError as RedisConnectionError + +import time + import redis import redis.asyncio from fakeredis import FakeRedis, FakeServer @@ -20,6 +25,7 @@ import litellm from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.exceptions import Timeout from litellm.scheduler import FlowItem, Scheduler, SchedulerCacheKeys from litellm.utils import _invalidate_model_cost_lowercase_map from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -282,7 +288,6 @@ async def test_wait_for_turn_during_cooldown_survives_the_queue_key_expiring(): assert await scheduler.get_queue("sched-model") == [] - class _JsonRoundTripRedisCache: def __init__(self) -> None: self.store: dict[str, str] = {} @@ -312,6 +317,7 @@ async def test_second_replica_enqueues_behind_a_queue_decoded_from_redis(): await replica_b.remove_request(request_id="waiting-on-a", model_name="sched-model") assert await replica_b.get_queue("sched-model") == [] + class _PausesAfterEnqueueScheduler(Scheduler): def __init__(self) -> None: super().__init__() @@ -387,6 +393,160 @@ async def test_wait_for_turn_finishes_removal_when_cancelled_again_during_cleanu assert await scheduler.get_queue("sched-model") == [(0, "head")] +@pytest.fixture +def scheduler_redis_replies() -> Iterator[tuple[RedisCache, AsyncMock, AsyncMock]]: + reads: Final = AsyncMock(return_value=None) + writes: Final = AsyncMock(return_value=1) + + async def execute( + *, keys: Sequence[str], args: Sequence[str | bytes | int | float], client: object = None + ) -> object: + return await (writes if args else reads)(keys=keys, args=args, client=client) + + with ( + patch("redis.Redis.execute_command", return_value={"redis_version": "unit-fixture"}), + patch("redis.asyncio.Redis.execute_command", new_callable=AsyncMock, return_value=True), + patch("redis.commands.core.AsyncScript.__call__", side_effect=execute), + ): + cache: Final = RedisCache(host="scheduler-unit.invalid", port=6379) + yield cache, reads, writes + cache.redis_client.close() + + +@pytest.mark.asyncio +async def test_shared_scheduler_retries_conflicts_using_the_new_peer_snapshot( + scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock], +) -> None: + cache, reads, writes = scheduler_redis_replies + scheduler: Final = Scheduler(redis_cache=cache) + reads.side_effect = (None, b'[[0,"peer"]]', RedisConnectionError("offline")) + writes.side_effect = (0, 1) + + await scheduler.add_request(FlowItem(priority=1, request_id="new", model_name="shared")) + + assert await scheduler.get_queue("shared") == [(0, "peer"), (1, "new")] + + +@pytest.mark.asyncio +async def test_shared_scheduler_stops_after_conflicts_without_publishing_an_uncommitted_queue( + scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock], +) -> None: + cache, _reads, writes = scheduler_redis_replies + scheduler: Final = Scheduler(redis_cache=cache) + writes.return_value = 0 + + with patch("random.uniform", return_value=0), pytest.raises(Timeout, match="queue remained busy"): + await scheduler.add_request(FlowItem(priority=1, request_id="never-committed", model_name="shared")) + + assert await scheduler.cache.async_get_cache("scheduler:queue:shared", local_only=True) is None + + +@pytest.mark.parametrize("operation", ["enqueue", "read"]) +@pytest.mark.asyncio +async def test_shared_scheduler_rejects_malformed_snapshots_without_replacing_local_history( + scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock], operation: str +) -> None: + cache, reads, _writes = scheduler_redis_replies + scheduler: Final = Scheduler(redis_cache=cache) + await scheduler.cache.async_set_cache("scheduler:queue:shared", [(0, "last-known")], local_only=True) + reads.side_effect = (b'{"invalid":"queue"}',) + + call: Final = ( + scheduler.add_request(FlowItem(priority=1, request_id="new", model_name="shared")) + if operation == "enqueue" + else scheduler.get_queue("shared") + ) + with pytest.raises(ValidationError): + await call + + assert await scheduler.cache.async_get_cache("scheduler:queue:shared", local_only=True) == [(0, "last-known")] + + +@pytest.mark.parametrize("outcome", ["admitted", "failed", "cancelled"]) +@pytest.mark.asyncio +async def test_shared_scheduler_cleanup_failure_preserves_the_request_outcome( + scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock], + outcome: Literal["admitted", "failed", "cancelled"], +) -> None: + cache, reads, _writes = scheduler_redis_replies + scheduler: Final = Scheduler(redis_cache=cache) + original: Final = RuntimeError("health lookup failed") + reads.side_effect = (None, b'{"invalid":"queue"}') + + async def health() -> Sequence[object]: + match outcome: + case "failed": + raise original + case "cancelled": + raise asyncio.CancelledError + case "admitted": + return ("healthy",) + + async def wait() -> None: + await scheduler.wait_for_turn( + FlowItem(priority=1, request_id="waiting", model_name="shared"), timeout=5, get_healthy_deployments=health + ) + + match outcome: + case "admitted": + await wait() + case "failed": + with pytest.raises(RuntimeError) as caught: + await wait() + assert caught.value is original + case "cancelled": + with pytest.raises(asyncio.CancelledError): + await wait() + assert await scheduler.cache.async_get_cache("scheduler:queue:shared", local_only=True) == [(1, "waiting")] + + +@pytest.mark.asyncio +async def test_shared_scheduler_keeps_admission_and_removal_in_its_fallback_snapshot( + scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock], +) -> None: + cache, reads, _writes = scheduler_redis_replies + scheduler: Final = Scheduler(redis_cache=cache) + head: Final = FlowItem(priority=0, request_id="head", model_name="shared") + later: Final = FlowItem(priority=2, request_id="later", model_name="shared") + reads.side_effect = ( + b'[[0,"head"],[2,"later"]]', + b'[[0,"head"],[2,"later"]]', + b'[[0,"head"],[2,"later"]]', + RedisConnectionError("offline after admission"), + b'[[2,"later"],[3,"peer"]]', + RedisConnectionError("offline after removal"), + b'[[3,"peer"]]', + ) + + await scheduler.add_request(head) + assert not await scheduler.poll(later, []) + assert await scheduler.poll(head, []) + assert await scheduler.get_queue("shared") == [(2, "later")] + await scheduler.remove_request("later", "shared") + assert await scheduler.get_queue("shared") == [(3, "peer")] + assert await scheduler.get_queue("shared") == [(3, "peer")] + + +@pytest.mark.asyncio +async def test_shared_scheduler_does_not_upload_outage_only_requests_when_redis_recovers( + scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock], +) -> None: + cache, reads, writes = scheduler_redis_replies + scheduler: Final = Scheduler(redis_cache=cache) + reads.side_effect = ( + RedisConnectionError("offline during enqueue"), + RedisConnectionError("still offline"), + b'[[0,"peer"]]', + RedisConnectionError("offline after recovery"), + ) + + await scheduler.add_request(FlowItem(priority=1, request_id="local-only", model_name="shared")) + assert await scheduler.get_queue("shared") == [(1, "local-only")] + await scheduler.add_request(FlowItem(priority=2, request_id="new", model_name="shared")) + assert await scheduler.get_queue("shared") == [(0, "peer"), (2, "new")] + assert json.loads(writes.call_args.kwargs["args"][2]) == [[0, "peer"], [2, "new"]] + + @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): install_live_call_probe(request, vcr) diff --git a/uv.lock b/uv.lock index 8712b78bbd6..4408dc6ca86 100644 --- a/uv.lock +++ b/uv.lock @@ -2127,6 +2127,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/b5/82f89307d0d769cd9bf46a54fb9136be08e4e57c5570ae421db4c9a2ba62/fakeredis-2.34.1-py3-none-any.whl", hash = "sha256:0107ec99d48913e7eec2a5e3e2403d1bd5f8aa6489d1a634571b975289c48f12", size = 122160, upload-time = "2026-02-25T13:17:49.701Z" }, ] +[package.optional-dependencies] +lua = [ + { name = "lupa" }, +] + [[package]] name = "fastapi" version = "0.136.3" @@ -5222,7 +5227,7 @@ dev = [ { name = "basedpyright" }, { name = "botocore-stubs" }, { name = "diff-cover" }, - { name = "fakeredis" }, + { name = "fakeredis", extra = ["lua"] }, { name = "fastapi-offline" }, { name = "hypothesis" }, { name = "keyring" }, @@ -5459,7 +5464,7 @@ dev = [ { name = "basedpyright", specifier = "==1.39.7" }, { name = "botocore-stubs", specifier = "==1.43.14" }, { name = "diff-cover", specifier = "==9.7.2" }, - { name = "fakeredis", specifier = "==2.34.1" }, + { name = "fakeredis", extras = ["lua"], specifier = "==2.34.1" }, { name = "fastapi-offline", specifier = "==1.7.6" }, { name = "hypothesis", specifier = "==6.165.10" }, { name = "keyring", specifier = "==25.7.0" }, @@ -5693,6 +5698,69 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/cf/e4e016820516c1f0c85549e356865c2319cf6afba8ab386611b1d03cf2b6/lunary-1.4.37-py3-none-any.whl", hash = "sha256:7289330e851a481404c92213ce480c0c7be9bfa56982a1385e08e3665abefe05", size = 25581, upload-time = "2026-02-12T08:15:03.892Z" }, ] +[[package]] +name = "lupa" +version = "2.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/a6/0f869fbb07c393f15473b1eefefb7b5bec162fb7481803d040ed4dc46002/lupa-2.8.tar.gz", hash = "sha256:d8022641b9ec8ecf2c5ecbe9f47e5a70e0b87c4b5ae921b92cb02a638e0acd08", size = 6156370, upload-time = "2026-04-15T20:08:30.534Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/21/9be4516ddd22f8eadba336d9ba065d17d79108465ae1b7f71424ab99b9d0/lupa-2.8-cp310-abi3-win32.whl", hash = "sha256:c2a5fd15dc62374e1661a55f01744c9ec1c56f291ba4a0749d3af2174556e78f", size = 1594887, upload-time = "2026-04-15T20:05:23.377Z" }, + { url = "https://files.pythonhosted.org/packages/2d/99/1557c9685d7034d9ce8dd2b54c40a26d6deb7c67c1fdb5c801abd1a02c3f/lupa-2.8-cp310-abi3-win_arm64.whl", hash = "sha256:9e304fb1c50cf23fd8882afbe1aa87525ef8a72667bcab3b37b2bbb2bc542269", size = 1371742, upload-time = "2026-04-15T20:05:27.417Z" }, + { url = "https://files.pythonhosted.org/packages/1c/34/05ce4745b191633f90ff1ab50f1a19a37da282bb0a41fb500d9157fc9b8f/lupa-2.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:97bd01e90b8031e56a5fd5bb70605aea09f1dba675c1140308a52780f93d06f1", size = 1202714, upload-time = "2026-04-15T20:05:31.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d2/f70fdbeec2d4c69ee6a469e6cddde9635fff4af4e13fb652e6a1229eef51/lupa-2.8-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b5ebe1a13c45767919c86750b84fe2da9f6288b6f3cea4ce7660bb2abc9d921", size = 1857453, upload-time = "2026-04-15T20:05:34.611Z" }, + { url = "https://files.pythonhosted.org/packages/97/dc/6fcda0e36e75eb6cb98dc9190fa4737d727eeae29e58f892980b2c96b656/lupa-2.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:097e7d0f1719a88020b67c82e05d53d7973c166952393afcecfd8434c7e19a15", size = 2408890, upload-time = "2026-04-15T20:05:37.994Z" }, + { url = "https://files.pythonhosted.org/packages/58/29/7ea176eac3c1dac83d059762daa875ad1390decc0bf2c3b4c7bbfc1f1665/lupa-2.8-cp310-cp310-win_amd64.whl", hash = "sha256:7bb223ee8f72d0dc076b0d65296ee72f1c69450f9d2fed5315f7707d98c4a03d", size = 1910396, upload-time = "2026-04-15T20:05:41.163Z" }, + { url = "https://files.pythonhosted.org/packages/b7/0a/5a740717f27aa77481e6a61b97cf79d1e0c1ede729b1268caacded915326/lupa-2.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b12e43c1fb787189dfc28cd604aef0baa2cb95e27da19498d520361d0ace070a", size = 1202376, upload-time = "2026-04-15T20:05:44.049Z" }, + { url = "https://files.pythonhosted.org/packages/1b/75/6b64d0098c64275a801896cb7a6a30e7e653d25fa102c64e747292afcdbb/lupa-2.8-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6f603391dffb256e36a79fd2044084d5f4b8a0a4c0e5ad291cd3ab3aaf1fd0a", size = 1839271, upload-time = "2026-04-15T20:05:47.399Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2f/0d4f00563046ff616ef6a421f8b776a5ffb327f7b32ed69e856d52b917a8/lupa-2.8-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f6f41c91366e7d0d474f87d81c1274af861f40812bf729c9f97ab4c8f3c7ac8", size = 2376251, upload-time = "2026-04-15T20:05:49.891Z" }, + { url = "https://files.pythonhosted.org/packages/4c/8e/caa83237f427d9e85b7f02c816e7270c9c9571dec1673e06b0180402f70e/lupa-2.8-cp311-cp311-win_amd64.whl", hash = "sha256:f5a6af145b0ea818f01d27bfe2583a4b538570bef61d22c8773e0eccf011234c", size = 1923488, upload-time = "2026-04-15T20:05:52.954Z" }, + { url = "https://files.pythonhosted.org/packages/ad/0b/368f2f0bc750b25c69d4563e44f677925ab5dd3d2887f9b0c15465d21a2a/lupa-2.8-cp312-abi3-macosx_10_13_x86_64.whl", hash = "sha256:f4342f4de76ae7ce2ab0672d36003bdb7e1a33252f293b569298ddd792e70e33", size = 1194056, upload-time = "2026-04-15T20:05:55.794Z" }, + { url = "https://files.pythonhosted.org/packages/5b/0f/c89eb8dd36fdea4e50ae3f7f5275bea3b0cc5d4057b8ee7b3bbc78010422/lupa-2.8-cp312-abi3-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:4203fa1659315e939a5304e75001b8cc14234fb3cbb3ed86c049b0cc5d90fcee", size = 1434278, upload-time = "2026-04-15T20:05:57.94Z" }, + { url = "https://files.pythonhosted.org/packages/47/30/c3b4d2cd8733621b404b8a4214e5f852955c4ba632546dc84123bea9ee89/lupa-2.8-cp312-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:81f2d843ce668b653146c007467570210ae44be51dac6926666c51d49536f307", size = 1150068, upload-time = "2026-04-15T20:06:01.04Z" }, + { url = "https://files.pythonhosted.org/packages/8d/d2/bac12c398519efafc6af84be1974edd0d7a4895fb4735b5c8d615d298595/lupa-2.8-cp312-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d3d0cde2c77588d1c60875a4f34f059513476c6e1775351897195b51e0f3df08", size = 1409532, upload-time = "2026-04-15T20:06:03.592Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6a/18b52e11962014026e07813530b0b108ee8bc0a2a13ef0eaea5d41dce023/lupa-2.8-cp312-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e0d11b8f3a8dac6413f704fef7161d048bb10c58bdac6cbffa5e60efa56e9a3", size = 1242687, upload-time = "2026-04-15T20:06:06.863Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8e/7fd4eb049875f61429b96780d2eae4700f0e78fe0a52db8edb231b1cd09f/lupa-2.8-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:54cff414f21f8cd8c6be4aae52541f3b9cd39602b59e3a3db9b5c9f9f674ff18", size = 1856038, upload-time = "2026-04-15T20:06:09.358Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f9/37ad9d2773d30f2931890d310a4bdce28d45484206e6f48bc18b0325eabd/lupa-2.8-cp312-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:24b4d8af5558e549b70daf1547f5c1c1d664ecea9fc790f83efe5d75e9a93797", size = 1128982, upload-time = "2026-04-15T20:06:12.312Z" }, + { url = "https://files.pythonhosted.org/packages/57/31/c0fd7984c24844ea79caa45c0235f61a06b38fd69a839f6c62770f8d684a/lupa-2.8-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:ce86dff1ee7f7cf45f5622065ae991949dd7bb1703581cbc58a630137bb7ccf9", size = 1457594, upload-time = "2026-04-15T20:06:15.881Z" }, + { url = "https://files.pythonhosted.org/packages/11/f5/a28e411be30ec1bf0db1eb0c087eebc73be9e7a1adcfe6ac209861ccc446/lupa-2.8-cp312-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:f4d01b2a08c70bbb883a9e082b6b36b89121ed5910b710f1ba11c73295ff4fba", size = 1425721, upload-time = "2026-04-15T20:06:18.009Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c1/359f767c4ae024be30d909fe8a9f0e9af266bad47ce2bd2ed248fb986fcf/lupa-2.8-cp312-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:7f210d5a8353e510ea1199c42cf3cbdd630553bf2bc8fb4c00fea06fdec7c798", size = 1253258, upload-time = "2026-04-15T20:06:21.17Z" }, + { url = "https://files.pythonhosted.org/packages/17/52/473f11790c261fd02bbf318a546fe040e9ec9f677181272fa78d3b4112a4/lupa-2.8-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4f81a02806e7c7ad26d8c6fa222c8bef1b0c1b124347c879be880b41339d41e4", size = 2395272, upload-time = "2026-04-15T20:06:24.137Z" }, + { url = "https://files.pythonhosted.org/packages/94/bf/75c8795655a8836eab6a11a630352c4b7c5dc5c54d075077bc9bffdeee45/lupa-2.8-cp312-abi3-win32.whl", hash = "sha256:360056453a7a4eaa4ac5a204c31a5a014b1eb2ee5490603234d2ba831684f1f2", size = 1606136, upload-time = "2026-04-15T20:06:27.815Z" }, + { url = "https://files.pythonhosted.org/packages/d8/29/11a2cdd612b6f55e506292dfb6ba343216e80a693e7fe3f876ef204ce9c6/lupa-2.8-cp312-abi3-win_arm64.whl", hash = "sha256:1628371c6592a6d5650497a9e31fb2bb3a7e9883c1f301d1111265e484045af9", size = 1364495, upload-time = "2026-04-15T20:06:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/4d/17/fa834b6b09ad17e7df5d0f7715d64877a125a3776ada689751a1f9dc2959/lupa-2.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:450650f91c48c2415b0d59ab3abfcfda3b6efb5b858205f4d4bda8ad141fa529", size = 1190111, upload-time = "2026-04-15T20:06:32.84Z" }, + { url = "https://files.pythonhosted.org/packages/ab/43/45589901b7d1a0e3a9d91d19a311fb6a56924e8571536c3f2212160fd953/lupa-2.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27044f3363047f946b3d3aab9157cbd172b3538ada9ec1baef43432bf7d03a78", size = 1812999, upload-time = "2026-04-15T20:06:35.664Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ac/4ade7d15ff5c61758d7943ac6f0a496bf1cc65b6c09f842b52a0702e664c/lupa-2.8-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8cf4f064a0e5531afce2d7d750120c10c10f9529139af6ca6150d13151034398", size = 2368731, upload-time = "2026-04-15T20:06:37.959Z" }, + { url = "https://files.pythonhosted.org/packages/0c/27/05f950d15b8ab120b39c43588b438ff3ace70c1b1b0225a960393a497483/lupa-2.8-cp312-cp312-win_amd64.whl", hash = "sha256:281bedc5deb92d31e649a3552edd662449365a635904fa4d5cb4509c7245e34e", size = 1941809, upload-time = "2026-04-15T20:06:40.302Z" }, + { url = "https://files.pythonhosted.org/packages/a6/3f/19f83c3a0c84dc8bea8a58e7416dca6a3ede662c33c8d1ec758e5afc754a/lupa-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45fc9da0145ecb0083ef5ff9975116cc784bd0258bdc2bd131ba15483ce18398", size = 1201203, upload-time = "2026-04-15T20:06:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/89/0f/a14f0073f09610158038582e230618a48c14da6bd88185289461aa4cb854/lupa-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:58e18afed57955b41130e269c78f53d4123ab86e236b53816f4cbffa25cb5d30", size = 1806210, upload-time = "2026-04-15T20:06:45.486Z" }, + { url = "https://files.pythonhosted.org/packages/2f/14/48fff156c63a136001a7620878af7d31aa07e66b495ed621e3eddd73c294/lupa-2.8-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc47f536ac13a79cef47d29a2b205576a22841f042a2bcec1676b95806e7706a", size = 2359005, upload-time = "2026-04-15T20:06:47.819Z" }, + { url = "https://files.pythonhosted.org/packages/fe/18/3ac638ec90edf178242b8a2b2f00f8adae694248c03a26341ef941bb746e/lupa-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:ce9404c661dbac65cc9bed351ad45e797af93d30d70be309a3fa8209ac86d93b", size = 1936754, upload-time = "2026-04-15T20:06:50.448Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ef/5ee5fed6ea7459a671196359ce04bfeeaf26be1dac8ff24bf28e5c7a6e81/lupa-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:348c3f8ecabb6324dcbc05c2740d762ef8fcec7b06c79e45262ab97a217684e3", size = 1209388, upload-time = "2026-04-15T20:06:53.022Z" }, + { url = "https://files.pythonhosted.org/packages/6e/b1/67a940d5542cb0384b443fe951b5a83ea9340d1333a733a258fdd1c619ba/lupa-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:951496471056061598a7d1729a6cdf48d662fec777a9f2d8aa5a1e62fd30e5a5", size = 1826821, upload-time = "2026-04-15T20:06:55.699Z" }, + { url = "https://files.pythonhosted.org/packages/a1/a2/b354e5ba3b911ec50686003dc8897e892b9e8c5c036b33219b03d54c4daf/lupa-2.8-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a591b9947ca347b41a63370e121d6e2b1458fe6dde9ae065029ec10a37f25ff4", size = 2366893, upload-time = "2026-04-15T20:06:58.9Z" }, + { url = "https://files.pythonhosted.org/packages/8e/52/d76066401f29539df5352f70ecded66576f32933b6045cd0bfc56cb770b9/lupa-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:3903c9cf628dae2f56405503247b77a61a3a61bd2dda470e336950c74776d55d", size = 1994716, upload-time = "2026-04-15T20:07:19.194Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bd/3efc437a4361c16d25e66478c50357c9a8e8ecfb718fe749eb9ca3176ef6/lupa-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f711a8ab0486b9ac6fdda94a22ddcfbc9f0d4a27e3a8cf1bf79c6e48b33017c1", size = 1251217, upload-time = "2026-04-15T20:07:01.64Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f4/2e9f8ecbaca854bfdf14af8a9b505ec0cbc640377b3b218921594b7563cd/lupa-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc51250e76367a3e27fcd01dc769b9bfcbbc34f48df48dde53d6af6e75b7eaa5", size = 1814701, upload-time = "2026-04-15T20:07:04.149Z" }, + { url = "https://files.pythonhosted.org/packages/ba/53/4000b1acaa8b1f3827fcff0cfcdff44d3befddda42cab7e685a49689b5a1/lupa-2.8-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f8a22088a552828958603323f0a5c4b3e11e03b75d0bf4c965ef879de9b60a8d", size = 2348414, upload-time = "2026-04-15T20:07:07.285Z" }, + { url = "https://files.pythonhosted.org/packages/d5/78/26ee48d3890cddf03cefb65f433e3492759c0b3c0582180755bddbaab7bd/lupa-2.8-cp314-cp314t-win32.whl", hash = "sha256:4f7c553c1d8cfffbe85d81daef730d12cae4b6002d457542914da0ac8a1145b3", size = 1831611, upload-time = "2026-04-15T20:07:09.752Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d1/4a5cc64a3cad22821ae4c3f7a90456a08ca19457d8354f4abf46ad03c7e8/lupa-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:d8766aff03a78c80ad2d188a8bdb216de5ec838359cd87e05bbdfa56394a6105", size = 2209250, upload-time = "2026-04-15T20:07:11.906Z" }, + { url = "https://files.pythonhosted.org/packages/37/7c/cdcb654daf668192aaf36b0aeb94f2281dad092aaa5003688691131736ea/lupa-2.8-cp314-cp314t-win_arm64.whl", hash = "sha256:91d622777febda3ab1bed1d45295f2f32a4680c7b3d7caf8c669998ed5c44118", size = 1126735, upload-time = "2026-04-15T20:07:15.434Z" }, + { url = "https://files.pythonhosted.org/packages/1d/44/de1961ad38e17cd326a53c246c7e3b91178ed578f4cf22ffcd5e7e11b041/lupa-2.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b036738282a5acd2e71fdddb317c9df8b87c1673aa57f403d05fcc2be8abc4ba", size = 1186020, upload-time = "2026-04-15T20:07:35.017Z" }, + { url = "https://files.pythonhosted.org/packages/13/c2/276f0b9dc8bcc5a8a58af5316dfa0e6f56be3613dd6dbcc8d3d2cb6559ba/lupa-2.8-cp39-abi3-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:ac6b6e8d0e617e26a98cbb44880bcd75de5d32b3ad7b3b3793583909292b47ed", size = 1468944, upload-time = "2026-04-15T20:07:37.782Z" }, + { url = "https://files.pythonhosted.org/packages/63/38/52934e52a5180dc6425d20284d004fe4b27a4f9171a82dc99fb67af250bf/lupa-2.8-cp39-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ba3a7dd839f90c3d2e53bebe3c192b1f3f9fd720a6781256405123211fd0dce6", size = 1172998, upload-time = "2026-04-15T20:07:40.812Z" }, + { url = "https://files.pythonhosted.org/packages/c7/82/76b3809bd0839d9b3b4ec58d06591e08f17337b6d9576877cb9d48b34e94/lupa-2.8-cp39-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d7edb13a7a5250b5c6c22d1495d9e842b5c9fc5081c8fe6b5efe2112fe3e41f9", size = 1449975, upload-time = "2026-04-15T20:07:44.262Z" }, + { url = "https://files.pythonhosted.org/packages/16/07/2f89d54f747c67c23b4b9ae4aa8c8dd06bb409155dedcf406157f2736b66/lupa-2.8-cp39-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:891f72e0bffbed1e4175f975aeb2a083956586a100066525e1be485f617f7b25", size = 1281944, upload-time = "2026-04-15T20:07:46.458Z" }, + { url = "https://files.pythonhosted.org/packages/e7/bd/7375d2b0fcae79d806baf52a76f26c96964593f58e1372d13ae5ac09c676/lupa-2.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a295f87b5b7ebbfd5191932e8cb0e51df3c7769101ac6b6c7d7c9fb27bfd1307", size = 1910455, upload-time = "2026-04-15T20:07:49.75Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0c/8abb3bc0e08b311fc01db05b6e9f9ff31a8f65e4fc3f0aeb05cfef75c8ac/lupa-2.8-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4fe5d7a810b64ea8511eb885fc8cdde042ee5ff7b7d08ae78f32449756acb177", size = 1155548, upload-time = "2026-04-15T20:07:52.657Z" }, + { url = "https://files.pythonhosted.org/packages/80/2e/9eeecd3f493099721c1d3f31beeca23a4237db1a54223684df4dc96aa1bd/lupa-2.8-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:bfc470012ef66ad064c7bd77416af03a3452ef630b04b9012595ea13f2e54518", size = 1489232, upload-time = "2026-04-15T20:07:54.92Z" }, + { url = "https://files.pythonhosted.org/packages/c3/13/731c99dc2e7652ae818a6de45bdf0142049f7cb566049061c898355f1891/lupa-2.8-cp39-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:250e035fdaffe8c87093e3ebc206ac29a26131b1568ea711d780c26001ce96e7", size = 1466321, upload-time = "2026-04-15T20:07:57.627Z" }, + { url = "https://files.pythonhosted.org/packages/de/71/3ad8cc4fc05a77dc0d3f7079348bd1cad4675a0d14c24f8e6a3ce5f008f7/lupa-2.8-cp39-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:b9bddb09acfffb4f828f790f444b11dc0cca591afea1a244d9329eea2d20c003", size = 1288577, upload-time = "2026-04-15T20:07:59.913Z" }, + { url = "https://files.pythonhosted.org/packages/d8/b2/1175f6d0aa7b68627fbe2f58bd1e8bea36a89d10dfd67671d2b024c96162/lupa-2.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2e64acbbd47e9b82a64405a39e0d2b36a5a7dad8ab41c0f3437f572f7d282ba3", size = 2444866, upload-time = "2026-04-15T20:08:02.753Z" }, + { url = "https://files.pythonhosted.org/packages/92/f7/e78df680c7a0ea452daac07467ca188d63c2c00ca1c884c0a50e27eb83b5/lupa-2.8-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32e4e5103bbddcdd2458fb2ccae6c8ba11c9997c711d7e379e0d45551d109c76", size = 1778509, upload-time = "2026-04-15T20:08:21.784Z" }, + { url = "https://files.pythonhosted.org/packages/e6/23/0e53cabb16b2a8aa9cf1fde499c097d8942c5dab709fc8e921f3b824b18b/lupa-2.8-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7667001804657496dee9feced2daae5000b4604a3218dd8e6b7b754982ba88b8", size = 2300480, upload-time = "2026-04-15T20:08:24.394Z" }, + { url = "https://files.pythonhosted.org/packages/7e/85/0271227eab939921a12ebba5d17aa4cd18346aa534ca7f5da09cd0b63dd4/lupa-2.8-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:86f6f668966965b15247dc32d064cfe7be67b71e584ccfacbe2f637575296878", size = 1847445, upload-time = "2026-04-15T20:08:27.031Z" }, +] + [[package]] name = "lxml" version = "6.1.1"