mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
Merge 47b18d5c0f into 0eb6816081
This commit is contained in:
yi-coding
GitHub
commit
a859787d50
5 changed files with 614 additions and 11 deletions
|
|
@ -2,8 +2,10 @@ import asyncio
|
|||
import enum
|
||||
import heapq
|
||||
import logging as _logging
|
||||
import random as _random
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from functools import partial as _partial
|
||||
from typing import Final, TypeAlias
|
||||
from typing import Protocol as _Protocol
|
||||
|
||||
|
|
@ -12,9 +14,17 @@ from pydantic import TypeAdapter
|
|||
from litellm import print_verbose
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_logger as _verbose_logger
|
||||
from litellm._logging import verbose_router_logger as _verbose_router_logger
|
||||
from litellm.caching.caching import DualCache, RedisCache
|
||||
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError as _RedisCircuitBreakerOpenError
|
||||
from litellm.caching.redis_cache import log_redis_failure as _log_redis_failure
|
||||
from litellm.caching.redis_cache import (
|
||||
RedisCircuitBreakerOpenError as _RedisCircuitBreakerOpenError,
|
||||
)
|
||||
from litellm.caching.redis_cache import (
|
||||
_is_redis_health_failure, # pyright: ignore[reportPrivateUsage] # RedisCache's connectivity classifier has no public counterpart
|
||||
)
|
||||
from litellm.caching.redis_cache import (
|
||||
log_redis_failure as _log_redis_failure,
|
||||
)
|
||||
from litellm.constants import DEFAULT_IN_MEMORY_TTL, DEFAULT_POLLING_INTERVAL
|
||||
from litellm.exceptions import Timeout
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -22,6 +32,25 @@ from litellm.types.llms.base import LiteLLMBaseModel
|
|||
SCHEDULER_QUEUE_TARGET: Final = "scheduler_queue"
|
||||
QueueEntry: TypeAlias = tuple[int, str]
|
||||
_QUEUE_ENTRIES: Final = TypeAdapter(list[QueueEntry])
|
||||
_Queue: TypeAlias = tuple[QueueEntry, ...]
|
||||
_QueueResult: TypeAlias = tuple[bool, _Queue]
|
||||
_QUEUE_SNAPSHOT: Final = TypeAdapter(_Queue)
|
||||
_RAW_QUEUE: Final = TypeAdapter[str | bytes | None](str | bytes | None)
|
||||
_CAS_RESULT: Final = TypeAdapter(bool)
|
||||
_QUEUE_READ: Final = "return redis.call('GET', KEYS[1])"
|
||||
_QUEUE_COMPARE_SET: Final = """
|
||||
local current = redis.call('GET', KEYS[1])
|
||||
if (ARGV[1] == '0' and current ~= false) or
|
||||
(ARGV[1] == '1' and current ~= ARGV[2]) then
|
||||
return 0
|
||||
end
|
||||
if ARGV[4] == '' then
|
||||
redis.call('SET', KEYS[1], ARGV[3])
|
||||
else
|
||||
redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[4])
|
||||
end
|
||||
return 1
|
||||
"""
|
||||
|
||||
|
||||
class SchedulerCacheKeys(enum.Enum):
|
||||
|
|
@ -35,6 +64,98 @@ class FlowItem(LiteLLMBaseModel):
|
|||
model_name: str
|
||||
|
||||
|
||||
class _LocalQueueCache(_Protocol):
|
||||
@property
|
||||
def default_in_memory_ttl(self) -> float | None: ...
|
||||
|
||||
async def async_get_cache(self, key: str, *, local_only: bool) -> object: ...
|
||||
|
||||
async def async_set_cache(self, key: str, value: Sequence[QueueEntry], *, local_only: bool) -> None: ...
|
||||
|
||||
|
||||
async def _read_local_queue(cache: _LocalQueueCache, key: str) -> _Queue:
|
||||
local: Final = await cache.async_get_cache(key=key, local_only=True)
|
||||
return _QUEUE_SNAPSHOT.validate_python(local or ())
|
||||
|
||||
|
||||
async def _write_local_queue(cache: _LocalQueueCache, key: str, queue: _Queue) -> None:
|
||||
await cache.async_set_cache(key=key, value=list(queue), local_only=True)
|
||||
|
||||
|
||||
def _enqueue(entry: QueueEntry, queue: _Queue) -> _QueueResult:
|
||||
return False, queue if entry in queue else tuple(sorted((*queue, entry)))
|
||||
|
||||
|
||||
def _remove(request_id: str, queue: _Queue) -> _QueueResult:
|
||||
return False, tuple(sorted(item for item in queue if item[1] != request_id))
|
||||
|
||||
|
||||
def _poll(entry: QueueEntry, queue: _Queue) -> _QueueResult:
|
||||
pending: Final = tuple(sorted(queue if entry in queue else (*queue, entry)))
|
||||
return (True, pending[1:]) if pending[0] == entry else (False, pending)
|
||||
|
||||
|
||||
def _redis_unavailable(error: Exception) -> bool:
|
||||
return isinstance(error, _RedisCircuitBreakerOpenError) or _is_redis_health_failure(error)
|
||||
|
||||
|
||||
async def _read_redis_queue(cache: RedisCache, key: str) -> tuple[str | bytes | None, _Queue]:
|
||||
raw: Final = _RAW_QUEUE.validate_python(await cache.async_register_script(_QUEUE_READ)(keys=(key,), args=()))
|
||||
return raw, () if raw is None else _QUEUE_SNAPSHOT.validate_json(raw)
|
||||
|
||||
|
||||
async def _mutate_redis_queue(
|
||||
cache: RedisCache, key: str, model_name: str, ttl: int | None, change: Callable[[_Queue], _QueueResult]
|
||||
) -> _QueueResult:
|
||||
script: Final = cache.async_register_script(_QUEUE_COMPARE_SET)
|
||||
for attempt in range(100):
|
||||
raw, queue = await _read_redis_queue(cache, key)
|
||||
admitted, updated = change(queue)
|
||||
if updated == queue:
|
||||
return admitted, updated
|
||||
if _CAS_RESULT.validate_python(
|
||||
await script(
|
||||
keys=(key,),
|
||||
args=(
|
||||
"0" if raw is None else "1",
|
||||
raw or "",
|
||||
_QUEUE_SNAPSHOT.dump_json(updated),
|
||||
"" if ttl is None else str(ttl),
|
||||
),
|
||||
)
|
||||
):
|
||||
return admitted, updated
|
||||
if attempt < 99:
|
||||
await asyncio.sleep(_random.uniform(0, min(0.05, 0.001 * 2 ** min(attempt, 6))))
|
||||
raise Timeout(
|
||||
message="Scheduler queue remained busy after concurrent updates", model=model_name, llm_provider="openai"
|
||||
)
|
||||
|
||||
|
||||
async def _shared_queue_result(
|
||||
cache: _LocalQueueCache,
|
||||
redis_cache: RedisCache,
|
||||
key: str,
|
||||
model_name: str,
|
||||
change: Callable[[_Queue], _QueueResult],
|
||||
) -> _QueueResult:
|
||||
try:
|
||||
return await _mutate_redis_queue(
|
||||
redis_cache,
|
||||
key,
|
||||
model_name,
|
||||
redis_cache.default_ttl if cache.default_in_memory_ttl is None else int(cache.default_in_memory_ttl),
|
||||
change,
|
||||
)
|
||||
except Exception as error:
|
||||
if not _redis_unavailable(error):
|
||||
raise
|
||||
_log_redis_failure(
|
||||
_verbose_router_logger, _logging.WARNING, "Scheduler using local queue while Redis is unavailable", error
|
||||
)
|
||||
return change(await _read_local_queue(cache, key))
|
||||
|
||||
|
||||
class _QueueCache(_Protocol):
|
||||
async def async_get_cache(self, key: str, *, local_only: bool = False, raise_on_error: bool = False) -> object: ...
|
||||
|
||||
|
|
@ -74,7 +195,22 @@ class Scheduler:
|
|||
self.cache = DualCache(redis_cache=redis_cache, default_in_memory_ttl=default_in_memory_ttl)
|
||||
self.polling_interval = polling_interval or DEFAULT_POLLING_INTERVAL # default to 3ms
|
||||
|
||||
@with_service_target(SCHEDULER_QUEUE_TARGET)
|
||||
async def _update_shared_queue(
|
||||
self, redis_cache: RedisCache, model_name: str, change: Callable[[_Queue], _QueueResult]
|
||||
) -> bool:
|
||||
key: Final = f"{SchedulerCacheKeys.queue.value}:{model_name}"
|
||||
result: Final = await _shared_queue_result(self.cache, redis_cache, key, model_name, change)
|
||||
await _write_local_queue(self.cache, key, result[1])
|
||||
return result[0]
|
||||
|
||||
async def add_request(self, request: FlowItem):
|
||||
redis_cache: Final = self.cache.redis_cache
|
||||
if isinstance(redis_cache, RedisCache):
|
||||
await self._update_shared_queue(
|
||||
redis_cache, request.model_name, _partial(_enqueue, (request.priority, request.request_id))
|
||||
)
|
||||
return
|
||||
# We use the priority directly, as lower values indicate higher priority
|
||||
# get the queue
|
||||
queue: Final = await self.get_queue(model_name=request.model_name)
|
||||
|
|
@ -103,6 +239,11 @@ class Scheduler:
|
|||
if len(health_deployments) > 0:
|
||||
return True
|
||||
|
||||
redis_cache: Final = self.cache.redis_cache
|
||||
if isinstance(redis_cache, RedisCache):
|
||||
return await self._update_shared_queue(
|
||||
redis_cache, request.model_name, _partial(_poll, (request.priority, request.request_id))
|
||||
)
|
||||
queue: Final = await self.get_queue(model_name=request.model_name)
|
||||
entry: Final = (request.priority, request.request_id)
|
||||
print_verbose(f"queue: {queue}, seeking {entry}")
|
||||
|
|
@ -134,14 +275,24 @@ class Scheduler:
|
|||
return
|
||||
await asyncio.sleep(self.polling_interval)
|
||||
finally:
|
||||
await asyncio.shield(self.remove_request(request_id=request.request_id, model_name=request.model_name))
|
||||
await asyncio.shield(self._cleanup_request(request))
|
||||
raise Timeout(message="Request timed out while polling queue", model=request.model_name, llm_provider="openai")
|
||||
|
||||
async def _cleanup_request(self, request: FlowItem) -> None:
|
||||
try:
|
||||
await self.remove_request(request_id=request.request_id, model_name=request.model_name)
|
||||
except Exception as error: # noqa: BLE001 # Cleanup must preserve every request outcome, including cancellation
|
||||
_log_redis_failure(_verbose_router_logger, _logging.WARNING, "Scheduler queue cleanup failed", error)
|
||||
|
||||
async def remove_request(self, request_id: str, model_name: str) -> None:
|
||||
"""
|
||||
Remove a specific request from the priority queue for a model.
|
||||
Used when a request times out while waiting in the queue.
|
||||
"""
|
||||
redis_cache: Final = self.cache.redis_cache
|
||||
if isinstance(redis_cache, RedisCache):
|
||||
await self._update_shared_queue(redis_cache, model_name, _partial(_remove, request_id))
|
||||
return
|
||||
queue: Final = await self.get_queue(model_name=model_name)
|
||||
filtered_queue: Final = [item for item in queue if item[1] != request_id]
|
||||
heapq.heapify(filtered_queue) # restore heap invariant after filtering
|
||||
|
|
@ -176,6 +327,17 @@ class Scheduler:
|
|||
Redis hands the queue back as JSON lists, so every entry is validated into the
|
||||
(priority, request_id) tuple the heap operations compare against.
|
||||
"""
|
||||
redis_cache: Final = self.cache.redis_cache
|
||||
if isinstance(redis_cache, RedisCache):
|
||||
key: Final = f"{SchedulerCacheKeys.queue.value}:{model_name}"
|
||||
try:
|
||||
_, snapshot = await _read_redis_queue(redis_cache, key)
|
||||
except Exception as error:
|
||||
if not _redis_unavailable(error):
|
||||
raise
|
||||
return list(await _read_local_queue(self.cache, key))
|
||||
await _write_local_queue(self.cache, key, snapshot)
|
||||
return list(snapshot)
|
||||
if self.cache is not None:
|
||||
_cache_key: Final = f"{SchedulerCacheKeys.queue.value}:{model_name}"
|
||||
response: Final = await _read_queue(self.cache, self.cache.redis_cache, _cache_key)
|
||||
|
|
|
|||
|
|
@ -236,7 +236,7 @@ dev = [
|
|||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"langfuse>=4.7,<5.0",
|
||||
"fastapi-offline==1.7.6",
|
||||
"fakeredis==2.34.1",
|
||||
"fakeredis[lua]==2.34.1",
|
||||
"pytest-rerunfailures==15.1",
|
||||
"pytest-cov==5.0.0",
|
||||
"parameterized==0.9.0",
|
||||
|
|
|
|||
|
|
@ -1,15 +1,22 @@
|
|||
import asyncio
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Final
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from integration._support.async_server import LoopbackServer, close_writer
|
||||
from integration._support.ports import allocate_port
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.caching.affinity_cache import claim_affinity_pin, read_affinity_pin
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.caching.redis_cache import RedisCache
|
||||
from litellm.scheduler import FlowItem, Scheduler, SchedulerCacheKeys
|
||||
|
||||
|
||||
def _redis_cache() -> RedisCache:
|
||||
|
|
@ -95,6 +102,212 @@ async def test_sync_and_async_batch_reads_share_one_redis_without_sync_reads_goi
|
|||
assert await dual_cache.async_batch_get_cache(keys=[survivor_key]) == [expected[survivor_key]]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(loop_scope="function")
|
||||
async def shared_scheduler_cache() -> AsyncIterator[RedisCache]:
|
||||
cache: Final = RedisCache(
|
||||
host=os.environ["REDIS_HOST"],
|
||||
port=int(os.environ["REDIS_PORT"]),
|
||||
namespace="scheduler-atomic-" + uuid.uuid4().hex,
|
||||
)
|
||||
try:
|
||||
yield cache
|
||||
finally:
|
||||
client: Final = cache.init_async_client()
|
||||
await client.delete(cache.check_and_fix_namespace(f"{SchedulerCacheKeys.queue.value}:shared"))
|
||||
await client.aclose()
|
||||
cache.redis_client.close()
|
||||
|
||||
|
||||
async def test_shared_scheduler_preserves_concurrent_admissions(shared_scheduler_cache: RedisCache) -> None:
|
||||
first: Final = Scheduler(redis_cache=shared_scheduler_cache)
|
||||
second: Final = Scheduler(redis_cache=shared_scheduler_cache)
|
||||
await asyncio.gather(first.get_queue("shared"), second.get_queue("shared"))
|
||||
await asyncio.gather(
|
||||
first.add_request(FlowItem(priority=1, request_id="first", model_name="shared")),
|
||||
second.add_request(FlowItem(priority=2, request_id="second", model_name="shared")),
|
||||
)
|
||||
assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == [[1, "first"], [2, "second"]]
|
||||
assert await first.get_queue("shared") == [(1, "first"), (2, "second")]
|
||||
assert await second.get_queue("shared") == [(1, "first"), (2, "second")]
|
||||
|
||||
|
||||
async def test_shared_scheduler_does_not_resurrect_peer_removal(shared_scheduler_cache: RedisCache) -> None:
|
||||
await shared_scheduler_cache.async_set_cache("scheduler:queue:shared", [[1, "removed"], [2, "waiting"]])
|
||||
first: Final = Scheduler(redis_cache=shared_scheduler_cache)
|
||||
second: Final = Scheduler(redis_cache=shared_scheduler_cache)
|
||||
await asyncio.gather(first.get_queue("shared"), second.get_queue("shared"))
|
||||
await first.remove_request("removed", "shared")
|
||||
await second.add_request(FlowItem(priority=3, request_id="new", model_name="shared"))
|
||||
assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == [[2, "waiting"], [3, "new"]]
|
||||
|
||||
|
||||
async def test_shared_scheduler_observes_peer_pop(shared_scheduler_cache: RedisCache) -> None:
|
||||
await shared_scheduler_cache.async_set_cache("scheduler:queue:shared", [[1, "first"], [2, "second"]])
|
||||
first: Final = Scheduler(redis_cache=shared_scheduler_cache)
|
||||
second: Final = Scheduler(redis_cache=shared_scheduler_cache)
|
||||
await asyncio.gather(first.get_queue("shared"), second.get_queue("shared"))
|
||||
assert await first.poll(FlowItem(priority=1, request_id="first", model_name="shared"), [])
|
||||
assert await second.poll(FlowItem(priority=2, request_id="second", model_name="shared"), [])
|
||||
assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == []
|
||||
|
||||
|
||||
async def test_shared_scheduler_keeps_other_requests_during_concurrent_cleanup(
|
||||
shared_scheduler_cache: RedisCache,
|
||||
) -> None:
|
||||
schedulers: Final = tuple(Scheduler(redis_cache=shared_scheduler_cache) for _ in range(16))
|
||||
await asyncio.gather(
|
||||
*(
|
||||
scheduler.add_request(FlowItem(priority=i, request_id=str(i), model_name="shared"))
|
||||
for i, scheduler in enumerate(schedulers)
|
||||
)
|
||||
)
|
||||
await asyncio.gather(
|
||||
*(scheduler.remove_request(str(i), "shared") for i, scheduler in enumerate(schedulers) if i % 2 == 0)
|
||||
)
|
||||
assert await schedulers[0].get_queue("shared") == [(i, str(i)) for i in range(16) if i % 2]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def interrupted_scheduler_cache(shared_scheduler_cache: RedisCache) -> Iterator[tuple[RedisCache, threading.Event]]:
|
||||
interrupted: Final = threading.Event()
|
||||
|
||||
async def relay(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
||||
upstream_reader, upstream_writer = await asyncio.open_connection(
|
||||
os.environ["REDIS_HOST"], int(os.environ["REDIS_PORT"])
|
||||
)
|
||||
|
||||
async def forward(source: asyncio.StreamReader, destination: asyncio.StreamWriter) -> None:
|
||||
try:
|
||||
while chunk := await source.read(65536):
|
||||
if interrupted.is_set():
|
||||
return
|
||||
destination.write(chunk)
|
||||
await destination.drain()
|
||||
except ConnectionError:
|
||||
return
|
||||
finally:
|
||||
await close_writer(destination)
|
||||
|
||||
await asyncio.gather(forward(reader, upstream_writer), forward(upstream_reader, writer))
|
||||
|
||||
port: Final = allocate_port()
|
||||
server: Final = LoopbackServer(port, relay)
|
||||
server.start()
|
||||
cache: Final = RedisCache(host="127.0.0.1", port=port, namespace=shared_scheduler_cache.namespace)
|
||||
try:
|
||||
yield cache, interrupted
|
||||
finally:
|
||||
cache.redis_client.close()
|
||||
server.stop()
|
||||
|
||||
|
||||
async def test_shared_scheduler_keeps_local_fallback_out_of_recovered_shared_queue(
|
||||
shared_scheduler_cache: RedisCache,
|
||||
interrupted_scheduler_cache: tuple[RedisCache, threading.Event],
|
||||
) -> None:
|
||||
cache, interrupted = interrupted_scheduler_cache
|
||||
scheduler: Final = Scheduler(redis_cache=cache)
|
||||
try:
|
||||
await scheduler.add_request(FlowItem(priority=1, request_id="removed", model_name="shared"))
|
||||
interrupted.set()
|
||||
await scheduler.add_request(FlowItem(priority=2, request_id="local", model_name="shared"))
|
||||
assert await scheduler.get_queue("shared") == [(1, "removed"), (2, "local")]
|
||||
await Scheduler(redis_cache=shared_scheduler_cache).remove_request("removed", "shared")
|
||||
interrupted.clear()
|
||||
await scheduler.add_request(FlowItem(priority=3, request_id="new", model_name="shared"))
|
||||
assert await scheduler.get_queue("shared") == [(3, "new")]
|
||||
assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == [[3, "new"]]
|
||||
finally:
|
||||
interrupted.clear()
|
||||
await cache.init_async_client().aclose()
|
||||
|
||||
|
||||
async def test_shared_scheduler_preserves_invalid_queue_for_diagnosis(shared_scheduler_cache: RedisCache) -> None:
|
||||
await shared_scheduler_cache.async_set_cache("scheduler:queue:shared", {"invalid": "queue"})
|
||||
scheduler: Final = Scheduler(redis_cache=shared_scheduler_cache)
|
||||
with pytest.raises(ValidationError):
|
||||
await scheduler.add_request(FlowItem(priority=1, request_id="first", model_name="shared"))
|
||||
assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == {"invalid": "queue"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("priority", [0, -1, 2**80])
|
||||
async def test_shared_scheduler_retains_priority_and_request_id(
|
||||
shared_scheduler_cache: RedisCache, priority: int
|
||||
) -> None:
|
||||
scheduler: Final = Scheduler(redis_cache=shared_scheduler_cache)
|
||||
scheduler.cache.update_cache_ttl(default_in_memory_ttl=17, default_redis_ttl=None)
|
||||
await scheduler.add_request(FlowItem(priority=priority, request_id="请求-é", model_name="shared"))
|
||||
assert await scheduler.get_queue("shared") == [(priority, "请求-é")]
|
||||
assert (
|
||||
0
|
||||
< shared_scheduler_cache.redis_client.ttl(
|
||||
shared_scheduler_cache.check_and_fix_namespace("scheduler:queue:shared")
|
||||
)
|
||||
<= 17
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outcome", ["admitted", "failed", "cancelled"])
|
||||
async def test_shared_scheduler_cleanup_failure_preserves_request_outcome(
|
||||
shared_scheduler_cache: RedisCache,
|
||||
outcome: Literal["admitted", "failed", "cancelled"],
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
scheduler: Final = Scheduler(redis_cache=shared_scheduler_cache)
|
||||
original: Final = RuntimeError("health lookup failed")
|
||||
|
||||
async def health_after_corrupting_queue() -> tuple[object, ...]:
|
||||
await shared_scheduler_cache.async_set_cache("scheduler:queue:shared", {"invalid": "queue"})
|
||||
match outcome:
|
||||
case "failed":
|
||||
raise original
|
||||
case "cancelled":
|
||||
raise asyncio.CancelledError
|
||||
case "admitted":
|
||||
return ("healthy",)
|
||||
|
||||
async def wait() -> None:
|
||||
await scheduler.wait_for_turn(
|
||||
FlowItem(priority=1, request_id="cleanup", model_name="shared"),
|
||||
timeout=5,
|
||||
get_healthy_deployments=health_after_corrupting_queue,
|
||||
)
|
||||
|
||||
match outcome:
|
||||
case "admitted":
|
||||
await wait()
|
||||
case "failed":
|
||||
with pytest.raises(RuntimeError) as caught:
|
||||
await wait()
|
||||
assert caught.value is original
|
||||
case "cancelled":
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wait()
|
||||
assert "Scheduler queue cleanup failed" in caplog.text
|
||||
assert await shared_scheduler_cache.async_get_cache("scheduler:queue:shared") == {"invalid": "queue"}
|
||||
|
||||
|
||||
async def test_shared_scheduler_handles_three_hundred_concurrent_enqueues_and_removals(
|
||||
shared_scheduler_cache: RedisCache,
|
||||
) -> None:
|
||||
schedulers: Final = tuple(Scheduler(redis_cache=shared_scheduler_cache) for _ in range(300))
|
||||
for scheduler in schedulers:
|
||||
scheduler.cache.update_cache_ttl(default_in_memory_ttl=60, default_redis_ttl=None)
|
||||
requests: Final = tuple(
|
||||
FlowItem(priority=i % 8, request_id=f"burst-{i:03}", model_name="shared") for i in range(300)
|
||||
)
|
||||
|
||||
await asyncio.gather(*(scheduler.add_request(request) for scheduler, request in zip(schedulers, requests)))
|
||||
|
||||
assert await schedulers[0].get_queue("shared") == sorted(
|
||||
(request.priority, request.request_id) for request in requests
|
||||
)
|
||||
await asyncio.gather(
|
||||
*(scheduler.remove_request(request.request_id, "shared") for scheduler, request in zip(schedulers, requests))
|
||||
)
|
||||
assert await schedulers[0].get_queue("shared") == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_session_repin_claims_keep_the_shared_winner_after_recovery() -> None:
|
||||
redis: Final = _redis_cache()
|
||||
|
|
|
|||
|
|
@ -5,11 +5,16 @@ import asyncio
|
|||
import importlib
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from typing import Final
|
||||
from collections.abc import Iterator, Sequence
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from redis.exceptions import ConnectionError as RedisConnectionError
|
||||
|
||||
import time
|
||||
|
||||
import redis
|
||||
import redis.asyncio
|
||||
from fakeredis import FakeRedis, FakeServer
|
||||
|
|
@ -20,6 +25,7 @@ import litellm
|
|||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.caching.redis_cache import RedisCache
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.exceptions import Timeout
|
||||
from litellm.scheduler import FlowItem, Scheduler, SchedulerCacheKeys
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
|
|
@ -282,7 +288,6 @@ async def test_wait_for_turn_during_cooldown_survives_the_queue_key_expiring():
|
|||
assert await scheduler.get_queue("sched-model") == []
|
||||
|
||||
|
||||
|
||||
class _JsonRoundTripRedisCache:
|
||||
def __init__(self) -> None:
|
||||
self.store: dict[str, str] = {}
|
||||
|
|
@ -312,6 +317,7 @@ async def test_second_replica_enqueues_behind_a_queue_decoded_from_redis():
|
|||
await replica_b.remove_request(request_id="waiting-on-a", model_name="sched-model")
|
||||
assert await replica_b.get_queue("sched-model") == []
|
||||
|
||||
|
||||
class _PausesAfterEnqueueScheduler(Scheduler):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
|
@ -387,6 +393,160 @@ async def test_wait_for_turn_finishes_removal_when_cancelled_again_during_cleanu
|
|||
assert await scheduler.get_queue("sched-model") == [(0, "head")]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scheduler_redis_replies() -> Iterator[tuple[RedisCache, AsyncMock, AsyncMock]]:
|
||||
reads: Final = AsyncMock(return_value=None)
|
||||
writes: Final = AsyncMock(return_value=1)
|
||||
|
||||
async def execute(
|
||||
*, keys: Sequence[str], args: Sequence[str | bytes | int | float], client: object = None
|
||||
) -> object:
|
||||
return await (writes if args else reads)(keys=keys, args=args, client=client)
|
||||
|
||||
with (
|
||||
patch("redis.Redis.execute_command", return_value={"redis_version": "unit-fixture"}),
|
||||
patch("redis.asyncio.Redis.execute_command", new_callable=AsyncMock, return_value=True),
|
||||
patch("redis.commands.core.AsyncScript.__call__", side_effect=execute),
|
||||
):
|
||||
cache: Final = RedisCache(host="scheduler-unit.invalid", port=6379)
|
||||
yield cache, reads, writes
|
||||
cache.redis_client.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shared_scheduler_retries_conflicts_using_the_new_peer_snapshot(
|
||||
scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock],
|
||||
) -> None:
|
||||
cache, reads, writes = scheduler_redis_replies
|
||||
scheduler: Final = Scheduler(redis_cache=cache)
|
||||
reads.side_effect = (None, b'[[0,"peer"]]', RedisConnectionError("offline"))
|
||||
writes.side_effect = (0, 1)
|
||||
|
||||
await scheduler.add_request(FlowItem(priority=1, request_id="new", model_name="shared"))
|
||||
|
||||
assert await scheduler.get_queue("shared") == [(0, "peer"), (1, "new")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shared_scheduler_stops_after_conflicts_without_publishing_an_uncommitted_queue(
|
||||
scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock],
|
||||
) -> None:
|
||||
cache, _reads, writes = scheduler_redis_replies
|
||||
scheduler: Final = Scheduler(redis_cache=cache)
|
||||
writes.return_value = 0
|
||||
|
||||
with patch("random.uniform", return_value=0), pytest.raises(Timeout, match="queue remained busy"):
|
||||
await scheduler.add_request(FlowItem(priority=1, request_id="never-committed", model_name="shared"))
|
||||
|
||||
assert await scheduler.cache.async_get_cache("scheduler:queue:shared", local_only=True) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["enqueue", "read"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_shared_scheduler_rejects_malformed_snapshots_without_replacing_local_history(
|
||||
scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock], operation: str
|
||||
) -> None:
|
||||
cache, reads, _writes = scheduler_redis_replies
|
||||
scheduler: Final = Scheduler(redis_cache=cache)
|
||||
await scheduler.cache.async_set_cache("scheduler:queue:shared", [(0, "last-known")], local_only=True)
|
||||
reads.side_effect = (b'{"invalid":"queue"}',)
|
||||
|
||||
call: Final = (
|
||||
scheduler.add_request(FlowItem(priority=1, request_id="new", model_name="shared"))
|
||||
if operation == "enqueue"
|
||||
else scheduler.get_queue("shared")
|
||||
)
|
||||
with pytest.raises(ValidationError):
|
||||
await call
|
||||
|
||||
assert await scheduler.cache.async_get_cache("scheduler:queue:shared", local_only=True) == [(0, "last-known")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outcome", ["admitted", "failed", "cancelled"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_shared_scheduler_cleanup_failure_preserves_the_request_outcome(
|
||||
scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock],
|
||||
outcome: Literal["admitted", "failed", "cancelled"],
|
||||
) -> None:
|
||||
cache, reads, _writes = scheduler_redis_replies
|
||||
scheduler: Final = Scheduler(redis_cache=cache)
|
||||
original: Final = RuntimeError("health lookup failed")
|
||||
reads.side_effect = (None, b'{"invalid":"queue"}')
|
||||
|
||||
async def health() -> Sequence[object]:
|
||||
match outcome:
|
||||
case "failed":
|
||||
raise original
|
||||
case "cancelled":
|
||||
raise asyncio.CancelledError
|
||||
case "admitted":
|
||||
return ("healthy",)
|
||||
|
||||
async def wait() -> None:
|
||||
await scheduler.wait_for_turn(
|
||||
FlowItem(priority=1, request_id="waiting", model_name="shared"), timeout=5, get_healthy_deployments=health
|
||||
)
|
||||
|
||||
match outcome:
|
||||
case "admitted":
|
||||
await wait()
|
||||
case "failed":
|
||||
with pytest.raises(RuntimeError) as caught:
|
||||
await wait()
|
||||
assert caught.value is original
|
||||
case "cancelled":
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wait()
|
||||
assert await scheduler.cache.async_get_cache("scheduler:queue:shared", local_only=True) == [(1, "waiting")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shared_scheduler_keeps_admission_and_removal_in_its_fallback_snapshot(
|
||||
scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock],
|
||||
) -> None:
|
||||
cache, reads, _writes = scheduler_redis_replies
|
||||
scheduler: Final = Scheduler(redis_cache=cache)
|
||||
head: Final = FlowItem(priority=0, request_id="head", model_name="shared")
|
||||
later: Final = FlowItem(priority=2, request_id="later", model_name="shared")
|
||||
reads.side_effect = (
|
||||
b'[[0,"head"],[2,"later"]]',
|
||||
b'[[0,"head"],[2,"later"]]',
|
||||
b'[[0,"head"],[2,"later"]]',
|
||||
RedisConnectionError("offline after admission"),
|
||||
b'[[2,"later"],[3,"peer"]]',
|
||||
RedisConnectionError("offline after removal"),
|
||||
b'[[3,"peer"]]',
|
||||
)
|
||||
|
||||
await scheduler.add_request(head)
|
||||
assert not await scheduler.poll(later, [])
|
||||
assert await scheduler.poll(head, [])
|
||||
assert await scheduler.get_queue("shared") == [(2, "later")]
|
||||
await scheduler.remove_request("later", "shared")
|
||||
assert await scheduler.get_queue("shared") == [(3, "peer")]
|
||||
assert await scheduler.get_queue("shared") == [(3, "peer")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shared_scheduler_does_not_upload_outage_only_requests_when_redis_recovers(
|
||||
scheduler_redis_replies: tuple[RedisCache, AsyncMock, AsyncMock],
|
||||
) -> None:
|
||||
cache, reads, writes = scheduler_redis_replies
|
||||
scheduler: Final = Scheduler(redis_cache=cache)
|
||||
reads.side_effect = (
|
||||
RedisConnectionError("offline during enqueue"),
|
||||
RedisConnectionError("still offline"),
|
||||
b'[[0,"peer"]]',
|
||||
RedisConnectionError("offline after recovery"),
|
||||
)
|
||||
|
||||
await scheduler.add_request(FlowItem(priority=1, request_id="local-only", model_name="shared"))
|
||||
assert await scheduler.get_queue("shared") == [(1, "local-only")]
|
||||
await scheduler.add_request(FlowItem(priority=2, request_id="new", model_name="shared"))
|
||||
assert await scheduler.get_queue("shared") == [(0, "peer"), (2, "new")]
|
||||
assert json.loads(writes.call_args.kwargs["args"][2]) == [[0, "peer"], [2, "new"]]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _vcr_outcome_gate(request, vcr):
|
||||
install_live_call_probe(request, vcr)
|
||||
|
|
|
|||
72
uv.lock
generated
72
uv.lock
generated
|
|
@ -2127,6 +2127,11 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/49/b5/82f89307d0d769cd9bf46a54fb9136be08e4e57c5570ae421db4c9a2ba62/fakeredis-2.34.1-py3-none-any.whl", hash = "sha256:0107ec99d48913e7eec2a5e3e2403d1bd5f8aa6489d1a634571b975289c48f12", size = 122160, upload-time = "2026-02-25T13:17:49.701Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
lua = [
|
||||
{ name = "lupa" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastapi"
|
||||
version = "0.136.3"
|
||||
|
|
@ -5222,7 +5227,7 @@ dev = [
|
|||
{ name = "basedpyright" },
|
||||
{ name = "botocore-stubs" },
|
||||
{ name = "diff-cover" },
|
||||
{ name = "fakeredis" },
|
||||
{ name = "fakeredis", extra = ["lua"] },
|
||||
{ name = "fastapi-offline" },
|
||||
{ name = "hypothesis" },
|
||||
{ name = "keyring" },
|
||||
|
|
@ -5459,7 +5464,7 @@ dev = [
|
|||
{ name = "basedpyright", specifier = "==1.39.7" },
|
||||
{ name = "botocore-stubs", specifier = "==1.43.14" },
|
||||
{ name = "diff-cover", specifier = "==9.7.2" },
|
||||
{ name = "fakeredis", specifier = "==2.34.1" },
|
||||
{ name = "fakeredis", extras = ["lua"], specifier = "==2.34.1" },
|
||||
{ name = "fastapi-offline", specifier = "==1.7.6" },
|
||||
{ name = "hypothesis", specifier = "==6.165.10" },
|
||||
{ name = "keyring", specifier = "==25.7.0" },
|
||||
|
|
@ -5693,6 +5698,69 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/eb/cf/e4e016820516c1f0c85549e356865c2319cf6afba8ab386611b1d03cf2b6/lunary-1.4.37-py3-none-any.whl", hash = "sha256:7289330e851a481404c92213ce480c0c7be9bfa56982a1385e08e3665abefe05", size = 25581, upload-time = "2026-02-12T08:15:03.892Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lupa"
|
||||
version = "2.8"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c3/a6/0f869fbb07c393f15473b1eefefb7b5bec162fb7481803d040ed4dc46002/lupa-2.8.tar.gz", hash = "sha256:d8022641b9ec8ecf2c5ecbe9f47e5a70e0b87c4b5ae921b92cb02a638e0acd08", size = 6156370, upload-time = "2026-04-15T20:08:30.534Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/09/21/9be4516ddd22f8eadba336d9ba065d17d79108465ae1b7f71424ab99b9d0/lupa-2.8-cp310-abi3-win32.whl", hash = "sha256:c2a5fd15dc62374e1661a55f01744c9ec1c56f291ba4a0749d3af2174556e78f", size = 1594887, upload-time = "2026-04-15T20:05:23.377Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2d/99/1557c9685d7034d9ce8dd2b54c40a26d6deb7c67c1fdb5c801abd1a02c3f/lupa-2.8-cp310-abi3-win_arm64.whl", hash = "sha256:9e304fb1c50cf23fd8882afbe1aa87525ef8a72667bcab3b37b2bbb2bc542269", size = 1371742, upload-time = "2026-04-15T20:05:27.417Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/34/05ce4745b191633f90ff1ab50f1a19a37da282bb0a41fb500d9157fc9b8f/lupa-2.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:97bd01e90b8031e56a5fd5bb70605aea09f1dba675c1140308a52780f93d06f1", size = 1202714, upload-time = "2026-04-15T20:05:31.088Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7d/d2/f70fdbeec2d4c69ee6a469e6cddde9635fff4af4e13fb652e6a1229eef51/lupa-2.8-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b5ebe1a13c45767919c86750b84fe2da9f6288b6f3cea4ce7660bb2abc9d921", size = 1857453, upload-time = "2026-04-15T20:05:34.611Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/dc/6fcda0e36e75eb6cb98dc9190fa4737d727eeae29e58f892980b2c96b656/lupa-2.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:097e7d0f1719a88020b67c82e05d53d7973c166952393afcecfd8434c7e19a15", size = 2408890, upload-time = "2026-04-15T20:05:37.994Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/58/29/7ea176eac3c1dac83d059762daa875ad1390decc0bf2c3b4c7bbfc1f1665/lupa-2.8-cp310-cp310-win_amd64.whl", hash = "sha256:7bb223ee8f72d0dc076b0d65296ee72f1c69450f9d2fed5315f7707d98c4a03d", size = 1910396, upload-time = "2026-04-15T20:05:41.163Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/0a/5a740717f27aa77481e6a61b97cf79d1e0c1ede729b1268caacded915326/lupa-2.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b12e43c1fb787189dfc28cd604aef0baa2cb95e27da19498d520361d0ace070a", size = 1202376, upload-time = "2026-04-15T20:05:44.049Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/75/6b64d0098c64275a801896cb7a6a30e7e653d25fa102c64e747292afcdbb/lupa-2.8-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6f603391dffb256e36a79fd2044084d5f4b8a0a4c0e5ad291cd3ab3aaf1fd0a", size = 1839271, upload-time = "2026-04-15T20:05:47.399Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/2f/0d4f00563046ff616ef6a421f8b776a5ffb327f7b32ed69e856d52b917a8/lupa-2.8-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f6f41c91366e7d0d474f87d81c1274af861f40812bf729c9f97ab4c8f3c7ac8", size = 2376251, upload-time = "2026-04-15T20:05:49.891Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4c/8e/caa83237f427d9e85b7f02c816e7270c9c9571dec1673e06b0180402f70e/lupa-2.8-cp311-cp311-win_amd64.whl", hash = "sha256:f5a6af145b0ea818f01d27bfe2583a4b538570bef61d22c8773e0eccf011234c", size = 1923488, upload-time = "2026-04-15T20:05:52.954Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ad/0b/368f2f0bc750b25c69d4563e44f677925ab5dd3d2887f9b0c15465d21a2a/lupa-2.8-cp312-abi3-macosx_10_13_x86_64.whl", hash = "sha256:f4342f4de76ae7ce2ab0672d36003bdb7e1a33252f293b569298ddd792e70e33", size = 1194056, upload-time = "2026-04-15T20:05:55.794Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/0f/c89eb8dd36fdea4e50ae3f7f5275bea3b0cc5d4057b8ee7b3bbc78010422/lupa-2.8-cp312-abi3-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:4203fa1659315e939a5304e75001b8cc14234fb3cbb3ed86c049b0cc5d90fcee", size = 1434278, upload-time = "2026-04-15T20:05:57.94Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/47/30/c3b4d2cd8733621b404b8a4214e5f852955c4ba632546dc84123bea9ee89/lupa-2.8-cp312-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:81f2d843ce668b653146c007467570210ae44be51dac6926666c51d49536f307", size = 1150068, upload-time = "2026-04-15T20:06:01.04Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/d2/bac12c398519efafc6af84be1974edd0d7a4895fb4735b5c8d615d298595/lupa-2.8-cp312-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d3d0cde2c77588d1c60875a4f34f059513476c6e1775351897195b51e0f3df08", size = 1409532, upload-time = "2026-04-15T20:06:03.592Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9c/6a/18b52e11962014026e07813530b0b108ee8bc0a2a13ef0eaea5d41dce023/lupa-2.8-cp312-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e0d11b8f3a8dac6413f704fef7161d048bb10c58bdac6cbffa5e60efa56e9a3", size = 1242687, upload-time = "2026-04-15T20:06:06.863Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/8e/7fd4eb049875f61429b96780d2eae4700f0e78fe0a52db8edb231b1cd09f/lupa-2.8-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:54cff414f21f8cd8c6be4aae52541f3b9cd39602b59e3a3db9b5c9f9f674ff18", size = 1856038, upload-time = "2026-04-15T20:06:09.358Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/f9/37ad9d2773d30f2931890d310a4bdce28d45484206e6f48bc18b0325eabd/lupa-2.8-cp312-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:24b4d8af5558e549b70daf1547f5c1c1d664ecea9fc790f83efe5d75e9a93797", size = 1128982, upload-time = "2026-04-15T20:06:12.312Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/57/31/c0fd7984c24844ea79caa45c0235f61a06b38fd69a839f6c62770f8d684a/lupa-2.8-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:ce86dff1ee7f7cf45f5622065ae991949dd7bb1703581cbc58a630137bb7ccf9", size = 1457594, upload-time = "2026-04-15T20:06:15.881Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/11/f5/a28e411be30ec1bf0db1eb0c087eebc73be9e7a1adcfe6ac209861ccc446/lupa-2.8-cp312-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:f4d01b2a08c70bbb883a9e082b6b36b89121ed5910b710f1ba11c73295ff4fba", size = 1425721, upload-time = "2026-04-15T20:06:18.009Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/c1/359f767c4ae024be30d909fe8a9f0e9af266bad47ce2bd2ed248fb986fcf/lupa-2.8-cp312-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:7f210d5a8353e510ea1199c42cf3cbdd630553bf2bc8fb4c00fea06fdec7c798", size = 1253258, upload-time = "2026-04-15T20:06:21.17Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/17/52/473f11790c261fd02bbf318a546fe040e9ec9f677181272fa78d3b4112a4/lupa-2.8-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4f81a02806e7c7ad26d8c6fa222c8bef1b0c1b124347c879be880b41339d41e4", size = 2395272, upload-time = "2026-04-15T20:06:24.137Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/bf/75c8795655a8836eab6a11a630352c4b7c5dc5c54d075077bc9bffdeee45/lupa-2.8-cp312-abi3-win32.whl", hash = "sha256:360056453a7a4eaa4ac5a204c31a5a014b1eb2ee5490603234d2ba831684f1f2", size = 1606136, upload-time = "2026-04-15T20:06:27.815Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/29/11a2cdd612b6f55e506292dfb6ba343216e80a693e7fe3f876ef204ce9c6/lupa-2.8-cp312-abi3-win_arm64.whl", hash = "sha256:1628371c6592a6d5650497a9e31fb2bb3a7e9883c1f301d1111265e484045af9", size = 1364495, upload-time = "2026-04-15T20:06:30.254Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4d/17/fa834b6b09ad17e7df5d0f7715d64877a125a3776ada689751a1f9dc2959/lupa-2.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:450650f91c48c2415b0d59ab3abfcfda3b6efb5b858205f4d4bda8ad141fa529", size = 1190111, upload-time = "2026-04-15T20:06:32.84Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ab/43/45589901b7d1a0e3a9d91d19a311fb6a56924e8571536c3f2212160fd953/lupa-2.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27044f3363047f946b3d3aab9157cbd172b3538ada9ec1baef43432bf7d03a78", size = 1812999, upload-time = "2026-04-15T20:06:35.664Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/ac/4ade7d15ff5c61758d7943ac6f0a496bf1cc65b6c09f842b52a0702e664c/lupa-2.8-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8cf4f064a0e5531afce2d7d750120c10c10f9529139af6ca6150d13151034398", size = 2368731, upload-time = "2026-04-15T20:06:37.959Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/27/05f950d15b8ab120b39c43588b438ff3ace70c1b1b0225a960393a497483/lupa-2.8-cp312-cp312-win_amd64.whl", hash = "sha256:281bedc5deb92d31e649a3552edd662449365a635904fa4d5cb4509c7245e34e", size = 1941809, upload-time = "2026-04-15T20:06:40.302Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/3f/19f83c3a0c84dc8bea8a58e7416dca6a3ede662c33c8d1ec758e5afc754a/lupa-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45fc9da0145ecb0083ef5ff9975116cc784bd0258bdc2bd131ba15483ce18398", size = 1201203, upload-time = "2026-04-15T20:06:42.169Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/0f/a14f0073f09610158038582e230618a48c14da6bd88185289461aa4cb854/lupa-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:58e18afed57955b41130e269c78f53d4123ab86e236b53816f4cbffa25cb5d30", size = 1806210, upload-time = "2026-04-15T20:06:45.486Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2f/14/48fff156c63a136001a7620878af7d31aa07e66b495ed621e3eddd73c294/lupa-2.8-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc47f536ac13a79cef47d29a2b205576a22841f042a2bcec1676b95806e7706a", size = 2359005, upload-time = "2026-04-15T20:06:47.819Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fe/18/3ac638ec90edf178242b8a2b2f00f8adae694248c03a26341ef941bb746e/lupa-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:ce9404c661dbac65cc9bed351ad45e797af93d30d70be309a3fa8209ac86d93b", size = 1936754, upload-time = "2026-04-15T20:06:50.448Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b0/ef/5ee5fed6ea7459a671196359ce04bfeeaf26be1dac8ff24bf28e5c7a6e81/lupa-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:348c3f8ecabb6324dcbc05c2740d762ef8fcec7b06c79e45262ab97a217684e3", size = 1209388, upload-time = "2026-04-15T20:06:53.022Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/b1/67a940d5542cb0384b443fe951b5a83ea9340d1333a733a258fdd1c619ba/lupa-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:951496471056061598a7d1729a6cdf48d662fec777a9f2d8aa5a1e62fd30e5a5", size = 1826821, upload-time = "2026-04-15T20:06:55.699Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/a2/b354e5ba3b911ec50686003dc8897e892b9e8c5c036b33219b03d54c4daf/lupa-2.8-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a591b9947ca347b41a63370e121d6e2b1458fe6dde9ae065029ec10a37f25ff4", size = 2366893, upload-time = "2026-04-15T20:06:58.9Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/52/d76066401f29539df5352f70ecded66576f32933b6045cd0bfc56cb770b9/lupa-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:3903c9cf628dae2f56405503247b77a61a3a61bd2dda470e336950c74776d55d", size = 1994716, upload-time = "2026-04-15T20:07:19.194Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/bd/3efc437a4361c16d25e66478c50357c9a8e8ecfb718fe749eb9ca3176ef6/lupa-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f711a8ab0486b9ac6fdda94a22ddcfbc9f0d4a27e3a8cf1bf79c6e48b33017c1", size = 1251217, upload-time = "2026-04-15T20:07:01.64Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ea/f4/2e9f8ecbaca854bfdf14af8a9b505ec0cbc640377b3b218921594b7563cd/lupa-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc51250e76367a3e27fcd01dc769b9bfcbbc34f48df48dde53d6af6e75b7eaa5", size = 1814701, upload-time = "2026-04-15T20:07:04.149Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/53/4000b1acaa8b1f3827fcff0cfcdff44d3befddda42cab7e685a49689b5a1/lupa-2.8-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f8a22088a552828958603323f0a5c4b3e11e03b75d0bf4c965ef879de9b60a8d", size = 2348414, upload-time = "2026-04-15T20:07:07.285Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/78/26ee48d3890cddf03cefb65f433e3492759c0b3c0582180755bddbaab7bd/lupa-2.8-cp314-cp314t-win32.whl", hash = "sha256:4f7c553c1d8cfffbe85d81daef730d12cae4b6002d457542914da0ac8a1145b3", size = 1831611, upload-time = "2026-04-15T20:07:09.752Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/d1/4a5cc64a3cad22821ae4c3f7a90456a08ca19457d8354f4abf46ad03c7e8/lupa-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:d8766aff03a78c80ad2d188a8bdb216de5ec838359cd87e05bbdfa56394a6105", size = 2209250, upload-time = "2026-04-15T20:07:11.906Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/37/7c/cdcb654daf668192aaf36b0aeb94f2281dad092aaa5003688691131736ea/lupa-2.8-cp314-cp314t-win_arm64.whl", hash = "sha256:91d622777febda3ab1bed1d45295f2f32a4680c7b3d7caf8c669998ed5c44118", size = 1126735, upload-time = "2026-04-15T20:07:15.434Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1d/44/de1961ad38e17cd326a53c246c7e3b91178ed578f4cf22ffcd5e7e11b041/lupa-2.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b036738282a5acd2e71fdddb317c9df8b87c1673aa57f403d05fcc2be8abc4ba", size = 1186020, upload-time = "2026-04-15T20:07:35.017Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/c2/276f0b9dc8bcc5a8a58af5316dfa0e6f56be3613dd6dbcc8d3d2cb6559ba/lupa-2.8-cp39-abi3-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:ac6b6e8d0e617e26a98cbb44880bcd75de5d32b3ad7b3b3793583909292b47ed", size = 1468944, upload-time = "2026-04-15T20:07:37.782Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/63/38/52934e52a5180dc6425d20284d004fe4b27a4f9171a82dc99fb67af250bf/lupa-2.8-cp39-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ba3a7dd839f90c3d2e53bebe3c192b1f3f9fd720a6781256405123211fd0dce6", size = 1172998, upload-time = "2026-04-15T20:07:40.812Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/82/76b3809bd0839d9b3b4ec58d06591e08f17337b6d9576877cb9d48b34e94/lupa-2.8-cp39-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d7edb13a7a5250b5c6c22d1495d9e842b5c9fc5081c8fe6b5efe2112fe3e41f9", size = 1449975, upload-time = "2026-04-15T20:07:44.262Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/16/07/2f89d54f747c67c23b4b9ae4aa8c8dd06bb409155dedcf406157f2736b66/lupa-2.8-cp39-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:891f72e0bffbed1e4175f975aeb2a083956586a100066525e1be485f617f7b25", size = 1281944, upload-time = "2026-04-15T20:07:46.458Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/bd/7375d2b0fcae79d806baf52a76f26c96964593f58e1372d13ae5ac09c676/lupa-2.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a295f87b5b7ebbfd5191932e8cb0e51df3c7769101ac6b6c7d7c9fb27bfd1307", size = 1910455, upload-time = "2026-04-15T20:07:49.75Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/0c/8abb3bc0e08b311fc01db05b6e9f9ff31a8f65e4fc3f0aeb05cfef75c8ac/lupa-2.8-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4fe5d7a810b64ea8511eb885fc8cdde042ee5ff7b7d08ae78f32449756acb177", size = 1155548, upload-time = "2026-04-15T20:07:52.657Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/2e/9eeecd3f493099721c1d3f31beeca23a4237db1a54223684df4dc96aa1bd/lupa-2.8-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:bfc470012ef66ad064c7bd77416af03a3452ef630b04b9012595ea13f2e54518", size = 1489232, upload-time = "2026-04-15T20:07:54.92Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/13/731c99dc2e7652ae818a6de45bdf0142049f7cb566049061c898355f1891/lupa-2.8-cp39-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:250e035fdaffe8c87093e3ebc206ac29a26131b1568ea711d780c26001ce96e7", size = 1466321, upload-time = "2026-04-15T20:07:57.627Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/71/3ad8cc4fc05a77dc0d3f7079348bd1cad4675a0d14c24f8e6a3ce5f008f7/lupa-2.8-cp39-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:b9bddb09acfffb4f828f790f444b11dc0cca591afea1a244d9329eea2d20c003", size = 1288577, upload-time = "2026-04-15T20:07:59.913Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/b2/1175f6d0aa7b68627fbe2f58bd1e8bea36a89d10dfd67671d2b024c96162/lupa-2.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2e64acbbd47e9b82a64405a39e0d2b36a5a7dad8ab41c0f3437f572f7d282ba3", size = 2444866, upload-time = "2026-04-15T20:08:02.753Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/f7/e78df680c7a0ea452daac07467ca188d63c2c00ca1c884c0a50e27eb83b5/lupa-2.8-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32e4e5103bbddcdd2458fb2ccae6c8ba11c9997c711d7e379e0d45551d109c76", size = 1778509, upload-time = "2026-04-15T20:08:21.784Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/23/0e53cabb16b2a8aa9cf1fde499c097d8942c5dab709fc8e921f3b824b18b/lupa-2.8-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7667001804657496dee9feced2daae5000b4604a3218dd8e6b7b754982ba88b8", size = 2300480, upload-time = "2026-04-15T20:08:24.394Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/85/0271227eab939921a12ebba5d17aa4cd18346aa534ca7f5da09cd0b63dd4/lupa-2.8-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:86f6f668966965b15247dc32d064cfe7be67b71e584ccfacbe2f637575296878", size = 1847445, upload-time = "2026-04-15T20:08:27.031Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lxml"
|
||||
version = "6.1.1"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue