mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(accounting): isolate batch persistence and preserve trusted attribution
This commit is contained in:
parent
80f4a31bc2
commit
e517e65bc2
9 changed files with 247 additions and 45 deletions
|
|
@ -275,7 +275,7 @@ class AccountingLedger:
|
|||
)
|
||||
return tuple(
|
||||
UsageWrite(
|
||||
OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]),
|
||||
OutboundUsageEvent.model_validate_json(_json(_JSON.validate_python(row["payload"])["event"])),
|
||||
_json(row["payload"]),
|
||||
TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -630,7 +630,10 @@ class _ProxyDBLogger(CustomLogger):
|
|||
|
||||
if usage_capture.meter is None:
|
||||
return None
|
||||
captured_identity: Final = str(request_data.get("call_type")) in _CAPTURED_IDENTITY_CALL_TYPES
|
||||
captured_identity: Final = (
|
||||
_batch_accounting_for(request_data.get("litellm_call_id")) is not None
|
||||
and request_data.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is not REJECTED_BEFORE_REQUEST_SETUP
|
||||
)
|
||||
current_identity: Final = {
|
||||
**LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth),
|
||||
"user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth),
|
||||
|
|
|
|||
|
|
@ -619,6 +619,49 @@ def _batch_row_write(root: UsageWrite, row: Mapping[str, JsonValue], index: int)
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _BatchCapture:
|
||||
payload: SpendLogsPayload
|
||||
root: UsageWrite
|
||||
rows: tuple[Mapping[str, JsonValue], ...]
|
||||
prerequisites: tuple[asyncio.Future[None], ...]
|
||||
|
||||
|
||||
def _prepare_batch_capture(
|
||||
kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime
|
||||
) -> _BatchCapture:
|
||||
from litellm.proxy.db.db_spend_update_writer import get_llm_router
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
|
||||
rows: Final = tuple(sorted(_batch_rows(response), key=_json))
|
||||
builder: Final = _CaptureBuilder(kwargs, response)
|
||||
payload: Final = get_logging_payload(
|
||||
dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe
|
||||
)
|
||||
if len(builder.writes) != 1 or _terminal_facts(builder.writes[0]) is None:
|
||||
raise UsageCaptureError("Batch rows require one terminal job observation")
|
||||
return _BatchCapture(
|
||||
payload, _batch_root(builder.writes[0], rows), rows, tuple(prior.receipt for prior in _prior_captures(kwargs))
|
||||
)
|
||||
|
||||
|
||||
async def _persist_batch_capture(
|
||||
meter: DurableUsageMeter,
|
||||
load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]],
|
||||
capture: _BatchCapture,
|
||||
) -> None:
|
||||
for receipt in capture.prerequisites:
|
||||
await asyncio.shield(receipt)
|
||||
await meter.record_writes((capture.root,))
|
||||
accepted: Final = await load((capture.root.event.event_id,))
|
||||
if len(accepted) != 1:
|
||||
raise UsageCaptureError("Committed batch snapshot could not be loaded")
|
||||
frozen: Final = _reuse_terminal_snapshot(capture.root, accepted[0])
|
||||
remaining: Final = iter(enumerate(capture.rows))
|
||||
while group := tuple(islice(remaining, meter.settings.batch_size)):
|
||||
await meter.record_writes(tuple(_batch_row_write(frozen, row, index) for index, row in group))
|
||||
|
||||
|
||||
async def capture_batch_callback(
|
||||
meter: DurableUsageMeter,
|
||||
load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None,
|
||||
|
|
@ -627,32 +670,13 @@ async def capture_batch_callback(
|
|||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
) -> CapturedUsage:
|
||||
from litellm.proxy.db.db_spend_update_writer import get_llm_router
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
|
||||
if load is None:
|
||||
raise UsageCaptureError("Batch capture requires the existing ledger snapshot reader")
|
||||
rows: Final = tuple(sorted(_batch_rows(response), key=_json))
|
||||
builder: Final = _CaptureBuilder(kwargs, response)
|
||||
payload: Final = get_logging_payload(
|
||||
dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe
|
||||
)
|
||||
if len(builder.writes) != 1 or _terminal_facts(builder.writes[0]) is None:
|
||||
raise UsageCaptureError("Batch rows require one terminal job observation")
|
||||
root: Final = _batch_root(builder.writes[0], rows)
|
||||
for prior in _prior_captures(kwargs):
|
||||
await asyncio.shield(prior.receipt)
|
||||
await meter.record_writes((root,))
|
||||
accepted: Final = await load((root.event.event_id,))
|
||||
if len(accepted) != 1:
|
||||
raise UsageCaptureError("Committed batch snapshot could not be loaded")
|
||||
frozen: Final = _reuse_terminal_snapshot(root, accepted[0])
|
||||
remaining: Final = iter(enumerate(rows))
|
||||
while group := tuple(islice(remaining, meter.settings.batch_size)):
|
||||
await meter.record_writes(tuple(_batch_row_write(frozen, row, index) for index, row in group))
|
||||
capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time)
|
||||
await _persist_batch_capture(meter, load, capture)
|
||||
receipt: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future()
|
||||
receipt.set_result(None)
|
||||
return CapturedUsage(payload, (frozen,), receipt)
|
||||
return CapturedUsage(capture.payload, (capture.root,), receipt)
|
||||
|
||||
|
||||
async def captured_callback_kwargs(
|
||||
|
|
@ -669,7 +693,7 @@ async def captured_callback_kwargs(
|
|||
return kwargs
|
||||
try:
|
||||
captured: Final = (
|
||||
await capture_batch_callback(usage_meter, usage_capture.load, kwargs, response, start_time, end_time)
|
||||
usage_capture.submit_batch(kwargs, response, start_time, end_time)
|
||||
if _batch_rows(response)
|
||||
else capture_callback(usage_meter, kwargs, response, start_time, end_time)
|
||||
)
|
||||
|
|
@ -926,6 +950,8 @@ class UsageBatchWriter:
|
|||
|
||||
class UsageCaptureRuntime:
|
||||
def __init__(self) -> None:
|
||||
self._batches: Final[set[asyncio.Task[None]]] = set() # mutable-ok: owned producer lifecycle
|
||||
self._stopping = False
|
||||
self.meter: DurableUsageMeter | None = None
|
||||
self.load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None
|
||||
|
||||
|
|
@ -945,18 +971,51 @@ class UsageCaptureRuntime:
|
|||
meter: Final = DurableUsageMeter(write_batch, settings)
|
||||
self.meter = meter
|
||||
self.load = load
|
||||
self._stopping = False
|
||||
Logging.usage_capture_callback = lambda logger, response, start, end, freeze: capture_logging_usage(
|
||||
meter, logger, response, start, end, freeze
|
||||
)
|
||||
|
||||
def submit_batch(
|
||||
self, kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime
|
||||
) -> CapturedUsage:
|
||||
meter: Final = self.meter
|
||||
load: Final = self.load
|
||||
if meter is None or load is None or self._stopping:
|
||||
raise UsageCaptureError("Batch capture requires an active ledger writer and snapshot reader")
|
||||
if len(self._batches) >= meter.settings.max_pending_records:
|
||||
raise UsageCaptureError("Batch capture producer capacity exhausted")
|
||||
capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time)
|
||||
task: Final = asyncio.create_task(_persist_batch_capture(meter, load, capture), name="accounting batch capture")
|
||||
self._batches.add(task)
|
||||
task.add_done_callback(self._batch_finished)
|
||||
return CapturedUsage(capture.payload, (capture.root,), task)
|
||||
|
||||
def _batch_finished(self, task: asyncio.Task[None]) -> None:
|
||||
self._batches.discard(task)
|
||||
if task.cancelled():
|
||||
verbose_proxy_logger.error("Batch usage capture stopped before complete persistence")
|
||||
elif (error := task.exception()) is not None:
|
||||
verbose_proxy_logger.error("Batch usage capture failed", exc_info=error)
|
||||
|
||||
async def stop(self, timeout: float) -> None:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
if self.meter is None:
|
||||
return
|
||||
self._stopping = True
|
||||
deadline: Final = asyncio.get_running_loop().time() + timeout
|
||||
try:
|
||||
await self.meter.stop(timeout)
|
||||
if self._batches:
|
||||
await asyncio.wait(tuple(self._batches), timeout=max(0, timeout))
|
||||
finally:
|
||||
Logging.usage_capture_callback = None
|
||||
self.meter = None
|
||||
self.load = None
|
||||
remaining: Final = tuple(self._batches)
|
||||
for task in remaining:
|
||||
task.cancel()
|
||||
await asyncio.gather(*remaining, return_exceptions=True)
|
||||
try:
|
||||
await self.meter.stop(max(0, deadline - asyncio.get_running_loop().time()))
|
||||
finally:
|
||||
Logging.usage_capture_callback = None
|
||||
self.meter = None
|
||||
self.load = None
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ class DurableUsageMeter:
|
|||
self._admission: Final = Lock()
|
||||
self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue
|
||||
self._wakeup: Final = asyncio.Event()
|
||||
self._capacity_changed: Final = asyncio.Event()
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._closed = False
|
||||
self.pending_bytes = 0
|
||||
|
|
@ -165,11 +166,13 @@ class DurableUsageMeter:
|
|||
if size > self.settings.max_pending_bytes:
|
||||
raise UsageCaptureError("One usage observation exceeds capture capacity")
|
||||
while (receipt := self._submit_available(writes)) is None:
|
||||
await self.flush()
|
||||
self._wakeup.set()
|
||||
await self._capacity_changed.wait()
|
||||
self._wakeup.set()
|
||||
await asyncio.shield(receipt)
|
||||
|
||||
def _submit_available(self, writes: tuple[UsageWrite, ...]) -> asyncio.Future[None] | None:
|
||||
self._capacity_changed.clear()
|
||||
try:
|
||||
receipt: Final = self.submit(writes)
|
||||
except UsageCaptureError:
|
||||
|
|
@ -265,6 +268,7 @@ class DurableUsageMeter:
|
|||
with self._admission:
|
||||
self.pending_bytes -= item.size
|
||||
self.pending_records -= len(item.writes)
|
||||
self._capacity_changed.set()
|
||||
if item.committed.done():
|
||||
continue
|
||||
if error is None:
|
||||
|
|
|
|||
|
|
@ -540,7 +540,12 @@ async def test_usage_batch_is_atomic_and_replay_preserves_receipts(
|
|||
writer: _Writer, ledger_database: str, failure: str
|
||||
) -> None:
|
||||
client, store = writer
|
||||
events: Final = (_usage("caller", inbound=True), _usage("attempt"))
|
||||
events: Final = (
|
||||
_usage("caller", inbound=True),
|
||||
_usage("attempt").model_copy(
|
||||
update={"metrics": (Metric(name=MetricName.PROMPT_TOKENS, value=Decimal("120000")),)}
|
||||
),
|
||||
)
|
||||
writes: Final = tuple(UsageWrite(event, _envelope(event), _TARGETS) for event in events)
|
||||
|
||||
@asynccontextmanager
|
||||
|
|
|
|||
|
|
@ -116,3 +116,34 @@ async def test_conflicting_usage_within_one_batch_is_rejected_before_database_ac
|
|||
changed: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json"), "pricing": {"rate": 2}}))
|
||||
with pytest.raises(AccountingConflictError, match="conflicting copies"):
|
||||
await AccountingLedger(_no_transaction).insert_usage_batch((original, changed))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_loaded_usage_preserves_decimal_metrics_from_jsonb(event: InboundUsageEvent) -> None:
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import cast
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from prisma import Prisma
|
||||
|
||||
from litellm.accounting import OutboundUsageEvent
|
||||
|
||||
outbound: Final = OutboundUsageEvent(
|
||||
**event.model_dump(exclude={"metrics"}),
|
||||
attempt_id="attempt",
|
||||
retry_of_attempt_id=None,
|
||||
metrics=(Metric(name=MetricName.PROMPT_TOKENS, value=Decimal("120000")),),
|
||||
)
|
||||
payload: Final = {"event": outbound.model_dump(mode="json"), "usage": {"prompt_tokens": 120000}}
|
||||
database: Final = AsyncMock(spec=Prisma)
|
||||
database.query_raw.return_value = [{"payload": payload, "targets": []}]
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction() -> AsyncIterator[Prisma]:
|
||||
yield cast(Prisma, database)
|
||||
|
||||
recovered: Final = await AccountingLedger(transaction).load_usage_batch((outbound.event_id,))
|
||||
assert len(recovered) == 1
|
||||
assert recovered[0].event == outbound
|
||||
assert json.loads(recovered[0].envelope) == payload
|
||||
|
|
|
|||
|
|
@ -4001,6 +4001,7 @@ async def test_capture_and_legacy_share_resolved_key_attribution(
|
|||
from litellm.accounting import DeliveryTarget, DimensionName
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import track_batch_accounting
|
||||
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime
|
||||
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
|
||||
from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage
|
||||
|
|
@ -4025,14 +4026,15 @@ async def test_capture_and_legacy_share_resolved_key_attribution(
|
|||
runtime: Final = UsageCaptureRuntime()
|
||||
runtime.start(database.insert, UsageCaptureSettings())
|
||||
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
|
||||
owner: Final = "original-owner" if deferred else "owner"
|
||||
rejected: Final = failure and deferred and not complete
|
||||
owner: Final = "original-owner" if deferred and not rejected else "owner"
|
||||
metadata: Final = {
|
||||
"user_api_key": "a" * 64,
|
||||
"user_api_key": "b" * 64 if rejected else "a" * 64,
|
||||
**(
|
||||
{
|
||||
"user_api_key_user_id": owner,
|
||||
"user_api_key_team_id": "team",
|
||||
"user_api_key_org_id": "org",
|
||||
"user_api_key_user_id": "forged-owner" if rejected else owner,
|
||||
"user_api_key_team_id": "forged-team" if rejected else "team",
|
||||
"user_api_key_org_id": "forged-org" if rejected else "org",
|
||||
"user_api_key_project_id": None if deferred and not complete else "project",
|
||||
"user_api_key_alias": "key",
|
||||
"user_api_key_team_alias": "team-alias",
|
||||
|
|
@ -4049,21 +4051,28 @@ async def test_capture_and_legacy_share_resolved_key_attribution(
|
|||
"start_time": _CALL_START,
|
||||
"litellm_params": {"metadata": metadata},
|
||||
}
|
||||
if rejected:
|
||||
from litellm.proxy.utils import REJECTED_BEFORE_REQUEST_SETUP, REJECTED_BEFORE_REQUEST_SETUP_KEY
|
||||
|
||||
kwargs[REJECTED_BEFORE_REQUEST_SETUP_KEY] = REJECTED_BEFORE_REQUEST_SETUP
|
||||
logger: Final = ProxyDBLogger(spend_writer=lambda: cast(DBSpendUpdateWriter, writer))
|
||||
try:
|
||||
if failure:
|
||||
await logger.async_post_call_failure_hook(
|
||||
kwargs,
|
||||
HTTPException(401, "model not allowed"),
|
||||
key if complete else UserAPIKeyAuth(api_key="a" * 64, request_route="/v1/chat/completions"),
|
||||
)
|
||||
else:
|
||||
await logger.async_log_success_event(kwargs, {"id": "tool"}, _CALL_START, _CALL_START)
|
||||
with track_batch_accounting("attributed" if deferred and not rejected else "other"):
|
||||
if failure:
|
||||
await logger.async_post_call_failure_hook(
|
||||
kwargs,
|
||||
HTTPException(401, "model not allowed"),
|
||||
key if complete else UserAPIKeyAuth(api_key="a" * 64, request_route="/v1/chat/completions"),
|
||||
)
|
||||
else:
|
||||
await logger.async_log_success_event(kwargs, {"id": "tool"}, _CALL_START, _CALL_START)
|
||||
finally:
|
||||
await runtime.stop(1)
|
||||
assert len(writer.rows) == 1
|
||||
assert len(database.rows) == 2
|
||||
dimensions: Final = {item.name: item.value for item in database.rows[-1].event.dimensions}
|
||||
if rejected:
|
||||
assert dimensions.get(DimensionName.KEY) == key.api_key
|
||||
assert {
|
||||
name: dimensions.get(name)
|
||||
for name in (DimensionName.USER, DimensionName.TEAM, DimensionName.ORGANIZATION, DimensionName.PROJECT)
|
||||
|
|
@ -4071,7 +4080,7 @@ async def test_capture_and_legacy_share_resolved_key_attribution(
|
|||
DimensionName.USER: owner,
|
||||
DimensionName.TEAM: "team",
|
||||
DimensionName.ORGANIZATION: "org",
|
||||
DimensionName.PROJECT: None if deferred and not complete else "project",
|
||||
DimensionName.PROJECT: None if deferred and not complete and not rejected else "project",
|
||||
}
|
||||
assert set(database.rows[-1].targets) >= {DeliveryTarget.DAILY_TEAM_SPEND, DeliveryTarget.DAILY_ORGANIZATION_SPEND}
|
||||
legacy: Final = TypeAdapter(dict[str, object]).validate_python(writer.rows[0]["kwargs"])
|
||||
|
|
|
|||
|
|
@ -1551,3 +1551,59 @@ async def test_large_batch_recovers_partial_commit_with_frozen_pricing_and_rejec
|
|||
assert database.rows == before_conflict
|
||||
assert transactions.qsize() > 2
|
||||
assert meter.pending_bytes == meter.pending_records == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("outcome", ["drain", "cancel", "failure"])
|
||||
async def test_batch_callback_handoff_is_bounded_and_runtime_owns_persistence(
|
||||
outcome: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, UsageCaptureRuntime, captured_callback_kwargs
|
||||
|
||||
database: Final = StoredUsage()
|
||||
entered: Final = asyncio.Event()
|
||||
release: Final = asyncio.Event()
|
||||
|
||||
async def persist(writes: tuple[UsageWrite, ...]) -> None:
|
||||
entered.set()
|
||||
await release.wait()
|
||||
if outcome == "failure":
|
||||
raise RuntimeError("Persistence unavailable")
|
||||
await database.insert(writes)
|
||||
|
||||
runtime: Final = UsageCaptureRuntime()
|
||||
runtime.start(persist, UsageCaptureSettings(batch_size=2, max_pending_records=2), load=database.load)
|
||||
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
|
||||
kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}}
|
||||
response: Final = {
|
||||
"id": "batch",
|
||||
"status": "completed",
|
||||
"usage": {"batch_usage_by_request": ({"model": "model", "usage": {"prompt_tokens": 2}},) * 3},
|
||||
}
|
||||
callback: Final = asyncio.create_task(captured_callback_kwargs(kwargs, response, NOW, NOW))
|
||||
try:
|
||||
await entered.wait()
|
||||
assert callback.done(), "Batch persistence must not consume the legacy callback deadline"
|
||||
captured: Final = (await callback)["_accounting_capture"]
|
||||
assert isinstance(captured, CapturedUsage) and not captured.receipt.done()
|
||||
second: Final = await captured_callback_kwargs(kwargs, response, NOW, NOW)
|
||||
assert isinstance(second.get("_accounting_capture"), CapturedUsage)
|
||||
overflow: Final = await captured_callback_kwargs(kwargs, response, NOW, NOW)
|
||||
assert "_accounting_capture" not in overflow
|
||||
assert "Batch capture producer capacity exhausted" in caplog.text
|
||||
if outcome != "cancel":
|
||||
release.set()
|
||||
await runtime.stop(0 if outcome == "cancel" else 1)
|
||||
assert captured.receipt.done()
|
||||
assert captured.receipt.cancelled() is (outcome == "cancel")
|
||||
if outcome == "failure":
|
||||
assert isinstance(captured.receipt.exception(), RuntimeError)
|
||||
assert "Batch usage capture failed" in caplog.text
|
||||
assert len(database.rows) == (4 if outcome == "drain" else 0)
|
||||
with pytest.raises(UsageCaptureError, match="active ledger writer"):
|
||||
runtime.submit_batch(kwargs, response, NOW, NOW)
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.gather(callback, return_exceptions=True)
|
||||
await runtime.stop(1)
|
||||
|
|
|
|||
|
|
@ -427,3 +427,38 @@ async def test_cancelling_bounded_capture_preserves_admitted_writes_without_admi
|
|||
await meter.close()
|
||||
assert database.committed == ("0", "1")
|
||||
assert meter.pending_bytes == meter.pending_records == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capacity_wait_retries_admission_before_other_producers_finish() -> None:
|
||||
entered: Final = asyncio.Queue[tuple[str, ...]]()
|
||||
release: Final = asyncio.Queue[None]()
|
||||
|
||||
async def persist(writes: tuple[UsageWrite, ...]) -> None:
|
||||
entered.put_nowait(tuple(write.event.event_id for write in writes))
|
||||
await release.get()
|
||||
|
||||
meter: Final = DurableUsageMeter(
|
||||
persist, UsageCaptureSettings(batch_size=2, max_pending_records=3), wait_for_batch=_manual_flush
|
||||
)
|
||||
initial: Final = meter.submit((_write("first"), _write("second")))
|
||||
remaining: Final = meter.submit((_write("remaining"),))
|
||||
assert await entered.get() == ("first", "second")
|
||||
recording: Final = asyncio.create_task(meter.record_writes((_write("target"),)))
|
||||
try:
|
||||
await asyncio.sleep(0)
|
||||
release.put_nowait(None)
|
||||
assert await entered.get() == ("remaining",)
|
||||
await asyncio.sleep(0)
|
||||
assert meter.pending_records == 2
|
||||
release.put_nowait(None)
|
||||
assert await entered.get() == ("target",)
|
||||
release.put_nowait(None)
|
||||
await recording
|
||||
await asyncio.gather(initial, remaining)
|
||||
finally:
|
||||
recording.cancel()
|
||||
await asyncio.gather(recording, return_exceptions=True)
|
||||
release.put_nowait(None)
|
||||
release.put_nowait(None)
|
||||
await meter.close()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue