mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(alerts): persist notification progress before retrying destinations
This commit is contained in:
parent
58933fe12c
commit
c9d344bbbc
6 changed files with 332 additions and 30 deletions
|
|
@ -5,15 +5,18 @@ import hashlib
|
|||
import json
|
||||
import os
|
||||
from html import escape
|
||||
from itertools import islice
|
||||
from typing import TYPE_CHECKING, ClassVar, Final
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
from httpx import Response
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.models.issues import IssueStore
|
||||
from litellm.types.integrations.slack_alerting import AlertType
|
||||
from litellm.types.proxy.issues import IssuePendingDelivery
|
||||
from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertChannel
|
||||
from litellm.types.proxy.issues import IssueDeliveryBinding, IssueDeliveryInput, IssuePendingDelivery
|
||||
from litellm.types.proxy.usage_alerts import UsageAlert
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
|
|
@ -56,19 +59,43 @@ class UsageAlertDelivery:
|
|||
# One retained send per worker bounds SMTP threads even when a job is cancelled.
|
||||
_email_task: ClassVar[asyncio.Task[None] | None] = None
|
||||
_email_key: ClassVar[str | None] = None
|
||||
_email_store: ClassVar[IssueStore | None] = None
|
||||
|
||||
def __init__(self, slack: "SlackAlerting") -> None:
|
||||
def __init__(self, slack: "SlackAlerting", store: IssueStore | None = None) -> None:
|
||||
self._slack = slack
|
||||
self._store = store
|
||||
|
||||
async def send_issue(self, pending: IssuePendingDelivery) -> bool:
|
||||
urls: Final = self._webhooks() if pending.delivery.channel == "slack" else ()
|
||||
if len(urls) > 1 and pending.delivery.slack_destination is None:
|
||||
if self._store is None:
|
||||
raise RuntimeError("Multiple alert destinations require a delivery store")
|
||||
bindings: Final = (
|
||||
IssueDeliveryBinding(
|
||||
delivery=IssueDeliveryInput(
|
||||
id=str(uuid5(NAMESPACE_URL, f"{pending.delivery.id}:{destination}")),
|
||||
issue_id=pending.issue.id,
|
||||
channel="slack",
|
||||
slack_destination=destination,
|
||||
parent_delivery_id=pending.delivery.id,
|
||||
),
|
||||
rule_id=pending.rule.id,
|
||||
rule_revision=pending.rule.updated_at.isoformat(),
|
||||
source_id=pending.alert.id,
|
||||
)
|
||||
for destination in (hashlib.sha256(url.encode()).hexdigest() for url in urls)
|
||||
)
|
||||
while batch := tuple(islice(bindings, 10)):
|
||||
await self._store.queue_deliveries(batch)
|
||||
return True
|
||||
path: Final = "/alerts?" + urlencode({"issue": pending.issue.id})
|
||||
issue_url: Final = activity_url(
|
||||
pending.alert.model_copy(update={"activity_path": path}), os.getenv("PROXY_BASE_URL")
|
||||
)
|
||||
source_url: Final = activity_url(pending.alert, os.getenv("PROXY_BASE_URL"))
|
||||
return await self._send(
|
||||
pending.delivery.channel,
|
||||
pending.delivery.email_recipients,
|
||||
pending,
|
||||
urls,
|
||||
incident_message(pending.alert, None),
|
||||
f"LiteLLM alert: {pending.issue.title}",
|
||||
(("View alert", issue_url), ("View source activity", source_url)),
|
||||
|
|
@ -76,12 +103,14 @@ class UsageAlertDelivery:
|
|||
|
||||
async def _send(
|
||||
self,
|
||||
channel: UsageAlertChannel,
|
||||
recipients: tuple[str, ...],
|
||||
pending: IssuePendingDelivery,
|
||||
urls: tuple[str, ...],
|
||||
message: str,
|
||||
subject: str,
|
||||
links: tuple[tuple[str, str], ...],
|
||||
) -> bool:
|
||||
channel: Final = pending.delivery.channel
|
||||
recipients: Final = pending.delivery.email_recipients
|
||||
if channel not in configured_destinations(self._slack):
|
||||
return False
|
||||
if channel == "email":
|
||||
|
|
@ -89,8 +118,19 @@ class UsageAlertDelivery:
|
|||
f'<p><a href="{escape(url, quote=True)}">{escape(label)}</a></p>' for label, url in links
|
||||
)
|
||||
for recipient in recipients:
|
||||
await self._email(recipient, subject, html)
|
||||
await self._email(pending.delivery.id, recipient, subject, html)
|
||||
return bool(recipients)
|
||||
destination: Final = pending.delivery.slack_destination
|
||||
webhook: Final = next(
|
||||
(url for url in urls if destination is None or hashlib.sha256(url.encode()).hexdigest() == destination),
|
||||
None,
|
||||
)
|
||||
if webhook is None:
|
||||
return False
|
||||
await self._post(webhook, message + "\n" + "\n".join(f"{label}: {url}" for label, url in links))
|
||||
return True
|
||||
|
||||
def _webhooks(self) -> tuple[str, ...]:
|
||||
overrides: Final = self._slack.alert_to_webhook_url or {}
|
||||
configured_url: Final = (
|
||||
overrides.get(AlertType.usage_alerts)
|
||||
|
|
@ -99,38 +139,51 @@ class UsageAlertDelivery:
|
|||
or os.getenv("ALERTING_WEBHOOK_URL")
|
||||
)
|
||||
if not configured_url:
|
||||
return False
|
||||
urls: Final = (configured_url,) if isinstance(configured_url, str) else _URLS.validate_python(configured_url)
|
||||
for webhook in urls:
|
||||
await self._post(webhook, message + "\n" + "\n".join(f"{label}: {url}" for label, url in links))
|
||||
return True
|
||||
return ()
|
||||
return (configured_url,) if isinstance(configured_url, str) else _URLS.validate_python(configured_url)
|
||||
|
||||
@staticmethod
|
||||
def _email_finished(task: asyncio.Task[None]) -> None:
|
||||
if not task.cancelled():
|
||||
task.exception() # consume a late failure after the calling job was cancelled
|
||||
|
||||
@classmethod
|
||||
async def _email(cls, recipient: str, subject: str, html: str) -> None:
|
||||
key: Final = hashlib.sha256(json.dumps((recipient, subject, html)).encode()).hexdigest()
|
||||
async def _email(self, key: str, recipient: str, subject: str, html: str) -> None:
|
||||
cls: Final = type(self)
|
||||
previous: Final = cls._email_task
|
||||
if previous is not None:
|
||||
if not previous.done():
|
||||
raise RuntimeError("An alert email is still in flight; retry later")
|
||||
if cls._email_key == key and not previous.cancelled() and previous.exception() is None:
|
||||
cls._email_task = None
|
||||
cls._email_key = None
|
||||
return previous.result()
|
||||
if not previous.cancelled() and previous.exception() is None:
|
||||
if cls._email_store is not None and cls._email_key is not None:
|
||||
await cls._email_store.mark_delivery_sent(cls._email_key)
|
||||
if cls._email_task is not previous:
|
||||
raise RuntimeError("Another alert email is still in flight; retry later")
|
||||
elif cls._email_key != key:
|
||||
raise RuntimeError("A completed alert email is awaiting its receipt")
|
||||
if cls._email_key == key:
|
||||
cls._clear_email(previous)
|
||||
return previous.result()
|
||||
task: Final = asyncio.create_task(cls._send_email(recipient, subject, html))
|
||||
cls._email_task = task
|
||||
cls._email_key = key
|
||||
cls._email_store = self._store
|
||||
task.add_done_callback(cls._email_finished)
|
||||
try:
|
||||
await asyncio.shield(task)
|
||||
finally:
|
||||
if task.done() and cls._email_task is task:
|
||||
cls._email_task = None
|
||||
cls._email_key = None
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
cls._clear_email(task)
|
||||
raise
|
||||
else:
|
||||
cls._clear_email(task)
|
||||
|
||||
@classmethod
|
||||
def _clear_email(cls, task: asyncio.Task[None]) -> None:
|
||||
if cls._email_task is task:
|
||||
cls._email_task = None
|
||||
cls._email_key = None
|
||||
cls._email_store = None
|
||||
|
||||
@staticmethod
|
||||
async def _send_email(recipient: str, subject: str, html: str) -> None:
|
||||
|
|
|
|||
|
|
@ -145,7 +145,8 @@ class UsageAlertJob:
|
|||
background=True,
|
||||
)
|
||||
metrics: Final = UsageAlertMetricsRepository(database)
|
||||
delivery: Final = UsageAlertDelivery(slack)
|
||||
issues: Final = IssuesRepository(database)
|
||||
delivery: Final = UsageAlertDelivery(slack, issues)
|
||||
return cls(
|
||||
UsageAlertsRepository(database),
|
||||
metrics,
|
||||
|
|
@ -158,7 +159,7 @@ class UsageAlertJob:
|
|||
),
|
||||
deprecation_projection=deprecation_projection,
|
||||
performance_projection=PerformanceIssueProjector(
|
||||
IssuesRepository(database),
|
||||
issues,
|
||||
metrics,
|
||||
ProviderLatencyRepository(database),
|
||||
delivery,
|
||||
|
|
|
|||
|
|
@ -607,7 +607,7 @@ class IssuesRepository:
|
|||
SELECT r.record AS rule,a.record AS alert FROM "LiteLLM_IssueDeliveryRule" b
|
||||
JOIN "LiteLLM_UsageAlertRule" r ON r.id=b.rule_id
|
||||
JOIN "LiteLLM_UsageAlertIncident" a ON a.id=b.source_id AND a.rule_id=r.id
|
||||
WHERE b.delivery_id=d.id AND r.enabled AND NOT r.deleted AND a.breached AND a.status='open'
|
||||
WHERE b.delivery_id=COALESCE(d.record->>'parent_delivery_id',d.id) AND r.enabled AND NOT r.deleted AND a.breached AND a.status='open'
|
||||
AND (r.record->'notification_channels') ? (d.record->>'channel')
|
||||
AND (d.record->>'channel'<>'email' OR
|
||||
(jsonb_array_length(d.record->'email_recipients')>0
|
||||
|
|
|
|||
|
|
@ -207,6 +207,8 @@ class IssueDeliveryInput(BaseModel, frozen=True):
|
|||
issue_id: str = Field(min_length=1, max_length=200)
|
||||
channel: UsageAlertChannel
|
||||
email_recipients: tuple[str, ...] = Field(default=(), max_length=20)
|
||||
slack_destination: str | None = Field(default=None, pattern="^[a-f0-9]{64}$")
|
||||
parent_delivery_id: str | None = Field(default=None, min_length=1, max_length=200)
|
||||
|
||||
|
||||
class IssueDeliveryBinding(BaseModel, frozen=True):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import os
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
|
@ -7,11 +8,14 @@ from typing import Final
|
|||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.actors import ManagementDatabase
|
||||
from integration._support.prisma_clients import connected_proxy_prisma
|
||||
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy.spend_tracking.usage_alert_job import UsageAlertJob
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.repositories.daily_activity_buckets import api_key_bucket
|
||||
|
|
@ -154,8 +158,7 @@ async def test_retirement_usage_omits_oversized_models_before_materialization(db
|
|||
@asynccontextmanager
|
||||
async def _isolated(db: ManagementDatabase) -> AsyncIterator[PrismaClient]:
|
||||
schema: Final = "issue_test_" + uuid4().hex
|
||||
assert db.gateway.database_url is not None
|
||||
parsed: Final = urlsplit(db.gateway.database_url)
|
||||
parsed: Final = urlsplit(db.gateway.database_url or os.environ["DATABASE_URL"])
|
||||
query: Final = {**dict(parse_qsl(parsed.query)), "schema": schema}
|
||||
url: Final = urlunsplit(parsed._replace(query=urlencode(query)))
|
||||
await db.prisma.writer_db.execute_raw(f'CREATE SCHEMA "{schema}"')
|
||||
|
|
@ -871,3 +874,175 @@ async def test_projection_resume_keeps_spend_and_traffic_in_one_episode(db: Mana
|
|||
assert tuple(item.id for item in after.data) == tuple(item.id for item in before.data)
|
||||
links: Final = await IssuesRepository.from_prisma(prisma).list_signals(after.data[0].id)
|
||||
assert any(link.usage_alert is not None and link.usage_alert.metric == "requests" for link in links.data)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_slack_retry_only_repeats_the_failed_destination(db: ManagementDatabase) -> None:
|
||||
paths: Final[list[str]] = []
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
paths.append(request.url.path)
|
||||
return httpx.Response(503 if request.url.path == "/two" and paths.count("/two") == 1 else 200)
|
||||
|
||||
handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond))
|
||||
try:
|
||||
async with _isolated(db) as prisma:
|
||||
await _seed(prisma)
|
||||
rules: Final = UsageAlertsRepository.from_prisma(prisma)
|
||||
assert (
|
||||
await rules.save_rule(
|
||||
UsageAlertRuleInput(
|
||||
name="Project spend",
|
||||
scope_type="key",
|
||||
scope_id=_KEY_A,
|
||||
metric="spend",
|
||||
direction="increase",
|
||||
notification_channels=("slack",),
|
||||
),
|
||||
"qa-admin",
|
||||
)
|
||||
is not None
|
||||
)
|
||||
assert (
|
||||
await rules.save_rule(
|
||||
UsageAlertRuleInput(
|
||||
name="Another project rule",
|
||||
scope_type="key",
|
||||
scope_id=_KEY_A,
|
||||
metric="spend",
|
||||
direction="increase",
|
||||
notification_channels=("slack",),
|
||||
),
|
||||
"qa-admin",
|
||||
)
|
||||
is not None
|
||||
)
|
||||
slack: Final = SlackAlerting(
|
||||
alerting=["slack"],
|
||||
alert_types=[],
|
||||
async_http_handler=handler,
|
||||
alert_to_webhook_url={
|
||||
"usage_alerts": [
|
||||
"https://hooks.example.com/one",
|
||||
"https://hooks.example.com/two",
|
||||
"https://hooks.example.com/one",
|
||||
]
|
||||
},
|
||||
)
|
||||
job: Final = UsageAlertJob.from_prisma(prisma, slack)
|
||||
await job._run(_NOW)
|
||||
assert paths == ["/one", "/two"] or paths == ["/two", "/one"]
|
||||
await prisma.writer_db.query_raw(
|
||||
'UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP WHERE NOT sent RETURNING id'
|
||||
)
|
||||
retry: Final = (await IssuesRepository.from_prisma(prisma).pending_deliveries())[0]
|
||||
assert (
|
||||
await rules.save_rule(
|
||||
UsageAlertRuleInput.model_validate(
|
||||
retry.rule.model_dump(include=set(UsageAlertRuleInput.model_fields))
|
||||
).model_copy(update={"enabled": False}),
|
||||
"qa-admin",
|
||||
retry.rule,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
await job._performance_projection.notify()
|
||||
assert paths.count("/one") == 1 and paths.count("/two") == 2
|
||||
assert await IssuesRepository.from_prisma(prisma).pending_deliveries() == ()
|
||||
records: Final = await prisma.writer_db.query_raw(
|
||||
'SELECT record::text AS value FROM "LiteLLM_IssueDelivery"'
|
||||
)
|
||||
assert all("hooks.example.com" not in row["value"] for row in records)
|
||||
finally:
|
||||
await handler.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_cancelled_email_receipt_survives_a_different_delivery(
|
||||
db: ManagementDatabase, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
accepted: Final = asyncio.Event()
|
||||
release: Final = asyncio.Event()
|
||||
messages: Final[list[bytes]] = []
|
||||
|
||||
async def smtp(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
||||
writer.write(b"220 example.com ready\r\n")
|
||||
await writer.drain()
|
||||
try:
|
||||
while command := await reader.readline():
|
||||
if command.upper().startswith(b"DATA"):
|
||||
writer.write(b"354 End with a dot\r\n")
|
||||
await writer.drain()
|
||||
messages.append(await reader.readuntil(b"\r\n.\r\n"))
|
||||
accepted.set()
|
||||
await release.wait()
|
||||
writer.write(b"250 Accepted\r\n")
|
||||
elif command.upper().startswith(b"QUIT"):
|
||||
writer.write(b"221 Closing\r\n")
|
||||
break
|
||||
else:
|
||||
writer.write(b"250 OK\r\n")
|
||||
await writer.drain()
|
||||
finally:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
|
||||
server: Final = await asyncio.start_server(smtp, "127.0.0.1", 0)
|
||||
monkeypatch.setenv("SMTP_HOST", "127.0.0.1")
|
||||
monkeypatch.setenv("SMTP_PORT", str(server.sockets[0].getsockname()[1]))
|
||||
monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com")
|
||||
monkeypatch.setenv("SMTP_TLS", "False")
|
||||
monkeypatch.setenv("SMTP_USE_SSL", "False")
|
||||
for name in ("SMTP_TIMEOUT", "SMTP_USERNAME", "SMTP_PASSWORD"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
try:
|
||||
async with _isolated(db) as prisma:
|
||||
await _seed(prisma)
|
||||
rules: Final = UsageAlertsRepository.from_prisma(prisma)
|
||||
assert (
|
||||
await rules.save_rule(
|
||||
UsageAlertRuleInput(
|
||||
name="Project spend",
|
||||
scope_type="key",
|
||||
scope_id=_KEY_A,
|
||||
metric="spend",
|
||||
direction="increase",
|
||||
notification_channels=("email",),
|
||||
email_recipients=("qa@example.com",),
|
||||
),
|
||||
"qa-admin",
|
||||
)
|
||||
is not None
|
||||
)
|
||||
job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[]))
|
||||
await job._run(_NOW)
|
||||
store: Final = IssuesRepository.from_prisma(prisma)
|
||||
await prisma.writer_db.query_raw(
|
||||
'UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP WHERE NOT sent RETURNING id'
|
||||
)
|
||||
pending: Final = (await store.pending_deliveries())[0]
|
||||
delivery: Final = UsageAlertDelivery(SlackAlerting(alerting=["email"], alert_types=[]), store)
|
||||
sending: Final = asyncio.create_task(delivery.send_issue(pending))
|
||||
await asyncio.wait_for(accepted.wait(), 10)
|
||||
sending.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await sending
|
||||
release.set()
|
||||
retained: Final = UsageAlertDelivery._email_task
|
||||
assert retained is not None
|
||||
await asyncio.wait_for(asyncio.shield(retained), 10)
|
||||
different: Final = pending.model_copy(
|
||||
update={"delivery": pending.delivery.model_copy(update={"id": "another-delivery"})}
|
||||
)
|
||||
assert await delivery.send_issue(different)
|
||||
assert await store.pending_delivery(pending.delivery.id) is None
|
||||
assert len(messages) == 2
|
||||
finally:
|
||||
release.set()
|
||||
server.close()
|
||||
await server.wait_closed()
|
||||
if UsageAlertDelivery._email_task is not None:
|
||||
await asyncio.wait_for(asyncio.shield(UsageAlertDelivery._email_task), 10)
|
||||
UsageAlertDelivery._email_task = None
|
||||
UsageAlertDelivery._email_key = None
|
||||
UsageAlertDelivery._email_store = None
|
||||
|
|
|
|||
|
|
@ -1,10 +1,23 @@
|
|||
import asyncio
|
||||
import json
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
from typing import Final, cast # noqa: TID251 # generated Prisma transaction implements the accounting protocol
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from integration._support.actors import ManagementDatabase
|
||||
from integration.database.test_performance_issues import _isolated
|
||||
|
||||
from litellm.accounting.application import apply_deliveries
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.repositories.accounting_delivery_repository import (
|
||||
AccountingDeliveryRepository,
|
||||
DeliveryProjection,
|
||||
LedgerDatabase,
|
||||
LedgerTransaction,
|
||||
)
|
||||
from litellm.repositories.usage_alert_health import UsageAlertFlushRecorder, UsageAlertHealthRepository
|
||||
from litellm.repositories.usage_alerts import UsageAlertsRepository
|
||||
from litellm.types.proxy.usage_alerts import UsageAlertRuleInput
|
||||
|
|
@ -68,3 +81,61 @@ async def test_freshness_lock_cannot_stall_committed_spend(db: ManagementDatabas
|
|||
"SELECT source FROM \"LiteLLM_UsageAlertFlush\" WHERE source='user' AND hour=date_trunc('hour',CURRENT_TIMESTAMP AT TIME ZONE 'UTC')"
|
||||
)
|
||||
assert rows == [{"source": "user"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_freshness_waits_until_accounting_has_committed_the_daily_totals(db: ManagementDatabase) -> None:
|
||||
async with _isolated(db) as prisma:
|
||||
await prisma.writer_db.execute_raw(
|
||||
'CREATE TABLE "LiteLLM_DailyModelUsage" (LIKE public."LiteLLM_DailyModelUsage" INCLUDING ALL)'
|
||||
)
|
||||
identity: Final = "coverage-" + uuid4().hex
|
||||
now: Final = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) + timedelta(days=3)
|
||||
row: Final = {
|
||||
"request_id": identity,
|
||||
"user": identity,
|
||||
"api_key": "synthetic-key",
|
||||
"startTime": (now - timedelta(hours=2)).isoformat(),
|
||||
"model": "example-model",
|
||||
"model_group": "example-model",
|
||||
"custom_llm_provider": "example-provider",
|
||||
"spend": 5,
|
||||
"metadata": {},
|
||||
}
|
||||
health: Final = UsageAlertHealthRepository.from_prisma(prisma)
|
||||
writer: Final = DBSpendUpdateWriter()
|
||||
logging: Final = ProxyLogging(UserApiKeyCache())
|
||||
|
||||
async def apply(transaction: LedgerTransaction, deliveries: tuple[DeliveryProjection, ...]) -> None:
|
||||
await apply_deliveries(
|
||||
transaction, deliveries, prisma_client=prisma, writer=writer, proxy_logging_obj=logging
|
||||
)
|
||||
assert not await health.record("user", now)
|
||||
|
||||
try:
|
||||
await prisma.writer_db.query_raw(
|
||||
'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id,accounting_payload,created_at,is_batch) VALUES ($1,$2::jsonb,$3::timestamp,false) RETURNING delivery_id',
|
||||
identity,
|
||||
json.dumps(row),
|
||||
(now - timedelta(hours=1)).replace(tzinfo=None),
|
||||
)
|
||||
assert not await health.record("user", now)
|
||||
database: Final = cast(LedgerDatabase, prisma) # cast-ok: generated Prisma transaction delegate
|
||||
result: Final = await AccountingDeliveryRepository(database).apply_pending(apply, batch_size=1)
|
||||
assert result.applied == 1
|
||||
assert await health.record("user", now)
|
||||
totals: Final = await prisma.writer_db.query_raw(
|
||||
'SELECT spend,api_requests FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1', identity
|
||||
)
|
||||
assert totals == [{"spend": 5, "api_requests": 1}]
|
||||
finally:
|
||||
await prisma.writer_db.query_raw(
|
||||
'DELETE FROM "LiteLLM_SpendLogDelivery" WHERE request_id=$1 RETURNING delivery_id', identity
|
||||
)
|
||||
await prisma.writer_db.query_raw(
|
||||
'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1 RETURNING id', identity
|
||||
)
|
||||
await prisma.writer_db.query_raw(
|
||||
"DELETE FROM \"LiteLLM_UsageAlertFlush\" WHERE source='user' AND hour=$1::timestamp RETURNING source",
|
||||
now.replace(tzinfo=None),
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue