feat(ptu): daily rollup job writes flat cost to LiteLLM_DailyTeamSpend

Adds a scheduled APScheduler job that walks active PTU reservations once
per UTC day and upserts prorated flat cost into LiteLLM_DailyTeamSpend
using a sentinel api_key. Backfill CLI helper included.

- Two additive columns on LiteLLM_DailyTeamSpend: ptu_flat_cost (float,
  default 0.0) and ptu_reservation_id (nullable text) plus an index
- Rollup module in litellm/proxy/spend_tracking/ptu_reservation_rollup.py
  with calendar-month proration and idempotent upsert; job re-reads the
  feature flag at execution and no-ops when off
- Cron job registered at 00:15 UTC daily in
  initialize_scheduled_background_jobs, gated on the config flag
- CLI backfill at scripts/ptu_reservation_backfill.py accepting --date
  or --date-range
- 19 unit tests covering proration math for 28/29/30/31-day months,
  flag-off short-circuit, sentinel api_key, idempotency under repeated
  runs, per-reservation upsert failure isolation, effective_from boundary
This commit is contained in:
Yucheng Zhu 2026-07-13 16:42:16 -07:00
parent da4e19529a
commit 48fb3d4a2a
6 changed files with 580 additions and 0 deletions

View file

@ -0,0 +1,6 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "ptu_flat_cost" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "ptu_reservation_id" TEXT;
-- CreateIndex
CREATE INDEX IF NOT EXISTS "LiteLLM_DailyTeamSpend_ptu_reservation_id_idx" ON "LiteLLM_DailyTeamSpend"("ptu_reservation_id");

View file

@ -7929,6 +7929,25 @@ class ProxyStartupEvent:
await cls._initialize_spend_tracking_background_jobs(scheduler=scheduler)
### PTU RESERVATION DAILY ROLLUP ###
if general_settings.get("enable_ptu_cost_attribution", False):
from litellm.proxy.spend_tracking.ptu_reservation_rollup import (
PTU_ROLLUP_JOB_ID,
run_ptu_reservation_rollup,
)
scheduler.add_job(
run_ptu_reservation_rollup,
"cron",
hour=0,
minute=15,
args=[prisma_client],
id=PTU_ROLLUP_JOB_ID,
replace_existing=True,
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
)
verbose_proxy_logger.info("PTU reservation rollup job scheduled at 00:15 UTC daily")
### SPEND LOG CLEANUP ###
if general_settings.get("maximum_spend_logs_retention_period") is not None:
spend_log_cleanup = SpendLogCleanup()

View file

@ -0,0 +1,165 @@
"""
Daily rollup for admin-registered PTU reservations.
Writes prorated flat cost to LiteLLM_DailyTeamSpend using a sentinel api_key
so the rows are distinguishable from real per-request rows and share the
existing unique constraint.
"""
from calendar import monthrange
from dataclasses import dataclass
from datetime import date, datetime, time, timedelta, timezone
from typing import Any
from litellm._logging import verbose_proxy_logger
from litellm.repositories.ptu_reservation_repository import PTUReservationRepository
PTU_SENTINEL_API_KEY = "__ptu_reservation__"
PTU_ROLLUP_JOB_ID = "ptu_reservation_rollup_job"
@dataclass(frozen=True, slots=True)
class RollupResult:
day: date
reservations_processed: int
rows_written: int
skipped_flag_off: bool = False
def _days_in_month(day: date) -> int:
return monthrange(day.year, day.month)[1]
def _compute_daily_flat_cost(reservation: Any, day: date) -> float:
"""Return the flat cost attributable to ``day`` for a single reservation."""
if reservation.cost_source != "manual":
return 0.0
if reservation.ptu_count is None or reservation.cost_per_ptu is None:
return 0.0
monthly_total = float(reservation.ptu_count) * float(reservation.cost_per_ptu)
return monthly_total / float(_days_in_month(day))
async def _upsert_ptu_daily_row(
prisma_client: Any,
*,
team_id: str,
model: str,
date_str: str,
reservation_id: str,
flat_cost: float,
) -> None:
"""Idempotent upsert of a sentinel-api_key row on LiteLLM_DailyTeamSpend."""
where = {
"team_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint": {
"team_id": team_id,
"date": date_str,
"api_key": PTU_SENTINEL_API_KEY,
"model": model,
"custom_llm_provider": "",
"mcp_namespaced_tool_name": "",
"endpoint": "",
}
}
now = datetime.now(timezone.utc)
await prisma_client.db.litellm_dailyteamspend.upsert(
where=where,
data={
"create": {
"team_id": team_id,
"date": date_str,
"api_key": PTU_SENTINEL_API_KEY,
"model": model,
"custom_llm_provider": "",
"mcp_namespaced_tool_name": "",
"endpoint": "",
"ptu_flat_cost": flat_cost,
"ptu_reservation_id": reservation_id,
},
"update": {
"ptu_flat_cost": flat_cost,
"ptu_reservation_id": reservation_id,
"updated_at": now,
},
},
)
async def run_ptu_reservation_rollup(
prisma_client: Any,
target_date: date | None = None,
) -> RollupResult:
"""Rollup one UTC day of flat PTU cost across all active reservations.
Defaults to yesterday UTC. Callable from the scheduler and from the CLI
backfill helper; both paths are idempotent under the ``LiteLLM_DailyTeamSpend``
unique constraint.
"""
from litellm.proxy.proxy_server import general_settings
if not general_settings.get("enable_ptu_cost_attribution", False):
verbose_proxy_logger.debug("PTU rollup: feature flag off, skipping")
return RollupResult(
day=target_date or (datetime.now(timezone.utc).date() - timedelta(days=1)),
reservations_processed=0,
rows_written=0,
skipped_flag_off=True,
)
if prisma_client is None:
verbose_proxy_logger.warning("PTU rollup: prisma_client is None, skipping")
return RollupResult(
day=target_date or (datetime.now(timezone.utc).date() - timedelta(days=1)),
reservations_processed=0,
rows_written=0,
)
day = target_date or (datetime.now(timezone.utc).date() - timedelta(days=1))
day_start = datetime.combine(day, time.min, tzinfo=timezone.utc)
date_str = day.isoformat()
repo = PTUReservationRepository(prisma_client)
reservations = await repo.find_active(as_of=day_start)
rows_written = 0
for reservation in reservations:
flat_cost = _compute_daily_flat_cost(reservation, day)
if flat_cost <= 0:
continue
try:
await _upsert_ptu_daily_row(
prisma_client,
team_id=reservation.team_id,
model=reservation.model,
date_str=date_str,
reservation_id=reservation.id,
flat_cost=flat_cost,
)
rows_written += 1
except Exception as exc:
verbose_proxy_logger.error(
"PTU rollup: upsert failed for reservation=%s day=%s: %s",
reservation.id,
date_str,
exc,
)
verbose_proxy_logger.info(
"PTU rollup for %s: %d reservations processed, %d rows written",
date_str,
len(reservations),
rows_written,
)
return RollupResult(
day=day,
reservations_processed=len(reservations),
rows_written=rows_written,
)
__all__ = [
"PTU_ROLLUP_JOB_ID",
"PTU_SENTINEL_API_KEY",
"RollupResult",
"run_ptu_reservation_rollup",
]

View file

@ -875,6 +875,8 @@ model LiteLLM_DailyTeamSpend {
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
ptu_flat_cost Float @default(0.0)
ptu_reservation_id String?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@ -885,6 +887,7 @@ model LiteLLM_DailyTeamSpend {
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
@@index([ptu_reservation_id])
}
// Track daily team spend metrics per model and key

View file

@ -0,0 +1,76 @@
"""Re-run the PTU reservation daily rollup for a specific UTC date.
Idempotent under the LiteLLM_DailyTeamSpend unique constraint.
Usage:
python scripts/ptu_reservation_backfill.py --date 2026-07-12
python scripts/ptu_reservation_backfill.py --date-range 2026-07-01:2026-07-12
"""
import argparse
import asyncio
import os
import sys
from datetime import date, datetime, timedelta
def _parse_date(s: str) -> date:
return datetime.strptime(s, "%Y-%m-%d").date()
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("--date", type=_parse_date, help="Single UTC date (YYYY-MM-DD)")
group.add_argument(
"--date-range",
type=str,
help="Inclusive UTC range as YYYY-MM-DD:YYYY-MM-DD",
)
return parser.parse_args()
def _dates_from_args(args: argparse.Namespace) -> list[date]:
if args.date is not None:
return [args.date]
start_str, _, end_str = args.date_range.partition(":")
start = _parse_date(start_str)
end = _parse_date(end_str)
if end < start:
raise ValueError(f"end date {end} is before start date {start}")
return [start + timedelta(days=i) for i in range((end - start).days + 1)]
async def _run(dates: list[date]) -> int:
from litellm.proxy.proxy_server import prisma_client
from litellm.proxy.spend_tracking.ptu_reservation_rollup import (
run_ptu_reservation_rollup,
)
if prisma_client is None:
print("ERROR: prisma_client is None; is DATABASE_URL set?", file=sys.stderr)
return 2
total_rows = 0
for target in dates:
result = await run_ptu_reservation_rollup(prisma_client, target_date=target)
print(
f"[{result.day.isoformat()}] "
f"reservations={result.reservations_processed} rows_written={result.rows_written}"
f"{' (flag off, skipped)' if result.skipped_flag_off else ''}"
)
total_rows += result.rows_written
print(f"total rows written: {total_rows}")
return 0
def main() -> int:
args = _parse_args()
dates = _dates_from_args(args)
if "PYTHONPATH" not in os.environ:
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
return asyncio.run(_run(dates))
if __name__ == "__main__":
sys.exit(main())

View file

@ -0,0 +1,311 @@
import types
from dataclasses import dataclass
from datetime import date, datetime, timedelta, timezone
from unittest.mock import AsyncMock, MagicMock
import pytest
import litellm.proxy.proxy_server as ps
from litellm.proxy.spend_tracking.ptu_reservation_rollup import (
PTU_SENTINEL_API_KEY,
_compute_daily_flat_cost,
_days_in_month,
run_ptu_reservation_rollup,
)
@dataclass
class _Reservation:
id: str
team_id: str
model: str
cost_source: str
ptu_count: int | None
cost_per_ptu: float | None
effective_from: datetime
effective_to: datetime | None
def _r(
*,
id: str = "res_1",
team_id: str = "team_x",
model: str = "gpt-4",
cost_source: str = "manual",
ptu_count: int | None = 1,
cost_per_ptu: float | None = 200.0,
effective_from: datetime | None = None,
effective_to: datetime | None = None,
) -> _Reservation:
return _Reservation(
id=id,
team_id=team_id,
model=model,
cost_source=cost_source,
ptu_count=ptu_count,
cost_per_ptu=cost_per_ptu,
effective_from=effective_from or datetime(2026, 7, 1, tzinfo=timezone.utc),
effective_to=effective_to,
)
@pytest.fixture
def mock_prisma(monkeypatch):
mock_daily = MagicMock()
mock_daily.upsert = AsyncMock()
mock_reservation = MagicMock()
mock_reservation.find_many = AsyncMock(return_value=[])
prisma = MagicMock()
prisma.db = types.SimpleNamespace(
litellm_dailyteamspend=mock_daily,
litellm_ptureservation=mock_reservation,
)
monkeypatch.setattr(ps, "prisma_client", prisma)
monkeypatch.setitem(ps.general_settings, "enable_ptu_cost_attribution", True)
return prisma, mock_daily, mock_reservation
def test_days_in_month_covers_calendar_variants():
assert _days_in_month(date(2026, 1, 15)) == 31
assert _days_in_month(date(2026, 2, 15)) == 28
assert _days_in_month(date(2024, 2, 15)) == 29
assert _days_in_month(date(2026, 4, 1)) == 30
assert _days_in_month(date(2026, 7, 31)) == 31
def test_compute_flat_cost_calendar_month_31():
r = _r(ptu_count=1, cost_per_ptu=200.0)
assert _compute_daily_flat_cost(r, date(2026, 7, 15)) == pytest.approx(200.0 / 31)
def test_compute_flat_cost_calendar_month_28():
r = _r(ptu_count=1, cost_per_ptu=200.0)
assert _compute_daily_flat_cost(r, date(2026, 2, 10)) == pytest.approx(200.0 / 28)
def test_compute_flat_cost_leap_february():
r = _r(ptu_count=1, cost_per_ptu=200.0)
assert _compute_daily_flat_cost(r, date(2024, 2, 10)) == pytest.approx(200.0 / 29)
def test_compute_flat_cost_scales_with_ptu_count():
small = _compute_daily_flat_cost(_r(ptu_count=1, cost_per_ptu=200.0), date(2026, 7, 1))
big = _compute_daily_flat_cost(_r(ptu_count=100, cost_per_ptu=200.0), date(2026, 7, 1))
assert big == pytest.approx(small * 100)
def test_compute_flat_cost_zero_for_non_manual_source():
r = _r(cost_source="azure_billing", ptu_count=None, cost_per_ptu=None)
assert _compute_daily_flat_cost(r, date(2026, 7, 1)) == 0.0
def test_compute_flat_cost_zero_when_manual_fields_missing():
r = _r(ptu_count=None, cost_per_ptu=None)
assert _compute_daily_flat_cost(r, date(2026, 7, 1)) == 0.0
@pytest.mark.asyncio
async def test_rollup_flag_off_short_circuits(mock_prisma, monkeypatch):
prisma, mock_daily, mock_reservation = mock_prisma
monkeypatch.setitem(ps.general_settings, "enable_ptu_cost_attribution", False)
result = await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
assert result.skipped_flag_off is True
assert result.rows_written == 0
mock_reservation.find_many.assert_not_awaited()
mock_daily.upsert.assert_not_awaited()
@pytest.mark.asyncio
async def test_rollup_prisma_none_returns_zero(monkeypatch):
monkeypatch.setattr(ps, "prisma_client", None)
monkeypatch.setitem(ps.general_settings, "enable_ptu_cost_attribution", True)
result = await run_ptu_reservation_rollup(None, target_date=date(2026, 7, 12))
assert result.rows_written == 0
assert result.skipped_flag_off is False
@pytest.mark.asyncio
async def test_rollup_writes_expected_row(mock_prisma):
prisma, mock_daily, mock_reservation = mock_prisma
reservation = _r(
id="res_1",
team_id="team_x",
model="gpt-4",
ptu_count=1,
cost_per_ptu=200.0,
effective_from=datetime(2026, 7, 1, tzinfo=timezone.utc),
)
mock_reservation.find_many = AsyncMock(return_value=[reservation])
result = await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
assert result.rows_written == 1
assert result.reservations_processed == 1
mock_daily.upsert.assert_awaited_once()
kwargs = mock_daily.upsert.await_args.kwargs
where_key = "team_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint"
assert kwargs["where"][where_key] == {
"team_id": "team_x",
"date": "2026-07-12",
"api_key": PTU_SENTINEL_API_KEY,
"model": "gpt-4",
"custom_llm_provider": "",
"mcp_namespaced_tool_name": "",
"endpoint": "",
}
create = kwargs["data"]["create"]
assert create["team_id"] == "team_x"
assert create["api_key"] == PTU_SENTINEL_API_KEY
assert create["ptu_reservation_id"] == "res_1"
assert create["ptu_flat_cost"] == pytest.approx(200.0 / 31)
assert create.get("spend", 0) == 0 or "spend" not in create
update = kwargs["data"]["update"]
assert update["ptu_flat_cost"] == pytest.approx(200.0 / 31)
assert update["ptu_reservation_id"] == "res_1"
@pytest.mark.asyncio
async def test_rollup_skips_azure_billing_reservations(mock_prisma):
prisma, mock_daily, mock_reservation = mock_prisma
mock_reservation.find_many = AsyncMock(
return_value=[
_r(cost_source="azure_billing", ptu_count=None, cost_per_ptu=None),
]
)
result = await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
assert result.reservations_processed == 1
assert result.rows_written == 0
mock_daily.upsert.assert_not_awaited()
@pytest.mark.asyncio
async def test_rollup_processes_multiple_reservations(mock_prisma):
prisma, mock_daily, mock_reservation = mock_prisma
mock_reservation.find_many = AsyncMock(
return_value=[
_r(id="a", team_id="team_x", model="gpt-4"),
_r(id="b", team_id="team_y", model="gpt-4"),
_r(id="c", team_id="team_x", model="gpt-4o"),
]
)
result = await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
assert result.rows_written == 3
assert mock_daily.upsert.await_count == 3
@pytest.mark.asyncio
async def test_rollup_defaults_target_date_to_yesterday_utc(mock_prisma):
prisma, _, mock_reservation = mock_prisma
mock_reservation.find_many = AsyncMock(return_value=[])
result = await run_ptu_reservation_rollup(prisma)
expected = datetime.now(timezone.utc).date() - timedelta(days=1)
assert result.day == expected
@pytest.mark.asyncio
async def test_rollup_queries_active_reservations_at_day_start(mock_prisma):
prisma, _, mock_reservation = mock_prisma
mock_reservation.find_many = AsyncMock(return_value=[])
await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
mock_reservation.find_many.assert_awaited_once()
where = mock_reservation.find_many.await_args.kwargs["where"]
day_start = datetime(2026, 7, 12, tzinfo=timezone.utc)
assert where["effective_from"] == {"lte": day_start}
assert {"effective_to": None} in where["OR"]
assert {"effective_to": {"gt": day_start}} in where["OR"]
@pytest.mark.asyncio
async def test_rollup_idempotent_second_run_upserts_same_row(mock_prisma):
prisma, mock_daily, mock_reservation = mock_prisma
mock_reservation.find_many = AsyncMock(
return_value=[_r(id="res_1", ptu_count=1, cost_per_ptu=200.0)]
)
r1 = await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
r2 = await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
assert r1.rows_written == 1
assert r2.rows_written == 1
assert mock_daily.upsert.await_count == 2
calls = mock_daily.upsert.await_args_list
where_key = "team_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint"
assert calls[0].kwargs["where"][where_key] == calls[1].kwargs["where"][where_key]
@pytest.mark.asyncio
async def test_rollup_continues_after_single_upsert_failure(mock_prisma):
prisma, mock_daily, mock_reservation = mock_prisma
mock_reservation.find_many = AsyncMock(
return_value=[
_r(id="a", team_id="team_x"),
_r(id="b", team_id="team_y"),
_r(id="c", team_id="team_z"),
]
)
call_count = {"n": 0}
async def flaky_upsert(**_kwargs):
call_count["n"] += 1
if call_count["n"] == 2:
raise RuntimeError("simulated db failure")
mock_daily.upsert = AsyncMock(side_effect=flaky_upsert)
result = await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
assert result.reservations_processed == 3
assert result.rows_written == 2
@pytest.mark.asyncio
async def test_rollup_writes_sentinel_api_key_not_real(mock_prisma):
prisma, mock_daily, mock_reservation = mock_prisma
mock_reservation.find_many = AsyncMock(return_value=[_r(id="res_1")])
await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
kwargs = mock_daily.upsert.await_args.kwargs
assert kwargs["data"]["create"]["api_key"] == PTU_SENTINEL_API_KEY
assert kwargs["data"]["create"]["api_key"] != "sk-real-token"
@pytest.mark.asyncio
async def test_rollup_upserts_zero_spend_tokens_on_sentinel_row(mock_prisma):
prisma, mock_daily, mock_reservation = mock_prisma
mock_reservation.find_many = AsyncMock(return_value=[_r(id="res_1")])
await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
create = mock_daily.upsert.await_args.kwargs["data"]["create"]
update = mock_daily.upsert.await_args.kwargs["data"]["update"]
for k in ("spend", "prompt_tokens", "completion_tokens", "api_requests"):
assert k not in update, f"{k} must not be part of the PTU update payload"
assert k not in create, f"{k} must not be part of the PTU create payload"
@pytest.mark.asyncio
async def test_rollup_boundary_effective_from_at_day_start_is_active(mock_prisma):
prisma, _, mock_reservation = mock_prisma
day_start = datetime(2026, 7, 12, tzinfo=timezone.utc)
r = _r(id="edge", effective_from=day_start)
mock_reservation.find_many = AsyncMock(return_value=[r])
result = await run_ptu_reservation_rollup(prisma, target_date=date(2026, 7, 12))
assert result.rows_written == 1