fix(passthrough): bill a background response once however many times it is read

This commit is contained in:
mateo-berri 2026-10-29 03:30:44 -07:00
parent dff474e450
commit c04506383e
3 changed files with 337 additions and 86 deletions

View file

@ -90,6 +90,10 @@ from litellm.proxy.spend_tracking.savings import (
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.repositories.prisma_protocols import BatchTable
from litellm.responses.streaming_iterator import spawn_logging_task
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.utils import CallTypes
if TYPE_CHECKING:
@ -157,23 +161,35 @@ def _org_member_transaction_key(org_id: str, user_id: str) -> str:
return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}"
def _is_batch_cost_row(payload: SpendLogsPayload) -> bool:
return payload.get("call_type") == CallTypes.aretrieve_batch.value and payload.get("status") == "success"
class _CostRowKwargs(TypedDict, total=False):
settles_background_response: ReadOnly[object]
_BATCH_COST_CLAIM_FIELDS: Final = frozenset({"request_id", "call_type", "spend", "startTime", "endTime", "status"})
_COST_ROW_KWARGS: Final = TypeAdapter(_CostRowKwargs)
def _batch_cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool) -> Mapping[str, object]:
"""Reduce a batch's cost row to what tells the retrieves apart when logging is off.
def _is_charge_once_cost_row(payload: SpendLogsPayload, kwargs: _CostRowKwargs) -> bool:
if payload.get("status") != "success":
return False
return (
payload.get("call_type") == CallTypes.aretrieve_batch.value
or kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES
)
A proxy run with spend logs disabled still needs one row per batch to charge it once,
so the row is written either way, but it carries no request of its own: no metadata,
no requester IP, no key, model, or token counts (LIT-7048).
_COST_ROW_CLAIM_FIELDS: Final = frozenset({"request_id", "call_type", "spend", "startTime", "endTime", "status"})
def _cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool) -> Mapping[str, object]:
"""Reduce a cost row to what tells the reads apart when logging is off.
A proxy run with spend logs disabled still needs one row per batch or background response
to charge it once, so the row is written either way, but it carries no request of its own:
no metadata, no requester IP, no key, model, or token counts (LIT-7048).
"""
if disable_spend_logs is False:
return payload
return MappingProxyType({field: value for field, value in payload.items() if field in _BATCH_COST_CLAIM_FIELDS})
return MappingProxyType({field: value for field, value in payload.items() if field in _COST_ROW_CLAIM_FIELDS})
class _SpendIncrement(TypedDict):
@ -608,8 +624,9 @@ class DBSpendUpdateWriter:
) -> bool:
"""Record the request's spend, answering whether its cost still needs charging.
False only for a batch retrieve whose cost row another retrieve already wrote,
so the caller leaves the key, team, and user counters alone (LIT-7048).
False only for a batch retrieve or a background response read whose cost row an
earlier read already charged, so the caller leaves the key, team, and user counters
alone (LIT-7048).
"""
from litellm.proxy.proxy_server import (
disable_spend_logs,
@ -663,6 +680,7 @@ class DBSpendUpdateWriter:
payload=payload,
prisma_client=prisma_client,
disable_spend_logs=disable_spend_logs,
charge_once=_is_charge_once_cost_row(payload, _COST_ROW_KWARGS.validate_python(kwargs or {})),
llm_router=llm_router,
):
return False
@ -732,26 +750,28 @@ class DBSpendUpdateWriter:
payload: SpendLogsPayload,
prisma_client: "PrismaClient | None",
disable_spend_logs: bool,
charge_once: bool,
llm_router: "Router | None" = None,
) -> bool:
if prisma_client is not None and _is_batch_cost_row(payload):
return await self._claim_batch_cost_spend_log(
if prisma_client is not None and charge_once:
return await self._claim_cost_spend_log(
payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs
)
if disable_spend_logs is False:
await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client, llm_router=llm_router)
return True
async def _claim_batch_cost_spend_log(
async def _claim_cost_spend_log(
self, payload: SpendLogsPayload, prisma_client: "PrismaClient", disable_spend_logs: bool
) -> bool:
"""Write the batch's cost row now, or learn that another retrieve already did.
"""Write the cost row now, or learn that another read already did.
Every retrieve of one batch shares this row, so the insert that lands first owns
the charge and every later one finds the row and charges nothing (LIT-7048). Only
a row that recorded a charge counts: a failed retrieve, a request whose client
picked the batch id as its call id, and the $0 row an older proxy left behind
while the batch was still running all leave the charge to be made.
Every retrieve of one batch, and every read of one finished background response,
shares this row, so the insert that lands first owns the charge and every later one
finds the row and charges nothing (LIT-7048). Only a row of the same call type that
recorded a charge counts: a failed read, a request whose client picked the row's id
as its call id, and the $0 row an older proxy left behind while a batch was still
running all leave the charge to be made.
"""
from litellm.repositories.table_repositories import SpendLogsRepository
@ -762,7 +782,7 @@ class DBSpendUpdateWriter:
)
row: Final = spend_log_row_with_retained_metadata(
_batch_cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields()
_cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields()
)
spend_logs: Final = SpendLogsRepository(prisma_client).table
try:
@ -774,40 +794,36 @@ class DBSpendUpdateWriter:
return True
existing: Final = await spend_logs.find_unique(where={"request_id": request_id})
except Exception as e: # noqa: BLE001 # prisma raises its own hierarchy; an unreachable DB queues the row like any other spend log
verbose_proxy_logger.warning(
"Could not claim spend row %s for a batch's cost, queueing it: %s", request_id, e
)
verbose_proxy_logger.warning("Could not claim cost spend row %s, queueing it: %s", request_id, e)
await self._insert_spend_log_to_db(payload=prisma_client.jsonify_object(row), prisma_client=prisma_client)
return True
if existing is None or existing.call_type != CallTypes.aretrieve_batch.value or existing.status != "success":
if existing is None or existing.call_type != payload["call_type"] or existing.status != "success":
verbose_proxy_logger.warning(
"Spend row %s belongs to a %s request, so this batch's cost is charged without a row of its own",
"Spend row %s belongs to a %s request, so this cost is charged without a row of its own",
request_id,
getattr(existing, "call_type", None),
)
return True
if existing.spend > 0:
verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this batch", request_id)
verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this cost", request_id)
return False
return await self._take_over_uncharged_batch_cost_row(payload=payload, prisma_client=prisma_client, row=row)
return await self._take_over_uncharged_cost_row(payload=payload, prisma_client=prisma_client, row=row)
async def _take_over_uncharged_batch_cost_row(
async def _take_over_uncharged_cost_row(
self, payload: SpendLogsPayload, prisma_client: "PrismaClient", row: Mapping[str, object]
) -> bool:
"""Take the batch's cost row over from the poll that left it charging nothing.
"""Take the cost row over from the read that left it charging nothing.
A pre-upgrade proxy wrote that row every time it polled the batch while it was still
A pre-upgrade proxy wrote a batch's row every time it polled the batch while it was still
running, so the charge is still to be made and the row still has to end up carrying
it. The row stops matching the moment it carries a charge, so it is one retrieve that
it. The row stops matching the moment it carries a charge, so it is one read that
takes it over and charges, and every later one reads the charge and charges nothing.
"""
from litellm.repositories.table_repositories import SpendLogsRepository
request_id: Final = payload["request_id"]
if payload["spend"] <= 0:
verbose_proxy_logger.debug(
"Cost tracking skipped: this batch costs nothing and spend row %s says so", request_id
)
verbose_proxy_logger.debug("Cost tracking skipped: this costs nothing and spend row %s says so", request_id)
return False
try:
taken_over: Final = await SpendLogsRepository(prisma_client).table.update_many(
@ -816,18 +832,18 @@ class DBSpendUpdateWriter:
),
where={
"request_id": request_id,
"call_type": CallTypes.aretrieve_batch.value,
"call_type": payload["call_type"],
"status": "success",
"spend": 0.0,
},
)
except Exception as e: # noqa: BLE001 # prisma raises its own hierarchy; the next retrieve takes the row over
verbose_proxy_logger.warning(
"Could not take over spend row %s, leaving this batch's cost to the next retrieve: %s", request_id, e
"Could not take over spend row %s, leaving this cost to the next read: %s", request_id, e
)
return False
if taken_over == 0:
verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this batch", request_id)
verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this cost", request_id)
return False
return True

View file

@ -0,0 +1,182 @@
import json
import math
import uuid
from collections.abc import Iterator, Mapping
from hashlib import sha256
from pathlib import Path
from typing import Final
from urllib.parse import urlsplit
import pytest
from integration._support.client import (
Gateway,
eventually,
gateway_from_environment,
held,
object_value,
string_value,
)
from integration._support.forward_proxy import tunnelling_forward_proxy
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.tls import server_context, write_self_signed_cert
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 180)
_MODEL: Final = "gpt-5.6"
_INPUT_TOKENS: Final = 1200
_OUTPUT_TOKENS: Final = 300
_READS: Final = 3
_SPEND_LANDING_SECONDS: Final = 70
_LATE_CHARGE_WINDOW_SECONDS: Final = 10
_COST_MAP: Final = Path("litellm/model_prices_and_context_window_backup.json")
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def _response(response_id: str, status: str) -> Mapping[str, JsonValue]:
finished: Final = status == "completed"
return {
"id": response_id,
"object": "response",
"created_at": 1,
"status": status,
"background": True,
"error": None,
"incomplete_details": None,
"model": _MODEL,
"output": [
{
"id": f"msg_{uuid.uuid4().hex}",
"type": "message",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "pong", "annotations": []}],
}
]
if finished
else [],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"usage": {
"input_tokens": _INPUT_TOKENS,
"output_tokens": _OUTPUT_TOKENS,
"total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS,
"input_tokens_details": {"cached_tokens": 0},
"output_tokens_details": {"reasoning_tokens": 0},
}
if finished
else None,
"metadata": {},
}
def _openai(request: Request) -> Reply:
path: Final = urlsplit(request.target).path
if request.method == "POST" and path == "/v1/responses":
return Reply(body=json.dumps(_response(f"resp_{uuid.uuid4().hex}", "queued")).encode())
return Reply(body=json.dumps(_response(path.rsplit("/", 1)[-1], "completed")).encode())
def _port(url: str) -> int:
port: Final = urlsplit(url).port
assert port is not None, url
return port
@pytest.fixture(scope="module")
def proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
directory: Final = tmp_path_factory.mktemp("passthrough-background-response")
certificate: Final = write_self_signed_cert(directory, ("api.openai.com",))
with (
wire_server(_openai, tls=server_context(*certificate)) as upstream,
tunnelling_forward_proxy(_port(upstream.url)) as tunnel,
gateway_from_environment() as base,
owned_proxy(
base,
directory,
{
"HTTPS_PROXY": tunnel.url,
"NO_PROXY": "127.0.0.1,localhost",
"AIOHTTP_TRUST_ENV": "True",
"SSL_VERIFY": "False",
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
"OPENAI_API_KEY": "sk-scripted-openai",
},
workers=2,
) as owned,
):
yield owned
def _price() -> float:
row: Final = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(_COST_MAP.read_bytes())[_MODEL])
input_rate: Final = row["input_cost_per_token"]
output_rate: Final = row["output_cost_per_token"]
assert isinstance(input_rate, float) and isinstance(output_rate, float), row
return _INPUT_TOKENS * input_rate + _OUTPUT_TOKENS * output_rate
def _passthrough(proxy: Gateway, method: str, path: str, key: str, body: Mapping[str, JsonValue] | None = None) -> str:
response: Final = proxy.request(method, f"/openai_passthrough/v1/responses{path}", body, key=key)
assert response.status_code == 200, response.text
payload: Final = object_value(response.json())
return string_value(payload["id"]) if method == "POST" else string_value(payload["status"])
def _number(value: JsonValue) -> float:
assert isinstance(value, int | float), value
return float(value)
def _spend_of(proxy: Gateway, table: str, column: str, identity: str) -> float:
return _number(proxy.rows(f'SELECT spend FROM "{table}" WHERE {column} = %s', (identity,))[0]["spend"])
def _key_spend(proxy: Gateway, key: str) -> float:
return _number(object_value(proxy.get("/key/info", {"key": key})["info"])["spend"])
def _logged_spend(proxy: Gateway, key: str) -> float:
rows: Final = proxy.rows(
'SELECT spend FROM "LiteLLM_SpendLogs" WHERE api_key = %s', (sha256(key.encode()).hexdigest(),)
)
return math.fsum(_number(row["spend"]) for row in rows)
def test_a_background_response_read_through_the_openai_passthrough_bills_its_key_team_user_and_org_once(
proxy: Gateway,
) -> None:
price: Final = _price()
with proxy.scenario() as scenario:
organization_id: Final = scenario.organization()
team_id: Final = scenario.team(organization_id=organization_id)
user_id: Final = scenario.org_member(organization_id, "internal_user")
proxy.post("/team/member_add", {"team_id": team_id, "member": {"role": "user", "user_id": user_id}})
key: Final = scenario.key(team_id=team_id, user_id=user_id)
response_id: Final = _passthrough(
proxy, "POST", "", key, {"model": _MODEL, "input": "bill me once", "background": True}
)
reads: Final = tuple(_passthrough(proxy, "GET", f"/{response_id}", key) for _ in range(_READS))
assert reads == ("completed",) * _READS
def spent() -> tuple[float, ...]:
return (
_key_spend(proxy, key),
_spend_of(proxy, "LiteLLM_TeamTable", "team_id", team_id),
_spend_of(proxy, "LiteLLM_UserTable", "user_id", user_id),
_spend_of(proxy, "LiteLLM_OrganizationTable", "organization_id", organization_id),
)
held(
spent,
lambda observed: all(math.isclose(value, price, rel_tol=1e-9) for value in observed),
holding=_LATE_CHARGE_WINDOW_SECONDS,
seconds=_SPEND_LANDING_SECONDS,
)
eventually(
lambda: (_logged_spend(proxy, key), _key_spend(proxy, key)),
lambda observed: math.isclose(observed[0], observed[1], rel_tol=1e-9),
seconds=_SPEND_LANDING_SECONDS,
)

View file

@ -3,10 +3,10 @@ import copy
import json
import logging
import re
from collections.abc import AsyncIterator, Callable
from collections.abc import AsyncIterator, Callable, Mapping
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from types import MappingProxyType, SimpleNamespace
from typing import Final, cast
from unittest.mock import AsyncMock, MagicMock, call, patch
@ -37,6 +37,10 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL
from litellm.responses.streaming_iterator import drain_logging_tasks
from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from tests.unit.proxy.db.fake_prisma_engine import engine_call
@ -4334,6 +4338,36 @@ def _batch_cost_payload() -> dict:
}
def _background_response_cost_payload() -> dict:
return {
**_minimal_spend_payload(),
"request_id": "resp_abc_background_cost",
"call_type": "pass_through_endpoint",
"status": "success",
}
_BATCH_RETRIEVE_KWARGS: Final = MappingProxyType({"model": "gpt-5.6-luna", "call_type": "aretrieve_batch"})
_PASSTHROUGH_CALL_KWARGS: Final = MappingProxyType(
{"model": "gpt-5.6", "call_type": "pass_through_endpoint", SETTLES_BACKGROUND_RESPONSE_KWARG: None}
)
_BACKGROUND_SETTLEMENT_KWARGS: Final = MappingProxyType(
{**_PASSTHROUGH_CALL_KWARGS, SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES}
)
_EVERY_CHARGE_ONCE_ROW: Final = pytest.mark.parametrize(
("cost_payload", "cost_kwargs"),
[
(_batch_cost_payload(), _BATCH_RETRIEVE_KWARGS),
(_background_response_cost_payload(), _BACKGROUND_SETTLEMENT_KWARGS),
],
ids=["batch_retrieve", "finished_background_response_read"],
)
def _existing_row(cost_payload: dict, **fields: object) -> SimpleNamespace:
return SimpleNamespace(**{"call_type": cost_payload["call_type"], "status": "success", **fields})
def _spend_logs_prisma(inserted: int, existing: object, taken_over: int = 1) -> MagicMock:
prisma = _tool_usage_prisma()
prisma.jsonify_object = lambda data: dict(data)
@ -4349,6 +4383,7 @@ async def _update_database_with(
payload: dict,
disable_spend_logs: bool = False,
response_cost: float = 0.25,
kwargs: Mapping[str, object] | None = _BATCH_RETRIEVE_KWARGS,
) -> bool:
with (
patch( # test-quality-ok: update_database reads this proxy_server global at call time, no seam
@ -4371,7 +4406,7 @@ async def _update_database_with(
end_user_id=None,
team_id=None,
org_id=None,
kwargs={"model": "gpt-5.6-luna", "call_type": "aretrieve_batch"},
kwargs=None if kwargs is None else dict(kwargs),
completion_response=None,
start_time=datetime.now(timezone.utc),
end_time=datetime.now(timezone.utc),
@ -4382,73 +4417,79 @@ async def _update_database_with(
@pytest.mark.asyncio
@_EVERY_CHARGE_ONCE_ROW
@pytest.mark.parametrize(
("inserted", "existing", "charged"),
("inserted", "existing_fields", "charged"),
[
(1, None, True),
(0, SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.25), False),
(0, SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.0), True),
(0, SimpleNamespace(call_type="aretrieve_batch", status="failure", spend=0.0), True),
(0, SimpleNamespace(call_type="aembedding", status="success", spend=0.25), True),
(0, {"spend": 0.25}, False),
(0, {"spend": 0.0}, True),
(0, {"status": "failure", "spend": 0.0}, True),
(0, {"call_type": "aembedding", "spend": 0.25}, True),
(0, None, True),
],
ids=[
"first_retrieve_owns_the_row",
"another_retrieve_already_charged",
"an_older_proxy_left_a_zero_row_while_the_batch_ran",
"failed_retrieve_holds_the_row",
"first_read_owns_the_row",
"another_read_already_charged",
"an_older_proxy_left_a_zero_row",
"failed_read_holds_the_row",
"client_chosen_call_id_holds_the_row",
"row_gone_between_insert_and_lookup",
],
)
async def test_update_database_charges_a_batch_only_from_the_retrieve_that_wrote_its_row(
inserted: int, existing: object, charged: bool
async def test_update_database_charges_a_cost_row_only_from_the_read_that_wrote_it(
cost_payload: dict,
cost_kwargs: Mapping[str, object],
inserted: int,
existing_fields: dict | None,
charged: bool,
):
"""
Every retrieve of one batch shares one spend row, so the insert that lands first is
the charge and every later retrieve must leave the counters alone (LIT-7048). A row
that recorded no charge must not be able to take the charge away: neither one a
client planted under the batch id, nor the $0 row a pre-upgrade proxy wrote every
time it polled the batch while it was still running.
Every retrieve of one batch, and every read of one finished background response, shares
one spend row, so the insert that lands first is the charge and every later read must
leave the counters alone (LIT-7048). A row that recorded no charge must not be able to
take the charge away: neither one a client planted under the row's id, nor the $0 row a
pre-upgrade proxy wrote every time it polled a batch while it was still running.
"""
db_writer = DBSpendUpdateWriter()
db_writer._batch_database_updates = AsyncMock()
existing = None if existing_fields is None else _existing_row(cost_payload, **existing_fields)
prisma = _spend_logs_prisma(inserted, existing)
assert await _update_database_with(db_writer, prisma, _batch_cost_payload()) is charged
assert await _update_database_with(db_writer, prisma, cost_payload, kwargs=cost_kwargs) is charged
claimed_rows = prisma.db.litellm_spendlogs.create_many.await_args.kwargs
assert claimed_rows["skip_duplicates"] is True
assert [(row["request_id"], row["spend"]) for row in claimed_rows["data"]] == [("batch_abc_batch_cost", 0.25)]
assert [(row["request_id"], row["spend"]) for row in claimed_rows["data"]] == [(cost_payload["request_id"], 0.25)]
assert prisma.spend_log_transactions == []
assert db_writer._batch_database_updates.await_count == (1 if charged else 0)
@pytest.mark.asyncio
@_EVERY_CHARGE_ONCE_ROW
@pytest.mark.parametrize(
("taken_over", "charged"),
[(1, True), (0, False)],
ids=["this_retrieve_takes_it_over", "another_one_got_there_first"],
ids=["this_read_takes_it_over", "another_one_got_there_first"],
)
async def test_update_database_charges_a_batch_whose_row_a_pre_upgrade_poll_left_at_zero(
taken_over: int, charged: bool
async def test_update_database_charges_a_cost_row_left_at_zero(
cost_payload: dict, cost_kwargs: Mapping[str, object], taken_over: int, charged: bool
):
"""
A proxy without this fix wrote the batch's row at $0 on every poll of a running batch,
A proxy without the batch fix wrote the batch's row at $0 on every poll of a running batch,
and the row outlives the upgrade, so the charge has to land on the row itself. Charging
without writing it there would charge again on every later retrieve (LIT-7048).
without writing it there would charge again on every later read (LIT-7048).
"""
db_writer = DBSpendUpdateWriter()
db_writer._batch_database_updates = AsyncMock()
existing = SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.0)
prisma = _spend_logs_prisma(0, existing, taken_over)
prisma = _spend_logs_prisma(0, _existing_row(cost_payload, spend=0.0), taken_over)
assert await _update_database_with(db_writer, prisma, _batch_cost_payload()) is charged
assert await _update_database_with(db_writer, prisma, cost_payload, kwargs=cost_kwargs) is charged
taken = prisma.db.litellm_spendlogs.update_many.await_args.kwargs
assert taken["where"] == {
"request_id": "batch_abc_batch_cost",
"call_type": "aretrieve_batch",
"request_id": cost_payload["request_id"],
"call_type": cost_payload["call_type"],
"status": "success",
"spend": 0.0,
}
@ -4494,26 +4535,24 @@ async def test_update_database_leaves_a_batch_that_cost_nothing_to_the_retrieve_
@pytest.mark.asyncio
@_EVERY_CHARGE_ONCE_ROW
@pytest.mark.parametrize(
("inserted", "existing", "charged"),
[
(1, None, True),
(0, SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.25), False),
],
ids=["first_retrieve_owns_the_row", "another_retrieve_already_charged"],
("inserted", "already_charged", "charged"),
[(1, False, True), (0, True, False)],
ids=["first_read_owns_the_row", "another_read_already_charged"],
)
async def test_update_database_charges_a_batch_once_even_with_spend_logs_disabled(
inserted: int, existing: object, charged: bool
async def test_update_database_charges_a_cost_row_once_even_with_spend_logs_disabled(
cost_payload: dict, cost_kwargs: Mapping[str, object], inserted: int, already_charged: bool, charged: bool
):
"""
disable_spend_logs drops the per-request logs, not the batch's charge, so the one row
that makes a batch chargeable exactly once is still written and still read back.
disable_spend_logs drops the per-request logs, not the charge, so the one row that makes
a batch or a background response chargeable exactly once is still written and read back.
"""
db_writer = DBSpendUpdateWriter()
db_writer._batch_database_updates = AsyncMock()
prisma = _spend_logs_prisma(inserted, existing)
prisma = _spend_logs_prisma(inserted, _existing_row(cost_payload, spend=0.25) if already_charged else None)
assert await _update_database_with(db_writer, prisma, _batch_cost_payload(), True) is charged
assert await _update_database_with(db_writer, prisma, cost_payload, True, kwargs=cost_kwargs) is charged
assert prisma.db.litellm_spendlogs.create_many.await_count == 1
assert db_writer._batch_database_updates.await_count == (1 if charged else 0)
@ -4606,16 +4645,30 @@ async def test_update_database_queues_a_batch_cost_row_it_could_not_claim():
@pytest.mark.asyncio
@pytest.mark.parametrize(
"payload",
[{**_batch_cost_payload(), "call_type": "acompletion"}, {**_batch_cost_payload(), "status": "failure"}],
ids=["not_a_batch_retrieve", "failed_batch_retrieve"],
("payload", "kwargs"),
[
({**_batch_cost_payload(), "call_type": "acompletion"}, _BATCH_RETRIEVE_KWARGS),
({**_batch_cost_payload(), "status": "failure"}, _BATCH_RETRIEVE_KWARGS),
(_background_response_cost_payload(), _PASSTHROUGH_CALL_KWARGS),
(_background_response_cost_payload(), None),
({**_background_response_cost_payload(), "status": "failure"}, _BACKGROUND_SETTLEMENT_KWARGS),
],
ids=[
"not_a_batch_retrieve",
"failed_batch_retrieve",
"passthrough_call_that_settles_nothing",
"call_without_kwargs",
"failed_background_response_read",
],
)
async def test_update_database_queues_every_other_spend_row_for_the_next_flush(payload: dict):
async def test_update_database_queues_every_other_spend_row_for_the_next_flush(
payload: dict, kwargs: Mapping[str, object] | None
):
db_writer = DBSpendUpdateWriter()
db_writer._batch_database_updates = AsyncMock()
prisma = _spend_logs_prisma(1, None)
assert await _update_database_with(db_writer, prisma, payload) is True
assert await _update_database_with(db_writer, prisma, payload, kwargs=kwargs) is True
prisma.db.litellm_spendlogs.create_many.assert_not_called()
assert prisma.spend_log_transactions == [payload]