From c04506383e2a375edc31402913eacd5362bf0492 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:30:44 -0700 Subject: [PATCH] fix(passthrough): bill a background response once however many times it is read --- litellm/proxy/db/db_spend_update_writer.py | 92 +++++---- ...through_background_response_billed_once.py | 182 ++++++++++++++++++ .../proxy/db/test_db_spend_update_writer.py | 149 +++++++++----- 3 files changed, 337 insertions(+), 86 deletions(-) create mode 100644 tests/integration/spend/test_passthrough_background_response_billed_once.py diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 8ac2597c99b..617ab1b0bfe 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -90,6 +90,10 @@ from litellm.proxy.spend_tracking.savings import ( from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.repositories.prisma_protocols import BatchTable from litellm.responses.streaming_iterator import spawn_logging_task +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) from litellm.types.utils import CallTypes if TYPE_CHECKING: @@ -157,23 +161,35 @@ def _org_member_transaction_key(org_id: str, user_id: str) -> str: return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}" -def _is_batch_cost_row(payload: SpendLogsPayload) -> bool: - return payload.get("call_type") == CallTypes.aretrieve_batch.value and payload.get("status") == "success" +class _CostRowKwargs(TypedDict, total=False): + settles_background_response: ReadOnly[object] -_BATCH_COST_CLAIM_FIELDS: Final = frozenset({"request_id", "call_type", "spend", "startTime", "endTime", "status"}) +_COST_ROW_KWARGS: Final = TypeAdapter(_CostRowKwargs) -def _batch_cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool) -> Mapping[str, object]: - """Reduce a batch's cost row to what tells the retrieves apart when logging is off. +def _is_charge_once_cost_row(payload: SpendLogsPayload, kwargs: _CostRowKwargs) -> bool: + if payload.get("status") != "success": + return False + return ( + payload.get("call_type") == CallTypes.aretrieve_batch.value + or kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES + ) - A proxy run with spend logs disabled still needs one row per batch to charge it once, - so the row is written either way, but it carries no request of its own: no metadata, - no requester IP, no key, model, or token counts (LIT-7048). + +_COST_ROW_CLAIM_FIELDS: Final = frozenset({"request_id", "call_type", "spend", "startTime", "endTime", "status"}) + + +def _cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool) -> Mapping[str, object]: + """Reduce a cost row to what tells the reads apart when logging is off. + + A proxy run with spend logs disabled still needs one row per batch or background response + to charge it once, so the row is written either way, but it carries no request of its own: + no metadata, no requester IP, no key, model, or token counts (LIT-7048). """ if disable_spend_logs is False: return payload - return MappingProxyType({field: value for field, value in payload.items() if field in _BATCH_COST_CLAIM_FIELDS}) + return MappingProxyType({field: value for field, value in payload.items() if field in _COST_ROW_CLAIM_FIELDS}) class _SpendIncrement(TypedDict): @@ -608,8 +624,9 @@ class DBSpendUpdateWriter: ) -> bool: """Record the request's spend, answering whether its cost still needs charging. - False only for a batch retrieve whose cost row another retrieve already wrote, - so the caller leaves the key, team, and user counters alone (LIT-7048). + False only for a batch retrieve or a background response read whose cost row an + earlier read already charged, so the caller leaves the key, team, and user counters + alone (LIT-7048). """ from litellm.proxy.proxy_server import ( disable_spend_logs, @@ -663,6 +680,7 @@ class DBSpendUpdateWriter: payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs, + charge_once=_is_charge_once_cost_row(payload, _COST_ROW_KWARGS.validate_python(kwargs or {})), llm_router=llm_router, ): return False @@ -732,26 +750,28 @@ class DBSpendUpdateWriter: payload: SpendLogsPayload, prisma_client: "PrismaClient | None", disable_spend_logs: bool, + charge_once: bool, llm_router: "Router | None" = None, ) -> bool: - if prisma_client is not None and _is_batch_cost_row(payload): - return await self._claim_batch_cost_spend_log( + if prisma_client is not None and charge_once: + return await self._claim_cost_spend_log( payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs ) if disable_spend_logs is False: await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client, llm_router=llm_router) return True - async def _claim_batch_cost_spend_log( + async def _claim_cost_spend_log( self, payload: SpendLogsPayload, prisma_client: "PrismaClient", disable_spend_logs: bool ) -> bool: - """Write the batch's cost row now, or learn that another retrieve already did. + """Write the cost row now, or learn that another read already did. - Every retrieve of one batch shares this row, so the insert that lands first owns - the charge and every later one finds the row and charges nothing (LIT-7048). Only - a row that recorded a charge counts: a failed retrieve, a request whose client - picked the batch id as its call id, and the $0 row an older proxy left behind - while the batch was still running all leave the charge to be made. + Every retrieve of one batch, and every read of one finished background response, + shares this row, so the insert that lands first owns the charge and every later one + finds the row and charges nothing (LIT-7048). Only a row of the same call type that + recorded a charge counts: a failed read, a request whose client picked the row's id + as its call id, and the $0 row an older proxy left behind while a batch was still + running all leave the charge to be made. """ from litellm.repositories.table_repositories import SpendLogsRepository @@ -762,7 +782,7 @@ class DBSpendUpdateWriter: ) row: Final = spend_log_row_with_retained_metadata( - _batch_cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields() + _cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields() ) spend_logs: Final = SpendLogsRepository(prisma_client).table try: @@ -774,40 +794,36 @@ class DBSpendUpdateWriter: return True existing: Final = await spend_logs.find_unique(where={"request_id": request_id}) except Exception as e: # noqa: BLE001 # prisma raises its own hierarchy; an unreachable DB queues the row like any other spend log - verbose_proxy_logger.warning( - "Could not claim spend row %s for a batch's cost, queueing it: %s", request_id, e - ) + verbose_proxy_logger.warning("Could not claim cost spend row %s, queueing it: %s", request_id, e) await self._insert_spend_log_to_db(payload=prisma_client.jsonify_object(row), prisma_client=prisma_client) return True - if existing is None or existing.call_type != CallTypes.aretrieve_batch.value or existing.status != "success": + if existing is None or existing.call_type != payload["call_type"] or existing.status != "success": verbose_proxy_logger.warning( - "Spend row %s belongs to a %s request, so this batch's cost is charged without a row of its own", + "Spend row %s belongs to a %s request, so this cost is charged without a row of its own", request_id, getattr(existing, "call_type", None), ) return True if existing.spend > 0: - verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this batch", request_id) + verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this cost", request_id) return False - return await self._take_over_uncharged_batch_cost_row(payload=payload, prisma_client=prisma_client, row=row) + return await self._take_over_uncharged_cost_row(payload=payload, prisma_client=prisma_client, row=row) - async def _take_over_uncharged_batch_cost_row( + async def _take_over_uncharged_cost_row( self, payload: SpendLogsPayload, prisma_client: "PrismaClient", row: Mapping[str, object] ) -> bool: - """Take the batch's cost row over from the poll that left it charging nothing. + """Take the cost row over from the read that left it charging nothing. - A pre-upgrade proxy wrote that row every time it polled the batch while it was still + A pre-upgrade proxy wrote a batch's row every time it polled the batch while it was still running, so the charge is still to be made and the row still has to end up carrying - it. The row stops matching the moment it carries a charge, so it is one retrieve that + it. The row stops matching the moment it carries a charge, so it is one read that takes it over and charges, and every later one reads the charge and charges nothing. """ from litellm.repositories.table_repositories import SpendLogsRepository request_id: Final = payload["request_id"] if payload["spend"] <= 0: - verbose_proxy_logger.debug( - "Cost tracking skipped: this batch costs nothing and spend row %s says so", request_id - ) + verbose_proxy_logger.debug("Cost tracking skipped: this costs nothing and spend row %s says so", request_id) return False try: taken_over: Final = await SpendLogsRepository(prisma_client).table.update_many( @@ -816,18 +832,18 @@ class DBSpendUpdateWriter: ), where={ "request_id": request_id, - "call_type": CallTypes.aretrieve_batch.value, + "call_type": payload["call_type"], "status": "success", "spend": 0.0, }, ) except Exception as e: # noqa: BLE001 # prisma raises its own hierarchy; the next retrieve takes the row over verbose_proxy_logger.warning( - "Could not take over spend row %s, leaving this batch's cost to the next retrieve: %s", request_id, e + "Could not take over spend row %s, leaving this cost to the next read: %s", request_id, e ) return False if taken_over == 0: - verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this batch", request_id) + verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this cost", request_id) return False return True diff --git a/tests/integration/spend/test_passthrough_background_response_billed_once.py b/tests/integration/spend/test_passthrough_background_response_billed_once.py new file mode 100644 index 00000000000..c7b8deda716 --- /dev/null +++ b/tests/integration/spend/test_passthrough_background_response_billed_once.py @@ -0,0 +1,182 @@ +import json +import math +import uuid +from collections.abc import Iterator, Mapping +from hashlib import sha256 +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit + +import pytest +from integration._support.client import ( + Gateway, + eventually, + gateway_from_environment, + held, + object_value, + string_value, +) +from integration._support.forward_proxy import tunnelling_forward_proxy +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 180) + +_MODEL: Final = "gpt-5.6" +_INPUT_TOKENS: Final = 1200 +_OUTPUT_TOKENS: Final = 300 +_READS: Final = 3 +_SPEND_LANDING_SECONDS: Final = 70 +_LATE_CHARGE_WINDOW_SECONDS: Final = 10 +_COST_MAP: Final = Path("litellm/model_prices_and_context_window_backup.json") +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _response(response_id: str, status: str) -> Mapping[str, JsonValue]: + finished: Final = status == "completed" + return { + "id": response_id, + "object": "response", + "created_at": 1, + "status": status, + "background": True, + "error": None, + "incomplete_details": None, + "model": _MODEL, + "output": [ + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], + } + ] + if finished + else [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": _INPUT_TOKENS, + "output_tokens": _OUTPUT_TOKENS, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + } + if finished + else None, + "metadata": {}, + } + + +def _openai(request: Request) -> Reply: + path: Final = urlsplit(request.target).path + if request.method == "POST" and path == "/v1/responses": + return Reply(body=json.dumps(_response(f"resp_{uuid.uuid4().hex}", "queued")).encode()) + return Reply(body=json.dumps(_response(path.rsplit("/", 1)[-1], "completed")).encode()) + + +def _port(url: str) -> int: + port: Final = urlsplit(url).port + assert port is not None, url + return port + + +@pytest.fixture(scope="module") +def proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("passthrough-background-response") + certificate: Final = write_self_signed_cert(directory, ("api.openai.com",)) + with ( + wire_server(_openai, tls=server_context(*certificate)) as upstream, + tunnelling_forward_proxy(_port(upstream.url)) as tunnel, + gateway_from_environment() as base, + owned_proxy( + base, + directory, + { + "HTTPS_PROXY": tunnel.url, + "NO_PROXY": "127.0.0.1,localhost", + "AIOHTTP_TRUST_ENV": "True", + "SSL_VERIFY": "False", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "OPENAI_API_KEY": "sk-scripted-openai", + }, + workers=2, + ) as owned, + ): + yield owned + + +def _price() -> float: + row: Final = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(_COST_MAP.read_bytes())[_MODEL]) + input_rate: Final = row["input_cost_per_token"] + output_rate: Final = row["output_cost_per_token"] + assert isinstance(input_rate, float) and isinstance(output_rate, float), row + return _INPUT_TOKENS * input_rate + _OUTPUT_TOKENS * output_rate + + +def _passthrough(proxy: Gateway, method: str, path: str, key: str, body: Mapping[str, JsonValue] | None = None) -> str: + response: Final = proxy.request(method, f"/openai_passthrough/v1/responses{path}", body, key=key) + assert response.status_code == 200, response.text + payload: Final = object_value(response.json()) + return string_value(payload["id"]) if method == "POST" else string_value(payload["status"]) + + +def _number(value: JsonValue) -> float: + assert isinstance(value, int | float), value + return float(value) + + +def _spend_of(proxy: Gateway, table: str, column: str, identity: str) -> float: + return _number(proxy.rows(f'SELECT spend FROM "{table}" WHERE {column} = %s', (identity,))[0]["spend"]) + + +def _key_spend(proxy: Gateway, key: str) -> float: + return _number(object_value(proxy.get("/key/info", {"key": key})["info"])["spend"]) + + +def _logged_spend(proxy: Gateway, key: str) -> float: + rows: Final = proxy.rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE api_key = %s', (sha256(key.encode()).hexdigest(),) + ) + return math.fsum(_number(row["spend"]) for row in rows) + + +def test_a_background_response_read_through_the_openai_passthrough_bills_its_key_team_user_and_org_once( + proxy: Gateway, +) -> None: + price: Final = _price() + with proxy.scenario() as scenario: + organization_id: Final = scenario.organization() + team_id: Final = scenario.team(organization_id=organization_id) + user_id: Final = scenario.org_member(organization_id, "internal_user") + proxy.post("/team/member_add", {"team_id": team_id, "member": {"role": "user", "user_id": user_id}}) + key: Final = scenario.key(team_id=team_id, user_id=user_id) + response_id: Final = _passthrough( + proxy, "POST", "", key, {"model": _MODEL, "input": "bill me once", "background": True} + ) + reads: Final = tuple(_passthrough(proxy, "GET", f"/{response_id}", key) for _ in range(_READS)) + assert reads == ("completed",) * _READS + + def spent() -> tuple[float, ...]: + return ( + _key_spend(proxy, key), + _spend_of(proxy, "LiteLLM_TeamTable", "team_id", team_id), + _spend_of(proxy, "LiteLLM_UserTable", "user_id", user_id), + _spend_of(proxy, "LiteLLM_OrganizationTable", "organization_id", organization_id), + ) + + held( + spent, + lambda observed: all(math.isclose(value, price, rel_tol=1e-9) for value in observed), + holding=_LATE_CHARGE_WINDOW_SECONDS, + seconds=_SPEND_LANDING_SECONDS, + ) + eventually( + lambda: (_logged_spend(proxy, key), _key_spend(proxy, key)), + lambda observed: math.isclose(observed[0], observed[1], rel_tol=1e-9), + seconds=_SPEND_LANDING_SECONDS, + ) diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index a42b6404278..a86d5dcf4c3 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -3,10 +3,10 @@ import copy import json import logging import re -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncIterator, Callable, Mapping from contextlib import AbstractAsyncContextManager, asynccontextmanager from datetime import datetime, timedelta, timezone -from types import SimpleNamespace +from types import MappingProxyType, SimpleNamespace from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, call, patch @@ -37,6 +37,10 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -4334,6 +4338,36 @@ def _batch_cost_payload() -> dict: } +def _background_response_cost_payload() -> dict: + return { + **_minimal_spend_payload(), + "request_id": "resp_abc_background_cost", + "call_type": "pass_through_endpoint", + "status": "success", + } + + +_BATCH_RETRIEVE_KWARGS: Final = MappingProxyType({"model": "gpt-5.6-luna", "call_type": "aretrieve_batch"}) +_PASSTHROUGH_CALL_KWARGS: Final = MappingProxyType( + {"model": "gpt-5.6", "call_type": "pass_through_endpoint", SETTLES_BACKGROUND_RESPONSE_KWARG: None} +) +_BACKGROUND_SETTLEMENT_KWARGS: Final = MappingProxyType( + {**_PASSTHROUGH_CALL_KWARGS, SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES} +) +_EVERY_CHARGE_ONCE_ROW: Final = pytest.mark.parametrize( + ("cost_payload", "cost_kwargs"), + [ + (_batch_cost_payload(), _BATCH_RETRIEVE_KWARGS), + (_background_response_cost_payload(), _BACKGROUND_SETTLEMENT_KWARGS), + ], + ids=["batch_retrieve", "finished_background_response_read"], +) + + +def _existing_row(cost_payload: dict, **fields: object) -> SimpleNamespace: + return SimpleNamespace(**{"call_type": cost_payload["call_type"], "status": "success", **fields}) + + def _spend_logs_prisma(inserted: int, existing: object, taken_over: int = 1) -> MagicMock: prisma = _tool_usage_prisma() prisma.jsonify_object = lambda data: dict(data) @@ -4349,6 +4383,7 @@ async def _update_database_with( payload: dict, disable_spend_logs: bool = False, response_cost: float = 0.25, + kwargs: Mapping[str, object] | None = _BATCH_RETRIEVE_KWARGS, ) -> bool: with ( patch( # test-quality-ok: update_database reads this proxy_server global at call time, no seam @@ -4371,7 +4406,7 @@ async def _update_database_with( end_user_id=None, team_id=None, org_id=None, - kwargs={"model": "gpt-5.6-luna", "call_type": "aretrieve_batch"}, + kwargs=None if kwargs is None else dict(kwargs), completion_response=None, start_time=datetime.now(timezone.utc), end_time=datetime.now(timezone.utc), @@ -4382,73 +4417,79 @@ async def _update_database_with( @pytest.mark.asyncio +@_EVERY_CHARGE_ONCE_ROW @pytest.mark.parametrize( - ("inserted", "existing", "charged"), + ("inserted", "existing_fields", "charged"), [ (1, None, True), - (0, SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.25), False), - (0, SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.0), True), - (0, SimpleNamespace(call_type="aretrieve_batch", status="failure", spend=0.0), True), - (0, SimpleNamespace(call_type="aembedding", status="success", spend=0.25), True), + (0, {"spend": 0.25}, False), + (0, {"spend": 0.0}, True), + (0, {"status": "failure", "spend": 0.0}, True), + (0, {"call_type": "aembedding", "spend": 0.25}, True), (0, None, True), ], ids=[ - "first_retrieve_owns_the_row", - "another_retrieve_already_charged", - "an_older_proxy_left_a_zero_row_while_the_batch_ran", - "failed_retrieve_holds_the_row", + "first_read_owns_the_row", + "another_read_already_charged", + "an_older_proxy_left_a_zero_row", + "failed_read_holds_the_row", "client_chosen_call_id_holds_the_row", "row_gone_between_insert_and_lookup", ], ) -async def test_update_database_charges_a_batch_only_from_the_retrieve_that_wrote_its_row( - inserted: int, existing: object, charged: bool +async def test_update_database_charges_a_cost_row_only_from_the_read_that_wrote_it( + cost_payload: dict, + cost_kwargs: Mapping[str, object], + inserted: int, + existing_fields: dict | None, + charged: bool, ): """ - Every retrieve of one batch shares one spend row, so the insert that lands first is - the charge and every later retrieve must leave the counters alone (LIT-7048). A row - that recorded no charge must not be able to take the charge away: neither one a - client planted under the batch id, nor the $0 row a pre-upgrade proxy wrote every - time it polled the batch while it was still running. + Every retrieve of one batch, and every read of one finished background response, shares + one spend row, so the insert that lands first is the charge and every later read must + leave the counters alone (LIT-7048). A row that recorded no charge must not be able to + take the charge away: neither one a client planted under the row's id, nor the $0 row a + pre-upgrade proxy wrote every time it polled a batch while it was still running. """ db_writer = DBSpendUpdateWriter() db_writer._batch_database_updates = AsyncMock() + existing = None if existing_fields is None else _existing_row(cost_payload, **existing_fields) prisma = _spend_logs_prisma(inserted, existing) - assert await _update_database_with(db_writer, prisma, _batch_cost_payload()) is charged + assert await _update_database_with(db_writer, prisma, cost_payload, kwargs=cost_kwargs) is charged claimed_rows = prisma.db.litellm_spendlogs.create_many.await_args.kwargs assert claimed_rows["skip_duplicates"] is True - assert [(row["request_id"], row["spend"]) for row in claimed_rows["data"]] == [("batch_abc_batch_cost", 0.25)] + assert [(row["request_id"], row["spend"]) for row in claimed_rows["data"]] == [(cost_payload["request_id"], 0.25)] assert prisma.spend_log_transactions == [] assert db_writer._batch_database_updates.await_count == (1 if charged else 0) @pytest.mark.asyncio +@_EVERY_CHARGE_ONCE_ROW @pytest.mark.parametrize( ("taken_over", "charged"), [(1, True), (0, False)], - ids=["this_retrieve_takes_it_over", "another_one_got_there_first"], + ids=["this_read_takes_it_over", "another_one_got_there_first"], ) -async def test_update_database_charges_a_batch_whose_row_a_pre_upgrade_poll_left_at_zero( - taken_over: int, charged: bool +async def test_update_database_charges_a_cost_row_left_at_zero( + cost_payload: dict, cost_kwargs: Mapping[str, object], taken_over: int, charged: bool ): """ - A proxy without this fix wrote the batch's row at $0 on every poll of a running batch, + A proxy without the batch fix wrote the batch's row at $0 on every poll of a running batch, and the row outlives the upgrade, so the charge has to land on the row itself. Charging - without writing it there would charge again on every later retrieve (LIT-7048). + without writing it there would charge again on every later read (LIT-7048). """ db_writer = DBSpendUpdateWriter() db_writer._batch_database_updates = AsyncMock() - existing = SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.0) - prisma = _spend_logs_prisma(0, existing, taken_over) + prisma = _spend_logs_prisma(0, _existing_row(cost_payload, spend=0.0), taken_over) - assert await _update_database_with(db_writer, prisma, _batch_cost_payload()) is charged + assert await _update_database_with(db_writer, prisma, cost_payload, kwargs=cost_kwargs) is charged taken = prisma.db.litellm_spendlogs.update_many.await_args.kwargs assert taken["where"] == { - "request_id": "batch_abc_batch_cost", - "call_type": "aretrieve_batch", + "request_id": cost_payload["request_id"], + "call_type": cost_payload["call_type"], "status": "success", "spend": 0.0, } @@ -4494,26 +4535,24 @@ async def test_update_database_leaves_a_batch_that_cost_nothing_to_the_retrieve_ @pytest.mark.asyncio +@_EVERY_CHARGE_ONCE_ROW @pytest.mark.parametrize( - ("inserted", "existing", "charged"), - [ - (1, None, True), - (0, SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.25), False), - ], - ids=["first_retrieve_owns_the_row", "another_retrieve_already_charged"], + ("inserted", "already_charged", "charged"), + [(1, False, True), (0, True, False)], + ids=["first_read_owns_the_row", "another_read_already_charged"], ) -async def test_update_database_charges_a_batch_once_even_with_spend_logs_disabled( - inserted: int, existing: object, charged: bool +async def test_update_database_charges_a_cost_row_once_even_with_spend_logs_disabled( + cost_payload: dict, cost_kwargs: Mapping[str, object], inserted: int, already_charged: bool, charged: bool ): """ - disable_spend_logs drops the per-request logs, not the batch's charge, so the one row - that makes a batch chargeable exactly once is still written and still read back. + disable_spend_logs drops the per-request logs, not the charge, so the one row that makes + a batch or a background response chargeable exactly once is still written and read back. """ db_writer = DBSpendUpdateWriter() db_writer._batch_database_updates = AsyncMock() - prisma = _spend_logs_prisma(inserted, existing) + prisma = _spend_logs_prisma(inserted, _existing_row(cost_payload, spend=0.25) if already_charged else None) - assert await _update_database_with(db_writer, prisma, _batch_cost_payload(), True) is charged + assert await _update_database_with(db_writer, prisma, cost_payload, True, kwargs=cost_kwargs) is charged assert prisma.db.litellm_spendlogs.create_many.await_count == 1 assert db_writer._batch_database_updates.await_count == (1 if charged else 0) @@ -4606,16 +4645,30 @@ async def test_update_database_queues_a_batch_cost_row_it_could_not_claim(): @pytest.mark.asyncio @pytest.mark.parametrize( - "payload", - [{**_batch_cost_payload(), "call_type": "acompletion"}, {**_batch_cost_payload(), "status": "failure"}], - ids=["not_a_batch_retrieve", "failed_batch_retrieve"], + ("payload", "kwargs"), + [ + ({**_batch_cost_payload(), "call_type": "acompletion"}, _BATCH_RETRIEVE_KWARGS), + ({**_batch_cost_payload(), "status": "failure"}, _BATCH_RETRIEVE_KWARGS), + (_background_response_cost_payload(), _PASSTHROUGH_CALL_KWARGS), + (_background_response_cost_payload(), None), + ({**_background_response_cost_payload(), "status": "failure"}, _BACKGROUND_SETTLEMENT_KWARGS), + ], + ids=[ + "not_a_batch_retrieve", + "failed_batch_retrieve", + "passthrough_call_that_settles_nothing", + "call_without_kwargs", + "failed_background_response_read", + ], ) -async def test_update_database_queues_every_other_spend_row_for_the_next_flush(payload: dict): +async def test_update_database_queues_every_other_spend_row_for_the_next_flush( + payload: dict, kwargs: Mapping[str, object] | None +): db_writer = DBSpendUpdateWriter() db_writer._batch_database_updates = AsyncMock() prisma = _spend_logs_prisma(1, None) - assert await _update_database_with(db_writer, prisma, payload) is True + assert await _update_database_with(db_writer, prisma, payload, kwargs=kwargs) is True prisma.db.litellm_spendlogs.create_many.assert_not_called() assert prisma.spend_log_transactions == [payload]