mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(passthrough): bill a background response once however many times it is read
This commit is contained in:
parent
dff474e450
commit
c04506383e
3 changed files with 337 additions and 86 deletions
|
|
@ -90,6 +90,10 @@ from litellm.proxy.spend_tracking.savings import (
|
|||
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
|
||||
from litellm.repositories.prisma_protocols import BatchTable
|
||||
from litellm.responses.streaming_iterator import spawn_logging_task
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
SETTLES_BACKGROUND_RESPONSE_KWARG,
|
||||
BackgroundResponseSettlement,
|
||||
)
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -157,23 +161,35 @@ def _org_member_transaction_key(org_id: str, user_id: str) -> str:
|
|||
return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}"
|
||||
|
||||
|
||||
def _is_batch_cost_row(payload: SpendLogsPayload) -> bool:
|
||||
return payload.get("call_type") == CallTypes.aretrieve_batch.value and payload.get("status") == "success"
|
||||
class _CostRowKwargs(TypedDict, total=False):
|
||||
settles_background_response: ReadOnly[object]
|
||||
|
||||
|
||||
_BATCH_COST_CLAIM_FIELDS: Final = frozenset({"request_id", "call_type", "spend", "startTime", "endTime", "status"})
|
||||
_COST_ROW_KWARGS: Final = TypeAdapter(_CostRowKwargs)
|
||||
|
||||
|
||||
def _batch_cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool) -> Mapping[str, object]:
|
||||
"""Reduce a batch's cost row to what tells the retrieves apart when logging is off.
|
||||
def _is_charge_once_cost_row(payload: SpendLogsPayload, kwargs: _CostRowKwargs) -> bool:
|
||||
if payload.get("status") != "success":
|
||||
return False
|
||||
return (
|
||||
payload.get("call_type") == CallTypes.aretrieve_batch.value
|
||||
or kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES
|
||||
)
|
||||
|
||||
A proxy run with spend logs disabled still needs one row per batch to charge it once,
|
||||
so the row is written either way, but it carries no request of its own: no metadata,
|
||||
no requester IP, no key, model, or token counts (LIT-7048).
|
||||
|
||||
_COST_ROW_CLAIM_FIELDS: Final = frozenset({"request_id", "call_type", "spend", "startTime", "endTime", "status"})
|
||||
|
||||
|
||||
def _cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool) -> Mapping[str, object]:
|
||||
"""Reduce a cost row to what tells the reads apart when logging is off.
|
||||
|
||||
A proxy run with spend logs disabled still needs one row per batch or background response
|
||||
to charge it once, so the row is written either way, but it carries no request of its own:
|
||||
no metadata, no requester IP, no key, model, or token counts (LIT-7048).
|
||||
"""
|
||||
if disable_spend_logs is False:
|
||||
return payload
|
||||
return MappingProxyType({field: value for field, value in payload.items() if field in _BATCH_COST_CLAIM_FIELDS})
|
||||
return MappingProxyType({field: value for field, value in payload.items() if field in _COST_ROW_CLAIM_FIELDS})
|
||||
|
||||
|
||||
class _SpendIncrement(TypedDict):
|
||||
|
|
@ -608,8 +624,9 @@ class DBSpendUpdateWriter:
|
|||
) -> bool:
|
||||
"""Record the request's spend, answering whether its cost still needs charging.
|
||||
|
||||
False only for a batch retrieve whose cost row another retrieve already wrote,
|
||||
so the caller leaves the key, team, and user counters alone (LIT-7048).
|
||||
False only for a batch retrieve or a background response read whose cost row an
|
||||
earlier read already charged, so the caller leaves the key, team, and user counters
|
||||
alone (LIT-7048).
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
disable_spend_logs,
|
||||
|
|
@ -663,6 +680,7 @@ class DBSpendUpdateWriter:
|
|||
payload=payload,
|
||||
prisma_client=prisma_client,
|
||||
disable_spend_logs=disable_spend_logs,
|
||||
charge_once=_is_charge_once_cost_row(payload, _COST_ROW_KWARGS.validate_python(kwargs or {})),
|
||||
llm_router=llm_router,
|
||||
):
|
||||
return False
|
||||
|
|
@ -732,26 +750,28 @@ class DBSpendUpdateWriter:
|
|||
payload: SpendLogsPayload,
|
||||
prisma_client: "PrismaClient | None",
|
||||
disable_spend_logs: bool,
|
||||
charge_once: bool,
|
||||
llm_router: "Router | None" = None,
|
||||
) -> bool:
|
||||
if prisma_client is not None and _is_batch_cost_row(payload):
|
||||
return await self._claim_batch_cost_spend_log(
|
||||
if prisma_client is not None and charge_once:
|
||||
return await self._claim_cost_spend_log(
|
||||
payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs
|
||||
)
|
||||
if disable_spend_logs is False:
|
||||
await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client, llm_router=llm_router)
|
||||
return True
|
||||
|
||||
async def _claim_batch_cost_spend_log(
|
||||
async def _claim_cost_spend_log(
|
||||
self, payload: SpendLogsPayload, prisma_client: "PrismaClient", disable_spend_logs: bool
|
||||
) -> bool:
|
||||
"""Write the batch's cost row now, or learn that another retrieve already did.
|
||||
"""Write the cost row now, or learn that another read already did.
|
||||
|
||||
Every retrieve of one batch shares this row, so the insert that lands first owns
|
||||
the charge and every later one finds the row and charges nothing (LIT-7048). Only
|
||||
a row that recorded a charge counts: a failed retrieve, a request whose client
|
||||
picked the batch id as its call id, and the $0 row an older proxy left behind
|
||||
while the batch was still running all leave the charge to be made.
|
||||
Every retrieve of one batch, and every read of one finished background response,
|
||||
shares this row, so the insert that lands first owns the charge and every later one
|
||||
finds the row and charges nothing (LIT-7048). Only a row of the same call type that
|
||||
recorded a charge counts: a failed read, a request whose client picked the row's id
|
||||
as its call id, and the $0 row an older proxy left behind while a batch was still
|
||||
running all leave the charge to be made.
|
||||
"""
|
||||
from litellm.repositories.table_repositories import SpendLogsRepository
|
||||
|
||||
|
|
@ -762,7 +782,7 @@ class DBSpendUpdateWriter:
|
|||
)
|
||||
|
||||
row: Final = spend_log_row_with_retained_metadata(
|
||||
_batch_cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields()
|
||||
_cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields()
|
||||
)
|
||||
spend_logs: Final = SpendLogsRepository(prisma_client).table
|
||||
try:
|
||||
|
|
@ -774,40 +794,36 @@ class DBSpendUpdateWriter:
|
|||
return True
|
||||
existing: Final = await spend_logs.find_unique(where={"request_id": request_id})
|
||||
except Exception as e: # noqa: BLE001 # prisma raises its own hierarchy; an unreachable DB queues the row like any other spend log
|
||||
verbose_proxy_logger.warning(
|
||||
"Could not claim spend row %s for a batch's cost, queueing it: %s", request_id, e
|
||||
)
|
||||
verbose_proxy_logger.warning("Could not claim cost spend row %s, queueing it: %s", request_id, e)
|
||||
await self._insert_spend_log_to_db(payload=prisma_client.jsonify_object(row), prisma_client=prisma_client)
|
||||
return True
|
||||
if existing is None or existing.call_type != CallTypes.aretrieve_batch.value or existing.status != "success":
|
||||
if existing is None or existing.call_type != payload["call_type"] or existing.status != "success":
|
||||
verbose_proxy_logger.warning(
|
||||
"Spend row %s belongs to a %s request, so this batch's cost is charged without a row of its own",
|
||||
"Spend row %s belongs to a %s request, so this cost is charged without a row of its own",
|
||||
request_id,
|
||||
getattr(existing, "call_type", None),
|
||||
)
|
||||
return True
|
||||
if existing.spend > 0:
|
||||
verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this batch", request_id)
|
||||
verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this cost", request_id)
|
||||
return False
|
||||
return await self._take_over_uncharged_batch_cost_row(payload=payload, prisma_client=prisma_client, row=row)
|
||||
return await self._take_over_uncharged_cost_row(payload=payload, prisma_client=prisma_client, row=row)
|
||||
|
||||
async def _take_over_uncharged_batch_cost_row(
|
||||
async def _take_over_uncharged_cost_row(
|
||||
self, payload: SpendLogsPayload, prisma_client: "PrismaClient", row: Mapping[str, object]
|
||||
) -> bool:
|
||||
"""Take the batch's cost row over from the poll that left it charging nothing.
|
||||
"""Take the cost row over from the read that left it charging nothing.
|
||||
|
||||
A pre-upgrade proxy wrote that row every time it polled the batch while it was still
|
||||
A pre-upgrade proxy wrote a batch's row every time it polled the batch while it was still
|
||||
running, so the charge is still to be made and the row still has to end up carrying
|
||||
it. The row stops matching the moment it carries a charge, so it is one retrieve that
|
||||
it. The row stops matching the moment it carries a charge, so it is one read that
|
||||
takes it over and charges, and every later one reads the charge and charges nothing.
|
||||
"""
|
||||
from litellm.repositories.table_repositories import SpendLogsRepository
|
||||
|
||||
request_id: Final = payload["request_id"]
|
||||
if payload["spend"] <= 0:
|
||||
verbose_proxy_logger.debug(
|
||||
"Cost tracking skipped: this batch costs nothing and spend row %s says so", request_id
|
||||
)
|
||||
verbose_proxy_logger.debug("Cost tracking skipped: this costs nothing and spend row %s says so", request_id)
|
||||
return False
|
||||
try:
|
||||
taken_over: Final = await SpendLogsRepository(prisma_client).table.update_many(
|
||||
|
|
@ -816,18 +832,18 @@ class DBSpendUpdateWriter:
|
|||
),
|
||||
where={
|
||||
"request_id": request_id,
|
||||
"call_type": CallTypes.aretrieve_batch.value,
|
||||
"call_type": payload["call_type"],
|
||||
"status": "success",
|
||||
"spend": 0.0,
|
||||
},
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # prisma raises its own hierarchy; the next retrieve takes the row over
|
||||
verbose_proxy_logger.warning(
|
||||
"Could not take over spend row %s, leaving this batch's cost to the next retrieve: %s", request_id, e
|
||||
"Could not take over spend row %s, leaving this cost to the next read: %s", request_id, e
|
||||
)
|
||||
return False
|
||||
if taken_over == 0:
|
||||
verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this batch", request_id)
|
||||
verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this cost", request_id)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,182 @@
|
|||
import json
|
||||
import math
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import pytest
|
||||
from integration._support.client import (
|
||||
Gateway,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
held,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.forward_proxy import tunnelling_forward_proxy
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.tls import server_context, write_self_signed_cert
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 180)
|
||||
|
||||
_MODEL: Final = "gpt-5.6"
|
||||
_INPUT_TOKENS: Final = 1200
|
||||
_OUTPUT_TOKENS: Final = 300
|
||||
_READS: Final = 3
|
||||
_SPEND_LANDING_SECONDS: Final = 70
|
||||
_LATE_CHARGE_WINDOW_SECONDS: Final = 10
|
||||
_COST_MAP: Final = Path("litellm/model_prices_and_context_window_backup.json")
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _response(response_id: str, status: str) -> Mapping[str, JsonValue]:
|
||||
finished: Final = status == "completed"
|
||||
return {
|
||||
"id": response_id,
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": status,
|
||||
"background": True,
|
||||
"error": None,
|
||||
"incomplete_details": None,
|
||||
"model": _MODEL,
|
||||
"output": [
|
||||
{
|
||||
"id": f"msg_{uuid.uuid4().hex}",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "pong", "annotations": []}],
|
||||
}
|
||||
]
|
||||
if finished
|
||||
else [],
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"usage": {
|
||||
"input_tokens": _INPUT_TOKENS,
|
||||
"output_tokens": _OUTPUT_TOKENS,
|
||||
"total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS,
|
||||
"input_tokens_details": {"cached_tokens": 0},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
}
|
||||
if finished
|
||||
else None,
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
|
||||
def _openai(request: Request) -> Reply:
|
||||
path: Final = urlsplit(request.target).path
|
||||
if request.method == "POST" and path == "/v1/responses":
|
||||
return Reply(body=json.dumps(_response(f"resp_{uuid.uuid4().hex}", "queued")).encode())
|
||||
return Reply(body=json.dumps(_response(path.rsplit("/", 1)[-1], "completed")).encode())
|
||||
|
||||
|
||||
def _port(url: str) -> int:
|
||||
port: Final = urlsplit(url).port
|
||||
assert port is not None, url
|
||||
return port
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("passthrough-background-response")
|
||||
certificate: Final = write_self_signed_cert(directory, ("api.openai.com",))
|
||||
with (
|
||||
wire_server(_openai, tls=server_context(*certificate)) as upstream,
|
||||
tunnelling_forward_proxy(_port(upstream.url)) as tunnel,
|
||||
gateway_from_environment() as base,
|
||||
owned_proxy(
|
||||
base,
|
||||
directory,
|
||||
{
|
||||
"HTTPS_PROXY": tunnel.url,
|
||||
"NO_PROXY": "127.0.0.1,localhost",
|
||||
"AIOHTTP_TRUST_ENV": "True",
|
||||
"SSL_VERIFY": "False",
|
||||
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
|
||||
"OPENAI_API_KEY": "sk-scripted-openai",
|
||||
},
|
||||
workers=2,
|
||||
) as owned,
|
||||
):
|
||||
yield owned
|
||||
|
||||
|
||||
def _price() -> float:
|
||||
row: Final = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(_COST_MAP.read_bytes())[_MODEL])
|
||||
input_rate: Final = row["input_cost_per_token"]
|
||||
output_rate: Final = row["output_cost_per_token"]
|
||||
assert isinstance(input_rate, float) and isinstance(output_rate, float), row
|
||||
return _INPUT_TOKENS * input_rate + _OUTPUT_TOKENS * output_rate
|
||||
|
||||
|
||||
def _passthrough(proxy: Gateway, method: str, path: str, key: str, body: Mapping[str, JsonValue] | None = None) -> str:
|
||||
response: Final = proxy.request(method, f"/openai_passthrough/v1/responses{path}", body, key=key)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = object_value(response.json())
|
||||
return string_value(payload["id"]) if method == "POST" else string_value(payload["status"])
|
||||
|
||||
|
||||
def _number(value: JsonValue) -> float:
|
||||
assert isinstance(value, int | float), value
|
||||
return float(value)
|
||||
|
||||
|
||||
def _spend_of(proxy: Gateway, table: str, column: str, identity: str) -> float:
|
||||
return _number(proxy.rows(f'SELECT spend FROM "{table}" WHERE {column} = %s', (identity,))[0]["spend"])
|
||||
|
||||
|
||||
def _key_spend(proxy: Gateway, key: str) -> float:
|
||||
return _number(object_value(proxy.get("/key/info", {"key": key})["info"])["spend"])
|
||||
|
||||
|
||||
def _logged_spend(proxy: Gateway, key: str) -> float:
|
||||
rows: Final = proxy.rows(
|
||||
'SELECT spend FROM "LiteLLM_SpendLogs" WHERE api_key = %s', (sha256(key.encode()).hexdigest(),)
|
||||
)
|
||||
return math.fsum(_number(row["spend"]) for row in rows)
|
||||
|
||||
|
||||
def test_a_background_response_read_through_the_openai_passthrough_bills_its_key_team_user_and_org_once(
|
||||
proxy: Gateway,
|
||||
) -> None:
|
||||
price: Final = _price()
|
||||
with proxy.scenario() as scenario:
|
||||
organization_id: Final = scenario.organization()
|
||||
team_id: Final = scenario.team(organization_id=organization_id)
|
||||
user_id: Final = scenario.org_member(organization_id, "internal_user")
|
||||
proxy.post("/team/member_add", {"team_id": team_id, "member": {"role": "user", "user_id": user_id}})
|
||||
key: Final = scenario.key(team_id=team_id, user_id=user_id)
|
||||
response_id: Final = _passthrough(
|
||||
proxy, "POST", "", key, {"model": _MODEL, "input": "bill me once", "background": True}
|
||||
)
|
||||
reads: Final = tuple(_passthrough(proxy, "GET", f"/{response_id}", key) for _ in range(_READS))
|
||||
assert reads == ("completed",) * _READS
|
||||
|
||||
def spent() -> tuple[float, ...]:
|
||||
return (
|
||||
_key_spend(proxy, key),
|
||||
_spend_of(proxy, "LiteLLM_TeamTable", "team_id", team_id),
|
||||
_spend_of(proxy, "LiteLLM_UserTable", "user_id", user_id),
|
||||
_spend_of(proxy, "LiteLLM_OrganizationTable", "organization_id", organization_id),
|
||||
)
|
||||
|
||||
held(
|
||||
spent,
|
||||
lambda observed: all(math.isclose(value, price, rel_tol=1e-9) for value in observed),
|
||||
holding=_LATE_CHARGE_WINDOW_SECONDS,
|
||||
seconds=_SPEND_LANDING_SECONDS,
|
||||
)
|
||||
eventually(
|
||||
lambda: (_logged_spend(proxy, key), _key_spend(proxy, key)),
|
||||
lambda observed: math.isclose(observed[0], observed[1], rel_tol=1e-9),
|
||||
seconds=_SPEND_LANDING_SECONDS,
|
||||
)
|
||||
|
|
@ -3,10 +3,10 @@ import copy
|
|||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||||
|
||||
|
|
@ -37,6 +37,10 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
|
|||
from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL
|
||||
from litellm.responses.streaming_iterator import drain_logging_tasks
|
||||
from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
SETTLES_BACKGROUND_RESPONSE_KWARG,
|
||||
BackgroundResponseSettlement,
|
||||
)
|
||||
from tests.unit.proxy.db.fake_prisma_engine import engine_call
|
||||
|
||||
|
||||
|
|
@ -4334,6 +4338,36 @@ def _batch_cost_payload() -> dict:
|
|||
}
|
||||
|
||||
|
||||
def _background_response_cost_payload() -> dict:
|
||||
return {
|
||||
**_minimal_spend_payload(),
|
||||
"request_id": "resp_abc_background_cost",
|
||||
"call_type": "pass_through_endpoint",
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
|
||||
_BATCH_RETRIEVE_KWARGS: Final = MappingProxyType({"model": "gpt-5.6-luna", "call_type": "aretrieve_batch"})
|
||||
_PASSTHROUGH_CALL_KWARGS: Final = MappingProxyType(
|
||||
{"model": "gpt-5.6", "call_type": "pass_through_endpoint", SETTLES_BACKGROUND_RESPONSE_KWARG: None}
|
||||
)
|
||||
_BACKGROUND_SETTLEMENT_KWARGS: Final = MappingProxyType(
|
||||
{**_PASSTHROUGH_CALL_KWARGS, SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES}
|
||||
)
|
||||
_EVERY_CHARGE_ONCE_ROW: Final = pytest.mark.parametrize(
|
||||
("cost_payload", "cost_kwargs"),
|
||||
[
|
||||
(_batch_cost_payload(), _BATCH_RETRIEVE_KWARGS),
|
||||
(_background_response_cost_payload(), _BACKGROUND_SETTLEMENT_KWARGS),
|
||||
],
|
||||
ids=["batch_retrieve", "finished_background_response_read"],
|
||||
)
|
||||
|
||||
|
||||
def _existing_row(cost_payload: dict, **fields: object) -> SimpleNamespace:
|
||||
return SimpleNamespace(**{"call_type": cost_payload["call_type"], "status": "success", **fields})
|
||||
|
||||
|
||||
def _spend_logs_prisma(inserted: int, existing: object, taken_over: int = 1) -> MagicMock:
|
||||
prisma = _tool_usage_prisma()
|
||||
prisma.jsonify_object = lambda data: dict(data)
|
||||
|
|
@ -4349,6 +4383,7 @@ async def _update_database_with(
|
|||
payload: dict,
|
||||
disable_spend_logs: bool = False,
|
||||
response_cost: float = 0.25,
|
||||
kwargs: Mapping[str, object] | None = _BATCH_RETRIEVE_KWARGS,
|
||||
) -> bool:
|
||||
with (
|
||||
patch( # test-quality-ok: update_database reads this proxy_server global at call time, no seam
|
||||
|
|
@ -4371,7 +4406,7 @@ async def _update_database_with(
|
|||
end_user_id=None,
|
||||
team_id=None,
|
||||
org_id=None,
|
||||
kwargs={"model": "gpt-5.6-luna", "call_type": "aretrieve_batch"},
|
||||
kwargs=None if kwargs is None else dict(kwargs),
|
||||
completion_response=None,
|
||||
start_time=datetime.now(timezone.utc),
|
||||
end_time=datetime.now(timezone.utc),
|
||||
|
|
@ -4382,73 +4417,79 @@ async def _update_database_with(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_EVERY_CHARGE_ONCE_ROW
|
||||
@pytest.mark.parametrize(
|
||||
("inserted", "existing", "charged"),
|
||||
("inserted", "existing_fields", "charged"),
|
||||
[
|
||||
(1, None, True),
|
||||
(0, SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.25), False),
|
||||
(0, SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.0), True),
|
||||
(0, SimpleNamespace(call_type="aretrieve_batch", status="failure", spend=0.0), True),
|
||||
(0, SimpleNamespace(call_type="aembedding", status="success", spend=0.25), True),
|
||||
(0, {"spend": 0.25}, False),
|
||||
(0, {"spend": 0.0}, True),
|
||||
(0, {"status": "failure", "spend": 0.0}, True),
|
||||
(0, {"call_type": "aembedding", "spend": 0.25}, True),
|
||||
(0, None, True),
|
||||
],
|
||||
ids=[
|
||||
"first_retrieve_owns_the_row",
|
||||
"another_retrieve_already_charged",
|
||||
"an_older_proxy_left_a_zero_row_while_the_batch_ran",
|
||||
"failed_retrieve_holds_the_row",
|
||||
"first_read_owns_the_row",
|
||||
"another_read_already_charged",
|
||||
"an_older_proxy_left_a_zero_row",
|
||||
"failed_read_holds_the_row",
|
||||
"client_chosen_call_id_holds_the_row",
|
||||
"row_gone_between_insert_and_lookup",
|
||||
],
|
||||
)
|
||||
async def test_update_database_charges_a_batch_only_from_the_retrieve_that_wrote_its_row(
|
||||
inserted: int, existing: object, charged: bool
|
||||
async def test_update_database_charges_a_cost_row_only_from_the_read_that_wrote_it(
|
||||
cost_payload: dict,
|
||||
cost_kwargs: Mapping[str, object],
|
||||
inserted: int,
|
||||
existing_fields: dict | None,
|
||||
charged: bool,
|
||||
):
|
||||
"""
|
||||
Every retrieve of one batch shares one spend row, so the insert that lands first is
|
||||
the charge and every later retrieve must leave the counters alone (LIT-7048). A row
|
||||
that recorded no charge must not be able to take the charge away: neither one a
|
||||
client planted under the batch id, nor the $0 row a pre-upgrade proxy wrote every
|
||||
time it polled the batch while it was still running.
|
||||
Every retrieve of one batch, and every read of one finished background response, shares
|
||||
one spend row, so the insert that lands first is the charge and every later read must
|
||||
leave the counters alone (LIT-7048). A row that recorded no charge must not be able to
|
||||
take the charge away: neither one a client planted under the row's id, nor the $0 row a
|
||||
pre-upgrade proxy wrote every time it polled a batch while it was still running.
|
||||
"""
|
||||
db_writer = DBSpendUpdateWriter()
|
||||
db_writer._batch_database_updates = AsyncMock()
|
||||
existing = None if existing_fields is None else _existing_row(cost_payload, **existing_fields)
|
||||
prisma = _spend_logs_prisma(inserted, existing)
|
||||
|
||||
assert await _update_database_with(db_writer, prisma, _batch_cost_payload()) is charged
|
||||
assert await _update_database_with(db_writer, prisma, cost_payload, kwargs=cost_kwargs) is charged
|
||||
|
||||
claimed_rows = prisma.db.litellm_spendlogs.create_many.await_args.kwargs
|
||||
assert claimed_rows["skip_duplicates"] is True
|
||||
assert [(row["request_id"], row["spend"]) for row in claimed_rows["data"]] == [("batch_abc_batch_cost", 0.25)]
|
||||
assert [(row["request_id"], row["spend"]) for row in claimed_rows["data"]] == [(cost_payload["request_id"], 0.25)]
|
||||
assert prisma.spend_log_transactions == []
|
||||
assert db_writer._batch_database_updates.await_count == (1 if charged else 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_EVERY_CHARGE_ONCE_ROW
|
||||
@pytest.mark.parametrize(
|
||||
("taken_over", "charged"),
|
||||
[(1, True), (0, False)],
|
||||
ids=["this_retrieve_takes_it_over", "another_one_got_there_first"],
|
||||
ids=["this_read_takes_it_over", "another_one_got_there_first"],
|
||||
)
|
||||
async def test_update_database_charges_a_batch_whose_row_a_pre_upgrade_poll_left_at_zero(
|
||||
taken_over: int, charged: bool
|
||||
async def test_update_database_charges_a_cost_row_left_at_zero(
|
||||
cost_payload: dict, cost_kwargs: Mapping[str, object], taken_over: int, charged: bool
|
||||
):
|
||||
"""
|
||||
A proxy without this fix wrote the batch's row at $0 on every poll of a running batch,
|
||||
A proxy without the batch fix wrote the batch's row at $0 on every poll of a running batch,
|
||||
and the row outlives the upgrade, so the charge has to land on the row itself. Charging
|
||||
without writing it there would charge again on every later retrieve (LIT-7048).
|
||||
without writing it there would charge again on every later read (LIT-7048).
|
||||
"""
|
||||
db_writer = DBSpendUpdateWriter()
|
||||
db_writer._batch_database_updates = AsyncMock()
|
||||
existing = SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.0)
|
||||
prisma = _spend_logs_prisma(0, existing, taken_over)
|
||||
prisma = _spend_logs_prisma(0, _existing_row(cost_payload, spend=0.0), taken_over)
|
||||
|
||||
assert await _update_database_with(db_writer, prisma, _batch_cost_payload()) is charged
|
||||
assert await _update_database_with(db_writer, prisma, cost_payload, kwargs=cost_kwargs) is charged
|
||||
|
||||
taken = prisma.db.litellm_spendlogs.update_many.await_args.kwargs
|
||||
assert taken["where"] == {
|
||||
"request_id": "batch_abc_batch_cost",
|
||||
"call_type": "aretrieve_batch",
|
||||
"request_id": cost_payload["request_id"],
|
||||
"call_type": cost_payload["call_type"],
|
||||
"status": "success",
|
||||
"spend": 0.0,
|
||||
}
|
||||
|
|
@ -4494,26 +4535,24 @@ async def test_update_database_leaves_a_batch_that_cost_nothing_to_the_retrieve_
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_EVERY_CHARGE_ONCE_ROW
|
||||
@pytest.mark.parametrize(
|
||||
("inserted", "existing", "charged"),
|
||||
[
|
||||
(1, None, True),
|
||||
(0, SimpleNamespace(call_type="aretrieve_batch", status="success", spend=0.25), False),
|
||||
],
|
||||
ids=["first_retrieve_owns_the_row", "another_retrieve_already_charged"],
|
||||
("inserted", "already_charged", "charged"),
|
||||
[(1, False, True), (0, True, False)],
|
||||
ids=["first_read_owns_the_row", "another_read_already_charged"],
|
||||
)
|
||||
async def test_update_database_charges_a_batch_once_even_with_spend_logs_disabled(
|
||||
inserted: int, existing: object, charged: bool
|
||||
async def test_update_database_charges_a_cost_row_once_even_with_spend_logs_disabled(
|
||||
cost_payload: dict, cost_kwargs: Mapping[str, object], inserted: int, already_charged: bool, charged: bool
|
||||
):
|
||||
"""
|
||||
disable_spend_logs drops the per-request logs, not the batch's charge, so the one row
|
||||
that makes a batch chargeable exactly once is still written and still read back.
|
||||
disable_spend_logs drops the per-request logs, not the charge, so the one row that makes
|
||||
a batch or a background response chargeable exactly once is still written and read back.
|
||||
"""
|
||||
db_writer = DBSpendUpdateWriter()
|
||||
db_writer._batch_database_updates = AsyncMock()
|
||||
prisma = _spend_logs_prisma(inserted, existing)
|
||||
prisma = _spend_logs_prisma(inserted, _existing_row(cost_payload, spend=0.25) if already_charged else None)
|
||||
|
||||
assert await _update_database_with(db_writer, prisma, _batch_cost_payload(), True) is charged
|
||||
assert await _update_database_with(db_writer, prisma, cost_payload, True, kwargs=cost_kwargs) is charged
|
||||
|
||||
assert prisma.db.litellm_spendlogs.create_many.await_count == 1
|
||||
assert db_writer._batch_database_updates.await_count == (1 if charged else 0)
|
||||
|
|
@ -4606,16 +4645,30 @@ async def test_update_database_queues_a_batch_cost_row_it_could_not_claim():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[{**_batch_cost_payload(), "call_type": "acompletion"}, {**_batch_cost_payload(), "status": "failure"}],
|
||||
ids=["not_a_batch_retrieve", "failed_batch_retrieve"],
|
||||
("payload", "kwargs"),
|
||||
[
|
||||
({**_batch_cost_payload(), "call_type": "acompletion"}, _BATCH_RETRIEVE_KWARGS),
|
||||
({**_batch_cost_payload(), "status": "failure"}, _BATCH_RETRIEVE_KWARGS),
|
||||
(_background_response_cost_payload(), _PASSTHROUGH_CALL_KWARGS),
|
||||
(_background_response_cost_payload(), None),
|
||||
({**_background_response_cost_payload(), "status": "failure"}, _BACKGROUND_SETTLEMENT_KWARGS),
|
||||
],
|
||||
ids=[
|
||||
"not_a_batch_retrieve",
|
||||
"failed_batch_retrieve",
|
||||
"passthrough_call_that_settles_nothing",
|
||||
"call_without_kwargs",
|
||||
"failed_background_response_read",
|
||||
],
|
||||
)
|
||||
async def test_update_database_queues_every_other_spend_row_for_the_next_flush(payload: dict):
|
||||
async def test_update_database_queues_every_other_spend_row_for_the_next_flush(
|
||||
payload: dict, kwargs: Mapping[str, object] | None
|
||||
):
|
||||
db_writer = DBSpendUpdateWriter()
|
||||
db_writer._batch_database_updates = AsyncMock()
|
||||
prisma = _spend_logs_prisma(1, None)
|
||||
|
||||
assert await _update_database_with(db_writer, prisma, payload) is True
|
||||
assert await _update_database_with(db_writer, prisma, payload, kwargs=kwargs) is True
|
||||
|
||||
prisma.db.litellm_spendlogs.create_many.assert_not_called()
|
||||
assert prisma.spend_log_transactions == [payload]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue