mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
test(spend): move the /spend/logs end_date repro to tests/integration
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
0d5ccb6d5e
commit
d2ee416342
7 changed files with 72 additions and 134 deletions
|
|
@ -210,7 +210,6 @@ quota_management.<behavior>.<variant>.<assertion>
|
|||
<spend_tracking> chat_completions | stream | messages_bridge | embeddings
|
||||
| cache_hit | key_rollup | concurrent_burst | tags | end_user
|
||||
| per_model | failure | spend_calculate | pagination | key_attribution
|
||||
| date_range
|
||||
assertion : blocks_over_limit | resets_after_window | headers_report_remaining | picks_under_tpm
|
||||
| blocks_then_resets | resets_windows_independently | alerts_without_blocking
|
||||
| isolates_per_model | isolates_per_member | isolates_per_group | enforced_across_keys
|
||||
|
|
@ -218,7 +217,7 @@ quota_management.<behavior>.<variant>.<assertion>
|
|||
| matches_sum_of_logs | loses_no_spend | attributes_spend | writes_own_rows
|
||||
| writes_failure_row | returns_cost | keeps_total | joins_key | reports_alias_and_email
|
||||
| health_rows_keep_service_account | retrieve_batch_cost_joins_retrieving_key
|
||||
| poller_batch_cost_joins_creating_key | includes_end_date
|
||||
| poller_batch_cost_joins_creating_key
|
||||
e.g. quota_management.ratelimit.rpm.blocks_over_limit exercised_on=[chat_completions, messages]
|
||||
quota_management.budget.key.blocks_over_limit exercised_on=[chat_completions]
|
||||
```
|
||||
|
|
|
|||
|
|
@ -51,7 +51,6 @@
|
|||
- {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"}
|
||||
- {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"}
|
||||
- {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"}
|
||||
- {id: quota_management.spend_tracking.date_range.includes_end_date, module: quota_management, tier: P1, behavior: spend_tracking, variant: date_range, assertions: [includes_end_date], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "A /spend/logs start_date/end_date window treats the end date as a whole calendar day, so the final day's spend is reported, not dropped to a zero pad (#40216)"}
|
||||
- {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"}
|
||||
- {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"}
|
||||
- {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"}
|
||||
|
|
|
|||
|
|
@ -909,7 +909,6 @@ class SpendLogMetadata(BaseModel):
|
|||
|
||||
class SpendLogRow(BaseModel):
|
||||
request_id: str | None = None
|
||||
startTime: str | None = None
|
||||
api_key: str | None = None
|
||||
model: str | None = None
|
||||
spend: float | None = None
|
||||
|
|
@ -949,27 +948,6 @@ class SpendLogsParams(BaseModel):
|
|||
return self
|
||||
|
||||
|
||||
class SpendLogsDateRangeParams(BaseModel):
|
||||
"""Query for legacy /spend/logs with a YYYY-MM-DD window; api_key keeps the
|
||||
read bounded."""
|
||||
start_date: str
|
||||
end_date: str
|
||||
api_key: str
|
||||
summarize: bool | None = None
|
||||
|
||||
|
||||
class SpendDailySummaryRow(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
startTime: str
|
||||
spend: float
|
||||
users: dict[str, float]
|
||||
models: dict[str, float]
|
||||
|
||||
|
||||
class SpendDailySummary(RootModel[list[SpendDailySummaryRow]]):
|
||||
pass
|
||||
|
||||
|
||||
class SpendLogsPageParams(BaseModel):
|
||||
"""Query for /spend/logs/v2, which requires an explicit date window and
|
||||
serves pages of at most 100 rows."""
|
||||
|
|
|
|||
|
|
@ -87,11 +87,8 @@ from models import (
|
|||
RouterSettingsResponse,
|
||||
SearchToolCreateBody,
|
||||
SearchToolCreateResponse,
|
||||
SpendDailySummary,
|
||||
SpendDailySummaryRow,
|
||||
SpendLogRow,
|
||||
SpendLogs,
|
||||
SpendLogsDateRangeParams,
|
||||
SpendLogsPage,
|
||||
SpendLogsPageParams,
|
||||
SpendLogsParams,
|
||||
|
|
@ -1036,28 +1033,6 @@ class ProxyClient:
|
|||
case _:
|
||||
return []
|
||||
|
||||
def spend_logs_daily(self, params: SpendLogsDateRangeParams) -> list[SpendDailySummaryRow]:
|
||||
"""GET /spend/logs with a date window (summarize=true default): one row per day."""
|
||||
return unwrap(
|
||||
self.transport.get(
|
||||
"/spend/logs",
|
||||
headers=self.management_headers(),
|
||||
params=params,
|
||||
response_type=SpendDailySummary,
|
||||
)
|
||||
).root
|
||||
|
||||
def spend_logs_in_window(self, params: SpendLogsDateRangeParams) -> list[SpendLogRow]:
|
||||
"""GET /spend/logs with a date window and summarize=false: the individual rows."""
|
||||
return unwrap(
|
||||
self.transport.get(
|
||||
"/spend/logs",
|
||||
headers=self.management_headers(),
|
||||
params=params,
|
||||
response_type=SpendLogs,
|
||||
)
|
||||
).root
|
||||
|
||||
def spend_logs_window(self, *, start: datetime, end: datetime) -> list[SpendLogRow]:
|
||||
def fetch(page: int) -> SpendLogsPage:
|
||||
return unwrap(
|
||||
|
|
|
|||
|
|
@ -17,14 +17,13 @@ fails the test; a pricing or token-count drift does not.
|
|||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from datetime import date, timedelta
|
||||
from math import isclose
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_http import Success
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, SpendLogs, SpendLogsDateRangeParams, SpendLogsParams
|
||||
from models import LiteLLMParamsBody, SpendLogs, SpendLogsParams
|
||||
from spend_e2e_client import SpendClient, SpendLogRow, is_ok, unique_marker, unwrap
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
|
@ -510,88 +509,6 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date")
|
||||
def test_spend_logs_single_day_range_returns_that_days_spend(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
"""start_date == end_date on /spend/logs must return the spend logged on that
|
||||
day, both as the daily summary (default) and as the individual rows
|
||||
(summarize=false). The end date is a calendar day, so its rows after
|
||||
midnight belong to the range; the window is the calendar day of the
|
||||
stored spend row."""
|
||||
chat = unwrap(
|
||||
client.chat(
|
||||
scoped_key,
|
||||
"gemini-2.5-flash",
|
||||
f"one word {unique_marker()}",
|
||||
max_tokens=16,
|
||||
)
|
||||
)
|
||||
assert chat.id, f"chat response carried no id: {chat}"
|
||||
rows = client.poll_logs_for_key(
|
||||
scoped_key,
|
||||
predicate=lambda rs: any(
|
||||
r.request_id == chat.id and (r.spend or 0) > 0 for r in rs
|
||||
),
|
||||
)
|
||||
logged = next(r for r in rows if r.request_id == chat.id)
|
||||
assert logged.startTime, f"spend row carries no startTime: {logged}"
|
||||
day: Final = logged.startTime[:10]
|
||||
window = SpendLogsDateRangeParams(start_date=day, end_date=day, api_key=scoped_key)
|
||||
|
||||
summary = client.proxy.spend_logs_daily(window)
|
||||
assert [r.startTime for r in summary] == [
|
||||
day
|
||||
], f"single-day window must yield exactly that day: {summary}"
|
||||
assert summary[0].spend > 0, (
|
||||
f"the end day's spend was dropped from the window: {summary}"
|
||||
)
|
||||
|
||||
raw = client.proxy.spend_logs_in_window(window.model_copy(update={"summarize": False}))
|
||||
assert any(r.request_id == chat.id for r in raw), (
|
||||
f"summarize=false window missing request {chat.id}: "
|
||||
f"{_summarize(raw)}; polled rows {_summarize(rows)}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date")
|
||||
def test_spend_logs_multi_day_range_includes_final_day(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
"""A multi-day window ending on the calendar day of the stored spend row
|
||||
reports that day's real spend as its final row, never a zero pad."""
|
||||
chat = unwrap(
|
||||
client.chat(
|
||||
scoped_key,
|
||||
"gemini-2.5-flash",
|
||||
f"one word {unique_marker()}",
|
||||
max_tokens=16,
|
||||
)
|
||||
)
|
||||
assert chat.id, f"chat response carried no id: {chat}"
|
||||
rows = client.poll_logs_for_key(
|
||||
scoped_key,
|
||||
predicate=lambda rs: any(
|
||||
r.request_id == chat.id and (r.spend or 0) > 0 for r in rs
|
||||
),
|
||||
)
|
||||
logged = next(r for r in rows if r.request_id == chat.id)
|
||||
assert logged.startTime, f"spend row carries no startTime: {logged}"
|
||||
end: Final = date.fromisoformat(logged.startTime[:10])
|
||||
start = end - timedelta(days=2)
|
||||
|
||||
summary = client.proxy.spend_logs_daily(
|
||||
SpendLogsDateRangeParams(
|
||||
start_date=start.isoformat(), end_date=end.isoformat(), api_key=scoped_key
|
||||
)
|
||||
)
|
||||
assert summary, "window ending today returned no rows"
|
||||
assert summary[-1].startTime == end.isoformat(), (
|
||||
f"final row is not the end date: {summary}"
|
||||
)
|
||||
assert summary[-1].spend > 0, f"end day reported as zero-spend padding: {summary}"
|
||||
|
||||
|
||||
def test_spend_logs_endpoint_returns_spend(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -245,6 +245,12 @@
|
|||
"tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [
|
||||
"quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals"
|
||||
],
|
||||
"tests/integration/spend/test_spend_logs_date_window.py::test_single_day_window_returns_that_days_summary_and_rows": [
|
||||
"quota_management.spend_tracking.date_range.includes_end_date"
|
||||
],
|
||||
"tests/integration/spend/test_spend_logs_date_window.py::test_multi_day_window_ends_with_the_end_days_real_spend": [
|
||||
"quota_management.spend_tracking.date_range.includes_end_date"
|
||||
],
|
||||
"tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [
|
||||
"mgmt.key.update.project_detach_denied_to_restricted_actor"
|
||||
],
|
||||
|
|
|
|||
64
tests/integration/spend/test_spend_logs_date_window.py
Normal file
64
tests/integration/spend/test_spend_logs_date_window.py
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
import uuid
|
||||
from datetime import date, timedelta
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
|
||||
from tests.integration._support.client import Gateway, Scenario, eventually, object_value, string_value
|
||||
from tests.integration._support.database import read_rows
|
||||
|
||||
EXPECTED_SPEND: Final = 20 * 0.001 + 20 * 0.002
|
||||
|
||||
|
||||
def _logged_request(gateway: Gateway, scenario: Scenario) -> tuple[str, str, str]:
|
||||
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002, num_retries=0)
|
||||
key: Final = scenario.key(models=[model])
|
||||
body: Final = gateway.chat(model, key=key, text=uuid.uuid4().hex)
|
||||
request_id: Final = string_value(body["id"])
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT to_char("startTime", \'YYYY-MM-DD\') AS day, spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
|
||||
(request_id,),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert float(rows[0]["spend"]) == pytest.approx(EXPECTED_SPEND)
|
||||
return key, request_id, string_value(rows[0]["day"])
|
||||
|
||||
|
||||
def _spend(row: dict[str, JsonValue]) -> float:
|
||||
value: Final = row["spend"]
|
||||
assert isinstance(value, (int, float)), f"spend is not numeric: {row}"
|
||||
return float(value)
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date")
|
||||
def test_single_day_window_returns_that_days_summary_and_rows(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
key, request_id, day = _logged_request(gateway, scenario)
|
||||
window: Final = {"start_date": day, "end_date": day, "api_key": key}
|
||||
summary: Final = gateway.request("GET", "/spend/logs", params=window)
|
||||
assert summary.status_code == 200, summary.text
|
||||
days: Final = tuple(object_value(row) for row in summary.json())
|
||||
assert [row["startTime"] for row in days] == [day], summary.text
|
||||
assert _spend(days[0]) == pytest.approx(EXPECTED_SPEND), summary.text
|
||||
raw: Final = gateway.request("GET", "/spend/logs", params={**window, "summarize": "false"})
|
||||
assert raw.status_code == 200, raw.text
|
||||
assert [object_value(row)["request_id"] for row in raw.json()] == [request_id], raw.text
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date")
|
||||
def test_multi_day_window_ends_with_the_end_days_real_spend(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
key, _, day = _logged_request(gateway, scenario)
|
||||
start: Final = (date.fromisoformat(day) - timedelta(days=2)).isoformat()
|
||||
summary: Final = gateway.request(
|
||||
"GET", "/spend/logs", params={"start_date": start, "end_date": day, "api_key": key}
|
||||
)
|
||||
assert summary.status_code == 200, summary.text
|
||||
days: Final = tuple(object_value(row) for row in summary.json())
|
||||
assert days, summary.text
|
||||
assert days[-1]["startTime"] == day, summary.text
|
||||
assert _spend(days[-1]) == pytest.approx(EXPECTED_SPEND), summary.text
|
||||
Loading…
Add table
Reference in a new issue