test(spend): move the /spend/logs end_date repro to tests/integration

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-22 18:43:46 +00:00
parent 0d5ccb6d5e
commit d2ee416342
7 changed files with 72 additions and 134 deletions

View file

@ -210,7 +210,6 @@ quota_management.<behavior>.<variant>.<assertion>
<spend_tracking> chat_completions | stream | messages_bridge | embeddings
| cache_hit | key_rollup | concurrent_burst | tags | end_user
| per_model | failure | spend_calculate | pagination | key_attribution
| date_range
assertion : blocks_over_limit | resets_after_window | headers_report_remaining | picks_under_tpm
| blocks_then_resets | resets_windows_independently | alerts_without_blocking
| isolates_per_model | isolates_per_member | isolates_per_group | enforced_across_keys
@ -218,7 +217,7 @@ quota_management.<behavior>.<variant>.<assertion>
| matches_sum_of_logs | loses_no_spend | attributes_spend | writes_own_rows
| writes_failure_row | returns_cost | keeps_total | joins_key | reports_alias_and_email
| health_rows_keep_service_account | retrieve_batch_cost_joins_retrieving_key
| poller_batch_cost_joins_creating_key | includes_end_date
| poller_batch_cost_joins_creating_key
e.g. quota_management.ratelimit.rpm.blocks_over_limit exercised_on=[chat_completions, messages]
quota_management.budget.key.blocks_over_limit exercised_on=[chat_completions]
```

View file

@ -51,7 +51,6 @@
- {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"}
- {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"}
- {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"}
- {id: quota_management.spend_tracking.date_range.includes_end_date, module: quota_management, tier: P1, behavior: spend_tracking, variant: date_range, assertions: [includes_end_date], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "A /spend/logs start_date/end_date window treats the end date as a whole calendar day, so the final day's spend is reported, not dropped to a zero pad (#40216)"}
- {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"}
- {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"}
- {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"}

View file

@ -909,7 +909,6 @@ class SpendLogMetadata(BaseModel):
class SpendLogRow(BaseModel):
request_id: str | None = None
startTime: str | None = None
api_key: str | None = None
model: str | None = None
spend: float | None = None
@ -949,27 +948,6 @@ class SpendLogsParams(BaseModel):
return self
class SpendLogsDateRangeParams(BaseModel):
"""Query for legacy /spend/logs with a YYYY-MM-DD window; api_key keeps the
read bounded."""
start_date: str
end_date: str
api_key: str
summarize: bool | None = None
class SpendDailySummaryRow(BaseModel):
model_config = ConfigDict(extra="allow")
startTime: str
spend: float
users: dict[str, float]
models: dict[str, float]
class SpendDailySummary(RootModel[list[SpendDailySummaryRow]]):
pass
class SpendLogsPageParams(BaseModel):
"""Query for /spend/logs/v2, which requires an explicit date window and
serves pages of at most 100 rows."""

View file

@ -87,11 +87,8 @@ from models import (
RouterSettingsResponse,
SearchToolCreateBody,
SearchToolCreateResponse,
SpendDailySummary,
SpendDailySummaryRow,
SpendLogRow,
SpendLogs,
SpendLogsDateRangeParams,
SpendLogsPage,
SpendLogsPageParams,
SpendLogsParams,
@ -1036,28 +1033,6 @@ class ProxyClient:
case _:
return []
def spend_logs_daily(self, params: SpendLogsDateRangeParams) -> list[SpendDailySummaryRow]:
"""GET /spend/logs with a date window (summarize=true default): one row per day."""
return unwrap(
self.transport.get(
"/spend/logs",
headers=self.management_headers(),
params=params,
response_type=SpendDailySummary,
)
).root
def spend_logs_in_window(self, params: SpendLogsDateRangeParams) -> list[SpendLogRow]:
"""GET /spend/logs with a date window and summarize=false: the individual rows."""
return unwrap(
self.transport.get(
"/spend/logs",
headers=self.management_headers(),
params=params,
response_type=SpendLogs,
)
).root
def spend_logs_window(self, *, start: datetime, end: datetime) -> list[SpendLogRow]:
def fetch(page: int) -> SpendLogsPage:
return unwrap(

View file

@ -17,14 +17,13 @@ fails the test; a pricing or token-count drift does not.
import time
from collections.abc import Callable
from datetime import date, timedelta
from math import isclose
from typing import Final
import pytest
from e2e_http import Success
from lifecycle import ResourceManager
from models import LiteLLMParamsBody, SpendLogs, SpendLogsDateRangeParams, SpendLogsParams
from models import LiteLLMParamsBody, SpendLogs, SpendLogsParams
from spend_e2e_client import SpendClient, SpendLogRow, is_ok, unique_marker, unwrap
pytestmark = pytest.mark.e2e
@ -510,88 +509,6 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
)
@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date")
def test_spend_logs_single_day_range_returns_that_days_spend(
client: SpendClient, scoped_key: str
) -> None:
"""start_date == end_date on /spend/logs must return the spend logged on that
day, both as the daily summary (default) and as the individual rows
(summarize=false). The end date is a calendar day, so its rows after
midnight belong to the range; the window is the calendar day of the
stored spend row."""
chat = unwrap(
client.chat(
scoped_key,
"gemini-2.5-flash",
f"one word {unique_marker()}",
max_tokens=16,
)
)
assert chat.id, f"chat response carried no id: {chat}"
rows = client.poll_logs_for_key(
scoped_key,
predicate=lambda rs: any(
r.request_id == chat.id and (r.spend or 0) > 0 for r in rs
),
)
logged = next(r for r in rows if r.request_id == chat.id)
assert logged.startTime, f"spend row carries no startTime: {logged}"
day: Final = logged.startTime[:10]
window = SpendLogsDateRangeParams(start_date=day, end_date=day, api_key=scoped_key)
summary = client.proxy.spend_logs_daily(window)
assert [r.startTime for r in summary] == [
day
], f"single-day window must yield exactly that day: {summary}"
assert summary[0].spend > 0, (
f"the end day's spend was dropped from the window: {summary}"
)
raw = client.proxy.spend_logs_in_window(window.model_copy(update={"summarize": False}))
assert any(r.request_id == chat.id for r in raw), (
f"summarize=false window missing request {chat.id}: "
f"{_summarize(raw)}; polled rows {_summarize(rows)}"
)
@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date")
def test_spend_logs_multi_day_range_includes_final_day(
client: SpendClient, scoped_key: str
) -> None:
"""A multi-day window ending on the calendar day of the stored spend row
reports that day's real spend as its final row, never a zero pad."""
chat = unwrap(
client.chat(
scoped_key,
"gemini-2.5-flash",
f"one word {unique_marker()}",
max_tokens=16,
)
)
assert chat.id, f"chat response carried no id: {chat}"
rows = client.poll_logs_for_key(
scoped_key,
predicate=lambda rs: any(
r.request_id == chat.id and (r.spend or 0) > 0 for r in rs
),
)
logged = next(r for r in rows if r.request_id == chat.id)
assert logged.startTime, f"spend row carries no startTime: {logged}"
end: Final = date.fromisoformat(logged.startTime[:10])
start = end - timedelta(days=2)
summary = client.proxy.spend_logs_daily(
SpendLogsDateRangeParams(
start_date=start.isoformat(), end_date=end.isoformat(), api_key=scoped_key
)
)
assert summary, "window ending today returned no rows"
assert summary[-1].startTime == end.isoformat(), (
f"final row is not the end date: {summary}"
)
assert summary[-1].spend > 0, f"end day reported as zero-spend padding: {summary}"
def test_spend_logs_endpoint_returns_spend(
client: SpendClient, scoped_key: str
) -> None:

View file

@ -245,6 +245,12 @@
"tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [
"quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals"
],
"tests/integration/spend/test_spend_logs_date_window.py::test_single_day_window_returns_that_days_summary_and_rows": [
"quota_management.spend_tracking.date_range.includes_end_date"
],
"tests/integration/spend/test_spend_logs_date_window.py::test_multi_day_window_ends_with_the_end_days_real_spend": [
"quota_management.spend_tracking.date_range.includes_end_date"
],
"tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [
"mgmt.key.update.project_detach_denied_to_restricted_actor"
],

View file

@ -0,0 +1,64 @@
import uuid
from datetime import date, timedelta
from typing import Final
import pytest
from pydantic import JsonValue
from tests.integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from tests.integration._support.database import read_rows
EXPECTED_SPEND: Final = 20 * 0.001 + 20 * 0.002
def _logged_request(gateway: Gateway, scenario: Scenario) -> tuple[str, str, str]:
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002, num_retries=0)
key: Final = scenario.key(models=[model])
body: Final = gateway.chat(model, key=key, text=uuid.uuid4().hex)
request_id: Final = string_value(body["id"])
rows: Final = eventually(
lambda: read_rows(
'SELECT to_char("startTime", \'YYYY-MM-DD\') AS day, spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
(request_id,),
),
lambda values: len(values) == 1,
seconds=70,
)
assert float(rows[0]["spend"]) == pytest.approx(EXPECTED_SPEND)
return key, request_id, string_value(rows[0]["day"])
def _spend(row: dict[str, JsonValue]) -> float:
value: Final = row["spend"]
assert isinstance(value, (int, float)), f"spend is not numeric: {row}"
return float(value)
@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date")
def test_single_day_window_returns_that_days_summary_and_rows(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
key, request_id, day = _logged_request(gateway, scenario)
window: Final = {"start_date": day, "end_date": day, "api_key": key}
summary: Final = gateway.request("GET", "/spend/logs", params=window)
assert summary.status_code == 200, summary.text
days: Final = tuple(object_value(row) for row in summary.json())
assert [row["startTime"] for row in days] == [day], summary.text
assert _spend(days[0]) == pytest.approx(EXPECTED_SPEND), summary.text
raw: Final = gateway.request("GET", "/spend/logs", params={**window, "summarize": "false"})
assert raw.status_code == 200, raw.text
assert [object_value(row)["request_id"] for row in raw.json()] == [request_id], raw.text
@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date")
def test_multi_day_window_ends_with_the_end_days_real_spend(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
key, _, day = _logged_request(gateway, scenario)
start: Final = (date.fromisoformat(day) - timedelta(days=2)).isoformat()
summary: Final = gateway.request(
"GET", "/spend/logs", params={"start_date": start, "end_date": day, "api_key": key}
)
assert summary.status_code == 200, summary.text
days: Final = tuple(object_value(row) for row in summary.json())
assert days, summary.text
assert days[-1]["startTime"] == day, summary.text
assert _spend(days[-1]) == pytest.approx(EXPECTED_SPEND), summary.text