From d2ee416342dbe71a15105f5b2cb41b4180eff141 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 22 Sep 2026 18:43:46 +0000 Subject: [PATCH] test(spend): move the /spend/logs end_date repro to tests/integration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/AGENTS.md | 3 +- .../coverage_registry/quota_management.yaml | 1 - tests/e2e/models.py | 22 ----- tests/e2e/proxy_client.py | 25 ------ .../spend_tracking/test_spend_tracking_e2e.py | 85 +------------------ tests/integration/contracts.json | 6 ++ .../spend/test_spend_logs_date_window.py | 64 ++++++++++++++ 7 files changed, 72 insertions(+), 134 deletions(-) create mode 100644 tests/integration/spend/test_spend_logs_date_window.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 11d30b3ff62..b00b7dfac95 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -210,7 +210,6 @@ quota_management... chat_completions | stream | messages_bridge | embeddings | cache_hit | key_rollup | concurrent_burst | tags | end_user | per_model | failure | spend_calculate | pagination | key_attribution - | date_range assertion : blocks_over_limit | resets_after_window | headers_report_remaining | picks_under_tpm | blocks_then_resets | resets_windows_independently | alerts_without_blocking | isolates_per_model | isolates_per_member | isolates_per_group | enforced_across_keys @@ -218,7 +217,7 @@ quota_management... | matches_sum_of_logs | loses_no_spend | attributes_spend | writes_own_rows | writes_failure_row | returns_cost | keeps_total | joins_key | reports_alias_and_email | health_rows_keep_service_account | retrieve_batch_cost_joins_retrieving_key - | poller_batch_cost_joins_creating_key | includes_end_date + | poller_batch_cost_joins_creating_key e.g. quota_management.ratelimit.rpm.blocks_over_limit exercised_on=[chat_completions, messages] quota_management.budget.key.blocks_over_limit exercised_on=[chat_completions] ``` diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 1ed38736985..ad0914d455b 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -51,7 +51,6 @@ - {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"} - {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"} - {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"} -- {id: quota_management.spend_tracking.date_range.includes_end_date, module: quota_management, tier: P1, behavior: spend_tracking, variant: date_range, assertions: [includes_end_date], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "A /spend/logs start_date/end_date window treats the end date as a whole calendar day, so the final day's spend is reported, not dropped to a zero pad (#40216)"} - {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"} - {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"} - {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 90f6e350921..5278dbd287a 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -909,7 +909,6 @@ class SpendLogMetadata(BaseModel): class SpendLogRow(BaseModel): request_id: str | None = None - startTime: str | None = None api_key: str | None = None model: str | None = None spend: float | None = None @@ -949,27 +948,6 @@ class SpendLogsParams(BaseModel): return self -class SpendLogsDateRangeParams(BaseModel): - """Query for legacy /spend/logs with a YYYY-MM-DD window; api_key keeps the - read bounded.""" - start_date: str - end_date: str - api_key: str - summarize: bool | None = None - - -class SpendDailySummaryRow(BaseModel): - model_config = ConfigDict(extra="allow") - startTime: str - spend: float - users: dict[str, float] - models: dict[str, float] - - -class SpendDailySummary(RootModel[list[SpendDailySummaryRow]]): - pass - - class SpendLogsPageParams(BaseModel): """Query for /spend/logs/v2, which requires an explicit date window and serves pages of at most 100 rows.""" diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index d1efe882727..f1981e0aa5e 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -87,11 +87,8 @@ from models import ( RouterSettingsResponse, SearchToolCreateBody, SearchToolCreateResponse, - SpendDailySummary, - SpendDailySummaryRow, SpendLogRow, SpendLogs, - SpendLogsDateRangeParams, SpendLogsPage, SpendLogsPageParams, SpendLogsParams, @@ -1036,28 +1033,6 @@ class ProxyClient: case _: return [] - def spend_logs_daily(self, params: SpendLogsDateRangeParams) -> list[SpendDailySummaryRow]: - """GET /spend/logs with a date window (summarize=true default): one row per day.""" - return unwrap( - self.transport.get( - "/spend/logs", - headers=self.management_headers(), - params=params, - response_type=SpendDailySummary, - ) - ).root - - def spend_logs_in_window(self, params: SpendLogsDateRangeParams) -> list[SpendLogRow]: - """GET /spend/logs with a date window and summarize=false: the individual rows.""" - return unwrap( - self.transport.get( - "/spend/logs", - headers=self.management_headers(), - params=params, - response_type=SpendLogs, - ) - ).root - def spend_logs_window(self, *, start: datetime, end: datetime) -> list[SpendLogRow]: def fetch(page: int) -> SpendLogsPage: return unwrap( diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index a95e2288ee2..8a91e53e7d7 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -17,14 +17,13 @@ fails the test; a pricing or token-count drift does not. import time from collections.abc import Callable -from datetime import date, timedelta from math import isclose from typing import Final import pytest from e2e_http import Success from lifecycle import ResourceManager -from models import LiteLLMParamsBody, SpendLogs, SpendLogsDateRangeParams, SpendLogsParams +from models import LiteLLMParamsBody, SpendLogs, SpendLogsParams from spend_e2e_client import SpendClient, SpendLogRow, is_ok, unique_marker, unwrap pytestmark = pytest.mark.e2e @@ -510,88 +509,6 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None: ) -@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date") -def test_spend_logs_single_day_range_returns_that_days_spend( - client: SpendClient, scoped_key: str -) -> None: - """start_date == end_date on /spend/logs must return the spend logged on that - day, both as the daily summary (default) and as the individual rows - (summarize=false). The end date is a calendar day, so its rows after - midnight belong to the range; the window is the calendar day of the - stored spend row.""" - chat = unwrap( - client.chat( - scoped_key, - "gemini-2.5-flash", - f"one word {unique_marker()}", - max_tokens=16, - ) - ) - assert chat.id, f"chat response carried no id: {chat}" - rows = client.poll_logs_for_key( - scoped_key, - predicate=lambda rs: any( - r.request_id == chat.id and (r.spend or 0) > 0 for r in rs - ), - ) - logged = next(r for r in rows if r.request_id == chat.id) - assert logged.startTime, f"spend row carries no startTime: {logged}" - day: Final = logged.startTime[:10] - window = SpendLogsDateRangeParams(start_date=day, end_date=day, api_key=scoped_key) - - summary = client.proxy.spend_logs_daily(window) - assert [r.startTime for r in summary] == [ - day - ], f"single-day window must yield exactly that day: {summary}" - assert summary[0].spend > 0, ( - f"the end day's spend was dropped from the window: {summary}" - ) - - raw = client.proxy.spend_logs_in_window(window.model_copy(update={"summarize": False})) - assert any(r.request_id == chat.id for r in raw), ( - f"summarize=false window missing request {chat.id}: " - f"{_summarize(raw)}; polled rows {_summarize(rows)}" - ) - - -@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date") -def test_spend_logs_multi_day_range_includes_final_day( - client: SpendClient, scoped_key: str -) -> None: - """A multi-day window ending on the calendar day of the stored spend row - reports that day's real spend as its final row, never a zero pad.""" - chat = unwrap( - client.chat( - scoped_key, - "gemini-2.5-flash", - f"one word {unique_marker()}", - max_tokens=16, - ) - ) - assert chat.id, f"chat response carried no id: {chat}" - rows = client.poll_logs_for_key( - scoped_key, - predicate=lambda rs: any( - r.request_id == chat.id and (r.spend or 0) > 0 for r in rs - ), - ) - logged = next(r for r in rows if r.request_id == chat.id) - assert logged.startTime, f"spend row carries no startTime: {logged}" - end: Final = date.fromisoformat(logged.startTime[:10]) - start = end - timedelta(days=2) - - summary = client.proxy.spend_logs_daily( - SpendLogsDateRangeParams( - start_date=start.isoformat(), end_date=end.isoformat(), api_key=scoped_key - ) - ) - assert summary, "window ending today returned no rows" - assert summary[-1].startTime == end.isoformat(), ( - f"final row is not the end date: {summary}" - ) - assert summary[-1].spend > 0, f"end day reported as zero-spend padding: {summary}" - - def test_spend_logs_endpoint_returns_spend( client: SpendClient, scoped_key: str ) -> None: diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index e1b5940935f..0bff208d22f 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -245,6 +245,12 @@ "tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [ "quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals" ], + "tests/integration/spend/test_spend_logs_date_window.py::test_single_day_window_returns_that_days_summary_and_rows": [ + "quota_management.spend_tracking.date_range.includes_end_date" + ], + "tests/integration/spend/test_spend_logs_date_window.py::test_multi_day_window_ends_with_the_end_days_real_spend": [ + "quota_management.spend_tracking.date_range.includes_end_date" + ], "tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [ "mgmt.key.update.project_detach_denied_to_restricted_actor" ], diff --git a/tests/integration/spend/test_spend_logs_date_window.py b/tests/integration/spend/test_spend_logs_date_window.py new file mode 100644 index 00000000000..fc87439c243 --- /dev/null +++ b/tests/integration/spend/test_spend_logs_date_window.py @@ -0,0 +1,64 @@ +import uuid +from datetime import date, timedelta +from typing import Final + +import pytest +from pydantic import JsonValue + +from tests.integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from tests.integration._support.database import read_rows + +EXPECTED_SPEND: Final = 20 * 0.001 + 20 * 0.002 + + +def _logged_request(gateway: Gateway, scenario: Scenario) -> tuple[str, str, str]: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002, num_retries=0) + key: Final = scenario.key(models=[model]) + body: Final = gateway.chat(model, key=key, text=uuid.uuid4().hex) + request_id: Final = string_value(body["id"]) + rows: Final = eventually( + lambda: read_rows( + 'SELECT to_char("startTime", \'YYYY-MM-DD\') AS day, spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (request_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert float(rows[0]["spend"]) == pytest.approx(EXPECTED_SPEND) + return key, request_id, string_value(rows[0]["day"]) + + +def _spend(row: dict[str, JsonValue]) -> float: + value: Final = row["spend"] + assert isinstance(value, (int, float)), f"spend is not numeric: {row}" + return float(value) + + +@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date") +def test_single_day_window_returns_that_days_summary_and_rows(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + key, request_id, day = _logged_request(gateway, scenario) + window: Final = {"start_date": day, "end_date": day, "api_key": key} + summary: Final = gateway.request("GET", "/spend/logs", params=window) + assert summary.status_code == 200, summary.text + days: Final = tuple(object_value(row) for row in summary.json()) + assert [row["startTime"] for row in days] == [day], summary.text + assert _spend(days[0]) == pytest.approx(EXPECTED_SPEND), summary.text + raw: Final = gateway.request("GET", "/spend/logs", params={**window, "summarize": "false"}) + assert raw.status_code == 200, raw.text + assert [object_value(row)["request_id"] for row in raw.json()] == [request_id], raw.text + + +@pytest.mark.covers("quota_management.spend_tracking.date_range.includes_end_date") +def test_multi_day_window_ends_with_the_end_days_real_spend(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + key, _, day = _logged_request(gateway, scenario) + start: Final = (date.fromisoformat(day) - timedelta(days=2)).isoformat() + summary: Final = gateway.request( + "GET", "/spend/logs", params={"start_date": start, "end_date": day, "api_key": key} + ) + assert summary.status_code == 200, summary.text + days: Final = tuple(object_value(row) for row in summary.json()) + assert days, summary.text + assert days[-1]["startTime"] == day, summary.text + assert _spend(days[-1]) == pytest.approx(EXPECTED_SPEND), summary.text