diff --git a/litellm/proxy/spend_tracking/session_page.py b/litellm/proxy/spend_tracking/session_page.py new file mode 100644 index 00000000000..0b512326c92 --- /dev/null +++ b/litellm/proxy/spend_tracking/session_page.py @@ -0,0 +1,212 @@ +"""Which sessions a logs page shows, and how many sessions its filters match. + +A session is the matching ``LiteLLM_SpendLogs`` rows that share a session key and an api_key, ranked by +``(last_activity, session_key, api_key)``. A descending page reads only the newest matching rows. Every matching row +newer than the oldest row read is in the read, so a session last active after that row has its true last activity and +no session outside the read can rank before it. A read that settles both the page and the capped count is used. One +that does not is discarded for a larger one, so the result is the same as grouping every matching row. +""" + +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from typing import Final, Protocol, TypeAlias + +from typing_extensions import ReadOnly, TypedDict + +SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" + +# Each read takes this many times the first read's rows, ``None`` taking every matching row. +_GROWTH: Final[tuple[int | None, ...]] = (1, 8, 64, None) + + +class _ReadRow(TypedDict): + session_key: ReadOnly[str | None] + api_key: ReadOnly[str | None] + last_activity: ReadOnly[str | None] + settled: ReadOnly[bool | None] + rows_read: ReadOnly[int] + session_count: ReadOnly[int] + + +class _SupportsQueryRaw(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[_ReadRow]: ... + + +@dataclass(frozen=True, slots=True) +class Session: + key: str + api_key: str + last_activity: str + + +@dataclass(frozen=True, slots=True) +class SessionPage: + sessions: tuple[Session, ...] + has_more: bool + next_cursor: str | None + total: int + total_is_capped: bool + + +@dataclass(frozen=True, slots=True) +class _Request: + cursor: tuple[str, str, str] | None + offset: int + page_limit: int + sort_desc: bool + cap: int + + +@dataclass(frozen=True, slots=True) +class _Read: + candidates: tuple[Session, ...] + last_settled: bool + rows_read: int + session_count: int + read_everything: bool + + def falls_short(self, budget: int | None, sessions_needed: int) -> bool: + """Whether ``budget`` newest rows, as dense in sessions as this read, would hold fewer than ``sessions_needed``.""" + return budget is not None and budget * self.session_count < sessions_needed * self.rows_read + + def decides(self, request: _Request) -> bool: + page_settled: Final = ( + request.page_limit <= 0 + or self.read_everything + or (len(self.candidates) > request.page_limit and self.last_settled) + ) + return page_settled and (self.read_everything or self.session_count > request.cap) + + +_Reader: TypeAlias = Callable[[int | None], Awaitable[_Read]] + + +def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None: + if session_cursor is None or session_cursor.count("|") < 2: + return None + last_activity, _, rest = session_cursor.partition("|") + api_key, _, session_key = rest.partition("|") + if not last_activity or not session_key: + return None + return (last_activity, session_key, api_key) + + +async def read_session_page( + db: _SupportsQueryRaw, + where_clause: str, + params: Sequence[object], + ctes: str, + page: int, + page_size: int, + sort_desc: bool, + session_cursor: str | None, + cap: int, +) -> SessionPage: + """The sessions of one page, resumed after ``session_cursor`` or, without one, after ``(page - 1) * page_size`` + sessions. ``where_clause`` reads ``params`` as ``$1`` onward and may name the CTEs that ``ctes`` declares, each + followed by a comma. No page runs past the ``cap`` sessions that the capped total promises.""" + cursor: Final = _parse_session_cursor(session_cursor) + offset: Final = (page - 1) * page_size if cursor is None else 0 + request: Final = _Request(cursor, offset, min(page_size, cap - offset), sort_desc, cap) + return await _session_page(_sql_reader(db, where_clause, params, ctes, request), request) + + +async def _session_page(read: _Reader, request: _Request) -> SessionPage: + # offset + page_limit never passes the cap, so the first read stays bounded however large the page number is. + first_read: Final = request.cap + request.offset + request.page_limit + 2 + budgets: Final = ( + tuple(None if growth is None else first_read * growth for growth in _GROWTH) if request.sort_desc else (None,) + ) + last = await read(budgets[0]) # rebind-ok: each read replaces the last until one decides + for budget in budgets[1:]: + if last.decides(request): + break + if last.falls_short(budget, request.cap + 1): + continue + last = await read(budget) + has_more: Final = len(last.candidates) > request.page_limit + sessions: Final = last.candidates[: request.page_limit] + final: Final = sessions[-1] if has_more and sessions else None + return SessionPage( + sessions=sessions, + has_more=has_more, + next_cursor=f"{final.last_activity}|{final.api_key}|{final.key}" if final else None, + total=min(last.session_count, request.cap), + total_is_capped=last.session_count > request.cap, + ) + + +def _sql_reader( + db: _SupportsQueryRaw, where_clause: str, params: Sequence[object], ctes: str, request: _Request +) -> _Reader: + cursor_index: Final = len(params) + 1 + cursor_params: Final[tuple[object, ...]] = request.cursor or () + limit_index: Final = cursor_index + len(cursor_params) + offset_params: Final[tuple[int, ...]] = (request.offset,) if request.offset and request.page_limit > 0 else () + direction: Final = "DESC" if request.sort_desc else "ASC" + after_cursor: Final = ( + f"WHERE (last_activity, session_key, api_key) {'<' if request.sort_desc else '>'} " + f"(${cursor_index}::timestamp, ${cursor_index + 1}, ${cursor_index + 2})" + if request.cursor + else "" + ) + page_query_args: Final = ( + *params, + *cursor_params, + request.page_limit + 1 if request.page_limit > 0 else 0, + *offset_params, + ) + + async def read(budget: int | None) -> _Read: + # ``scanned`` is read twice, so Postgres computes it once without MATERIALIZED, which Postgres 10 and 11 + # cannot parse. ``summary`` is one row, and the LEFT JOIN keeps it when the page is empty. + rows: Final = await db.query_raw( + f""" + WITH {ctes} scanned AS ( + SELECT {SESSION_KEY_EXPR} AS session_key, api_key, "startTime" + FROM "LiteLLM_SpendLogs" + WHERE {where_clause} + {f'ORDER BY "startTime" DESC LIMIT {budget}' if budget is not None else ""} + ), + sessions AS ( + SELECT session_key, api_key, MAX("startTime") AS last_activity + FROM scanned + GROUP BY session_key, api_key + ), + summary AS ( + SELECT COUNT(*) AS rows_read, + MIN("startTime") AS oldest, + (SELECT COUNT(*) FROM sessions) AS session_count + FROM scanned + ) + SELECT page.session_key, + page.api_key, + page.last_activity::text AS last_activity, + page.last_activity > summary.oldest AS settled, + summary.rows_read, + summary.session_count + FROM summary + LEFT JOIN LATERAL ( + SELECT session_key, api_key, last_activity + FROM sessions + {after_cursor} + ORDER BY last_activity {direction}, session_key {direction}, api_key {direction} + LIMIT ${limit_index} {f"OFFSET ${limit_index + 1}" if offset_params else ""} + ) AS page ON TRUE + """, + *page_query_args, + ) + rows_read: Final = int(rows[0]["rows_read"]) + return _Read( + candidates=tuple( + Session(str(row["session_key"]), str(row["api_key"]), str(row["last_activity"])) + for row in rows + if row["session_key"] is not None + ), + last_settled=bool(rows[-1]["settled"]), + rows_read=rows_read, + session_count=int(rows[0]["session_count"]), + read_everything=budget is None or rows_read < budget, + ) + + return read diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 89200792c0b..177f23996b0 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -52,6 +52,7 @@ from litellm.proxy.auth.authorization_dependencies import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.spend_tracking.session_page import SESSION_KEY_EXPR, read_session_page from litellm.proxy.spend_tracking.spend_capture_rate import ( ProviderBillingCredentialMissing, ProviderBillingRequestFailed, @@ -87,8 +88,7 @@ router: Final = APIRouter() SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000 _DECODED_JSON: Final = TypeAdapter(object) -_SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" -_SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key" +_SESSION_GROUP_KEY_SQL: Final = f"{SESSION_KEY_EXPR}, api_key" _MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" _AGENT_CALL_TYPE_SQL: Final = "'asend_message'" _SESSION_REPRESENTATIVE_ORDER_SQL: Final = ( @@ -2843,12 +2843,14 @@ async def ui_view_spend_logs( # Date range. Wrap the param side with `AT TIME ZONE 'UTC'` so comparison # against the plain `timestamp` column does not depend on the DB session # timezone (see #22529). Absent for a request_id-only lookup (see above). - if start_date_obj is not None and end_date_obj is not None and not search_owns_window: - sql_conditions.append(f"\"startTime\" >= (${p}::timestamptz AT TIME ZONE 'UTC')") - sql_params.append(start_date_obj) - p += 1 - sql_conditions.append(f"\"startTime\" <= (${p}::timestamptz AT TIME ZONE 'UTC')") - sql_params.append(end_date_obj) + window_bounds: Final = ( + ((">=", start_date_obj), ("<=", end_date_obj)) + if start_date_obj is not None and end_date_obj is not None and not search_owns_window + else () + ) + for op, bound in window_bounds: + sql_conditions.append(f"\"startTime\" {op} (${p}::timestamptz AT TIME ZONE 'UTC')") + sql_params.append(bound) p += 1 if search is not None and start_date_obj is not None and end_date_obj is not None: @@ -2948,23 +2950,6 @@ async def ui_view_spend_logs( sql_params.append(json.dumps(used_client_oauth_token)) p += 1 - if status_filter is not None and group_by_session is True and not is_search_lookup: - session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE" - sql_conditions.append( - f"""({_SESSION_GROUP_KEY_SQL}) IN ( - SELECT session_key, api_key FROM ( - SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) - {_SESSION_KEY_EXPR} AS session_key, api_key, status - FROM "LiteLLM_SpendLogs" - WHERE {session_filter_conditions} - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} - ) AS session_outcomes - WHERE COALESCE(status, 'success') = ${p} - )""" - ) - sql_params.append(status_filter) - p += 1 - if ( group_by_session is True and not is_v2 @@ -2976,13 +2961,22 @@ async def ui_view_spend_logs( prisma_client=prisma_client, sql_conditions=sql_conditions, sql_params=sql_params, - next_param_index=p, + window_bounds=window_bounds, + status_filter=status_filter, page=page, page_size=page_size, sort_desc=order_direction != "asc", session_cursor=session_cursor, ) + if status_filter is not None and group_by_session is True and not is_search_lookup: + sql_conditions.append( + # bounded-ok: a subquery, not a list of values + f"({_SESSION_GROUP_KEY_SQL}) IN ({_sessions_with_status_sql(sql_conditions, window_bounds, p)})" + ) + sql_params.append(status_filter) + p += 1 + # Build the ORDER BY expression. ttft_ms is computed from # completionStartTime - startTime; non-streaming rows (where # completionStartTime is null or equals endTime) yield NULL, so we @@ -3080,41 +3074,63 @@ async def ui_view_spend_logs( raise handle_exception_on_proxy(e) -class _SessionPageRow(TypedDict): - session_key: ReadOnly[str] - api_key: ReadOnly[str] - last_activity: ReadOnly[str] +def _utc_timestamp_literal(bound: datetime) -> str: + """SQL literal for a UTC datetime, refusing anything else so no caller text reaches the SQL.""" + if type(bound) is not datetime or bound.tzinfo is not timezone.utc: + raise TypeError(f"expected a UTC datetime, got {type(bound).__name__}") + return f"('{bound.isoformat()}'::timestamptz AT TIME ZONE 'UTC')" -def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None: - if session_cursor is None or session_cursor.count("|") < 2: - return None - last_activity, _, rest = session_cursor.partition("|") - api_key, _, session_key = rest.partition("|") - if not last_activity or not session_key: - return None - return (last_activity, session_key, api_key) +def _sessions_with_status_sql( + sql_conditions: Sequence[str], window_bounds: Sequence[tuple[str, datetime]], status_index: int +) -> str: + """The ``(session_key, api_key)`` of each session whose outcome has the status bound to ``$status_index``. + + The window, which leads ``sql_conditions``, goes in as literals: Prisma reuses prepared statements, and from the + 6th run Postgres plans a ``$N`` window as 0.5% of the table, then probes every session by api_key, ~2 min instead + of ~2.5 s over 700k rows. + """ + window_literals: Final = tuple(f'"startTime" {op} {_utc_timestamp_literal(bound)}' for op, bound in window_bounds) + conditions: Final = " AND ".join((*window_literals, *sql_conditions[len(window_literals) :])) or "TRUE" + return f""" + SELECT session_key, api_key FROM ( + SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) + {SESSION_KEY_EXPR} AS session_key, api_key, status + FROM "LiteLLM_SpendLogs" + WHERE {conditions} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} + ) AS session_outcomes + WHERE COALESCE(status, 'success') = ${status_index} + """ async def _fetch_session_representatives( prisma_client: "PrismaClient", where_clause: str, sql_params: Sequence[object], - next_param_index: int, session_keys: Sequence[tuple[str, str]], + status_filter: str | None, ) -> list[dict[str, object]]: # mutable-ok: _build_ui_spend_logs_response writes session counts onto each row - """Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order.""" + """Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order. + + With ``status_filter``, a session whose outcome no longer has that status is dropped, as the session status + filter would drop it in this statement's snapshot. + """ + keys_index: Final = len(sql_params) + 1 + # The session_id / request_id test finds the rows through those columns' indexes, which the key expression cannot use. rep_query: Final = f""" SELECT * FROM ( SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) {_SPEND_LOG_LIST_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {where_clause} + AND (session_id = ANY(${keys_index}::text[]) OR request_id = ANY(${keys_index}::text[])) AND ({_SESSION_GROUP_KEY_SQL}) IN ( - SELECT * FROM unnest(${next_param_index}::text[], ${next_param_index + 1}::text[]) + SELECT * FROM unnest(${keys_index}::text[], ${keys_index + 1}::text[]) ) ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} ) AS session_representatives + {f"WHERE COALESCE(status, 'success') = ${keys_index + 2}" if status_filter is not None else ""} """ rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place prisma_client, @@ -3122,6 +3138,7 @@ async def _fetch_session_representatives( *sql_params, [session_key for session_key, _ in session_keys], [api_key for _, api_key in session_keys], + *((status_filter,) if status_filter is not None else ()), ) rep_by_key: Final[Mapping[tuple[str, str], dict[str, object]]] = MappingProxyType( # mutable-ok: same rows {(str(row["session_id"] or row["request_id"]), str(row["api_key"])): row for row in rep_rows} @@ -3129,142 +3146,77 @@ async def _fetch_session_representatives( return [rep_by_key[key] for key in session_keys if key in rep_by_key] -async def _count_grouped_sessions( - prisma_client: "PrismaClient", - where_clause: str, - sql_params: Sequence[object], - next_param_index: int, -) -> tuple[int, bool]: - """Count the sessions matching the filter, returning ``(total, total_is_capped)`` bounded by the count cap.""" - count_query: Final = f""" - SELECT COUNT(*) AS total_count - FROM ( - SELECT 1 - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - LIMIT ${next_param_index} - ) AS bounded_sessions - """ - count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_raw( - prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1 - ) - raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0 - return ( - (SPEND_LOGS_PAGINATION_COUNT_CAP, True) if raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP else (raw_total, False) - ) - - async def _ui_session_grouped_spend_logs( prisma_client: "PrismaClient", sql_conditions: Sequence[str], sql_params: Sequence[object], - next_param_index: int, + window_bounds: Sequence[tuple[str, datetime]], + status_filter: str | None, page: int, page_size: int, sort_desc: bool, session_cursor: str | None, ) -> Mapping[str, object]: - """ - One row per session, keyset-paginated by session last activity. - - Sessions are derived on the fly from ``LiteLLM_SpendLogs`` (no extra - table): rows sharing a ``session_id`` and ``api_key`` form a session, rows - without a session id are singletons keyed by ``request_id``. A page is the - next ``page_size`` sessions ordered by ``(MAX(startTime), session_key, - api_key)``, resumed from the ``session_cursor`` keyset - ``'||'`` instead of an OFFSET, so - page depth does not degrade the query plan. A request for ``page > 1`` - without a cursor (the UI jumping straight to the last page, or back to a - page it never walked through) falls back to ``OFFSET (page - 1) * - page_size``, trimmed to the end of the ``SPEND_LOGS_PAGINATION_COUNT_CAP`` - window the capped ``total`` promises, so a page never runs past that total - and one starting at or past it returns no rows without a query. Each session is represented - by its final agent outcome (or newest non-MCP row), enriched by ``_build_ui_spend_logs_response`` - exactly like the flat listing, and the response carries - ``next_session_cursor`` / ``has_more`` while ``total`` counts sessions - (capped like the flat total). A page that runs out of sessions while still - holding some is itself the end of the list, so its ``total`` is - ``offset + len(page)`` and the grouped count query is skipped; a page that - starts past the end says nothing about the total, so that one is counted. - """ - where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE" - cmp_op: Final = "<" if sort_desc else ">" - direction: Final = "DESC" if sort_desc else "ASC" - - cursor: Final = _parse_session_cursor(session_cursor) - having_clause: Final = ( - f'HAVING (MAX("startTime"), {_SESSION_GROUP_KEY_SQL}) {cmp_op} ' - f"(${next_param_index}::timestamp, ${next_param_index + 1}, ${next_param_index + 2})" - if cursor - else "" + """One row per session, keyset-paginated by session last activity: ``read_session_page`` picks the sessions and + counts them, and each session is shown by its final agent outcome (or newest non-MCP row), enriched by + ``_build_ui_spend_logs_response`` exactly like the flat listing.""" + outcomes_sql: Final = ( + None + if status_filter is None + else _sessions_with_status_sql(sql_conditions, window_bounds, status_index=len(sql_params) + 1) ) - cursor_params: Final[tuple[object, ...]] = cursor if cursor else () - limit_index: Final = next_param_index + len(cursor_params) - offset: Final = (page - 1) * page_size if cursor is None else 0 - page_limit: Final = min(page_size, SPEND_LOGS_PAGINATION_COUNT_CAP - offset) - offset_params: Final[tuple[int, ...]] = (offset,) if offset and page_limit > 0 else () - offset_clause: Final = f"OFFSET ${limit_index + 1}" if offset_params else "" - - page_query: Final = f""" - SELECT {_SESSION_KEY_EXPR} AS session_key, - api_key, - MAX("startTime")::text AS last_activity - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - {having_clause} - ORDER BY MAX("startTime") {direction}, {_SESSION_KEY_EXPR} {direction}, api_key {direction} - LIMIT ${limit_index} {offset_clause} - """ - page_rows: Final[Sequence[_SessionPageRow]] = ( + # A failure filter keeps few sessions, so they are computed once in a CTE: inlined, each parallel worker re-sorts + # the whole window for them (3x the blocks and temp on 2M rows). Inlined is faster for a success filter. The CTE is + # named twice so Postgres 12+ keeps it as one computation, as MATERIALIZED would, which Postgres 10 and 11 cannot + # parse. + materialized: Final = outcomes_sql is not None and status_filter == "failure" + status_conditions: Final = ( () - if page_limit <= 0 - else await _query_raw(prisma_client, page_query, *sql_params, *cursor_params, page_limit + 1, *offset_params) + if outcomes_sql is None + else ( + f"EXISTS (SELECT 1 FROM status_sessions) AND ({_SESSION_GROUP_KEY_SQL}) IN " + "(SELECT session_key, api_key FROM status_sessions)" + if materialized + # bounded-ok: a subquery, not a list of values + else f"({_SESSION_GROUP_KEY_SQL}) IN ({outcomes_sql})", + ) ) - - has_more: Final = len(page_rows) > page_limit - visible_rows: Final = page_rows[:page_limit] - next_cursor: Final = ( - f"{visible_rows[-1]['last_activity']}|{visible_rows[-1]['api_key']}|{visible_rows[-1]['session_key']}" - if has_more and visible_rows - else None + session_page: Final = await read_session_page( + db=prisma_client.db, # pyright: ignore[reportArgumentType] # PrismaWrapper forwards raw queries via __getattr__ + where_clause=" AND ".join((*sql_conditions, *status_conditions)) or "TRUE", + params=(*sql_params, *((status_filter,) if status_filter is not None else ())), + ctes=f"status_sessions AS ({outcomes_sql})," if materialized else "", + page=page, + page_size=page_size, + sort_desc=sort_desc, + session_cursor=session_cursor, + cap=SPEND_LOGS_PAGINATION_COUNT_CAP, ) - - page_starts_inside_the_list: Final = offset == 0 or len(page_rows) > 0 - page_ends_the_list: Final = cursor is None and page_limit > 0 and not has_more and page_starts_inside_the_list - total_records, total_is_capped = ( - (offset + len(page_rows), False) - if page_ends_the_list - else await _count_grouped_sessions(prisma_client, where_clause, sql_params, next_param_index) - ) - - session_keys: Final = tuple((row["session_key"], row["api_key"]) for row in visible_rows) + # The status filter keeps whole sessions, so the page's own sessions skip it here; their outcomes are re-checked. data: Final[list[dict[str, object]]] = ( # mutable-ok: _build_ui_spend_logs_response writes onto each row await _fetch_session_representatives( prisma_client=prisma_client, - where_clause=where_clause, + where_clause=" AND ".join(sql_conditions) or "TRUE", sql_params=sql_params, - next_param_index=next_param_index, - session_keys=session_keys, + session_keys=tuple((session.key, session.api_key) for session in session_page.sessions), + status_filter=status_filter, ) - if session_keys + if session_page.sessions else [] ) _hydrate_spend_log_metadata(data) - total_pages: Final = (total_records + page_size - 1) // page_size response: Final[Mapping[str, object]] = await _build_ui_spend_logs_response( prisma_client, data, - total_records, + session_page.total, page, page_size, - total_pages, + (session_page.total + page_size - 1) // page_size, enrich_session_counts=True, - total_is_capped=total_is_capped, + total_is_capped=session_page.total_is_capped, ) - return {**response, "next_session_cursor": next_cursor, "has_more": has_more} + return {**response, "next_session_cursor": session_page.next_cursor, "has_more": session_page.has_more} class RequestResponsePayload(NamedTuple): diff --git a/tests/integration/spend/test_session_grouped_logs_pages.py b/tests/integration/spend/test_session_grouped_logs_pages.py new file mode 100644 index 00000000000..5591d2b7fab --- /dev/null +++ b/tests/integration/spend/test_session_grouped_logs_pages.py @@ -0,0 +1,276 @@ +import json +import os +import uuid +from collections.abc import AsyncIterator, Iterator, Mapping +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any, Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import psycopg +import pytest +from prisma import Prisma +from psycopg import sql +from pydantic import BaseModel, ConfigDict + +from litellm.proxy.spend_tracking.session_page import read_session_page +from tests.integration._support.client import Gateway +from tests.integration._support.database import write_rows + +_URL: Final = "/spend/logs/ui" +_WINDOW_SECONDS: Final = 3600 + + +@dataclass(frozen=True, slots=True) +class _Row: + request: str + session: str | None + api_key: str + second: int + status: str = "success" + model: str = "m-a" + + +_ROWS: Final = ( + _Row("z1", "z", "key-1", -1), + _Row("a1", "a", "key-1", 10), + _Row("a2", "a", "key-1", 50), + _Row("b1", "b", "key-1", 20, model="m-b"), + _Row("b2", "b", "key-1", 60), + _Row("b3", "b", "key-1", 30), + _Row("c1", "c", "key-2", 60), + _Row("d1", "b", "key-2", 40, model="m-b"), + _Row("e1", None, "key-1", 55, model="m-b"), + _Row("f1", "", "key-1", 45), + _Row("g1", "g", "key-3", 5, status="failure"), + _Row("g2", "g", "key-3", 15, status="failure"), + *(_Row(f"h{second - 69}", "h", "key-1", second) for second in range(70, 76)), + _Row("i1", "i", "key-3", 35, status="failure"), + _Row("i2", "i", "key-3", _WINDOW_SECONDS + 1), +) +_NEWEST_FIRST: Final = ( + ("h", "key-1"), + ("c", "key-2"), + ("b", "key-1"), + ("e1", "key-1"), + ("a", "key-1"), + ("f1", "key-1"), + ("b", "key-2"), + ("i", "key-3"), + ("g", "key-3"), +) + + +class _LogRow(BaseModel): + model_config = ConfigDict(frozen=True) + + request_id: str + session_id: str | None + api_key: str + session_total_count: int + + +class _Page(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[_LogRow, ...] + total: int + total_is_capped: bool + has_more: bool + next_session_cursor: str | None + + +@dataclass(frozen=True, slots=True) +class _Seeded: + prefix: str + window: Mapping[str, str] + + def session(self, row: _LogRow) -> tuple[str, str]: + key: Final = row.session_id or row.request_id + assert key.startswith(self.prefix), key + return key.removeprefix(self.prefix), row.api_key + + +@pytest.fixture +def seeded(gateway: Gateway) -> Iterator[_Seeded]: + prefix: Final = f"sess{uuid.uuid4().hex[:12]}-" + start: Final = datetime(1900, 1, 1) + timedelta(days=int(prefix[4:16], 16) % 200000) + rows: Final = [ + { + "request_id": f"{prefix}{row.request}", + "session_id": None if row.session is None else f"{prefix}{row.session}" if row.session else "", + "api_key": row.api_key, + "start_time": (start + timedelta(seconds=row.second)).isoformat(), + "status": row.status, + "model": row.model, + } + for row in _ROWS + ] + with gateway.scenario() as scenario: + scenario.cleanups.callback( + write_rows, 'DELETE FROM "LiteLLM_SpendLogs" WHERE request_id LIKE %s', (f"{prefix}%",) + ) + write_rows( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model, ' + "session_id, status) SELECT request_id, 'acompletion', api_key, start_time, start_time, model, session_id, " + "status FROM jsonb_to_recordset(%s::jsonb) AS seeded(request_id text, session_id text, api_key text, " + "start_time timestamp, status text, model text)", + (json.dumps(rows),), + ) + yield _Seeded( + prefix, + { + "start_date": f"{start:%Y-%m-%d %H:%M:%S}", + "end_date": f"{start + timedelta(seconds=_WINDOW_SECONDS):%Y-%m-%d %H:%M:%S}", + "group_by_session": "true", + }, + ) + + +def _page(gateway: Gateway, seeded: _Seeded, **params: str) -> _Page: + return _Page.model_validate(gateway.get(_URL, {**seeded.window, **params})) + + +def _cursor_walk(gateway: Gateway, seeded: _Seeded, number: int = 1, **params: str) -> tuple[_Page, ...]: + page: Final = _page(gateway, seeded, page_size="2", page=str(number), **params) + if page.next_session_cursor is None or number > len(_NEWEST_FIRST): + return (page,) + following: Final = {**params, "session_cursor": page.next_session_cursor} + return (page, *_cursor_walk(gateway, seeded, number + 1, **following)) + + +def _sessions(seeded: _Seeded, pages: tuple[_Page, ...]) -> tuple[tuple[str, str], ...]: + return tuple(seeded.session(row) for page in pages for row in page.data) + + +def test_cursor_pages_show_every_session_once_newest_first_with_its_newest_row( + gateway: Gateway, seeded: _Seeded +) -> None: + pages: Final = _cursor_walk(gateway, seeded) + assert _sessions(seeded, pages) == _NEWEST_FIRST + assert [(page.total, page.total_is_capped) for page in pages] == [(len(_NEWEST_FIRST), False)] * len(pages) + assert [page.has_more for page in pages] == [True] * (len(pages) - 1) + [False] + representatives: Final = tuple( + (row.request_id.removeprefix(seeded.prefix), row.session_total_count) for page in pages for row in page.data + ) + assert representatives == ( + ("h6", 6), + ("c1", 1), + ("b2", 3), + ("e1", 1), + ("a2", 2), + ("f1", 1), + ("d1", 1), + ("i1", 2), + ("g2", 2), + ) + + +@pytest.mark.parametrize( + "sort_order, expected", + [("desc", _NEWEST_FIRST), ("asc", tuple(reversed(_NEWEST_FIRST)))], + ids=["newest-first", "oldest-first"], +) +def test_numbered_pages_show_the_same_sessions_as_the_cursor_walk( + gateway: Gateway, seeded: _Seeded, sort_order: str, expected: tuple[tuple[str, str], ...] +) -> None: + pages: Final = tuple( + _page(gateway, seeded, page_size="2", page=str(number), sort_order=sort_order) for number in range(1, 7) + ) + assert _sessions(seeded, pages) == expected + assert [len(page.data) for page in pages] == [2, 2, 2, 2, 1, 0] + assert [page.has_more for page in pages] == [True, True, True, True, False, False] + assert {page.total for page in pages} == {len(expected)} + + +def test_failure_filter_keeps_only_sessions_whose_outcome_failed(gateway: Gateway, seeded: _Seeded) -> None: + pages: Final = _cursor_walk(gateway, seeded, status_filter="failure") + assert _sessions(seeded, pages) == (("i", "key-3"), ("g", "key-3")) + assert pages[0].total == 2 + + +def test_model_filter_ranks_each_session_by_its_matching_rows_only(gateway: Gateway, seeded: _Seeded) -> None: + pages: Final = _cursor_walk(gateway, seeded, model="m-b") + assert _sessions(seeded, pages) == (("e1", "key-1"), ("b", "key-2"), ("b", "key-1")) + assert [row.request_id.removeprefix(seeded.prefix) for page in pages for row in page.data] == ["e1", "d1", "b1"] + assert pages[0].total == 3 + + +_COPIED_ROWS: Final = 20_000 +_COPY_START: Final = datetime(2000, 1, 1, tzinfo=UTC) + + +@asynccontextmanager +async def _spend_logs_copy() -> AsyncIterator[Prisma]: + """A private schema holding a copy of the migrated `LiteLLM_SpendLogs` with its indexes, seeded with + `_COPIED_ROWS` one-row sessions a second apart.""" + schema: Final = f"integration_{uuid.uuid4().hex}" + url: Final = os.environ["DATABASE_URL"] + table: Final = sql.Identifier(schema, "LiteLLM_SpendLogs") + with psycopg.connect(url, autocommit=True) as setup: + setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + try: + setup.execute(sql.SQL('CREATE TABLE {} (LIKE "LiteLLM_SpendLogs" INCLUDING ALL)').format(table)) + setup.execute( + sql.SQL( + 'INSERT INTO {} (request_id, call_type, api_key, "startTime", "endTime") ' + "SELECT 'r' || n, 'acompletion', 'key-1', moment, moment FROM generate_series(1, %s) n, " + "LATERAL (SELECT %s::timestamp + n * INTERVAL '1 second' AS moment) seconds" + ).format(table), + (_COPIED_ROWS, _COPY_START.replace(tzinfo=None)), + ) + setup.execute(sql.SQL("ANALYZE {}").format(table)) + parsed: Final = urlsplit(url) + query: Final = urlencode({**dict(parse_qsl(parsed.query)), "schema": schema}) + database: Final = Prisma(datasource={"url": urlunsplit(parsed._replace(query=query))}) + await database.connect() + try: + yield database + finally: + await database.disconnect() + finally: + setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + + +def _spend_log_rows_scanned(node: Mapping[str, Any]) -> int: + own: Final = ( + (node["Actual Rows"] + node.get("Rows Removed by Filter", 0)) * node["Actual Loops"] + if node.get("Relation Name") == "LiteLLM_SpendLogs" + else 0 + ) + return own + sum(_spend_log_rows_scanned(child) for child in node.get("Plans", ())) + + +@dataclass(frozen=True, slots=True) +class _ExplainedReads: + """Runs each read through Prisma, first under EXPLAIN ANALYZE to count the `LiteLLM_SpendLogs` rows it scans.""" + + database: Prisma + rows_scanned: list[int] + + async def query_raw(self, query: str, *args: object) -> list[dict[str, Any]]: + explained: Final = await self.database.query_raw(f"EXPLAIN (ANALYZE, FORMAT JSON) {query}", *args) + self.rows_scanned.append(_spend_log_rows_scanned(explained[0]["QUERY PLAN"][0]["Plan"])) + return await self.database.query_raw(query, *args) + + +@pytest.mark.parametrize("page, rows_asked_for", [(1, 112), (3, 132)], ids=["first-page", "third-page"]) +async def test_a_newest_first_page_scans_only_the_rows_its_first_read_asks_for(page: int, rows_asked_for: int) -> None: + async with _spend_logs_copy() as database: + rows_scanned: Final[list[int]] = [] + session_page: Final = await read_session_page( + _ExplainedReads(database, rows_scanned), + "\"startTime\" >= ($1::timestamptz AT TIME ZONE 'UTC') AND \"startTime\" <= ($2::timestamptz AT TIME ZONE 'UTC')", + (_COPY_START, _COPY_START + timedelta(days=1)), + "", + page=page, + page_size=10, + sort_desc=True, + session_cursor=None, + cap=100, + ) + newest: Final = _COPIED_ROWS - (page - 1) * 10 + assert [session.key for session in session_page.sessions] == [f"r{n}" for n in range(newest, newest - 10, -1)] + assert (session_page.total, session_page.total_is_capped) == (100, True) + assert rows_scanned == [rows_asked_for] diff --git a/tests/unit/proxy/spend_tracking/test_session_page.py b/tests/unit/proxy/spend_tracking/test_session_page.py new file mode 100644 index 00000000000..dd45e21d6f3 --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_session_page.py @@ -0,0 +1,259 @@ +import random +import re + +import pytest + +from litellm.proxy.spend_tracking.session_page import ( + Session, + _parse_session_cursor, + _Read, + _Request, + _session_page, + read_session_page, +) + +_Row = tuple[int, str, str] # (startTime, session key, api_key) + + +def _ranked(rows: list[_Row], request: _Request) -> list[tuple[int, str, str]]: + """The sessions in ``rows`` in page order, after the request's cursor or offset.""" + last_activity: dict[tuple[str, str], int] = {} + for start, key, api_key in rows: + last_activity[(key, api_key)] = max(last_activity.get((key, api_key), start), start) + ranked = sorted(((at, key, api_key) for (key, api_key), at in last_activity.items()), reverse=request.sort_desc) + if request.cursor is None: + return ranked[request.offset :] + cursor = (int(request.cursor[0]), request.cursor[1], request.cursor[2]) + return [session for session in ranked if (session < cursor if request.sort_desc else session > cursor)] + + +def _reader(rows: list[_Row], request: _Request, budgets: list[int | None], every_row: bool = False): + """Reads like the page query: the newest ``budget`` rows, where a LIMIT cuts tied rows lowest key first, the + opposite of page order, so a session at the read's oldest time can hide a tied session that ranks before it. + With ``every_row`` it ignores the budget, which is what grouping every row returns.""" + newest_first = sorted(rows, key=lambda row: (-row[0], row[1], row[2])) + + async def read(budget: int | None) -> _Read: + budgets.append(budget) + taken = newest_first if budget is None or every_row else newest_first[:budget] + page = _ranked(taken, request)[: request.page_limit + 1] if request.page_limit > 0 else [] + return _Read( + candidates=tuple(Session(key, api_key, f"{at:06d}") for at, key, api_key in page), + last_settled=bool(page) and page[-1][0] > taken[-1][0], + rows_read=len(taken), + session_count=len({(key, api_key) for _, key, api_key in taken}), + read_everything=budget is None or every_row or len(taken) < budget, + ) + + return read + + +def _sessions(rng: random.Random) -> list[_Row]: + """Sessions of one to forty rows over a short clock, so many rows share a time.""" + rows: list[_Row] = [] + clock = rng.choice((3, 40, 400)) + for index in range(rng.randrange(1, 120)): + start, length = rng.randrange(0, clock), rng.choice((1, 1, 2, 3, 8, 40)) + rows.extend((start + rng.randrange(0, 30), f"s{index:03d}", rng.choice(("k1", "k2"))) for _ in range(length)) + return rows + + +def _requests(rng: random.Random, rows: list[_Row]): + for _ in range(12): + cap, page_size, sort_desc = rng.randrange(2, 60), rng.randrange(1, 12), rng.random() < 0.8 + mode = rng.choice(("first", "offset", "cursor")) + full = _ranked(rows, _Request(None, 0, 0, sort_desc, cap)) + cursor = None + if mode == "cursor" and full: + at, key, api_key = full[rng.randrange(len(full))] + cursor = (f"{at:06d}", key, api_key) + offset = (rng.randrange(1, 2 * cap // page_size + 3) - 1) * page_size if mode == "offset" else 0 + yield _Request(cursor, offset, min(page_size, cap - offset), sort_desc, cap) + + +@pytest.mark.parametrize("seed", range(60)) +@pytest.mark.asyncio +async def test_a_page_read_from_the_newest_rows_matches_grouping_every_row(seed): + rng = random.Random(seed) + rows = _sessions(rng) + for request in _requests(rng, rows): + grouped = await _session_page(_reader(rows, request, [], every_row=True), request) + assert await _session_page(_reader(rows, request, []), request) == grouped, request + + +@pytest.mark.parametrize( + "rows_per_session, expected_budgets", + [ + pytest.param(1, [112], id="one-read-holds-the-cap"), + pytest.param(2, [112, 896], id="grows-8x"), + pytest.param(50, [112, 7168], id="skips-8x-when-its-density-cannot-pass-the-cap"), + pytest.param(500, [112, None], id="skips-to-every-row"), + ], +) +@pytest.mark.asyncio +async def test_reads_grow_8x_then_64x_then_every_row_skipping_reads_too_sparse_to_pass_the_cap( + rows_per_session, expected_budgets +): + """A cap of 100 and a page of 10 start at 100 + 10 + 2 rows.""" + rows = [(index, f"s{index // rows_per_session:05d}", "k") for index in range(150 * rows_per_session)] + request = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, cap=100) + budgets: list[int | None] = [] + await _session_page(_reader(rows, request, budgets), request) + assert budgets == expected_budgets + + +@pytest.mark.asyncio +async def test_a_session_tied_with_the_reads_oldest_row_is_not_trusted(): + """Thirty one-row sessions share one time and the first read takes 17 of them. The page's sixth session sits at the + read's oldest time, where the LIMIT may have cut a session that ranks before it, so the page reads on.""" + rows = [(0, f"s{index:02d}", "k") for index in range(30)] + request = _Request(cursor=None, offset=0, page_limit=5, sort_desc=True, cap=10) + budgets: list[int | None] = [] + page = await _session_page(_reader(rows, request, budgets), request) + assert [session.key for session in page.sessions] == ["s29", "s28", "s27", "s26", "s25"] + assert budgets == [17, 136] + + +@pytest.mark.parametrize("session_count, expected_total", [(100, (100, False)), (101, (100, True))]) +@pytest.mark.asyncio +async def test_the_total_is_capped_only_past_the_cap(session_count, expected_total): + rows = [(index, f"s{index:05d}", "k") for index in range(session_count)] + request = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, cap=100) + page = await _session_page(_reader(rows, request, []), request) + assert (page.total, page.total_is_capped) == expected_total + + +@pytest.mark.asyncio +async def test_a_page_number_near_the_bigint_limit_reads_at_most_twice_the_cap(): + rows = [(index, f"s{index:05d}", "k") for index in range(500)] + offset = (9_300_000_000_000_000 - 1) * 1000 + request = _Request(cursor=None, offset=offset, page_limit=min(1000, 100 - offset), sort_desc=True, cap=100) + budgets: list[int | None] = [] + page = await _session_page(_reader(rows, request, budgets), request) + assert budgets == [2 * 100 + 2] + assert (page.sessions, page.total, page.total_is_capped) == ((), 100, True) + + +@pytest.mark.asyncio +async def test_an_ascending_page_reads_every_row_once(): + rows = [(index, f"s{index:05d}", "k") for index in range(500)] + request = _Request(cursor=None, offset=0, page_limit=10, sort_desc=False, cap=100) + budgets: list[int | None] = [] + await _session_page(_reader(rows, request, budgets), request) + assert budgets == [None] + + +def test_parse_session_cursor(): + assert _parse_session_cursor(None) is None + assert _parse_session_cursor("no-delimiter") is None + assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None + assert _parse_session_cursor("|hashed-key|sess-1") is None + assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None + assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == ( + "2026-08-29 10:00:00", + "sess-1", + "hashed-key", + ) + assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == ( + "2026-08-29 10:00:00.123", + "sess|with|pipes", + "hashed-key", + ) + + +def _bound(sql: str, args: tuple[object, ...], pattern: str) -> tuple[object, ...]: + """The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``.""" + found = re.search(pattern, sql) + assert found, f"{pattern!r} not found in:\n{sql}" + assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}" + return tuple(args[int(index) - 1] for index in found.groups()) + + +class _TableDb: + """A table of ``rows`` matching rows in ``sessions`` sessions, whose reads return no page row.""" + + def __init__(self, calls: list[tuple[str, tuple[object, ...]]], rows: int, sessions: int): + self.calls, self.rows, self.sessions = calls, rows, sessions + + async def query_raw(self, query: str, *args: object): + self.calls.append((query, args)) + budget = re.search(r"DESC LIMIT (\d+)", query) + rows_read = min(int(budget.group(1)), self.rows) if budget else self.rows + summary = {"rows_read": rows_read, "session_count": self.sessions} + return [{"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary}] + + +@pytest.mark.asyncio +async def test_a_read_that_fills_its_budget_is_not_taken_for_every_row(): + """A read that took every row its budget allowed may have stopped short, so its 50 sessions are no total: the reads + grow until one takes fewer rows than it asked for.""" + calls: list[tuple[str, tuple[object, ...]]] = [] + page = await read_session_page( + _TableDb(calls, rows=9_000, sessions=50), + "TRUE", + (), + "", + page=1, + page_size=10, + sort_desc=True, + session_cursor=None, + cap=100, + ) + budgets = [re.search(r"DESC LIMIT (\d+)", sql) for sql, _ in calls] + assert [int(found.group(1)) if found else None for found in budgets] == [112, 896, 7168, None] + assert (page.total, page.total_is_capped) == (50, False) + + +@pytest.mark.asyncio +async def test_a_page_starting_at_the_cap_reads_once_and_fetches_no_session(): + calls: list[tuple[str, tuple[object, ...]]] = [] + page = await read_session_page( + _TableDb(calls, rows=1_000, sessions=150), + "TRUE", + (), + "", + page=5, + page_size=25, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert len(calls) == 1, "the first read already holds more sessions than the cap" + assert _bound(*calls[0], r"LIMIT \$(\d+)") == (0,) + assert (page.sessions, page.has_more, page.total, page.total_is_capped) == ((), False, 100, True) + + +@pytest.mark.parametrize( + "cap, expected_budgets", + [ + pytest.param(103, [115, 920, 7360, None], id="13-sessions-per-115-rows-just-reach-104-in-920"), + pytest.param(104, [116, 7424, None], id="13-sessions-per-116-rows-fall-one-short-of-105-in-928"), + ], +) +@pytest.mark.asyncio +async def test_the_density_skip_reads_a_budget_whose_density_exactly_reaches_past_the_cap(cap, expected_budgets): + """A budget is skipped only when its rows, as dense as the last read, would hold fewer than cap + 1 sessions.""" + calls: list[tuple[str, tuple[object, ...]]] = [] + await read_session_page(_TableDb(calls, rows=100_000, sessions=13), "TRUE", (), "", 1, 10, True, None, cap) + budgets = [re.search(r"DESC LIMIT (\d+)", sql) for sql, _ in calls] + assert [int(found.group(1)) if found else None for found in budgets] == expected_budgets + + +@pytest.mark.asyncio +async def test_the_read_binds_filters_cursor_limit_and_offset_to_their_placeholders(): + calls: list[tuple[str, tuple[object, ...]]] = [] + db = _TableDb(calls, rows=50, sessions=40) + where, params = "model = $1 AND api_key = $2", ("gpt-4o", "key-a") + await read_session_page(db, where, params, "", 1, 3, True, "2026-10-28 09:00:00|key-a|sess|9", cap=100) + await read_session_page(db, where, params, "", 3, 1, True, None, cap=100) + (cursor_sql, cursor_args), (offset_sql, offset_args) = calls + for sql, args in calls: + assert _bound(sql, args, r"model = \$(\d+) AND api_key = \$(\d+)") == params + assert "page.last_activity > summary.oldest AS settled" in sql, ( + "a session tied with the read's oldest row may rank after a session the LIMIT cut, so it is not settled" + ) + assert _bound( + cursor_sql, cursor_args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)" + ) == ("2026-10-28 09:00:00", "sess|9", "key-a") + assert _bound(cursor_sql, cursor_args, r"LIMIT \$(\d+)") == (4,) + assert _bound(offset_sql, offset_args, r"LIMIT \$(\d+) OFFSET \$(\d+)") == (2, 2) diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 6598b94e60f..ae382c0dca8 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -7151,28 +7151,6 @@ def test_scoped_spend_report_range_at_max_allowed(client, monkeypatch): app.dependency_overrides.pop(ps.user_api_key_auth, None) -def test_parse_session_cursor(): - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - _parse_session_cursor, - ) - - assert _parse_session_cursor(None) is None - assert _parse_session_cursor("no-delimiter") is None - assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None - assert _parse_session_cursor("|hashed-key|sess-1") is None - assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None - assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == ( - "2026-08-29 10:00:00", - "sess-1", - "hashed-key", - ) - assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == ( - "2026-08-29 10:00:00.123", - "sess|with|pipes", - "hashed-key", - ) - - SESSION_GROUP_KEY_SQL = "COALESCE(NULLIF(session_id, ''), request_id), api_key" @@ -7180,8 +7158,8 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ """Mock prisma dispatching the raw queries the keyset grouped path emits.""" async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": session_total}] + if "rows_read" in sql_query: + return _session_read_rows(session_page_rows, session_total) if "DISTINCT ON" in sql_query: return representative_rows if "COALESCE(SUM(spend)" in sql_query: @@ -7199,7 +7177,7 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ "session_models": ["gpt-4o"], } ] - return session_page_rows + raise AssertionError(f"unexpected query:\n{sql_query}") mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -7228,12 +7206,18 @@ def _session_page_row(session_key, last_activity): return {"session_key": session_key, "api_key": "hashed-key", "last_activity": last_activity} -def _session_grouped_paginating_prisma(sessions, counted_total=None): +def _session_read_rows(page_rows, session_count): + """The read query's rows for ``page_rows``, as if the read held every matching row.""" + summary = {"rows_read": 0, "session_count": session_count} + if not page_rows: + return [{"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary}] + return [{**row, "settled": True, **summary} for row in page_rows] + + +def _session_grouped_paginating_prisma(sessions): """Mock prisma serving the grouped page query out of ``sessions``, honoring the LIMIT and OFFSET it asks for.""" async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": min(len(sessions) if counted_total is None else counted_total, params[-1])}] if "DISTINCT ON" in sql_query: return [_session_representative_row(f"req-{session_key}", session_key) for session_key in params[-2]] if "COALESCE(SUM(spend)" in sql_query: @@ -7241,10 +7225,13 @@ def _session_grouped_paginating_prisma(sessions, counted_total=None): bounds = re.search(r"LIMIT \$(\d+)(?: OFFSET \$(\d+))?", sql_query) limit = params[int(bounds.group(1)) - 1] offset = params[int(bounds.group(2)) - 1] if bounds.group(2) else 0 - return [ - _session_page_row(session_key, last_activity) - for session_key, last_activity in sessions[offset : offset + limit] - ] + return _session_read_rows( + [ + _session_page_row(session_key, last_activity) + for session_key, last_activity in sessions[offset : offset + limit] + ], + len(sessions), + ) mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -7252,6 +7239,29 @@ def _session_grouped_paginating_prisma(sessions, counted_total=None): return mock_prisma +def _bound(sql, args, pattern): + """The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``.""" + found = re.search(pattern, sql) + assert found, f"{pattern!r} not found in:\n{sql}" + assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}" + return tuple(args[int(index) - 1] for index in found.groups()) + + +def _get_session_page(client, monkeypatch, mock_prisma, **params): + """GET the session-grouped logs page over the default window as a proxy admin.""" + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + app.dependency_overrides[ps.user_api_key_auth] = _admin_auth + try: + start_date, end_date = _default_date_range() + return client.get( + "/spend/logs/ui", + params={"start_date": start_date, "end_date": end_date, "group_by_session": "true", **params}, + headers={"Authorization": "Bearer sk-test"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatch): """One row per (session, api_key), session-count total, and a keyset cursor for the next page.""" @@ -7288,6 +7298,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc assert response.status_code == 200, response.text data = response.json() assert data["total"] == 3 + assert data["total_pages"] == 2 assert data["has_more"] is True assert data["next_session_cursor"] == "2026-08-29 09:00:00|hashed-key|req-solo" request_ids = [row["request_id"] for row in data["data"]] @@ -7296,17 +7307,10 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc assert data["data"][1]["session_total_count"] == 1 emitted = [call.args for call in mock_prisma.db.query_raw.await_args_list] - page_query_sql = emitted[0][0] - assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in page_query_sql - assert "OFFSET" not in page_query_sql - assert "HAVING" not in page_query_sql - assert emitted[0][-1] == 3, "page query fetches page_size + 1 sessions to detect has_more" + assert len(emitted) == 3, "one read decides the page and the total, then the representatives and their totals" + assert emitted[0][-1] == 3, "the read fetches page_size + 1 sessions to detect has_more" - count_sql = emitted[1][0] - assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in count_sql - assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" - - rep_call = emitted[2] + rep_call = emitted[1] assert f"DISTINCT ON ({SESSION_GROUP_KEY_SQL})" in rep_call[0] assert ( f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL @@ -7319,7 +7323,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypatch): - """A session_cursor becomes a HAVING keyset predicate instead of an OFFSET.""" + """A session_cursor becomes a keyset predicate on the read's sessions instead of an OFFSET.""" page_rows = [_session_page_row("sess-2", "2026-08-29 07:00:00")] reps = [_session_representative_row("req-2", "sess-2")] mock_prisma = _session_grouped_mock_prisma(page_rows, 3, reps) @@ -7352,7 +7356,7 @@ async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypat page_query_call = mock_prisma.db.query_raw.await_args_list[0] page_query_sql = page_query_call.args[0] - assert f'HAVING (MAX("startTime"), {SESSION_GROUP_KEY_SQL}) <' in page_query_sql + assert "WHERE (last_activity, session_key, api_key) <" in page_query_sql assert "OFFSET" not in page_query_sql cursor_index = page_query_call.args.index("2026-08-29 09:00:00") assert page_query_call.args[cursor_index : cursor_index + 3] == ( @@ -7403,9 +7407,9 @@ async def test_ui_view_spend_logs_group_by_session_jumps_to_page_without_cursor( @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(client, monkeypatch): - """A page that runs out of sessions is the end of the list, so the total comes from it and nothing is counted.""" + """A page that runs out of sessions has read every matching row, so that one read gives the total.""" sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(10)) - mock_prisma = _session_grouped_paginating_prisma(sessions, counted_total=999) + mock_prisma = _session_grouped_paginating_prisma(sessions) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) monkeypatch.setattr( "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", @@ -7429,10 +7433,12 @@ async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(clie ) assert response.status_code == 200, response.text data = response.json() - assert data["total"] == 10, "the count query's 999 would have won if it had been asked" + assert data["total"] == 10 assert data["total_is_capped"] is False assert data["total_pages"] == 1 assert len(data["data"]) == 10 + reads = [call for call in mock_prisma.db.query_raw.await_args_list if "rows_read" in call.args[0]] + assert len(reads) == 1, "a page that ends the list is not read again for its total" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -7517,7 +7523,7 @@ async def test_ui_view_spend_logs_group_by_session_page_past_count_cap_is_empty( assert past_cap_data["data"] == [] assert past_cap_data["has_more"] is False assert past_cap_data["total"] == cap - assert mock_prisma.db.query_raw.await_count == 1, "only the bounded count query runs past the capped window" + assert mock_prisma.db.query_raw.await_count == 1, "only the bounded read runs past the capped window" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -7561,6 +7567,114 @@ async def test_ui_view_spend_logs_group_by_session_last_page_stops_at_the_capped app.dependency_overrides.pop(ps.user_api_key_auth, None) +class _DatetimeWithInjectedIsoformat(datetime.datetime): + def isoformat(self, sep: str = "T", timespec: str = "auto") -> str: + return "2026-01-01'); DROP TABLE x; --" + + +@pytest.mark.parametrize( + "bound", + [ + "2026-01-01'); DROP TABLE x; --", + datetime.datetime(2026, 1, 1), + datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone(datetime.timedelta(hours=5))), + _DatetimeWithInjectedIsoformat(2026, 1, 1, tzinfo=timezone.utc), + datetime.date(2026, 1, 1), + ], + ids=["text", "naive", "not-utc", "datetime-subclass", "date"], +) +def test_utc_timestamp_literal_refuses_anything_but_a_utc_datetime(bound): + with pytest.raises(TypeError): + spend_management_endpoints._utc_timestamp_literal(bound) + + +def test_utc_timestamp_literal_formats_a_utc_datetime(): + literal = spend_management_endpoints._utc_timestamp_literal( + datetime.datetime(2026, 1, 2, 3, 4, 5, 6, tzinfo=timezone.utc) + ) + assert literal == "('2026-01-02T03:04:05.000006+00:00'::timestamptz AT TIME ZONE 'UTC')" + + +@pytest.mark.parametrize("status_filter", ["failure", "success"]) +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_status_reads_outcomes_over_a_literal_window( + client, monkeypatch, status_filter +): + """Reused prepared statements get a generic plan from the 6th run, which takes a $N window for a sliver of the + table and turned the outcome subquery into a ~2 min nested loop, so the window goes in as literals. A failure + filter's sessions are computed once in a CTE, which measured faster only for failure. The outcomes, the read and + the representatives each bind the status, the model and the page's sessions to their own placeholders.""" + page_rows = [_session_page_row("sess-1", "2026-08-29 08:00:00")] + mock_prisma = _session_grouped_mock_prisma(page_rows, 1, [_session_representative_row("req-1", "sess-1")]) + response = _get_session_page(client, monkeypatch, mock_prisma, status_filter=status_filter, model="gpt-4o") + assert response.status_code == 200, response.text + calls = [call.args for call in mock_prisma.db.query_raw.await_args_list] + read_sql, *read_args = next(args for args in calls if "rows_read" in args[0]) + outcomes = read_sql.split("SELECT DISTINCT ON", 1)[1].split("AS session_outcomes", 1)[0] + assert "$1::timestamptz" not in outcomes and "$2::timestamptz" not in outcomes + assert all(f"('{bound.isoformat()}'::timestamptz AT TIME ZONE 'UTC')" in outcomes for bound in read_args[:2]) + assert _bound(outcomes, read_args, r"model = \$(\d+)") == ("gpt-4o",) + outcome_status = r"AS session_outcomes\s+WHERE COALESCE\(status, 'success'\) = \$(\d+)" + assert _bound(read_sql, read_args, outcome_status) == (status_filter,) + assert read_sql.lstrip().startswith("WITH status_sessions AS (") is (status_filter == "failure") + if status_filter == "failure": + assert read_sql.count("FROM status_sessions") == 2, "named once, Postgres 12+ would inline it per worker" + assert "MATERIALIZED" not in read_sql, "Postgres 10 and 11 cannot parse MATERIALIZED" + + rep_sql, *rep_args = next(args for args in calls if "AS session_representatives" in args[0]) + assert _bound(rep_sql, rep_args, r"model = \$(\d+)") == ("gpt-4o",), "representatives keep the row filters" + assert _bound(rep_sql, rep_args, r"session_id = ANY\(\$(\d+)::text\[\]\)") == (["sess-1"],) + assert _bound(rep_sql, rep_args, r"unnest\(\$(\d+)::text\[\], \$(\d+)::text\[\]\)") == (["sess-1"], ["hashed-key"]) + assert _bound(rep_sql.split("AS session_representatives", 1)[1], rep_args, r"= \$(\d+)") == (status_filter,) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_failure_off_the_session_page_stays_inlined(client, monkeypatch): + """Only the session page declares the status_sessions CTE, so a grouped page sorted by another column must inline + the failure subquery or Postgres rejects the statement.""" + calls = [] + + async def mock_query_raw(sql_query, *params): + calls.append((sql_query, params)) + return [{"total_count": 0}] if "total_count" in sql_query else [] + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw) + response = _get_session_page( + client, monkeypatch, mock_prisma, status_filter="failure", sort_by="spend", page=3, page_size=7 + ) + assert response.status_code == 200, response.text + assert calls and all("status_sessions" not in sql for sql, _ in calls) + page_sql, page_args = next(call for call in calls if "AS session_representatives" in call[0]) + assert _bound(page_sql, page_args, r"COALESCE\(status, 'success'\) = \$(\d+)") == ("failure",) + assert _bound(page_sql, page_args, r"LIMIT \$(\d+) OFFSET \$(\d+)") == (7, 14) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_status_drops_a_session_whose_outcome_changed(client, monkeypatch): + """A row written between the page read and the representatives can flip a session's outcome; like the status + filter, the representatives statement then drops the session instead of showing a row of the other status.""" + flipped = {**_session_representative_row("req-new", "sess-1"), "status": "success"} + + async def mock_query_raw(sql_query, *params): + if "rows_read" in sql_query: + return _session_read_rows([_session_page_row("sess-1", "2026-08-29 08:00:00")], 1) + if "AS session_representatives" in sql_query: + check = re.search(r"WHERE COALESCE\(status, 'success'\) = \$(\d+)\s*$", sql_query.strip()) + return [flipped] if check is None or params[int(check.group(1)) - 1] == flipped["status"] else [] + if "COALESCE(SUM(spend)" in sql_query: + return [] + raise AssertionError(f"unexpected query:\n{sql_query}") + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw) + response = _get_session_page(client, monkeypatch, mock_prisma, status_filter="failure") + assert response.status_code == 200, response.text + assert response.json()["data"] == [] + + @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_offset_for_non_starttime_sort(client, monkeypatch): """Sorting by another column keeps session grouping but pages with OFFSET, without a keyset cursor.""" diff --git a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py index 93fae093340..5649d46dbb2 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py @@ -512,16 +512,13 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): With group_by_session=true, /spend/logs/ui must page and count SESSIONS, not raw calls: the page is one representative row per session (DISTINCT ON the session group key, preferring non-MCP calls, newest first), the - bounded count counts groups, and on the default startTime sort the page is + total counts groups, and on the default startTime sort the page is selected by keyset (no OFFSET) so deep pages do not degrade. Otherwise the UI collapses a server page of N calls into fewer visible rows while the footer still claims N (issue #38060). """ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - SPEND_LOGS_PAGINATION_COUNT_CAP, - ui_view_spend_logs, - ) + from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key" session_rows = [ @@ -534,11 +531,11 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): ] async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": 60}] + if "rows_read" in sql_query: + return [{**row, "settled": True, "rows_read": 0, "session_count": 60} for row in session_rows] if "AS session_representatives" in sql_query: return representative_rows - return session_rows + return [] mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -569,23 +566,16 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): emitted = [call[0] for call in mock_prisma.db.query_raw.call_args_list] page_sql = emitted[0][0] - assert f"GROUP BY {group_key}" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}" + assert "GROUP BY session_key, api_key" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}" assert "OFFSET" not in page_sql, "the startTime page must be keyset-selected, not offset-selected" assert emitted[0][-1] == 51, "the page query fetches page_size + 1 sessions to detect has_more" - count_call = emitted[1] - count_sql = count_call[0] - assert f"GROUP BY {group_key}" in count_sql, f"grouped total must count sessions. SQL was:\n{count_sql}" - assert "COUNT(*) OVER ()" not in count_sql - assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" - assert count_call[-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1 - - rep_sql = emitted[2][0] + rep_sql = emitted[1][0] assert f"DISTINCT ON ({group_key})" in rep_sql, f"page must return one row per session. SQL was:\n{rep_sql}" assert ( f"ORDER BY {group_key}, (call_type = 'asend_message') DESC, " "CASE WHEN call_type = 'asend_message' THEN \"endTime\" END DESC NULLS LAST, " - "call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC" + "call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC, request_id" ) in rep_sql, "the session representative must prefer the final agent outcome, then the newest non-MCP call" assert "COUNT(*) OVER ()" not in rep_sql