From b7f17df47d0d30346745811368f950483641ecee Mon Sep 17 00:00:00 2001 From: shrey-berri Date: Fri, 30 Oct 2026 18:08:29 -0700 Subject: [PATCH] fix(spend): read only the newest rows when paging sessions (LIT-8513) The session view grouped every row in the date window to show 50 sessions, then grouped it again to count them, so each page read the whole window twice. On a 108 GiB table that took 45 to 95 s per page A page now reads only the newest rows that match its filters and groups those. Every matching row newer than the oldest row read is in the read, so a session last active after that row has its true last activity and no unread session can rank before it. A read is used once it settles both the page and the capped count, otherwise it is discarded for a read 8x, then 64x larger, then for every row, so results are the same as before. A read that the last read's rows per session say cannot pass the cap is skipped. Cursor and offset pages read from the newest row the same way, with the cursor as a keyset test on the read's sessions, since the capped count needs the newest rows anyway. Ascending order still reads every row in one query The reading lives in its own module, session_page.py, behind one function. The endpoint keeps the filters, the representatives and the response A status filter keeps or drops whole sessions. Its outcome subquery gets the date window as literals, because from the 6th run of a reused prepared statement Postgres plans a $N window as tiny, which turned the subquery into a ~2 min nested loop. A failure filter's sessions are computed once in a CTE that the read names twice, since inlined each parallel worker re-sorted the window for it. Naming it twice keeps Postgres 12+ from inlining it without MATERIALIZED, which Postgres 10 and 11 cannot parse. The representatives skip the subquery and re-check their own outcome, so a session whose outcome changed between the two statements is dropped An integration test seeds sessions in a unique window and walks them by cursor and by offset, both orders, with failure and model filters, on real Postgres. Another runs a page read under EXPLAIN ANALYZE on a seeded copy of the table and checks it scans only the rows its first read asks for --- litellm/proxy/spend_tracking/session_page.py | 212 ++++++++++++++ .../spend_management_endpoints.py | 248 +++++++--------- .../spend/test_session_grouped_logs_pages.py | 276 ++++++++++++++++++ .../proxy/spend_tracking/test_session_page.py | 259 ++++++++++++++++ .../test_spend_management_endpoints.py | 210 ++++++++++--- .../test_spend_query_optimization.py | 26 +- 6 files changed, 1017 insertions(+), 214 deletions(-) create mode 100644 litellm/proxy/spend_tracking/session_page.py create mode 100644 tests/integration/spend/test_session_grouped_logs_pages.py create mode 100644 tests/unit/proxy/spend_tracking/test_session_page.py diff --git a/litellm/proxy/spend_tracking/session_page.py b/litellm/proxy/spend_tracking/session_page.py new file mode 100644 index 00000000000..0b512326c92 --- /dev/null +++ b/litellm/proxy/spend_tracking/session_page.py @@ -0,0 +1,212 @@ +"""Which sessions a logs page shows, and how many sessions its filters match. + +A session is the matching ``LiteLLM_SpendLogs`` rows that share a session key and an api_key, ranked by +``(last_activity, session_key, api_key)``. A descending page reads only the newest matching rows. Every matching row +newer than the oldest row read is in the read, so a session last active after that row has its true last activity and +no session outside the read can rank before it. A read that settles both the page and the capped count is used. One +that does not is discarded for a larger one, so the result is the same as grouping every matching row. +""" + +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from typing import Final, Protocol, TypeAlias + +from typing_extensions import ReadOnly, TypedDict + +SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" + +# Each read takes this many times the first read's rows, ``None`` taking every matching row. +_GROWTH: Final[tuple[int | None, ...]] = (1, 8, 64, None) + + +class _ReadRow(TypedDict): + session_key: ReadOnly[str | None] + api_key: ReadOnly[str | None] + last_activity: ReadOnly[str | None] + settled: ReadOnly[bool | None] + rows_read: ReadOnly[int] + session_count: ReadOnly[int] + + +class _SupportsQueryRaw(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[_ReadRow]: ... + + +@dataclass(frozen=True, slots=True) +class Session: + key: str + api_key: str + last_activity: str + + +@dataclass(frozen=True, slots=True) +class SessionPage: + sessions: tuple[Session, ...] + has_more: bool + next_cursor: str | None + total: int + total_is_capped: bool + + +@dataclass(frozen=True, slots=True) +class _Request: + cursor: tuple[str, str, str] | None + offset: int + page_limit: int + sort_desc: bool + cap: int + + +@dataclass(frozen=True, slots=True) +class _Read: + candidates: tuple[Session, ...] + last_settled: bool + rows_read: int + session_count: int + read_everything: bool + + def falls_short(self, budget: int | None, sessions_needed: int) -> bool: + """Whether ``budget`` newest rows, as dense in sessions as this read, would hold fewer than ``sessions_needed``.""" + return budget is not None and budget * self.session_count < sessions_needed * self.rows_read + + def decides(self, request: _Request) -> bool: + page_settled: Final = ( + request.page_limit <= 0 + or self.read_everything + or (len(self.candidates) > request.page_limit and self.last_settled) + ) + return page_settled and (self.read_everything or self.session_count > request.cap) + + +_Reader: TypeAlias = Callable[[int | None], Awaitable[_Read]] + + +def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None: + if session_cursor is None or session_cursor.count("|") < 2: + return None + last_activity, _, rest = session_cursor.partition("|") + api_key, _, session_key = rest.partition("|") + if not last_activity or not session_key: + return None + return (last_activity, session_key, api_key) + + +async def read_session_page( + db: _SupportsQueryRaw, + where_clause: str, + params: Sequence[object], + ctes: str, + page: int, + page_size: int, + sort_desc: bool, + session_cursor: str | None, + cap: int, +) -> SessionPage: + """The sessions of one page, resumed after ``session_cursor`` or, without one, after ``(page - 1) * page_size`` + sessions. ``where_clause`` reads ``params`` as ``$1`` onward and may name the CTEs that ``ctes`` declares, each + followed by a comma. No page runs past the ``cap`` sessions that the capped total promises.""" + cursor: Final = _parse_session_cursor(session_cursor) + offset: Final = (page - 1) * page_size if cursor is None else 0 + request: Final = _Request(cursor, offset, min(page_size, cap - offset), sort_desc, cap) + return await _session_page(_sql_reader(db, where_clause, params, ctes, request), request) + + +async def _session_page(read: _Reader, request: _Request) -> SessionPage: + # offset + page_limit never passes the cap, so the first read stays bounded however large the page number is. + first_read: Final = request.cap + request.offset + request.page_limit + 2 + budgets: Final = ( + tuple(None if growth is None else first_read * growth for growth in _GROWTH) if request.sort_desc else (None,) + ) + last = await read(budgets[0]) # rebind-ok: each read replaces the last until one decides + for budget in budgets[1:]: + if last.decides(request): + break + if last.falls_short(budget, request.cap + 1): + continue + last = await read(budget) + has_more: Final = len(last.candidates) > request.page_limit + sessions: Final = last.candidates[: request.page_limit] + final: Final = sessions[-1] if has_more and sessions else None + return SessionPage( + sessions=sessions, + has_more=has_more, + next_cursor=f"{final.last_activity}|{final.api_key}|{final.key}" if final else None, + total=min(last.session_count, request.cap), + total_is_capped=last.session_count > request.cap, + ) + + +def _sql_reader( + db: _SupportsQueryRaw, where_clause: str, params: Sequence[object], ctes: str, request: _Request +) -> _Reader: + cursor_index: Final = len(params) + 1 + cursor_params: Final[tuple[object, ...]] = request.cursor or () + limit_index: Final = cursor_index + len(cursor_params) + offset_params: Final[tuple[int, ...]] = (request.offset,) if request.offset and request.page_limit > 0 else () + direction: Final = "DESC" if request.sort_desc else "ASC" + after_cursor: Final = ( + f"WHERE (last_activity, session_key, api_key) {'<' if request.sort_desc else '>'} " + f"(${cursor_index}::timestamp, ${cursor_index + 1}, ${cursor_index + 2})" + if request.cursor + else "" + ) + page_query_args: Final = ( + *params, + *cursor_params, + request.page_limit + 1 if request.page_limit > 0 else 0, + *offset_params, + ) + + async def read(budget: int | None) -> _Read: + # ``scanned`` is read twice, so Postgres computes it once without MATERIALIZED, which Postgres 10 and 11 + # cannot parse. ``summary`` is one row, and the LEFT JOIN keeps it when the page is empty. + rows: Final = await db.query_raw( + f""" + WITH {ctes} scanned AS ( + SELECT {SESSION_KEY_EXPR} AS session_key, api_key, "startTime" + FROM "LiteLLM_SpendLogs" + WHERE {where_clause} + {f'ORDER BY "startTime" DESC LIMIT {budget}' if budget is not None else ""} + ), + sessions AS ( + SELECT session_key, api_key, MAX("startTime") AS last_activity + FROM scanned + GROUP BY session_key, api_key + ), + summary AS ( + SELECT COUNT(*) AS rows_read, + MIN("startTime") AS oldest, + (SELECT COUNT(*) FROM sessions) AS session_count + FROM scanned + ) + SELECT page.session_key, + page.api_key, + page.last_activity::text AS last_activity, + page.last_activity > summary.oldest AS settled, + summary.rows_read, + summary.session_count + FROM summary + LEFT JOIN LATERAL ( + SELECT session_key, api_key, last_activity + FROM sessions + {after_cursor} + ORDER BY last_activity {direction}, session_key {direction}, api_key {direction} + LIMIT ${limit_index} {f"OFFSET ${limit_index + 1}" if offset_params else ""} + ) AS page ON TRUE + """, + *page_query_args, + ) + rows_read: Final = int(rows[0]["rows_read"]) + return _Read( + candidates=tuple( + Session(str(row["session_key"]), str(row["api_key"]), str(row["last_activity"])) + for row in rows + if row["session_key"] is not None + ), + last_settled=bool(rows[-1]["settled"]), + rows_read=rows_read, + session_count=int(rows[0]["session_count"]), + read_everything=budget is None or rows_read < budget, + ) + + return read diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 84278f4619d..8c69708664f 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -51,6 +51,7 @@ from litellm.proxy.auth.authorization_dependencies import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.spend_tracking.session_page import SESSION_KEY_EXPR, read_session_page from litellm.proxy.spend_tracking.spend_capture_rate import ( ProviderBillingCredentialMissing, ProviderBillingRequestFailed, @@ -86,8 +87,7 @@ router: Final = APIRouter() SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000 _DECODED_JSON: Final = TypeAdapter(object) -_SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" -_SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key" +_SESSION_GROUP_KEY_SQL: Final = f"{SESSION_KEY_EXPR}, api_key" _MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" _AGENT_CALL_TYPE_SQL: Final = "'asend_message'" _SESSION_REPRESENTATIVE_ORDER_SQL: Final = ( @@ -2840,12 +2840,14 @@ async def ui_view_spend_logs( # Date range. Wrap the param side with `AT TIME ZONE 'UTC'` so comparison # against the plain `timestamp` column does not depend on the DB session # timezone (see #22529). Absent for a request_id-only lookup (see above). - if start_date_obj is not None and end_date_obj is not None and not search_owns_window: - sql_conditions.append(f"\"startTime\" >= (${p}::timestamptz AT TIME ZONE 'UTC')") - sql_params.append(start_date_obj) - p += 1 - sql_conditions.append(f"\"startTime\" <= (${p}::timestamptz AT TIME ZONE 'UTC')") - sql_params.append(end_date_obj) + window_bounds: Final = ( + ((">=", start_date_obj), ("<=", end_date_obj)) + if start_date_obj is not None and end_date_obj is not None and not search_owns_window + else () + ) + for op, bound in window_bounds: + sql_conditions.append(f"\"startTime\" {op} (${p}::timestamptz AT TIME ZONE 'UTC')") + sql_params.append(bound) p += 1 if search is not None and start_date_obj is not None and end_date_obj is not None: @@ -2945,23 +2947,6 @@ async def ui_view_spend_logs( sql_params.append(json.dumps(used_client_oauth_token)) p += 1 - if status_filter is not None and group_by_session is True and not is_search_lookup: - session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE" - sql_conditions.append( - f"""({_SESSION_GROUP_KEY_SQL}) IN ( - SELECT session_key, api_key FROM ( - SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) - {_SESSION_KEY_EXPR} AS session_key, api_key, status - FROM "LiteLLM_SpendLogs" - WHERE {session_filter_conditions} - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} - ) AS session_outcomes - WHERE COALESCE(status, 'success') = ${p} - )""" - ) - sql_params.append(status_filter) - p += 1 - if ( group_by_session is True and not is_v2 @@ -2973,13 +2958,22 @@ async def ui_view_spend_logs( prisma_client=prisma_client, sql_conditions=sql_conditions, sql_params=sql_params, - next_param_index=p, + window_bounds=window_bounds, + status_filter=status_filter, page=page, page_size=page_size, sort_desc=order_direction != "asc", session_cursor=session_cursor, ) + if status_filter is not None and group_by_session is True and not is_search_lookup: + sql_conditions.append( + # bounded-ok: a subquery, not a list of values + f"({_SESSION_GROUP_KEY_SQL}) IN ({_sessions_with_status_sql(sql_conditions, window_bounds, p)})" + ) + sql_params.append(status_filter) + p += 1 + # Build the ORDER BY expression. ttft_ms is computed from # completionStartTime - startTime; non-streaming rows (where # completionStartTime is null or equals endTime) yield NULL, so we @@ -3077,41 +3071,63 @@ async def ui_view_spend_logs( raise handle_exception_on_proxy(e) -class _SessionPageRow(TypedDict): - session_key: ReadOnly[str] - api_key: ReadOnly[str] - last_activity: ReadOnly[str] +def _utc_timestamp_literal(bound: datetime) -> str: + """SQL literal for a UTC datetime, refusing anything else so no caller text reaches the SQL.""" + if type(bound) is not datetime or bound.tzinfo is not timezone.utc: + raise TypeError(f"expected a UTC datetime, got {type(bound).__name__}") + return f"('{bound.isoformat()}'::timestamptz AT TIME ZONE 'UTC')" -def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None: - if session_cursor is None or session_cursor.count("|") < 2: - return None - last_activity, _, rest = session_cursor.partition("|") - api_key, _, session_key = rest.partition("|") - if not last_activity or not session_key: - return None - return (last_activity, session_key, api_key) +def _sessions_with_status_sql( + sql_conditions: Sequence[str], window_bounds: Sequence[tuple[str, datetime]], status_index: int +) -> str: + """The ``(session_key, api_key)`` of each session whose outcome has the status bound to ``$status_index``. + + The window, which leads ``sql_conditions``, goes in as literals: Prisma reuses prepared statements, and from the + 6th run Postgres plans a ``$N`` window as 0.5% of the table, then probes every session by api_key, ~2 min instead + of ~2.5 s over 700k rows. + """ + window_literals: Final = tuple(f'"startTime" {op} {_utc_timestamp_literal(bound)}' for op, bound in window_bounds) + conditions: Final = " AND ".join((*window_literals, *sql_conditions[len(window_literals) :])) or "TRUE" + return f""" + SELECT session_key, api_key FROM ( + SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) + {SESSION_KEY_EXPR} AS session_key, api_key, status + FROM "LiteLLM_SpendLogs" + WHERE {conditions} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} + ) AS session_outcomes + WHERE COALESCE(status, 'success') = ${status_index} + """ async def _fetch_session_representatives( prisma_client: "PrismaClient", where_clause: str, sql_params: Sequence[object], - next_param_index: int, session_keys: Sequence[tuple[str, str]], + status_filter: str | None, ) -> list[dict[str, object]]: # mutable-ok: _build_ui_spend_logs_response writes session counts onto each row - """Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order.""" + """Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order. + + With ``status_filter``, a session whose outcome no longer has that status is dropped, as the session status + filter would drop it in this statement's snapshot. + """ + keys_index: Final = len(sql_params) + 1 + # The session_id / request_id test finds the rows through those columns' indexes, which the key expression cannot use. rep_query: Final = f""" SELECT * FROM ( SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) {_SPEND_LOG_LIST_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {where_clause} + AND (session_id = ANY(${keys_index}::text[]) OR request_id = ANY(${keys_index}::text[])) AND ({_SESSION_GROUP_KEY_SQL}) IN ( - SELECT * FROM unnest(${next_param_index}::text[], ${next_param_index + 1}::text[]) + SELECT * FROM unnest(${keys_index}::text[], ${keys_index + 1}::text[]) ) ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} ) AS session_representatives + {f"WHERE COALESCE(status, 'success') = ${keys_index + 2}" if status_filter is not None else ""} """ rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place prisma_client, @@ -3119,6 +3135,7 @@ async def _fetch_session_representatives( *sql_params, [session_key for session_key, _ in session_keys], [api_key for _, api_key in session_keys], + *((status_filter,) if status_filter is not None else ()), ) rep_by_key: Final[Mapping[tuple[str, str], dict[str, object]]] = MappingProxyType( # mutable-ok: same rows {(str(row["session_id"] or row["request_id"]), str(row["api_key"])): row for row in rep_rows} @@ -3126,142 +3143,77 @@ async def _fetch_session_representatives( return [rep_by_key[key] for key in session_keys if key in rep_by_key] -async def _count_grouped_sessions( - prisma_client: "PrismaClient", - where_clause: str, - sql_params: Sequence[object], - next_param_index: int, -) -> tuple[int, bool]: - """Count the sessions matching the filter, returning ``(total, total_is_capped)`` bounded by the count cap.""" - count_query: Final = f""" - SELECT COUNT(*) AS total_count - FROM ( - SELECT 1 - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - LIMIT ${next_param_index} - ) AS bounded_sessions - """ - count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_raw( - prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1 - ) - raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0 - return ( - (SPEND_LOGS_PAGINATION_COUNT_CAP, True) if raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP else (raw_total, False) - ) - - async def _ui_session_grouped_spend_logs( prisma_client: "PrismaClient", sql_conditions: Sequence[str], sql_params: Sequence[object], - next_param_index: int, + window_bounds: Sequence[tuple[str, datetime]], + status_filter: str | None, page: int, page_size: int, sort_desc: bool, session_cursor: str | None, ) -> Mapping[str, object]: - """ - One row per session, keyset-paginated by session last activity. - - Sessions are derived on the fly from ``LiteLLM_SpendLogs`` (no extra - table): rows sharing a ``session_id`` and ``api_key`` form a session, rows - without a session id are singletons keyed by ``request_id``. A page is the - next ``page_size`` sessions ordered by ``(MAX(startTime), session_key, - api_key)``, resumed from the ``session_cursor`` keyset - ``'||'`` instead of an OFFSET, so - page depth does not degrade the query plan. A request for ``page > 1`` - without a cursor (the UI jumping straight to the last page, or back to a - page it never walked through) falls back to ``OFFSET (page - 1) * - page_size``, trimmed to the end of the ``SPEND_LOGS_PAGINATION_COUNT_CAP`` - window the capped ``total`` promises, so a page never runs past that total - and one starting at or past it returns no rows without a query. Each session is represented - by its final agent outcome (or newest non-MCP row), enriched by ``_build_ui_spend_logs_response`` - exactly like the flat listing, and the response carries - ``next_session_cursor`` / ``has_more`` while ``total`` counts sessions - (capped like the flat total). A page that runs out of sessions while still - holding some is itself the end of the list, so its ``total`` is - ``offset + len(page)`` and the grouped count query is skipped; a page that - starts past the end says nothing about the total, so that one is counted. - """ - where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE" - cmp_op: Final = "<" if sort_desc else ">" - direction: Final = "DESC" if sort_desc else "ASC" - - cursor: Final = _parse_session_cursor(session_cursor) - having_clause: Final = ( - f'HAVING (MAX("startTime"), {_SESSION_GROUP_KEY_SQL}) {cmp_op} ' - f"(${next_param_index}::timestamp, ${next_param_index + 1}, ${next_param_index + 2})" - if cursor - else "" + """One row per session, keyset-paginated by session last activity: ``read_session_page`` picks the sessions and + counts them, and each session is shown by its final agent outcome (or newest non-MCP row), enriched by + ``_build_ui_spend_logs_response`` exactly like the flat listing.""" + outcomes_sql: Final = ( + None + if status_filter is None + else _sessions_with_status_sql(sql_conditions, window_bounds, status_index=len(sql_params) + 1) ) - cursor_params: Final[tuple[object, ...]] = cursor if cursor else () - limit_index: Final = next_param_index + len(cursor_params) - offset: Final = (page - 1) * page_size if cursor is None else 0 - page_limit: Final = min(page_size, SPEND_LOGS_PAGINATION_COUNT_CAP - offset) - offset_params: Final[tuple[int, ...]] = (offset,) if offset and page_limit > 0 else () - offset_clause: Final = f"OFFSET ${limit_index + 1}" if offset_params else "" - - page_query: Final = f""" - SELECT {_SESSION_KEY_EXPR} AS session_key, - api_key, - MAX("startTime")::text AS last_activity - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - {having_clause} - ORDER BY MAX("startTime") {direction}, {_SESSION_KEY_EXPR} {direction}, api_key {direction} - LIMIT ${limit_index} {offset_clause} - """ - page_rows: Final[Sequence[_SessionPageRow]] = ( + # A failure filter keeps few sessions, so they are computed once in a CTE: inlined, each parallel worker re-sorts + # the whole window for them (3x the blocks and temp on 2M rows). Inlined is faster for a success filter. The CTE is + # named twice so Postgres 12+ keeps it as one computation, as MATERIALIZED would, which Postgres 10 and 11 cannot + # parse. + materialized: Final = outcomes_sql is not None and status_filter == "failure" + status_conditions: Final = ( () - if page_limit <= 0 - else await _query_raw(prisma_client, page_query, *sql_params, *cursor_params, page_limit + 1, *offset_params) + if outcomes_sql is None + else ( + f"EXISTS (SELECT 1 FROM status_sessions) AND ({_SESSION_GROUP_KEY_SQL}) IN " + "(SELECT session_key, api_key FROM status_sessions)" + if materialized + # bounded-ok: a subquery, not a list of values + else f"({_SESSION_GROUP_KEY_SQL}) IN ({outcomes_sql})", + ) ) - - has_more: Final = len(page_rows) > page_limit - visible_rows: Final = page_rows[:page_limit] - next_cursor: Final = ( - f"{visible_rows[-1]['last_activity']}|{visible_rows[-1]['api_key']}|{visible_rows[-1]['session_key']}" - if has_more and visible_rows - else None + session_page: Final = await read_session_page( + db=prisma_client.db, # pyright: ignore[reportArgumentType] # PrismaWrapper forwards raw queries via __getattr__ + where_clause=" AND ".join((*sql_conditions, *status_conditions)) or "TRUE", + params=(*sql_params, *((status_filter,) if status_filter is not None else ())), + ctes=f"status_sessions AS ({outcomes_sql})," if materialized else "", + page=page, + page_size=page_size, + sort_desc=sort_desc, + session_cursor=session_cursor, + cap=SPEND_LOGS_PAGINATION_COUNT_CAP, ) - - page_starts_inside_the_list: Final = offset == 0 or len(page_rows) > 0 - page_ends_the_list: Final = cursor is None and page_limit > 0 and not has_more and page_starts_inside_the_list - total_records, total_is_capped = ( - (offset + len(page_rows), False) - if page_ends_the_list - else await _count_grouped_sessions(prisma_client, where_clause, sql_params, next_param_index) - ) - - session_keys: Final = tuple((row["session_key"], row["api_key"]) for row in visible_rows) + # The status filter keeps whole sessions, so the page's own sessions skip it here; their outcomes are re-checked. data: Final[list[dict[str, object]]] = ( # mutable-ok: _build_ui_spend_logs_response writes onto each row await _fetch_session_representatives( prisma_client=prisma_client, - where_clause=where_clause, + where_clause=" AND ".join(sql_conditions) or "TRUE", sql_params=sql_params, - next_param_index=next_param_index, - session_keys=session_keys, + session_keys=tuple((session.key, session.api_key) for session in session_page.sessions), + status_filter=status_filter, ) - if session_keys + if session_page.sessions else [] ) _hydrate_spend_log_metadata(data) - total_pages: Final = (total_records + page_size - 1) // page_size response: Final[Mapping[str, object]] = await _build_ui_spend_logs_response( prisma_client, data, - total_records, + session_page.total, page, page_size, - total_pages, + (session_page.total + page_size - 1) // page_size, enrich_session_counts=True, - total_is_capped=total_is_capped, + total_is_capped=session_page.total_is_capped, ) - return {**response, "next_session_cursor": next_cursor, "has_more": has_more} + return {**response, "next_session_cursor": session_page.next_cursor, "has_more": session_page.has_more} class RequestResponsePayload(NamedTuple): diff --git a/tests/integration/spend/test_session_grouped_logs_pages.py b/tests/integration/spend/test_session_grouped_logs_pages.py new file mode 100644 index 00000000000..5591d2b7fab --- /dev/null +++ b/tests/integration/spend/test_session_grouped_logs_pages.py @@ -0,0 +1,276 @@ +import json +import os +import uuid +from collections.abc import AsyncIterator, Iterator, Mapping +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any, Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import psycopg +import pytest +from prisma import Prisma +from psycopg import sql +from pydantic import BaseModel, ConfigDict + +from litellm.proxy.spend_tracking.session_page import read_session_page +from tests.integration._support.client import Gateway +from tests.integration._support.database import write_rows + +_URL: Final = "/spend/logs/ui" +_WINDOW_SECONDS: Final = 3600 + + +@dataclass(frozen=True, slots=True) +class _Row: + request: str + session: str | None + api_key: str + second: int + status: str = "success" + model: str = "m-a" + + +_ROWS: Final = ( + _Row("z1", "z", "key-1", -1), + _Row("a1", "a", "key-1", 10), + _Row("a2", "a", "key-1", 50), + _Row("b1", "b", "key-1", 20, model="m-b"), + _Row("b2", "b", "key-1", 60), + _Row("b3", "b", "key-1", 30), + _Row("c1", "c", "key-2", 60), + _Row("d1", "b", "key-2", 40, model="m-b"), + _Row("e1", None, "key-1", 55, model="m-b"), + _Row("f1", "", "key-1", 45), + _Row("g1", "g", "key-3", 5, status="failure"), + _Row("g2", "g", "key-3", 15, status="failure"), + *(_Row(f"h{second - 69}", "h", "key-1", second) for second in range(70, 76)), + _Row("i1", "i", "key-3", 35, status="failure"), + _Row("i2", "i", "key-3", _WINDOW_SECONDS + 1), +) +_NEWEST_FIRST: Final = ( + ("h", "key-1"), + ("c", "key-2"), + ("b", "key-1"), + ("e1", "key-1"), + ("a", "key-1"), + ("f1", "key-1"), + ("b", "key-2"), + ("i", "key-3"), + ("g", "key-3"), +) + + +class _LogRow(BaseModel): + model_config = ConfigDict(frozen=True) + + request_id: str + session_id: str | None + api_key: str + session_total_count: int + + +class _Page(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[_LogRow, ...] + total: int + total_is_capped: bool + has_more: bool + next_session_cursor: str | None + + +@dataclass(frozen=True, slots=True) +class _Seeded: + prefix: str + window: Mapping[str, str] + + def session(self, row: _LogRow) -> tuple[str, str]: + key: Final = row.session_id or row.request_id + assert key.startswith(self.prefix), key + return key.removeprefix(self.prefix), row.api_key + + +@pytest.fixture +def seeded(gateway: Gateway) -> Iterator[_Seeded]: + prefix: Final = f"sess{uuid.uuid4().hex[:12]}-" + start: Final = datetime(1900, 1, 1) + timedelta(days=int(prefix[4:16], 16) % 200000) + rows: Final = [ + { + "request_id": f"{prefix}{row.request}", + "session_id": None if row.session is None else f"{prefix}{row.session}" if row.session else "", + "api_key": row.api_key, + "start_time": (start + timedelta(seconds=row.second)).isoformat(), + "status": row.status, + "model": row.model, + } + for row in _ROWS + ] + with gateway.scenario() as scenario: + scenario.cleanups.callback( + write_rows, 'DELETE FROM "LiteLLM_SpendLogs" WHERE request_id LIKE %s', (f"{prefix}%",) + ) + write_rows( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model, ' + "session_id, status) SELECT request_id, 'acompletion', api_key, start_time, start_time, model, session_id, " + "status FROM jsonb_to_recordset(%s::jsonb) AS seeded(request_id text, session_id text, api_key text, " + "start_time timestamp, status text, model text)", + (json.dumps(rows),), + ) + yield _Seeded( + prefix, + { + "start_date": f"{start:%Y-%m-%d %H:%M:%S}", + "end_date": f"{start + timedelta(seconds=_WINDOW_SECONDS):%Y-%m-%d %H:%M:%S}", + "group_by_session": "true", + }, + ) + + +def _page(gateway: Gateway, seeded: _Seeded, **params: str) -> _Page: + return _Page.model_validate(gateway.get(_URL, {**seeded.window, **params})) + + +def _cursor_walk(gateway: Gateway, seeded: _Seeded, number: int = 1, **params: str) -> tuple[_Page, ...]: + page: Final = _page(gateway, seeded, page_size="2", page=str(number), **params) + if page.next_session_cursor is None or number > len(_NEWEST_FIRST): + return (page,) + following: Final = {**params, "session_cursor": page.next_session_cursor} + return (page, *_cursor_walk(gateway, seeded, number + 1, **following)) + + +def _sessions(seeded: _Seeded, pages: tuple[_Page, ...]) -> tuple[tuple[str, str], ...]: + return tuple(seeded.session(row) for page in pages for row in page.data) + + +def test_cursor_pages_show_every_session_once_newest_first_with_its_newest_row( + gateway: Gateway, seeded: _Seeded +) -> None: + pages: Final = _cursor_walk(gateway, seeded) + assert _sessions(seeded, pages) == _NEWEST_FIRST + assert [(page.total, page.total_is_capped) for page in pages] == [(len(_NEWEST_FIRST), False)] * len(pages) + assert [page.has_more for page in pages] == [True] * (len(pages) - 1) + [False] + representatives: Final = tuple( + (row.request_id.removeprefix(seeded.prefix), row.session_total_count) for page in pages for row in page.data + ) + assert representatives == ( + ("h6", 6), + ("c1", 1), + ("b2", 3), + ("e1", 1), + ("a2", 2), + ("f1", 1), + ("d1", 1), + ("i1", 2), + ("g2", 2), + ) + + +@pytest.mark.parametrize( + "sort_order, expected", + [("desc", _NEWEST_FIRST), ("asc", tuple(reversed(_NEWEST_FIRST)))], + ids=["newest-first", "oldest-first"], +) +def test_numbered_pages_show_the_same_sessions_as_the_cursor_walk( + gateway: Gateway, seeded: _Seeded, sort_order: str, expected: tuple[tuple[str, str], ...] +) -> None: + pages: Final = tuple( + _page(gateway, seeded, page_size="2", page=str(number), sort_order=sort_order) for number in range(1, 7) + ) + assert _sessions(seeded, pages) == expected + assert [len(page.data) for page in pages] == [2, 2, 2, 2, 1, 0] + assert [page.has_more for page in pages] == [True, True, True, True, False, False] + assert {page.total for page in pages} == {len(expected)} + + +def test_failure_filter_keeps_only_sessions_whose_outcome_failed(gateway: Gateway, seeded: _Seeded) -> None: + pages: Final = _cursor_walk(gateway, seeded, status_filter="failure") + assert _sessions(seeded, pages) == (("i", "key-3"), ("g", "key-3")) + assert pages[0].total == 2 + + +def test_model_filter_ranks_each_session_by_its_matching_rows_only(gateway: Gateway, seeded: _Seeded) -> None: + pages: Final = _cursor_walk(gateway, seeded, model="m-b") + assert _sessions(seeded, pages) == (("e1", "key-1"), ("b", "key-2"), ("b", "key-1")) + assert [row.request_id.removeprefix(seeded.prefix) for page in pages for row in page.data] == ["e1", "d1", "b1"] + assert pages[0].total == 3 + + +_COPIED_ROWS: Final = 20_000 +_COPY_START: Final = datetime(2000, 1, 1, tzinfo=UTC) + + +@asynccontextmanager +async def _spend_logs_copy() -> AsyncIterator[Prisma]: + """A private schema holding a copy of the migrated `LiteLLM_SpendLogs` with its indexes, seeded with + `_COPIED_ROWS` one-row sessions a second apart.""" + schema: Final = f"integration_{uuid.uuid4().hex}" + url: Final = os.environ["DATABASE_URL"] + table: Final = sql.Identifier(schema, "LiteLLM_SpendLogs") + with psycopg.connect(url, autocommit=True) as setup: + setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + try: + setup.execute(sql.SQL('CREATE TABLE {} (LIKE "LiteLLM_SpendLogs" INCLUDING ALL)').format(table)) + setup.execute( + sql.SQL( + 'INSERT INTO {} (request_id, call_type, api_key, "startTime", "endTime") ' + "SELECT 'r' || n, 'acompletion', 'key-1', moment, moment FROM generate_series(1, %s) n, " + "LATERAL (SELECT %s::timestamp + n * INTERVAL '1 second' AS moment) seconds" + ).format(table), + (_COPIED_ROWS, _COPY_START.replace(tzinfo=None)), + ) + setup.execute(sql.SQL("ANALYZE {}").format(table)) + parsed: Final = urlsplit(url) + query: Final = urlencode({**dict(parse_qsl(parsed.query)), "schema": schema}) + database: Final = Prisma(datasource={"url": urlunsplit(parsed._replace(query=query))}) + await database.connect() + try: + yield database + finally: + await database.disconnect() + finally: + setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + + +def _spend_log_rows_scanned(node: Mapping[str, Any]) -> int: + own: Final = ( + (node["Actual Rows"] + node.get("Rows Removed by Filter", 0)) * node["Actual Loops"] + if node.get("Relation Name") == "LiteLLM_SpendLogs" + else 0 + ) + return own + sum(_spend_log_rows_scanned(child) for child in node.get("Plans", ())) + + +@dataclass(frozen=True, slots=True) +class _ExplainedReads: + """Runs each read through Prisma, first under EXPLAIN ANALYZE to count the `LiteLLM_SpendLogs` rows it scans.""" + + database: Prisma + rows_scanned: list[int] + + async def query_raw(self, query: str, *args: object) -> list[dict[str, Any]]: + explained: Final = await self.database.query_raw(f"EXPLAIN (ANALYZE, FORMAT JSON) {query}", *args) + self.rows_scanned.append(_spend_log_rows_scanned(explained[0]["QUERY PLAN"][0]["Plan"])) + return await self.database.query_raw(query, *args) + + +@pytest.mark.parametrize("page, rows_asked_for", [(1, 112), (3, 132)], ids=["first-page", "third-page"]) +async def test_a_newest_first_page_scans_only_the_rows_its_first_read_asks_for(page: int, rows_asked_for: int) -> None: + async with _spend_logs_copy() as database: + rows_scanned: Final[list[int]] = [] + session_page: Final = await read_session_page( + _ExplainedReads(database, rows_scanned), + "\"startTime\" >= ($1::timestamptz AT TIME ZONE 'UTC') AND \"startTime\" <= ($2::timestamptz AT TIME ZONE 'UTC')", + (_COPY_START, _COPY_START + timedelta(days=1)), + "", + page=page, + page_size=10, + sort_desc=True, + session_cursor=None, + cap=100, + ) + newest: Final = _COPIED_ROWS - (page - 1) * 10 + assert [session.key for session in session_page.sessions] == [f"r{n}" for n in range(newest, newest - 10, -1)] + assert (session_page.total, session_page.total_is_capped) == (100, True) + assert rows_scanned == [rows_asked_for] diff --git a/tests/unit/proxy/spend_tracking/test_session_page.py b/tests/unit/proxy/spend_tracking/test_session_page.py new file mode 100644 index 00000000000..dd45e21d6f3 --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_session_page.py @@ -0,0 +1,259 @@ +import random +import re + +import pytest + +from litellm.proxy.spend_tracking.session_page import ( + Session, + _parse_session_cursor, + _Read, + _Request, + _session_page, + read_session_page, +) + +_Row = tuple[int, str, str] # (startTime, session key, api_key) + + +def _ranked(rows: list[_Row], request: _Request) -> list[tuple[int, str, str]]: + """The sessions in ``rows`` in page order, after the request's cursor or offset.""" + last_activity: dict[tuple[str, str], int] = {} + for start, key, api_key in rows: + last_activity[(key, api_key)] = max(last_activity.get((key, api_key), start), start) + ranked = sorted(((at, key, api_key) for (key, api_key), at in last_activity.items()), reverse=request.sort_desc) + if request.cursor is None: + return ranked[request.offset :] + cursor = (int(request.cursor[0]), request.cursor[1], request.cursor[2]) + return [session for session in ranked if (session < cursor if request.sort_desc else session > cursor)] + + +def _reader(rows: list[_Row], request: _Request, budgets: list[int | None], every_row: bool = False): + """Reads like the page query: the newest ``budget`` rows, where a LIMIT cuts tied rows lowest key first, the + opposite of page order, so a session at the read's oldest time can hide a tied session that ranks before it. + With ``every_row`` it ignores the budget, which is what grouping every row returns.""" + newest_first = sorted(rows, key=lambda row: (-row[0], row[1], row[2])) + + async def read(budget: int | None) -> _Read: + budgets.append(budget) + taken = newest_first if budget is None or every_row else newest_first[:budget] + page = _ranked(taken, request)[: request.page_limit + 1] if request.page_limit > 0 else [] + return _Read( + candidates=tuple(Session(key, api_key, f"{at:06d}") for at, key, api_key in page), + last_settled=bool(page) and page[-1][0] > taken[-1][0], + rows_read=len(taken), + session_count=len({(key, api_key) for _, key, api_key in taken}), + read_everything=budget is None or every_row or len(taken) < budget, + ) + + return read + + +def _sessions(rng: random.Random) -> list[_Row]: + """Sessions of one to forty rows over a short clock, so many rows share a time.""" + rows: list[_Row] = [] + clock = rng.choice((3, 40, 400)) + for index in range(rng.randrange(1, 120)): + start, length = rng.randrange(0, clock), rng.choice((1, 1, 2, 3, 8, 40)) + rows.extend((start + rng.randrange(0, 30), f"s{index:03d}", rng.choice(("k1", "k2"))) for _ in range(length)) + return rows + + +def _requests(rng: random.Random, rows: list[_Row]): + for _ in range(12): + cap, page_size, sort_desc = rng.randrange(2, 60), rng.randrange(1, 12), rng.random() < 0.8 + mode = rng.choice(("first", "offset", "cursor")) + full = _ranked(rows, _Request(None, 0, 0, sort_desc, cap)) + cursor = None + if mode == "cursor" and full: + at, key, api_key = full[rng.randrange(len(full))] + cursor = (f"{at:06d}", key, api_key) + offset = (rng.randrange(1, 2 * cap // page_size + 3) - 1) * page_size if mode == "offset" else 0 + yield _Request(cursor, offset, min(page_size, cap - offset), sort_desc, cap) + + +@pytest.mark.parametrize("seed", range(60)) +@pytest.mark.asyncio +async def test_a_page_read_from_the_newest_rows_matches_grouping_every_row(seed): + rng = random.Random(seed) + rows = _sessions(rng) + for request in _requests(rng, rows): + grouped = await _session_page(_reader(rows, request, [], every_row=True), request) + assert await _session_page(_reader(rows, request, []), request) == grouped, request + + +@pytest.mark.parametrize( + "rows_per_session, expected_budgets", + [ + pytest.param(1, [112], id="one-read-holds-the-cap"), + pytest.param(2, [112, 896], id="grows-8x"), + pytest.param(50, [112, 7168], id="skips-8x-when-its-density-cannot-pass-the-cap"), + pytest.param(500, [112, None], id="skips-to-every-row"), + ], +) +@pytest.mark.asyncio +async def test_reads_grow_8x_then_64x_then_every_row_skipping_reads_too_sparse_to_pass_the_cap( + rows_per_session, expected_budgets +): + """A cap of 100 and a page of 10 start at 100 + 10 + 2 rows.""" + rows = [(index, f"s{index // rows_per_session:05d}", "k") for index in range(150 * rows_per_session)] + request = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, cap=100) + budgets: list[int | None] = [] + await _session_page(_reader(rows, request, budgets), request) + assert budgets == expected_budgets + + +@pytest.mark.asyncio +async def test_a_session_tied_with_the_reads_oldest_row_is_not_trusted(): + """Thirty one-row sessions share one time and the first read takes 17 of them. The page's sixth session sits at the + read's oldest time, where the LIMIT may have cut a session that ranks before it, so the page reads on.""" + rows = [(0, f"s{index:02d}", "k") for index in range(30)] + request = _Request(cursor=None, offset=0, page_limit=5, sort_desc=True, cap=10) + budgets: list[int | None] = [] + page = await _session_page(_reader(rows, request, budgets), request) + assert [session.key for session in page.sessions] == ["s29", "s28", "s27", "s26", "s25"] + assert budgets == [17, 136] + + +@pytest.mark.parametrize("session_count, expected_total", [(100, (100, False)), (101, (100, True))]) +@pytest.mark.asyncio +async def test_the_total_is_capped_only_past_the_cap(session_count, expected_total): + rows = [(index, f"s{index:05d}", "k") for index in range(session_count)] + request = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, cap=100) + page = await _session_page(_reader(rows, request, []), request) + assert (page.total, page.total_is_capped) == expected_total + + +@pytest.mark.asyncio +async def test_a_page_number_near_the_bigint_limit_reads_at_most_twice_the_cap(): + rows = [(index, f"s{index:05d}", "k") for index in range(500)] + offset = (9_300_000_000_000_000 - 1) * 1000 + request = _Request(cursor=None, offset=offset, page_limit=min(1000, 100 - offset), sort_desc=True, cap=100) + budgets: list[int | None] = [] + page = await _session_page(_reader(rows, request, budgets), request) + assert budgets == [2 * 100 + 2] + assert (page.sessions, page.total, page.total_is_capped) == ((), 100, True) + + +@pytest.mark.asyncio +async def test_an_ascending_page_reads_every_row_once(): + rows = [(index, f"s{index:05d}", "k") for index in range(500)] + request = _Request(cursor=None, offset=0, page_limit=10, sort_desc=False, cap=100) + budgets: list[int | None] = [] + await _session_page(_reader(rows, request, budgets), request) + assert budgets == [None] + + +def test_parse_session_cursor(): + assert _parse_session_cursor(None) is None + assert _parse_session_cursor("no-delimiter") is None + assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None + assert _parse_session_cursor("|hashed-key|sess-1") is None + assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None + assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == ( + "2026-08-29 10:00:00", + "sess-1", + "hashed-key", + ) + assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == ( + "2026-08-29 10:00:00.123", + "sess|with|pipes", + "hashed-key", + ) + + +def _bound(sql: str, args: tuple[object, ...], pattern: str) -> tuple[object, ...]: + """The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``.""" + found = re.search(pattern, sql) + assert found, f"{pattern!r} not found in:\n{sql}" + assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}" + return tuple(args[int(index) - 1] for index in found.groups()) + + +class _TableDb: + """A table of ``rows`` matching rows in ``sessions`` sessions, whose reads return no page row.""" + + def __init__(self, calls: list[tuple[str, tuple[object, ...]]], rows: int, sessions: int): + self.calls, self.rows, self.sessions = calls, rows, sessions + + async def query_raw(self, query: str, *args: object): + self.calls.append((query, args)) + budget = re.search(r"DESC LIMIT (\d+)", query) + rows_read = min(int(budget.group(1)), self.rows) if budget else self.rows + summary = {"rows_read": rows_read, "session_count": self.sessions} + return [{"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary}] + + +@pytest.mark.asyncio +async def test_a_read_that_fills_its_budget_is_not_taken_for_every_row(): + """A read that took every row its budget allowed may have stopped short, so its 50 sessions are no total: the reads + grow until one takes fewer rows than it asked for.""" + calls: list[tuple[str, tuple[object, ...]]] = [] + page = await read_session_page( + _TableDb(calls, rows=9_000, sessions=50), + "TRUE", + (), + "", + page=1, + page_size=10, + sort_desc=True, + session_cursor=None, + cap=100, + ) + budgets = [re.search(r"DESC LIMIT (\d+)", sql) for sql, _ in calls] + assert [int(found.group(1)) if found else None for found in budgets] == [112, 896, 7168, None] + assert (page.total, page.total_is_capped) == (50, False) + + +@pytest.mark.asyncio +async def test_a_page_starting_at_the_cap_reads_once_and_fetches_no_session(): + calls: list[tuple[str, tuple[object, ...]]] = [] + page = await read_session_page( + _TableDb(calls, rows=1_000, sessions=150), + "TRUE", + (), + "", + page=5, + page_size=25, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert len(calls) == 1, "the first read already holds more sessions than the cap" + assert _bound(*calls[0], r"LIMIT \$(\d+)") == (0,) + assert (page.sessions, page.has_more, page.total, page.total_is_capped) == ((), False, 100, True) + + +@pytest.mark.parametrize( + "cap, expected_budgets", + [ + pytest.param(103, [115, 920, 7360, None], id="13-sessions-per-115-rows-just-reach-104-in-920"), + pytest.param(104, [116, 7424, None], id="13-sessions-per-116-rows-fall-one-short-of-105-in-928"), + ], +) +@pytest.mark.asyncio +async def test_the_density_skip_reads_a_budget_whose_density_exactly_reaches_past_the_cap(cap, expected_budgets): + """A budget is skipped only when its rows, as dense as the last read, would hold fewer than cap + 1 sessions.""" + calls: list[tuple[str, tuple[object, ...]]] = [] + await read_session_page(_TableDb(calls, rows=100_000, sessions=13), "TRUE", (), "", 1, 10, True, None, cap) + budgets = [re.search(r"DESC LIMIT (\d+)", sql) for sql, _ in calls] + assert [int(found.group(1)) if found else None for found in budgets] == expected_budgets + + +@pytest.mark.asyncio +async def test_the_read_binds_filters_cursor_limit_and_offset_to_their_placeholders(): + calls: list[tuple[str, tuple[object, ...]]] = [] + db = _TableDb(calls, rows=50, sessions=40) + where, params = "model = $1 AND api_key = $2", ("gpt-4o", "key-a") + await read_session_page(db, where, params, "", 1, 3, True, "2026-10-28 09:00:00|key-a|sess|9", cap=100) + await read_session_page(db, where, params, "", 3, 1, True, None, cap=100) + (cursor_sql, cursor_args), (offset_sql, offset_args) = calls + for sql, args in calls: + assert _bound(sql, args, r"model = \$(\d+) AND api_key = \$(\d+)") == params + assert "page.last_activity > summary.oldest AS settled" in sql, ( + "a session tied with the read's oldest row may rank after a session the LIMIT cut, so it is not settled" + ) + assert _bound( + cursor_sql, cursor_args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)" + ) == ("2026-10-28 09:00:00", "sess|9", "key-a") + assert _bound(cursor_sql, cursor_args, r"LIMIT \$(\d+)") == (4,) + assert _bound(offset_sql, offset_args, r"LIMIT \$(\d+) OFFSET \$(\d+)") == (2, 2) diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 8d53817f51e..253c0875402 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -7151,28 +7151,6 @@ def test_scoped_spend_report_range_at_max_allowed(client, monkeypatch): app.dependency_overrides.pop(ps.user_api_key_auth, None) -def test_parse_session_cursor(): - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - _parse_session_cursor, - ) - - assert _parse_session_cursor(None) is None - assert _parse_session_cursor("no-delimiter") is None - assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None - assert _parse_session_cursor("|hashed-key|sess-1") is None - assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None - assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == ( - "2026-08-29 10:00:00", - "sess-1", - "hashed-key", - ) - assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == ( - "2026-08-29 10:00:00.123", - "sess|with|pipes", - "hashed-key", - ) - - SESSION_GROUP_KEY_SQL = "COALESCE(NULLIF(session_id, ''), request_id), api_key" @@ -7180,8 +7158,8 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ """Mock prisma dispatching the raw queries the keyset grouped path emits.""" async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": session_total}] + if "rows_read" in sql_query: + return _session_read_rows(session_page_rows, session_total) if "DISTINCT ON" in sql_query: return representative_rows if "COALESCE(SUM(spend)" in sql_query: @@ -7199,7 +7177,7 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ "session_models": ["gpt-4o"], } ] - return session_page_rows + raise AssertionError(f"unexpected query:\n{sql_query}") mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -7228,12 +7206,18 @@ def _session_page_row(session_key, last_activity): return {"session_key": session_key, "api_key": "hashed-key", "last_activity": last_activity} -def _session_grouped_paginating_prisma(sessions, counted_total=None): +def _session_read_rows(page_rows, session_count): + """The read query's rows for ``page_rows``, as if the read held every matching row.""" + summary = {"rows_read": 0, "session_count": session_count} + if not page_rows: + return [{"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary}] + return [{**row, "settled": True, **summary} for row in page_rows] + + +def _session_grouped_paginating_prisma(sessions): """Mock prisma serving the grouped page query out of ``sessions``, honoring the LIMIT and OFFSET it asks for.""" async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": min(len(sessions) if counted_total is None else counted_total, params[-1])}] if "DISTINCT ON" in sql_query: return [_session_representative_row(f"req-{session_key}", session_key) for session_key in params[-2]] if "COALESCE(SUM(spend)" in sql_query: @@ -7241,10 +7225,13 @@ def _session_grouped_paginating_prisma(sessions, counted_total=None): bounds = re.search(r"LIMIT \$(\d+)(?: OFFSET \$(\d+))?", sql_query) limit = params[int(bounds.group(1)) - 1] offset = params[int(bounds.group(2)) - 1] if bounds.group(2) else 0 - return [ - _session_page_row(session_key, last_activity) - for session_key, last_activity in sessions[offset : offset + limit] - ] + return _session_read_rows( + [ + _session_page_row(session_key, last_activity) + for session_key, last_activity in sessions[offset : offset + limit] + ], + len(sessions), + ) mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -7252,6 +7239,29 @@ def _session_grouped_paginating_prisma(sessions, counted_total=None): return mock_prisma +def _bound(sql, args, pattern): + """The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``.""" + found = re.search(pattern, sql) + assert found, f"{pattern!r} not found in:\n{sql}" + assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}" + return tuple(args[int(index) - 1] for index in found.groups()) + + +def _get_session_page(client, monkeypatch, mock_prisma, **params): + """GET the session-grouped logs page over the default window as a proxy admin.""" + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + app.dependency_overrides[ps.user_api_key_auth] = _admin_auth + try: + start_date, end_date = _default_date_range() + return client.get( + "/spend/logs/ui", + params={"start_date": start_date, "end_date": end_date, "group_by_session": "true", **params}, + headers={"Authorization": "Bearer sk-test"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatch): """One row per (session, api_key), session-count total, and a keyset cursor for the next page.""" @@ -7288,6 +7298,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc assert response.status_code == 200, response.text data = response.json() assert data["total"] == 3 + assert data["total_pages"] == 2 assert data["has_more"] is True assert data["next_session_cursor"] == "2026-08-29 09:00:00|hashed-key|req-solo" request_ids = [row["request_id"] for row in data["data"]] @@ -7296,17 +7307,10 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc assert data["data"][1]["session_total_count"] == 1 emitted = [call.args for call in mock_prisma.db.query_raw.await_args_list] - page_query_sql = emitted[0][0] - assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in page_query_sql - assert "OFFSET" not in page_query_sql - assert "HAVING" not in page_query_sql - assert emitted[0][-1] == 3, "page query fetches page_size + 1 sessions to detect has_more" + assert len(emitted) == 3, "one read decides the page and the total, then the representatives and their totals" + assert emitted[0][-1] == 3, "the read fetches page_size + 1 sessions to detect has_more" - count_sql = emitted[1][0] - assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in count_sql - assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" - - rep_call = emitted[2] + rep_call = emitted[1] assert f"DISTINCT ON ({SESSION_GROUP_KEY_SQL})" in rep_call[0] assert ( f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL @@ -7319,7 +7323,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypatch): - """A session_cursor becomes a HAVING keyset predicate instead of an OFFSET.""" + """A session_cursor becomes a keyset predicate on the read's sessions instead of an OFFSET.""" page_rows = [_session_page_row("sess-2", "2026-08-29 07:00:00")] reps = [_session_representative_row("req-2", "sess-2")] mock_prisma = _session_grouped_mock_prisma(page_rows, 3, reps) @@ -7352,7 +7356,7 @@ async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypat page_query_call = mock_prisma.db.query_raw.await_args_list[0] page_query_sql = page_query_call.args[0] - assert f'HAVING (MAX("startTime"), {SESSION_GROUP_KEY_SQL}) <' in page_query_sql + assert "WHERE (last_activity, session_key, api_key) <" in page_query_sql assert "OFFSET" not in page_query_sql cursor_index = page_query_call.args.index("2026-08-29 09:00:00") assert page_query_call.args[cursor_index : cursor_index + 3] == ( @@ -7403,9 +7407,9 @@ async def test_ui_view_spend_logs_group_by_session_jumps_to_page_without_cursor( @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(client, monkeypatch): - """A page that runs out of sessions is the end of the list, so the total comes from it and nothing is counted.""" + """A page that runs out of sessions has read every matching row, so that one read gives the total.""" sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(10)) - mock_prisma = _session_grouped_paginating_prisma(sessions, counted_total=999) + mock_prisma = _session_grouped_paginating_prisma(sessions) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) monkeypatch.setattr( "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", @@ -7429,10 +7433,12 @@ async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(clie ) assert response.status_code == 200, response.text data = response.json() - assert data["total"] == 10, "the count query's 999 would have won if it had been asked" + assert data["total"] == 10 assert data["total_is_capped"] is False assert data["total_pages"] == 1 assert len(data["data"]) == 10 + reads = [call for call in mock_prisma.db.query_raw.await_args_list if "rows_read" in call.args[0]] + assert len(reads) == 1, "a page that ends the list is not read again for its total" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -7517,7 +7523,7 @@ async def test_ui_view_spend_logs_group_by_session_page_past_count_cap_is_empty( assert past_cap_data["data"] == [] assert past_cap_data["has_more"] is False assert past_cap_data["total"] == cap - assert mock_prisma.db.query_raw.await_count == 1, "only the bounded count query runs past the capped window" + assert mock_prisma.db.query_raw.await_count == 1, "only the bounded read runs past the capped window" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -7561,6 +7567,114 @@ async def test_ui_view_spend_logs_group_by_session_last_page_stops_at_the_capped app.dependency_overrides.pop(ps.user_api_key_auth, None) +class _DatetimeWithInjectedIsoformat(datetime.datetime): + def isoformat(self, sep: str = "T", timespec: str = "auto") -> str: + return "2026-01-01'); DROP TABLE x; --" + + +@pytest.mark.parametrize( + "bound", + [ + "2026-01-01'); DROP TABLE x; --", + datetime.datetime(2026, 1, 1), + datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone(datetime.timedelta(hours=5))), + _DatetimeWithInjectedIsoformat(2026, 1, 1, tzinfo=timezone.utc), + datetime.date(2026, 1, 1), + ], + ids=["text", "naive", "not-utc", "datetime-subclass", "date"], +) +def test_utc_timestamp_literal_refuses_anything_but_a_utc_datetime(bound): + with pytest.raises(TypeError): + spend_management_endpoints._utc_timestamp_literal(bound) + + +def test_utc_timestamp_literal_formats_a_utc_datetime(): + literal = spend_management_endpoints._utc_timestamp_literal( + datetime.datetime(2026, 1, 2, 3, 4, 5, 6, tzinfo=timezone.utc) + ) + assert literal == "('2026-01-02T03:04:05.000006+00:00'::timestamptz AT TIME ZONE 'UTC')" + + +@pytest.mark.parametrize("status_filter", ["failure", "success"]) +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_status_reads_outcomes_over_a_literal_window( + client, monkeypatch, status_filter +): + """Reused prepared statements get a generic plan from the 6th run, which takes a $N window for a sliver of the + table and turned the outcome subquery into a ~2 min nested loop, so the window goes in as literals. A failure + filter's sessions are computed once in a CTE, which measured faster only for failure. The outcomes, the read and + the representatives each bind the status, the model and the page's sessions to their own placeholders.""" + page_rows = [_session_page_row("sess-1", "2026-08-29 08:00:00")] + mock_prisma = _session_grouped_mock_prisma(page_rows, 1, [_session_representative_row("req-1", "sess-1")]) + response = _get_session_page(client, monkeypatch, mock_prisma, status_filter=status_filter, model="gpt-4o") + assert response.status_code == 200, response.text + calls = [call.args for call in mock_prisma.db.query_raw.await_args_list] + read_sql, *read_args = next(args for args in calls if "rows_read" in args[0]) + outcomes = read_sql.split("SELECT DISTINCT ON", 1)[1].split("AS session_outcomes", 1)[0] + assert "$1::timestamptz" not in outcomes and "$2::timestamptz" not in outcomes + assert all(f"('{bound.isoformat()}'::timestamptz AT TIME ZONE 'UTC')" in outcomes for bound in read_args[:2]) + assert _bound(outcomes, read_args, r"model = \$(\d+)") == ("gpt-4o",) + outcome_status = r"AS session_outcomes\s+WHERE COALESCE\(status, 'success'\) = \$(\d+)" + assert _bound(read_sql, read_args, outcome_status) == (status_filter,) + assert read_sql.lstrip().startswith("WITH status_sessions AS (") is (status_filter == "failure") + if status_filter == "failure": + assert read_sql.count("FROM status_sessions") == 2, "named once, Postgres 12+ would inline it per worker" + assert "MATERIALIZED" not in read_sql, "Postgres 10 and 11 cannot parse MATERIALIZED" + + rep_sql, *rep_args = next(args for args in calls if "AS session_representatives" in args[0]) + assert _bound(rep_sql, rep_args, r"model = \$(\d+)") == ("gpt-4o",), "representatives keep the row filters" + assert _bound(rep_sql, rep_args, r"session_id = ANY\(\$(\d+)::text\[\]\)") == (["sess-1"],) + assert _bound(rep_sql, rep_args, r"unnest\(\$(\d+)::text\[\], \$(\d+)::text\[\]\)") == (["sess-1"], ["hashed-key"]) + assert _bound(rep_sql.split("AS session_representatives", 1)[1], rep_args, r"= \$(\d+)") == (status_filter,) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_failure_off_the_session_page_stays_inlined(client, monkeypatch): + """Only the session page declares the status_sessions CTE, so a grouped page sorted by another column must inline + the failure subquery or Postgres rejects the statement.""" + calls = [] + + async def mock_query_raw(sql_query, *params): + calls.append((sql_query, params)) + return [{"total_count": 0}] if "total_count" in sql_query else [] + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw) + response = _get_session_page( + client, monkeypatch, mock_prisma, status_filter="failure", sort_by="spend", page=3, page_size=7 + ) + assert response.status_code == 200, response.text + assert calls and all("status_sessions" not in sql for sql, _ in calls) + page_sql, page_args = next(call for call in calls if "AS session_representatives" in call[0]) + assert _bound(page_sql, page_args, r"COALESCE\(status, 'success'\) = \$(\d+)") == ("failure",) + assert _bound(page_sql, page_args, r"LIMIT \$(\d+) OFFSET \$(\d+)") == (7, 14) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_status_drops_a_session_whose_outcome_changed(client, monkeypatch): + """A row written between the page read and the representatives can flip a session's outcome; like the status + filter, the representatives statement then drops the session instead of showing a row of the other status.""" + flipped = {**_session_representative_row("req-new", "sess-1"), "status": "success"} + + async def mock_query_raw(sql_query, *params): + if "rows_read" in sql_query: + return _session_read_rows([_session_page_row("sess-1", "2026-08-29 08:00:00")], 1) + if "AS session_representatives" in sql_query: + check = re.search(r"WHERE COALESCE\(status, 'success'\) = \$(\d+)\s*$", sql_query.strip()) + return [flipped] if check is None or params[int(check.group(1)) - 1] == flipped["status"] else [] + if "COALESCE(SUM(spend)" in sql_query: + return [] + raise AssertionError(f"unexpected query:\n{sql_query}") + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw) + response = _get_session_page(client, monkeypatch, mock_prisma, status_filter="failure") + assert response.status_code == 200, response.text + assert response.json()["data"] == [] + + @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_offset_for_non_starttime_sort(client, monkeypatch): """Sorting by another column keeps session grouping but pages with OFFSET, without a keyset cursor.""" diff --git a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py index 93fae093340..5649d46dbb2 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py @@ -512,16 +512,13 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): With group_by_session=true, /spend/logs/ui must page and count SESSIONS, not raw calls: the page is one representative row per session (DISTINCT ON the session group key, preferring non-MCP calls, newest first), the - bounded count counts groups, and on the default startTime sort the page is + total counts groups, and on the default startTime sort the page is selected by keyset (no OFFSET) so deep pages do not degrade. Otherwise the UI collapses a server page of N calls into fewer visible rows while the footer still claims N (issue #38060). """ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - SPEND_LOGS_PAGINATION_COUNT_CAP, - ui_view_spend_logs, - ) + from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key" session_rows = [ @@ -534,11 +531,11 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): ] async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": 60}] + if "rows_read" in sql_query: + return [{**row, "settled": True, "rows_read": 0, "session_count": 60} for row in session_rows] if "AS session_representatives" in sql_query: return representative_rows - return session_rows + return [] mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -569,23 +566,16 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): emitted = [call[0] for call in mock_prisma.db.query_raw.call_args_list] page_sql = emitted[0][0] - assert f"GROUP BY {group_key}" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}" + assert "GROUP BY session_key, api_key" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}" assert "OFFSET" not in page_sql, "the startTime page must be keyset-selected, not offset-selected" assert emitted[0][-1] == 51, "the page query fetches page_size + 1 sessions to detect has_more" - count_call = emitted[1] - count_sql = count_call[0] - assert f"GROUP BY {group_key}" in count_sql, f"grouped total must count sessions. SQL was:\n{count_sql}" - assert "COUNT(*) OVER ()" not in count_sql - assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" - assert count_call[-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1 - - rep_sql = emitted[2][0] + rep_sql = emitted[1][0] assert f"DISTINCT ON ({group_key})" in rep_sql, f"page must return one row per session. SQL was:\n{rep_sql}" assert ( f"ORDER BY {group_key}, (call_type = 'asend_message') DESC, " "CASE WHEN call_type = 'asend_message' THEN \"endTime\" END DESC NULLS LAST, " - "call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC" + "call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC, request_id" ) in rep_sql, "the session representative must prefer the final agent outcome, then the newest non-MCP call" assert "COUNT(*) OVER ()" not in rep_sql