mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge b7f17df47d into 837f36e872
This commit is contained in:
commit
9c002dec5f
6 changed files with 1017 additions and 214 deletions
212
litellm/proxy/spend_tracking/session_page.py
Normal file
212
litellm/proxy/spend_tracking/session_page.py
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
"""Which sessions a logs page shows, and how many sessions its filters match.
|
||||
|
||||
A session is the matching ``LiteLLM_SpendLogs`` rows that share a session key and an api_key, ranked by
|
||||
``(last_activity, session_key, api_key)``. A descending page reads only the newest matching rows. Every matching row
|
||||
newer than the oldest row read is in the read, so a session last active after that row has its true last activity and
|
||||
no session outside the read can rank before it. A read that settles both the page and the capped count is used. One
|
||||
that does not is discarded for a larger one, so the result is the same as grouping every matching row.
|
||||
"""
|
||||
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Protocol, TypeAlias
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)"
|
||||
|
||||
# Each read takes this many times the first read's rows, ``None`` taking every matching row.
|
||||
_GROWTH: Final[tuple[int | None, ...]] = (1, 8, 64, None)
|
||||
|
||||
|
||||
class _ReadRow(TypedDict):
|
||||
session_key: ReadOnly[str | None]
|
||||
api_key: ReadOnly[str | None]
|
||||
last_activity: ReadOnly[str | None]
|
||||
settled: ReadOnly[bool | None]
|
||||
rows_read: ReadOnly[int]
|
||||
session_count: ReadOnly[int]
|
||||
|
||||
|
||||
class _SupportsQueryRaw(Protocol):
|
||||
async def query_raw(self, query: str, *args: object) -> Sequence[_ReadRow]: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Session:
|
||||
key: str
|
||||
api_key: str
|
||||
last_activity: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SessionPage:
|
||||
sessions: tuple[Session, ...]
|
||||
has_more: bool
|
||||
next_cursor: str | None
|
||||
total: int
|
||||
total_is_capped: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Request:
|
||||
cursor: tuple[str, str, str] | None
|
||||
offset: int
|
||||
page_limit: int
|
||||
sort_desc: bool
|
||||
cap: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Read:
|
||||
candidates: tuple[Session, ...]
|
||||
last_settled: bool
|
||||
rows_read: int
|
||||
session_count: int
|
||||
read_everything: bool
|
||||
|
||||
def falls_short(self, budget: int | None, sessions_needed: int) -> bool:
|
||||
"""Whether ``budget`` newest rows, as dense in sessions as this read, would hold fewer than ``sessions_needed``."""
|
||||
return budget is not None and budget * self.session_count < sessions_needed * self.rows_read
|
||||
|
||||
def decides(self, request: _Request) -> bool:
|
||||
page_settled: Final = (
|
||||
request.page_limit <= 0
|
||||
or self.read_everything
|
||||
or (len(self.candidates) > request.page_limit and self.last_settled)
|
||||
)
|
||||
return page_settled and (self.read_everything or self.session_count > request.cap)
|
||||
|
||||
|
||||
_Reader: TypeAlias = Callable[[int | None], Awaitable[_Read]]
|
||||
|
||||
|
||||
def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None:
|
||||
if session_cursor is None or session_cursor.count("|") < 2:
|
||||
return None
|
||||
last_activity, _, rest = session_cursor.partition("|")
|
||||
api_key, _, session_key = rest.partition("|")
|
||||
if not last_activity or not session_key:
|
||||
return None
|
||||
return (last_activity, session_key, api_key)
|
||||
|
||||
|
||||
async def read_session_page(
|
||||
db: _SupportsQueryRaw,
|
||||
where_clause: str,
|
||||
params: Sequence[object],
|
||||
ctes: str,
|
||||
page: int,
|
||||
page_size: int,
|
||||
sort_desc: bool,
|
||||
session_cursor: str | None,
|
||||
cap: int,
|
||||
) -> SessionPage:
|
||||
"""The sessions of one page, resumed after ``session_cursor`` or, without one, after ``(page - 1) * page_size``
|
||||
sessions. ``where_clause`` reads ``params`` as ``$1`` onward and may name the CTEs that ``ctes`` declares, each
|
||||
followed by a comma. No page runs past the ``cap`` sessions that the capped total promises."""
|
||||
cursor: Final = _parse_session_cursor(session_cursor)
|
||||
offset: Final = (page - 1) * page_size if cursor is None else 0
|
||||
request: Final = _Request(cursor, offset, min(page_size, cap - offset), sort_desc, cap)
|
||||
return await _session_page(_sql_reader(db, where_clause, params, ctes, request), request)
|
||||
|
||||
|
||||
async def _session_page(read: _Reader, request: _Request) -> SessionPage:
|
||||
# offset + page_limit never passes the cap, so the first read stays bounded however large the page number is.
|
||||
first_read: Final = request.cap + request.offset + request.page_limit + 2
|
||||
budgets: Final = (
|
||||
tuple(None if growth is None else first_read * growth for growth in _GROWTH) if request.sort_desc else (None,)
|
||||
)
|
||||
last = await read(budgets[0]) # rebind-ok: each read replaces the last until one decides
|
||||
for budget in budgets[1:]:
|
||||
if last.decides(request):
|
||||
break
|
||||
if last.falls_short(budget, request.cap + 1):
|
||||
continue
|
||||
last = await read(budget)
|
||||
has_more: Final = len(last.candidates) > request.page_limit
|
||||
sessions: Final = last.candidates[: request.page_limit]
|
||||
final: Final = sessions[-1] if has_more and sessions else None
|
||||
return SessionPage(
|
||||
sessions=sessions,
|
||||
has_more=has_more,
|
||||
next_cursor=f"{final.last_activity}|{final.api_key}|{final.key}" if final else None,
|
||||
total=min(last.session_count, request.cap),
|
||||
total_is_capped=last.session_count > request.cap,
|
||||
)
|
||||
|
||||
|
||||
def _sql_reader(
|
||||
db: _SupportsQueryRaw, where_clause: str, params: Sequence[object], ctes: str, request: _Request
|
||||
) -> _Reader:
|
||||
cursor_index: Final = len(params) + 1
|
||||
cursor_params: Final[tuple[object, ...]] = request.cursor or ()
|
||||
limit_index: Final = cursor_index + len(cursor_params)
|
||||
offset_params: Final[tuple[int, ...]] = (request.offset,) if request.offset and request.page_limit > 0 else ()
|
||||
direction: Final = "DESC" if request.sort_desc else "ASC"
|
||||
after_cursor: Final = (
|
||||
f"WHERE (last_activity, session_key, api_key) {'<' if request.sort_desc else '>'} "
|
||||
f"(${cursor_index}::timestamp, ${cursor_index + 1}, ${cursor_index + 2})"
|
||||
if request.cursor
|
||||
else ""
|
||||
)
|
||||
page_query_args: Final = (
|
||||
*params,
|
||||
*cursor_params,
|
||||
request.page_limit + 1 if request.page_limit > 0 else 0,
|
||||
*offset_params,
|
||||
)
|
||||
|
||||
async def read(budget: int | None) -> _Read:
|
||||
# ``scanned`` is read twice, so Postgres computes it once without MATERIALIZED, which Postgres 10 and 11
|
||||
# cannot parse. ``summary`` is one row, and the LEFT JOIN keeps it when the page is empty.
|
||||
rows: Final = await db.query_raw(
|
||||
f"""
|
||||
WITH {ctes} scanned AS (
|
||||
SELECT {SESSION_KEY_EXPR} AS session_key, api_key, "startTime"
|
||||
FROM "LiteLLM_SpendLogs"
|
||||
WHERE {where_clause}
|
||||
{f'ORDER BY "startTime" DESC LIMIT {budget}' if budget is not None else ""}
|
||||
),
|
||||
sessions AS (
|
||||
SELECT session_key, api_key, MAX("startTime") AS last_activity
|
||||
FROM scanned
|
||||
GROUP BY session_key, api_key
|
||||
),
|
||||
summary AS (
|
||||
SELECT COUNT(*) AS rows_read,
|
||||
MIN("startTime") AS oldest,
|
||||
(SELECT COUNT(*) FROM sessions) AS session_count
|
||||
FROM scanned
|
||||
)
|
||||
SELECT page.session_key,
|
||||
page.api_key,
|
||||
page.last_activity::text AS last_activity,
|
||||
page.last_activity > summary.oldest AS settled,
|
||||
summary.rows_read,
|
||||
summary.session_count
|
||||
FROM summary
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT session_key, api_key, last_activity
|
||||
FROM sessions
|
||||
{after_cursor}
|
||||
ORDER BY last_activity {direction}, session_key {direction}, api_key {direction}
|
||||
LIMIT ${limit_index} {f"OFFSET ${limit_index + 1}" if offset_params else ""}
|
||||
) AS page ON TRUE
|
||||
""",
|
||||
*page_query_args,
|
||||
)
|
||||
rows_read: Final = int(rows[0]["rows_read"])
|
||||
return _Read(
|
||||
candidates=tuple(
|
||||
Session(str(row["session_key"]), str(row["api_key"]), str(row["last_activity"]))
|
||||
for row in rows
|
||||
if row["session_key"] is not None
|
||||
),
|
||||
last_settled=bool(rows[-1]["settled"]),
|
||||
rows_read=rows_read,
|
||||
session_count=int(rows[0]["session_count"]),
|
||||
read_everything=budget is None or rows_read < budget,
|
||||
)
|
||||
|
||||
return read
|
||||
|
|
@ -52,6 +52,7 @@ from litellm.proxy.auth.authorization_dependencies import (
|
|||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.proxy.spend_tracking.session_page import SESSION_KEY_EXPR, read_session_page
|
||||
from litellm.proxy.spend_tracking.spend_capture_rate import (
|
||||
ProviderBillingCredentialMissing,
|
||||
ProviderBillingRequestFailed,
|
||||
|
|
@ -87,8 +88,7 @@ router: Final = APIRouter()
|
|||
SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
_SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)"
|
||||
_SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key"
|
||||
_SESSION_GROUP_KEY_SQL: Final = f"{SESSION_KEY_EXPR}, api_key"
|
||||
_MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')"
|
||||
_AGENT_CALL_TYPE_SQL: Final = "'asend_message'"
|
||||
_SESSION_REPRESENTATIVE_ORDER_SQL: Final = (
|
||||
|
|
@ -2843,12 +2843,14 @@ async def ui_view_spend_logs(
|
|||
# Date range. Wrap the param side with `AT TIME ZONE 'UTC'` so comparison
|
||||
# against the plain `timestamp` column does not depend on the DB session
|
||||
# timezone (see #22529). Absent for a request_id-only lookup (see above).
|
||||
if start_date_obj is not None and end_date_obj is not None and not search_owns_window:
|
||||
sql_conditions.append(f"\"startTime\" >= (${p}::timestamptz AT TIME ZONE 'UTC')")
|
||||
sql_params.append(start_date_obj)
|
||||
p += 1
|
||||
sql_conditions.append(f"\"startTime\" <= (${p}::timestamptz AT TIME ZONE 'UTC')")
|
||||
sql_params.append(end_date_obj)
|
||||
window_bounds: Final = (
|
||||
((">=", start_date_obj), ("<=", end_date_obj))
|
||||
if start_date_obj is not None and end_date_obj is not None and not search_owns_window
|
||||
else ()
|
||||
)
|
||||
for op, bound in window_bounds:
|
||||
sql_conditions.append(f"\"startTime\" {op} (${p}::timestamptz AT TIME ZONE 'UTC')")
|
||||
sql_params.append(bound)
|
||||
p += 1
|
||||
|
||||
if search is not None and start_date_obj is not None and end_date_obj is not None:
|
||||
|
|
@ -2948,23 +2950,6 @@ async def ui_view_spend_logs(
|
|||
sql_params.append(json.dumps(used_client_oauth_token))
|
||||
p += 1
|
||||
|
||||
if status_filter is not None and group_by_session is True and not is_search_lookup:
|
||||
session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE"
|
||||
sql_conditions.append(
|
||||
f"""({_SESSION_GROUP_KEY_SQL}) IN (
|
||||
SELECT session_key, api_key FROM (
|
||||
SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL})
|
||||
{_SESSION_KEY_EXPR} AS session_key, api_key, status
|
||||
FROM "LiteLLM_SpendLogs"
|
||||
WHERE {session_filter_conditions}
|
||||
ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL}
|
||||
) AS session_outcomes
|
||||
WHERE COALESCE(status, 'success') = ${p}
|
||||
)"""
|
||||
)
|
||||
sql_params.append(status_filter)
|
||||
p += 1
|
||||
|
||||
if (
|
||||
group_by_session is True
|
||||
and not is_v2
|
||||
|
|
@ -2976,13 +2961,22 @@ async def ui_view_spend_logs(
|
|||
prisma_client=prisma_client,
|
||||
sql_conditions=sql_conditions,
|
||||
sql_params=sql_params,
|
||||
next_param_index=p,
|
||||
window_bounds=window_bounds,
|
||||
status_filter=status_filter,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
sort_desc=order_direction != "asc",
|
||||
session_cursor=session_cursor,
|
||||
)
|
||||
|
||||
if status_filter is not None and group_by_session is True and not is_search_lookup:
|
||||
sql_conditions.append(
|
||||
# bounded-ok: a subquery, not a list of values
|
||||
f"({_SESSION_GROUP_KEY_SQL}) IN ({_sessions_with_status_sql(sql_conditions, window_bounds, p)})"
|
||||
)
|
||||
sql_params.append(status_filter)
|
||||
p += 1
|
||||
|
||||
# Build the ORDER BY expression. ttft_ms is computed from
|
||||
# completionStartTime - startTime; non-streaming rows (where
|
||||
# completionStartTime is null or equals endTime) yield NULL, so we
|
||||
|
|
@ -3080,41 +3074,63 @@ async def ui_view_spend_logs(
|
|||
raise handle_exception_on_proxy(e)
|
||||
|
||||
|
||||
class _SessionPageRow(TypedDict):
|
||||
session_key: ReadOnly[str]
|
||||
api_key: ReadOnly[str]
|
||||
last_activity: ReadOnly[str]
|
||||
def _utc_timestamp_literal(bound: datetime) -> str:
|
||||
"""SQL literal for a UTC datetime, refusing anything else so no caller text reaches the SQL."""
|
||||
if type(bound) is not datetime or bound.tzinfo is not timezone.utc:
|
||||
raise TypeError(f"expected a UTC datetime, got {type(bound).__name__}")
|
||||
return f"('{bound.isoformat()}'::timestamptz AT TIME ZONE 'UTC')"
|
||||
|
||||
|
||||
def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None:
|
||||
if session_cursor is None or session_cursor.count("|") < 2:
|
||||
return None
|
||||
last_activity, _, rest = session_cursor.partition("|")
|
||||
api_key, _, session_key = rest.partition("|")
|
||||
if not last_activity or not session_key:
|
||||
return None
|
||||
return (last_activity, session_key, api_key)
|
||||
def _sessions_with_status_sql(
|
||||
sql_conditions: Sequence[str], window_bounds: Sequence[tuple[str, datetime]], status_index: int
|
||||
) -> str:
|
||||
"""The ``(session_key, api_key)`` of each session whose outcome has the status bound to ``$status_index``.
|
||||
|
||||
The window, which leads ``sql_conditions``, goes in as literals: Prisma reuses prepared statements, and from the
|
||||
6th run Postgres plans a ``$N`` window as 0.5% of the table, then probes every session by api_key, ~2 min instead
|
||||
of ~2.5 s over 700k rows.
|
||||
"""
|
||||
window_literals: Final = tuple(f'"startTime" {op} {_utc_timestamp_literal(bound)}' for op, bound in window_bounds)
|
||||
conditions: Final = " AND ".join((*window_literals, *sql_conditions[len(window_literals) :])) or "TRUE"
|
||||
return f"""
|
||||
SELECT session_key, api_key FROM (
|
||||
SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL})
|
||||
{SESSION_KEY_EXPR} AS session_key, api_key, status
|
||||
FROM "LiteLLM_SpendLogs"
|
||||
WHERE {conditions}
|
||||
ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL}
|
||||
) AS session_outcomes
|
||||
WHERE COALESCE(status, 'success') = ${status_index}
|
||||
"""
|
||||
|
||||
|
||||
async def _fetch_session_representatives(
|
||||
prisma_client: "PrismaClient",
|
||||
where_clause: str,
|
||||
sql_params: Sequence[object],
|
||||
next_param_index: int,
|
||||
session_keys: Sequence[tuple[str, str]],
|
||||
status_filter: str | None,
|
||||
) -> list[dict[str, object]]: # mutable-ok: _build_ui_spend_logs_response writes session counts onto each row
|
||||
"""Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order."""
|
||||
"""Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order.
|
||||
|
||||
With ``status_filter``, a session whose outcome no longer has that status is dropped, as the session status
|
||||
filter would drop it in this statement's snapshot.
|
||||
"""
|
||||
keys_index: Final = len(sql_params) + 1
|
||||
# The session_id / request_id test finds the rows through those columns' indexes, which the key expression cannot use.
|
||||
rep_query: Final = f"""
|
||||
SELECT * FROM (
|
||||
SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL})
|
||||
{_SPEND_LOG_LIST_COLUMNS}
|
||||
FROM "LiteLLM_SpendLogs"
|
||||
WHERE {where_clause}
|
||||
AND (session_id = ANY(${keys_index}::text[]) OR request_id = ANY(${keys_index}::text[]))
|
||||
AND ({_SESSION_GROUP_KEY_SQL}) IN (
|
||||
SELECT * FROM unnest(${next_param_index}::text[], ${next_param_index + 1}::text[])
|
||||
SELECT * FROM unnest(${keys_index}::text[], ${keys_index + 1}::text[])
|
||||
)
|
||||
ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL}
|
||||
) AS session_representatives
|
||||
{f"WHERE COALESCE(status, 'success') = ${keys_index + 2}" if status_filter is not None else ""}
|
||||
"""
|
||||
rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place
|
||||
prisma_client,
|
||||
|
|
@ -3122,6 +3138,7 @@ async def _fetch_session_representatives(
|
|||
*sql_params,
|
||||
[session_key for session_key, _ in session_keys],
|
||||
[api_key for _, api_key in session_keys],
|
||||
*((status_filter,) if status_filter is not None else ()),
|
||||
)
|
||||
rep_by_key: Final[Mapping[tuple[str, str], dict[str, object]]] = MappingProxyType( # mutable-ok: same rows
|
||||
{(str(row["session_id"] or row["request_id"]), str(row["api_key"])): row for row in rep_rows}
|
||||
|
|
@ -3129,142 +3146,77 @@ async def _fetch_session_representatives(
|
|||
return [rep_by_key[key] for key in session_keys if key in rep_by_key]
|
||||
|
||||
|
||||
async def _count_grouped_sessions(
|
||||
prisma_client: "PrismaClient",
|
||||
where_clause: str,
|
||||
sql_params: Sequence[object],
|
||||
next_param_index: int,
|
||||
) -> tuple[int, bool]:
|
||||
"""Count the sessions matching the filter, returning ``(total, total_is_capped)`` bounded by the count cap."""
|
||||
count_query: Final = f"""
|
||||
SELECT COUNT(*) AS total_count
|
||||
FROM (
|
||||
SELECT 1
|
||||
FROM "LiteLLM_SpendLogs"
|
||||
WHERE {where_clause}
|
||||
GROUP BY {_SESSION_GROUP_KEY_SQL}
|
||||
LIMIT ${next_param_index}
|
||||
) AS bounded_sessions
|
||||
"""
|
||||
count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_raw(
|
||||
prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1
|
||||
)
|
||||
raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0
|
||||
return (
|
||||
(SPEND_LOGS_PAGINATION_COUNT_CAP, True) if raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP else (raw_total, False)
|
||||
)
|
||||
|
||||
|
||||
async def _ui_session_grouped_spend_logs(
|
||||
prisma_client: "PrismaClient",
|
||||
sql_conditions: Sequence[str],
|
||||
sql_params: Sequence[object],
|
||||
next_param_index: int,
|
||||
window_bounds: Sequence[tuple[str, datetime]],
|
||||
status_filter: str | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
sort_desc: bool,
|
||||
session_cursor: str | None,
|
||||
) -> Mapping[str, object]:
|
||||
"""
|
||||
One row per session, keyset-paginated by session last activity.
|
||||
|
||||
Sessions are derived on the fly from ``LiteLLM_SpendLogs`` (no extra
|
||||
table): rows sharing a ``session_id`` and ``api_key`` form a session, rows
|
||||
without a session id are singletons keyed by ``request_id``. A page is the
|
||||
next ``page_size`` sessions ordered by ``(MAX(startTime), session_key,
|
||||
api_key)``, resumed from the ``session_cursor`` keyset
|
||||
``'<last_activity>|<api_key>|<session_key>'`` instead of an OFFSET, so
|
||||
page depth does not degrade the query plan. A request for ``page > 1``
|
||||
without a cursor (the UI jumping straight to the last page, or back to a
|
||||
page it never walked through) falls back to ``OFFSET (page - 1) *
|
||||
page_size``, trimmed to the end of the ``SPEND_LOGS_PAGINATION_COUNT_CAP``
|
||||
window the capped ``total`` promises, so a page never runs past that total
|
||||
and one starting at or past it returns no rows without a query. Each session is represented
|
||||
by its final agent outcome (or newest non-MCP row), enriched by ``_build_ui_spend_logs_response``
|
||||
exactly like the flat listing, and the response carries
|
||||
``next_session_cursor`` / ``has_more`` while ``total`` counts sessions
|
||||
(capped like the flat total). A page that runs out of sessions while still
|
||||
holding some is itself the end of the list, so its ``total`` is
|
||||
``offset + len(page)`` and the grouped count query is skipped; a page that
|
||||
starts past the end says nothing about the total, so that one is counted.
|
||||
"""
|
||||
where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE"
|
||||
cmp_op: Final = "<" if sort_desc else ">"
|
||||
direction: Final = "DESC" if sort_desc else "ASC"
|
||||
|
||||
cursor: Final = _parse_session_cursor(session_cursor)
|
||||
having_clause: Final = (
|
||||
f'HAVING (MAX("startTime"), {_SESSION_GROUP_KEY_SQL}) {cmp_op} '
|
||||
f"(${next_param_index}::timestamp, ${next_param_index + 1}, ${next_param_index + 2})"
|
||||
if cursor
|
||||
else ""
|
||||
"""One row per session, keyset-paginated by session last activity: ``read_session_page`` picks the sessions and
|
||||
counts them, and each session is shown by its final agent outcome (or newest non-MCP row), enriched by
|
||||
``_build_ui_spend_logs_response`` exactly like the flat listing."""
|
||||
outcomes_sql: Final = (
|
||||
None
|
||||
if status_filter is None
|
||||
else _sessions_with_status_sql(sql_conditions, window_bounds, status_index=len(sql_params) + 1)
|
||||
)
|
||||
cursor_params: Final[tuple[object, ...]] = cursor if cursor else ()
|
||||
limit_index: Final = next_param_index + len(cursor_params)
|
||||
offset: Final = (page - 1) * page_size if cursor is None else 0
|
||||
page_limit: Final = min(page_size, SPEND_LOGS_PAGINATION_COUNT_CAP - offset)
|
||||
offset_params: Final[tuple[int, ...]] = (offset,) if offset and page_limit > 0 else ()
|
||||
offset_clause: Final = f"OFFSET ${limit_index + 1}" if offset_params else ""
|
||||
|
||||
page_query: Final = f"""
|
||||
SELECT {_SESSION_KEY_EXPR} AS session_key,
|
||||
api_key,
|
||||
MAX("startTime")::text AS last_activity
|
||||
FROM "LiteLLM_SpendLogs"
|
||||
WHERE {where_clause}
|
||||
GROUP BY {_SESSION_GROUP_KEY_SQL}
|
||||
{having_clause}
|
||||
ORDER BY MAX("startTime") {direction}, {_SESSION_KEY_EXPR} {direction}, api_key {direction}
|
||||
LIMIT ${limit_index} {offset_clause}
|
||||
"""
|
||||
page_rows: Final[Sequence[_SessionPageRow]] = (
|
||||
# A failure filter keeps few sessions, so they are computed once in a CTE: inlined, each parallel worker re-sorts
|
||||
# the whole window for them (3x the blocks and temp on 2M rows). Inlined is faster for a success filter. The CTE is
|
||||
# named twice so Postgres 12+ keeps it as one computation, as MATERIALIZED would, which Postgres 10 and 11 cannot
|
||||
# parse.
|
||||
materialized: Final = outcomes_sql is not None and status_filter == "failure"
|
||||
status_conditions: Final = (
|
||||
()
|
||||
if page_limit <= 0
|
||||
else await _query_raw(prisma_client, page_query, *sql_params, *cursor_params, page_limit + 1, *offset_params)
|
||||
if outcomes_sql is None
|
||||
else (
|
||||
f"EXISTS (SELECT 1 FROM status_sessions) AND ({_SESSION_GROUP_KEY_SQL}) IN "
|
||||
"(SELECT session_key, api_key FROM status_sessions)"
|
||||
if materialized
|
||||
# bounded-ok: a subquery, not a list of values
|
||||
else f"({_SESSION_GROUP_KEY_SQL}) IN ({outcomes_sql})",
|
||||
)
|
||||
)
|
||||
|
||||
has_more: Final = len(page_rows) > page_limit
|
||||
visible_rows: Final = page_rows[:page_limit]
|
||||
next_cursor: Final = (
|
||||
f"{visible_rows[-1]['last_activity']}|{visible_rows[-1]['api_key']}|{visible_rows[-1]['session_key']}"
|
||||
if has_more and visible_rows
|
||||
else None
|
||||
session_page: Final = await read_session_page(
|
||||
db=prisma_client.db, # pyright: ignore[reportArgumentType] # PrismaWrapper forwards raw queries via __getattr__
|
||||
where_clause=" AND ".join((*sql_conditions, *status_conditions)) or "TRUE",
|
||||
params=(*sql_params, *((status_filter,) if status_filter is not None else ())),
|
||||
ctes=f"status_sessions AS ({outcomes_sql})," if materialized else "",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
sort_desc=sort_desc,
|
||||
session_cursor=session_cursor,
|
||||
cap=SPEND_LOGS_PAGINATION_COUNT_CAP,
|
||||
)
|
||||
|
||||
page_starts_inside_the_list: Final = offset == 0 or len(page_rows) > 0
|
||||
page_ends_the_list: Final = cursor is None and page_limit > 0 and not has_more and page_starts_inside_the_list
|
||||
total_records, total_is_capped = (
|
||||
(offset + len(page_rows), False)
|
||||
if page_ends_the_list
|
||||
else await _count_grouped_sessions(prisma_client, where_clause, sql_params, next_param_index)
|
||||
)
|
||||
|
||||
session_keys: Final = tuple((row["session_key"], row["api_key"]) for row in visible_rows)
|
||||
# The status filter keeps whole sessions, so the page's own sessions skip it here; their outcomes are re-checked.
|
||||
data: Final[list[dict[str, object]]] = ( # mutable-ok: _build_ui_spend_logs_response writes onto each row
|
||||
await _fetch_session_representatives(
|
||||
prisma_client=prisma_client,
|
||||
where_clause=where_clause,
|
||||
where_clause=" AND ".join(sql_conditions) or "TRUE",
|
||||
sql_params=sql_params,
|
||||
next_param_index=next_param_index,
|
||||
session_keys=session_keys,
|
||||
session_keys=tuple((session.key, session.api_key) for session in session_page.sessions),
|
||||
status_filter=status_filter,
|
||||
)
|
||||
if session_keys
|
||||
if session_page.sessions
|
||||
else []
|
||||
)
|
||||
_hydrate_spend_log_metadata(data)
|
||||
|
||||
total_pages: Final = (total_records + page_size - 1) // page_size
|
||||
response: Final[Mapping[str, object]] = await _build_ui_spend_logs_response(
|
||||
prisma_client,
|
||||
data,
|
||||
total_records,
|
||||
session_page.total,
|
||||
page,
|
||||
page_size,
|
||||
total_pages,
|
||||
(session_page.total + page_size - 1) // page_size,
|
||||
enrich_session_counts=True,
|
||||
total_is_capped=total_is_capped,
|
||||
total_is_capped=session_page.total_is_capped,
|
||||
)
|
||||
return {**response, "next_session_cursor": next_cursor, "has_more": has_more}
|
||||
return {**response, "next_session_cursor": session_page.next_cursor, "has_more": session_page.has_more}
|
||||
|
||||
|
||||
class RequestResponsePayload(NamedTuple):
|
||||
|
|
|
|||
276
tests/integration/spend/test_session_grouped_logs_pages.py
Normal file
276
tests/integration/spend/test_session_grouped_logs_pages.py
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
import json
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any, Final
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from prisma import Prisma
|
||||
from psycopg import sql
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from litellm.proxy.spend_tracking.session_page import read_session_page
|
||||
from tests.integration._support.client import Gateway
|
||||
from tests.integration._support.database import write_rows
|
||||
|
||||
_URL: Final = "/spend/logs/ui"
|
||||
_WINDOW_SECONDS: Final = 3600
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Row:
|
||||
request: str
|
||||
session: str | None
|
||||
api_key: str
|
||||
second: int
|
||||
status: str = "success"
|
||||
model: str = "m-a"
|
||||
|
||||
|
||||
_ROWS: Final = (
|
||||
_Row("z1", "z", "key-1", -1),
|
||||
_Row("a1", "a", "key-1", 10),
|
||||
_Row("a2", "a", "key-1", 50),
|
||||
_Row("b1", "b", "key-1", 20, model="m-b"),
|
||||
_Row("b2", "b", "key-1", 60),
|
||||
_Row("b3", "b", "key-1", 30),
|
||||
_Row("c1", "c", "key-2", 60),
|
||||
_Row("d1", "b", "key-2", 40, model="m-b"),
|
||||
_Row("e1", None, "key-1", 55, model="m-b"),
|
||||
_Row("f1", "", "key-1", 45),
|
||||
_Row("g1", "g", "key-3", 5, status="failure"),
|
||||
_Row("g2", "g", "key-3", 15, status="failure"),
|
||||
*(_Row(f"h{second - 69}", "h", "key-1", second) for second in range(70, 76)),
|
||||
_Row("i1", "i", "key-3", 35, status="failure"),
|
||||
_Row("i2", "i", "key-3", _WINDOW_SECONDS + 1),
|
||||
)
|
||||
_NEWEST_FIRST: Final = (
|
||||
("h", "key-1"),
|
||||
("c", "key-2"),
|
||||
("b", "key-1"),
|
||||
("e1", "key-1"),
|
||||
("a", "key-1"),
|
||||
("f1", "key-1"),
|
||||
("b", "key-2"),
|
||||
("i", "key-3"),
|
||||
("g", "key-3"),
|
||||
)
|
||||
|
||||
|
||||
class _LogRow(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
request_id: str
|
||||
session_id: str | None
|
||||
api_key: str
|
||||
session_total_count: int
|
||||
|
||||
|
||||
class _Page(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
data: tuple[_LogRow, ...]
|
||||
total: int
|
||||
total_is_capped: bool
|
||||
has_more: bool
|
||||
next_session_cursor: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Seeded:
|
||||
prefix: str
|
||||
window: Mapping[str, str]
|
||||
|
||||
def session(self, row: _LogRow) -> tuple[str, str]:
|
||||
key: Final = row.session_id or row.request_id
|
||||
assert key.startswith(self.prefix), key
|
||||
return key.removeprefix(self.prefix), row.api_key
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def seeded(gateway: Gateway) -> Iterator[_Seeded]:
|
||||
prefix: Final = f"sess{uuid.uuid4().hex[:12]}-"
|
||||
start: Final = datetime(1900, 1, 1) + timedelta(days=int(prefix[4:16], 16) % 200000)
|
||||
rows: Final = [
|
||||
{
|
||||
"request_id": f"{prefix}{row.request}",
|
||||
"session_id": None if row.session is None else f"{prefix}{row.session}" if row.session else "",
|
||||
"api_key": row.api_key,
|
||||
"start_time": (start + timedelta(seconds=row.second)).isoformat(),
|
||||
"status": row.status,
|
||||
"model": row.model,
|
||||
}
|
||||
for row in _ROWS
|
||||
]
|
||||
with gateway.scenario() as scenario:
|
||||
scenario.cleanups.callback(
|
||||
write_rows, 'DELETE FROM "LiteLLM_SpendLogs" WHERE request_id LIKE %s', (f"{prefix}%",)
|
||||
)
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model, '
|
||||
"session_id, status) SELECT request_id, 'acompletion', api_key, start_time, start_time, model, session_id, "
|
||||
"status FROM jsonb_to_recordset(%s::jsonb) AS seeded(request_id text, session_id text, api_key text, "
|
||||
"start_time timestamp, status text, model text)",
|
||||
(json.dumps(rows),),
|
||||
)
|
||||
yield _Seeded(
|
||||
prefix,
|
||||
{
|
||||
"start_date": f"{start:%Y-%m-%d %H:%M:%S}",
|
||||
"end_date": f"{start + timedelta(seconds=_WINDOW_SECONDS):%Y-%m-%d %H:%M:%S}",
|
||||
"group_by_session": "true",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _page(gateway: Gateway, seeded: _Seeded, **params: str) -> _Page:
|
||||
return _Page.model_validate(gateway.get(_URL, {**seeded.window, **params}))
|
||||
|
||||
|
||||
def _cursor_walk(gateway: Gateway, seeded: _Seeded, number: int = 1, **params: str) -> tuple[_Page, ...]:
|
||||
page: Final = _page(gateway, seeded, page_size="2", page=str(number), **params)
|
||||
if page.next_session_cursor is None or number > len(_NEWEST_FIRST):
|
||||
return (page,)
|
||||
following: Final = {**params, "session_cursor": page.next_session_cursor}
|
||||
return (page, *_cursor_walk(gateway, seeded, number + 1, **following))
|
||||
|
||||
|
||||
def _sessions(seeded: _Seeded, pages: tuple[_Page, ...]) -> tuple[tuple[str, str], ...]:
|
||||
return tuple(seeded.session(row) for page in pages for row in page.data)
|
||||
|
||||
|
||||
def test_cursor_pages_show_every_session_once_newest_first_with_its_newest_row(
|
||||
gateway: Gateway, seeded: _Seeded
|
||||
) -> None:
|
||||
pages: Final = _cursor_walk(gateway, seeded)
|
||||
assert _sessions(seeded, pages) == _NEWEST_FIRST
|
||||
assert [(page.total, page.total_is_capped) for page in pages] == [(len(_NEWEST_FIRST), False)] * len(pages)
|
||||
assert [page.has_more for page in pages] == [True] * (len(pages) - 1) + [False]
|
||||
representatives: Final = tuple(
|
||||
(row.request_id.removeprefix(seeded.prefix), row.session_total_count) for page in pages for row in page.data
|
||||
)
|
||||
assert representatives == (
|
||||
("h6", 6),
|
||||
("c1", 1),
|
||||
("b2", 3),
|
||||
("e1", 1),
|
||||
("a2", 2),
|
||||
("f1", 1),
|
||||
("d1", 1),
|
||||
("i1", 2),
|
||||
("g2", 2),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sort_order, expected",
|
||||
[("desc", _NEWEST_FIRST), ("asc", tuple(reversed(_NEWEST_FIRST)))],
|
||||
ids=["newest-first", "oldest-first"],
|
||||
)
|
||||
def test_numbered_pages_show_the_same_sessions_as_the_cursor_walk(
|
||||
gateway: Gateway, seeded: _Seeded, sort_order: str, expected: tuple[tuple[str, str], ...]
|
||||
) -> None:
|
||||
pages: Final = tuple(
|
||||
_page(gateway, seeded, page_size="2", page=str(number), sort_order=sort_order) for number in range(1, 7)
|
||||
)
|
||||
assert _sessions(seeded, pages) == expected
|
||||
assert [len(page.data) for page in pages] == [2, 2, 2, 2, 1, 0]
|
||||
assert [page.has_more for page in pages] == [True, True, True, True, False, False]
|
||||
assert {page.total for page in pages} == {len(expected)}
|
||||
|
||||
|
||||
def test_failure_filter_keeps_only_sessions_whose_outcome_failed(gateway: Gateway, seeded: _Seeded) -> None:
|
||||
pages: Final = _cursor_walk(gateway, seeded, status_filter="failure")
|
||||
assert _sessions(seeded, pages) == (("i", "key-3"), ("g", "key-3"))
|
||||
assert pages[0].total == 2
|
||||
|
||||
|
||||
def test_model_filter_ranks_each_session_by_its_matching_rows_only(gateway: Gateway, seeded: _Seeded) -> None:
|
||||
pages: Final = _cursor_walk(gateway, seeded, model="m-b")
|
||||
assert _sessions(seeded, pages) == (("e1", "key-1"), ("b", "key-2"), ("b", "key-1"))
|
||||
assert [row.request_id.removeprefix(seeded.prefix) for page in pages for row in page.data] == ["e1", "d1", "b1"]
|
||||
assert pages[0].total == 3
|
||||
|
||||
|
||||
_COPIED_ROWS: Final = 20_000
|
||||
_COPY_START: Final = datetime(2000, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _spend_logs_copy() -> AsyncIterator[Prisma]:
|
||||
"""A private schema holding a copy of the migrated `LiteLLM_SpendLogs` with its indexes, seeded with
|
||||
`_COPIED_ROWS` one-row sessions a second apart."""
|
||||
schema: Final = f"integration_{uuid.uuid4().hex}"
|
||||
url: Final = os.environ["DATABASE_URL"]
|
||||
table: Final = sql.Identifier(schema, "LiteLLM_SpendLogs")
|
||||
with psycopg.connect(url, autocommit=True) as setup:
|
||||
setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
|
||||
try:
|
||||
setup.execute(sql.SQL('CREATE TABLE {} (LIKE "LiteLLM_SpendLogs" INCLUDING ALL)').format(table))
|
||||
setup.execute(
|
||||
sql.SQL(
|
||||
'INSERT INTO {} (request_id, call_type, api_key, "startTime", "endTime") '
|
||||
"SELECT 'r' || n, 'acompletion', 'key-1', moment, moment FROM generate_series(1, %s) n, "
|
||||
"LATERAL (SELECT %s::timestamp + n * INTERVAL '1 second' AS moment) seconds"
|
||||
).format(table),
|
||||
(_COPIED_ROWS, _COPY_START.replace(tzinfo=None)),
|
||||
)
|
||||
setup.execute(sql.SQL("ANALYZE {}").format(table))
|
||||
parsed: Final = urlsplit(url)
|
||||
query: Final = urlencode({**dict(parse_qsl(parsed.query)), "schema": schema})
|
||||
database: Final = Prisma(datasource={"url": urlunsplit(parsed._replace(query=query))})
|
||||
await database.connect()
|
||||
try:
|
||||
yield database
|
||||
finally:
|
||||
await database.disconnect()
|
||||
finally:
|
||||
setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
|
||||
|
||||
|
||||
def _spend_log_rows_scanned(node: Mapping[str, Any]) -> int:
|
||||
own: Final = (
|
||||
(node["Actual Rows"] + node.get("Rows Removed by Filter", 0)) * node["Actual Loops"]
|
||||
if node.get("Relation Name") == "LiteLLM_SpendLogs"
|
||||
else 0
|
||||
)
|
||||
return own + sum(_spend_log_rows_scanned(child) for child in node.get("Plans", ()))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ExplainedReads:
|
||||
"""Runs each read through Prisma, first under EXPLAIN ANALYZE to count the `LiteLLM_SpendLogs` rows it scans."""
|
||||
|
||||
database: Prisma
|
||||
rows_scanned: list[int]
|
||||
|
||||
async def query_raw(self, query: str, *args: object) -> list[dict[str, Any]]:
|
||||
explained: Final = await self.database.query_raw(f"EXPLAIN (ANALYZE, FORMAT JSON) {query}", *args)
|
||||
self.rows_scanned.append(_spend_log_rows_scanned(explained[0]["QUERY PLAN"][0]["Plan"]))
|
||||
return await self.database.query_raw(query, *args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("page, rows_asked_for", [(1, 112), (3, 132)], ids=["first-page", "third-page"])
|
||||
async def test_a_newest_first_page_scans_only_the_rows_its_first_read_asks_for(page: int, rows_asked_for: int) -> None:
|
||||
async with _spend_logs_copy() as database:
|
||||
rows_scanned: Final[list[int]] = []
|
||||
session_page: Final = await read_session_page(
|
||||
_ExplainedReads(database, rows_scanned),
|
||||
"\"startTime\" >= ($1::timestamptz AT TIME ZONE 'UTC') AND \"startTime\" <= ($2::timestamptz AT TIME ZONE 'UTC')",
|
||||
(_COPY_START, _COPY_START + timedelta(days=1)),
|
||||
"",
|
||||
page=page,
|
||||
page_size=10,
|
||||
sort_desc=True,
|
||||
session_cursor=None,
|
||||
cap=100,
|
||||
)
|
||||
newest: Final = _COPIED_ROWS - (page - 1) * 10
|
||||
assert [session.key for session in session_page.sessions] == [f"r{n}" for n in range(newest, newest - 10, -1)]
|
||||
assert (session_page.total, session_page.total_is_capped) == (100, True)
|
||||
assert rows_scanned == [rows_asked_for]
|
||||
259
tests/unit/proxy/spend_tracking/test_session_page.py
Normal file
259
tests/unit/proxy/spend_tracking/test_session_page.py
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
import random
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.spend_tracking.session_page import (
|
||||
Session,
|
||||
_parse_session_cursor,
|
||||
_Read,
|
||||
_Request,
|
||||
_session_page,
|
||||
read_session_page,
|
||||
)
|
||||
|
||||
_Row = tuple[int, str, str] # (startTime, session key, api_key)
|
||||
|
||||
|
||||
def _ranked(rows: list[_Row], request: _Request) -> list[tuple[int, str, str]]:
|
||||
"""The sessions in ``rows`` in page order, after the request's cursor or offset."""
|
||||
last_activity: dict[tuple[str, str], int] = {}
|
||||
for start, key, api_key in rows:
|
||||
last_activity[(key, api_key)] = max(last_activity.get((key, api_key), start), start)
|
||||
ranked = sorted(((at, key, api_key) for (key, api_key), at in last_activity.items()), reverse=request.sort_desc)
|
||||
if request.cursor is None:
|
||||
return ranked[request.offset :]
|
||||
cursor = (int(request.cursor[0]), request.cursor[1], request.cursor[2])
|
||||
return [session for session in ranked if (session < cursor if request.sort_desc else session > cursor)]
|
||||
|
||||
|
||||
def _reader(rows: list[_Row], request: _Request, budgets: list[int | None], every_row: bool = False):
|
||||
"""Reads like the page query: the newest ``budget`` rows, where a LIMIT cuts tied rows lowest key first, the
|
||||
opposite of page order, so a session at the read's oldest time can hide a tied session that ranks before it.
|
||||
With ``every_row`` it ignores the budget, which is what grouping every row returns."""
|
||||
newest_first = sorted(rows, key=lambda row: (-row[0], row[1], row[2]))
|
||||
|
||||
async def read(budget: int | None) -> _Read:
|
||||
budgets.append(budget)
|
||||
taken = newest_first if budget is None or every_row else newest_first[:budget]
|
||||
page = _ranked(taken, request)[: request.page_limit + 1] if request.page_limit > 0 else []
|
||||
return _Read(
|
||||
candidates=tuple(Session(key, api_key, f"{at:06d}") for at, key, api_key in page),
|
||||
last_settled=bool(page) and page[-1][0] > taken[-1][0],
|
||||
rows_read=len(taken),
|
||||
session_count=len({(key, api_key) for _, key, api_key in taken}),
|
||||
read_everything=budget is None or every_row or len(taken) < budget,
|
||||
)
|
||||
|
||||
return read
|
||||
|
||||
|
||||
def _sessions(rng: random.Random) -> list[_Row]:
|
||||
"""Sessions of one to forty rows over a short clock, so many rows share a time."""
|
||||
rows: list[_Row] = []
|
||||
clock = rng.choice((3, 40, 400))
|
||||
for index in range(rng.randrange(1, 120)):
|
||||
start, length = rng.randrange(0, clock), rng.choice((1, 1, 2, 3, 8, 40))
|
||||
rows.extend((start + rng.randrange(0, 30), f"s{index:03d}", rng.choice(("k1", "k2"))) for _ in range(length))
|
||||
return rows
|
||||
|
||||
|
||||
def _requests(rng: random.Random, rows: list[_Row]):
|
||||
for _ in range(12):
|
||||
cap, page_size, sort_desc = rng.randrange(2, 60), rng.randrange(1, 12), rng.random() < 0.8
|
||||
mode = rng.choice(("first", "offset", "cursor"))
|
||||
full = _ranked(rows, _Request(None, 0, 0, sort_desc, cap))
|
||||
cursor = None
|
||||
if mode == "cursor" and full:
|
||||
at, key, api_key = full[rng.randrange(len(full))]
|
||||
cursor = (f"{at:06d}", key, api_key)
|
||||
offset = (rng.randrange(1, 2 * cap // page_size + 3) - 1) * page_size if mode == "offset" else 0
|
||||
yield _Request(cursor, offset, min(page_size, cap - offset), sort_desc, cap)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seed", range(60))
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_page_read_from_the_newest_rows_matches_grouping_every_row(seed):
|
||||
rng = random.Random(seed)
|
||||
rows = _sessions(rng)
|
||||
for request in _requests(rng, rows):
|
||||
grouped = await _session_page(_reader(rows, request, [], every_row=True), request)
|
||||
assert await _session_page(_reader(rows, request, []), request) == grouped, request
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rows_per_session, expected_budgets",
|
||||
[
|
||||
pytest.param(1, [112], id="one-read-holds-the-cap"),
|
||||
pytest.param(2, [112, 896], id="grows-8x"),
|
||||
pytest.param(50, [112, 7168], id="skips-8x-when-its-density-cannot-pass-the-cap"),
|
||||
pytest.param(500, [112, None], id="skips-to-every-row"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_reads_grow_8x_then_64x_then_every_row_skipping_reads_too_sparse_to_pass_the_cap(
|
||||
rows_per_session, expected_budgets
|
||||
):
|
||||
"""A cap of 100 and a page of 10 start at 100 + 10 + 2 rows."""
|
||||
rows = [(index, f"s{index // rows_per_session:05d}", "k") for index in range(150 * rows_per_session)]
|
||||
request = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, cap=100)
|
||||
budgets: list[int | None] = []
|
||||
await _session_page(_reader(rows, request, budgets), request)
|
||||
assert budgets == expected_budgets
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_session_tied_with_the_reads_oldest_row_is_not_trusted():
|
||||
"""Thirty one-row sessions share one time and the first read takes 17 of them. The page's sixth session sits at the
|
||||
read's oldest time, where the LIMIT may have cut a session that ranks before it, so the page reads on."""
|
||||
rows = [(0, f"s{index:02d}", "k") for index in range(30)]
|
||||
request = _Request(cursor=None, offset=0, page_limit=5, sort_desc=True, cap=10)
|
||||
budgets: list[int | None] = []
|
||||
page = await _session_page(_reader(rows, request, budgets), request)
|
||||
assert [session.key for session in page.sessions] == ["s29", "s28", "s27", "s26", "s25"]
|
||||
assert budgets == [17, 136]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("session_count, expected_total", [(100, (100, False)), (101, (100, True))])
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_total_is_capped_only_past_the_cap(session_count, expected_total):
|
||||
rows = [(index, f"s{index:05d}", "k") for index in range(session_count)]
|
||||
request = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, cap=100)
|
||||
page = await _session_page(_reader(rows, request, []), request)
|
||||
assert (page.total, page.total_is_capped) == expected_total
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_page_number_near_the_bigint_limit_reads_at_most_twice_the_cap():
|
||||
rows = [(index, f"s{index:05d}", "k") for index in range(500)]
|
||||
offset = (9_300_000_000_000_000 - 1) * 1000
|
||||
request = _Request(cursor=None, offset=offset, page_limit=min(1000, 100 - offset), sort_desc=True, cap=100)
|
||||
budgets: list[int | None] = []
|
||||
page = await _session_page(_reader(rows, request, budgets), request)
|
||||
assert budgets == [2 * 100 + 2]
|
||||
assert (page.sessions, page.total, page.total_is_capped) == ((), 100, True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_ascending_page_reads_every_row_once():
|
||||
rows = [(index, f"s{index:05d}", "k") for index in range(500)]
|
||||
request = _Request(cursor=None, offset=0, page_limit=10, sort_desc=False, cap=100)
|
||||
budgets: list[int | None] = []
|
||||
await _session_page(_reader(rows, request, budgets), request)
|
||||
assert budgets == [None]
|
||||
|
||||
|
||||
def test_parse_session_cursor():
|
||||
assert _parse_session_cursor(None) is None
|
||||
assert _parse_session_cursor("no-delimiter") is None
|
||||
assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None
|
||||
assert _parse_session_cursor("|hashed-key|sess-1") is None
|
||||
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None
|
||||
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == (
|
||||
"2026-08-29 10:00:00",
|
||||
"sess-1",
|
||||
"hashed-key",
|
||||
)
|
||||
assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == (
|
||||
"2026-08-29 10:00:00.123",
|
||||
"sess|with|pipes",
|
||||
"hashed-key",
|
||||
)
|
||||
|
||||
|
||||
def _bound(sql: str, args: tuple[object, ...], pattern: str) -> tuple[object, ...]:
|
||||
"""The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``."""
|
||||
found = re.search(pattern, sql)
|
||||
assert found, f"{pattern!r} not found in:\n{sql}"
|
||||
assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}"
|
||||
return tuple(args[int(index) - 1] for index in found.groups())
|
||||
|
||||
|
||||
class _TableDb:
|
||||
"""A table of ``rows`` matching rows in ``sessions`` sessions, whose reads return no page row."""
|
||||
|
||||
def __init__(self, calls: list[tuple[str, tuple[object, ...]]], rows: int, sessions: int):
|
||||
self.calls, self.rows, self.sessions = calls, rows, sessions
|
||||
|
||||
async def query_raw(self, query: str, *args: object):
|
||||
self.calls.append((query, args))
|
||||
budget = re.search(r"DESC LIMIT (\d+)", query)
|
||||
rows_read = min(int(budget.group(1)), self.rows) if budget else self.rows
|
||||
summary = {"rows_read": rows_read, "session_count": self.sessions}
|
||||
return [{"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_read_that_fills_its_budget_is_not_taken_for_every_row():
|
||||
"""A read that took every row its budget allowed may have stopped short, so its 50 sessions are no total: the reads
|
||||
grow until one takes fewer rows than it asked for."""
|
||||
calls: list[tuple[str, tuple[object, ...]]] = []
|
||||
page = await read_session_page(
|
||||
_TableDb(calls, rows=9_000, sessions=50),
|
||||
"TRUE",
|
||||
(),
|
||||
"",
|
||||
page=1,
|
||||
page_size=10,
|
||||
sort_desc=True,
|
||||
session_cursor=None,
|
||||
cap=100,
|
||||
)
|
||||
budgets = [re.search(r"DESC LIMIT (\d+)", sql) for sql, _ in calls]
|
||||
assert [int(found.group(1)) if found else None for found in budgets] == [112, 896, 7168, None]
|
||||
assert (page.total, page.total_is_capped) == (50, False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_page_starting_at_the_cap_reads_once_and_fetches_no_session():
|
||||
calls: list[tuple[str, tuple[object, ...]]] = []
|
||||
page = await read_session_page(
|
||||
_TableDb(calls, rows=1_000, sessions=150),
|
||||
"TRUE",
|
||||
(),
|
||||
"",
|
||||
page=5,
|
||||
page_size=25,
|
||||
sort_desc=True,
|
||||
session_cursor=None,
|
||||
cap=100,
|
||||
)
|
||||
assert len(calls) == 1, "the first read already holds more sessions than the cap"
|
||||
assert _bound(*calls[0], r"LIMIT \$(\d+)") == (0,)
|
||||
assert (page.sessions, page.has_more, page.total, page.total_is_capped) == ((), False, 100, True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cap, expected_budgets",
|
||||
[
|
||||
pytest.param(103, [115, 920, 7360, None], id="13-sessions-per-115-rows-just-reach-104-in-920"),
|
||||
pytest.param(104, [116, 7424, None], id="13-sessions-per-116-rows-fall-one-short-of-105-in-928"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_density_skip_reads_a_budget_whose_density_exactly_reaches_past_the_cap(cap, expected_budgets):
|
||||
"""A budget is skipped only when its rows, as dense as the last read, would hold fewer than cap + 1 sessions."""
|
||||
calls: list[tuple[str, tuple[object, ...]]] = []
|
||||
await read_session_page(_TableDb(calls, rows=100_000, sessions=13), "TRUE", (), "", 1, 10, True, None, cap)
|
||||
budgets = [re.search(r"DESC LIMIT (\d+)", sql) for sql, _ in calls]
|
||||
assert [int(found.group(1)) if found else None for found in budgets] == expected_budgets
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_read_binds_filters_cursor_limit_and_offset_to_their_placeholders():
|
||||
calls: list[tuple[str, tuple[object, ...]]] = []
|
||||
db = _TableDb(calls, rows=50, sessions=40)
|
||||
where, params = "model = $1 AND api_key = $2", ("gpt-4o", "key-a")
|
||||
await read_session_page(db, where, params, "", 1, 3, True, "2026-10-28 09:00:00|key-a|sess|9", cap=100)
|
||||
await read_session_page(db, where, params, "", 3, 1, True, None, cap=100)
|
||||
(cursor_sql, cursor_args), (offset_sql, offset_args) = calls
|
||||
for sql, args in calls:
|
||||
assert _bound(sql, args, r"model = \$(\d+) AND api_key = \$(\d+)") == params
|
||||
assert "page.last_activity > summary.oldest AS settled" in sql, (
|
||||
"a session tied with the read's oldest row may rank after a session the LIMIT cut, so it is not settled"
|
||||
)
|
||||
assert _bound(
|
||||
cursor_sql, cursor_args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)"
|
||||
) == ("2026-10-28 09:00:00", "sess|9", "key-a")
|
||||
assert _bound(cursor_sql, cursor_args, r"LIMIT \$(\d+)") == (4,)
|
||||
assert _bound(offset_sql, offset_args, r"LIMIT \$(\d+) OFFSET \$(\d+)") == (2, 2)
|
||||
|
|
@ -7151,28 +7151,6 @@ def test_scoped_spend_report_range_at_max_allowed(client, monkeypatch):
|
|||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
||||
def test_parse_session_cursor():
|
||||
from litellm.proxy.spend_tracking.spend_management_endpoints import (
|
||||
_parse_session_cursor,
|
||||
)
|
||||
|
||||
assert _parse_session_cursor(None) is None
|
||||
assert _parse_session_cursor("no-delimiter") is None
|
||||
assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None
|
||||
assert _parse_session_cursor("|hashed-key|sess-1") is None
|
||||
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None
|
||||
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == (
|
||||
"2026-08-29 10:00:00",
|
||||
"sess-1",
|
||||
"hashed-key",
|
||||
)
|
||||
assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == (
|
||||
"2026-08-29 10:00:00.123",
|
||||
"sess|with|pipes",
|
||||
"hashed-key",
|
||||
)
|
||||
|
||||
|
||||
SESSION_GROUP_KEY_SQL = "COALESCE(NULLIF(session_id, ''), request_id), api_key"
|
||||
|
||||
|
||||
|
|
@ -7180,8 +7158,8 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ
|
|||
"""Mock prisma dispatching the raw queries the keyset grouped path emits."""
|
||||
|
||||
async def mock_query_raw(sql_query, *params):
|
||||
if "COUNT(*) AS total_count" in sql_query:
|
||||
return [{"total_count": session_total}]
|
||||
if "rows_read" in sql_query:
|
||||
return _session_read_rows(session_page_rows, session_total)
|
||||
if "DISTINCT ON" in sql_query:
|
||||
return representative_rows
|
||||
if "COALESCE(SUM(spend)" in sql_query:
|
||||
|
|
@ -7199,7 +7177,7 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ
|
|||
"session_models": ["gpt-4o"],
|
||||
}
|
||||
]
|
||||
return session_page_rows
|
||||
raise AssertionError(f"unexpected query:\n{sql_query}")
|
||||
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db = MagicMock()
|
||||
|
|
@ -7228,12 +7206,18 @@ def _session_page_row(session_key, last_activity):
|
|||
return {"session_key": session_key, "api_key": "hashed-key", "last_activity": last_activity}
|
||||
|
||||
|
||||
def _session_grouped_paginating_prisma(sessions, counted_total=None):
|
||||
def _session_read_rows(page_rows, session_count):
|
||||
"""The read query's rows for ``page_rows``, as if the read held every matching row."""
|
||||
summary = {"rows_read": 0, "session_count": session_count}
|
||||
if not page_rows:
|
||||
return [{"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary}]
|
||||
return [{**row, "settled": True, **summary} for row in page_rows]
|
||||
|
||||
|
||||
def _session_grouped_paginating_prisma(sessions):
|
||||
"""Mock prisma serving the grouped page query out of ``sessions``, honoring the LIMIT and OFFSET it asks for."""
|
||||
|
||||
async def mock_query_raw(sql_query, *params):
|
||||
if "COUNT(*) AS total_count" in sql_query:
|
||||
return [{"total_count": min(len(sessions) if counted_total is None else counted_total, params[-1])}]
|
||||
if "DISTINCT ON" in sql_query:
|
||||
return [_session_representative_row(f"req-{session_key}", session_key) for session_key in params[-2]]
|
||||
if "COALESCE(SUM(spend)" in sql_query:
|
||||
|
|
@ -7241,10 +7225,13 @@ def _session_grouped_paginating_prisma(sessions, counted_total=None):
|
|||
bounds = re.search(r"LIMIT \$(\d+)(?: OFFSET \$(\d+))?", sql_query)
|
||||
limit = params[int(bounds.group(1)) - 1]
|
||||
offset = params[int(bounds.group(2)) - 1] if bounds.group(2) else 0
|
||||
return [
|
||||
_session_page_row(session_key, last_activity)
|
||||
for session_key, last_activity in sessions[offset : offset + limit]
|
||||
]
|
||||
return _session_read_rows(
|
||||
[
|
||||
_session_page_row(session_key, last_activity)
|
||||
for session_key, last_activity in sessions[offset : offset + limit]
|
||||
],
|
||||
len(sessions),
|
||||
)
|
||||
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db = MagicMock()
|
||||
|
|
@ -7252,6 +7239,29 @@ def _session_grouped_paginating_prisma(sessions, counted_total=None):
|
|||
return mock_prisma
|
||||
|
||||
|
||||
def _bound(sql, args, pattern):
|
||||
"""The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``."""
|
||||
found = re.search(pattern, sql)
|
||||
assert found, f"{pattern!r} not found in:\n{sql}"
|
||||
assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}"
|
||||
return tuple(args[int(index) - 1] for index in found.groups())
|
||||
|
||||
|
||||
def _get_session_page(client, monkeypatch, mock_prisma, **params):
|
||||
"""GET the session-grouped logs page over the default window as a proxy admin."""
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma)
|
||||
app.dependency_overrides[ps.user_api_key_auth] = _admin_auth
|
||||
try:
|
||||
start_date, end_date = _default_date_range()
|
||||
return client.get(
|
||||
"/spend/logs/ui",
|
||||
params={"start_date": start_date, "end_date": end_date, "group_by_session": "true", **params},
|
||||
headers={"Authorization": "Bearer sk-test"},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatch):
|
||||
"""One row per (session, api_key), session-count total, and a keyset cursor for the next page."""
|
||||
|
|
@ -7288,6 +7298,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc
|
|||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
assert data["total"] == 3
|
||||
assert data["total_pages"] == 2
|
||||
assert data["has_more"] is True
|
||||
assert data["next_session_cursor"] == "2026-08-29 09:00:00|hashed-key|req-solo"
|
||||
request_ids = [row["request_id"] for row in data["data"]]
|
||||
|
|
@ -7296,17 +7307,10 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc
|
|||
assert data["data"][1]["session_total_count"] == 1
|
||||
|
||||
emitted = [call.args for call in mock_prisma.db.query_raw.await_args_list]
|
||||
page_query_sql = emitted[0][0]
|
||||
assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in page_query_sql
|
||||
assert "OFFSET" not in page_query_sql
|
||||
assert "HAVING" not in page_query_sql
|
||||
assert emitted[0][-1] == 3, "page query fetches page_size + 1 sessions to detect has_more"
|
||||
assert len(emitted) == 3, "one read decides the page and the total, then the representatives and their totals"
|
||||
assert emitted[0][-1] == 3, "the read fetches page_size + 1 sessions to detect has_more"
|
||||
|
||||
count_sql = emitted[1][0]
|
||||
assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in count_sql
|
||||
assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded"
|
||||
|
||||
rep_call = emitted[2]
|
||||
rep_call = emitted[1]
|
||||
assert f"DISTINCT ON ({SESSION_GROUP_KEY_SQL})" in rep_call[0]
|
||||
assert (
|
||||
f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL
|
||||
|
|
@ -7319,7 +7323,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypatch):
|
||||
"""A session_cursor becomes a HAVING keyset predicate instead of an OFFSET."""
|
||||
"""A session_cursor becomes a keyset predicate on the read's sessions instead of an OFFSET."""
|
||||
page_rows = [_session_page_row("sess-2", "2026-08-29 07:00:00")]
|
||||
reps = [_session_representative_row("req-2", "sess-2")]
|
||||
mock_prisma = _session_grouped_mock_prisma(page_rows, 3, reps)
|
||||
|
|
@ -7352,7 +7356,7 @@ async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypat
|
|||
|
||||
page_query_call = mock_prisma.db.query_raw.await_args_list[0]
|
||||
page_query_sql = page_query_call.args[0]
|
||||
assert f'HAVING (MAX("startTime"), {SESSION_GROUP_KEY_SQL}) <' in page_query_sql
|
||||
assert "WHERE (last_activity, session_key, api_key) <" in page_query_sql
|
||||
assert "OFFSET" not in page_query_sql
|
||||
cursor_index = page_query_call.args.index("2026-08-29 09:00:00")
|
||||
assert page_query_call.args[cursor_index : cursor_index + 3] == (
|
||||
|
|
@ -7403,9 +7407,9 @@ async def test_ui_view_spend_logs_group_by_session_jumps_to_page_without_cursor(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(client, monkeypatch):
|
||||
"""A page that runs out of sessions is the end of the list, so the total comes from it and nothing is counted."""
|
||||
"""A page that runs out of sessions has read every matching row, so that one read gives the total."""
|
||||
sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(10))
|
||||
mock_prisma = _session_grouped_paginating_prisma(sessions, counted_total=999)
|
||||
mock_prisma = _session_grouped_paginating_prisma(sessions)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe",
|
||||
|
|
@ -7429,10 +7433,12 @@ async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(clie
|
|||
)
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
assert data["total"] == 10, "the count query's 999 would have won if it had been asked"
|
||||
assert data["total"] == 10
|
||||
assert data["total_is_capped"] is False
|
||||
assert data["total_pages"] == 1
|
||||
assert len(data["data"]) == 10
|
||||
reads = [call for call in mock_prisma.db.query_raw.await_args_list if "rows_read" in call.args[0]]
|
||||
assert len(reads) == 1, "a page that ends the list is not read again for its total"
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
|
@ -7517,7 +7523,7 @@ async def test_ui_view_spend_logs_group_by_session_page_past_count_cap_is_empty(
|
|||
assert past_cap_data["data"] == []
|
||||
assert past_cap_data["has_more"] is False
|
||||
assert past_cap_data["total"] == cap
|
||||
assert mock_prisma.db.query_raw.await_count == 1, "only the bounded count query runs past the capped window"
|
||||
assert mock_prisma.db.query_raw.await_count == 1, "only the bounded read runs past the capped window"
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
|
@ -7561,6 +7567,114 @@ async def test_ui_view_spend_logs_group_by_session_last_page_stops_at_the_capped
|
|||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
||||
class _DatetimeWithInjectedIsoformat(datetime.datetime):
|
||||
def isoformat(self, sep: str = "T", timespec: str = "auto") -> str:
|
||||
return "2026-01-01'); DROP TABLE x; --"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bound",
|
||||
[
|
||||
"2026-01-01'); DROP TABLE x; --",
|
||||
datetime.datetime(2026, 1, 1),
|
||||
datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone(datetime.timedelta(hours=5))),
|
||||
_DatetimeWithInjectedIsoformat(2026, 1, 1, tzinfo=timezone.utc),
|
||||
datetime.date(2026, 1, 1),
|
||||
],
|
||||
ids=["text", "naive", "not-utc", "datetime-subclass", "date"],
|
||||
)
|
||||
def test_utc_timestamp_literal_refuses_anything_but_a_utc_datetime(bound):
|
||||
with pytest.raises(TypeError):
|
||||
spend_management_endpoints._utc_timestamp_literal(bound)
|
||||
|
||||
|
||||
def test_utc_timestamp_literal_formats_a_utc_datetime():
|
||||
literal = spend_management_endpoints._utc_timestamp_literal(
|
||||
datetime.datetime(2026, 1, 2, 3, 4, 5, 6, tzinfo=timezone.utc)
|
||||
)
|
||||
assert literal == "('2026-01-02T03:04:05.000006+00:00'::timestamptz AT TIME ZONE 'UTC')"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_filter", ["failure", "success"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_view_spend_logs_group_by_session_status_reads_outcomes_over_a_literal_window(
|
||||
client, monkeypatch, status_filter
|
||||
):
|
||||
"""Reused prepared statements get a generic plan from the 6th run, which takes a $N window for a sliver of the
|
||||
table and turned the outcome subquery into a ~2 min nested loop, so the window goes in as literals. A failure
|
||||
filter's sessions are computed once in a CTE, which measured faster only for failure. The outcomes, the read and
|
||||
the representatives each bind the status, the model and the page's sessions to their own placeholders."""
|
||||
page_rows = [_session_page_row("sess-1", "2026-08-29 08:00:00")]
|
||||
mock_prisma = _session_grouped_mock_prisma(page_rows, 1, [_session_representative_row("req-1", "sess-1")])
|
||||
response = _get_session_page(client, monkeypatch, mock_prisma, status_filter=status_filter, model="gpt-4o")
|
||||
assert response.status_code == 200, response.text
|
||||
calls = [call.args for call in mock_prisma.db.query_raw.await_args_list]
|
||||
read_sql, *read_args = next(args for args in calls if "rows_read" in args[0])
|
||||
outcomes = read_sql.split("SELECT DISTINCT ON", 1)[1].split("AS session_outcomes", 1)[0]
|
||||
assert "$1::timestamptz" not in outcomes and "$2::timestamptz" not in outcomes
|
||||
assert all(f"('{bound.isoformat()}'::timestamptz AT TIME ZONE 'UTC')" in outcomes for bound in read_args[:2])
|
||||
assert _bound(outcomes, read_args, r"model = \$(\d+)") == ("gpt-4o",)
|
||||
outcome_status = r"AS session_outcomes\s+WHERE COALESCE\(status, 'success'\) = \$(\d+)"
|
||||
assert _bound(read_sql, read_args, outcome_status) == (status_filter,)
|
||||
assert read_sql.lstrip().startswith("WITH status_sessions AS (") is (status_filter == "failure")
|
||||
if status_filter == "failure":
|
||||
assert read_sql.count("FROM status_sessions") == 2, "named once, Postgres 12+ would inline it per worker"
|
||||
assert "MATERIALIZED" not in read_sql, "Postgres 10 and 11 cannot parse MATERIALIZED"
|
||||
|
||||
rep_sql, *rep_args = next(args for args in calls if "AS session_representatives" in args[0])
|
||||
assert _bound(rep_sql, rep_args, r"model = \$(\d+)") == ("gpt-4o",), "representatives keep the row filters"
|
||||
assert _bound(rep_sql, rep_args, r"session_id = ANY\(\$(\d+)::text\[\]\)") == (["sess-1"],)
|
||||
assert _bound(rep_sql, rep_args, r"unnest\(\$(\d+)::text\[\], \$(\d+)::text\[\]\)") == (["sess-1"], ["hashed-key"])
|
||||
assert _bound(rep_sql.split("AS session_representatives", 1)[1], rep_args, r"= \$(\d+)") == (status_filter,)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_view_spend_logs_group_by_session_failure_off_the_session_page_stays_inlined(client, monkeypatch):
|
||||
"""Only the session page declares the status_sessions CTE, so a grouped page sorted by another column must inline
|
||||
the failure subquery or Postgres rejects the statement."""
|
||||
calls = []
|
||||
|
||||
async def mock_query_raw(sql_query, *params):
|
||||
calls.append((sql_query, params))
|
||||
return [{"total_count": 0}] if "total_count" in sql_query else []
|
||||
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db = MagicMock()
|
||||
mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw)
|
||||
response = _get_session_page(
|
||||
client, monkeypatch, mock_prisma, status_filter="failure", sort_by="spend", page=3, page_size=7
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert calls and all("status_sessions" not in sql for sql, _ in calls)
|
||||
page_sql, page_args = next(call for call in calls if "AS session_representatives" in call[0])
|
||||
assert _bound(page_sql, page_args, r"COALESCE\(status, 'success'\) = \$(\d+)") == ("failure",)
|
||||
assert _bound(page_sql, page_args, r"LIMIT \$(\d+) OFFSET \$(\d+)") == (7, 14)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_view_spend_logs_group_by_session_status_drops_a_session_whose_outcome_changed(client, monkeypatch):
|
||||
"""A row written between the page read and the representatives can flip a session's outcome; like the status
|
||||
filter, the representatives statement then drops the session instead of showing a row of the other status."""
|
||||
flipped = {**_session_representative_row("req-new", "sess-1"), "status": "success"}
|
||||
|
||||
async def mock_query_raw(sql_query, *params):
|
||||
if "rows_read" in sql_query:
|
||||
return _session_read_rows([_session_page_row("sess-1", "2026-08-29 08:00:00")], 1)
|
||||
if "AS session_representatives" in sql_query:
|
||||
check = re.search(r"WHERE COALESCE\(status, 'success'\) = \$(\d+)\s*$", sql_query.strip())
|
||||
return [flipped] if check is None or params[int(check.group(1)) - 1] == flipped["status"] else []
|
||||
if "COALESCE(SUM(spend)" in sql_query:
|
||||
return []
|
||||
raise AssertionError(f"unexpected query:\n{sql_query}")
|
||||
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db = MagicMock()
|
||||
mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw)
|
||||
response = _get_session_page(client, monkeypatch, mock_prisma, status_filter="failure")
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["data"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_view_spend_logs_group_by_session_offset_for_non_starttime_sort(client, monkeypatch):
|
||||
"""Sorting by another column keeps session grouping but pages with OFFSET, without a keyset cursor."""
|
||||
|
|
|
|||
|
|
@ -512,16 +512,13 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch):
|
|||
With group_by_session=true, /spend/logs/ui must page and count SESSIONS,
|
||||
not raw calls: the page is one representative row per session (DISTINCT
|
||||
ON the session group key, preferring non-MCP calls, newest first), the
|
||||
bounded count counts groups, and on the default startTime sort the page is
|
||||
total counts groups, and on the default startTime sort the page is
|
||||
selected by keyset (no OFFSET) so deep pages do not degrade. Otherwise the
|
||||
UI collapses a server page of N calls into fewer visible rows while the
|
||||
footer still claims N (issue #38060).
|
||||
"""
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.spend_tracking.spend_management_endpoints import (
|
||||
SPEND_LOGS_PAGINATION_COUNT_CAP,
|
||||
ui_view_spend_logs,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs
|
||||
|
||||
group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key"
|
||||
session_rows = [
|
||||
|
|
@ -534,11 +531,11 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch):
|
|||
]
|
||||
|
||||
async def mock_query_raw(sql_query, *params):
|
||||
if "COUNT(*) AS total_count" in sql_query:
|
||||
return [{"total_count": 60}]
|
||||
if "rows_read" in sql_query:
|
||||
return [{**row, "settled": True, "rows_read": 0, "session_count": 60} for row in session_rows]
|
||||
if "AS session_representatives" in sql_query:
|
||||
return representative_rows
|
||||
return session_rows
|
||||
return []
|
||||
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db = MagicMock()
|
||||
|
|
@ -569,23 +566,16 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch):
|
|||
|
||||
emitted = [call[0] for call in mock_prisma.db.query_raw.call_args_list]
|
||||
page_sql = emitted[0][0]
|
||||
assert f"GROUP BY {group_key}" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}"
|
||||
assert "GROUP BY session_key, api_key" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}"
|
||||
assert "OFFSET" not in page_sql, "the startTime page must be keyset-selected, not offset-selected"
|
||||
assert emitted[0][-1] == 51, "the page query fetches page_size + 1 sessions to detect has_more"
|
||||
|
||||
count_call = emitted[1]
|
||||
count_sql = count_call[0]
|
||||
assert f"GROUP BY {group_key}" in count_sql, f"grouped total must count sessions. SQL was:\n{count_sql}"
|
||||
assert "COUNT(*) OVER ()" not in count_sql
|
||||
assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded"
|
||||
assert count_call[-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1
|
||||
|
||||
rep_sql = emitted[2][0]
|
||||
rep_sql = emitted[1][0]
|
||||
assert f"DISTINCT ON ({group_key})" in rep_sql, f"page must return one row per session. SQL was:\n{rep_sql}"
|
||||
assert (
|
||||
f"ORDER BY {group_key}, (call_type = 'asend_message') DESC, "
|
||||
"CASE WHEN call_type = 'asend_message' THEN \"endTime\" END DESC NULLS LAST, "
|
||||
"call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC"
|
||||
"call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC, request_id"
|
||||
) in rep_sql, "the session representative must prefer the final agent outcome, then the newest non-MCP call"
|
||||
assert "COUNT(*) OVER ()" not in rep_sql
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue