mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(proxy): let the Admin UI cancel /team/spend/by_user at 60s with an opt-in statement timeout
GET /team/spend/by_user takes an optional timeout_seconds (1 to 600). When set, the raw LiteLLM_SpendLogs aggregation runs under SET LOCAL statement_timeout and Postgres cancels it, and the route returns 503 instead of letting the query run for hours. Callers that omit the param are unchanged. The Admin UI sends timeout_seconds=60 and the Spend Per User Within Team card shows the server's error instead of an empty table
This commit is contained in:
parent
0aa7079f04
commit
096a4bb737
9 changed files with 203 additions and 9 deletions
|
|
@ -34,6 +34,8 @@ _CONNECTION_CAPACITY_PHRASES: Final = (
|
|||
_PRISMA_POOL_TIMEOUT_CODE: Final = "P2024"
|
||||
_LOCK_TIMEOUT_SQLSTATE: Final = "55P03"
|
||||
_LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout"
|
||||
_STATEMENT_TIMEOUT_SQLSTATE: Final = "57014"
|
||||
_STATEMENT_TIMEOUT_PHRASE: Final = "canceling statement due to statement timeout"
|
||||
|
||||
|
||||
def exception_chain(e: BaseException) -> Iterator[BaseException]:
|
||||
|
|
@ -302,6 +304,17 @@ class PrismaDBExceptionHandler:
|
|||
return True
|
||||
return _LOCK_TIMEOUT_PHRASE in str(e).lower()
|
||||
|
||||
@staticmethod
|
||||
def is_statement_timeout_error(e: Exception) -> bool:
|
||||
"""True iff Postgres cancelled a statement for exceeding ``statement_timeout`` (SQLSTATE 57014)."""
|
||||
import prisma
|
||||
|
||||
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
|
||||
return False
|
||||
if PrismaDBExceptionHandler.postgres_sqlstate(e) == _STATEMENT_TIMEOUT_SQLSTATE:
|
||||
return True
|
||||
return _STATEMENT_TIMEOUT_PHRASE in str(e).lower()
|
||||
|
||||
@staticmethod
|
||||
def postgres_sqlstate(e: Exception) -> str | None:
|
||||
"""The SQLSTATE Postgres attached to a failed statement, as prisma surfaces it, or None."""
|
||||
|
|
|
|||
|
|
@ -140,6 +140,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
team_membership_reservation_cache_key,
|
||||
)
|
||||
from litellm.proxy.db.db_span import db_span
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
|
||||
from litellm.proxy.hooks.model_max_budget_limiter import (
|
||||
RoutedModels,
|
||||
|
|
@ -7473,6 +7474,16 @@ class _TeamUserSpendDbRow(TypedDict):
|
|||
failed_requests: ReadOnly[int]
|
||||
|
||||
|
||||
async def _team_user_spend_rows(
|
||||
prisma_client: PrismaClient, sql: str, params: tuple[str, ...], timeout_seconds: int | None
|
||||
) -> Sequence[_TeamUserSpendDbRow]:
|
||||
if timeout_seconds is None:
|
||||
return await prisma_client.db.query_raw(sql, *params)
|
||||
async with prisma_client.db.tx(timeout=timedelta(seconds=timeout_seconds + 5)) as tx:
|
||||
await tx.execute_raw(f"SET LOCAL statement_timeout = {timeout_seconds * 1000}")
|
||||
return await tx.query_raw(sql, *params)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/team/spend/by_user",
|
||||
response_model=TeamUserSpendResponse,
|
||||
|
|
@ -7483,6 +7494,14 @@ async def get_team_spend_by_user(
|
|||
team_ids: str | None = None,
|
||||
start_date: str | None = None,
|
||||
end_date: str | None = None,
|
||||
timeout_seconds: Annotated[
|
||||
int | None,
|
||||
fastapi.Query(
|
||||
ge=1,
|
||||
le=600,
|
||||
description="Cancel the database query after this many seconds and return 503. Unset means no limit.",
|
||||
),
|
||||
] = None,
|
||||
) -> TeamUserSpendResponse:
|
||||
"""
|
||||
Spend per user within the given teams, attributed per request from spend logs.
|
||||
|
|
@ -7522,14 +7541,21 @@ async def get_team_spend_by_user(
|
|||
|
||||
own_user_only: Final = scope.api_key_filter is not None
|
||||
user_param: Final = (user_api_key_dict.user_id or "",) if own_user_only else ()
|
||||
async with db_span("team_user_spend", "LiteLLM_SpendLogs"):
|
||||
rows: Final[Sequence[_TeamUserSpendDbRow]] = await prisma_client.db.query_raw(
|
||||
_team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only),
|
||||
start_date,
|
||||
end_date,
|
||||
*scoped_team_ids,
|
||||
*user_param,
|
||||
)
|
||||
try:
|
||||
async with db_span("team_user_spend", "LiteLLM_SpendLogs"):
|
||||
rows: Final = await _team_user_spend_rows(
|
||||
prisma_client,
|
||||
_team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only),
|
||||
(start_date, end_date, *scoped_team_ids, *user_param),
|
||||
timeout_seconds,
|
||||
)
|
||||
except Exception as e:
|
||||
if timeout_seconds is not None and PrismaDBExceptionHandler.is_statement_timeout_error(e):
|
||||
raise _daily_activity_error(
|
||||
status_code=503,
|
||||
message=f"Spend per user took longer than {timeout_seconds}s. Pick a shorter date range or fewer teams",
|
||||
) from e
|
||||
raise
|
||||
results: Final = tuple(
|
||||
TeamUserSpendRow(
|
||||
team_id=row["team_id"],
|
||||
|
|
|
|||
|
|
@ -56,3 +56,15 @@ async def test_team_spend_by_user_matrix(actor: Actor, team: str, expected_statu
|
|||
body = resp.json()
|
||||
assert (body["start_date"], body["end_date"]) == ("2024-01-01", "2024-12-31")
|
||||
assert all(row["team_id"] == team_id for row in body["results"])
|
||||
|
||||
|
||||
async def test_team_spend_by_user_timeout_seconds_runs_under_a_statement_timeout(gateway_client, actors):
|
||||
headers = {"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}
|
||||
query = f"{_DATES}&team_ids={actors.team_alpha_id}"
|
||||
|
||||
resp = await gateway_client.get(f"/team/spend/by_user?{query}&timeout_seconds=60", headers=headers)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert all(row["team_id"] == actors.team_alpha_id for row in resp.json()["results"])
|
||||
|
||||
too_small = await gateway_client.get(f"/team/spend/by_user?{query}&timeout_seconds=0", headers=headers)
|
||||
assert too_small.status_code == 422, too_small.text
|
||||
|
|
|
|||
|
|
@ -953,6 +953,21 @@ def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None
|
|||
assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "expected"),
|
||||
[
|
||||
(_lock_timeout_error(message="canceling statement due to statement timeout", code="57014"), True),
|
||||
(_lock_timeout_error(message="canceling statement due to statement timeout", code=None), True),
|
||||
(_lock_timeout_error(), False),
|
||||
(_capacity_error("sorry, too many clients already"), False),
|
||||
(RuntimeError("canceling statement due to statement timeout"), False),
|
||||
],
|
||||
ids=["meta_57014", "message_only", "lock_timeout_55P03", "capacity_53300", "not_prisma"],
|
||||
)
|
||||
def test_is_statement_timeout_error(error: Exception, expected: bool) -> None:
|
||||
assert PrismaDBExceptionHandler.is_statement_timeout_error(error) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "retryable"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import litellm
|
|||
import pytest
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.testclient import TestClient
|
||||
from prisma.errors import DataError
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -17413,6 +17414,87 @@ async def test_get_team_spend_by_user_rejects_bad_input(mock_db_client, team_ids
|
|||
mock_db_client.db.query_raw.assert_not_called()
|
||||
|
||||
|
||||
class _RecordingTransaction:
|
||||
def __init__(self, query_raw: AsyncMock) -> None:
|
||||
self.statements: list[str] = []
|
||||
self.query_raw = query_raw
|
||||
|
||||
async def execute_raw(self, sql: str) -> int:
|
||||
self.statements.append(sql)
|
||||
return 0
|
||||
|
||||
async def __aenter__(self) -> "_RecordingTransaction":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc_info: object) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _statement_timeout_error() -> DataError:
|
||||
message = "canceling statement due to statement timeout"
|
||||
return DataError(
|
||||
data={
|
||||
"user_facing_error": {
|
||||
"is_panic": False,
|
||||
"message": f"Error querying the database: {message}",
|
||||
"meta": {"code": "57014", "message": message},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_team_spend_by_user_timeout_runs_query_under_statement_timeout(mock_db_client):
|
||||
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
|
||||
|
||||
admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), [])
|
||||
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha])
|
||||
mock_db_client.db.query_raw = AsyncMock(return_value=[])
|
||||
transaction = _RecordingTransaction(
|
||||
AsyncMock(return_value=[_team_spend_by_user_db_row("team-alpha", "alice", 0.5, 3)])
|
||||
)
|
||||
mock_db_client.db.tx = MagicMock(return_value=transaction)
|
||||
|
||||
response = await get_team_spend_by_user(
|
||||
user_api_key_dict=admin,
|
||||
team_ids="team-alpha",
|
||||
start_date="2026-09-01",
|
||||
end_date="2026-09-04",
|
||||
timeout_seconds=60,
|
||||
)
|
||||
|
||||
assert transaction.statements == ["SET LOCAL statement_timeout = 60000"]
|
||||
_sql, *params = transaction.query_raw.call_args.args
|
||||
assert params == ["2026-09-01", "2026-09-04", "team-alpha"]
|
||||
mock_db_client.db.query_raw.assert_not_called()
|
||||
assert [(r.user_id, r.spend) for r in response.results] == [("alice", 0.5)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_team_spend_by_user_statement_timeout_returns_503(mock_db_client):
|
||||
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
|
||||
|
||||
admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), [])
|
||||
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha])
|
||||
mock_db_client.db.tx = MagicMock(
|
||||
return_value=_RecordingTransaction(AsyncMock(side_effect=_statement_timeout_error()))
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await get_team_spend_by_user(
|
||||
user_api_key_dict=admin,
|
||||
team_ids="team-alpha",
|
||||
start_date="2026-09-01",
|
||||
end_date="2026-09-04",
|
||||
timeout_seconds=60,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 503
|
||||
assert "longer than 60s" in str(exc_info.value.detail)
|
||||
|
||||
|
||||
class _TeamRowWithOrganization(LiteLLM_TeamTable):
|
||||
litellm_organization_table: LiteLLM_OrganizationTable | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import React from "react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { teamSpendByUserCall } from "@/components/networking";
|
||||
|
||||
import TeamUserSpendCard from "./TeamUserSpendCard";
|
||||
|
||||
vi.mock("@/components/networking", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@/components/networking")>()),
|
||||
teamSpendByUserCall: vi.fn(),
|
||||
}));
|
||||
|
||||
const renderCard = () => {
|
||||
const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } });
|
||||
return render(
|
||||
<QueryClientProvider client={client}>
|
||||
<TeamUserSpendCard
|
||||
accessToken="sk-test"
|
||||
startTime={new Date("2026-09-01T00:00:00Z")}
|
||||
endTime={new Date("2026-09-30T00:00:00Z")}
|
||||
teamIds={["team-alpha"]}
|
||||
/>
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
};
|
||||
|
||||
describe("TeamUserSpendCard", () => {
|
||||
it("shows the server's error instead of an empty table when the request fails", async () => {
|
||||
const timeoutMessage = "Spend per user took longer than 60s. Pick a shorter date range or fewer teams";
|
||||
vi.mocked(teamSpendByUserCall).mockRejectedValueOnce(new Error(timeoutMessage));
|
||||
|
||||
renderCard();
|
||||
|
||||
expect(await screen.findByText(timeoutMessage)).toBeInTheDocument();
|
||||
expect(screen.queryByText("No user spend in this range")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -73,7 +73,7 @@ const columns: ColumnDef<TeamUserSpendRow>[] = [
|
|||
|
||||
const TeamUserSpendCard: React.FC<TeamUserSpendCardProps> = ({ accessToken, startTime, endTime, teamIds }) => {
|
||||
const hasTeams = teamIds.length > 0;
|
||||
const { data, isLoading } = useQuery({
|
||||
const { data, isLoading, error } = useQuery({
|
||||
queryKey: ["teamSpendByUser", startTime?.toISOString(), endTime?.toISOString(), teamIds],
|
||||
queryFn: () =>
|
||||
accessToken && startTime && endTime ? teamSpendByUserCall(accessToken, startTime, endTime, teamIds) : null,
|
||||
|
|
@ -105,6 +105,8 @@ const TeamUserSpendCard: React.FC<TeamUserSpendCardProps> = ({ accessToken, star
|
|||
data={rows}
|
||||
getRowId={teamUserSpendRowId}
|
||||
isLoading={isLoading}
|
||||
isError={error !== null}
|
||||
errorMessage={error instanceof Error ? error.message : "Spend per user failed to load"}
|
||||
maxBodyHeight={320}
|
||||
noDataMessage={teamIds.length === 0 ? "Select a team to see spend per user" : "No user spend in this range"}
|
||||
size="compact"
|
||||
|
|
|
|||
|
|
@ -1487,6 +1487,8 @@ export const cacheLeakageKeysCall = (req: DailyActivityRequest, limit?: number):
|
|||
|
||||
export type TeamUserSpendResponse = components["schemas"]["TeamUserSpendResponse"];
|
||||
|
||||
export const TEAM_SPEND_BY_USER_TIMEOUT_SECONDS = 60;
|
||||
|
||||
export const teamSpendByUserCall = async (
|
||||
accessToken: string,
|
||||
startTime: Date,
|
||||
|
|
@ -1499,6 +1501,7 @@ export const teamSpendByUserCall = async (
|
|||
start_date: formatDate(startTime),
|
||||
end_date: formatDate(endTime),
|
||||
team_ids: teamIds.join(","),
|
||||
timeout_seconds: TEAM_SPEND_BY_USER_TIMEOUT_SECONDS,
|
||||
},
|
||||
});
|
||||
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -75686,6 +75686,8 @@ export interface operations {
|
|||
team_ids?: string | null;
|
||||
start_date?: string | null;
|
||||
end_date?: string | null;
|
||||
/** @description Cancel the database query after this many seconds and return 503. Unset means no limit. */
|
||||
timeout_seconds?: number | null;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue