scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-28 06:23:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 94

refactor(proxy): paginated team tags report replaces group_by=tag

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mrinal 2026-10-21 15:49:53 +00:00
parent 19fd432f55
commit 9ea95f8deb
12 changed files with 419 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/constants.py
View file

@ -2234,6 +2234,8 @@ USAGE_TOP_API_KEYS_DEFAULT: Final[int] = 100
USAGE_TOP_API_KEYS_MAX: Final[int] = 1000
USAGE_KEY_PAGE_DEFAULT: Final[int] = 50
USAGE_KEY_PAGE_MAX: Final[int] = 100
USAGE_TAG_PAGE_DEFAULT: Final[int] = 50
USAGE_TAG_PAGE_MAX: Final[int] = 100
USAGE_USER_PAGE_DEFAULT: Final[int] = 50
USAGE_USER_PAGE_MAX: Final[int] = 100
USAGE_KEY_SEARCH_DEFAULT: Final[int] = 100

4
litellm/proxy/_types.py
View file

@ -601,6 +601,7 @@ class LiteLLMRoutes(enum.Enum):
KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value,
KeyManagementRoutes.TEAM_DAILY_ACTIVITY.value,
KeyManagementRoutes.TEAM_DAILY_ACTIVITY_AGGREGATED.value,
"/team/daily/activity/aggregated/tags",
"/team/daily/activity/aggregated/keys",
"/team/daily/activity/aggregated/search",
"/team/daily/activity/aggregated/model_top_keys",
@ -654,6 +655,7 @@ class LiteLLMRoutes(enum.Enum):
"/team/permissions_bulk_update",
"/team/daily/activity",
"/team/daily/activity/aggregated",
"/team/daily/activity/aggregated/tags",
"/team/daily/activity/aggregated/keys",
"/team/daily/activity/aggregated/search",
"/team/daily/activity/aggregated/model_top_keys",
@ -844,6 +846,7 @@ class LiteLLMRoutes(enum.Enum):
"/team/permissions_update",
"/team/daily/activity",
"/team/daily/activity/aggregated",
"/team/daily/activity/aggregated/tags",
"/team/daily/activity/aggregated/keys",
"/team/daily/activity/aggregated/search",
"/team/daily/activity/aggregated/model_top_keys",
@ -968,6 +971,7 @@ class LiteLLMRoutes(enum.Enum):
"/user/daily/activity/aggregated/users",
"/team/daily/activity",
"/team/daily/activity/aggregated",
"/team/daily/activity/aggregated/tags",
"/team/daily/activity/aggregated/keys",
"/team/daily/activity/aggregated/search",
"/team/daily/activity/aggregated/model_top_keys",

62
litellm/proxy/management_endpoints/daily_activity_routes.py
View file

@ -33,7 +33,7 @@ from litellm.proxy.management_endpoints.daily_activity_scopes import (
EntityQuery,
EntityScopeResolver,
ResolvedScope,
group_by_tag_error,
named_team_ids,
)
from litellm.proxy.management_endpoints.ptu_consumption import single_team_id, with_ptu_consumption
from litellm.proxy.management_endpoints.team_endpoints import aggregated_date_range_error
@ -45,6 +45,7 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import (
CacheLeakageKeysResponse,
DailyActivityKeyPageResponse,
DailyActivityKeySearchResponse,
DailyActivityTagPageResponse,
DailyActivityUserPageResponse,
KeyActivityRow,
KeyMetadata,
@ -102,9 +103,6 @@ async def _resolved_scope(
date_error: Final[InvalidDateRange | None] = _date_range_error(query, user_aggregated=user_aggregated)
if date_error is not None:
raise_public(date_error)
group_by_error: Final[str | None] = group_by_tag_error(query)
if group_by_error is not None:
raise_public(InvalidDateRange(reason=group_by_error))
result: ResolvedScope | ScopeDenied = await resolver.resolve(user_api_key_dict, query, prisma_client)
if isinstance(result, ScopeDenied):
raise_public(result)
@ -401,6 +399,60 @@ def _register_key_page_route(router: APIRouter, resolver: EntityScopeResolver, p
)
def _register_tag_page_route(router: APIRouter, resolver: EntityScopeResolver, prefix: str) -> None:
@management_endpoint_wrapper
async def tag_page(
entity_query: Annotated[EntityQuery, Depends(resolver.query)],
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
repository: Annotated[DailyActivityRepository, Depends(get_daily_activity_repository)],
prisma_client: Annotated[PrismaClient, Depends(get_daily_activity_prisma_client)],
offset: Annotated[int, Query(ge=0)] = 0,
limit: Annotated[int, Query(ge=1, le=constants.USAGE_TAG_PAGE_MAX)] = constants.USAGE_TAG_PAGE_DEFAULT,
) -> DailyActivityTagPageResponse:
try:
team_ids: Final = named_team_ids(entity_query)
if len(team_ids) != 1:
raise_public(
InvalidDateRange(reason=f"tags page needs exactly one team in team_ids, got {len(team_ids)}")
)
normalized_query: Final = replace(entity_query, entity_ids=team_ids)
resolved: Final = await _resolved_scope(
resolver,
normalized_query,
user_api_key_dict,
prisma_client,
user_aggregated=False,
)
page: Final = await repository.tag_page(
resolved.scope,
team_id=team_ids[0],
offset=offset,
limit=limit,
)
return DailyActivityTagPageResponse(
tags=list(page.rows),
total_tags=page.total_tags,
offset=offset,
limit=limit,
)
except HTTPException:
raise
except Exception as exc: # noqa: BLE001, LIT003 # keep sibling routes' 500 error mapping
verbose_proxy_logger.exception("Daily activity tag page failed: %s", exc)
raise HTTPException(status_code=500, detail={"error": f"Failed to fetch analytics: {exc}"})
router.add_api_route(
f"{prefix}/daily/activity/aggregated/tags",
tag_page,
methods=["GET"],
name=resolver.operation_names["tag_page"],
tags=list(resolver.tags),
dependencies=(Depends(user_api_key_auth),),
response_model=DailyActivityTagPageResponse,
include_in_schema=prefix != "/end_user",
)
def _register_search_route(router: APIRouter, resolver: EntityScopeResolver, prefix: str) -> None:
@management_endpoint_wrapper
async def search(
@ -652,6 +704,8 @@ def register_daily_activity_routes(router: APIRouter, resolver: EntityScopeResol
for prefix in resolver.route_prefixes:
_register_aggregated_route(router, resolver, prefix)
_register_key_page_route(router, resolver, prefix)
if resolver.entity == "team":
_register_tag_page_route(router, resolver, prefix)
_register_search_route(router, resolver, prefix)
_register_model_top_keys_route(router, resolver, prefix)
_register_export_route(router, resolver, prefix)

42
litellm/proxy/management_endpoints/daily_activity_scopes.py
View file

@ -1,7 +1,7 @@
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass, replace
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from typing import Final, Literal
from fastapi import Query
@ -30,7 +30,6 @@ class EntityQuery:
model: str | None
timezone_offset_minutes: int | None
include_current_utc_day: bool
group_by: "TeamGroupBy" = "team"
@dataclass(frozen=True, slots=True)
@ -40,7 +39,6 @@ class ResolvedScope:
Entity = Literal["user", "team", "tag", "organization", "customer", "agent"]
TeamGroupBy: TypeAlias = Literal["team", "tag"]
EntityScopeResolution = ResolvedScope | ScopeDenied
EntityScopeQuery = Callable[..., EntityQuery]
EntityScopeResolve = Callable[[UserAPIKeyAuth, EntityQuery, PrismaClient], Awaitable[EntityScopeResolution]]
@ -126,9 +124,8 @@ async def _resolve_team(
) -> EntityScopeResolution:
from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache
requested: Final = _named_team_ids(query) if query.group_by == "tag" else query.entity_ids
team_scope: Final = await resolve_team_daily_activity_scope(
team_ids=",".join(requested) if requested is not None else None,
team_ids=",".join(query.entity_ids) if query.entity_ids is not None else None,
exclude_team_ids=",".join(query.exclude_entity_ids) if query.exclude_entity_ids else None,
api_key=query.api_key,
user_api_key_dict=user_api_key_dict,
@ -136,7 +133,7 @@ async def _resolve_team(
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
team: Final = _build_scope(
return _build_scope(
TEAM_RESOLVER,
query,
team_scope.team_ids,
@ -144,22 +141,6 @@ async def _resolve_team(
team_scope.api_key_filter,
team_scope.team_alias_metadata,
)
if query.group_by == "team":
return team
permitted: Final = tuple(team_scope.team_ids or ())
if len(permitted) != 1:
raise ValueError(f"group_by=tag resolved to {len(permitted)} teams, expected exactly one")
return ResolvedScope(
scope=replace(
team.scope,
table=DailyActivityTable.TAG,
entity_id_field="tag",
entity_ids=None,
exclude_entity_ids=(),
team_id=permitted[0],
),
entity_metadata=None,
)
async def _resolve_tag(
@ -278,7 +259,6 @@ def _team_query(
api_key: str | None = None,
exclude_team_ids: str | None = None,
timezone: int | None = None,
group_by: TeamGroupBy = "team",
) -> EntityQuery:
return EntityQuery(
entity_ids=_query_ids(team_ids),
@ -289,24 +269,13 @@ def _team_query(
model=model,
timezone_offset_minutes=timezone,
include_current_utc_day=False,
group_by=group_by,
)
def _named_team_ids(query: EntityQuery) -> tuple[str, ...]:
def named_team_ids(query: EntityQuery) -> tuple[str, ...]:
return tuple(team_id for team_id in query.entity_ids or () if team_id)
def group_by_tag_error(query: EntityQuery) -> str | None:
"""Checked before any permission lookup, which fills in every team a non-admin belongs to."""
if query.group_by != "tag":
return None
named: Final = _named_team_ids(query)
if len(named) == 1:
return None
return f"group_by=tag needs exactly one team in team_ids, got {len(named)}"
def _tag_query(
start_date: str | None = None,
end_date: str | None = None,
@ -425,6 +394,7 @@ TEAM_RESOLVER = EntityScopeResolver(
"aggregated": "get_team_daily_activity_aggregated",
"search": "get_team_daily_activity_aggregated_search",
"key_page": "get_team_daily_activity_aggregated_keys",
"tag_page": "get_team_daily_activity_aggregated_tags",
"model_top_keys": "get_team_daily_activity_model_top_keys",
"export": "get_team_daily_activity_export",
}

16
litellm/repositories/daily_activity_repository.py
View file

@ -22,6 +22,7 @@ from litellm.repositories.daily_activity_sql import (
build_key_page_sql,
build_key_search_sql,
build_model_top_keys_sql,
build_tag_page_sql,
build_user_page_sql,
)
from litellm.repositories.prisma_protocols import TableActions
@ -40,6 +41,8 @@ from litellm.types.repositories.daily_activity import (
KeyPage,
KeySpendRow,
SpendLogsWindow,
TagPage,
TagSpendRow,
UserMetadataRow,
UserPage,
UserSpendRow,
@ -102,6 +105,7 @@ _GROUPING_ADAPTER: Final = TypeAdapter(tuple[GroupingSetsRow, ...])
_ENTITY_ADAPTER: Final = TypeAdapter(tuple[EntityRollupRow, ...])
_KEY_SPEND_ADAPTER: Final = TypeAdapter(tuple[KeySpendRow, ...])
_KEY_PAGE_TOTAL_ADAPTER: Final[TypeAdapter[int]] = TypeAdapter(int)
_TAG_SPEND_ADAPTER: Final = TypeAdapter(tuple[TagSpendRow, ...])
_USER_SPEND_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...])
_EXPORT_ADAPTER: Final = TypeAdapter(tuple[ExportRow, ...])
_METADATA_TAGS_ADAPTER: Final = TypeAdapter(list[StrictStr])
@ -208,6 +212,18 @@ class DailyActivityRepository:
rows: Final = _KEY_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("api_key") is not None))
return KeyPage(rows=rows, total_api_keys=total_api_keys)
async def tag_page(
self, scope: DailyActivityScope, *, team_id: str, offset: int, limit: int
) -> TagPage:
query: Final = build_tag_page_sql(scope, team_id=team_id, offset=offset, limit=limit)
result: Final = await self._query(query)
total_tags_value: Final = result[0].get("total_tags") if result else 0
total_tags: Final = (
_KEY_PAGE_TOTAL_ADAPTER.validate_python(total_tags_value) if total_tags_value is not None else 0
)
rows: Final = _TAG_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("tag") is not None))
return TagPage(rows=rows, total_tags=total_tags)
async def user_page(self, scope: DailyActivityScope, *, offset: int, limit: int) -> UserPage:
query: Final = build_user_page_sql(scope, offset=offset, limit=limit)
result: Final = await self._query(query)

41
litellm/repositories/daily_activity_sql.py
View file

@ -1,5 +1,5 @@
from collections.abc import Mapping
from dataclasses import dataclass
from dataclasses import dataclass, replace
from datetime import datetime, timedelta, timezone
from itertools import count, islice
from types import MappingProxyType
@ -68,7 +68,6 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
exclusion_index: Final = entity_index + int(has_entity_array)
model_index: Final = exclusion_index + int(bool(scope.exclude_entity_ids))
api_keys_index: Final = model_index + int(bool(scope.model))
team_index: Final = api_keys_index + int(bool(scope.api_keys))
conditions: Final = (
f"date >= ${start_index}",
f"date <= ${start_index + 1}",
@ -95,7 +94,6 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
if scope.api_keys
else ()
),
*((f"team_id = ${team_index}", _NOT_USER_AGENT_TAG) if scope.team_id is not None else ()),
)
params: Final = (
adjusted_start,
@ -104,7 +102,6 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
*((list(scope.exclude_entity_ids),) if scope.exclude_entity_ids else ()),
*((scope.model,) if scope.model else ()),
*((list(scope.api_keys),) if scope.api_keys else ()),
*((scope.team_id,) if scope.team_id is not None else ()),
)
return " AND ".join(conditions), params
@ -308,6 +305,42 @@ def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) ->
return SqlQuery(sql=sql, params=(*where_params, PTU_SENTINEL_API_KEY, limit, offset))
def build_tag_page_sql(scope: DailyActivityScope, *, team_id: str, offset: int, limit: int) -> SqlQuery:
if not 1 <= limit <= constants.USAGE_TAG_PAGE_MAX:
raise ValueError(f"limit must be between 1 and {constants.USAGE_TAG_PAGE_MAX}")
if offset < 0:
raise ValueError("offset must be non-negative")
tag_scope: Final = replace(
scope,
table=DailyActivityTable.TAG,
entity_id_field="tag",
entity_ids=None,
exclude_entity_ids=(),
)
where_clause, where_params = build_where_clause(tag_scope)
team_param: Final = len(where_params) + 1
sentinel_param: Final = team_param + 1
limit_param: Final = sentinel_param + 1
offset_param: Final = limit_param + 1
sql: Final = f"""
WITH ranked AS (
SELECT tag,{_key_spend_select()}, SUM(spend::numeric) AS rank_spend
FROM "{PRISMA_TO_PG_TABLE[DailyActivityTable.TAG]}"
WHERE {where_clause} AND team_id = ${team_param} AND api_key <> ${sentinel_param}
AND tag IS NOT NULL AND {_NOT_USER_AGENT_TAG}
GROUP BY tag
)
SELECT (SELECT COUNT(*) FROM ranked)::bigint AS total_tags, page.*
FROM (SELECT 1) AS one
LEFT JOIN LATERAL (
SELECT * FROM ranked
ORDER BY rank_spend DESC, tag ASC
LIMIT ${limit_param} OFFSET ${offset_param}
) AS page ON TRUE
"""
return SqlQuery(sql=sql, params=(*where_params, team_id, PTU_SENTINEL_API_KEY, limit, offset))
def _user_spend_select() -> str:
return """
COALESCE(SUM(spend), 0)::float AS spend,

10
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -7,6 +7,7 @@ from pydantic import ConfigDict, Field
from typing_extensions import TypedDict
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.repositories.daily_activity import TagSpendRow
class GroupByDimension(str, Enum):
@ -164,6 +165,15 @@ class DailyActivityKeyPageResponse(LiteLLMBaseModel):
limit: int
class DailyActivityTagPageResponse(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
tags: list[TagSpendRow] # mutable-ok: response contract requires a JSON array
total_tags: int
offset: int
limit: int
class UserActivityRow(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)

23
litellm/types/repositories/daily_activity.py
View file

@ -39,13 +39,10 @@ class DailyActivityScope:
model: str | None
timezone_offset_minutes: int | None
include_current_utc_day: bool = False
team_id: str | None = None
def __post_init__(self) -> None:
if self.entity_id_field not in _ENTITY_FIELDS[self.table]:
raise ValueError(f"Invalid entity_id_field {self.entity_id_field!r} for {self.table.value}")
if self.team_id is not None and self.table is not DailyActivityTable.TAG:
raise ValueError(f"team_id applies only to {DailyActivityTable.TAG.value}, not {self.table.value}")
@dataclass(frozen=True, slots=True)
@ -68,6 +65,26 @@ class KeyPage:
total_api_keys: int
@dataclass(frozen=True, slots=True)
class TagSpendRow:
tag: str
spend: float
prompt_tokens: int
completion_tokens: int
total_tokens: int
api_requests: int
successful_requests: int
failed_requests: int
cache_read_input_tokens: int
cache_creation_input_tokens: int
@dataclass(frozen=True, slots=True)
class TagPage:
rows: tuple[TagSpendRow, ...]
total_tags: int
@dataclass(frozen=True, slots=True)
class UserSpendRow:
user_id: str | None

66
tests/integration/spend/test_team_tag_spend_attribution.py
View file

@ -3,7 +3,7 @@ from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from integration._support.client import Gateway, eventually, object_value
from integration._support.client import Gateway, eventually, list_value, object_value
from integration._support.database import read_rows
_SPEND_PER_CALL: Final = 20 * 0.001 + 20 * 0.002
@ -22,29 +22,30 @@ def _tagged_chat(gateway: Gateway, model: str, key: str, tags: list[str]) -> Non
assert reply["usage"]["total_tokens"] == 40, reply
def _tag_breakdown(gateway: Gateway, team: str, *, key: str | None = None) -> dict[str, tuple[float, int]]:
"""The team's per-tag (spend, requests) from /team/daily/activity/aggregated?group_by=tag."""
def _tag_page(
gateway: Gateway,
team: str,
*,
offset: int = 0,
limit: int = 50,
key: str | None = None,
) -> dict[str, object]:
today: Final = datetime.now(timezone.utc)
response: Final = gateway.request(
"GET",
"/team/daily/activity/aggregated",
"/team/daily/activity/aggregated/tags",
params={
"team_ids": team,
"group_by": "tag",
"start_date": (today - timedelta(days=1)).strftime("%Y-%m-%d"),
"end_date": (today + timedelta(days=1)).strftime("%Y-%m-%d"),
"timezone": "0",
"offset": offset,
"limit": limit,
},
key=key,
)
assert response.status_code == 200, response.text
totals: dict[str, tuple[float, int]] = {}
for day in object_value(response.json())["results"]:
for tag, bucket in object_value(object_value(object_value(day)["breakdown"])["entities"]).items():
metrics = object_value(object_value(bucket)["metrics"])
spend, requests = totals.get(tag, (0.0, 0))
totals[tag] = (spend + float(metrics["spend"]), requests + int(metrics["api_requests"]))
return totals
return object_value(response.json())
def test_one_key_moved_between_teams_records_its_tag_once_per_team(gateway: Gateway) -> None:
@ -84,7 +85,7 @@ def test_one_key_moved_between_teams_records_its_tag_once_per_team(gateway: Gate
assert float(row["spend"]) == pytest.approx(calls_per_team[row["team_id"]] * _SPEND_PER_CALL), rows
def test_group_by_tag_reports_one_teams_spend_per_tag(gateway: Gateway) -> None:
def test_tag_page_reports_one_teams_spend_per_tag(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
shared: Final = f"cost-center:{uuid.uuid4().hex}"
@ -93,23 +94,36 @@ def test_group_by_tag_reports_one_teams_spend_per_tag(gateway: Gateway) -> None:
other_team: Final = scenario.team(models=[model])
key: Final = scenario.key(team_id=team, models=[model])
other_key: Final = scenario.key(team_id=other_team, models=[model])
no_team_key: Final = scenario.key(models=[model])
no_team_tag: Final = f"unassigned:{uuid.uuid4().hex}"
_tagged_chat(gateway, model, key, [shared, own, "User-Agent: integration/1.0"])
_tagged_chat(gateway, model, key, [shared])
_tagged_chat(gateway, model, other_key, [shared])
_tagged_chat(gateway, model, no_team_key, [no_team_tag])
eventually(
lambda: read_rows('SELECT api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag IN (%s, %s)', (shared, own)),
lambda values: sum(int(row["api_requests"]) for row in values) == 4,
lambda: read_rows(
'SELECT api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag IN (%s, %s, %s)',
(shared, own, no_team_tag),
),
lambda values: sum(int(row["api_requests"]) for row in values) == 5,
seconds=70,
)
breakdown: Final = _tag_breakdown(gateway, team)
assert set(breakdown) == {shared, own}, breakdown
assert breakdown[shared] == (pytest.approx(2 * _SPEND_PER_CALL), 2), breakdown
assert breakdown[own] == (pytest.approx(1 * _SPEND_PER_CALL), 1), breakdown
first_page: Final = _tag_page(gateway, team, limit=1)
next_page: Final = _tag_page(gateway, team, offset=1, limit=1)
assert first_page["total_tags"] == 2
assert next_page["total_tags"] == 2
first_tag: Final = object_value(list_value(first_page["tags"])[0])
next_tag: Final = object_value(list_value(next_page["tags"])[0])
assert first_tag["tag"] == shared
assert first_tag["spend"] == pytest.approx(2 * _SPEND_PER_CALL)
assert first_tag["api_requests"] == 2
assert next_tag["tag"] == own
assert next_tag["spend"] == pytest.approx(_SPEND_PER_CALL)
assert next_tag["api_requests"] == 1
def test_group_by_tag_keeps_the_team_routes_member_permissions(gateway: Gateway) -> None:
def test_tag_page_keeps_the_team_routes_member_permissions(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
tag: Final = f"cost-center:{uuid.uuid4().hex}"
@ -126,11 +140,15 @@ def test_group_by_tag_keeps_the_team_routes_member_permissions(gateway: Gateway)
seconds=70,
)
assert _tag_breakdown(gateway, team, key=member_key) == {tag: (pytest.approx(_SPEND_PER_CALL), 1)}
member_page: Final = _tag_page(gateway, team, key=member_key)
assert member_page["total_tags"] == 1
member_tag: Final = object_value(list_value(member_page["tags"])[0])
assert member_tag["tag"] == tag
assert member_tag["api_requests"] == 1
denied: Final = gateway.request(
"GET",
"/team/daily/activity/aggregated",
params={"team_ids": other_team, "group_by": "tag", "start_date": "2026-01-01", "end_date": "2026-01-02"},
"/team/daily/activity/aggregated/tags",
params={"team_ids": other_team, "start_date": "2026-01-01", "end_date": "2026-01-02"},
key=member_key,
)
assert denied.status_code == 404, denied.text

127
tests/unit/proxy/management_endpoints/test_daily_activity_routes.py
View file

@ -1,7 +1,7 @@
import csv
import io
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from dataclasses import dataclass, fields
from dataclasses import dataclass, fields, replace
from itertools import chain
from types import MappingProxyType, SimpleNamespace
from typing import Final
@ -35,6 +35,8 @@ from litellm.types.repositories.daily_activity import (
KeyMetadataRow,
KeyPage,
KeySpendRow,
TagPage,
TagSpendRow,
UserMetadataRow,
UserPage,
UserSpendRow,
@ -250,6 +252,8 @@ class _FakeRepository:
self.aggregated = AsyncMock(side_effect=self._aggregated)
self.key_page = AsyncMock(side_effect=self._key_page)
self.key_page_call: tuple[DailyActivityScope, int, int] | None = None
self.tag_page = AsyncMock(side_effect=self._tag_page)
self.tag_page_call: tuple[DailyActivityScope, str, int, int] | None = None
self.search_keys = AsyncMock(side_effect=self._search_keys)
self.model_top_keys = AsyncMock(side_effect=self._model_top_keys)
self.cache_leakage_keys = AsyncMock(side_effect=self._cache_leakage_keys)
@ -321,6 +325,29 @@ class _FakeRepository:
total_api_keys=len(api_keys),
)
async def _tag_page(
self, scope: DailyActivityScope, *, team_id: str, offset: int, limit: int
) -> TagPage:
self.tag_page_call = (scope, team_id, offset, limit)
tag_scope: Final = replace(
scope,
table=DailyActivityTable.TAG,
entity_id_field="tag",
entity_ids=None,
exclude_entity_ids=(),
)
rows: Final = self._matching_rows(tag_scope)
tags: Final = tuple(
sorted(
{row.entity_id for row in rows},
key=lambda tag: (-sum(row.spend for row in rows if row.entity_id == tag), tag),
)
)
return TagPage(
rows=tuple(_tag_spend_row(tag, rows) for tag in tags[offset : offset + limit]),
total_tags=len(tags),
)
async def _model_top_keys(
self, scope: DailyActivityScope, *, model_group: str, by_model_group: bool, limit: int
) -> tuple[KeySpendRow, ...]:
@ -445,6 +472,22 @@ def _key_spend_row(api_key: str, rows: Sequence[_Activity]) -> KeySpendRow:
)
def _tag_spend_row(tag: str, rows: Sequence[_Activity]) -> TagSpendRow:
matching: Final = tuple(row for row in rows if row.entity_id == tag)
return TagSpendRow(
tag=tag,
spend=sum(row.spend for row in matching),
prompt_tokens=sum(row.prompt_tokens for row in matching),
completion_tokens=sum(row.completion_tokens for row in matching),
total_tokens=sum(row.prompt_tokens + row.completion_tokens for row in matching),
api_requests=sum(row.api_requests for row in matching),
successful_requests=sum(row.successful_requests for row in matching),
failed_requests=sum(row.failed_requests for row in matching),
cache_read_input_tokens=sum(row.cache_read_input_tokens for row in matching),
cache_creation_input_tokens=sum(row.cache_creation_input_tokens for row in matching),
)
def _entity_rows_for_day(
date: str, rows: Sequence[_Activity], distinct_api_keys: int | None = None
) -> tuple[EntityRollupRow, ...]:
@ -1268,25 +1311,30 @@ def test_team_scope_applies_membership_and_user_key_filter(
@pytest.mark.parametrize("team_ids", ["team-a", "team-a,", ",team-a"])
def test_team_group_by_tag_reads_one_teams_tag_rows_with_the_members_key_filter(
def test_team_tag_page_normalizes_team_ids_and_applies_the_members_key_filter(
daily_activity_client: tuple[TestClient, _FakeRepository],
team_ids: str,
) -> None:
client, repository = daily_activity_client
response: Final = client.get(
"/team/daily/activity/aggregated",
params={**_entity_params("team_ids", team_ids), "group_by": "tag"},
"/team/daily/activity/aggregated/tags",
params={**_entity_params("team_ids", team_ids), "offset": 3, "limit": 7},
headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value, "x-user-id": "user-a"},
)
assert response.status_code == 200, response.text
scope: Final = repository.aggregated.call_args.args[0]
assert (scope.table, scope.entity_id_field, scope.entity_ids, scope.team_id, scope.api_keys) == (
DailyActivityTable.TAG,
"tag",
None,
"team-a",
assert response.json()["offset"] == 3
assert response.json()["limit"] == 7
tag_page_call: Final = repository.tag_page_call
assert tag_page_call is not None
scope, team_id, offset, limit = tag_page_call
assert (scope.table, scope.entity_ids, scope.api_keys, team_id, offset, limit) == (
DailyActivityTable.TEAM,
("team-a",),
("key-alpha",),
), scope
"team-a",
3,
7,
), tag_page_call
@pytest.mark.parametrize(
@ -1294,7 +1342,7 @@ def test_team_group_by_tag_reads_one_teams_tag_rows_with_the_members_key_filter(
[(None, 0), ("team-a,team-b", 2), ("team-a,,team-b", 2)],
)
@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER])
def test_team_group_by_tag_needs_exactly_one_named_team_for_every_caller(
def test_team_tag_page_needs_exactly_one_named_team_for_every_caller(
daily_activity_client: tuple[TestClient, _FakeRepository],
team_ids: str | None,
expected_count: int,
@ -1302,47 +1350,48 @@ def test_team_group_by_tag_needs_exactly_one_named_team_for_every_caller(
) -> None:
client, repository = daily_activity_client
response: Final = client.get(
"/team/daily/activity/aggregated",
params={**_DATE_PARAMS, "group_by": "tag", **({"team_ids": team_ids} if team_ids else {})},
"/team/daily/activity/aggregated/tags",
params={**_DATE_PARAMS, **({"team_ids": team_ids} if team_ids else {})},
headers={"x-user-role": role.value, "x-user-id": "user-a"},
)
assert response.status_code == 400, response.text
assert response.json() == {
"detail": {"error": f"group_by=tag needs exactly one team in team_ids, got {expected_count}"}
"detail": {"error": f"tags page needs exactly one team in team_ids, got {expected_count}"}
}
repository.aggregated.assert_not_called()
repository.tag_page.assert_not_awaited()
def test_team_group_by_tag_carries_no_ptu_hours(
daily_activity_client: tuple[TestClient, _FakeRepository],
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, _ = daily_activity_client
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
_inject_llm_router(client, _ptu_sized_router("rare-group"))
by_team: Final = client.get("/team/daily/activity/aggregated", params=_entity_params("team_ids", "team-a"))
by_tag: Final = client.get(
"/team/daily/activity/aggregated", params={**_entity_params("team_ids", "team-a"), "group_by": "tag"}
)
assert by_team.status_code == 200 and by_tag.status_code == 200, (by_team.text, by_tag.text)
ptu_hours: Final = (by_team.json()["metadata"]["total_ptu_hours"], by_tag.json()["metadata"]["total_ptu_hours"])
assert ptu_hours[0] > 0 and ptu_hours[1] == 0, f"(by team, by tag) total_ptu_hours = {ptu_hours}"
def test_team_route_without_group_by_reads_the_team_table_as_before(
def test_team_tag_page_returns_404_for_a_non_member(
daily_activity_client: tuple[TestClient, _FakeRepository],
) -> None:
client, repository = daily_activity_client
response: Final = client.get("/team/daily/activity/aggregated", params=_entity_params("team_ids", "team-a"))
assert response.status_code == 200, response.text
response: Final = client.get(
"/team/daily/activity/aggregated/tags",
params=_entity_params("team_ids", "team-b"),
headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value, "x-user-id": "user-a"},
)
assert response.status_code == 404, response.text
repository.tag_page.assert_not_awaited()
def test_team_aggregated_ignores_group_by_and_reads_the_team_table(
daily_activity_client: tuple[TestClient, _FakeRepository],
) -> None:
client, repository = daily_activity_client
params: Final = _entity_params("team_ids", "team-a")
without_group_by: Final = client.get("/team/daily/activity/aggregated", params=params)
with_group_by: Final = client.get(
"/team/daily/activity/aggregated",
params={**params, "group_by": "tag"},
)
assert without_group_by.status_code == with_group_by.status_code == 200
assert with_group_by.json() == without_group_by.json()
scope: Final = repository.aggregated.call_args.args[0]
assert (scope.table, scope.entity_id_field, scope.entity_ids, scope.team_id) == (
assert (scope.table, scope.entity_id_field, scope.entity_ids) == (
DailyActivityTable.TEAM,
"team_id",
("team-a",),
None,
), scope

67
tests/unit/repositories/test_daily_activity_sql.py
View file

@ -16,6 +16,7 @@ from litellm.repositories.daily_activity_sql import (
build_key_page_sql,
build_key_search_sql,
build_model_top_keys_sql,
build_tag_page_sql,
build_user_page_sql,
build_where_clause,
)
@ -112,27 +113,6 @@ def test_where_clause_distinguishes_no_filter_from_empty_membership(
assert params == expected_params
def test_where_clause_scopes_tag_rows_to_one_team_without_user_agent_tags() -> None:
scope = replace(
_scope(table=DailyActivityTable.TAG, entity_ids=None, api_keys=("key-1",), model="gpt-test"),
team_id="team-a",
)
sql, params = build_where_clause(scope)
assert sql == (
"date >= $1 AND date <= $2 AND model = $3 AND api_key = ANY($4::text[]) AND team_id = $5 "
"AND NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')"
)
assert params == ("2026-01-01", "2026-01-31", "gpt-test", ["key-1"], "team-a")
@pytest.mark.parametrize("table", [table for table in DailyActivityTable if table is not DailyActivityTable.TAG])
def test_scope_rejects_a_team_id_on_any_table_but_the_tag_table(table: DailyActivityTable) -> None:
with pytest.raises(ValueError, match="team_id applies only to litellm_dailytagspend"):
replace(_scope(table=table), team_id="team-a")
def test_key_page_sql_orders_exact_spend_and_binds_scope_before_page() -> None:
query = build_key_page_sql(_scope(), offset=7, limit=3)
@ -162,6 +142,51 @@ def test_key_page_sql_rejects_invalid_page_bounds(offset: int, limit: int, error
build_key_page_sql(_scope(), offset=offset, limit=limit)
def test_tag_page_sql_scopes_to_team_excludes_user_agents_and_orders_tags() -> None:
query: Final = build_tag_page_sql(
_scope(table=DailyActivityTable.TEAM, entity_ids=("team-a",), api_keys=("key-1",), model="gpt-test"),
team_id="team-a",
offset=7,
limit=3,
)
assert query.params == (
"2026-01-01",
"2026-01-31",
"gpt-test",
["key-1"],
"team-a",
PTU_SENTINEL_API_KEY,
3,
7,
)
assert 'FROM "LiteLLM_DailyTagSpend"' in query.sql
assert "team_id = $5" in query.sql
assert "api_key = ANY($4::text[])" in query.sql
assert "tag IS NOT NULL" in query.sql
assert "api_key <> $6" in query.sql
assert "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" in query.sql
assert "GROUP BY tag" in query.sql
assert "SUM(spend::numeric) AS rank_spend" in query.sql
assert "ORDER BY rank_spend DESC, tag ASC" in query.sql
assert "(SELECT COUNT(*) FROM ranked)::bigint AS total_tags" in query.sql
assert "LIMIT $7 OFFSET $8" in query.sql
assert '"tag" = ANY' not in query.sql
@pytest.mark.parametrize(
("offset", "limit", "error"),
(
(0, 0, "limit must be between"),
(0, constants.USAGE_TAG_PAGE_MAX + 1, "limit must be between"),
(-1, 1, "offset must be non-negative"),
),
)
def test_tag_page_sql_rejects_invalid_page_bounds(offset: int, limit: int, error: str) -> None:
with pytest.raises(ValueError, match=error):
build_tag_page_sql(_scope(table=DailyActivityTable.TEAM), team_id="team-a", offset=offset, limit=limit)
def test_user_page_sql_groups_by_user_id_with_deterministic_spend_order() -> None:
query: Final = build_user_page_sql(_scope(), offset=7, limit=3)

95
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -16631,6 +16631,23 @@ export interface paths {
patch?: never;
trace?: never;
};
"/team/daily/activity/aggregated/tags": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Get Team Daily Activity Aggregated Tags */
get: operations["get_team_daily_activity_aggregated_tags_team_daily_activity_aggregated_tags_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/team/daily/activity/export": {
parameters: {
query?: never;
@ -31256,6 +31273,17 @@ export interface components {
/** Api Keys */
api_keys: components["schemas"]["KeyActivityRow"][];
};
/** DailyActivityTagPageResponse */
DailyActivityTagPageResponse: {
/** Limit */
limit: number;
/** Offset */
offset: number;
/** Tags */
tags: components["schemas"]["TagSpendRow"][];
/** Total Tags */
total_tags: number;
};
/** DailyActivityUserPageResponse */
DailyActivityUserPageResponse: {
/** Limit */
@ -45854,6 +45882,29 @@ export interface components {
/** Tpm Limit */
tpm_limit?: number | null;
};
/** TagSpendRow */
TagSpendRow: {
/** Api Requests */
api_requests: number;
/** Cache Creation Input Tokens */
cache_creation_input_tokens: number;
/** Cache Read Input Tokens */
cache_read_input_tokens: number;
/** Completion Tokens */
completion_tokens: number;
/** Failed Requests */
failed_requests: number;
/** Prompt Tokens */
prompt_tokens: number;
/** Spend */
spend: number;
/** Successful Requests */
successful_requests: number;
/** Tag */
tag: string;
/** Total Tokens */
total_tokens: number;
};
/**
* TagSummaryMetrics
* @description Summary metrics for a tag
@ -72862,7 +72913,6 @@ export interface operations {
api_key?: string | null;
exclude_team_ids?: string | null;
timezone?: number | null;
group_by?: "team" | "tag";
};
header?: never;
path?: never;
@ -72902,7 +72952,6 @@ export interface operations {
api_key?: string | null;
exclude_team_ids?: string | null;
timezone?: number | null;
group_by?: "team" | "tag";
};
header?: never;
path?: never;
@ -72943,7 +72992,6 @@ export interface operations {
api_key?: string | null;
exclude_team_ids?: string | null;
timezone?: number | null;
group_by?: "team" | "tag";
};
header?: never;
path?: never;
@ -72983,7 +73031,6 @@ export interface operations {
api_key?: string | null;
exclude_team_ids?: string | null;
timezone?: number | null;
group_by?: "team" | "tag";
};
header?: never;
path?: never;
@ -73011,6 +73058,45 @@ export interface operations {
};
};
};
get_team_daily_activity_aggregated_tags_team_daily_activity_aggregated_tags_get: {
parameters: {
query?: {
offset?: number;
limit?: number;
team_ids?: string | null;
start_date?: string | null;
end_date?: string | null;
model?: string | null;
api_key?: string | null;
exclude_team_ids?: string | null;
timezone?: number | null;
};
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["DailyActivityTagPageResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_team_daily_activity_export_team_daily_activity_export_get: {
parameters: {
query?: {
@ -73023,7 +73109,6 @@ export interface operations {
api_key?: string | null;
exclude_team_ids?: string | null;
timezone?: number | null;
group_by?: "team" | "tag";
};
header?: never;
path?: never;