diff --git a/litellm/constants.py b/litellm/constants.py index d1daf2ee2d1..2072738053e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2268,6 +2268,8 @@ USAGE_TOP_API_KEYS_DEFAULT: Final[int] = 100 USAGE_TOP_API_KEYS_MAX: Final[int] = 1000 USAGE_KEY_PAGE_DEFAULT: Final[int] = 50 USAGE_KEY_PAGE_MAX: Final[int] = 100 +USAGE_TAG_PAGE_DEFAULT: Final[int] = 50 +USAGE_TAG_PAGE_MAX: Final[int] = 100 USAGE_USER_PAGE_DEFAULT: Final[int] = 50 USAGE_USER_PAGE_MAX: Final[int] = 100 USAGE_KEY_SEARCH_DEFAULT: Final[int] = 100 diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ddf1760a554..81ff25ee31b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -606,6 +606,7 @@ class LiteLLMRoutes(enum.Enum): KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value, KeyManagementRoutes.TEAM_DAILY_ACTIVITY.value, KeyManagementRoutes.TEAM_DAILY_ACTIVITY_AGGREGATED.value, + "/team/daily/activity/aggregated/tags", "/team/daily/activity/aggregated/keys", "/team/daily/activity/aggregated/search", "/team/daily/activity/aggregated/model_top_keys", @@ -659,6 +660,7 @@ class LiteLLMRoutes(enum.Enum): "/team/permissions_bulk_update", "/team/daily/activity", "/team/daily/activity/aggregated", + "/team/daily/activity/aggregated/tags", "/team/daily/activity/aggregated/keys", "/team/daily/activity/aggregated/search", "/team/daily/activity/aggregated/model_top_keys", @@ -849,6 +851,7 @@ class LiteLLMRoutes(enum.Enum): "/team/permissions_update", "/team/daily/activity", "/team/daily/activity/aggregated", + "/team/daily/activity/aggregated/tags", "/team/daily/activity/aggregated/keys", "/team/daily/activity/aggregated/search", "/team/daily/activity/aggregated/model_top_keys", @@ -973,6 +976,7 @@ class LiteLLMRoutes(enum.Enum): "/user/daily/activity/aggregated/users", "/team/daily/activity", "/team/daily/activity/aggregated", + "/team/daily/activity/aggregated/tags", "/team/daily/activity/aggregated/keys", "/team/daily/activity/aggregated/search", "/team/daily/activity/aggregated/model_top_keys", diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 842873129e4..7824540c54c 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -33,6 +33,7 @@ from litellm.proxy.management_endpoints.daily_activity_scopes import ( EntityQuery, EntityScopeResolver, ResolvedScope, + named_team_ids, ) from litellm.proxy.management_endpoints.ptu_consumption import single_team_id, with_ptu_consumption from litellm.proxy.management_endpoints.team_endpoints import aggregated_date_range_error @@ -44,6 +45,7 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( CacheLeakageKeysResponse, DailyActivityKeyPageResponse, DailyActivityKeySearchResponse, + DailyActivityTagPageResponse, DailyActivityUserPageResponse, KeyActivityRow, KeyMetadata, @@ -55,10 +57,12 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( UserActivityRow, ) from litellm.types.repositories.daily_activity import ( + DailyActivityTable, ExportRow, ExportType, KeyMetadataRow, KeySpendRow, + TagPage, UserMetadataRow, UserSpendRow, ) @@ -302,7 +306,8 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver, include_entity_breakdown=resolver.include_entity_breakdown, api_key_limit=api_key_limit, ) - if resolver.entity != "team": + # PTU hours are a whole-team figure, so a team's per-tag breakdown carries none. + if resolver.entity != "team" or resolved.scope.table is not DailyActivityTable.TEAM: return activity return with_ptu_consumption(activity, llm_router, single_team_id(resolved.scope.entity_ids)) except HTTPException: @@ -395,6 +400,59 @@ def _register_key_page_route(router: APIRouter, resolver: EntityScopeResolver, p ) +def _register_tag_page_route(router: APIRouter, resolver: EntityScopeResolver, prefix: str) -> None: + @management_endpoint_wrapper + async def tag_page( + entity_query: Annotated[EntityQuery, Depends(resolver.query)], + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[DailyActivityRepository, Depends(get_daily_activity_repository)], + prisma_client: Annotated[PrismaClient, Depends(get_daily_activity_prisma_client)], + offset: Annotated[int, Query(ge=0)] = 0, + limit: Annotated[int, Query(ge=1, le=constants.USAGE_TAG_PAGE_MAX)] = constants.USAGE_TAG_PAGE_DEFAULT, + ) -> DailyActivityTagPageResponse: + try: + team_ids: Final = named_team_ids(entity_query) + if len(team_ids) != 1: + raise_public( + InvalidDateRange(reason=f"tags page needs exactly one team in team_ids, got {len(team_ids)}") + ) + normalized_query: Final = replace(entity_query, entity_ids=team_ids) + resolved: Final = await _resolved_scope( + resolver, + normalized_query, + user_api_key_dict, + prisma_client, + user_aggregated=False, + ) + page: Final = ( + TagPage(rows=(), total_tags=0) + if team_ids[0] in resolved.scope.exclude_entity_ids + else await repository.tag_page(resolved.scope, team_id=team_ids[0], offset=offset, limit=limit) + ) + return DailyActivityTagPageResponse( + tags=list(page.rows), + total_tags=page.total_tags, + offset=offset, + limit=limit, + ) + except HTTPException: + raise + except Exception as exc: # noqa: BLE001, LIT003 # keep sibling routes' 500 error mapping + verbose_proxy_logger.exception("Daily activity tag page failed: %s", exc) + raise HTTPException(status_code=500, detail={"error": f"Failed to fetch analytics: {exc}"}) + + router.add_api_route( + f"{prefix}/daily/activity/aggregated/tags", + tag_page, + methods=["GET"], + name=resolver.operation_names["tag_page"], + tags=list(resolver.tags), + dependencies=(Depends(user_api_key_auth),), + response_model=DailyActivityTagPageResponse, + include_in_schema=prefix != "/end_user", + ) + + def _register_search_route(router: APIRouter, resolver: EntityScopeResolver, prefix: str) -> None: @management_endpoint_wrapper async def search( @@ -646,6 +704,8 @@ def register_daily_activity_routes(router: APIRouter, resolver: EntityScopeResol for prefix in resolver.route_prefixes: _register_aggregated_route(router, resolver, prefix) _register_key_page_route(router, resolver, prefix) + if resolver.entity == "team": + _register_tag_page_route(router, resolver, prefix) _register_search_route(router, resolver, prefix) _register_model_top_keys_route(router, resolver, prefix) _register_export_route(router, resolver, prefix) diff --git a/litellm/proxy/management_endpoints/daily_activity_scopes.py b/litellm/proxy/management_endpoints/daily_activity_scopes.py index bc373388a4e..5ddb12114a6 100644 --- a/litellm/proxy/management_endpoints/daily_activity_scopes.py +++ b/litellm/proxy/management_endpoints/daily_activity_scopes.py @@ -272,6 +272,10 @@ def _team_query( ) +def named_team_ids(query: EntityQuery) -> tuple[str, ...]: + return tuple(team_id for team_id in query.entity_ids or () if team_id) + + def _tag_query( start_date: str | None = None, end_date: str | None = None, @@ -390,6 +394,7 @@ TEAM_RESOLVER = EntityScopeResolver( "aggregated": "get_team_daily_activity_aggregated", "search": "get_team_daily_activity_aggregated_search", "key_page": "get_team_daily_activity_aggregated_keys", + "tag_page": "get_team_daily_activity_aggregated_tags", "model_top_keys": "get_team_daily_activity_model_top_keys", "export": "get_team_daily_activity_export", } diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 10c603fe56f..55530aeae48 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -22,6 +22,7 @@ from litellm.repositories.daily_activity_sql import ( build_key_page_sql, build_key_search_sql, build_model_top_keys_sql, + build_tag_page_sql, build_user_page_sql, ) from litellm.repositories.prisma_protocols import TableActions @@ -40,6 +41,8 @@ from litellm.types.repositories.daily_activity import ( KeyPage, KeySpendRow, SpendLogsWindow, + TagPage, + TagSpendRow, UserMetadataRow, UserPage, UserSpendRow, @@ -102,6 +105,7 @@ _GROUPING_ADAPTER: Final = TypeAdapter(tuple[GroupingSetsRow, ...]) _ENTITY_ADAPTER: Final = TypeAdapter(tuple[EntityRollupRow, ...]) _KEY_SPEND_ADAPTER: Final = TypeAdapter(tuple[KeySpendRow, ...]) _KEY_PAGE_TOTAL_ADAPTER: Final[TypeAdapter[int]] = TypeAdapter(int) +_TAG_SPEND_ADAPTER: Final = TypeAdapter(tuple[TagSpendRow, ...]) _USER_SPEND_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...]) _EXPORT_ADAPTER: Final = TypeAdapter(tuple[ExportRow, ...]) _METADATA_TAGS_ADAPTER: Final = TypeAdapter(list[StrictStr]) @@ -208,6 +212,16 @@ class DailyActivityRepository: rows: Final = _KEY_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("api_key") is not None)) return KeyPage(rows=rows, total_api_keys=total_api_keys) + async def tag_page(self, scope: DailyActivityScope, *, team_id: str, offset: int, limit: int) -> TagPage: + query: Final = build_tag_page_sql(scope, team_id=team_id, offset=offset, limit=limit) + result: Final = await self._query(query) + total_tags_value: Final = result[0].get("total_tags") if result else 0 + total_tags: Final = ( + _KEY_PAGE_TOTAL_ADAPTER.validate_python(total_tags_value) if total_tags_value is not None else 0 + ) + rows: Final = _TAG_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("tag") is not None)) + return TagPage(rows=rows, total_tags=total_tags) + async def user_page(self, scope: DailyActivityScope, *, offset: int, limit: int) -> UserPage: query: Final = build_user_page_sql(scope, offset=offset, limit=limit) result: Final = await self._query(query) diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 3d27e7d937e..7c3eb568971 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -1,5 +1,5 @@ from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timedelta, timezone from itertools import count, islice from types import MappingProxyType @@ -106,6 +106,9 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu return " AND ".join(conditions), params +_NOT_USER_AGENT_TAG: Final = "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" + + def _ptu_flat_cost_select(table: DailyActivityTable, *, aggregate: bool = True) -> str: if table is DailyActivityTable.TEAM: return "SUM(ptu_flat_cost)::float AS ptu_flat_cost" if aggregate else "SUM(scoped.ptu_flat_cost)::float" @@ -301,6 +304,42 @@ def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> return SqlQuery(sql=sql, params=(*where_params, PTU_SENTINEL_API_KEY, limit, offset)) +def build_tag_page_sql(scope: DailyActivityScope, *, team_id: str, offset: int, limit: int) -> SqlQuery: + if not 1 <= limit <= constants.USAGE_TAG_PAGE_MAX: + raise ValueError(f"limit must be between 1 and {constants.USAGE_TAG_PAGE_MAX}") + if offset < 0: + raise ValueError("offset must be non-negative") + tag_scope: Final = replace( + scope, + table=DailyActivityTable.TAG, + entity_id_field="tag", + entity_ids=None, + exclude_entity_ids=(), + ) + where_clause, where_params = build_where_clause(tag_scope) + team_param: Final = len(where_params) + 1 + sentinel_param: Final = team_param + 1 + limit_param: Final = sentinel_param + 1 + offset_param: Final = limit_param + 1 + sql: Final = f""" + WITH ranked AS ( + SELECT tag,{_key_spend_select()}, SUM(spend::numeric) AS rank_spend + FROM "{PRISMA_TO_PG_TABLE[DailyActivityTable.TAG]}" + WHERE {where_clause} AND team_id = ${team_param} AND api_key <> ${sentinel_param} + AND tag IS NOT NULL AND {_NOT_USER_AGENT_TAG} + GROUP BY tag + ) + SELECT (SELECT COUNT(*) FROM ranked)::bigint AS total_tags, page.* + FROM (SELECT 1) AS one + LEFT JOIN LATERAL ( + SELECT * FROM ranked + ORDER BY rank_spend DESC, tag ASC + LIMIT ${limit_param} OFFSET ${offset_param} + ) AS page ON TRUE + """ + return SqlQuery(sql=sql, params=(*where_params, team_id, PTU_SENTINEL_API_KEY, limit, offset)) + + def _user_spend_select() -> str: return """ COALESCE(SUM(spend), 0)::float AS spend, diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 213c911cfdc..f18b0264fbe 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -6,6 +6,7 @@ from pydantic import ConfigDict, Field from typing_extensions import TypedDict from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.repositories.daily_activity import TagSpendRow class GroupByDimension(str, Enum): @@ -163,6 +164,15 @@ class DailyActivityKeyPageResponse(LiteLLMBaseModel): limit: int +class DailyActivityTagPageResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + tags: list[TagSpendRow] # mutable-ok: response contract requires a JSON array + total_tags: int + offset: int + limit: int + + class UserActivityRow(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index 4111b83a891..2746b417965 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -65,6 +65,26 @@ class KeyPage: total_api_keys: int +@dataclass(frozen=True, slots=True) +class TagSpendRow: + tag: str + spend: float + prompt_tokens: int + completion_tokens: int + total_tokens: int + api_requests: int + successful_requests: int + failed_requests: int + cache_read_input_tokens: int + cache_creation_input_tokens: int + + +@dataclass(frozen=True, slots=True) +class TagPage: + rows: tuple[TagSpendRow, ...] + total_tags: int + + @dataclass(frozen=True, slots=True) class UserSpendRow: user_id: str | None diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt index 00f8f3b6031..473a44d20de 100644 --- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt +++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt @@ -48,6 +48,7 @@ GET /tag/user-agent/per-user-analytics GET /tag/wau GET /team/daily/activity GET /team/daily/activity/aggregated +GET /team/daily/activity/aggregated/tags # proxy-only analytics report; no provider resource GET /team/daily/activity/aggregated/keys GET /team/daily/activity/aggregated/model_top_keys GET /team/daily/activity/aggregated/search diff --git a/tests/integration/spend/test_team_tag_spend_attribution.py b/tests/integration/spend/test_team_tag_spend_attribution.py index d82394e143b..ae3389a7e88 100644 --- a/tests/integration/spend/test_team_tag_spend_attribution.py +++ b/tests/integration/spend/test_team_tag_spend_attribution.py @@ -1,13 +1,53 @@ import uuid +from datetime import datetime, timedelta, timezone from typing import Final import pytest -from integration._support.client import Gateway, eventually +from integration._support.client import Gateway, eventually, list_value, object_value from integration._support.database import read_rows _SPEND_PER_CALL: Final = 20 * 0.001 + 20 * 0.002 +def _tagged_chat(gateway: Gateway, model: str, key: str, tags: list[str]) -> None: + reply: Final = gateway.post( + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"tagged {uuid.uuid4().hex}"}], + "metadata": {"tags": tags}, + }, + key=key, + ) + assert reply["usage"]["total_tokens"] == 40, reply + + +def _tag_page( + gateway: Gateway, + team: str, + *, + offset: int = 0, + limit: int = 50, + key: str | None = None, +) -> dict[str, object]: + today: Final = datetime.now(timezone.utc) + response: Final = gateway.request( + "GET", + "/team/daily/activity/aggregated/tags", + params={ + "team_ids": team, + "start_date": (today - timedelta(days=1)).strftime("%Y-%m-%d"), + "end_date": (today + timedelta(days=1)).strftime("%Y-%m-%d"), + "timezone": "0", + "offset": offset, + "limit": limit, + }, + key=key, + ) + assert response.status_code == 200, response.text + return object_value(response.json()) + + def test_one_key_moved_between_teams_records_its_tag_once_per_team(gateway: Gateway) -> None: with gateway.scenario() as scenario: model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) @@ -43,3 +83,72 @@ def test_one_key_moved_between_teams_records_its_tag_once_per_team(gateway: Gate ) for row in rows: assert float(row["spend"]) == pytest.approx(calls_per_team[row["team_id"]] * _SPEND_PER_CALL), rows + + +def test_tag_page_reports_one_teams_spend_per_tag(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + shared: Final = f"cost-center:{uuid.uuid4().hex}" + own: Final = f"project:{uuid.uuid4().hex}" + team: Final = scenario.team(models=[model]) + other_team: Final = scenario.team(models=[model]) + key: Final = scenario.key(team_id=team, models=[model]) + other_key: Final = scenario.key(team_id=other_team, models=[model]) + no_team_key: Final = scenario.key(models=[model]) + no_team_tag: Final = f"unassigned:{uuid.uuid4().hex}" + _tagged_chat(gateway, model, key, [shared, own, "User-Agent: integration/1.0"]) + _tagged_chat(gateway, model, key, [shared]) + _tagged_chat(gateway, model, other_key, [shared]) + _tagged_chat(gateway, model, no_team_key, [no_team_tag]) + eventually( + lambda: read_rows( + 'SELECT api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag IN (%s, %s, %s)', + (shared, own, no_team_tag), + ), + lambda values: sum(int(row["api_requests"]) for row in values) == 5, + seconds=70, + ) + + first_page: Final = _tag_page(gateway, team, limit=1) + next_page: Final = _tag_page(gateway, team, offset=1, limit=1) + assert first_page["total_tags"] == 2 + assert next_page["total_tags"] == 2 + first_tag: Final = object_value(list_value(first_page["tags"])[0]) + next_tag: Final = object_value(list_value(next_page["tags"])[0]) + assert first_tag["tag"] == shared + assert first_tag["spend"] == pytest.approx(2 * _SPEND_PER_CALL) + assert first_tag["api_requests"] == 2 + assert next_tag["tag"] == own + assert next_tag["spend"] == pytest.approx(_SPEND_PER_CALL) + assert next_tag["api_requests"] == 1 + + +def test_tag_page_keeps_the_team_routes_member_permissions(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + tag: Final = f"cost-center:{uuid.uuid4().hex}" + team: Final = scenario.team(models=[model]) + other_team: Final = scenario.team(models=[model]) + member: Final = scenario.member(team) + member_key: Final = scenario.key(user_id=member, team_id=team, models=[model]) + teammate_key: Final = scenario.key(team_id=team, models=[model]) + _tagged_chat(gateway, model, member_key, [tag]) + _tagged_chat(gateway, model, teammate_key, [tag]) + eventually( + lambda: read_rows('SELECT api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag = %s', (tag,)), + lambda values: sum(int(row["api_requests"]) for row in values) == 2, + seconds=70, + ) + + member_page: Final = _tag_page(gateway, team, key=member_key) + assert member_page["total_tags"] == 1 + member_tag: Final = object_value(list_value(member_page["tags"])[0]) + assert member_tag["tag"] == tag + assert member_tag["api_requests"] == 1 + denied: Final = gateway.request( + "GET", + "/team/daily/activity/aggregated/tags", + params={"team_ids": other_team, "start_date": "2026-01-01", "end_date": "2026-01-02"}, + key=member_key, + ) + assert denied.status_code == 404, denied.text diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index 8fbead8c01c..0ca61a47da1 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -1,7 +1,7 @@ import csv import io from collections.abc import AsyncIterator, Iterator, Mapping, Sequence -from dataclasses import dataclass, fields +from dataclasses import dataclass, fields, replace from itertools import chain from types import MappingProxyType, SimpleNamespace from typing import Final @@ -35,6 +35,8 @@ from litellm.types.repositories.daily_activity import ( KeyMetadataRow, KeyPage, KeySpendRow, + TagPage, + TagSpendRow, UserMetadataRow, UserPage, UserSpendRow, @@ -250,6 +252,8 @@ class _FakeRepository: self.aggregated = AsyncMock(side_effect=self._aggregated) self.key_page = AsyncMock(side_effect=self._key_page) self.key_page_call: tuple[DailyActivityScope, int, int] | None = None + self.tag_page = AsyncMock(side_effect=self._tag_page) + self.tag_page_call: tuple[DailyActivityScope, str, int, int] | None = None self.search_keys = AsyncMock(side_effect=self._search_keys) self.model_top_keys = AsyncMock(side_effect=self._model_top_keys) self.cache_leakage_keys = AsyncMock(side_effect=self._cache_leakage_keys) @@ -321,6 +325,29 @@ class _FakeRepository: total_api_keys=len(api_keys), ) + async def _tag_page( + self, scope: DailyActivityScope, *, team_id: str, offset: int, limit: int + ) -> TagPage: + self.tag_page_call = (scope, team_id, offset, limit) + tag_scope: Final = replace( + scope, + table=DailyActivityTable.TAG, + entity_id_field="tag", + entity_ids=None, + exclude_entity_ids=(), + ) + rows: Final = self._matching_rows(tag_scope) + tags: Final = tuple( + sorted( + {row.entity_id for row in rows}, + key=lambda tag: (-sum(row.spend for row in rows if row.entity_id == tag), tag), + ) + ) + return TagPage( + rows=tuple(_tag_spend_row(tag, rows) for tag in tags[offset : offset + limit]), + total_tags=len(tags), + ) + async def _model_top_keys( self, scope: DailyActivityScope, *, model_group: str, by_model_group: bool, limit: int ) -> tuple[KeySpendRow, ...]: @@ -445,6 +472,22 @@ def _key_spend_row(api_key: str, rows: Sequence[_Activity]) -> KeySpendRow: ) +def _tag_spend_row(tag: str, rows: Sequence[_Activity]) -> TagSpendRow: + matching: Final = tuple(row for row in rows if row.entity_id == tag) + return TagSpendRow( + tag=tag, + spend=sum(row.spend for row in matching), + prompt_tokens=sum(row.prompt_tokens for row in matching), + completion_tokens=sum(row.completion_tokens for row in matching), + total_tokens=sum(row.prompt_tokens + row.completion_tokens for row in matching), + api_requests=sum(row.api_requests for row in matching), + successful_requests=sum(row.successful_requests for row in matching), + failed_requests=sum(row.failed_requests for row in matching), + cache_read_input_tokens=sum(row.cache_read_input_tokens for row in matching), + cache_creation_input_tokens=sum(row.cache_creation_input_tokens for row in matching), + ) + + def _entity_rows_for_day( date: str, rows: Sequence[_Activity], distinct_api_keys: int | None = None ) -> tuple[EntityRollupRow, ...]: @@ -1267,6 +1310,105 @@ def test_team_scope_applies_membership_and_user_key_filter( assert repository.aggregated.call_args.kwargs["include_entity_breakdown"] is True +@pytest.mark.parametrize("team_ids", ["team-a", "team-a,", ",team-a"]) +def test_team_tag_page_normalizes_team_ids_and_applies_the_members_key_filter( + daily_activity_client: tuple[TestClient, _FakeRepository], + team_ids: str, +) -> None: + client, repository = daily_activity_client + response: Final = client.get( + "/team/daily/activity/aggregated/tags", + params={**_entity_params("team_ids", team_ids), "offset": 3, "limit": 7}, + headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value, "x-user-id": "user-a"}, + ) + assert response.status_code == 200, response.text + assert response.json()["offset"] == 3 + assert response.json()["limit"] == 7 + tag_page_call: Final = repository.tag_page_call + assert tag_page_call is not None + scope, team_id, offset, limit = tag_page_call + assert (scope.table, scope.entity_ids, scope.api_keys, team_id, offset, limit) == ( + DailyActivityTable.TEAM, + ("team-a",), + ("key-alpha",), + "team-a", + 3, + 7, + ), tag_page_call + + +@pytest.mark.parametrize( + ("team_ids", "expected_count"), + [(None, 0), ("team-a,team-b", 2), ("team-a,,team-b", 2)], +) +@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER]) +def test_team_tag_page_needs_exactly_one_named_team_for_every_caller( + daily_activity_client: tuple[TestClient, _FakeRepository], + team_ids: str | None, + expected_count: int, + role: LitellmUserRoles, +) -> None: + client, repository = daily_activity_client + response: Final = client.get( + "/team/daily/activity/aggregated/tags", + params={**_DATE_PARAMS, **({"team_ids": team_ids} if team_ids else {})}, + headers={"x-user-role": role.value, "x-user-id": "user-a"}, + ) + assert response.status_code == 400, response.text + assert response.json() == { + "detail": {"error": f"tags page needs exactly one team in team_ids, got {expected_count}"} + } + repository.tag_page.assert_not_awaited() + + +def test_team_tag_page_returns_404_for_a_non_member( + daily_activity_client: tuple[TestClient, _FakeRepository], +) -> None: + client, repository = daily_activity_client + response: Final = client.get( + "/team/daily/activity/aggregated/tags", + params=_entity_params("team_ids", "team-b"), + headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value, "x-user-id": "user-a"}, + ) + + assert response.status_code == 404, response.text + repository.tag_page.assert_not_awaited() + + +def test_team_tag_page_is_empty_for_a_team_that_is_also_excluded( + daily_activity_client: tuple[TestClient, _FakeRepository], +) -> None: + client, repository = daily_activity_client + response: Final = client.get( + "/team/daily/activity/aggregated/tags", + params={**_entity_params("team_ids", "team-a"), "exclude_team_ids": "team-a", "offset": 0, "limit": 5}, + ) + + assert response.status_code == 200, response.text + assert response.json() == {"tags": [], "total_tags": 0, "offset": 0, "limit": 5} + repository.tag_page.assert_not_awaited() + + +def test_team_aggregated_ignores_group_by_and_reads_the_team_table( + daily_activity_client: tuple[TestClient, _FakeRepository], +) -> None: + client, repository = daily_activity_client + params: Final = _entity_params("team_ids", "team-a") + without_group_by: Final = client.get("/team/daily/activity/aggregated", params=params) + with_group_by: Final = client.get( + "/team/daily/activity/aggregated", + params={**params, "group_by": "tag"}, + ) + assert without_group_by.status_code == with_group_by.status_code == 200 + assert with_group_by.json() == without_group_by.json() + scope: Final = repository.aggregated.call_args.args[0] + assert (scope.table, scope.entity_id_field, scope.entity_ids) == ( + DailyActivityTable.TEAM, + "team_id", + ("team-a",), + ), scope + + def test_team_scope_does_not_allow_an_unowned_api_key( daily_activity_client: tuple[TestClient, _FakeRepository], ) -> None: diff --git a/tests/unit/repositories/test_daily_activity_sql.py b/tests/unit/repositories/test_daily_activity_sql.py index c57db08e976..ae10b931c69 100644 --- a/tests/unit/repositories/test_daily_activity_sql.py +++ b/tests/unit/repositories/test_daily_activity_sql.py @@ -1,3 +1,4 @@ +from dataclasses import replace from datetime import datetime, timezone from typing import Final @@ -15,6 +16,7 @@ from litellm.repositories.daily_activity_sql import ( build_key_page_sql, build_key_search_sql, build_model_top_keys_sql, + build_tag_page_sql, build_user_page_sql, build_where_clause, ) @@ -140,6 +142,51 @@ def test_key_page_sql_rejects_invalid_page_bounds(offset: int, limit: int, error build_key_page_sql(_scope(), offset=offset, limit=limit) +def test_tag_page_sql_scopes_to_team_excludes_user_agents_and_orders_tags() -> None: + query: Final = build_tag_page_sql( + _scope(table=DailyActivityTable.TEAM, entity_ids=("team-a",), api_keys=("key-1",), model="gpt-test"), + team_id="team-a", + offset=7, + limit=3, + ) + + assert query.params == ( + "2026-01-01", + "2026-01-31", + "gpt-test", + ["key-1"], + "team-a", + PTU_SENTINEL_API_KEY, + 3, + 7, + ) + assert 'FROM "LiteLLM_DailyTagSpend"' in query.sql + assert "team_id = $5" in query.sql + assert "api_key = ANY($4::text[])" in query.sql + assert "tag IS NOT NULL" in query.sql + assert "api_key <> $6" in query.sql + assert "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" in query.sql + assert "GROUP BY tag" in query.sql + assert "SUM(spend::numeric) AS rank_spend" in query.sql + assert "ORDER BY rank_spend DESC, tag ASC" in query.sql + assert "(SELECT COUNT(*) FROM ranked)::bigint AS total_tags" in query.sql + assert "LIMIT $7 OFFSET $8" in query.sql + assert '"tag" = ANY' not in query.sql + + +@pytest.mark.parametrize( + ("offset", "limit", "error"), + ( + (0, 0, "limit must be between"), + (0, constants.USAGE_TAG_PAGE_MAX + 1, "limit must be between"), + (-1, 1, "offset must be non-negative"), + ), +) +def test_tag_page_sql_rejects_invalid_page_bounds(offset: int, limit: int, error: str) -> None: + with pytest.raises(ValueError, match=error): + build_tag_page_sql(_scope(table=DailyActivityTable.TEAM), team_id="team-a", offset=offset, limit=limit) + + def test_user_page_sql_groups_by_user_id_with_deterministic_spend_order() -> None: query: Final = build_user_page_sql(_scope(), offset=7, limit=3) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 262e8b6deae..c025f27e67f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16704,6 +16704,23 @@ export interface paths { patch?: never; trace?: never; }; + "/team/daily/activity/aggregated/tags": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Team Daily Activity Aggregated Tags */ + get: operations["get_team_daily_activity_aggregated_tags_team_daily_activity_aggregated_tags_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/team/daily/activity/export": { parameters: { query?: never; @@ -31386,6 +31403,17 @@ export interface components { /** Api Keys */ api_keys: components["schemas"]["KeyActivityRow"][]; }; + /** DailyActivityTagPageResponse */ + DailyActivityTagPageResponse: { + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Tags */ + tags: components["schemas"]["TagSpendRow"][]; + /** Total Tags */ + total_tags: number; + }; /** DailyActivityUserPageResponse */ DailyActivityUserPageResponse: { /** Limit */ @@ -46145,6 +46173,29 @@ export interface components { /** Tpm Limit */ tpm_limit?: number | null; }; + /** TagSpendRow */ + TagSpendRow: { + /** Api Requests */ + api_requests: number; + /** Cache Creation Input Tokens */ + cache_creation_input_tokens: number; + /** Cache Read Input Tokens */ + cache_read_input_tokens: number; + /** Completion Tokens */ + completion_tokens: number; + /** Failed Requests */ + failed_requests: number; + /** Prompt Tokens */ + prompt_tokens: number; + /** Spend */ + spend: number; + /** Successful Requests */ + successful_requests: number; + /** Tag */ + tag: string; + /** Total Tokens */ + total_tokens: number; + }; /** * TagSummaryMetrics * @description Summary metrics for a tag @@ -73604,6 +73655,45 @@ export interface operations { }; }; }; + get_team_daily_activity_aggregated_tags_team_daily_activity_aggregated_tags_get: { + parameters: { + query?: { + offset?: number; + limit?: number; + team_ids?: string | null; + start_date?: string | null; + end_date?: string | null; + model?: string | null; + api_key?: string | null; + exclude_team_ids?: string | null; + timezone?: number | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DailyActivityTagPageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_team_daily_activity_export_team_daily_activity_export_get: { parameters: { query?: {