From 9638fe2d0320f64a1c9b76df54aba0d18f796324 Mon Sep 17 00:00:00 2001 From: Mrinal Chanshetty Date: Mon, 19 Oct 2026 17:14:59 -0700 Subject: [PATCH 1/6] feat(proxy): group_by=tag on the team daily activity report GET /team/daily/activity/aggregated?team_ids=&group_by=tag returns one team's spend per tag, keyed by tag in breakdown.entities, read from LiteLLM_DailyTagSpend filtered to that team. The response shape is unchanged, and without group_by the report is exactly as before. group_by=tag needs exactly one team in team_ids for every caller, else 400. It is checked before the permission lookup, which would otherwise fill in every team a non-admin belongs to. The team route's permissions are unchanged: a member without team-admin rights sees only their own keys, and another team is still a 404. The automatic User-Agent tags are left out of a team's tag breakdown, and PTU hours (a whole-team figure) are not attached to it. The keys, search, model_top_keys and export routes take the same parameter. The team filter binds after every existing WHERE filter, so no current query is renumbered. schema.d.ts is regenerated for the new parameter. LIT-8516 --- .../daily_activity_routes.py | 9 +- .../daily_activity_scopes.py | 42 ++++++++- litellm/repositories/daily_activity_sql.py | 11 +++ litellm/types/repositories/daily_activity.py | 5 + .../spend/test_team_tag_spend_attribution.py | 93 ++++++++++++++++++- .../test_daily_activity_routes.py | 83 +++++++++++++++++ .../repositories/test_daily_activity_sql.py | 25 +++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 8 files changed, 268 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 842873129e4..1244efb7d33 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -33,6 +33,7 @@ from litellm.proxy.management_endpoints.daily_activity_scopes import ( EntityQuery, EntityScopeResolver, ResolvedScope, + group_by_tag_error, ) from litellm.proxy.management_endpoints.ptu_consumption import single_team_id, with_ptu_consumption from litellm.proxy.management_endpoints.team_endpoints import aggregated_date_range_error @@ -55,6 +56,7 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( UserActivityRow, ) from litellm.types.repositories.daily_activity import ( + DailyActivityTable, ExportRow, ExportType, KeyMetadataRow, @@ -100,6 +102,10 @@ async def _resolved_scope( date_error: Final[InvalidDateRange | None] = _date_range_error(query, user_aggregated=user_aggregated) if date_error is not None: raise_public(date_error) + # A malformed request is a 400 before any permission lookup, the same as a bad date range. + group_by_error: Final[str | None] = group_by_tag_error(query) + if group_by_error is not None: + raise_public(InvalidDateRange(reason=group_by_error)) result: ResolvedScope | ScopeDenied = await resolver.resolve(user_api_key_dict, query, prisma_client) if isinstance(result, ScopeDenied): raise_public(result) @@ -302,7 +308,8 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver, include_entity_breakdown=resolver.include_entity_breakdown, api_key_limit=api_key_limit, ) - if resolver.entity != "team": + # PTU hours are a whole-team figure, so a team's per-tag breakdown carries none. + if resolver.entity != "team" or resolved.scope.table is not DailyActivityTable.TEAM: return activity return with_ptu_consumption(activity, llm_router, single_team_id(resolved.scope.entity_ids)) except HTTPException: diff --git a/litellm/proxy/management_endpoints/daily_activity_scopes.py b/litellm/proxy/management_endpoints/daily_activity_scopes.py index bc373388a4e..ac329c4affa 100644 --- a/litellm/proxy/management_endpoints/daily_activity_scopes.py +++ b/litellm/proxy/management_endpoints/daily_activity_scopes.py @@ -1,7 +1,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from types import MappingProxyType -from typing import Final, Literal +from typing import Final, Literal, TypeAlias from fastapi import Query @@ -30,6 +30,7 @@ class EntityQuery: model: str | None timezone_offset_minutes: int | None include_current_utc_day: bool + group_by: "TeamGroupBy" = "team" @dataclass(frozen=True, slots=True) @@ -39,6 +40,8 @@ class ResolvedScope: Entity = Literal["user", "team", "tag", "organization", "customer", "agent"] +# How the Team view buckets its breakdown: by team (default), or by tag for one team's spend. +TeamGroupBy: TypeAlias = Literal["team", "tag"] EntityScopeResolution = ResolvedScope | ScopeDenied EntityScopeQuery = Callable[..., EntityQuery] EntityScopeResolve = Callable[[UserAPIKeyAuth, EntityQuery, PrismaClient], Awaitable[EntityScopeResolution]] @@ -133,7 +136,7 @@ async def _resolve_team( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) - return _build_scope( + team: Final = _build_scope( TEAM_RESOLVER, query, team_scope.team_ids, @@ -141,6 +144,25 @@ async def _resolve_team( team_scope.api_key_filter, team_scope.team_alias_metadata, ) + if query.group_by == "team": + return team + # group_by=tag: the same permitted team and key filter, read from the tag table and bucketed + # by tag. group_by_tag_error already held the request to exactly one named team, and the + # resolver returns a named team list as given (or 404s it), so this is that one team. + permitted: Final = tuple(team_scope.team_ids or ()) + if len(permitted) != 1: + raise ValueError(f"group_by=tag resolved to {len(permitted)} teams, expected exactly one") + return ResolvedScope( + scope=replace( + team.scope, + table=DailyActivityTable.TAG, + entity_id_field="tag", + entity_ids=None, + exclude_entity_ids=(), + team_id=permitted[0], + ), + entity_metadata=None, + ) async def _resolve_tag( @@ -259,6 +281,7 @@ def _team_query( api_key: str | None = None, exclude_team_ids: str | None = None, timezone: int | None = None, + group_by: TeamGroupBy = "team", ) -> EntityQuery: return EntityQuery( entity_ids=_query_ids(team_ids), @@ -269,9 +292,22 @@ def _team_query( model=model, timezone_offset_minutes=timezone, include_current_utc_day=False, + group_by=group_by, ) +def group_by_tag_error(query: EntityQuery) -> str | None: + """Why a group_by=tag request is malformed, else None. A team's tag breakdown is one team's + spend, so it needs exactly one team named in team_ids, whoever is asking: checked before any + permission lookup, since that lookup fills in every team a non-admin belongs to.""" + if query.group_by != "tag": + return None + named: Final = tuple(team_id for team_id in query.entity_ids or () if team_id) + if len(named) == 1: + return None + return f"group_by=tag needs exactly one team in team_ids, got {len(named)}" + + def _tag_query( start_date: str | None = None, end_date: str | None = None, diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 3d27e7d937e..e2cb74b4060 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -68,6 +68,8 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu exclusion_index: Final = entity_index + int(has_entity_array) model_index: Final = exclusion_index + int(bool(scope.exclude_entity_ids)) api_keys_index: Final = model_index + int(bool(scope.model)) + # Appended after every existing filter so a scope without a team binds exactly as before. + team_index: Final = api_keys_index + int(bool(scope.api_keys)) conditions: Final = ( f"date >= ${start_index}", f"date <= ${start_index + 1}", @@ -94,6 +96,7 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu if scope.api_keys else () ), + *((f"team_id = ${team_index}", _NOT_USER_AGENT_TAG) if scope.team_id is not None else ()), ) params: Final = ( adjusted_start, @@ -102,10 +105,18 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu *((list(scope.exclude_entity_ids),) if scope.exclude_entity_ids else ()), *((scope.model,) if scope.model else ()), *((list(scope.api_keys),) if scope.api_keys else ()), + *((scope.team_id,) if scope.team_id is not None else ()), ) return " AND ".join(conditions), params +# The proxy stamps a User-Agent tag on every request, so in a team's tag breakdown these would +# outnumber its real tags. Same rule as common_daily_activity._is_user_agent_tag. +_NOT_USER_AGENT_TAG: Final = ( + "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" +) + + def _ptu_flat_cost_select(table: DailyActivityTable, *, aggregate: bool = True) -> str: if table is DailyActivityTable.TEAM: return "SUM(ptu_flat_cost)::float AS ptu_flat_cost" if aggregate else "SUM(scoped.ptu_flat_cost)::float" diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index 4111b83a891..f6e5e5c6641 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -39,10 +39,15 @@ class DailyActivityScope: model: str | None timezone_offset_minutes: int | None include_current_utc_day: bool = False + # One team's tag spend (the Team view's Spend Per Tag card). Only the tag table records the + # sending team, and a team-scoped tag read leaves out the automatic User-Agent tags. + team_id: str | None = None def __post_init__(self) -> None: if self.entity_id_field not in _ENTITY_FIELDS[self.table]: raise ValueError(f"Invalid entity_id_field {self.entity_id_field!r} for {self.table.value}") + if self.team_id is not None and self.table is not DailyActivityTable.TAG: + raise ValueError(f"team_id applies only to {DailyActivityTable.TAG.value}, not {self.table.value}") @dataclass(frozen=True, slots=True) diff --git a/tests/integration/spend/test_team_tag_spend_attribution.py b/tests/integration/spend/test_team_tag_spend_attribution.py index b0bd50ca44e..1a67ec2108a 100644 --- a/tests/integration/spend/test_team_tag_spend_attribution.py +++ b/tests/integration/spend/test_team_tag_spend_attribution.py @@ -1,13 +1,52 @@ import uuid +from datetime import datetime, timedelta, timezone from typing import Final import pytest -from integration._support.client import Gateway, eventually +from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows _SPEND_PER_CALL: Final = 20 * 0.001 + 20 * 0.002 +def _tagged_chat(gateway: Gateway, model: str, key: str, tags: list[str]) -> None: + reply: Final = gateway.post( + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"tagged {uuid.uuid4().hex}"}], + "metadata": {"tags": tags}, + }, + key=key, + ) + assert reply["usage"]["total_tokens"] == 40, reply + + +def _tag_breakdown(gateway: Gateway, team: str, *, key: str | None = None) -> dict[str, tuple[float, int]]: + """The team's per-tag (spend, requests) from /team/daily/activity/aggregated?group_by=tag.""" + today: Final = datetime.now(timezone.utc) + response: Final = gateway.request( + "GET", + "/team/daily/activity/aggregated", + params={ + "team_ids": team, + "group_by": "tag", + "start_date": (today - timedelta(days=1)).strftime("%Y-%m-%d"), + "end_date": (today + timedelta(days=1)).strftime("%Y-%m-%d"), + "timezone": "0", + }, + key=key, + ) + assert response.status_code == 200, response.text + totals: dict[str, tuple[float, int]] = {} + for day in object_value(response.json())["results"]: + for tag, bucket in object_value(object_value(object_value(day)["breakdown"])["entities"]).items(): + metrics = object_value(object_value(bucket)["metrics"]) + spend, requests = totals.get(tag, (0.0, 0)) + totals[tag] = (spend + float(metrics["spend"]), requests + int(metrics["api_requests"])) + return totals + + def test_one_tag_sent_by_two_teams_is_recorded_once_per_team(gateway: Gateway) -> None: with gateway.scenario() as scenario: model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) @@ -42,3 +81,55 @@ def test_one_tag_sent_by_two_teams_is_recorded_once_per_team(gateway: Gateway) - ) for row in rows: assert float(row["spend"]) == pytest.approx(calls_per_team[row["team_id"]] * _SPEND_PER_CALL), rows + + +def test_group_by_tag_reports_one_teams_spend_per_tag(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + shared: Final = f"cost-center:{uuid.uuid4().hex}" + own: Final = f"project:{uuid.uuid4().hex}" + team: Final = scenario.team(models=[model]) + other_team: Final = scenario.team(models=[model]) + key: Final = scenario.key(team_id=team, models=[model]) + other_key: Final = scenario.key(team_id=other_team, models=[model]) + _tagged_chat(gateway, model, key, [shared, own, "User-Agent: integration/1.0"]) + _tagged_chat(gateway, model, key, [shared]) + _tagged_chat(gateway, model, other_key, [shared]) + eventually( + lambda: read_rows('SELECT api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag IN (%s, %s)', (shared, own)), + lambda values: sum(int(row["api_requests"]) for row in values) == 4, + seconds=70, + ) + + breakdown: Final = _tag_breakdown(gateway, team) + + assert set(breakdown) == {shared, own}, breakdown + assert breakdown[shared] == (pytest.approx(2 * _SPEND_PER_CALL), 2), breakdown + assert breakdown[own] == (pytest.approx(1 * _SPEND_PER_CALL), 1), breakdown + + +def test_group_by_tag_keeps_the_team_routes_member_permissions(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + tag: Final = f"cost-center:{uuid.uuid4().hex}" + team: Final = scenario.team(models=[model]) + other_team: Final = scenario.team(models=[model]) + member: Final = scenario.member(team) + member_key: Final = scenario.key(user_id=member, team_id=team, models=[model]) + teammate_key: Final = scenario.key(team_id=team, models=[model]) + _tagged_chat(gateway, model, member_key, [tag]) + _tagged_chat(gateway, model, teammate_key, [tag]) + eventually( + lambda: read_rows('SELECT api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag = %s', (tag,)), + lambda values: sum(int(row["api_requests"]) for row in values) == 2, + seconds=70, + ) + + assert _tag_breakdown(gateway, team, key=member_key) == {tag: (pytest.approx(_SPEND_PER_CALL), 1)} + denied: Final = gateway.request( + "GET", + "/team/daily/activity/aggregated", + params={"team_ids": other_team, "group_by": "tag", "start_date": "2026-01-01", "end_date": "2026-01-02"}, + key=member_key, + ) + assert denied.status_code == 404, denied.text diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index 8fbead8c01c..b4c42c068b3 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -1267,6 +1267,89 @@ def test_team_scope_applies_membership_and_user_key_filter( assert repository.aggregated.call_args.kwargs["include_entity_breakdown"] is True +def test_team_group_by_tag_reads_one_teams_tag_rows_with_the_members_key_filter( + daily_activity_client: tuple[TestClient, _FakeRepository], +) -> None: + """group_by=tag keeps every team-route permission (membership, the member's own keys) and + only swaps the source to the tag table, scoped to that one team and bucketed by tag.""" + client, repository = daily_activity_client + response: Final = client.get( + "/team/daily/activity/aggregated", + params={**_entity_params("team_ids", "team-a"), "group_by": "tag"}, + headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value, "x-user-id": "user-a"}, + ) + assert response.status_code == 200, response.text + scope: Final = repository.aggregated.call_args.args[0] + assert (scope.table, scope.entity_id_field, scope.entity_ids, scope.team_id, scope.api_keys) == ( + DailyActivityTable.TAG, + "tag", + None, + "team-a", + ("key-alpha",), + ), scope + + +@pytest.mark.parametrize( + ("team_ids", "expected_count"), + [(None, 0), ("team-a,team-b", 2), ("team-a,,team-b", 2)], +) +@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER]) +def test_team_group_by_tag_needs_exactly_one_named_team_for_every_caller( + daily_activity_client: tuple[TestClient, _FakeRepository], + team_ids: str | None, + expected_count: int, + role: LitellmUserRoles, +) -> None: + """Answered before any permission lookup, which would otherwise fill in every team a + non-admin belongs to and quietly turn "no team" into "all of my teams".""" + client, repository = daily_activity_client + response: Final = client.get( + "/team/daily/activity/aggregated", + params={**_DATE_PARAMS, "group_by": "tag", **({"team_ids": team_ids} if team_ids else {})}, + headers={"x-user-role": role.value, "x-user-id": "user-a"}, + ) + assert response.status_code == 400, response.text + assert response.json() == { + "detail": {"error": f"group_by=tag needs exactly one team in team_ids, got {expected_count}"} + } + repository.aggregated.assert_not_called() + + +def test_team_group_by_tag_carries_no_ptu_hours( + daily_activity_client: tuple[TestClient, _FakeRepository], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """PTU hours are a whole-team figure; the same sized team that reports them by team reports + none per tag.""" + client, _ = daily_activity_client + monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True") + _inject_llm_router(client, _ptu_sized_router("rare-group")) + + by_team: Final = client.get("/team/daily/activity/aggregated", params=_entity_params("team_ids", "team-a")) + by_tag: Final = client.get( + "/team/daily/activity/aggregated", params={**_entity_params("team_ids", "team-a"), "group_by": "tag"} + ) + + assert by_team.status_code == 200 and by_tag.status_code == 200, (by_team.text, by_tag.text) + ptu_hours: Final = (by_team.json()["metadata"]["total_ptu_hours"], by_tag.json()["metadata"]["total_ptu_hours"]) + assert ptu_hours[0] > 0 and ptu_hours[1] == 0, f"(by team, by tag) total_ptu_hours = {ptu_hours}" + + +def test_team_route_without_group_by_reads_the_team_table_as_before( + daily_activity_client: tuple[TestClient, _FakeRepository], +) -> None: + client, repository = daily_activity_client + response: Final = client.get("/team/daily/activity/aggregated", params=_entity_params("team_ids", "team-a")) + assert response.status_code == 200, response.text + scope: Final = repository.aggregated.call_args.args[0] + assert (scope.table, scope.entity_id_field, scope.entity_ids, scope.team_id) == ( + DailyActivityTable.TEAM, + "team_id", + ("team-a",), + None, + ), scope + + def test_team_scope_does_not_allow_an_unowned_api_key( daily_activity_client: tuple[TestClient, _FakeRepository], ) -> None: diff --git a/tests/unit/repositories/test_daily_activity_sql.py b/tests/unit/repositories/test_daily_activity_sql.py index c57db08e976..2a31f764e47 100644 --- a/tests/unit/repositories/test_daily_activity_sql.py +++ b/tests/unit/repositories/test_daily_activity_sql.py @@ -1,3 +1,4 @@ +from dataclasses import replace from datetime import datetime, timezone from typing import Final @@ -111,6 +112,30 @@ def test_where_clause_distinguishes_no_filter_from_empty_membership( assert params == expected_params +def test_where_clause_scopes_tag_rows_to_one_team_without_user_agent_tags() -> None: + """The team filter binds after every existing filter, so it never renumbers them, and the + User-Agent tags the proxy stamps on every request are left out of a team's tag breakdown.""" + scope = replace( + _scope(table=DailyActivityTable.TAG, entity_ids=None, api_keys=("key-1",), model="gpt-test"), + team_id="team-a", + ) + + sql, params = build_where_clause(scope) + + assert sql == ( + "date >= $1 AND date <= $2 AND model = $3 AND api_key = ANY($4::text[]) AND team_id = $5 " + "AND NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" + ) + assert params == ("2026-01-01", "2026-01-31", "gpt-test", ["key-1"], "team-a") + + +@pytest.mark.parametrize("table", [table for table in DailyActivityTable if table is not DailyActivityTable.TAG]) +def test_scope_rejects_a_team_id_on_any_table_but_the_tag_table(table: DailyActivityTable) -> None: + """Only LiteLLM_DailyTagSpend records the sending team in a team_id column next to its own entity.""" + with pytest.raises(ValueError, match="team_id applies only to litellm_dailytagspend"): + replace(_scope(table=table), team_id="team-a") + + def test_key_page_sql_orders_exact_spend_and_binds_scope_before_page() -> None: query = build_key_page_sql(_scope(), offset=7, limit=3) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e5c1b9f2c26..1cf087232b8 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -72862,6 +72862,7 @@ export interface operations { api_key?: string | null; exclude_team_ids?: string | null; timezone?: number | null; + group_by?: "team" | "tag"; }; header?: never; path?: never; @@ -72901,6 +72902,7 @@ export interface operations { api_key?: string | null; exclude_team_ids?: string | null; timezone?: number | null; + group_by?: "team" | "tag"; }; header?: never; path?: never; @@ -72941,6 +72943,7 @@ export interface operations { api_key?: string | null; exclude_team_ids?: string | null; timezone?: number | null; + group_by?: "team" | "tag"; }; header?: never; path?: never; @@ -72980,6 +72983,7 @@ export interface operations { api_key?: string | null; exclude_team_ids?: string | null; timezone?: number | null; + group_by?: "team" | "tag"; }; header?: never; path?: never; @@ -73019,6 +73023,7 @@ export interface operations { api_key?: string | null; exclude_team_ids?: string | null; timezone?: number | null; + group_by?: "team" | "tag"; }; header?: never; path?: never; From 0f253ef01d943b85fec12f446ac1a24dcd539216 Mon Sep 17 00:00:00 2001 From: Mrinal Chanshetty Date: Tue, 20 Oct 2026 14:37:26 -0700 Subject: [PATCH 2/6] fix(proxy): resolve group_by=tag on the named team when team_ids has a stray comma group_by_tag_error counted team_ids without empty entries, but the resolver got the raw list back, so team_ids=team-a, passed the one-team check and then resolved to two teams. Admins got a 500 and members a 404 naming an empty team. Both now resolve the same non-empty names, so the request reads team-a's tags. Trims comments that restated the code. --- .../daily_activity_routes.py | 1 - .../daily_activity_scopes.py | 17 ++++++++--------- litellm/repositories/daily_activity_sql.py | 4 +--- litellm/types/repositories/daily_activity.py | 2 -- .../test_daily_activity_routes.py | 10 +++------- .../repositories/test_daily_activity_sql.py | 3 --- 6 files changed, 12 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 1244efb7d33..8783f8f1f65 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -102,7 +102,6 @@ async def _resolved_scope( date_error: Final[InvalidDateRange | None] = _date_range_error(query, user_aggregated=user_aggregated) if date_error is not None: raise_public(date_error) - # A malformed request is a 400 before any permission lookup, the same as a bad date range. group_by_error: Final[str | None] = group_by_tag_error(query) if group_by_error is not None: raise_public(InvalidDateRange(reason=group_by_error)) diff --git a/litellm/proxy/management_endpoints/daily_activity_scopes.py b/litellm/proxy/management_endpoints/daily_activity_scopes.py index ac329c4affa..974d8dede96 100644 --- a/litellm/proxy/management_endpoints/daily_activity_scopes.py +++ b/litellm/proxy/management_endpoints/daily_activity_scopes.py @@ -40,7 +40,6 @@ class ResolvedScope: Entity = Literal["user", "team", "tag", "organization", "customer", "agent"] -# How the Team view buckets its breakdown: by team (default), or by tag for one team's spend. TeamGroupBy: TypeAlias = Literal["team", "tag"] EntityScopeResolution = ResolvedScope | ScopeDenied EntityScopeQuery = Callable[..., EntityQuery] @@ -127,8 +126,9 @@ async def _resolve_team( ) -> EntityScopeResolution: from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + requested: Final = _named_team_ids(query) if query.group_by == "tag" else query.entity_ids team_scope: Final = await resolve_team_daily_activity_scope( - team_ids=",".join(query.entity_ids) if query.entity_ids is not None else None, + team_ids=",".join(requested) if requested is not None else None, exclude_team_ids=",".join(query.exclude_entity_ids) if query.exclude_entity_ids else None, api_key=query.api_key, user_api_key_dict=user_api_key_dict, @@ -146,9 +146,6 @@ async def _resolve_team( ) if query.group_by == "team": return team - # group_by=tag: the same permitted team and key filter, read from the tag table and bucketed - # by tag. group_by_tag_error already held the request to exactly one named team, and the - # resolver returns a named team list as given (or 404s it), so this is that one team. permitted: Final = tuple(team_scope.team_ids or ()) if len(permitted) != 1: raise ValueError(f"group_by=tag resolved to {len(permitted)} teams, expected exactly one") @@ -296,13 +293,15 @@ def _team_query( ) +def _named_team_ids(query: EntityQuery) -> tuple[str, ...]: + return tuple(team_id for team_id in query.entity_ids or () if team_id) + + def group_by_tag_error(query: EntityQuery) -> str | None: - """Why a group_by=tag request is malformed, else None. A team's tag breakdown is one team's - spend, so it needs exactly one team named in team_ids, whoever is asking: checked before any - permission lookup, since that lookup fills in every team a non-admin belongs to.""" + """Checked before any permission lookup, which fills in every team a non-admin belongs to.""" if query.group_by != "tag": return None - named: Final = tuple(team_id for team_id in query.entity_ids or () if team_id) + named: Final = _named_team_ids(query) if len(named) == 1: return None return f"group_by=tag needs exactly one team in team_ids, got {len(named)}" diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index e2cb74b4060..00eab423dd2 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -68,7 +68,6 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu exclusion_index: Final = entity_index + int(has_entity_array) model_index: Final = exclusion_index + int(bool(scope.exclude_entity_ids)) api_keys_index: Final = model_index + int(bool(scope.model)) - # Appended after every existing filter so a scope without a team binds exactly as before. team_index: Final = api_keys_index + int(bool(scope.api_keys)) conditions: Final = ( f"date >= ${start_index}", @@ -110,8 +109,7 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu return " AND ".join(conditions), params -# The proxy stamps a User-Agent tag on every request, so in a team's tag breakdown these would -# outnumber its real tags. Same rule as common_daily_activity._is_user_agent_tag. +# Same rule as common_daily_activity._is_user_agent_tag _NOT_USER_AGENT_TAG: Final = ( "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" ) diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index f6e5e5c6641..e279f64e00a 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -39,8 +39,6 @@ class DailyActivityScope: model: str | None timezone_offset_minutes: int | None include_current_utc_day: bool = False - # One team's tag spend (the Team view's Spend Per Tag card). Only the tag table records the - # sending team, and a team-scoped tag read leaves out the automatic User-Agent tags. team_id: str | None = None def __post_init__(self) -> None: diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index b4c42c068b3..8b08dce1478 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -1267,15 +1267,15 @@ def test_team_scope_applies_membership_and_user_key_filter( assert repository.aggregated.call_args.kwargs["include_entity_breakdown"] is True +@pytest.mark.parametrize("team_ids", ["team-a", "team-a,", ",team-a"]) def test_team_group_by_tag_reads_one_teams_tag_rows_with_the_members_key_filter( daily_activity_client: tuple[TestClient, _FakeRepository], + team_ids: str, ) -> None: - """group_by=tag keeps every team-route permission (membership, the member's own keys) and - only swaps the source to the tag table, scoped to that one team and bucketed by tag.""" client, repository = daily_activity_client response: Final = client.get( "/team/daily/activity/aggregated", - params={**_entity_params("team_ids", "team-a"), "group_by": "tag"}, + params={**_entity_params("team_ids", team_ids), "group_by": "tag"}, headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value, "x-user-id": "user-a"}, ) assert response.status_code == 200, response.text @@ -1300,8 +1300,6 @@ def test_team_group_by_tag_needs_exactly_one_named_team_for_every_caller( expected_count: int, role: LitellmUserRoles, ) -> None: - """Answered before any permission lookup, which would otherwise fill in every team a - non-admin belongs to and quietly turn "no team" into "all of my teams".""" client, repository = daily_activity_client response: Final = client.get( "/team/daily/activity/aggregated", @@ -1319,8 +1317,6 @@ def test_team_group_by_tag_carries_no_ptu_hours( daily_activity_client: tuple[TestClient, _FakeRepository], monkeypatch: pytest.MonkeyPatch, ) -> None: - """PTU hours are a whole-team figure; the same sized team that reports them by team reports - none per tag.""" client, _ = daily_activity_client monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True") _inject_llm_router(client, _ptu_sized_router("rare-group")) diff --git a/tests/unit/repositories/test_daily_activity_sql.py b/tests/unit/repositories/test_daily_activity_sql.py index 2a31f764e47..28ce51470df 100644 --- a/tests/unit/repositories/test_daily_activity_sql.py +++ b/tests/unit/repositories/test_daily_activity_sql.py @@ -113,8 +113,6 @@ def test_where_clause_distinguishes_no_filter_from_empty_membership( def test_where_clause_scopes_tag_rows_to_one_team_without_user_agent_tags() -> None: - """The team filter binds after every existing filter, so it never renumbers them, and the - User-Agent tags the proxy stamps on every request are left out of a team's tag breakdown.""" scope = replace( _scope(table=DailyActivityTable.TAG, entity_ids=None, api_keys=("key-1",), model="gpt-test"), team_id="team-a", @@ -131,7 +129,6 @@ def test_where_clause_scopes_tag_rows_to_one_team_without_user_agent_tags() -> N @pytest.mark.parametrize("table", [table for table in DailyActivityTable if table is not DailyActivityTable.TAG]) def test_scope_rejects_a_team_id_on_any_table_but_the_tag_table(table: DailyActivityTable) -> None: - """Only LiteLLM_DailyTagSpend records the sending team in a team_id column next to its own entity.""" with pytest.raises(ValueError, match="team_id applies only to litellm_dailytagspend"): replace(_scope(table=table), team_id="team-a") From fdd5b2fac9af3fe1bd4797284896774e516663d9 Mon Sep 17 00:00:00 2001 From: Mrinal Chanshetty Date: Tue, 20 Oct 2026 15:24:26 -0700 Subject: [PATCH 3/6] style(proxy): ruff format daily_activity_sql --- litellm/repositories/daily_activity_sql.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 00eab423dd2..6f8a030f83a 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -110,9 +110,7 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu # Same rule as common_daily_activity._is_user_agent_tag -_NOT_USER_AGENT_TAG: Final = ( - "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" -) +_NOT_USER_AGENT_TAG: Final = "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" def _ptu_flat_cost_select(table: DailyActivityTable, *, aggregate: bool = True) -> str: From 9ea95f8debc267ed283de649be81714ecd40a77d Mon Sep 17 00:00:00 2001 From: mrinal Date: Wed, 21 Oct 2026 15:49:53 +0000 Subject: [PATCH 4/6] refactor(proxy): paginated team tags report replaces group_by=tag Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 2 + litellm/proxy/_types.py | 4 + .../daily_activity_routes.py | 62 ++++++++- .../daily_activity_scopes.py | 42 +----- .../repositories/daily_activity_repository.py | 16 +++ litellm/repositories/daily_activity_sql.py | 41 +++++- .../common_daily_activity.py | 10 ++ litellm/types/repositories/daily_activity.py | 23 +++- .../spend/test_team_tag_spend_attribution.py | 66 +++++---- .../test_daily_activity_routes.py | 127 ++++++++++++------ .../repositories/test_daily_activity_sql.py | 67 ++++++--- ui/litellm-dashboard/src/lib/http/schema.d.ts | 95 ++++++++++++- 12 files changed, 419 insertions(+), 136 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 94ff02a8c26..eef982b2f9e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2234,6 +2234,8 @@ USAGE_TOP_API_KEYS_DEFAULT: Final[int] = 100 USAGE_TOP_API_KEYS_MAX: Final[int] = 1000 USAGE_KEY_PAGE_DEFAULT: Final[int] = 50 USAGE_KEY_PAGE_MAX: Final[int] = 100 +USAGE_TAG_PAGE_DEFAULT: Final[int] = 50 +USAGE_TAG_PAGE_MAX: Final[int] = 100 USAGE_USER_PAGE_DEFAULT: Final[int] = 50 USAGE_USER_PAGE_MAX: Final[int] = 100 USAGE_KEY_SEARCH_DEFAULT: Final[int] = 100 diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e34b1d7b9a4..362430ca16d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -601,6 +601,7 @@ class LiteLLMRoutes(enum.Enum): KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value, KeyManagementRoutes.TEAM_DAILY_ACTIVITY.value, KeyManagementRoutes.TEAM_DAILY_ACTIVITY_AGGREGATED.value, + "/team/daily/activity/aggregated/tags", "/team/daily/activity/aggregated/keys", "/team/daily/activity/aggregated/search", "/team/daily/activity/aggregated/model_top_keys", @@ -654,6 +655,7 @@ class LiteLLMRoutes(enum.Enum): "/team/permissions_bulk_update", "/team/daily/activity", "/team/daily/activity/aggregated", + "/team/daily/activity/aggregated/tags", "/team/daily/activity/aggregated/keys", "/team/daily/activity/aggregated/search", "/team/daily/activity/aggregated/model_top_keys", @@ -844,6 +846,7 @@ class LiteLLMRoutes(enum.Enum): "/team/permissions_update", "/team/daily/activity", "/team/daily/activity/aggregated", + "/team/daily/activity/aggregated/tags", "/team/daily/activity/aggregated/keys", "/team/daily/activity/aggregated/search", "/team/daily/activity/aggregated/model_top_keys", @@ -968,6 +971,7 @@ class LiteLLMRoutes(enum.Enum): "/user/daily/activity/aggregated/users", "/team/daily/activity", "/team/daily/activity/aggregated", + "/team/daily/activity/aggregated/tags", "/team/daily/activity/aggregated/keys", "/team/daily/activity/aggregated/search", "/team/daily/activity/aggregated/model_top_keys", diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 8783f8f1f65..13d12cbecee 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -33,7 +33,7 @@ from litellm.proxy.management_endpoints.daily_activity_scopes import ( EntityQuery, EntityScopeResolver, ResolvedScope, - group_by_tag_error, + named_team_ids, ) from litellm.proxy.management_endpoints.ptu_consumption import single_team_id, with_ptu_consumption from litellm.proxy.management_endpoints.team_endpoints import aggregated_date_range_error @@ -45,6 +45,7 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( CacheLeakageKeysResponse, DailyActivityKeyPageResponse, DailyActivityKeySearchResponse, + DailyActivityTagPageResponse, DailyActivityUserPageResponse, KeyActivityRow, KeyMetadata, @@ -102,9 +103,6 @@ async def _resolved_scope( date_error: Final[InvalidDateRange | None] = _date_range_error(query, user_aggregated=user_aggregated) if date_error is not None: raise_public(date_error) - group_by_error: Final[str | None] = group_by_tag_error(query) - if group_by_error is not None: - raise_public(InvalidDateRange(reason=group_by_error)) result: ResolvedScope | ScopeDenied = await resolver.resolve(user_api_key_dict, query, prisma_client) if isinstance(result, ScopeDenied): raise_public(result) @@ -401,6 +399,60 @@ def _register_key_page_route(router: APIRouter, resolver: EntityScopeResolver, p ) +def _register_tag_page_route(router: APIRouter, resolver: EntityScopeResolver, prefix: str) -> None: + @management_endpoint_wrapper + async def tag_page( + entity_query: Annotated[EntityQuery, Depends(resolver.query)], + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[DailyActivityRepository, Depends(get_daily_activity_repository)], + prisma_client: Annotated[PrismaClient, Depends(get_daily_activity_prisma_client)], + offset: Annotated[int, Query(ge=0)] = 0, + limit: Annotated[int, Query(ge=1, le=constants.USAGE_TAG_PAGE_MAX)] = constants.USAGE_TAG_PAGE_DEFAULT, + ) -> DailyActivityTagPageResponse: + try: + team_ids: Final = named_team_ids(entity_query) + if len(team_ids) != 1: + raise_public( + InvalidDateRange(reason=f"tags page needs exactly one team in team_ids, got {len(team_ids)}") + ) + normalized_query: Final = replace(entity_query, entity_ids=team_ids) + resolved: Final = await _resolved_scope( + resolver, + normalized_query, + user_api_key_dict, + prisma_client, + user_aggregated=False, + ) + page: Final = await repository.tag_page( + resolved.scope, + team_id=team_ids[0], + offset=offset, + limit=limit, + ) + return DailyActivityTagPageResponse( + tags=list(page.rows), + total_tags=page.total_tags, + offset=offset, + limit=limit, + ) + except HTTPException: + raise + except Exception as exc: # noqa: BLE001, LIT003 # keep sibling routes' 500 error mapping + verbose_proxy_logger.exception("Daily activity tag page failed: %s", exc) + raise HTTPException(status_code=500, detail={"error": f"Failed to fetch analytics: {exc}"}) + + router.add_api_route( + f"{prefix}/daily/activity/aggregated/tags", + tag_page, + methods=["GET"], + name=resolver.operation_names["tag_page"], + tags=list(resolver.tags), + dependencies=(Depends(user_api_key_auth),), + response_model=DailyActivityTagPageResponse, + include_in_schema=prefix != "/end_user", + ) + + def _register_search_route(router: APIRouter, resolver: EntityScopeResolver, prefix: str) -> None: @management_endpoint_wrapper async def search( @@ -652,6 +704,8 @@ def register_daily_activity_routes(router: APIRouter, resolver: EntityScopeResol for prefix in resolver.route_prefixes: _register_aggregated_route(router, resolver, prefix) _register_key_page_route(router, resolver, prefix) + if resolver.entity == "team": + _register_tag_page_route(router, resolver, prefix) _register_search_route(router, resolver, prefix) _register_model_top_keys_route(router, resolver, prefix) _register_export_route(router, resolver, prefix) diff --git a/litellm/proxy/management_endpoints/daily_activity_scopes.py b/litellm/proxy/management_endpoints/daily_activity_scopes.py index 974d8dede96..5ddb12114a6 100644 --- a/litellm/proxy/management_endpoints/daily_activity_scopes.py +++ b/litellm/proxy/management_endpoints/daily_activity_scopes.py @@ -1,7 +1,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence -from dataclasses import dataclass, replace +from dataclasses import dataclass from types import MappingProxyType -from typing import Final, Literal, TypeAlias +from typing import Final, Literal from fastapi import Query @@ -30,7 +30,6 @@ class EntityQuery: model: str | None timezone_offset_minutes: int | None include_current_utc_day: bool - group_by: "TeamGroupBy" = "team" @dataclass(frozen=True, slots=True) @@ -40,7 +39,6 @@ class ResolvedScope: Entity = Literal["user", "team", "tag", "organization", "customer", "agent"] -TeamGroupBy: TypeAlias = Literal["team", "tag"] EntityScopeResolution = ResolvedScope | ScopeDenied EntityScopeQuery = Callable[..., EntityQuery] EntityScopeResolve = Callable[[UserAPIKeyAuth, EntityQuery, PrismaClient], Awaitable[EntityScopeResolution]] @@ -126,9 +124,8 @@ async def _resolve_team( ) -> EntityScopeResolution: from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache - requested: Final = _named_team_ids(query) if query.group_by == "tag" else query.entity_ids team_scope: Final = await resolve_team_daily_activity_scope( - team_ids=",".join(requested) if requested is not None else None, + team_ids=",".join(query.entity_ids) if query.entity_ids is not None else None, exclude_team_ids=",".join(query.exclude_entity_ids) if query.exclude_entity_ids else None, api_key=query.api_key, user_api_key_dict=user_api_key_dict, @@ -136,7 +133,7 @@ async def _resolve_team( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) - team: Final = _build_scope( + return _build_scope( TEAM_RESOLVER, query, team_scope.team_ids, @@ -144,22 +141,6 @@ async def _resolve_team( team_scope.api_key_filter, team_scope.team_alias_metadata, ) - if query.group_by == "team": - return team - permitted: Final = tuple(team_scope.team_ids or ()) - if len(permitted) != 1: - raise ValueError(f"group_by=tag resolved to {len(permitted)} teams, expected exactly one") - return ResolvedScope( - scope=replace( - team.scope, - table=DailyActivityTable.TAG, - entity_id_field="tag", - entity_ids=None, - exclude_entity_ids=(), - team_id=permitted[0], - ), - entity_metadata=None, - ) async def _resolve_tag( @@ -278,7 +259,6 @@ def _team_query( api_key: str | None = None, exclude_team_ids: str | None = None, timezone: int | None = None, - group_by: TeamGroupBy = "team", ) -> EntityQuery: return EntityQuery( entity_ids=_query_ids(team_ids), @@ -289,24 +269,13 @@ def _team_query( model=model, timezone_offset_minutes=timezone, include_current_utc_day=False, - group_by=group_by, ) -def _named_team_ids(query: EntityQuery) -> tuple[str, ...]: +def named_team_ids(query: EntityQuery) -> tuple[str, ...]: return tuple(team_id for team_id in query.entity_ids or () if team_id) -def group_by_tag_error(query: EntityQuery) -> str | None: - """Checked before any permission lookup, which fills in every team a non-admin belongs to.""" - if query.group_by != "tag": - return None - named: Final = _named_team_ids(query) - if len(named) == 1: - return None - return f"group_by=tag needs exactly one team in team_ids, got {len(named)}" - - def _tag_query( start_date: str | None = None, end_date: str | None = None, @@ -425,6 +394,7 @@ TEAM_RESOLVER = EntityScopeResolver( "aggregated": "get_team_daily_activity_aggregated", "search": "get_team_daily_activity_aggregated_search", "key_page": "get_team_daily_activity_aggregated_keys", + "tag_page": "get_team_daily_activity_aggregated_tags", "model_top_keys": "get_team_daily_activity_model_top_keys", "export": "get_team_daily_activity_export", } diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 10c603fe56f..1d2ba530aaa 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -22,6 +22,7 @@ from litellm.repositories.daily_activity_sql import ( build_key_page_sql, build_key_search_sql, build_model_top_keys_sql, + build_tag_page_sql, build_user_page_sql, ) from litellm.repositories.prisma_protocols import TableActions @@ -40,6 +41,8 @@ from litellm.types.repositories.daily_activity import ( KeyPage, KeySpendRow, SpendLogsWindow, + TagPage, + TagSpendRow, UserMetadataRow, UserPage, UserSpendRow, @@ -102,6 +105,7 @@ _GROUPING_ADAPTER: Final = TypeAdapter(tuple[GroupingSetsRow, ...]) _ENTITY_ADAPTER: Final = TypeAdapter(tuple[EntityRollupRow, ...]) _KEY_SPEND_ADAPTER: Final = TypeAdapter(tuple[KeySpendRow, ...]) _KEY_PAGE_TOTAL_ADAPTER: Final[TypeAdapter[int]] = TypeAdapter(int) +_TAG_SPEND_ADAPTER: Final = TypeAdapter(tuple[TagSpendRow, ...]) _USER_SPEND_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...]) _EXPORT_ADAPTER: Final = TypeAdapter(tuple[ExportRow, ...]) _METADATA_TAGS_ADAPTER: Final = TypeAdapter(list[StrictStr]) @@ -208,6 +212,18 @@ class DailyActivityRepository: rows: Final = _KEY_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("api_key") is not None)) return KeyPage(rows=rows, total_api_keys=total_api_keys) + async def tag_page( + self, scope: DailyActivityScope, *, team_id: str, offset: int, limit: int + ) -> TagPage: + query: Final = build_tag_page_sql(scope, team_id=team_id, offset=offset, limit=limit) + result: Final = await self._query(query) + total_tags_value: Final = result[0].get("total_tags") if result else 0 + total_tags: Final = ( + _KEY_PAGE_TOTAL_ADAPTER.validate_python(total_tags_value) if total_tags_value is not None else 0 + ) + rows: Final = _TAG_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("tag") is not None)) + return TagPage(rows=rows, total_tags=total_tags) + async def user_page(self, scope: DailyActivityScope, *, offset: int, limit: int) -> UserPage: query: Final = build_user_page_sql(scope, offset=offset, limit=limit) result: Final = await self._query(query) diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 6f8a030f83a..366de5c68b2 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -1,5 +1,5 @@ from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timedelta, timezone from itertools import count, islice from types import MappingProxyType @@ -68,7 +68,6 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu exclusion_index: Final = entity_index + int(has_entity_array) model_index: Final = exclusion_index + int(bool(scope.exclude_entity_ids)) api_keys_index: Final = model_index + int(bool(scope.model)) - team_index: Final = api_keys_index + int(bool(scope.api_keys)) conditions: Final = ( f"date >= ${start_index}", f"date <= ${start_index + 1}", @@ -95,7 +94,6 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu if scope.api_keys else () ), - *((f"team_id = ${team_index}", _NOT_USER_AGENT_TAG) if scope.team_id is not None else ()), ) params: Final = ( adjusted_start, @@ -104,7 +102,6 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu *((list(scope.exclude_entity_ids),) if scope.exclude_entity_ids else ()), *((scope.model,) if scope.model else ()), *((list(scope.api_keys),) if scope.api_keys else ()), - *((scope.team_id,) if scope.team_id is not None else ()), ) return " AND ".join(conditions), params @@ -308,6 +305,42 @@ def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> return SqlQuery(sql=sql, params=(*where_params, PTU_SENTINEL_API_KEY, limit, offset)) +def build_tag_page_sql(scope: DailyActivityScope, *, team_id: str, offset: int, limit: int) -> SqlQuery: + if not 1 <= limit <= constants.USAGE_TAG_PAGE_MAX: + raise ValueError(f"limit must be between 1 and {constants.USAGE_TAG_PAGE_MAX}") + if offset < 0: + raise ValueError("offset must be non-negative") + tag_scope: Final = replace( + scope, + table=DailyActivityTable.TAG, + entity_id_field="tag", + entity_ids=None, + exclude_entity_ids=(), + ) + where_clause, where_params = build_where_clause(tag_scope) + team_param: Final = len(where_params) + 1 + sentinel_param: Final = team_param + 1 + limit_param: Final = sentinel_param + 1 + offset_param: Final = limit_param + 1 + sql: Final = f""" + WITH ranked AS ( + SELECT tag,{_key_spend_select()}, SUM(spend::numeric) AS rank_spend + FROM "{PRISMA_TO_PG_TABLE[DailyActivityTable.TAG]}" + WHERE {where_clause} AND team_id = ${team_param} AND api_key <> ${sentinel_param} + AND tag IS NOT NULL AND {_NOT_USER_AGENT_TAG} + GROUP BY tag + ) + SELECT (SELECT COUNT(*) FROM ranked)::bigint AS total_tags, page.* + FROM (SELECT 1) AS one + LEFT JOIN LATERAL ( + SELECT * FROM ranked + ORDER BY rank_spend DESC, tag ASC + LIMIT ${limit_param} OFFSET ${offset_param} + ) AS page ON TRUE + """ + return SqlQuery(sql=sql, params=(*where_params, team_id, PTU_SENTINEL_API_KEY, limit, offset)) + + def _user_spend_select() -> str: return """ COALESCE(SUM(spend), 0)::float AS spend, diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 66702abfb34..a82b0c8a77a 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -7,6 +7,7 @@ from pydantic import ConfigDict, Field from typing_extensions import TypedDict from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.repositories.daily_activity import TagSpendRow class GroupByDimension(str, Enum): @@ -164,6 +165,15 @@ class DailyActivityKeyPageResponse(LiteLLMBaseModel): limit: int +class DailyActivityTagPageResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + tags: list[TagSpendRow] # mutable-ok: response contract requires a JSON array + total_tags: int + offset: int + limit: int + + class UserActivityRow(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index e279f64e00a..2746b417965 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -39,13 +39,10 @@ class DailyActivityScope: model: str | None timezone_offset_minutes: int | None include_current_utc_day: bool = False - team_id: str | None = None def __post_init__(self) -> None: if self.entity_id_field not in _ENTITY_FIELDS[self.table]: raise ValueError(f"Invalid entity_id_field {self.entity_id_field!r} for {self.table.value}") - if self.team_id is not None and self.table is not DailyActivityTable.TAG: - raise ValueError(f"team_id applies only to {DailyActivityTable.TAG.value}, not {self.table.value}") @dataclass(frozen=True, slots=True) @@ -68,6 +65,26 @@ class KeyPage: total_api_keys: int +@dataclass(frozen=True, slots=True) +class TagSpendRow: + tag: str + spend: float + prompt_tokens: int + completion_tokens: int + total_tokens: int + api_requests: int + successful_requests: int + failed_requests: int + cache_read_input_tokens: int + cache_creation_input_tokens: int + + +@dataclass(frozen=True, slots=True) +class TagPage: + rows: tuple[TagSpendRow, ...] + total_tags: int + + @dataclass(frozen=True, slots=True) class UserSpendRow: user_id: str | None diff --git a/tests/integration/spend/test_team_tag_spend_attribution.py b/tests/integration/spend/test_team_tag_spend_attribution.py index 8152020ef4a..ae3389a7e88 100644 --- a/tests/integration/spend/test_team_tag_spend_attribution.py +++ b/tests/integration/spend/test_team_tag_spend_attribution.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta, timezone from typing import Final import pytest -from integration._support.client import Gateway, eventually, object_value +from integration._support.client import Gateway, eventually, list_value, object_value from integration._support.database import read_rows _SPEND_PER_CALL: Final = 20 * 0.001 + 20 * 0.002 @@ -22,29 +22,30 @@ def _tagged_chat(gateway: Gateway, model: str, key: str, tags: list[str]) -> Non assert reply["usage"]["total_tokens"] == 40, reply -def _tag_breakdown(gateway: Gateway, team: str, *, key: str | None = None) -> dict[str, tuple[float, int]]: - """The team's per-tag (spend, requests) from /team/daily/activity/aggregated?group_by=tag.""" +def _tag_page( + gateway: Gateway, + team: str, + *, + offset: int = 0, + limit: int = 50, + key: str | None = None, +) -> dict[str, object]: today: Final = datetime.now(timezone.utc) response: Final = gateway.request( "GET", - "/team/daily/activity/aggregated", + "/team/daily/activity/aggregated/tags", params={ "team_ids": team, - "group_by": "tag", "start_date": (today - timedelta(days=1)).strftime("%Y-%m-%d"), "end_date": (today + timedelta(days=1)).strftime("%Y-%m-%d"), "timezone": "0", + "offset": offset, + "limit": limit, }, key=key, ) assert response.status_code == 200, response.text - totals: dict[str, tuple[float, int]] = {} - for day in object_value(response.json())["results"]: - for tag, bucket in object_value(object_value(object_value(day)["breakdown"])["entities"]).items(): - metrics = object_value(object_value(bucket)["metrics"]) - spend, requests = totals.get(tag, (0.0, 0)) - totals[tag] = (spend + float(metrics["spend"]), requests + int(metrics["api_requests"])) - return totals + return object_value(response.json()) def test_one_key_moved_between_teams_records_its_tag_once_per_team(gateway: Gateway) -> None: @@ -84,7 +85,7 @@ def test_one_key_moved_between_teams_records_its_tag_once_per_team(gateway: Gate assert float(row["spend"]) == pytest.approx(calls_per_team[row["team_id"]] * _SPEND_PER_CALL), rows -def test_group_by_tag_reports_one_teams_spend_per_tag(gateway: Gateway) -> None: +def test_tag_page_reports_one_teams_spend_per_tag(gateway: Gateway) -> None: with gateway.scenario() as scenario: model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) shared: Final = f"cost-center:{uuid.uuid4().hex}" @@ -93,23 +94,36 @@ def test_group_by_tag_reports_one_teams_spend_per_tag(gateway: Gateway) -> None: other_team: Final = scenario.team(models=[model]) key: Final = scenario.key(team_id=team, models=[model]) other_key: Final = scenario.key(team_id=other_team, models=[model]) + no_team_key: Final = scenario.key(models=[model]) + no_team_tag: Final = f"unassigned:{uuid.uuid4().hex}" _tagged_chat(gateway, model, key, [shared, own, "User-Agent: integration/1.0"]) _tagged_chat(gateway, model, key, [shared]) _tagged_chat(gateway, model, other_key, [shared]) + _tagged_chat(gateway, model, no_team_key, [no_team_tag]) eventually( - lambda: read_rows('SELECT api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag IN (%s, %s)', (shared, own)), - lambda values: sum(int(row["api_requests"]) for row in values) == 4, + lambda: read_rows( + 'SELECT api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag IN (%s, %s, %s)', + (shared, own, no_team_tag), + ), + lambda values: sum(int(row["api_requests"]) for row in values) == 5, seconds=70, ) - breakdown: Final = _tag_breakdown(gateway, team) - - assert set(breakdown) == {shared, own}, breakdown - assert breakdown[shared] == (pytest.approx(2 * _SPEND_PER_CALL), 2), breakdown - assert breakdown[own] == (pytest.approx(1 * _SPEND_PER_CALL), 1), breakdown + first_page: Final = _tag_page(gateway, team, limit=1) + next_page: Final = _tag_page(gateway, team, offset=1, limit=1) + assert first_page["total_tags"] == 2 + assert next_page["total_tags"] == 2 + first_tag: Final = object_value(list_value(first_page["tags"])[0]) + next_tag: Final = object_value(list_value(next_page["tags"])[0]) + assert first_tag["tag"] == shared + assert first_tag["spend"] == pytest.approx(2 * _SPEND_PER_CALL) + assert first_tag["api_requests"] == 2 + assert next_tag["tag"] == own + assert next_tag["spend"] == pytest.approx(_SPEND_PER_CALL) + assert next_tag["api_requests"] == 1 -def test_group_by_tag_keeps_the_team_routes_member_permissions(gateway: Gateway) -> None: +def test_tag_page_keeps_the_team_routes_member_permissions(gateway: Gateway) -> None: with gateway.scenario() as scenario: model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) tag: Final = f"cost-center:{uuid.uuid4().hex}" @@ -126,11 +140,15 @@ def test_group_by_tag_keeps_the_team_routes_member_permissions(gateway: Gateway) seconds=70, ) - assert _tag_breakdown(gateway, team, key=member_key) == {tag: (pytest.approx(_SPEND_PER_CALL), 1)} + member_page: Final = _tag_page(gateway, team, key=member_key) + assert member_page["total_tags"] == 1 + member_tag: Final = object_value(list_value(member_page["tags"])[0]) + assert member_tag["tag"] == tag + assert member_tag["api_requests"] == 1 denied: Final = gateway.request( "GET", - "/team/daily/activity/aggregated", - params={"team_ids": other_team, "group_by": "tag", "start_date": "2026-01-01", "end_date": "2026-01-02"}, + "/team/daily/activity/aggregated/tags", + params={"team_ids": other_team, "start_date": "2026-01-01", "end_date": "2026-01-02"}, key=member_key, ) assert denied.status_code == 404, denied.text diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index 8b08dce1478..80cfd930607 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -1,7 +1,7 @@ import csv import io from collections.abc import AsyncIterator, Iterator, Mapping, Sequence -from dataclasses import dataclass, fields +from dataclasses import dataclass, fields, replace from itertools import chain from types import MappingProxyType, SimpleNamespace from typing import Final @@ -35,6 +35,8 @@ from litellm.types.repositories.daily_activity import ( KeyMetadataRow, KeyPage, KeySpendRow, + TagPage, + TagSpendRow, UserMetadataRow, UserPage, UserSpendRow, @@ -250,6 +252,8 @@ class _FakeRepository: self.aggregated = AsyncMock(side_effect=self._aggregated) self.key_page = AsyncMock(side_effect=self._key_page) self.key_page_call: tuple[DailyActivityScope, int, int] | None = None + self.tag_page = AsyncMock(side_effect=self._tag_page) + self.tag_page_call: tuple[DailyActivityScope, str, int, int] | None = None self.search_keys = AsyncMock(side_effect=self._search_keys) self.model_top_keys = AsyncMock(side_effect=self._model_top_keys) self.cache_leakage_keys = AsyncMock(side_effect=self._cache_leakage_keys) @@ -321,6 +325,29 @@ class _FakeRepository: total_api_keys=len(api_keys), ) + async def _tag_page( + self, scope: DailyActivityScope, *, team_id: str, offset: int, limit: int + ) -> TagPage: + self.tag_page_call = (scope, team_id, offset, limit) + tag_scope: Final = replace( + scope, + table=DailyActivityTable.TAG, + entity_id_field="tag", + entity_ids=None, + exclude_entity_ids=(), + ) + rows: Final = self._matching_rows(tag_scope) + tags: Final = tuple( + sorted( + {row.entity_id for row in rows}, + key=lambda tag: (-sum(row.spend for row in rows if row.entity_id == tag), tag), + ) + ) + return TagPage( + rows=tuple(_tag_spend_row(tag, rows) for tag in tags[offset : offset + limit]), + total_tags=len(tags), + ) + async def _model_top_keys( self, scope: DailyActivityScope, *, model_group: str, by_model_group: bool, limit: int ) -> tuple[KeySpendRow, ...]: @@ -445,6 +472,22 @@ def _key_spend_row(api_key: str, rows: Sequence[_Activity]) -> KeySpendRow: ) +def _tag_spend_row(tag: str, rows: Sequence[_Activity]) -> TagSpendRow: + matching: Final = tuple(row for row in rows if row.entity_id == tag) + return TagSpendRow( + tag=tag, + spend=sum(row.spend for row in matching), + prompt_tokens=sum(row.prompt_tokens for row in matching), + completion_tokens=sum(row.completion_tokens for row in matching), + total_tokens=sum(row.prompt_tokens + row.completion_tokens for row in matching), + api_requests=sum(row.api_requests for row in matching), + successful_requests=sum(row.successful_requests for row in matching), + failed_requests=sum(row.failed_requests for row in matching), + cache_read_input_tokens=sum(row.cache_read_input_tokens for row in matching), + cache_creation_input_tokens=sum(row.cache_creation_input_tokens for row in matching), + ) + + def _entity_rows_for_day( date: str, rows: Sequence[_Activity], distinct_api_keys: int | None = None ) -> tuple[EntityRollupRow, ...]: @@ -1268,25 +1311,30 @@ def test_team_scope_applies_membership_and_user_key_filter( @pytest.mark.parametrize("team_ids", ["team-a", "team-a,", ",team-a"]) -def test_team_group_by_tag_reads_one_teams_tag_rows_with_the_members_key_filter( +def test_team_tag_page_normalizes_team_ids_and_applies_the_members_key_filter( daily_activity_client: tuple[TestClient, _FakeRepository], team_ids: str, ) -> None: client, repository = daily_activity_client response: Final = client.get( - "/team/daily/activity/aggregated", - params={**_entity_params("team_ids", team_ids), "group_by": "tag"}, + "/team/daily/activity/aggregated/tags", + params={**_entity_params("team_ids", team_ids), "offset": 3, "limit": 7}, headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value, "x-user-id": "user-a"}, ) assert response.status_code == 200, response.text - scope: Final = repository.aggregated.call_args.args[0] - assert (scope.table, scope.entity_id_field, scope.entity_ids, scope.team_id, scope.api_keys) == ( - DailyActivityTable.TAG, - "tag", - None, - "team-a", + assert response.json()["offset"] == 3 + assert response.json()["limit"] == 7 + tag_page_call: Final = repository.tag_page_call + assert tag_page_call is not None + scope, team_id, offset, limit = tag_page_call + assert (scope.table, scope.entity_ids, scope.api_keys, team_id, offset, limit) == ( + DailyActivityTable.TEAM, + ("team-a",), ("key-alpha",), - ), scope + "team-a", + 3, + 7, + ), tag_page_call @pytest.mark.parametrize( @@ -1294,7 +1342,7 @@ def test_team_group_by_tag_reads_one_teams_tag_rows_with_the_members_key_filter( [(None, 0), ("team-a,team-b", 2), ("team-a,,team-b", 2)], ) @pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER]) -def test_team_group_by_tag_needs_exactly_one_named_team_for_every_caller( +def test_team_tag_page_needs_exactly_one_named_team_for_every_caller( daily_activity_client: tuple[TestClient, _FakeRepository], team_ids: str | None, expected_count: int, @@ -1302,47 +1350,48 @@ def test_team_group_by_tag_needs_exactly_one_named_team_for_every_caller( ) -> None: client, repository = daily_activity_client response: Final = client.get( - "/team/daily/activity/aggregated", - params={**_DATE_PARAMS, "group_by": "tag", **({"team_ids": team_ids} if team_ids else {})}, + "/team/daily/activity/aggregated/tags", + params={**_DATE_PARAMS, **({"team_ids": team_ids} if team_ids else {})}, headers={"x-user-role": role.value, "x-user-id": "user-a"}, ) assert response.status_code == 400, response.text assert response.json() == { - "detail": {"error": f"group_by=tag needs exactly one team in team_ids, got {expected_count}"} + "detail": {"error": f"tags page needs exactly one team in team_ids, got {expected_count}"} } - repository.aggregated.assert_not_called() + repository.tag_page.assert_not_awaited() -def test_team_group_by_tag_carries_no_ptu_hours( - daily_activity_client: tuple[TestClient, _FakeRepository], - monkeypatch: pytest.MonkeyPatch, -) -> None: - client, _ = daily_activity_client - monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True") - _inject_llm_router(client, _ptu_sized_router("rare-group")) - - by_team: Final = client.get("/team/daily/activity/aggregated", params=_entity_params("team_ids", "team-a")) - by_tag: Final = client.get( - "/team/daily/activity/aggregated", params={**_entity_params("team_ids", "team-a"), "group_by": "tag"} - ) - - assert by_team.status_code == 200 and by_tag.status_code == 200, (by_team.text, by_tag.text) - ptu_hours: Final = (by_team.json()["metadata"]["total_ptu_hours"], by_tag.json()["metadata"]["total_ptu_hours"]) - assert ptu_hours[0] > 0 and ptu_hours[1] == 0, f"(by team, by tag) total_ptu_hours = {ptu_hours}" - - -def test_team_route_without_group_by_reads_the_team_table_as_before( +def test_team_tag_page_returns_404_for_a_non_member( daily_activity_client: tuple[TestClient, _FakeRepository], ) -> None: client, repository = daily_activity_client - response: Final = client.get("/team/daily/activity/aggregated", params=_entity_params("team_ids", "team-a")) - assert response.status_code == 200, response.text + response: Final = client.get( + "/team/daily/activity/aggregated/tags", + params=_entity_params("team_ids", "team-b"), + headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value, "x-user-id": "user-a"}, + ) + + assert response.status_code == 404, response.text + repository.tag_page.assert_not_awaited() + + +def test_team_aggregated_ignores_group_by_and_reads_the_team_table( + daily_activity_client: tuple[TestClient, _FakeRepository], +) -> None: + client, repository = daily_activity_client + params: Final = _entity_params("team_ids", "team-a") + without_group_by: Final = client.get("/team/daily/activity/aggregated", params=params) + with_group_by: Final = client.get( + "/team/daily/activity/aggregated", + params={**params, "group_by": "tag"}, + ) + assert without_group_by.status_code == with_group_by.status_code == 200 + assert with_group_by.json() == without_group_by.json() scope: Final = repository.aggregated.call_args.args[0] - assert (scope.table, scope.entity_id_field, scope.entity_ids, scope.team_id) == ( + assert (scope.table, scope.entity_id_field, scope.entity_ids) == ( DailyActivityTable.TEAM, "team_id", ("team-a",), - None, ), scope diff --git a/tests/unit/repositories/test_daily_activity_sql.py b/tests/unit/repositories/test_daily_activity_sql.py index 28ce51470df..ae10b931c69 100644 --- a/tests/unit/repositories/test_daily_activity_sql.py +++ b/tests/unit/repositories/test_daily_activity_sql.py @@ -16,6 +16,7 @@ from litellm.repositories.daily_activity_sql import ( build_key_page_sql, build_key_search_sql, build_model_top_keys_sql, + build_tag_page_sql, build_user_page_sql, build_where_clause, ) @@ -112,27 +113,6 @@ def test_where_clause_distinguishes_no_filter_from_empty_membership( assert params == expected_params -def test_where_clause_scopes_tag_rows_to_one_team_without_user_agent_tags() -> None: - scope = replace( - _scope(table=DailyActivityTable.TAG, entity_ids=None, api_keys=("key-1",), model="gpt-test"), - team_id="team-a", - ) - - sql, params = build_where_clause(scope) - - assert sql == ( - "date >= $1 AND date <= $2 AND model = $3 AND api_key = ANY($4::text[]) AND team_id = $5 " - "AND NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" - ) - assert params == ("2026-01-01", "2026-01-31", "gpt-test", ["key-1"], "team-a") - - -@pytest.mark.parametrize("table", [table for table in DailyActivityTable if table is not DailyActivityTable.TAG]) -def test_scope_rejects_a_team_id_on_any_table_but_the_tag_table(table: DailyActivityTable) -> None: - with pytest.raises(ValueError, match="team_id applies only to litellm_dailytagspend"): - replace(_scope(table=table), team_id="team-a") - - def test_key_page_sql_orders_exact_spend_and_binds_scope_before_page() -> None: query = build_key_page_sql(_scope(), offset=7, limit=3) @@ -162,6 +142,51 @@ def test_key_page_sql_rejects_invalid_page_bounds(offset: int, limit: int, error build_key_page_sql(_scope(), offset=offset, limit=limit) +def test_tag_page_sql_scopes_to_team_excludes_user_agents_and_orders_tags() -> None: + query: Final = build_tag_page_sql( + _scope(table=DailyActivityTable.TEAM, entity_ids=("team-a",), api_keys=("key-1",), model="gpt-test"), + team_id="team-a", + offset=7, + limit=3, + ) + + assert query.params == ( + "2026-01-01", + "2026-01-31", + "gpt-test", + ["key-1"], + "team-a", + PTU_SENTINEL_API_KEY, + 3, + 7, + ) + assert 'FROM "LiteLLM_DailyTagSpend"' in query.sql + assert "team_id = $5" in query.sql + assert "api_key = ANY($4::text[])" in query.sql + assert "tag IS NOT NULL" in query.sql + assert "api_key <> $6" in query.sql + assert "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" in query.sql + assert "GROUP BY tag" in query.sql + assert "SUM(spend::numeric) AS rank_spend" in query.sql + assert "ORDER BY rank_spend DESC, tag ASC" in query.sql + assert "(SELECT COUNT(*) FROM ranked)::bigint AS total_tags" in query.sql + assert "LIMIT $7 OFFSET $8" in query.sql + assert '"tag" = ANY' not in query.sql + + +@pytest.mark.parametrize( + ("offset", "limit", "error"), + ( + (0, 0, "limit must be between"), + (0, constants.USAGE_TAG_PAGE_MAX + 1, "limit must be between"), + (-1, 1, "offset must be non-negative"), + ), +) +def test_tag_page_sql_rejects_invalid_page_bounds(offset: int, limit: int, error: str) -> None: + with pytest.raises(ValueError, match=error): + build_tag_page_sql(_scope(table=DailyActivityTable.TEAM), team_id="team-a", offset=offset, limit=limit) + + def test_user_page_sql_groups_by_user_id_with_deterministic_spend_order() -> None: query: Final = build_user_page_sql(_scope(), offset=7, limit=3) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1cf087232b8..d780bbef8e0 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16631,6 +16631,23 @@ export interface paths { patch?: never; trace?: never; }; + "/team/daily/activity/aggregated/tags": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Team Daily Activity Aggregated Tags */ + get: operations["get_team_daily_activity_aggregated_tags_team_daily_activity_aggregated_tags_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/team/daily/activity/export": { parameters: { query?: never; @@ -31256,6 +31273,17 @@ export interface components { /** Api Keys */ api_keys: components["schemas"]["KeyActivityRow"][]; }; + /** DailyActivityTagPageResponse */ + DailyActivityTagPageResponse: { + /** Limit */ + limit: number; + /** Offset */ + offset: number; + /** Tags */ + tags: components["schemas"]["TagSpendRow"][]; + /** Total Tags */ + total_tags: number; + }; /** DailyActivityUserPageResponse */ DailyActivityUserPageResponse: { /** Limit */ @@ -45854,6 +45882,29 @@ export interface components { /** Tpm Limit */ tpm_limit?: number | null; }; + /** TagSpendRow */ + TagSpendRow: { + /** Api Requests */ + api_requests: number; + /** Cache Creation Input Tokens */ + cache_creation_input_tokens: number; + /** Cache Read Input Tokens */ + cache_read_input_tokens: number; + /** Completion Tokens */ + completion_tokens: number; + /** Failed Requests */ + failed_requests: number; + /** Prompt Tokens */ + prompt_tokens: number; + /** Spend */ + spend: number; + /** Successful Requests */ + successful_requests: number; + /** Tag */ + tag: string; + /** Total Tokens */ + total_tokens: number; + }; /** * TagSummaryMetrics * @description Summary metrics for a tag @@ -72862,7 +72913,6 @@ export interface operations { api_key?: string | null; exclude_team_ids?: string | null; timezone?: number | null; - group_by?: "team" | "tag"; }; header?: never; path?: never; @@ -72902,7 +72952,6 @@ export interface operations { api_key?: string | null; exclude_team_ids?: string | null; timezone?: number | null; - group_by?: "team" | "tag"; }; header?: never; path?: never; @@ -72943,7 +72992,6 @@ export interface operations { api_key?: string | null; exclude_team_ids?: string | null; timezone?: number | null; - group_by?: "team" | "tag"; }; header?: never; path?: never; @@ -72983,7 +73031,6 @@ export interface operations { api_key?: string | null; exclude_team_ids?: string | null; timezone?: number | null; - group_by?: "team" | "tag"; }; header?: never; path?: never; @@ -73011,6 +73058,45 @@ export interface operations { }; }; }; + get_team_daily_activity_aggregated_tags_team_daily_activity_aggregated_tags_get: { + parameters: { + query?: { + offset?: number; + limit?: number; + team_ids?: string | null; + start_date?: string | null; + end_date?: string | null; + model?: string | null; + api_key?: string | null; + exclude_team_ids?: string | null; + timezone?: number | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DailyActivityTagPageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_team_daily_activity_export_team_daily_activity_export_get: { parameters: { query?: { @@ -73023,7 +73109,6 @@ export interface operations { api_key?: string | null; exclude_team_ids?: string | null; timezone?: number | null; - group_by?: "team" | "tag"; }; header?: never; path?: never; From b83e878929da4cacea241d7db13b5cbfde4daa72 Mon Sep 17 00:00:00 2001 From: mrinal Date: Wed, 21 Oct 2026 16:26:53 +0000 Subject: [PATCH 5/6] fix(proxy): allowlist team tag activity endpoint Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/repositories/daily_activity_repository.py | 4 +--- terraform/provider/tools/endpointaudit/coverage_allowlist.txt | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 1d2ba530aaa..55530aeae48 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -212,9 +212,7 @@ class DailyActivityRepository: rows: Final = _KEY_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("api_key") is not None)) return KeyPage(rows=rows, total_api_keys=total_api_keys) - async def tag_page( - self, scope: DailyActivityScope, *, team_id: str, offset: int, limit: int - ) -> TagPage: + async def tag_page(self, scope: DailyActivityScope, *, team_id: str, offset: int, limit: int) -> TagPage: query: Final = build_tag_page_sql(scope, team_id=team_id, offset=offset, limit=limit) result: Final = await self._query(query) total_tags_value: Final = result[0].get("total_tags") if result else 0 diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt index 3a45eef3606..eee32fac48f 100644 --- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt +++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt @@ -48,6 +48,7 @@ GET /tag/user-agent/per-user-analytics GET /tag/wau GET /team/daily/activity GET /team/daily/activity/aggregated +GET /team/daily/activity/aggregated/tags # proxy-only analytics report; no provider resource GET /team/daily/activity/aggregated/keys GET /team/daily/activity/aggregated/model_top_keys GET /team/daily/activity/aggregated/search From 76838c087f7f743aa3ad58ebf23c69564a26e596 Mon Sep 17 00:00:00 2001 From: Mrinal Chanshetty Date: Sat, 24 Oct 2026 20:20:49 -0500 Subject: [PATCH 6/6] fix(proxy): honor exclude_team_ids on the team tags page The tags page took its team from team_ids alone, so a team that was both named and excluded still returned its tag spend, while the aggregated team report returned nothing for the same request. It now returns an empty page after the permission check. Also drops a comment that restated the code. --- .../management_endpoints/daily_activity_routes.py | 10 +++++----- litellm/repositories/daily_activity_sql.py | 1 - .../test_daily_activity_routes.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 13d12cbecee..7824540c54c 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -62,6 +62,7 @@ from litellm.types.repositories.daily_activity import ( ExportType, KeyMetadataRow, KeySpendRow, + TagPage, UserMetadataRow, UserSpendRow, ) @@ -423,11 +424,10 @@ def _register_tag_page_route(router: APIRouter, resolver: EntityScopeResolver, p prisma_client, user_aggregated=False, ) - page: Final = await repository.tag_page( - resolved.scope, - team_id=team_ids[0], - offset=offset, - limit=limit, + page: Final = ( + TagPage(rows=(), total_tags=0) + if team_ids[0] in resolved.scope.exclude_entity_ids + else await repository.tag_page(resolved.scope, team_id=team_ids[0], offset=offset, limit=limit) ) return DailyActivityTagPageResponse( tags=list(page.rows), diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 366de5c68b2..7c3eb568971 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -106,7 +106,6 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu return " AND ".join(conditions), params -# Same rule as common_daily_activity._is_user_agent_tag _NOT_USER_AGENT_TAG: Final = "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index 80cfd930607..0ca61a47da1 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -1375,6 +1375,20 @@ def test_team_tag_page_returns_404_for_a_non_member( repository.tag_page.assert_not_awaited() +def test_team_tag_page_is_empty_for_a_team_that_is_also_excluded( + daily_activity_client: tuple[TestClient, _FakeRepository], +) -> None: + client, repository = daily_activity_client + response: Final = client.get( + "/team/daily/activity/aggregated/tags", + params={**_entity_params("team_ids", "team-a"), "exclude_team_ids": "team-a", "offset": 0, "limit": 5}, + ) + + assert response.status_code == 200, response.text + assert response.json() == {"tags": [], "total_tags": 0, "offset": 0, "limit": 5} + repository.tag_page.assert_not_awaited() + + def test_team_aggregated_ignores_group_by_and_reads_the_team_table( daily_activity_client: tuple[TestClient, _FakeRepository], ) -> None: