mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-04 07:27:51 +00:00
Merge 69ed71a908 into 73e91735dd
This commit is contained in:
commit
2172fe49c3
13 changed files with 547 additions and 4 deletions
|
|
@ -2268,6 +2268,8 @@ USAGE_TOP_API_KEYS_DEFAULT: Final[int] = 100
|
|||
USAGE_TOP_API_KEYS_MAX: Final[int] = 1000
|
||||
USAGE_KEY_PAGE_DEFAULT: Final[int] = 50
|
||||
USAGE_KEY_PAGE_MAX: Final[int] = 100
|
||||
USAGE_TAG_PAGE_DEFAULT: Final[int] = 50
|
||||
USAGE_TAG_PAGE_MAX: Final[int] = 100
|
||||
USAGE_USER_PAGE_DEFAULT: Final[int] = 50
|
||||
USAGE_USER_PAGE_MAX: Final[int] = 100
|
||||
USAGE_KEY_SEARCH_DEFAULT: Final[int] = 100
|
||||
|
|
|
|||
|
|
@ -606,6 +606,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value,
|
||||
KeyManagementRoutes.TEAM_DAILY_ACTIVITY.value,
|
||||
KeyManagementRoutes.TEAM_DAILY_ACTIVITY_AGGREGATED.value,
|
||||
"/team/daily/activity/aggregated/tags",
|
||||
"/team/daily/activity/aggregated/keys",
|
||||
"/team/daily/activity/aggregated/search",
|
||||
"/team/daily/activity/aggregated/model_top_keys",
|
||||
|
|
@ -659,6 +660,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/team/permissions_bulk_update",
|
||||
"/team/daily/activity",
|
||||
"/team/daily/activity/aggregated",
|
||||
"/team/daily/activity/aggregated/tags",
|
||||
"/team/daily/activity/aggregated/keys",
|
||||
"/team/daily/activity/aggregated/search",
|
||||
"/team/daily/activity/aggregated/model_top_keys",
|
||||
|
|
@ -849,6 +851,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/team/permissions_update",
|
||||
"/team/daily/activity",
|
||||
"/team/daily/activity/aggregated",
|
||||
"/team/daily/activity/aggregated/tags",
|
||||
"/team/daily/activity/aggregated/keys",
|
||||
"/team/daily/activity/aggregated/search",
|
||||
"/team/daily/activity/aggregated/model_top_keys",
|
||||
|
|
@ -973,6 +976,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/user/daily/activity/aggregated/users",
|
||||
"/team/daily/activity",
|
||||
"/team/daily/activity/aggregated",
|
||||
"/team/daily/activity/aggregated/tags",
|
||||
"/team/daily/activity/aggregated/keys",
|
||||
"/team/daily/activity/aggregated/search",
|
||||
"/team/daily/activity/aggregated/model_top_keys",
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.proxy.management_endpoints.daily_activity_scopes import (
|
|||
EntityQuery,
|
||||
EntityScopeResolver,
|
||||
ResolvedScope,
|
||||
named_team_ids,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.ptu_consumption import single_team_id, with_ptu_consumption
|
||||
from litellm.proxy.management_endpoints.team_endpoints import aggregated_date_range_error
|
||||
|
|
@ -44,6 +45,7 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
|||
CacheLeakageKeysResponse,
|
||||
DailyActivityKeyPageResponse,
|
||||
DailyActivityKeySearchResponse,
|
||||
DailyActivityTagPageResponse,
|
||||
DailyActivityUserPageResponse,
|
||||
KeyActivityRow,
|
||||
KeyMetadata,
|
||||
|
|
@ -55,10 +57,12 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
|||
UserActivityRow,
|
||||
)
|
||||
from litellm.types.repositories.daily_activity import (
|
||||
DailyActivityTable,
|
||||
ExportRow,
|
||||
ExportType,
|
||||
KeyMetadataRow,
|
||||
KeySpendRow,
|
||||
TagPage,
|
||||
UserMetadataRow,
|
||||
UserSpendRow,
|
||||
)
|
||||
|
|
@ -302,7 +306,8 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver,
|
|||
include_entity_breakdown=resolver.include_entity_breakdown,
|
||||
api_key_limit=api_key_limit,
|
||||
)
|
||||
if resolver.entity != "team":
|
||||
# PTU hours are a whole-team figure, so a team's per-tag breakdown carries none.
|
||||
if resolver.entity != "team" or resolved.scope.table is not DailyActivityTable.TEAM:
|
||||
return activity
|
||||
return with_ptu_consumption(activity, llm_router, single_team_id(resolved.scope.entity_ids))
|
||||
except HTTPException:
|
||||
|
|
@ -395,6 +400,59 @@ def _register_key_page_route(router: APIRouter, resolver: EntityScopeResolver, p
|
|||
)
|
||||
|
||||
|
||||
def _register_tag_page_route(router: APIRouter, resolver: EntityScopeResolver, prefix: str) -> None:
|
||||
@management_endpoint_wrapper
|
||||
async def tag_page(
|
||||
entity_query: Annotated[EntityQuery, Depends(resolver.query)],
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
repository: Annotated[DailyActivityRepository, Depends(get_daily_activity_repository)],
|
||||
prisma_client: Annotated[PrismaClient, Depends(get_daily_activity_prisma_client)],
|
||||
offset: Annotated[int, Query(ge=0)] = 0,
|
||||
limit: Annotated[int, Query(ge=1, le=constants.USAGE_TAG_PAGE_MAX)] = constants.USAGE_TAG_PAGE_DEFAULT,
|
||||
) -> DailyActivityTagPageResponse:
|
||||
try:
|
||||
team_ids: Final = named_team_ids(entity_query)
|
||||
if len(team_ids) != 1:
|
||||
raise_public(
|
||||
InvalidDateRange(reason=f"tags page needs exactly one team in team_ids, got {len(team_ids)}")
|
||||
)
|
||||
normalized_query: Final = replace(entity_query, entity_ids=team_ids)
|
||||
resolved: Final = await _resolved_scope(
|
||||
resolver,
|
||||
normalized_query,
|
||||
user_api_key_dict,
|
||||
prisma_client,
|
||||
user_aggregated=False,
|
||||
)
|
||||
page: Final = (
|
||||
TagPage(rows=(), total_tags=0)
|
||||
if team_ids[0] in resolved.scope.exclude_entity_ids
|
||||
else await repository.tag_page(resolved.scope, team_id=team_ids[0], offset=offset, limit=limit)
|
||||
)
|
||||
return DailyActivityTagPageResponse(
|
||||
tags=list(page.rows),
|
||||
total_tags=page.total_tags,
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001, LIT003 # keep sibling routes' 500 error mapping
|
||||
verbose_proxy_logger.exception("Daily activity tag page failed: %s", exc)
|
||||
raise HTTPException(status_code=500, detail={"error": f"Failed to fetch analytics: {exc}"})
|
||||
|
||||
router.add_api_route(
|
||||
f"{prefix}/daily/activity/aggregated/tags",
|
||||
tag_page,
|
||||
methods=["GET"],
|
||||
name=resolver.operation_names["tag_page"],
|
||||
tags=list(resolver.tags),
|
||||
dependencies=(Depends(user_api_key_auth),),
|
||||
response_model=DailyActivityTagPageResponse,
|
||||
include_in_schema=prefix != "/end_user",
|
||||
)
|
||||
|
||||
|
||||
def _register_search_route(router: APIRouter, resolver: EntityScopeResolver, prefix: str) -> None:
|
||||
@management_endpoint_wrapper
|
||||
async def search(
|
||||
|
|
@ -646,6 +704,8 @@ def register_daily_activity_routes(router: APIRouter, resolver: EntityScopeResol
|
|||
for prefix in resolver.route_prefixes:
|
||||
_register_aggregated_route(router, resolver, prefix)
|
||||
_register_key_page_route(router, resolver, prefix)
|
||||
if resolver.entity == "team":
|
||||
_register_tag_page_route(router, resolver, prefix)
|
||||
_register_search_route(router, resolver, prefix)
|
||||
_register_model_top_keys_route(router, resolver, prefix)
|
||||
_register_export_route(router, resolver, prefix)
|
||||
|
|
|
|||
|
|
@ -272,6 +272,10 @@ def _team_query(
|
|||
)
|
||||
|
||||
|
||||
def named_team_ids(query: EntityQuery) -> tuple[str, ...]:
|
||||
return tuple(team_id for team_id in query.entity_ids or () if team_id)
|
||||
|
||||
|
||||
def _tag_query(
|
||||
start_date: str | None = None,
|
||||
end_date: str | None = None,
|
||||
|
|
@ -390,6 +394,7 @@ TEAM_RESOLVER = EntityScopeResolver(
|
|||
"aggregated": "get_team_daily_activity_aggregated",
|
||||
"search": "get_team_daily_activity_aggregated_search",
|
||||
"key_page": "get_team_daily_activity_aggregated_keys",
|
||||
"tag_page": "get_team_daily_activity_aggregated_tags",
|
||||
"model_top_keys": "get_team_daily_activity_model_top_keys",
|
||||
"export": "get_team_daily_activity_export",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from litellm.repositories.daily_activity_sql import (
|
|||
build_key_page_sql,
|
||||
build_key_search_sql,
|
||||
build_model_top_keys_sql,
|
||||
build_tag_page_sql,
|
||||
build_user_page_sql,
|
||||
)
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
|
|
@ -40,6 +41,8 @@ from litellm.types.repositories.daily_activity import (
|
|||
KeyPage,
|
||||
KeySpendRow,
|
||||
SpendLogsWindow,
|
||||
TagPage,
|
||||
TagSpendRow,
|
||||
UserMetadataRow,
|
||||
UserPage,
|
||||
UserSpendRow,
|
||||
|
|
@ -102,6 +105,7 @@ _GROUPING_ADAPTER: Final = TypeAdapter(tuple[GroupingSetsRow, ...])
|
|||
_ENTITY_ADAPTER: Final = TypeAdapter(tuple[EntityRollupRow, ...])
|
||||
_KEY_SPEND_ADAPTER: Final = TypeAdapter(tuple[KeySpendRow, ...])
|
||||
_KEY_PAGE_TOTAL_ADAPTER: Final[TypeAdapter[int]] = TypeAdapter(int)
|
||||
_TAG_SPEND_ADAPTER: Final = TypeAdapter(tuple[TagSpendRow, ...])
|
||||
_USER_SPEND_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...])
|
||||
_EXPORT_ADAPTER: Final = TypeAdapter(tuple[ExportRow, ...])
|
||||
_METADATA_TAGS_ADAPTER: Final = TypeAdapter(list[StrictStr])
|
||||
|
|
@ -208,6 +212,16 @@ class DailyActivityRepository:
|
|||
rows: Final = _KEY_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("api_key") is not None))
|
||||
return KeyPage(rows=rows, total_api_keys=total_api_keys)
|
||||
|
||||
async def tag_page(self, scope: DailyActivityScope, *, team_id: str, offset: int, limit: int) -> TagPage:
|
||||
query: Final = build_tag_page_sql(scope, team_id=team_id, offset=offset, limit=limit)
|
||||
result: Final = await self._query(query)
|
||||
total_tags_value: Final = result[0].get("total_tags") if result else 0
|
||||
total_tags: Final = (
|
||||
_KEY_PAGE_TOTAL_ADAPTER.validate_python(total_tags_value) if total_tags_value is not None else 0
|
||||
)
|
||||
rows: Final = _TAG_SPEND_ADAPTER.validate_python(tuple(row for row in result if row.get("tag") is not None))
|
||||
return TagPage(rows=rows, total_tags=total_tags)
|
||||
|
||||
async def user_page(self, scope: DailyActivityScope, *, offset: int, limit: int) -> UserPage:
|
||||
query: Final = build_user_page_sql(scope, offset=offset, limit=limit)
|
||||
result: Final = await self._query(query)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import count, islice
|
||||
from types import MappingProxyType
|
||||
|
|
@ -106,6 +106,9 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
|
|||
return " AND ".join(conditions), params
|
||||
|
||||
|
||||
_NOT_USER_AGENT_TAG: Final = "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')"
|
||||
|
||||
|
||||
def _ptu_flat_cost_select(table: DailyActivityTable, *, aggregate: bool = True) -> str:
|
||||
if table is DailyActivityTable.TEAM:
|
||||
return "SUM(ptu_flat_cost)::float AS ptu_flat_cost" if aggregate else "SUM(scoped.ptu_flat_cost)::float"
|
||||
|
|
@ -301,6 +304,42 @@ def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) ->
|
|||
return SqlQuery(sql=sql, params=(*where_params, PTU_SENTINEL_API_KEY, limit, offset))
|
||||
|
||||
|
||||
def build_tag_page_sql(scope: DailyActivityScope, *, team_id: str, offset: int, limit: int) -> SqlQuery:
|
||||
if not 1 <= limit <= constants.USAGE_TAG_PAGE_MAX:
|
||||
raise ValueError(f"limit must be between 1 and {constants.USAGE_TAG_PAGE_MAX}")
|
||||
if offset < 0:
|
||||
raise ValueError("offset must be non-negative")
|
||||
tag_scope: Final = replace(
|
||||
scope,
|
||||
table=DailyActivityTable.TAG,
|
||||
entity_id_field="tag",
|
||||
entity_ids=None,
|
||||
exclude_entity_ids=(),
|
||||
)
|
||||
where_clause, where_params = build_where_clause(tag_scope)
|
||||
team_param: Final = len(where_params) + 1
|
||||
sentinel_param: Final = team_param + 1
|
||||
limit_param: Final = sentinel_param + 1
|
||||
offset_param: Final = limit_param + 1
|
||||
sql: Final = f"""
|
||||
WITH ranked AS (
|
||||
SELECT tag,{_key_spend_select()}, SUM(spend::numeric) AS rank_spend
|
||||
FROM "{PRISMA_TO_PG_TABLE[DailyActivityTable.TAG]}"
|
||||
WHERE {where_clause} AND team_id = ${team_param} AND api_key <> ${sentinel_param}
|
||||
AND tag IS NOT NULL AND {_NOT_USER_AGENT_TAG}
|
||||
GROUP BY tag
|
||||
)
|
||||
SELECT (SELECT COUNT(*) FROM ranked)::bigint AS total_tags, page.*
|
||||
FROM (SELECT 1) AS one
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT * FROM ranked
|
||||
ORDER BY rank_spend DESC, tag ASC
|
||||
LIMIT ${limit_param} OFFSET ${offset_param}
|
||||
) AS page ON TRUE
|
||||
"""
|
||||
return SqlQuery(sql=sql, params=(*where_params, team_id, PTU_SENTINEL_API_KEY, limit, offset))
|
||||
|
||||
|
||||
def _user_spend_select() -> str:
|
||||
return """
|
||||
COALESCE(SUM(spend), 0)::float AS spend,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from pydantic import ConfigDict, Field
|
|||
from typing_extensions import TypedDict
|
||||
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.repositories.daily_activity import TagSpendRow
|
||||
|
||||
|
||||
class GroupByDimension(str, Enum):
|
||||
|
|
@ -163,6 +164,15 @@ class DailyActivityKeyPageResponse(LiteLLMBaseModel):
|
|||
limit: int
|
||||
|
||||
|
||||
class DailyActivityTagPageResponse(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
tags: list[TagSpendRow] # mutable-ok: response contract requires a JSON array
|
||||
total_tags: int
|
||||
offset: int
|
||||
limit: int
|
||||
|
||||
|
||||
class UserActivityRow(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -65,6 +65,26 @@ class KeyPage:
|
|||
total_api_keys: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TagSpendRow:
|
||||
tag: str
|
||||
spend: float
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
total_tokens: int
|
||||
api_requests: int
|
||||
successful_requests: int
|
||||
failed_requests: int
|
||||
cache_read_input_tokens: int
|
||||
cache_creation_input_tokens: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TagPage:
|
||||
rows: tuple[TagSpendRow, ...]
|
||||
total_tags: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UserSpendRow:
|
||||
user_id: str | None
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ GET /tag/user-agent/per-user-analytics
|
|||
GET /tag/wau
|
||||
GET /team/daily/activity
|
||||
GET /team/daily/activity/aggregated
|
||||
GET /team/daily/activity/aggregated/tags # proxy-only analytics report; no provider resource
|
||||
GET /team/daily/activity/aggregated/keys
|
||||
GET /team/daily/activity/aggregated/model_top_keys
|
||||
GET /team/daily/activity/aggregated/search
|
||||
|
|
|
|||
|
|
@ -1,13 +1,53 @@
|
|||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.client import Gateway, eventually, list_value, object_value
|
||||
from integration._support.database import read_rows
|
||||
|
||||
_SPEND_PER_CALL: Final = 20 * 0.001 + 20 * 0.002
|
||||
|
||||
|
||||
def _tagged_chat(gateway: Gateway, model: str, key: str, tags: list[str]) -> None:
|
||||
reply: Final = gateway.post(
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": f"tagged {uuid.uuid4().hex}"}],
|
||||
"metadata": {"tags": tags},
|
||||
},
|
||||
key=key,
|
||||
)
|
||||
assert reply["usage"]["total_tokens"] == 40, reply
|
||||
|
||||
|
||||
def _tag_page(
|
||||
gateway: Gateway,
|
||||
team: str,
|
||||
*,
|
||||
offset: int = 0,
|
||||
limit: int = 50,
|
||||
key: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
today: Final = datetime.now(timezone.utc)
|
||||
response: Final = gateway.request(
|
||||
"GET",
|
||||
"/team/daily/activity/aggregated/tags",
|
||||
params={
|
||||
"team_ids": team,
|
||||
"start_date": (today - timedelta(days=1)).strftime("%Y-%m-%d"),
|
||||
"end_date": (today + timedelta(days=1)).strftime("%Y-%m-%d"),
|
||||
"timezone": "0",
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
},
|
||||
key=key,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
return object_value(response.json())
|
||||
|
||||
|
||||
def test_one_key_moved_between_teams_records_its_tag_once_per_team(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
|
||||
|
|
@ -43,3 +83,72 @@ def test_one_key_moved_between_teams_records_its_tag_once_per_team(gateway: Gate
|
|||
)
|
||||
for row in rows:
|
||||
assert float(row["spend"]) == pytest.approx(calls_per_team[row["team_id"]] * _SPEND_PER_CALL), rows
|
||||
|
||||
|
||||
def test_tag_page_reports_one_teams_spend_per_tag(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
|
||||
shared: Final = f"cost-center:{uuid.uuid4().hex}"
|
||||
own: Final = f"project:{uuid.uuid4().hex}"
|
||||
team: Final = scenario.team(models=[model])
|
||||
other_team: Final = scenario.team(models=[model])
|
||||
key: Final = scenario.key(team_id=team, models=[model])
|
||||
other_key: Final = scenario.key(team_id=other_team, models=[model])
|
||||
no_team_key: Final = scenario.key(models=[model])
|
||||
no_team_tag: Final = f"unassigned:{uuid.uuid4().hex}"
|
||||
_tagged_chat(gateway, model, key, [shared, own, "User-Agent: integration/1.0"])
|
||||
_tagged_chat(gateway, model, key, [shared])
|
||||
_tagged_chat(gateway, model, other_key, [shared])
|
||||
_tagged_chat(gateway, model, no_team_key, [no_team_tag])
|
||||
eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag IN (%s, %s, %s)',
|
||||
(shared, own, no_team_tag),
|
||||
),
|
||||
lambda values: sum(int(row["api_requests"]) for row in values) == 5,
|
||||
seconds=70,
|
||||
)
|
||||
|
||||
first_page: Final = _tag_page(gateway, team, limit=1)
|
||||
next_page: Final = _tag_page(gateway, team, offset=1, limit=1)
|
||||
assert first_page["total_tags"] == 2
|
||||
assert next_page["total_tags"] == 2
|
||||
first_tag: Final = object_value(list_value(first_page["tags"])[0])
|
||||
next_tag: Final = object_value(list_value(next_page["tags"])[0])
|
||||
assert first_tag["tag"] == shared
|
||||
assert first_tag["spend"] == pytest.approx(2 * _SPEND_PER_CALL)
|
||||
assert first_tag["api_requests"] == 2
|
||||
assert next_tag["tag"] == own
|
||||
assert next_tag["spend"] == pytest.approx(_SPEND_PER_CALL)
|
||||
assert next_tag["api_requests"] == 1
|
||||
|
||||
|
||||
def test_tag_page_keeps_the_team_routes_member_permissions(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
|
||||
tag: Final = f"cost-center:{uuid.uuid4().hex}"
|
||||
team: Final = scenario.team(models=[model])
|
||||
other_team: Final = scenario.team(models=[model])
|
||||
member: Final = scenario.member(team)
|
||||
member_key: Final = scenario.key(user_id=member, team_id=team, models=[model])
|
||||
teammate_key: Final = scenario.key(team_id=team, models=[model])
|
||||
_tagged_chat(gateway, model, member_key, [tag])
|
||||
_tagged_chat(gateway, model, teammate_key, [tag])
|
||||
eventually(
|
||||
lambda: read_rows('SELECT api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag = %s', (tag,)),
|
||||
lambda values: sum(int(row["api_requests"]) for row in values) == 2,
|
||||
seconds=70,
|
||||
)
|
||||
|
||||
member_page: Final = _tag_page(gateway, team, key=member_key)
|
||||
assert member_page["total_tags"] == 1
|
||||
member_tag: Final = object_value(list_value(member_page["tags"])[0])
|
||||
assert member_tag["tag"] == tag
|
||||
assert member_tag["api_requests"] == 1
|
||||
denied: Final = gateway.request(
|
||||
"GET",
|
||||
"/team/daily/activity/aggregated/tags",
|
||||
params={"team_ids": other_team, "start_date": "2026-01-01", "end_date": "2026-01-02"},
|
||||
key=member_key,
|
||||
)
|
||||
assert denied.status_code == 404, denied.text
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import csv
|
||||
import io
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass, fields
|
||||
from dataclasses import dataclass, fields, replace
|
||||
from itertools import chain
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final
|
||||
|
|
@ -35,6 +35,8 @@ from litellm.types.repositories.daily_activity import (
|
|||
KeyMetadataRow,
|
||||
KeyPage,
|
||||
KeySpendRow,
|
||||
TagPage,
|
||||
TagSpendRow,
|
||||
UserMetadataRow,
|
||||
UserPage,
|
||||
UserSpendRow,
|
||||
|
|
@ -250,6 +252,8 @@ class _FakeRepository:
|
|||
self.aggregated = AsyncMock(side_effect=self._aggregated)
|
||||
self.key_page = AsyncMock(side_effect=self._key_page)
|
||||
self.key_page_call: tuple[DailyActivityScope, int, int] | None = None
|
||||
self.tag_page = AsyncMock(side_effect=self._tag_page)
|
||||
self.tag_page_call: tuple[DailyActivityScope, str, int, int] | None = None
|
||||
self.search_keys = AsyncMock(side_effect=self._search_keys)
|
||||
self.model_top_keys = AsyncMock(side_effect=self._model_top_keys)
|
||||
self.cache_leakage_keys = AsyncMock(side_effect=self._cache_leakage_keys)
|
||||
|
|
@ -321,6 +325,29 @@ class _FakeRepository:
|
|||
total_api_keys=len(api_keys),
|
||||
)
|
||||
|
||||
async def _tag_page(
|
||||
self, scope: DailyActivityScope, *, team_id: str, offset: int, limit: int
|
||||
) -> TagPage:
|
||||
self.tag_page_call = (scope, team_id, offset, limit)
|
||||
tag_scope: Final = replace(
|
||||
scope,
|
||||
table=DailyActivityTable.TAG,
|
||||
entity_id_field="tag",
|
||||
entity_ids=None,
|
||||
exclude_entity_ids=(),
|
||||
)
|
||||
rows: Final = self._matching_rows(tag_scope)
|
||||
tags: Final = tuple(
|
||||
sorted(
|
||||
{row.entity_id for row in rows},
|
||||
key=lambda tag: (-sum(row.spend for row in rows if row.entity_id == tag), tag),
|
||||
)
|
||||
)
|
||||
return TagPage(
|
||||
rows=tuple(_tag_spend_row(tag, rows) for tag in tags[offset : offset + limit]),
|
||||
total_tags=len(tags),
|
||||
)
|
||||
|
||||
async def _model_top_keys(
|
||||
self, scope: DailyActivityScope, *, model_group: str, by_model_group: bool, limit: int
|
||||
) -> tuple[KeySpendRow, ...]:
|
||||
|
|
@ -445,6 +472,22 @@ def _key_spend_row(api_key: str, rows: Sequence[_Activity]) -> KeySpendRow:
|
|||
)
|
||||
|
||||
|
||||
def _tag_spend_row(tag: str, rows: Sequence[_Activity]) -> TagSpendRow:
|
||||
matching: Final = tuple(row for row in rows if row.entity_id == tag)
|
||||
return TagSpendRow(
|
||||
tag=tag,
|
||||
spend=sum(row.spend for row in matching),
|
||||
prompt_tokens=sum(row.prompt_tokens for row in matching),
|
||||
completion_tokens=sum(row.completion_tokens for row in matching),
|
||||
total_tokens=sum(row.prompt_tokens + row.completion_tokens for row in matching),
|
||||
api_requests=sum(row.api_requests for row in matching),
|
||||
successful_requests=sum(row.successful_requests for row in matching),
|
||||
failed_requests=sum(row.failed_requests for row in matching),
|
||||
cache_read_input_tokens=sum(row.cache_read_input_tokens for row in matching),
|
||||
cache_creation_input_tokens=sum(row.cache_creation_input_tokens for row in matching),
|
||||
)
|
||||
|
||||
|
||||
def _entity_rows_for_day(
|
||||
date: str, rows: Sequence[_Activity], distinct_api_keys: int | None = None
|
||||
) -> tuple[EntityRollupRow, ...]:
|
||||
|
|
@ -1267,6 +1310,105 @@ def test_team_scope_applies_membership_and_user_key_filter(
|
|||
assert repository.aggregated.call_args.kwargs["include_entity_breakdown"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("team_ids", ["team-a", "team-a,", ",team-a"])
|
||||
def test_team_tag_page_normalizes_team_ids_and_applies_the_members_key_filter(
|
||||
daily_activity_client: tuple[TestClient, _FakeRepository],
|
||||
team_ids: str,
|
||||
) -> None:
|
||||
client, repository = daily_activity_client
|
||||
response: Final = client.get(
|
||||
"/team/daily/activity/aggregated/tags",
|
||||
params={**_entity_params("team_ids", team_ids), "offset": 3, "limit": 7},
|
||||
headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value, "x-user-id": "user-a"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["offset"] == 3
|
||||
assert response.json()["limit"] == 7
|
||||
tag_page_call: Final = repository.tag_page_call
|
||||
assert tag_page_call is not None
|
||||
scope, team_id, offset, limit = tag_page_call
|
||||
assert (scope.table, scope.entity_ids, scope.api_keys, team_id, offset, limit) == (
|
||||
DailyActivityTable.TEAM,
|
||||
("team-a",),
|
||||
("key-alpha",),
|
||||
"team-a",
|
||||
3,
|
||||
7,
|
||||
), tag_page_call
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("team_ids", "expected_count"),
|
||||
[(None, 0), ("team-a,team-b", 2), ("team-a,,team-b", 2)],
|
||||
)
|
||||
@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER])
|
||||
def test_team_tag_page_needs_exactly_one_named_team_for_every_caller(
|
||||
daily_activity_client: tuple[TestClient, _FakeRepository],
|
||||
team_ids: str | None,
|
||||
expected_count: int,
|
||||
role: LitellmUserRoles,
|
||||
) -> None:
|
||||
client, repository = daily_activity_client
|
||||
response: Final = client.get(
|
||||
"/team/daily/activity/aggregated/tags",
|
||||
params={**_DATE_PARAMS, **({"team_ids": team_ids} if team_ids else {})},
|
||||
headers={"x-user-role": role.value, "x-user-id": "user-a"},
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
assert response.json() == {
|
||||
"detail": {"error": f"tags page needs exactly one team in team_ids, got {expected_count}"}
|
||||
}
|
||||
repository.tag_page.assert_not_awaited()
|
||||
|
||||
|
||||
def test_team_tag_page_returns_404_for_a_non_member(
|
||||
daily_activity_client: tuple[TestClient, _FakeRepository],
|
||||
) -> None:
|
||||
client, repository = daily_activity_client
|
||||
response: Final = client.get(
|
||||
"/team/daily/activity/aggregated/tags",
|
||||
params=_entity_params("team_ids", "team-b"),
|
||||
headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value, "x-user-id": "user-a"},
|
||||
)
|
||||
|
||||
assert response.status_code == 404, response.text
|
||||
repository.tag_page.assert_not_awaited()
|
||||
|
||||
|
||||
def test_team_tag_page_is_empty_for_a_team_that_is_also_excluded(
|
||||
daily_activity_client: tuple[TestClient, _FakeRepository],
|
||||
) -> None:
|
||||
client, repository = daily_activity_client
|
||||
response: Final = client.get(
|
||||
"/team/daily/activity/aggregated/tags",
|
||||
params={**_entity_params("team_ids", "team-a"), "exclude_team_ids": "team-a", "offset": 0, "limit": 5},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"tags": [], "total_tags": 0, "offset": 0, "limit": 5}
|
||||
repository.tag_page.assert_not_awaited()
|
||||
|
||||
|
||||
def test_team_aggregated_ignores_group_by_and_reads_the_team_table(
|
||||
daily_activity_client: tuple[TestClient, _FakeRepository],
|
||||
) -> None:
|
||||
client, repository = daily_activity_client
|
||||
params: Final = _entity_params("team_ids", "team-a")
|
||||
without_group_by: Final = client.get("/team/daily/activity/aggregated", params=params)
|
||||
with_group_by: Final = client.get(
|
||||
"/team/daily/activity/aggregated",
|
||||
params={**params, "group_by": "tag"},
|
||||
)
|
||||
assert without_group_by.status_code == with_group_by.status_code == 200
|
||||
assert with_group_by.json() == without_group_by.json()
|
||||
scope: Final = repository.aggregated.call_args.args[0]
|
||||
assert (scope.table, scope.entity_id_field, scope.entity_ids) == (
|
||||
DailyActivityTable.TEAM,
|
||||
"team_id",
|
||||
("team-a",),
|
||||
), scope
|
||||
|
||||
|
||||
def test_team_scope_does_not_allow_an_unowned_api_key(
|
||||
daily_activity_client: tuple[TestClient, _FakeRepository],
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from dataclasses import replace
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -15,6 +16,7 @@ from litellm.repositories.daily_activity_sql import (
|
|||
build_key_page_sql,
|
||||
build_key_search_sql,
|
||||
build_model_top_keys_sql,
|
||||
build_tag_page_sql,
|
||||
build_user_page_sql,
|
||||
build_where_clause,
|
||||
)
|
||||
|
|
@ -140,6 +142,51 @@ def test_key_page_sql_rejects_invalid_page_bounds(offset: int, limit: int, error
|
|||
build_key_page_sql(_scope(), offset=offset, limit=limit)
|
||||
|
||||
|
||||
def test_tag_page_sql_scopes_to_team_excludes_user_agents_and_orders_tags() -> None:
|
||||
query: Final = build_tag_page_sql(
|
||||
_scope(table=DailyActivityTable.TEAM, entity_ids=("team-a",), api_keys=("key-1",), model="gpt-test"),
|
||||
team_id="team-a",
|
||||
offset=7,
|
||||
limit=3,
|
||||
)
|
||||
|
||||
assert query.params == (
|
||||
"2026-01-01",
|
||||
"2026-01-31",
|
||||
"gpt-test",
|
||||
["key-1"],
|
||||
"team-a",
|
||||
PTU_SENTINEL_API_KEY,
|
||||
3,
|
||||
7,
|
||||
)
|
||||
assert 'FROM "LiteLLM_DailyTagSpend"' in query.sql
|
||||
assert "team_id = $5" in query.sql
|
||||
assert "api_key = ANY($4::text[])" in query.sql
|
||||
assert "tag IS NOT NULL" in query.sql
|
||||
assert "api_key <> $6" in query.sql
|
||||
assert "NOT (LOWER(BTRIM(tag)) LIKE 'user-agent:%' OR LOWER(BTRIM(tag)) LIKE 'user agent:%')" in query.sql
|
||||
assert "GROUP BY tag" in query.sql
|
||||
assert "SUM(spend::numeric) AS rank_spend" in query.sql
|
||||
assert "ORDER BY rank_spend DESC, tag ASC" in query.sql
|
||||
assert "(SELECT COUNT(*) FROM ranked)::bigint AS total_tags" in query.sql
|
||||
assert "LIMIT $7 OFFSET $8" in query.sql
|
||||
assert '"tag" = ANY' not in query.sql
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("offset", "limit", "error"),
|
||||
(
|
||||
(0, 0, "limit must be between"),
|
||||
(0, constants.USAGE_TAG_PAGE_MAX + 1, "limit must be between"),
|
||||
(-1, 1, "offset must be non-negative"),
|
||||
),
|
||||
)
|
||||
def test_tag_page_sql_rejects_invalid_page_bounds(offset: int, limit: int, error: str) -> None:
|
||||
with pytest.raises(ValueError, match=error):
|
||||
build_tag_page_sql(_scope(table=DailyActivityTable.TEAM), team_id="team-a", offset=offset, limit=limit)
|
||||
|
||||
|
||||
def test_user_page_sql_groups_by_user_id_with_deterministic_spend_order() -> None:
|
||||
query: Final = build_user_page_sql(_scope(), offset=7, limit=3)
|
||||
|
||||
|
|
|
|||
90
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
90
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -16704,6 +16704,23 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/team/daily/activity/aggregated/tags": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Get Team Daily Activity Aggregated Tags */
|
||||
get: operations["get_team_daily_activity_aggregated_tags_team_daily_activity_aggregated_tags_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/team/daily/activity/export": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -31386,6 +31403,17 @@ export interface components {
|
|||
/** Api Keys */
|
||||
api_keys: components["schemas"]["KeyActivityRow"][];
|
||||
};
|
||||
/** DailyActivityTagPageResponse */
|
||||
DailyActivityTagPageResponse: {
|
||||
/** Limit */
|
||||
limit: number;
|
||||
/** Offset */
|
||||
offset: number;
|
||||
/** Tags */
|
||||
tags: components["schemas"]["TagSpendRow"][];
|
||||
/** Total Tags */
|
||||
total_tags: number;
|
||||
};
|
||||
/** DailyActivityUserPageResponse */
|
||||
DailyActivityUserPageResponse: {
|
||||
/** Limit */
|
||||
|
|
@ -46145,6 +46173,29 @@ export interface components {
|
|||
/** Tpm Limit */
|
||||
tpm_limit?: number | null;
|
||||
};
|
||||
/** TagSpendRow */
|
||||
TagSpendRow: {
|
||||
/** Api Requests */
|
||||
api_requests: number;
|
||||
/** Cache Creation Input Tokens */
|
||||
cache_creation_input_tokens: number;
|
||||
/** Cache Read Input Tokens */
|
||||
cache_read_input_tokens: number;
|
||||
/** Completion Tokens */
|
||||
completion_tokens: number;
|
||||
/** Failed Requests */
|
||||
failed_requests: number;
|
||||
/** Prompt Tokens */
|
||||
prompt_tokens: number;
|
||||
/** Spend */
|
||||
spend: number;
|
||||
/** Successful Requests */
|
||||
successful_requests: number;
|
||||
/** Tag */
|
||||
tag: string;
|
||||
/** Total Tokens */
|
||||
total_tokens: number;
|
||||
};
|
||||
/**
|
||||
* TagSummaryMetrics
|
||||
* @description Summary metrics for a tag
|
||||
|
|
@ -73604,6 +73655,45 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_team_daily_activity_aggregated_tags_team_daily_activity_aggregated_tags_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
team_ids?: string | null;
|
||||
start_date?: string | null;
|
||||
end_date?: string | null;
|
||||
model?: string | null;
|
||||
api_key?: string | null;
|
||||
exclude_team_ids?: string | null;
|
||||
timezone?: number | null;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["DailyActivityTagPageResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_team_daily_activity_export_team_daily_activity_export_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue