mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
refactor(ui): move cache dashboard aggregation server-side with a typed response
The /global/activity/cache_hits endpoint previously returned raw per (key, call_type, model) spend-log aggregates typed as LiteLLM_SpendLogs (wrong), and the dashboard reduced them in the browser: grouping by call_type, relabeling empty call_type as Unknown, and computing the stat card totals. All of that now happens server-side. The SQL groups per call_type and splits cache hits vs successful vs failed requests, a new cache_activity module validates rows into Pydantic models and computes totals plus the key-alias/model filter options, and the endpoint declares a real response_model so schema.d.ts types it correctly. The dashboard consumes it through a typed $api react-query hook (filters ride the query key and are applied in SQL instead of the browser), the hand-rolled summarizeCacheActivity transform and the adminGlobalCacheActivity fetch helper are deleted, and the refresh button now actually refetches. The endpoint is UI-internal (hidden from the public swagger), so the response reshape is not a public API break.
This commit is contained in:
ryan-crabbe-berri
parent
770bf73a02
commit
4d5cab3143
12 changed files with 523 additions and 421 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -141,3 +141,4 @@ crash.*.log
|
|||
.coverage
|
||||
|
||||
ui/litellm-dashboard/out/
|
||||
litellm.log
|
||||
|
|
|
|||
|
|
@ -1,106 +1,61 @@
|
|||
#### Analytics Endpoints #####
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Optional
|
||||
from typing import Annotated
|
||||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.analytics_endpoints.cache_activity import CacheActivityResponse, get_cache_activity
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _parse_date(value: str, param_name: str) -> datetime:
|
||||
try:
|
||||
return datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={"error": f"{param_name} must be a YYYY-MM-DD date, got {value!r}"},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/global/activity/cache_hits",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
responses={
|
||||
200: {"model": List[LiteLLM_SpendLogs]},
|
||||
},
|
||||
response_model=CacheActivityResponse,
|
||||
include_in_schema=False,
|
||||
)
|
||||
async def get_global_activity(
|
||||
start_date: Optional[str] = fastapi.Query(
|
||||
default=None,
|
||||
description="Time from which to start viewing spend",
|
||||
),
|
||||
end_date: Optional[str] = fastapi.Query(
|
||||
default=None,
|
||||
description="Time till which to view spend",
|
||||
),
|
||||
):
|
||||
start_date: Annotated[str, fastapi.Query(description="Time from which to start viewing spend")],
|
||||
end_date: Annotated[str, fastapi.Query(description="Time till which to view spend")],
|
||||
key_aliases: Annotated[
|
||||
list[str] | None, fastapi.Query(description="Only include spend from these key aliases")
|
||||
] = None,
|
||||
models: Annotated[list[str] | None, fastapi.Query(description="Only include spend for these models")] = None,
|
||||
) -> CacheActivityResponse:
|
||||
"""
|
||||
Get number of cache hits, vs misses
|
||||
|
||||
{
|
||||
"daily_data": [
|
||||
const chartdata = [
|
||||
{
|
||||
date: 'Jan 22',
|
||||
cache_hits: 10,
|
||||
llm_api_calls: 2000
|
||||
},
|
||||
{
|
||||
date: 'Jan 23',
|
||||
cache_hits: 10,
|
||||
llm_api_calls: 12
|
||||
},
|
||||
],
|
||||
"sum_cache_hits": 20,
|
||||
"sum_llm_api_calls": 2012
|
||||
}
|
||||
Cache activity for the Admin UI cache dashboard, aggregated per call_type:
|
||||
cache hits vs successful LLM API requests vs failed requests, plus totals
|
||||
for the stat cards and the available key-alias/model filter options.
|
||||
"""
|
||||
|
||||
if start_date is None or end_date is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={"error": "Please provide start_date and end_date"},
|
||||
)
|
||||
|
||||
start_date_obj = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
end_date_obj = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
try:
|
||||
if prisma_client is None:
|
||||
raise ValueError(
|
||||
"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys"
|
||||
)
|
||||
|
||||
sql_query = """
|
||||
SELECT
|
||||
CASE
|
||||
WHEN vt."key_alias" IS NOT NULL THEN vt."key_alias"
|
||||
ELSE 'Unnamed Key'
|
||||
END AS api_key,
|
||||
sl."call_type",
|
||||
sl."model",
|
||||
COUNT(*) AS total_rows,
|
||||
SUM(CASE WHEN sl."cache_hit" = 'True' THEN 1 ELSE 0 END) AS cache_hit_true_rows,
|
||||
SUM(CASE WHEN sl."status" = 'failure' THEN 1 ELSE 0 END) AS failed_rows,
|
||||
SUM(CASE WHEN sl."cache_hit" = 'True' THEN sl."completion_tokens" ELSE 0 END) AS cached_completion_tokens,
|
||||
SUM(CASE WHEN sl."cache_hit" != 'True' THEN sl."completion_tokens" ELSE 0 END) AS generated_completion_tokens
|
||||
FROM "LiteLLM_SpendLogs" sl
|
||||
LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token"
|
||||
WHERE
|
||||
sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC')
|
||||
AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC')
|
||||
GROUP BY
|
||||
vt."key_alias",
|
||||
sl."call_type",
|
||||
sl."model"
|
||||
"""
|
||||
db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj)
|
||||
|
||||
if db_response is None:
|
||||
return []
|
||||
|
||||
return db_response
|
||||
|
||||
except Exception as e:
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={"error": str(e)},
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail={
|
||||
"error": "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys"
|
||||
},
|
||||
)
|
||||
|
||||
return await get_cache_activity(
|
||||
prisma_client=prisma_client,
|
||||
start_date=_parse_date(start_date, "start_date"),
|
||||
end_date=_parse_date(end_date, "end_date"),
|
||||
key_aliases=key_aliases or [],
|
||||
models=models or [],
|
||||
)
|
||||
|
|
|
|||
137
litellm/proxy/analytics_endpoints/cache_activity.py
Normal file
137
litellm/proxy/analytics_endpoints/cache_activity.py
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import asyncio
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Sequence
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
UNKNOWN_CALL_TYPE = "Unknown"
|
||||
|
||||
|
||||
class CacheActivityGroup(BaseModel):
|
||||
call_type: str
|
||||
api_requests: int
|
||||
cache_hits: int
|
||||
failed_requests: int
|
||||
cached_completion_tokens: int
|
||||
generated_completion_tokens: int
|
||||
|
||||
|
||||
class CacheActivityTotals(BaseModel):
|
||||
api_requests: int
|
||||
cache_hits: int
|
||||
failed_requests: int
|
||||
cached_completion_tokens: int
|
||||
cache_hit_ratio: float
|
||||
|
||||
|
||||
class CacheActivityFilterOptions(BaseModel):
|
||||
key_aliases: list[str]
|
||||
models: list[str]
|
||||
|
||||
|
||||
class CacheActivityResponse(BaseModel):
|
||||
groups: list[CacheActivityGroup]
|
||||
totals: CacheActivityTotals
|
||||
filter_options: CacheActivityFilterOptions
|
||||
|
||||
|
||||
GROUPS_SQL = """
|
||||
SELECT
|
||||
CASE WHEN sl."call_type" = '' THEN 'Unknown' ELSE sl."call_type" END AS call_type,
|
||||
(COUNT(*)
|
||||
- SUM(CASE WHEN COALESCE(sl."cache_hit", '') = 'True' THEN 1 ELSE 0 END)
|
||||
- SUM(CASE WHEN sl."status" = 'failure' THEN 1 ELSE 0 END))::int AS api_requests,
|
||||
SUM(CASE WHEN COALESCE(sl."cache_hit", '') = 'True' THEN 1 ELSE 0 END)::int AS cache_hits,
|
||||
SUM(CASE WHEN sl."status" = 'failure' THEN 1 ELSE 0 END)::int AS failed_requests,
|
||||
SUM(CASE WHEN COALESCE(sl."cache_hit", '') = 'True' THEN sl."completion_tokens" ELSE 0 END)::int
|
||||
AS cached_completion_tokens,
|
||||
SUM(CASE WHEN COALESCE(sl."cache_hit", '') != 'True' THEN sl."completion_tokens" ELSE 0 END)::int
|
||||
AS generated_completion_tokens
|
||||
FROM "LiteLLM_SpendLogs" sl
|
||||
LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token"
|
||||
WHERE
|
||||
sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC')
|
||||
AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC')
|
||||
AND ($3::jsonb = '[]'::jsonb
|
||||
OR COALESCE(vt."key_alias", 'Unnamed Key') IN (SELECT jsonb_array_elements_text($3::jsonb)))
|
||||
AND ($4::jsonb = '[]'::jsonb
|
||||
OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb)))
|
||||
GROUP BY 1
|
||||
ORDER BY (COUNT(*)) DESC
|
||||
"""
|
||||
|
||||
KEY_ALIAS_OPTIONS_SQL = """
|
||||
SELECT DISTINCT COALESCE(vt."key_alias", 'Unnamed Key') AS key_alias
|
||||
FROM "LiteLLM_SpendLogs" sl
|
||||
LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token"
|
||||
WHERE
|
||||
sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC')
|
||||
AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC')
|
||||
ORDER BY 1
|
||||
"""
|
||||
|
||||
MODEL_OPTIONS_SQL = """
|
||||
SELECT DISTINCT sl."model" AS model
|
||||
FROM "LiteLLM_SpendLogs" sl
|
||||
WHERE
|
||||
sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC')
|
||||
AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC')
|
||||
AND sl."model" != ''
|
||||
ORDER BY 1
|
||||
"""
|
||||
|
||||
|
||||
class _KeyAliasRow(BaseModel):
|
||||
key_alias: str
|
||||
|
||||
|
||||
class _ModelRow(BaseModel):
|
||||
model: str
|
||||
|
||||
|
||||
_groups_adapter = TypeAdapter(list[CacheActivityGroup])
|
||||
_key_alias_rows_adapter = TypeAdapter(list[_KeyAliasRow])
|
||||
_model_rows_adapter = TypeAdapter(list[_ModelRow])
|
||||
|
||||
|
||||
def compute_totals(groups: Sequence[CacheActivityGroup]) -> CacheActivityTotals:
|
||||
api_requests = sum(group.api_requests for group in groups)
|
||||
cache_hits = sum(group.cache_hits for group in groups)
|
||||
failed_requests = sum(group.failed_requests for group in groups)
|
||||
all_requests = api_requests + cache_hits + failed_requests
|
||||
return CacheActivityTotals(
|
||||
api_requests=api_requests,
|
||||
cache_hits=cache_hits,
|
||||
failed_requests=failed_requests,
|
||||
cached_completion_tokens=sum(group.cached_completion_tokens for group in groups),
|
||||
cache_hit_ratio=(cache_hits / all_requests) * 100 if all_requests > 0 else 0.0,
|
||||
)
|
||||
|
||||
|
||||
async def get_cache_activity(
|
||||
prisma_client: "PrismaClient",
|
||||
start_date: datetime,
|
||||
end_date: datetime,
|
||||
key_aliases: Sequence[str],
|
||||
models: Sequence[str],
|
||||
) -> CacheActivityResponse:
|
||||
group_rows, key_alias_rows, model_rows = await asyncio.gather(
|
||||
prisma_client.db.query_raw(
|
||||
GROUPS_SQL, start_date, end_date, json.dumps(list(key_aliases)), json.dumps(list(models))
|
||||
),
|
||||
prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date),
|
||||
prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date),
|
||||
)
|
||||
groups = _groups_adapter.validate_python(group_rows or [])
|
||||
return CacheActivityResponse(
|
||||
groups=groups,
|
||||
totals=compute_totals(groups),
|
||||
filter_options=CacheActivityFilterOptions(
|
||||
key_aliases=[row.key_alias for row in _key_alias_rows_adapter.validate_python(key_alias_rows or [])],
|
||||
models=[row.model for row in _model_rows_adapter.validate_python(model_rows or [])],
|
||||
),
|
||||
)
|
||||
|
|
@ -1,53 +1,133 @@
|
|||
"""
|
||||
The cache dashboard buckets spend-log rows whose call_type is empty as "Unknown";
|
||||
those rows are failed requests. The activity SQL must therefore report a
|
||||
failed_rows count per group so the UI can chart failures as their own series.
|
||||
The cache dashboard chart is fed by /global/activity/cache_hits. Aggregation
|
||||
lives server-side: the SQL groups per call_type (splitting cache hits vs
|
||||
successful vs failed requests; failed spend logs have call_type '' today and
|
||||
must surface as 'Unknown'), and the endpoint returns chart-ready groups,
|
||||
totals for the stat cards, and the filter options for the UI dropdowns.
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.analytics_endpoints.analytics_endpoints import get_global_activity
|
||||
from litellm.proxy.analytics_endpoints.cache_activity import (
|
||||
GROUPS_SQL,
|
||||
CacheActivityGroup,
|
||||
compute_totals,
|
||||
)
|
||||
|
||||
GROUP_ROWS = [
|
||||
{
|
||||
"call_type": "acompletion",
|
||||
"api_requests": 1000,
|
||||
"cache_hits": 300,
|
||||
"failed_requests": 200,
|
||||
"cached_completion_tokens": 12000,
|
||||
"generated_completion_tokens": 48000,
|
||||
},
|
||||
{
|
||||
"call_type": "Unknown",
|
||||
"api_requests": 0,
|
||||
"cache_hits": 0,
|
||||
"failed_requests": 110,
|
||||
"cached_completion_tokens": 0,
|
||||
"generated_completion_tokens": 0,
|
||||
},
|
||||
]
|
||||
KEY_ALIAS_ROWS = [{"key_alias": "Unnamed Key"}, {"key_alias": "my-key"}]
|
||||
MODEL_ROWS = [{"model": "gpt-5.1"}]
|
||||
|
||||
|
||||
def build_prisma(query_raw: AsyncMock) -> MagicMock:
|
||||
prisma = MagicMock()
|
||||
prisma.db.query_raw = query_raw
|
||||
return prisma
|
||||
|
||||
|
||||
def dispatching_query_raw() -> AsyncMock:
|
||||
async def dispatch(sql: str, *params: object) -> list[dict[str, object]]:
|
||||
if "GROUP BY" in sql:
|
||||
return GROUP_ROWS
|
||||
if "key_alias" in sql:
|
||||
return KEY_ALIAS_ROWS
|
||||
return MODEL_ROWS
|
||||
|
||||
return AsyncMock(side_effect=dispatch)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_prisma(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
|
||||
prisma = MagicMock()
|
||||
prisma.db.query_raw = AsyncMock(return_value=[])
|
||||
prisma = build_prisma(dispatching_query_raw())
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma)
|
||||
return prisma
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_hits_query_counts_failed_rows_per_group(mock_prisma: MagicMock):
|
||||
rows = [
|
||||
{
|
||||
"api_key": "my-key-alias",
|
||||
"call_type": "acompletion",
|
||||
"model": "gpt-5.1",
|
||||
"total_rows": 10,
|
||||
"cache_hit_true_rows": 3,
|
||||
"failed_rows": 2,
|
||||
"cached_completion_tokens": 100,
|
||||
"generated_completion_tokens": 900,
|
||||
}
|
||||
]
|
||||
mock_prisma.db.query_raw = AsyncMock(return_value=rows)
|
||||
async def test_returns_groups_totals_and_filter_options(mock_prisma: MagicMock):
|
||||
response = await get_global_activity(
|
||||
start_date="2026-07-01", end_date="2026-07-27", key_aliases=[], models=[]
|
||||
)
|
||||
|
||||
response = await get_global_activity(start_date="2026-07-01", end_date="2026-07-27")
|
||||
|
||||
assert response == rows
|
||||
sql_query = mock_prisma.db.query_raw.call_args.args[0]
|
||||
assert 'SUM(CASE WHEN sl."status" = \'failure\' THEN 1 ELSE 0 END) AS failed_rows' in sql_query
|
||||
assert 'SUM(CASE WHEN sl."cache_hit" = \'True\' THEN 1 ELSE 0 END) AS cache_hit_true_rows' in sql_query
|
||||
assert [group.call_type for group in response.groups] == ["acompletion", "Unknown"]
|
||||
assert response.groups[0].api_requests == 1000
|
||||
assert response.groups[0].failed_requests == 200
|
||||
assert response.totals.api_requests == 1000
|
||||
assert response.totals.cache_hits == 300
|
||||
assert response.totals.failed_requests == 310
|
||||
assert response.totals.cached_completion_tokens == 12000
|
||||
assert response.totals.cache_hit_ratio == pytest.approx((300 / 1610) * 100)
|
||||
assert response.filter_options.key_aliases == ["Unnamed Key", "my-key"]
|
||||
assert response.filter_options.models == ["gpt-5.1"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_hits_requires_date_range(mock_prisma: MagicMock):
|
||||
async def test_filters_are_passed_to_sql_as_json_arrays(mock_prisma: MagicMock):
|
||||
await get_global_activity(
|
||||
start_date="2026-07-01",
|
||||
end_date="2026-07-27",
|
||||
key_aliases=["my-key"],
|
||||
models=["gpt-5.1", "claude-opus-4-8"],
|
||||
)
|
||||
|
||||
groups_call = next(
|
||||
call for call in mock_prisma.db.query_raw.call_args_list if "GROUP BY" in call.args[0]
|
||||
)
|
||||
assert groups_call.args[3] == json.dumps(["my-key"])
|
||||
assert groups_call.args[4] == json.dumps(["gpt-5.1", "claude-opus-4-8"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_malformed_dates_with_400(mock_prisma: MagicMock):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await get_global_activity(start_date=None, end_date=None)
|
||||
await get_global_activity(start_date="07/01/2026", end_date="2026-07-27", key_aliases=[], models=[])
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
mock_prisma.db.query_raw.assert_not_called()
|
||||
|
||||
|
||||
def test_totals_ratio_is_zero_without_requests():
|
||||
totals = compute_totals([])
|
||||
|
||||
assert totals.cache_hit_ratio == 0.0
|
||||
assert totals.api_requests == 0
|
||||
|
||||
|
||||
def test_totals_denominator_includes_failed_requests():
|
||||
group = CacheActivityGroup(
|
||||
call_type="acompletion",
|
||||
api_requests=60,
|
||||
cache_hits=20,
|
||||
failed_requests=20,
|
||||
cached_completion_tokens=0,
|
||||
generated_completion_tokens=0,
|
||||
)
|
||||
|
||||
assert compute_totals([group]).cache_hit_ratio == pytest.approx(20.0)
|
||||
|
||||
|
||||
def test_groups_sql_splits_failures_and_labels_empty_call_type_unknown():
|
||||
assert "SUM(CASE WHEN sl.\"status\" = 'failure' THEN 1 ELSE 0 END)" in GROUPS_SQL
|
||||
assert "CASE WHEN sl.\"call_type\" = '' THEN 'Unknown' ELSE sl.\"call_type\" END" in GROUPS_SQL
|
||||
|
|
|
|||
|
|
@ -4,38 +4,50 @@ import { screen, waitFor, within } from "@testing-library/react";
|
|||
import { renderWithProviders } from "../../../../../tests/test-utils";
|
||||
import CacheDashboard from "./cache_dashboard";
|
||||
|
||||
const { adminGlobalCacheActivity, cachingHealthCheckCall } = vi.hoisted(() => ({
|
||||
adminGlobalCacheActivity: vi.fn(),
|
||||
const { useCacheActivity, cachingHealthCheckCall } = vi.hoisted(() => ({
|
||||
useCacheActivity: vi.fn(),
|
||||
cachingHealthCheckCall: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
adminGlobalCacheActivity,
|
||||
cachingHealthCheckCall,
|
||||
}));
|
||||
|
||||
const cacheActivity = [
|
||||
{
|
||||
api_key: "sk-1",
|
||||
model: "gpt-5.1",
|
||||
call_type: "acompletion",
|
||||
total_rows: 1500,
|
||||
cache_hit_true_rows: 300,
|
||||
failed_rows: 200,
|
||||
cached_completion_tokens: 12000,
|
||||
generated_completion_tokens: 48000,
|
||||
vi.mock("@/app/(dashboard)/hooks/caching/useCacheActivity", () => ({
|
||||
useCacheActivity,
|
||||
}));
|
||||
|
||||
const cacheActivity = {
|
||||
groups: [
|
||||
{
|
||||
call_type: "acompletion",
|
||||
api_requests: 1000,
|
||||
cache_hits: 300,
|
||||
failed_requests: 200,
|
||||
cached_completion_tokens: 12000,
|
||||
generated_completion_tokens: 48000,
|
||||
},
|
||||
{
|
||||
call_type: "aembedding",
|
||||
api_requests: 550,
|
||||
cache_hits: 100,
|
||||
failed_requests: 50,
|
||||
cached_completion_tokens: 2000,
|
||||
generated_completion_tokens: 9000,
|
||||
},
|
||||
],
|
||||
totals: {
|
||||
api_requests: 1550,
|
||||
cache_hits: 400,
|
||||
failed_requests: 250,
|
||||
cached_completion_tokens: 14000,
|
||||
cache_hit_ratio: (400 / 2200) * 100,
|
||||
},
|
||||
{
|
||||
api_key: "sk-2",
|
||||
model: "text-embedding-3-large",
|
||||
call_type: "aembedding",
|
||||
total_rows: 700,
|
||||
cache_hit_true_rows: 100,
|
||||
failed_rows: 50,
|
||||
cached_completion_tokens: 2000,
|
||||
generated_completion_tokens: 9000,
|
||||
filter_options: {
|
||||
key_aliases: ["my-key", "Unnamed Key"],
|
||||
models: ["gpt-5.1", "text-embedding-3-large"],
|
||||
},
|
||||
];
|
||||
};
|
||||
|
||||
const renderDashboard = () =>
|
||||
renderWithProviders(
|
||||
|
|
@ -77,7 +89,7 @@ const legendFillByCategory = (card: HTMLElement) =>
|
|||
describe("CacheDashboard cache analytics charts", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
adminGlobalCacheActivity.mockResolvedValue(cacheActivity);
|
||||
useCacheActivity.mockReturnValue({ data: cacheActivity, refetch: vi.fn() });
|
||||
});
|
||||
|
||||
it("renders both chart card titles", async () => {
|
||||
|
|
@ -156,12 +168,23 @@ describe("CacheDashboard cache analytics charts", () => {
|
|||
}
|
||||
});
|
||||
|
||||
it("keeps failed requests in the cache hit ratio denominator", async () => {
|
||||
it("renders the server-computed cache hit ratio", async () => {
|
||||
renderDashboard();
|
||||
|
||||
expect(await screen.findByText("18.18%")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("passes the date range and selected filters to the activity query", () => {
|
||||
renderDashboard();
|
||||
|
||||
expect(useCacheActivity).toHaveBeenCalledWith({
|
||||
startDate: expect.stringMatching(/^\d{4}-\d{2}-\d{2}$/),
|
||||
endDate: expect.stringMatching(/^\d{4}-\d{2}-\d{2}$/),
|
||||
keyAliases: [],
|
||||
models: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("formats y-axis ticks with compact notation", async () => {
|
||||
renderDashboard();
|
||||
const { requestsCard, tokensCard } = await findChartCards();
|
||||
|
|
|
|||
|
|
@ -19,14 +19,29 @@ import {
|
|||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
|
||||
import { RefreshCw } from "lucide-react";
|
||||
import { adminGlobalCacheActivity, cachingHealthCheckCall } from "@/components/networking";
|
||||
import { cachingHealthCheckCall } from "@/components/networking";
|
||||
import { useCacheActivity, type CacheActivityGroup } from "@/app/(dashboard)/hooks/caching/useCacheActivity";
|
||||
|
||||
// Import the new component
|
||||
import { CacheHealthTab } from "./cache_health";
|
||||
import { REQUEST_SERIES, summarizeCacheActivity, type CacheActivityRow, type CacheChartDatum } from "./cache_data";
|
||||
import CacheSettings from "./cache_settings";
|
||||
import CoordinationRedisSettings from "./coordination_redis_settings";
|
||||
|
||||
const REQUEST_SERIES = {
|
||||
apiRequests: "LLM API requests",
|
||||
cacheHits: "Cache hit",
|
||||
failed: "Failed requests",
|
||||
} as const;
|
||||
|
||||
const toChartDatum = (group: CacheActivityGroup) => ({
|
||||
name: group.call_type,
|
||||
[REQUEST_SERIES.apiRequests]: group.api_requests,
|
||||
[REQUEST_SERIES.cacheHits]: group.cache_hits,
|
||||
[REQUEST_SERIES.failed]: group.failed_requests,
|
||||
"Cached Completion Tokens": group.cached_completion_tokens,
|
||||
"Generated Completion Tokens": group.generated_completion_tokens,
|
||||
});
|
||||
|
||||
const formatDateWithoutTZ = (date: Date | undefined) => {
|
||||
if (!date) return undefined;
|
||||
return date.toISOString().split("T")[0];
|
||||
|
|
@ -78,13 +93,8 @@ const deepParse = (input: any) => {
|
|||
};
|
||||
|
||||
const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole, userID, premiumUser }) => {
|
||||
const [filteredData, setFilteredData] = useState<CacheChartDatum[]>([]);
|
||||
const [selectedApiKeys, setSelectedApiKeys] = useState<string[]>([]);
|
||||
const [selectedModels, setSelectedModels] = useState<string[]>([]);
|
||||
const [data, setData] = useState<CacheActivityRow[]>([]);
|
||||
const [cachedResponses, setCachedResponses] = useState("0");
|
||||
const [cachedTokens, setCachedTokens] = useState("0");
|
||||
const [cacheHitRatio, setCacheHitRatio] = useState("0");
|
||||
|
||||
const [dateValue, setDateValue] = useState<DateRangePickerValue>({
|
||||
from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000),
|
||||
|
|
@ -94,70 +104,24 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
|
|||
const [lastRefreshed, setLastRefreshed] = useState("");
|
||||
const [healthCheckResponse, setHealthCheckResponse] = useState<any>("");
|
||||
|
||||
useEffect(() => {
|
||||
if (!accessToken || !dateValue) {
|
||||
return;
|
||||
}
|
||||
const fetchData = async () => {
|
||||
const response = await adminGlobalCacheActivity(
|
||||
accessToken,
|
||||
formatDateWithoutTZ(dateValue.from),
|
||||
formatDateWithoutTZ(dateValue.to),
|
||||
);
|
||||
setData(response);
|
||||
};
|
||||
fetchData();
|
||||
|
||||
const currentDate = new Date();
|
||||
setLastRefreshed(currentDate.toLocaleString());
|
||||
}, [accessToken]);
|
||||
|
||||
const uniqueApiKeys = Array.from(new Set(data.map((item) => item?.api_key ?? "")));
|
||||
const uniqueModels = Array.from(new Set(data.map((item) => item?.model ?? "")));
|
||||
const uniqueCallTypes = Array.from(new Set(data.map((item) => item?.call_type ?? "")));
|
||||
|
||||
const updateCachingData = async (startTime: Date | undefined, endTime: Date | undefined) => {
|
||||
if (!startTime || !endTime || !accessToken) {
|
||||
return;
|
||||
}
|
||||
|
||||
let new_cache_data = await adminGlobalCacheActivity(
|
||||
accessToken,
|
||||
formatDateWithoutTZ(startTime),
|
||||
formatDateWithoutTZ(endTime),
|
||||
);
|
||||
|
||||
setData(new_cache_data);
|
||||
};
|
||||
const { data: activity, refetch } = useCacheActivity({
|
||||
startDate: formatDateWithoutTZ(dateValue.from),
|
||||
endDate: formatDateWithoutTZ(dateValue.to),
|
||||
keyAliases: selectedApiKeys,
|
||||
models: selectedModels,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
let newData: CacheActivityRow[] = data;
|
||||
if (selectedApiKeys.length > 0) {
|
||||
newData = newData.filter((item) => selectedApiKeys.includes(item.api_key));
|
||||
}
|
||||
setLastRefreshed(new Date().toLocaleString());
|
||||
}, []);
|
||||
|
||||
if (selectedModels.length > 0) {
|
||||
newData = newData.filter((item) => selectedModels.includes(item.model));
|
||||
}
|
||||
|
||||
const summary = summarizeCacheActivity(newData);
|
||||
|
||||
setCachedResponses(valueFormatterNumbers(summary.cacheHits));
|
||||
setCachedTokens(valueFormatterNumbers(summary.cachedCompletionTokens));
|
||||
const allRequests = summary.cacheHits + summary.llmApiRequests + summary.failedRequests;
|
||||
if (allRequests > 0) {
|
||||
setCacheHitRatio(((summary.cacheHits / allRequests) * 100).toFixed(2));
|
||||
} else {
|
||||
setCacheHitRatio("0");
|
||||
}
|
||||
|
||||
setFilteredData(summary.chartData);
|
||||
}, [selectedApiKeys, selectedModels, dateValue, data]);
|
||||
const uniqueApiKeys = activity?.filter_options.key_aliases ?? [];
|
||||
const uniqueModels = activity?.filter_options.models ?? [];
|
||||
const chartData = (activity?.groups ?? []).map(toChartDatum);
|
||||
|
||||
const handleRefreshClick = () => {
|
||||
// Update the 'lastRefreshed' state to the current date and time
|
||||
const currentDate = new Date();
|
||||
setLastRefreshed(currentDate.toLocaleString());
|
||||
refetch();
|
||||
setLastRefreshed(new Date().toLocaleString());
|
||||
};
|
||||
|
||||
const runCachingHealthCheck = async () => {
|
||||
|
|
@ -188,10 +152,12 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
|
|||
}
|
||||
};
|
||||
|
||||
const totals = activity?.totals;
|
||||
const hasRequests = totals != null && totals.api_requests + totals.cache_hits + totals.failed_requests > 0;
|
||||
const statCards = [
|
||||
{ label: "Cache Hit Ratio", value: `${cacheHitRatio}%` },
|
||||
{ label: "Cache Hits", value: cachedResponses },
|
||||
{ label: "Cached Completion Tokens", value: cachedTokens },
|
||||
{ label: "Cache Hit Ratio", value: `${hasRequests ? totals.cache_hit_ratio.toFixed(2) : "0"}%` },
|
||||
{ label: "Cache Hits", value: valueFormatterNumbers(totals?.cache_hits ?? 0) },
|
||||
{ label: "Cached Completion Tokens", value: valueFormatterNumbers(totals?.cached_completion_tokens ?? 0) },
|
||||
];
|
||||
|
||||
return (
|
||||
|
|
@ -311,7 +277,6 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
|
|||
value={dateValue}
|
||||
onValueChange={(value) => {
|
||||
setDateValue(value);
|
||||
updateCachingData(value.from, value.to);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
|
@ -335,7 +300,7 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
|
|||
</CardHeader>
|
||||
<CardContent>
|
||||
<BarChart
|
||||
data={filteredData}
|
||||
data={chartData}
|
||||
stack={true}
|
||||
index="name"
|
||||
valueFormatter={valueFormatterNumbers}
|
||||
|
|
@ -354,7 +319,7 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
|
|||
</CardHeader>
|
||||
<CardContent>
|
||||
<BarChart
|
||||
data={filteredData}
|
||||
data={chartData}
|
||||
stack={true}
|
||||
index="name"
|
||||
valueFormatter={valueFormatterNumbers}
|
||||
|
|
|
|||
|
|
@ -1,84 +0,0 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { summarizeCacheActivity, UNKNOWN_CALL_TYPE, type CacheActivityRow } from "./cache_data";
|
||||
|
||||
const row = (overrides: Partial<CacheActivityRow>): CacheActivityRow => ({
|
||||
api_key: "sk-1",
|
||||
model: "gpt-5.1",
|
||||
call_type: "acompletion",
|
||||
total_rows: 0,
|
||||
cache_hit_true_rows: 0,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe("summarizeCacheActivity", () => {
|
||||
it("splits each call_type into api requests, cache hits, and failed requests", () => {
|
||||
const summary = summarizeCacheActivity([row({ total_rows: 1000, cache_hit_true_rows: 300, failed_rows: 100 })]);
|
||||
|
||||
expect(summary.chartData).toEqual([
|
||||
expect.objectContaining({
|
||||
name: "acompletion",
|
||||
"LLM API requests": 600,
|
||||
"Cache hit": 300,
|
||||
"Failed requests": 100,
|
||||
}),
|
||||
]);
|
||||
expect(summary.llmApiRequests).toBe(600);
|
||||
expect(summary.cacheHits).toBe(300);
|
||||
expect(summary.failedRequests).toBe(100);
|
||||
});
|
||||
|
||||
it("treats a missing failed_rows as zero so older proxy responses keep the old math", () => {
|
||||
const summary = summarizeCacheActivity([row({ total_rows: 50, cache_hit_true_rows: 20 })]);
|
||||
|
||||
expect(summary.chartData[0]["LLM API requests"]).toBe(30);
|
||||
expect(summary.chartData[0]["Failed requests"]).toBe(0);
|
||||
});
|
||||
|
||||
it("buckets rows with an empty call_type under Unknown", () => {
|
||||
const summary = summarizeCacheActivity([
|
||||
row({ call_type: "", total_rows: 8, failed_rows: 8 }),
|
||||
row({ call_type: "", api_key: "sk-2", total_rows: 3, failed_rows: 3 }),
|
||||
]);
|
||||
|
||||
expect(summary.chartData).toHaveLength(1);
|
||||
expect(summary.chartData[0]).toEqual(
|
||||
expect.objectContaining({ name: UNKNOWN_CALL_TYPE, "Failed requests": 11, "LLM API requests": 0 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("merges rows sharing a call_type across keys and models", () => {
|
||||
const summary = summarizeCacheActivity([
|
||||
row({ total_rows: 10, cache_hit_true_rows: 4, failed_rows: 1, cached_completion_tokens: 100 }),
|
||||
row({
|
||||
api_key: "sk-2",
|
||||
model: "claude-opus-4-8",
|
||||
total_rows: 20,
|
||||
cache_hit_true_rows: 6,
|
||||
failed_rows: 2,
|
||||
generated_completion_tokens: 500,
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(summary.chartData).toEqual([
|
||||
{
|
||||
name: "acompletion",
|
||||
"LLM API requests": 17,
|
||||
"Cache hit": 10,
|
||||
"Failed requests": 3,
|
||||
"Cached Completion Tokens": 100,
|
||||
"Generated Completion Tokens": 500,
|
||||
},
|
||||
]);
|
||||
expect(summary.cachedCompletionTokens).toBe(100);
|
||||
});
|
||||
|
||||
it("returns empty totals for no rows", () => {
|
||||
expect(summarizeCacheActivity([])).toEqual({
|
||||
chartData: [],
|
||||
cacheHits: 0,
|
||||
llmApiRequests: 0,
|
||||
failedRequests: 0,
|
||||
cachedCompletionTokens: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
export interface CacheActivityRow {
|
||||
api_key: string;
|
||||
model: string;
|
||||
call_type: string;
|
||||
total_rows: number;
|
||||
cache_hit_true_rows: number;
|
||||
failed_rows?: number;
|
||||
cached_completion_tokens?: number;
|
||||
generated_completion_tokens?: number;
|
||||
}
|
||||
|
||||
export const UNKNOWN_CALL_TYPE = "Unknown";
|
||||
|
||||
export const REQUEST_SERIES = {
|
||||
apiRequests: "LLM API requests",
|
||||
cacheHits: "Cache hit",
|
||||
failed: "Failed requests",
|
||||
} as const;
|
||||
|
||||
export type CacheChartDatum = {
|
||||
name: string;
|
||||
[REQUEST_SERIES.apiRequests]: number;
|
||||
[REQUEST_SERIES.cacheHits]: number;
|
||||
[REQUEST_SERIES.failed]: number;
|
||||
"Cached Completion Tokens": number;
|
||||
"Generated Completion Tokens": number;
|
||||
};
|
||||
|
||||
export interface CacheActivitySummary {
|
||||
chartData: CacheChartDatum[];
|
||||
cacheHits: number;
|
||||
llmApiRequests: number;
|
||||
failedRequests: number;
|
||||
cachedCompletionTokens: number;
|
||||
}
|
||||
|
||||
export function summarizeCacheActivity(rows: readonly CacheActivityRow[]): CacheActivitySummary {
|
||||
const groups = new Map<string, CacheChartDatum>();
|
||||
|
||||
for (const row of rows) {
|
||||
const name = row.call_type || UNKNOWN_CALL_TYPE;
|
||||
const hits = row.cache_hit_true_rows || 0;
|
||||
const failed = row.failed_rows || 0;
|
||||
const apiRequests = (row.total_rows || 0) - hits - failed;
|
||||
|
||||
const group = groups.get(name) ?? {
|
||||
name,
|
||||
[REQUEST_SERIES.apiRequests]: 0,
|
||||
[REQUEST_SERIES.cacheHits]: 0,
|
||||
[REQUEST_SERIES.failed]: 0,
|
||||
"Cached Completion Tokens": 0,
|
||||
"Generated Completion Tokens": 0,
|
||||
};
|
||||
|
||||
groups.set(name, {
|
||||
...group,
|
||||
[REQUEST_SERIES.apiRequests]: group[REQUEST_SERIES.apiRequests] + apiRequests,
|
||||
[REQUEST_SERIES.cacheHits]: group[REQUEST_SERIES.cacheHits] + hits,
|
||||
[REQUEST_SERIES.failed]: group[REQUEST_SERIES.failed] + failed,
|
||||
"Cached Completion Tokens": group["Cached Completion Tokens"] + (row.cached_completion_tokens || 0),
|
||||
"Generated Completion Tokens": group["Generated Completion Tokens"] + (row.generated_completion_tokens || 0),
|
||||
});
|
||||
}
|
||||
|
||||
const chartData = Array.from(groups.values());
|
||||
return {
|
||||
chartData,
|
||||
cacheHits: chartData.reduce((sum, g) => sum + g[REQUEST_SERIES.cacheHits], 0),
|
||||
llmApiRequests: chartData.reduce((sum, g) => sum + g[REQUEST_SERIES.apiRequests], 0),
|
||||
failedRequests: chartData.reduce((sum, g) => sum + g[REQUEST_SERIES.failed], 0),
|
||||
cachedCompletionTokens: chartData.reduce((sum, g) => sum + g["Cached Completion Tokens"], 0),
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
import { renderHook } from "@testing-library/react";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useCacheActivity, type CacheActivityParams } from "./useCacheActivity";
|
||||
|
||||
const useQueryMock = vi.fn();
|
||||
vi.mock("@/lib/http/api", () => ({
|
||||
$api: { useQuery: (...args: unknown[]) => useQueryMock(...args) },
|
||||
}));
|
||||
|
||||
const mockUseAuthorized = vi.fn();
|
||||
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
|
||||
default: () => mockUseAuthorized(),
|
||||
}));
|
||||
|
||||
const params: CacheActivityParams = {
|
||||
startDate: "2026-07-20",
|
||||
endDate: "2026-07-27",
|
||||
keyAliases: ["my-key"],
|
||||
models: ["gpt-5.1"],
|
||||
};
|
||||
|
||||
const lastCallOptions = (): { enabled: boolean } => {
|
||||
const calls = useQueryMock.mock.calls;
|
||||
return calls[calls.length - 1][3] as { enabled: boolean };
|
||||
};
|
||||
|
||||
describe("useCacheActivity", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
useQueryMock.mockReturnValue({ data: undefined });
|
||||
mockUseAuthorized.mockReturnValue({ accessToken: "test-access-token" });
|
||||
});
|
||||
|
||||
it("queries GET /global/activity/cache_hits with dates and filters as query params", () => {
|
||||
renderHook(() => useCacheActivity(params));
|
||||
|
||||
expect(useQueryMock).toHaveBeenCalledWith(
|
||||
"get",
|
||||
"/global/activity/cache_hits",
|
||||
{
|
||||
params: {
|
||||
query: {
|
||||
start_date: "2026-07-20",
|
||||
end_date: "2026-07-27",
|
||||
key_aliases: ["my-key"],
|
||||
models: ["gpt-5.1"],
|
||||
},
|
||||
},
|
||||
},
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it("enables the query when authorized and both dates are set", () => {
|
||||
renderHook(() => useCacheActivity(params));
|
||||
|
||||
expect(lastCallOptions().enabled).toBe(true);
|
||||
});
|
||||
|
||||
it("disables the query without an access token", () => {
|
||||
mockUseAuthorized.mockReturnValue({ accessToken: null });
|
||||
renderHook(() => useCacheActivity(params));
|
||||
|
||||
expect(lastCallOptions().enabled).toBe(false);
|
||||
});
|
||||
|
||||
it("disables the query while the date range is incomplete", () => {
|
||||
renderHook(() => useCacheActivity({ ...params, endDate: undefined }));
|
||||
|
||||
expect(lastCallOptions().enabled).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
import { $api } from "@/lib/http/api";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import type { components } from "@/lib/http/schema";
|
||||
|
||||
export type CacheActivityResponse = components["schemas"]["CacheActivityResponse"];
|
||||
export type CacheActivityGroup = components["schemas"]["CacheActivityGroup"];
|
||||
|
||||
export interface CacheActivityParams {
|
||||
startDate: string | undefined;
|
||||
endDate: string | undefined;
|
||||
keyAliases: string[];
|
||||
models: string[];
|
||||
}
|
||||
|
||||
export const useCacheActivity = ({ startDate, endDate, keyAliases, models }: CacheActivityParams) => {
|
||||
const { accessToken } = useAuthorized();
|
||||
return $api.useQuery(
|
||||
"get",
|
||||
"/global/activity/cache_hits",
|
||||
{
|
||||
params: {
|
||||
query: {
|
||||
start_date: startDate ?? "",
|
||||
end_date: endDate ?? "",
|
||||
key_aliases: keyAliases,
|
||||
models,
|
||||
},
|
||||
},
|
||||
},
|
||||
{ enabled: Boolean(accessToken && startDate && endDate) },
|
||||
);
|
||||
};
|
||||
|
|
@ -2094,42 +2094,6 @@ export const adminGlobalActivity = async (
|
|||
}
|
||||
};
|
||||
|
||||
export const adminGlobalCacheActivity = async (
|
||||
accessToken: string,
|
||||
startTime: string | undefined,
|
||||
endTime: string | undefined,
|
||||
) => {
|
||||
try {
|
||||
let url = proxyBaseUrl ? `${proxyBaseUrl}/global/activity/cache_hits` : `/global/activity/cache_hits`;
|
||||
|
||||
if (startTime && endTime) {
|
||||
url += `?start_date=${startTime}&end_date=${endTime}`;
|
||||
}
|
||||
|
||||
const requestOptions = {
|
||||
method: "GET",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
},
|
||||
};
|
||||
|
||||
const response = await fetch(url, requestOptions);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
const errorMessage = deriveErrorMessage(errorData);
|
||||
handleError(errorMessage);
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch spend data:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const adminGlobalActivityPerModel = async (
|
||||
accessToken: string,
|
||||
startTime: string | undefined,
|
||||
|
|
|
|||
76
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
76
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -4356,25 +4356,9 @@ export interface paths {
|
|||
};
|
||||
/**
|
||||
* Get Global Activity
|
||||
* @description Get number of cache hits, vs misses
|
||||
*
|
||||
* {
|
||||
* "daily_data": [
|
||||
* const chartdata = [
|
||||
* {
|
||||
* date: 'Jan 22',
|
||||
* cache_hits: 10,
|
||||
* llm_api_calls: 2000
|
||||
* },
|
||||
* {
|
||||
* date: 'Jan 23',
|
||||
* cache_hits: 10,
|
||||
* llm_api_calls: 12
|
||||
* },
|
||||
* ],
|
||||
* "sum_cache_hits": 20,
|
||||
* "sum_llm_api_calls": 2012
|
||||
* }
|
||||
* @description Cache activity for the Admin UI cache dashboard, aggregated per call_type:
|
||||
* cache hits vs successful LLM API requests vs failed requests, plus totals
|
||||
* for the stat cards and the available key-alias/model filter options.
|
||||
*/
|
||||
get: operations["get_global_activity_global_activity_cache_hits_get"];
|
||||
put?: never;
|
||||
|
|
@ -21741,6 +21725,48 @@ export interface components {
|
|||
/** Total Requested */
|
||||
total_requested: number;
|
||||
};
|
||||
/** CacheActivityFilterOptions */
|
||||
CacheActivityFilterOptions: {
|
||||
/** Key Aliases */
|
||||
key_aliases: string[];
|
||||
/** Models */
|
||||
models: string[];
|
||||
};
|
||||
/** CacheActivityGroup */
|
||||
CacheActivityGroup: {
|
||||
/** Api Requests */
|
||||
api_requests: number;
|
||||
/** Cache Hits */
|
||||
cache_hits: number;
|
||||
/** Cached Completion Tokens */
|
||||
cached_completion_tokens: number;
|
||||
/** Call Type */
|
||||
call_type: string;
|
||||
/** Failed Requests */
|
||||
failed_requests: number;
|
||||
/** Generated Completion Tokens */
|
||||
generated_completion_tokens: number;
|
||||
};
|
||||
/** CacheActivityResponse */
|
||||
CacheActivityResponse: {
|
||||
filter_options: components["schemas"]["CacheActivityFilterOptions"];
|
||||
/** Groups */
|
||||
groups: components["schemas"]["CacheActivityGroup"][];
|
||||
totals: components["schemas"]["CacheActivityTotals"];
|
||||
};
|
||||
/** CacheActivityTotals */
|
||||
CacheActivityTotals: {
|
||||
/** Api Requests */
|
||||
api_requests: number;
|
||||
/** Cache Hit Ratio */
|
||||
cache_hit_ratio: number;
|
||||
/** Cache Hits */
|
||||
cache_hits: number;
|
||||
/** Cached Completion Tokens */
|
||||
cached_completion_tokens: number;
|
||||
/** Failed Requests */
|
||||
failed_requests: number;
|
||||
};
|
||||
/** CachePingResponse */
|
||||
CachePingResponse: {
|
||||
/** Cache Type */
|
||||
|
|
@ -40618,11 +40644,15 @@ export interface operations {
|
|||
};
|
||||
get_global_activity_global_activity_cache_hits_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
query: {
|
||||
/** @description Time from which to start viewing spend */
|
||||
start_date?: string | null;
|
||||
start_date: string;
|
||||
/** @description Time till which to view spend */
|
||||
end_date?: string | null;
|
||||
end_date: string;
|
||||
/** @description Only include spend from these key aliases */
|
||||
key_aliases?: string[] | null;
|
||||
/** @description Only include spend for these models */
|
||||
models?: string[] | null;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
|
|
@ -40636,7 +40666,7 @@ export interface operations {
|
|||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["LiteLLM_SpendLogs"][];
|
||||
"application/json": components["schemas"]["CacheActivityResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue