mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
feat(cost-optimization): show recent optimized requests on dashboard
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Krrish Dholakia
parent
ac1b4b35ce
commit
20cf663a1c
10 changed files with 680 additions and 16 deletions
|
|
@ -29,8 +29,8 @@ import litellm
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching import RedisCache
|
||||
from litellm.constants import (
|
||||
DB_SPEND_UPDATE_JOB_NAME,
|
||||
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME,
|
||||
DB_SPEND_UPDATE_JOB_NAME,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -62,6 +62,7 @@ from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
|
|||
from litellm.proxy.spend_tracking.compression_savings import (
|
||||
extract_compression_saved_tokens,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.optimization import extract_cache_read_tokens
|
||||
from litellm.proxy.spend_tracking.savings import compute_savings_spend
|
||||
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
|
||||
|
||||
|
|
@ -72,18 +73,6 @@ else:
|
|||
ProxyLogging = Any
|
||||
|
||||
|
||||
def _extract_cache_read_tokens(usage_obj: dict) -> int:
|
||||
"""
|
||||
Anthropic: top-level cache_read_input_tokens field.
|
||||
OpenAI-compatible (moonshotai, openai, deepseek, etc.): prompt_tokens_details.cached_tokens.
|
||||
"""
|
||||
explicit = usage_obj.get("cache_read_input_tokens", 0) or 0
|
||||
if explicit:
|
||||
return int(explicit)
|
||||
details = usage_obj.get("prompt_tokens_details") or {}
|
||||
return int(details.get("cached_tokens", 0) or 0)
|
||||
|
||||
|
||||
def _extract_cache_creation_tokens(usage_obj: dict) -> int:
|
||||
"""
|
||||
Anthropic: top-level cache_creation_input_tokens field.
|
||||
|
|
@ -1854,7 +1843,7 @@ class DBSpendUpdateWriter:
|
|||
if call_type:
|
||||
endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None)
|
||||
|
||||
cache_read_input_tokens = _extract_cache_read_tokens(usage_obj)
|
||||
cache_read_input_tokens = extract_cache_read_tokens({"usage_object": usage_obj})
|
||||
compression_saved_tokens = extract_compression_saved_tokens(_metadata)
|
||||
savings_spend = compute_savings_spend(
|
||||
model=payload.get("model", None),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,180 @@
|
|||
import json
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Awaitable, Callable, Literal, cast
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
|
||||
from litellm.proxy.spend_tracking.optimization import extract_cache_read_tokens
|
||||
from litellm.proxy.spend_tracking.savings import compute_savings_spend
|
||||
from litellm.types.proxy.management_endpoints.cost_optimization import (
|
||||
OptimizedRequestLog,
|
||||
OptimizedRequestLogsResponse,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _metadata_dict(value: object) -> Mapping[str, object]:
|
||||
if isinstance(value, dict):
|
||||
return cast(Mapping[str, object], value) # cast-ok: JSON metadata is validated as a mapping above
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
return (
|
||||
cast(Mapping[str, object], parsed) # cast-ok: parsed JSON is checked as a dict
|
||||
if isinstance(parsed, dict)
|
||||
else {}
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
return {}
|
||||
|
||||
|
||||
def _nonnegative_int(value: object) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return 0
|
||||
return max(int(value), 0)
|
||||
|
||||
|
||||
def build_optimized_request_log(row: Mapping[str, object]) -> OptimizedRequestLog | None:
|
||||
metadata = _metadata_dict(row.get("metadata"))
|
||||
cache_read_tokens = extract_cache_read_tokens(metadata)
|
||||
tokens_saved = extract_compression_saved_tokens(metadata)
|
||||
if cache_read_tokens <= 0 and tokens_saved <= 0:
|
||||
return None
|
||||
|
||||
model = str(row.get("model", "") or "")
|
||||
provider_value = row.get("custom_llm_provider")
|
||||
provider = str(provider_value) if provider_value is not None else None
|
||||
savings_spend = compute_savings_spend(
|
||||
model=model,
|
||||
custom_llm_provider=provider,
|
||||
compression_saved_tokens=tokens_saved,
|
||||
cache_read_input_tokens=cache_read_tokens,
|
||||
)
|
||||
compression_savings = float(savings_spend.compression)
|
||||
caching_savings = float(savings_spend.prompt_caching)
|
||||
if tokens_saved > 0 and cache_read_tokens > 0:
|
||||
optimization_type: Literal["compression", "caching", "both"] = "both"
|
||||
elif tokens_saved > 0:
|
||||
optimization_type = "compression"
|
||||
else:
|
||||
optimization_type = "caching"
|
||||
|
||||
spend_value = row.get("spend")
|
||||
spend = float(spend_value) if isinstance(spend_value, (int, float)) else 0.0
|
||||
timestamp = row.get("startTime")
|
||||
if isinstance(timestamp, datetime):
|
||||
timestamp = timestamp.isoformat()
|
||||
else:
|
||||
timestamp = str(timestamp or "")
|
||||
|
||||
return OptimizedRequestLog(
|
||||
request_id=str(row.get("request_id", "")),
|
||||
timestamp=timestamp,
|
||||
model=str(model or ""),
|
||||
total_tokens=_nonnegative_int(row.get("total_tokens")),
|
||||
optimization_type=optimization_type,
|
||||
spend=spend,
|
||||
savings=compression_savings + caching_savings,
|
||||
original_cost=spend + compression_savings + caching_savings,
|
||||
compression_savings_spend=compression_savings,
|
||||
prompt_caching_savings_spend=caching_savings,
|
||||
tokens_saved=tokens_saved,
|
||||
cache_read_tokens=cache_read_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _cache_read_filter_sql() -> str:
|
||||
candidates = (
|
||||
"(metadata->'usage_object'->>'cache_read_input_tokens')",
|
||||
"(metadata->'usage_object'->'prompt_tokens_details'->>'cached_tokens')",
|
||||
"(metadata->'additional_usage_values'->>'cache_read_input_tokens')",
|
||||
"(metadata->'additional_usage_values'->'prompt_tokens_details'->>'cached_tokens')",
|
||||
)
|
||||
return " OR ".join(f"({candidate}) ~ '^[0-9]+$' AND ({candidate})::bigint > 0" for candidate in candidates)
|
||||
|
||||
|
||||
def _optimized_where_sql() -> str:
|
||||
return f"""(
|
||||
(metadata->'compression_savings' IS NOT NULL
|
||||
AND metadata->'compression_savings' <> 'null'::jsonb)
|
||||
OR {_cache_read_filter_sql()}
|
||||
)"""
|
||||
|
||||
|
||||
def _parse_date(value: str | None, *, end: bool = False) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
parsed = datetime.strptime(value, "%Y-%m-%d %H:%M:%S" if " " in value else "%Y-%m-%d")
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
if end and " " not in value:
|
||||
parsed += timedelta(days=1)
|
||||
return parsed
|
||||
|
||||
|
||||
@router.get(
|
||||
"/cost_optimization/usage/logs",
|
||||
tags=["Cost Optimization"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=OptimizedRequestLogsResponse,
|
||||
)
|
||||
async def cost_optimization_usage_logs(
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=50, ge=1, le=100),
|
||||
start_date: str | None = Query(default=None),
|
||||
end_date: str | None = Query(default=None),
|
||||
) -> OptimizedRequestLogsResponse:
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if prisma_client is None:
|
||||
return OptimizedRequestLogsResponse(logs=[], total=0, page=page, page_size=page_size, total_pages=0)
|
||||
|
||||
conditions = [_optimized_where_sql()]
|
||||
params: list[object] = []
|
||||
if start_date:
|
||||
conditions.append('"startTime" >= $%d' % (len(params) + 1))
|
||||
params.append(_parse_date(start_date))
|
||||
if end_date:
|
||||
conditions.append('"startTime" < $%d' % (len(params) + 1))
|
||||
params.append(_parse_date(end_date, end=True))
|
||||
where_sql = " AND ".join(conditions)
|
||||
|
||||
query_raw: Callable[..., Awaitable[object]] = cast( # cast-ok: Prisma exposes query_raw dynamically
|
||||
Callable[..., Awaitable[object]], prisma_client.db.query_raw
|
||||
)
|
||||
total_rows = cast( # cast-ok: Prisma raw query returns rows with the selected columns
|
||||
list[Mapping[str, object]],
|
||||
await query_raw(
|
||||
f'SELECT COUNT(*)::bigint AS total FROM "LiteLLM_SpendLogs" WHERE {where_sql}',
|
||||
*params,
|
||||
),
|
||||
)
|
||||
total_value = total_rows[0].get("total") if total_rows else 0
|
||||
total = int(total_value) if isinstance(total_value, (int, float)) else 0
|
||||
offset_placeholder = len(params) + 1
|
||||
limit_placeholder = len(params) + 2
|
||||
rows = cast( # cast-ok: Prisma raw query returns rows with the selected columns
|
||||
list[Mapping[str, object]],
|
||||
await query_raw(
|
||||
f"""SELECT request_id, "startTime", model, custom_llm_provider, total_tokens, spend, metadata
|
||||
FROM "LiteLLM_SpendLogs"
|
||||
WHERE {where_sql}
|
||||
ORDER BY "startTime" DESC
|
||||
LIMIT ${limit_placeholder} OFFSET ${offset_placeholder}""",
|
||||
*params,
|
||||
page_size,
|
||||
(page - 1) * page_size,
|
||||
),
|
||||
)
|
||||
logs = [entry for row in rows if (entry := build_optimized_request_log(row)) is not None]
|
||||
return OptimizedRequestLogsResponse(
|
||||
logs=logs,
|
||||
total=total,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
total_pages=(total + page_size - 1) // page_size,
|
||||
)
|
||||
|
|
@ -506,6 +506,9 @@ from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_st
|
|||
from litellm.proxy.spend_tracking.spend_management_endpoints import (
|
||||
router as spend_management_router,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.cost_optimization_endpoints import (
|
||||
router as cost_optimization_router,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
from litellm.proxy.types_utils.utils import get_instance_fn
|
||||
from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import (
|
||||
|
|
@ -16191,6 +16194,7 @@ app.include_router(ui_sso_router)
|
|||
app.include_router(organization_router)
|
||||
app.include_router(customer_router)
|
||||
app.include_router(spend_management_router)
|
||||
app.include_router(cost_optimization_router)
|
||||
app.include_router(caching_router)
|
||||
app.include_router(analytics_router)
|
||||
app.include_router(callback_management_endpoints_router)
|
||||
|
|
|
|||
29
litellm/proxy/spend_tracking/optimization.py
Normal file
29
litellm/proxy/spend_tracking/optimization.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import cast
|
||||
|
||||
|
||||
def _positive_int(value: object) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return 0
|
||||
return max(int(value), 0)
|
||||
|
||||
|
||||
def _extract_cache_read_tokens_from_usage(usage_obj: object) -> int:
|
||||
if not isinstance(usage_obj, Mapping):
|
||||
return 0
|
||||
typed_usage = cast(Mapping[str, object], usage_obj) # cast-ok: usage object is checked as a mapping above
|
||||
explicit = _positive_int(typed_usage.get("cache_read_input_tokens"))
|
||||
if explicit:
|
||||
return explicit
|
||||
details = typed_usage.get("prompt_tokens_details")
|
||||
if isinstance(details, Mapping):
|
||||
typed_details = cast(Mapping[str, object], details) # cast-ok: details is checked as a mapping above
|
||||
return _positive_int(typed_details.get("cached_tokens"))
|
||||
return 0
|
||||
|
||||
|
||||
def extract_cache_read_tokens(metadata: Mapping[str, object]) -> int:
|
||||
return max(
|
||||
_extract_cache_read_tokens_from_usage(metadata.get("usage_object")),
|
||||
_extract_cache_read_tokens_from_usage(metadata.get("additional_usage_values")),
|
||||
)
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
from typing import List, Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class OptimizedRequestLog(BaseModel):
|
||||
request_id: str
|
||||
timestamp: str
|
||||
model: str
|
||||
total_tokens: int
|
||||
optimization_type: Literal["compression", "caching", "both"]
|
||||
spend: float
|
||||
savings: float
|
||||
original_cost: float
|
||||
compression_savings_spend: float
|
||||
prompt_caching_savings_spend: float
|
||||
tokens_saved: int
|
||||
cache_read_tokens: int
|
||||
|
||||
|
||||
class OptimizedRequestLogsResponse(BaseModel):
|
||||
logs: List[OptimizedRequestLog]
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
total_pages: int
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.management_endpoints.cost_optimization_endpoints import (
|
||||
build_optimized_request_log,
|
||||
cost_optimization_usage_logs,
|
||||
)
|
||||
|
||||
|
||||
def _row(metadata: Mapping[str, object], **overrides: object) -> dict[str, object]:
|
||||
return {
|
||||
"request_id": "req-1",
|
||||
"startTime": datetime(2026, 7, 13, 12, 0, tzinfo=timezone.utc),
|
||||
"model": "test-model",
|
||||
"custom_llm_provider": "openai",
|
||||
"total_tokens": 150,
|
||||
"spend": 0.02,
|
||||
"metadata": metadata,
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
def test_build_optimized_request_log_computes_both_savings():
|
||||
row = _row(
|
||||
{
|
||||
"compression_savings": {"tokens_saved": 100},
|
||||
"usage_object": {"cache_read_input_tokens": 50},
|
||||
}
|
||||
)
|
||||
with patch(
|
||||
"litellm.proxy.spend_tracking.savings._input_and_cache_read_cost",
|
||||
return_value=(0.001, 0.0002),
|
||||
):
|
||||
result = build_optimized_request_log(row)
|
||||
|
||||
assert result is not None
|
||||
assert result.optimization_type == "both"
|
||||
assert result.tokens_saved == 100
|
||||
assert result.cache_read_tokens == 50
|
||||
assert result.compression_savings_spend == 0.1
|
||||
assert result.prompt_caching_savings_spend == 0.04
|
||||
assert result.savings == 0.14
|
||||
assert result.original_cost == 0.16
|
||||
|
||||
|
||||
def test_build_optimized_request_log_ignores_unoptimized_row():
|
||||
assert build_optimized_request_log(_row({"usage_object": {"prompt_tokens": 10}})) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cost_optimization_usage_logs_returns_paginated_entries():
|
||||
db = MagicMock()
|
||||
query_raw_mock = AsyncMock(
|
||||
side_effect=[
|
||||
[{"total": 1}],
|
||||
[_row({"compression_savings": {"tokens_saved": 25}})],
|
||||
]
|
||||
)
|
||||
db.query_raw = query_raw_mock
|
||||
prisma = MagicMock()
|
||||
prisma.db = db
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", prisma),
|
||||
patch(
|
||||
"litellm.proxy.spend_tracking.savings._input_and_cache_read_cost",
|
||||
return_value=(0.001, 0.001),
|
||||
),
|
||||
):
|
||||
result = await cost_optimization_usage_logs(
|
||||
page=1,
|
||||
page_size=50,
|
||||
start_date="2026-07-01",
|
||||
end_date="2026-07-31",
|
||||
)
|
||||
|
||||
assert result.total == 1
|
||||
assert result.total_pages == 1
|
||||
assert result.logs[0].optimization_type == "compression"
|
||||
assert result.logs[0].tokens_saved == 25
|
||||
assert query_raw_mock.await_count == 2
|
||||
|
|
@ -4,6 +4,11 @@ import { describe, expect, it, vi } from "vitest";
|
|||
import type { DailyData, SpendMetrics } from "@/components/UsagePage/types";
|
||||
|
||||
const mockUsePaginatedDailyActivity = vi.fn();
|
||||
const mockUseQuery = vi.fn();
|
||||
|
||||
vi.mock("@tanstack/react-query", () => ({
|
||||
useQuery: (args: unknown) => mockUseQuery(args),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({
|
||||
usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args),
|
||||
|
|
@ -11,6 +16,7 @@ vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", (
|
|||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
userDailyActivityCall: vi.fn(),
|
||||
getCostOptimizationUsageLogs: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/shared/advanced_date_picker", () => ({
|
||||
|
|
@ -57,6 +63,29 @@ const day = (date: string, metrics: Partial<SpendMetrics>): DailyData => ({
|
|||
|
||||
const renderWith = (results: DailyData[]) => {
|
||||
mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false });
|
||||
mockUseQuery.mockReturnValue({
|
||||
data: {
|
||||
logs: [
|
||||
{
|
||||
request_id: "req-123456789",
|
||||
timestamp: "2026-07-13T12:00:00Z",
|
||||
model: "test-model",
|
||||
total_tokens: 150,
|
||||
optimization_type: "both",
|
||||
spend: 0.02,
|
||||
savings: 0.14,
|
||||
original_cost: 0.16,
|
||||
},
|
||||
],
|
||||
total: 1,
|
||||
page: 1,
|
||||
page_size: 50,
|
||||
total_pages: 1,
|
||||
},
|
||||
isLoading: false,
|
||||
isFetching: false,
|
||||
error: null,
|
||||
});
|
||||
return render(<CostOptimizationView accessToken="test-token" userId="u1" userRole="proxy_admin" />);
|
||||
};
|
||||
|
||||
|
|
@ -106,4 +135,15 @@ describe("CostOptimizationView", () => {
|
|||
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
|
||||
expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]);
|
||||
});
|
||||
|
||||
it("renders recent optimized requests with savings and optimization type", () => {
|
||||
const { getByText } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);
|
||||
|
||||
expect(getByText("Recent Optimized Requests")).toBeInTheDocument();
|
||||
expect(getByText("req-123456789")).toBeInTheDocument();
|
||||
expect(getByText("Both")).toBeInTheDocument();
|
||||
expect(getByText("$0.1600")).toBeInTheDocument();
|
||||
expect(getByText("$0.0200")).toBeInTheDocument();
|
||||
expect(getByText("$0.1400")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,11 +2,17 @@
|
|||
|
||||
import React, { useMemo, useState } from "react";
|
||||
import { PiggyBank } from "lucide-react";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
|
||||
import { AreaChart, DonutChart } from "@/components/shared/charts";
|
||||
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { userDailyActivityCall } from "@/components/networking";
|
||||
import {
|
||||
getCostOptimizationUsageLogs,
|
||||
type OptimizedRequestLog,
|
||||
type OptimizedRequestLogsResponse,
|
||||
userDailyActivityCall,
|
||||
} from "@/components/networking";
|
||||
import { DailyData, SpendMetrics } from "@/components/UsagePage/types";
|
||||
import { formatNumberWithCommas } from "@/utils/dataUtils";
|
||||
import { all_admin_roles } from "@/utils/roles";
|
||||
|
|
@ -46,6 +52,112 @@ const SummaryCard = ({ label, value, hint }: { label: string; value: string; hin
|
|||
</Card>
|
||||
);
|
||||
|
||||
const optimizationTypeLabel = (type: OptimizedRequestLog["optimization_type"]): string => {
|
||||
if (type === "both") return "Both";
|
||||
if (type === "caching") return "Caching";
|
||||
return "Compression";
|
||||
};
|
||||
|
||||
const optimizationTypeClass = (type: OptimizedRequestLog["optimization_type"]): string => {
|
||||
if (type === "both") return "bg-purple-50 text-purple-700 border-purple-200";
|
||||
if (type === "caching") return "bg-green-50 text-green-700 border-green-200";
|
||||
return "bg-blue-50 text-blue-700 border-blue-200";
|
||||
};
|
||||
|
||||
const optimizedLogsDescription = (
|
||||
loading: boolean,
|
||||
fetching: boolean,
|
||||
error: unknown,
|
||||
data: OptimizedRequestLogsResponse | undefined,
|
||||
): string => {
|
||||
if (loading || fetching) return "Loading...";
|
||||
if (error) return "Failed to load optimized requests";
|
||||
if (data?.total) return `Showing ${data.logs.length} of ${data.total} requests`;
|
||||
return "No optimized requests for this period";
|
||||
};
|
||||
|
||||
const OptimizedRequestsTable = ({
|
||||
data,
|
||||
logsPage,
|
||||
onPrevious,
|
||||
onNext,
|
||||
}: {
|
||||
data: OptimizedRequestLogsResponse;
|
||||
logsPage: number;
|
||||
onPrevious: () => void;
|
||||
onNext: () => void;
|
||||
}) => (
|
||||
<>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full text-sm">
|
||||
<thead>
|
||||
<tr className="border-b text-left text-xs text-muted-foreground">
|
||||
<th className="px-3 py-3 font-medium">Request ID</th>
|
||||
<th className="px-3 py-3 font-medium">Timestamp</th>
|
||||
<th className="px-3 py-3 font-medium">Model</th>
|
||||
<th className="px-3 py-3 text-right font-medium">Tokens</th>
|
||||
<th className="px-3 py-3 font-medium">Type</th>
|
||||
<th className="px-3 py-3 text-right font-medium">Original Cost</th>
|
||||
<th className="px-3 py-3 text-right font-medium">Optimized Cost</th>
|
||||
<th className="px-3 py-3 text-right font-medium">Savings</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{data.logs.map((log) => (
|
||||
<tr key={log.request_id} className="border-b last:border-0">
|
||||
<td className="max-w-40 truncate px-3 py-3 font-mono text-xs" title={log.request_id}>
|
||||
{log.request_id}
|
||||
</td>
|
||||
<td className="whitespace-nowrap px-3 py-3 text-muted-foreground">
|
||||
{new Date(log.timestamp).toLocaleString()}
|
||||
</td>
|
||||
<td className="max-w-48 truncate px-3 py-3" title={log.model}>
|
||||
{log.model}
|
||||
</td>
|
||||
<td className="px-3 py-3 text-right">{formatNumberWithCommas(log.total_tokens)}</td>
|
||||
<td className="px-3 py-3">
|
||||
<span
|
||||
className={`inline-flex rounded border px-2 py-0.5 text-xs font-medium ${optimizationTypeClass(log.optimization_type)}`}
|
||||
>
|
||||
{optimizationTypeLabel(log.optimization_type)}
|
||||
</span>
|
||||
</td>
|
||||
<td className="px-3 py-3 text-right text-muted-foreground line-through">{usd(log.original_cost)}</td>
|
||||
<td className="px-3 py-3 text-right">{usd(log.spend)}</td>
|
||||
<td className="px-3 py-3 text-right font-medium text-emerald-600">{usd(log.savings)}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{data.total_pages > 1 && (
|
||||
<div className="mt-4 flex items-center justify-between">
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Page {data.page} of {data.total_pages}
|
||||
</p>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
type="button"
|
||||
className="rounded border px-3 py-1 text-sm disabled:cursor-not-allowed disabled:opacity-50"
|
||||
disabled={logsPage === 1}
|
||||
onClick={onPrevious}
|
||||
>
|
||||
Previous
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="rounded border px-3 py-1 text-sm disabled:cursor-not-allowed disabled:opacity-50"
|
||||
disabled={logsPage >= data.total_pages}
|
||||
onClick={onNext}
|
||||
>
|
||||
Next
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
const CostOptimizationView: React.FC<CostOptimizationViewProps> = ({ accessToken, userId, userRole }) => {
|
||||
const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []);
|
||||
const initialTo = useMemo(() => new Date(), []);
|
||||
|
|
@ -68,6 +180,32 @@ const CostOptimizationView: React.FC<CostOptimizationViewProps> = ({ accessToken
|
|||
const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]);
|
||||
const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]);
|
||||
const totalSaved = compressionTotal + cachingTotal;
|
||||
const [logsPage, setLogsPage] = useState(1);
|
||||
const startDate = startTime ? startTime.toISOString().slice(0, 10) : "";
|
||||
const endDate = endTime ? endTime.toISOString().slice(0, 10) : "";
|
||||
const {
|
||||
data: optimizedLogsData,
|
||||
isLoading: optimizedLogsLoading,
|
||||
isFetching: optimizedLogsFetching,
|
||||
error: optimizedLogsError,
|
||||
} = useQuery({
|
||||
queryKey: ["cost-optimization-usage-logs", startDate, endDate, logsPage],
|
||||
queryFn: () =>
|
||||
getCostOptimizationUsageLogs({
|
||||
accessToken: accessToken!,
|
||||
startDate,
|
||||
endDate,
|
||||
page: logsPage,
|
||||
pageSize: 50,
|
||||
}),
|
||||
enabled: !!accessToken && !!startDate && !!endDate,
|
||||
});
|
||||
const logsDescription = optimizedLogsDescription(
|
||||
optimizedLogsLoading,
|
||||
optimizedLogsFetching,
|
||||
optimizedLogsError,
|
||||
optimizedLogsData,
|
||||
);
|
||||
|
||||
const overTime = useMemo(
|
||||
() =>
|
||||
|
|
@ -100,7 +238,13 @@ const CostOptimizationView: React.FC<CostOptimizationViewProps> = ({ accessToken
|
|||
Money saved by prompt compression and prompt caching across your requests
|
||||
</p>
|
||||
</div>
|
||||
<AdvancedDatePicker value={dateValue} onValueChange={(v) => setDateValue(v)} />
|
||||
<AdvancedDatePicker
|
||||
value={dateValue}
|
||||
onValueChange={(v) => {
|
||||
setDateValue(v);
|
||||
setLogsPage(1);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
|
||||
|
|
@ -150,6 +294,32 @@ const CostOptimizationView: React.FC<CostOptimizationViewProps> = ({ accessToken
|
|||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Recent Optimized Requests</CardTitle>
|
||||
<p className="text-sm text-muted-foreground">{logsDescription}</p>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{optimizedLogsLoading && (
|
||||
<div className="py-10 text-center text-sm text-muted-foreground">Loading optimized requests...</div>
|
||||
)}
|
||||
{!optimizedLogsLoading && optimizedLogsError && (
|
||||
<div className="py-10 text-center text-sm text-red-600">Failed to load optimized requests.</div>
|
||||
)}
|
||||
{!optimizedLogsLoading && !optimizedLogsError && optimizedLogsData?.logs.length ? (
|
||||
<OptimizedRequestsTable
|
||||
data={optimizedLogsData}
|
||||
logsPage={logsPage}
|
||||
onPrevious={() => setLogsPage((page) => Math.max(1, page - 1))}
|
||||
onNext={() => setLogsPage((page) => Math.min(optimizedLogsData.total_pages, page + 1))}
|
||||
/>
|
||||
) : null}
|
||||
{!optimizedLogsLoading && !optimizedLogsError && !optimizedLogsData?.logs.length && (
|
||||
<div className="py-10 text-center text-sm text-muted-foreground">No optimized requests to display.</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1403,6 +1403,55 @@ export const userDailyActivityCall = async (
|
|||
});
|
||||
};
|
||||
|
||||
export interface OptimizedRequestLog {
|
||||
request_id: string;
|
||||
timestamp: string;
|
||||
model: string;
|
||||
total_tokens: number;
|
||||
optimization_type: "compression" | "caching" | "both";
|
||||
spend: number;
|
||||
savings: number;
|
||||
original_cost: number;
|
||||
compression_savings_spend: number;
|
||||
prompt_caching_savings_spend: number;
|
||||
tokens_saved: number;
|
||||
cache_read_tokens: number;
|
||||
}
|
||||
|
||||
export interface OptimizedRequestLogsResponse {
|
||||
logs: OptimizedRequestLog[];
|
||||
total: number;
|
||||
page: number;
|
||||
page_size: number;
|
||||
total_pages: number;
|
||||
}
|
||||
|
||||
export interface CostOptimizationUsageLogsArgs {
|
||||
accessToken: string;
|
||||
startDate: string;
|
||||
endDate: string;
|
||||
page?: number;
|
||||
pageSize?: number;
|
||||
}
|
||||
|
||||
export const getCostOptimizationUsageLogs = async ({
|
||||
accessToken,
|
||||
startDate,
|
||||
endDate,
|
||||
page = 1,
|
||||
pageSize = 50,
|
||||
}: CostOptimizationUsageLogsArgs): Promise<OptimizedRequestLogsResponse> => {
|
||||
return apiClient.get<OptimizedRequestLogsResponse>("/cost_optimization/usage/logs", {
|
||||
accessToken,
|
||||
query: {
|
||||
start_date: startDate,
|
||||
end_date: endDate,
|
||||
page,
|
||||
page_size: pageSize,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
export const tagDailyActivityCall = async (
|
||||
accessToken: string,
|
||||
startTime: Date,
|
||||
|
|
|
|||
94
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
94
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -2474,6 +2474,23 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/cost_optimization/usage/logs": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Cost Optimization Usage Logs */
|
||||
get: operations["cost_optimization_usage_logs_cost_optimization_usage_logs_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/credentials": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -28573,6 +28590,49 @@ export interface components {
|
|||
*/
|
||||
type: "openIdConnect";
|
||||
};
|
||||
/** OptimizedRequestLog */
|
||||
OptimizedRequestLog: {
|
||||
/** Cache Read Tokens */
|
||||
cache_read_tokens: number;
|
||||
/** Compression Savings Spend */
|
||||
compression_savings_spend: number;
|
||||
/** Model */
|
||||
model: string;
|
||||
/**
|
||||
* Optimization Type
|
||||
* @enum {string}
|
||||
*/
|
||||
optimization_type: "compression" | "caching" | "both";
|
||||
/** Original Cost */
|
||||
original_cost: number;
|
||||
/** Prompt Caching Savings Spend */
|
||||
prompt_caching_savings_spend: number;
|
||||
/** Request Id */
|
||||
request_id: string;
|
||||
/** Savings */
|
||||
savings: number;
|
||||
/** Spend */
|
||||
spend: number;
|
||||
/** Timestamp */
|
||||
timestamp: string;
|
||||
/** Tokens Saved */
|
||||
tokens_saved: number;
|
||||
/** Total Tokens */
|
||||
total_tokens: number;
|
||||
};
|
||||
/** OptimizedRequestLogsResponse */
|
||||
OptimizedRequestLogsResponse: {
|
||||
/** Logs */
|
||||
logs: components["schemas"]["OptimizedRequestLog"][];
|
||||
/** Page */
|
||||
page: number;
|
||||
/** Page Size */
|
||||
page_size: number;
|
||||
/** Total */
|
||||
total: number;
|
||||
/** Total Pages */
|
||||
total_pages: number;
|
||||
};
|
||||
/** OrgMember */
|
||||
OrgMember: {
|
||||
/**
|
||||
|
|
@ -37642,6 +37702,40 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
cost_optimization_usage_logs_cost_optimization_usage_logs_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
page?: number;
|
||||
page_size?: number;
|
||||
start_date?: string | null;
|
||||
end_date?: string | null;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["OptimizedRequestLogsResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_credentials_credentials_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue