diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 449457bd123..d464d55453d 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -1,8 +1,8 @@ import base64 -import json # <--- NEW +import json import os from datetime import datetime -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from litellm._logging import verbose_logger from litellm.integrations.arize import _utils @@ -25,6 +25,8 @@ else: LANGFUSE_CLOUD_EU_ENDPOINT = "https://cloud.langfuse.com/api/public/otel" LANGFUSE_CLOUD_US_ENDPOINT = "https://us.cloud.langfuse.com/api/public/otel" +LANGFUSE_INGESTION_VERSION_HEADER = "x-langfuse-ingestion-version" +LANGFUSE_INGESTION_VERSION = "4" class LangfuseOtelLogger(OpenTelemetry): @@ -326,7 +328,9 @@ class LangfuseOtelLogger(OpenTelemetry): return OpenTelemetryConfig( exporter="otlp_http", endpoint=endpoint, - headers=f"Authorization={auth_header}", + headers=LangfuseOtelLogger._format_otel_headers( + LangfuseOtelLogger._build_langfuse_otel_headers(auth_header) + ), ) @staticmethod @@ -338,6 +342,26 @@ class LangfuseOtelLogger(OpenTelemetry): auth_header = base64.b64encode(auth_string.encode()).decode() return f"Basic {auth_header}" + @staticmethod + def _build_langfuse_otel_headers(auth_header: str) -> Dict[str, str]: + """ + Build the OTLP header set Langfuse expects. + + `x-langfuse-ingestion-version: 4` selects Langfuse's v4 ingestion path; + without it spans fall back to the older transformation path. + """ + return { + "Authorization": auth_header, + LANGFUSE_INGESTION_VERSION_HEADER: LANGFUSE_INGESTION_VERSION, + } + + @staticmethod + def _format_otel_headers(headers: Dict[str, str]) -> str: + """ + Serialize a header mapping into the comma-separated OTLP header string + """ + return ",".join(f"{key}={value}" for key, value in headers.items()) + def construct_dynamic_otel_headers( self, standard_callback_dynamic_params: StandardCallbackDynamicParams ) -> Optional[dict]: @@ -358,7 +382,7 @@ class LangfuseOtelLogger(OpenTelemetry): public_key=dynamic_langfuse_public_key, secret_key=dynamic_langfuse_secret_key, ) - dynamic_headers["Authorization"] = auth_header + dynamic_headers.update(LangfuseOtelLogger._build_langfuse_otel_headers(auth_header)) return dynamic_headers diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 5347fffca4d..609da6a9b07 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -14,14 +14,14 @@ shared fixtures build on it. """ import functools -import sys +import os from collections.abc import Iterator -from pathlib import Path import pytest import requests from e2e_config import CONTROL_PLANE_BASE_URL, PROXY_BASE_URL +from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager from proxy_client import ProxyClient, build_proxy_client @@ -107,26 +107,17 @@ def pytest_runtest_call(item: pytest.Item) -> None: def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: - """Once the whole e2e session is done (all suites), truncate the spend logs so - the DB doesn't accumulate test rows. Sessions where no e2e test body ran leave - the DB alone so a `DATABASE_URL` pointing at a shared instance is never wiped - without an e2e run. Best-effort: a cleanup failure (no DB reachable) must not - fail the run. The spend_tracking dir goes on sys.path only for this import and - is removed after, so a broader `pytest tests/` run is not left with a mutated - path.""" - if not session.stash.get(_E2E_TEST_RAN, False): - return - spend_dir = str(Path(__file__).parent / "quota_management" / "spend_tracking") - sys.path.insert(0, spend_dir) - try: - from spend_e2e_client import reset_spend_logs # pyright: ignore - - reset_spend_logs() - except Exception as exc: # noqa: BLE001 - cleanup is best-effort - print(f"spend-log cleanup best-effort failed: {exc}") - finally: - if spend_dir in sys.path: - sys.path.remove(spend_dir) + """Once the whole e2e session is done (all suites), optionally truncate the + spend logs so the DB doesn't accumulate test rows. The truncate is destructive + and irreversible, so it runs only when the operator explicitly opts in + (`E2E_RESET_SPEND_LOGS=1`) and an e2e test body actually ran; otherwise a + `DATABASE_URL` pointing at a shared or staging instance is left untouched. + Best-effort: a cleanup failure (no DB reachable) must not fail the run.""" + run_spend_log_cleanup( + opt_in=os.environ.get(RESET_OPT_IN_ENV), + e2e_test_ran=session.stash.get(_E2E_TEST_RAN, False), + truncate=reset_spend_logs, + ) @pytest.fixture(scope="session") diff --git a/tests/e2e/e2e_db.py b/tests/e2e/e2e_db.py new file mode 100644 index 00000000000..439dd519c03 --- /dev/null +++ b/tests/e2e/e2e_db.py @@ -0,0 +1,56 @@ +"""Shared, destructive DB helpers for the e2e harness. + +Kept at the top level next to e2e_config and lifecycle so every suite imports it +by name (`from e2e_db import ...`); no suite reaches into another's directory by +mutating sys.path. + +reset_spend_logs truncates LiteLLM_SpendLogs and cannot be undone, so the +session-finish cleanup routes through run_spend_log_cleanup, which fires the +truncate only on an explicit operator opt-in. "An e2e test ran" is necessary but +never sufficient: a DATABASE_URL pointing at a shared or staging instance must +not be wiped by a routine local run that merely exercised a test. +""" + +import os +from collections.abc import Callable + +RESET_OPT_IN_ENV = "E2E_RESET_SPEND_LOGS" + + +def run_spend_log_cleanup( + *, opt_in: str | None, e2e_test_ran: bool, truncate: Callable[[], None] +) -> bool: + """Invoke `truncate` iff the destructive spend-log reset is both opted into + and warranted, returning whether the truncate was attempted. + + The truncate fires only when the opt-in value is exactly "1" AND an e2e test + body actually ran. Any other opt-in value (unset, "0", "true", "") leaves the + DB untouched, so the destructive path is never armed by the env var's mere + presence or by a test run on its own. Best-effort: a truncate failure is + swallowed so cleanup never fails the session, so the returned bool reports + that the reset was attempted, not that the DB call succeeded. + """ + if opt_in != "1" or not e2e_test_ran: + return False + try: + truncate() + except Exception as exc: # noqa: BLE001 - cleanup is best-effort + print(f"spend-log cleanup best-effort failed: {exc}") + return True + + +def reset_spend_logs() -> None: + """Truncate LiteLLM_SpendLogs for a clean slate. No proxy endpoint deletes + spend logs (/global/spend/reset keeps them), so go to the DB directly. Uses + DATABASE_URL (default: the local docker postgres on its mapped host port; the + in-container `@db` host isn't resolvable from the host, so default to + localhost). + """ + import psycopg + + url = os.environ.get( + "DATABASE_URL", + "postgresql://llmproxy:dbpassword9090@localhost:5432/litellm", + ) + with psycopg.connect(url) as conn: + _ = conn.execute('TRUNCATE TABLE "LiteLLM_SpendLogs"') diff --git a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py index 0d49869aa91..26860212fa3 100644 --- a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py +++ b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py @@ -11,7 +11,6 @@ helpers from one place. from __future__ import annotations -import os import time from collections.abc import Callable from dataclasses import dataclass @@ -50,7 +49,6 @@ from models import ( __all__ = [ "SpendClient", "build_client", - "reset_spend_logs", "unique_marker", "unwrap", "is_ok", @@ -59,23 +57,6 @@ __all__ = [ ] -def reset_spend_logs() -> None: - """Truncate LiteLLM_SpendLogs for a clean slate. No proxy endpoint deletes - spend logs (/global/spend/reset keeps them), so go to the DB directly. Uses - DATABASE_URL (default: the local docker postgres on its mapped host port; note - the in-container `@db` host isn't resolvable from the host, so default to - localhost). - """ - import psycopg - - url = os.environ.get( - "DATABASE_URL", - "postgresql://llmproxy:dbpassword9090@localhost:5432/litellm", - ) - with psycopg.connect(url) as conn: - _ = conn.execute('TRUNCATE TABLE "LiteLLM_SpendLogs"') - - def _chat_body( model: str, content: str, diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 2f2675ca790..28f138c7acd 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -456,7 +456,7 @@ class TestLangfuseOtelKeyDynamicConfig: import base64 expected_auth = base64.b64encode(b"key_public:key_secret").decode() - assert config.headers == f"Authorization=Basic {expected_auth}" + assert config.headers == f"Authorization=Basic {expected_auth},x-langfuse-ingestion-version=4" def test_construct_dynamic_otel_config_host_without_protocol(self): with self._clean_env(): @@ -521,7 +521,10 @@ class TestLangfuseOtelKeyDynamicConfig: import base64 expected_auth = base64.b64encode(b"key_public:key_secret").decode() - assert exporter._headers == {"Authorization": f"Basic {expected_auth}"} + assert exporter._headers == { + "Authorization": f"Basic {expected_auth}", + "x-langfuse-ingestion-version": "4", + } def test_key_dynamic_params_reuse_cached_provider(self): with self._clean_env(): @@ -574,7 +577,10 @@ class TestLangfuseOtelKeyDynamicConfig: provider = next(iter(logger._tracer_provider_cache.values())) exporter = provider._active_span_processor._span_processors[0].span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._headers == {"Authorization": f"Basic {secret}"} + assert exporter._headers == { + "Authorization": f"Basic {secret}", + "x-langfuse-ingestion-version": "4", + } class TestLangfuseOtelResponsesAPI: diff --git a/tests/test_service_logger_otel.py b/tests/test_service_logger_otel.py index 35070d55546..044d37d6781 100644 --- a/tests/test_service_logger_otel.py +++ b/tests/test_service_logger_otel.py @@ -12,6 +12,7 @@ from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger from litellm.integrations.opentelemetry import OpenTelemetry from litellm.types.services import ServiceTypes from litellm._service_logger import ServiceLogging +from litellm.types.utils import StandardCallbackDynamicParams class TestServiceLoggerOTEL(unittest.IsolatedAsyncioTestCase): @@ -108,6 +109,44 @@ class TestServiceLoggerOTEL(unittest.IsolatedAsyncioTestCase): "Generic OTEL logger should have received the log exactly once.", ) + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_tracing") + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_metrics") + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_logs") + async def test_langfuse_otel_env_config_includes_v4_ingestion_header( + self, mock_logs, mock_metrics, mock_tracing + ): + logger = LangfuseOtelLogger() + + headers = OpenTelemetry._get_headers_dictionary(logger.config.headers) + + self.assertEqual( + headers["x-langfuse-ingestion-version"], + "4", + ) + self.assertTrue(headers["Authorization"].startswith("Basic ")) + + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_tracing") + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_metrics") + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_logs") + async def test_langfuse_otel_dynamic_headers_include_v4_ingestion_header( + self, mock_logs, mock_metrics, mock_tracing + ): + logger = LangfuseOtelLogger() + + headers = logger.construct_dynamic_otel_headers( + StandardCallbackDynamicParams( + langfuse_public_key="pk-lf-dynamic", + langfuse_secret_key="sk-lf-dynamic", + ) + ) + + self.assertIsNotNone(headers) + self.assertEqual( + headers["x-langfuse-ingestion-version"], + "4", + ) + self.assertTrue(headers["Authorization"].startswith("Basic ")) + if __name__ == "__main__": unittest.main() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx new file mode 100644 index 00000000000..3474036528a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx @@ -0,0 +1,28 @@ +"use client"; + +import React from "react"; +import { Form } from "antd"; + +import AddAutoRouterTab from "@/components/add_model/add_auto_router_tab"; + +interface AutorouterTabProps { + accessToken: string | null; + userId: string | null; + userRole: string; +} + +const AutorouterTab: React.FC = ({ accessToken, userRole }) => { + const [form] = Form.useForm(); + + if (!accessToken) { + return null; + } + + return ( +
+ form.resetFields()} accessToken={accessToken} userRole={userRole} /> +
+ ); +}; + +export default AutorouterTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx index 7820e19cd60..46aa23fcfc0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx @@ -1,207 +1,34 @@ import { fireEvent, render } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; -import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; - -const mockUsePaginatedDailyActivity = vi.fn(); -const mockUseQuery = vi.fn(); -const mockUiSpendLogsCall = vi.fn(); - -vi.mock("@tanstack/react-query", () => ({ - useQuery: (args: unknown) => mockUseQuery(args), -})); - -vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({ - usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args), -})); - -vi.mock("@/components/networking", () => ({ - userDailyActivityCall: vi.fn(), - getCostOptimizationUsageLogs: vi.fn(), - uiSpendLogsCall: (args: unknown) => mockUiSpendLogsCall(args), -})); - -vi.mock("@/components/view_logs/LogDetailsDrawer", () => ({ - LogDetailsDrawer: ({ open, logEntry }: { open: boolean; logEntry: { request_id: string } | null }) => - open && logEntry ?
{logEntry.request_id}
: null, -})); - -vi.mock("@/components/shared/advanced_date_picker", () => ({ - __esModule: true, - default: () =>
, -})); - -vi.mock("@/components/shared/charts", () => ({ - AreaChart: ({ data, categories }: { data: unknown; categories: string[] }) => ( -
- ), - DonutChart: ({ data, label }: { data: unknown; label: string }) => ( -
- ), -})); +vi.mock("./UsageTab", () => ({ __esModule: true, default: () =>
})); +vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () =>
})); +vi.mock("./AutorouterTab", () => ({ __esModule: true, default: () =>
})); +vi.mock("./PromptCachingTab", () => ({ __esModule: true, default: () =>
})); import CostOptimizationView from "./CostOptimizationView"; -const baseMetrics = (overrides: Partial): SpendMetrics => ({ - spend: 0, - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - api_requests: 0, - successful_requests: 0, - failed_requests: 0, - cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, - ...overrides, -}); - -const day = (date: string, metrics: Partial): DailyData => ({ - date, - metrics: baseMetrics(metrics), - breakdown: { - models: {}, - model_groups: {}, - mcp_servers: {}, - providers: {}, - api_keys: {}, - entities: {}, - }, -}); - -const detailLog = { - request_id: "req-123456789", - api_key: "key", - team_id: "team", - model: "test-model", - model_id: "model-id", - call_type: "completion", - spend: 0.02, - total_tokens: 150, - prompt_tokens: 100, - completion_tokens: 50, - startTime: "2026-07-13T12:00:00Z", - endTime: "2026-07-13T12:00:01Z", - messages: [], - response: {}, - cache_hit: "", - metadata: {}, -}; - -const renderWith = (results: DailyData[]) => { - mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false }); - mockUseQuery.mockImplementation((args: { queryKey: string[] }) => - args.queryKey[0] === "cost-optimization-usage-logs" - ? { - data: { - logs: [ - { - request_id: "req-123456789", - timestamp: "2026-07-13T12:00:00Z", - model: "test-model", - total_tokens: 150, - optimization_type: "both", - spend: 0.02, - savings: 0.14, - original_cost: 0.16, - compression_savings_spend: 0.1, - prompt_caching_savings_spend: 0.04, - tokens_saved: 100, - cache_read_tokens: 50, - }, - ], - total: 1, - page: 1, - page_size: 50, - total_pages: 1, - }, - isLoading: false, - isFetching: false, - error: null, - } - : { - data: { data: [detailLog], total: 1 }, - isLoading: false, - isFetching: false, - error: null, - }, - ); - return render(); -}; +const renderView = () => render(); describe("CostOptimizationView", () => { - it("sums compression and caching dollars across days into the summary cards", () => { - const { getByText } = renderWith([ - day("2026-07-12", { - compression_savings_spend: 0.04, - prompt_caching_savings_spend: 0.006, - compression_saved_tokens: 40000, - }), - day("2026-07-13", { - compression_savings_spend: 0.1, - prompt_caching_savings_spend: 0.01, - compression_saved_tokens: 100000, - }), - ]); + it("renders all four cost-optimization tabs", () => { + const { getByText } = renderView(); - // compression 0.14 + caching 0.016 = 0.156 - expect(getByText("$0.1560")).toBeInTheDocument(); - expect(getByText("$0.1400")).toBeInTheDocument(); - expect(getByText("$0.0160")).toBeInTheDocument(); - expect(getByText("140,000 tokens compressed")).toBeInTheDocument(); + expect(getByText("Usage")).toBeInTheDocument(); + expect(getByText("Prompt Compression")).toBeInTheDocument(); + expect(getByText("Autorouter")).toBeInTheDocument(); + expect(getByText("Prompt Caching")).toBeInTheDocument(); }); - it("builds a per-day time series and per-driver donut from the daily rows", () => { - const { getByTestId } = renderWith([ - day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }), - day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }), - ]); + it("defaults to the Usage tab and switches the active tab on click", () => { + const { getByRole } = renderView(); - const series = JSON.parse(getByTestId("area-chart").getAttribute("data-series") ?? "[]"); - expect(series).toHaveLength(2); - expect(series[0]).toMatchObject({ Compression: 0.04, "Prompt caching": 0.006 }); - expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.01 }); + expect(getByRole("tab", { name: "Usage" })).toHaveAttribute("aria-selected", "true"); + expect(getByRole("tab", { name: "Prompt Compression" })).toHaveAttribute("aria-selected", "false"); - const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); - expect(slices).toEqual([ - { driver: "Compression", usd: expect.closeTo(0.14, 5) }, - { driver: "Prompt caching", usd: expect.closeTo(0.016, 5) }, - ]); - }); + fireEvent.click(getByRole("tab", { name: "Prompt Compression" })); - it("omits a driver slice when that driver has no savings", () => { - const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); - - const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); - expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); - }); - - it("renders recent optimized requests with savings and optimization type", () => { - const { getByText } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); - - expect(getByText("Recent Optimized Requests")).toBeInTheDocument(); - expect(getByText("req-123456789")).toBeInTheDocument(); - expect(getByText("Both")).toBeInTheDocument(); - expect(getByText("$0.1600")).toBeInTheDocument(); - expect(getByText("$0.0200")).toBeInTheDocument(); - expect(getByText("$0.1400")).toBeInTheDocument(); - }); - - it("fetches and opens request details when an optimized request is clicked", async () => { - const { getByText, getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); - - fireEvent.click(getByText("req-123456789")); - - const detailQuery = mockUseQuery.mock.calls - .map(([args]) => args as { queryKey: string[]; queryFn: () => Promise }) - .find((args) => args.queryKey[0] === "cost-optimization-spend-log"); - expect(detailQuery).toBeDefined(); - await detailQuery?.queryFn(); - expect(mockUiSpendLogsCall).toHaveBeenCalledWith( - expect.objectContaining({ - accessToken: "test-token", - params: { request_id: "req-123456789" }, - }), - ); - expect(getByTestId("log-details-drawer")).toHaveTextContent("req-123456789"); + expect(getByRole("tab", { name: "Usage" })).toHaveAttribute("aria-selected", "false"); + expect(getByRole("tab", { name: "Prompt Compression" })).toHaveAttribute("aria-selected", "true"); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx index b4be35f2fdc..6e6830b8451 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx @@ -1,26 +1,13 @@ "use client"; -import React, { useMemo, useState } from "react"; +import React from "react"; import { PiggyBank } from "lucide-react"; -import { useQuery } from "@tanstack/react-query"; -import moment from "moment"; +import { Alert, Tabs } from "antd"; -import { AreaChart, DonutChart } from "@/components/shared/charts"; -import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { - getCostOptimizationUsageLogs, - uiSpendLogsCall, - type OptimizedRequestLog, - type OptimizedRequestLogsResponse, - userDailyActivityCall, -} from "@/components/networking"; -import { DailyData, SpendMetrics } from "@/components/UsagePage/types"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { all_admin_roles } from "@/utils/roles"; -import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity"; -import { LogDetailsDrawer } from "@/components/view_logs/LogDetailsDrawer"; -import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/columns"; +import UsageTab from "./UsageTab"; +import PromptCompressionTab from "./PromptCompressionTab"; +import AutorouterTab from "./AutorouterTab"; +import PromptCachingTab from "./PromptCachingTab"; interface CostOptimizationViewProps { accessToken: string | null; @@ -28,356 +15,62 @@ interface CostOptimizationViewProps { userRole: string; } -type DateRange = { from?: Date; to?: Date }; - -const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; - -const usd = (value: number): string => { - const decimals = value > 0 && value < 1 ? 4 : 2; - return `$${formatNumberWithCommas(value, decimals)}`; -}; - -const shortDate = (iso: string): string => - new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); - -const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; -const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; -const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; - -const SummaryCard = ({ label, value, hint }: { label: string; value: string; hint?: string }) => ( - - - {label} - - -

{value}

- {hint &&

{hint}

} -
-
-); - -const optimizationTypeLabel = (type: OptimizedRequestLog["optimization_type"]): string => { - if (type === "both") return "Both"; - if (type === "caching") return "Caching"; - return "Compression"; -}; - -const optimizationTypeClass = (type: OptimizedRequestLog["optimization_type"]): string => { - if (type === "both") return "bg-purple-50 text-purple-700 border-purple-200"; - if (type === "caching") return "bg-green-50 text-green-700 border-green-200"; - return "bg-blue-50 text-blue-700 border-blue-200"; -}; - -const optimizedLogsDescription = ( - loading: boolean, - fetching: boolean, - error: unknown, - data: OptimizedRequestLogsResponse | undefined, -): string => { - if (loading || fetching) return "Loading..."; - if (error) return "Failed to load optimized requests"; - if (data?.total) return `Showing ${data.logs.length} of ${data.total} requests`; - return "No optimized requests for this period"; -}; - -const OptimizedRequestsTable = ({ - data, - logsPage, - onPrevious, - onNext, - onRowClick, -}: { - data: OptimizedRequestLogsResponse; - logsPage: number; - onPrevious: () => void; - onNext: () => void; - onRowClick: (log: OptimizedRequestLog) => void; -}) => ( - <> -
- - - - - - - - - - - - - - - {data.logs.map((log) => ( - onRowClick(log)} - onKeyDown={(event) => { - if (event.key === "Enter" || event.key === " ") { - event.preventDefault(); - onRowClick(log); - } - }} - role="button" - tabIndex={0} - > - - - - - - - - - - ))} - -
Request IDTimestampModelTokensTypeOriginal CostOptimized CostSavings
- {log.request_id} - - {new Date(log.timestamp).toLocaleString()} - - {log.model} - {formatNumberWithCommas(log.total_tokens)} - - {optimizationTypeLabel(log.optimization_type)} - - {usd(log.original_cost)}{usd(log.spend)}{usd(log.savings)}
-
- {data.total_pages > 1 && ( -
-

- Page {data.page} of {data.total_pages} -

-
- - -
-
- )} - -); - const CostOptimizationView: React.FC = ({ accessToken, userId, userRole }) => { - const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []); - const initialTo = useMemo(() => new Date(), []); - const [dateValue, setDateValue] = useState({ from: initialFrom, to: initialTo }); - - const startTime = dateValue.from ?? null; - const endTime = dateValue.to ?? null; - const isAdmin = all_admin_roles.includes(userRole); - const effectiveUserId = isAdmin ? null : userId; - - const { data, loading, isFetchingMore } = usePaginatedDailyActivity({ - fetchFn: userDailyActivityCall, - args: [accessToken, startTime, endTime, effectiveUserId], - enabled: !!accessToken && !!startTime && !!endTime, - }); - - const results = data.results as DailyData[]; - - const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]); - const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]); - const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]); - const totalSaved = compressionTotal + cachingTotal; - const [logsPage, setLogsPage] = useState(1); - const startDate = startTime ? startTime.toISOString().slice(0, 10) : ""; - const endDate = endTime ? endTime.toISOString().slice(0, 10) : ""; - const [selectedRequestId, setSelectedRequestId] = useState(null); - const [drawerOpen, setDrawerOpen] = useState(false); - const { - data: optimizedLogsData, - isLoading: optimizedLogsLoading, - isFetching: optimizedLogsFetching, - error: optimizedLogsError, - } = useQuery({ - queryKey: ["cost-optimization-usage-logs", startDate, endDate, logsPage], - queryFn: () => - getCostOptimizationUsageLogs({ - accessToken: accessToken!, - startDate, - endDate, - page: logsPage, - pageSize: 50, - }), - enabled: !!accessToken && !!startDate && !!endDate, - }); - const logsDescription = optimizedLogsDescription( - optimizedLogsLoading, - optimizedLogsFetching, - optimizedLogsError, - optimizedLogsData, - ); - const drawerStartTime = startTime - ? moment(startTime).utc().format("YYYY-MM-DD HH:mm:ss") - : moment().subtract(24, "hours").utc().format("YYYY-MM-DD HH:mm:ss"); - const drawerEndTime = endTime - ? moment(endTime).utc().endOf("day").format("YYYY-MM-DD HH:mm:ss") - : moment().utc().format("YYYY-MM-DD HH:mm:ss"); - const { data: fullLogResponse } = useQuery({ - queryKey: ["cost-optimization-spend-log", selectedRequestId, drawerStartTime, drawerEndTime], - queryFn: async () => { - if (!accessToken || !selectedRequestId) return null; - const response = await uiSpendLogsCall({ - accessToken, - start_date: drawerStartTime, - end_date: drawerEndTime, - page: 1, - page_size: 10, - params: { request_id: selectedRequestId }, - }); - return response as { data: ViewLogsLogEntry[]; total: number }; + const items = [ + { + key: "usage", + label: "Usage", + children: , }, - enabled: Boolean(accessToken && selectedRequestId && drawerOpen), - }); - const selectedLog: ViewLogsLogEntry | null = fullLogResponse?.data?.[0] ?? null; - - const overTime = useMemo( - () => - results.map((d) => ({ - date: shortDate(d.date), - Compression: compressionOf(d.metrics), - "Prompt caching": cachingOf(d.metrics), - })), - [results], - ); - - const byDriver = useMemo( - () => - [ - { driver: "Compression", usd: compressionTotal }, - { driver: "Prompt caching", usd: cachingTotal }, - ].filter((d) => d.usd > 0), - [compressionTotal, cachingTotal], - ); + { + key: "compression", + label: "Prompt Compression", + children: , + }, + { + key: "autorouter", + label: "Autorouter", + children: , + }, + { + key: "caching", + label: "Prompt Caching", + children: , + }, + ]; return (
-
-
-
- -

Cost Optimization

-
-

- Money saved by prompt compression and prompt caching across your requests -

+
+
+ +

Cost Optimization

- { - setDateValue(v); - setLogsPage(1); - }} - /> +

+ Track and configure the mechanisms that save you money: prompt compression, prompt caching, and auto routing +

-
- - - -
- -
- - - Savings over time - - - - - - - - Savings by driver - - - - - -
- - - - Recent Optimized Requests -

{logsDescription}

-
- - {optimizedLogsLoading && ( -
Loading optimized requests...
- )} - {!optimizedLogsLoading && optimizedLogsError && ( -
Failed to load optimized requests.
- )} - {!optimizedLogsLoading && !optimizedLogsError && optimizedLogsData?.logs.length ? ( - setLogsPage((page) => Math.max(1, page - 1))} - onNext={() => setLogsPage((page) => Math.min(optimizedLogsData.total_pages, page + 1))} - onRowClick={(log) => { - setSelectedRequestId(log.request_id); - setDrawerOpen(true); - }} - /> - ) : null} - {!optimizedLogsLoading && !optimizedLogsError && !optimizedLogsData?.logs.length && ( -
No optimized requests to display.
- )} -
-
- { - setDrawerOpen(false); - setSelectedRequestId(null); - }} - logEntry={selectedLog} - accessToken={accessToken} - allLogs={selectedLog ? [selectedLog] : []} - startTime={drawerStartTime} + + Have feedback? Join the discussion{" "} + + here + + + } /> + +
); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx new file mode 100644 index 00000000000..e6f73088824 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx @@ -0,0 +1,52 @@ +"use client"; + +import React, { useCallback, useEffect, useState } from "react"; + +import { getGeneralSettingsCall } from "@/components/networking"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { + PromptCachingPanel, + generalSettingsItem, +} from "@/app/(dashboard)/router-settings/_components/general_settings"; + +interface PromptCachingTabProps { + accessToken: string | null; +} + +const PromptCachingTab: React.FC = ({ accessToken }) => { + const [settings, setSettings] = useState([]); + + const loadSettings = useCallback(() => { + if (!accessToken) { + return; + } + getGeneralSettingsCall(accessToken) + .then((data: generalSettingsItem[]) => setSettings(data)) + .catch((error) => { + console.error("Failed to load prompt caching settings:", error); + NotificationsManager.fromBackend("Failed to load prompt caching settings"); + }); + }, [accessToken]); + + useEffect(() => { + loadSettings(); + }, [loadSettings]); + + const handleChange = (fieldName: string, newValue: unknown) => { + setSettings((prev) => + prev.map((setting) => (setting.field_name === fieldName ? { ...setting, field_value: newValue } : setting)), + ); + }; + + if (!accessToken) { + return null; + } + + return ( +
+ +
+ ); +}; + +export default PromptCachingTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCompressionTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCompressionTab.tsx new file mode 100644 index 00000000000..071ad1d6bd5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCompressionTab.tsx @@ -0,0 +1,176 @@ +"use client"; + +import React, { useCallback, useEffect, useState } from "react"; +import { Button, Form, Input, Switch } from "antd"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { createGuardrailCall, getGuardrailsList } from "@/components/networking"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { + buildCompressionGuardrailPayload, + compressionGuardrailsOf, + GuardrailListItem, + GuardrailListResponse, +} from "./helpers"; + +interface PromptCompressionTabProps { + accessToken: string | null; +} + +interface CompressionFormValues { + name: string; + apiBase: string; + defaultOn: boolean; +} + +const PromptCompressionTab: React.FC = ({ accessToken }) => { + const [form] = Form.useForm(); + const [guardrails, setGuardrails] = useState([]); + const [isLoading, setIsLoading] = useState(true); + const [isSaving, setIsSaving] = useState(false); + + const loadGuardrails = useCallback(() => { + if (!accessToken) { + return; + } + getGuardrailsList(accessToken) + .then((response) => setGuardrails(compressionGuardrailsOf(response as GuardrailListResponse))) + .catch((error) => { + console.error("Failed to load compression guardrails:", error); + NotificationsManager.fromBackend("Failed to load compression guardrails"); + }) + .finally(() => setIsLoading(false)); + }, [accessToken]); + + useEffect(() => { + loadGuardrails(); + }, [loadGuardrails]); + + const handleAdd = async (values: CompressionFormValues) => { + if (!accessToken) { + return; + } + setIsSaving(true); + try { + await createGuardrailCall( + accessToken, + buildCompressionGuardrailPayload({ + name: values.name, + apiBase: values.apiBase, + defaultOn: values.defaultOn ?? true, + }), + ); + NotificationsManager.success("Compression guardrail created"); + form.resetFields(); + await loadGuardrails(); + } catch (error) { + console.error("Failed to create compression guardrail:", error); + NotificationsManager.fromBackend("Failed to create compression guardrail"); + } finally { + setIsSaving(false); + } + }; + + return ( +
+ + + Headroom prompt compression + + +

+ Headroom is a native LiteLLM guardrail that compresses your prompts before they reach the model, so you pay + for fewer input tokens. The tokens it removes are priced and shown on the Usage tab as compression savings.{" "} + + Headroom setup docs + +

+ {isLoading &&

Loading...

} + {!isLoading && guardrails.length === 0 && ( +

+ No prompt compression guardrails configured yet. Add one below to start saving on input tokens +

+ )} + {!isLoading && guardrails.length > 0 && ( +
    + {guardrails.map((guardrail) => ( +
  • +
    +

    {guardrail.guardrail_name}

    +

    {guardrail.litellm_params?.api_base ?? ""}

    +
    + + {guardrail.litellm_params?.default_on ? "Always on" : "Opt-in"} + +
  • + ))} +
+ )} +
+
+ + + + Add Headroom compression guardrail + + +
+ + + + + + + + + +
+

+ Applying compression to all requests is available to all users. Enabling it selectively per key or team + is a LiteLLM Enterprise feature. Get a trial key{" "} + + here + +

+
+
+ +
+
+
+
+
+ ); +}; + +export default PromptCompressionTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx new file mode 100644 index 00000000000..10bbb7c2b06 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx @@ -0,0 +1,206 @@ +import { fireEvent, render } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; + +const mockUsePaginatedDailyActivity = vi.fn(); +const mockUseQuery = vi.fn(); +const mockUiSpendLogsCall = vi.fn(); + +vi.mock("@tanstack/react-query", () => ({ + useQuery: (args: unknown) => mockUseQuery(args), +})); + +vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({ + usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args), +})); + +vi.mock("@/components/networking", () => ({ + userDailyActivityCall: vi.fn(), + getCostOptimizationUsageLogs: vi.fn(), + uiSpendLogsCall: (args: unknown) => mockUiSpendLogsCall(args), +})); + +vi.mock("@/components/view_logs/LogDetailsDrawer", () => ({ + LogDetailsDrawer: ({ open, logEntry }: { open: boolean; logEntry: { request_id: string } | null }) => + open && logEntry ?
{logEntry.request_id}
: null, +})); + +vi.mock("@/components/shared/advanced_date_picker", () => ({ + __esModule: true, + default: () =>
, +})); + +vi.mock("@/components/shared/charts", () => ({ + AreaChart: ({ data, categories }: { data: unknown; categories: string[] }) => ( +
+ ), + DonutChart: ({ data, label }: { data: unknown; label: string }) => ( +
+ ), +})); + +import UsageTab from "./UsageTab"; + +const baseMetrics = (overrides: Partial): SpendMetrics => ({ + spend: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + api_requests: 0, + successful_requests: 0, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + ...overrides, +}); + +const day = (date: string, metrics: Partial): DailyData => ({ + date, + metrics: baseMetrics(metrics), + breakdown: { + models: {}, + model_groups: {}, + mcp_servers: {}, + providers: {}, + api_keys: {}, + entities: {}, + }, +}); + +const detailLog = { + request_id: "req-123456789", + api_key: "key", + team_id: "team", + model: "test-model", + model_id: "model-id", + call_type: "completion", + spend: 0.02, + total_tokens: 150, + prompt_tokens: 100, + completion_tokens: 50, + startTime: "2026-07-13T12:00:00Z", + endTime: "2026-07-13T12:00:01Z", + messages: [], + response: {}, + cache_hit: "", + metadata: {}, +}; + +const renderWith = (results: DailyData[]) => { + mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false }); + mockUseQuery.mockImplementation((args: { queryKey: string[] }) => + args.queryKey[0] === "cost-optimization-usage-logs" + ? { + data: { + logs: [ + { + request_id: "req-123456789", + timestamp: "2026-07-13T12:00:00Z", + model: "test-model", + total_tokens: 150, + optimization_type: "both", + spend: 0.02, + savings: 0.14, + original_cost: 0.16, + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.04, + tokens_saved: 100, + cache_read_tokens: 50, + }, + ], + total: 1, + page: 1, + page_size: 50, + total_pages: 1, + }, + isLoading: false, + isFetching: false, + error: null, + } + : { + data: { data: [detailLog], total: 1 }, + isLoading: false, + isFetching: false, + error: null, + }, + ); + return render(); +}; + +describe("UsageTab", () => { + it("sums compression and caching dollars across days into the summary cards", () => { + const { getByText } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.04, + prompt_caching_savings_spend: 0.006, + compression_saved_tokens: 40000, + }), + day("2026-07-13", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.01, + compression_saved_tokens: 100000, + }), + ]); + + expect(getByText("$0.1560")).toBeInTheDocument(); + expect(getByText("$0.1400")).toBeInTheDocument(); + expect(getByText("$0.0160")).toBeInTheDocument(); + expect(getByText("140,000 tokens compressed")).toBeInTheDocument(); + }); + + it("builds a per-day time series and per-driver donut from the daily rows", () => { + const { getByTestId } = renderWith([ + day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }), + day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }), + ]); + + const series = JSON.parse(getByTestId("area-chart").getAttribute("data-series") ?? "[]"); + expect(series).toHaveLength(2); + expect(series[0]).toMatchObject({ Compression: 0.04, "Prompt caching": 0.006 }); + expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.01 }); + + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices).toEqual([ + { driver: "Compression", usd: expect.closeTo(0.14, 5) }, + { driver: "Prompt caching", usd: expect.closeTo(0.016, 5) }, + ]); + }); + + it("omits a driver slice when that driver has no savings", () => { + const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); + + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); + }); + + it("renders recent optimized requests with savings and optimization type", () => { + const { getByText } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); + + expect(getByText("Recent Optimized Requests")).toBeInTheDocument(); + expect(getByText("req-123456789")).toBeInTheDocument(); + expect(getByText("Both")).toBeInTheDocument(); + expect(getByText("$0.1600")).toBeInTheDocument(); + expect(getByText("$0.0200")).toBeInTheDocument(); + expect(getByText("$0.1400")).toBeInTheDocument(); + }); + + it("fetches and opens request details when an optimized request is clicked", async () => { + const { getByText, getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); + + fireEvent.click(getByText("req-123456789")); + + const detailQuery = mockUseQuery.mock.calls + .map(([args]) => args as { queryKey: string[]; queryFn: () => Promise }) + .find((args) => args.queryKey[0] === "cost-optimization-spend-log"); + expect(detailQuery).toBeDefined(); + await detailQuery?.queryFn(); + expect(mockUiSpendLogsCall).toHaveBeenCalledWith( + expect.objectContaining({ + accessToken: "test-token", + params: { request_id: "req-123456789" }, + }), + ); + expect(getByTestId("log-details-drawer")).toHaveTextContent("req-123456789"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx new file mode 100644 index 00000000000..e069b82d0f1 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx @@ -0,0 +1,412 @@ +"use client"; + +import React, { useMemo, useState } from "react"; +import { Collapse } from "antd"; +import { useQuery } from "@tanstack/react-query"; +import moment from "moment"; + +import { AreaChart, DonutChart } from "@/components/shared/charts"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { + getCostOptimizationUsageLogs, + uiSpendLogsCall, + type OptimizedRequestLog, + type OptimizedRequestLogsResponse, + userDailyActivityCall, +} from "@/components/networking"; +import { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { all_admin_roles } from "@/utils/roles"; +import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity"; +import { LogDetailsDrawer } from "@/components/view_logs/LogDetailsDrawer"; +import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/columns"; + +interface UsageTabProps { + accessToken: string | null; + userId: string | null; + userRole: string; +} + +type DateRange = { from?: Date; to?: Date }; + +const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; + +const usd = (value: number): string => { + const decimals = value > 0 && value < 1 ? 4 : 2; + return `$${formatNumberWithCommas(value, decimals)}`; +}; + +const shortDate = (iso: string): string => + new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); + +const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; +const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; +const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; + +const MethodologyNote = () => ( + How savings are calculated, + children: ( +
+

+ Savings are computed for each request when it is logged, using the provider's reported usage and the + model's pricing, then summed into a daily rollup. Totals below are read from that rollup over the + selected date range, so the numbers never require a scan of raw request logs. +

+

+ Compression savings are the tokens Headroom removed before the call, priced at the model's input + rate: compression_saved_tokens * input_cost_per_token +

+

+ Prompt caching savings are the tokens the provider served from cache (Anthropic{" "} + cache_read_input_tokens, or OpenAI-style prompt_tokens_details.cached_tokens), + priced at the discount between the normal input rate and the cache-read rate:{" "} + cache_read_input_tokens * max(input_cost_per_token - cache_read_input_token_cost, 0) +

+

+ Total saved is the sum of both drivers. Models without a separate cache-read price in the pricing map + contribute zero caching savings rather than erroring. +

+
+ ), + }, + ]} + /> +); + +const SummaryCard = ({ label, value, hint }: { label: string; value: string; hint?: string }) => ( + + + {label} + + +

{value}

+ {hint &&

{hint}

} +
+
+); + +const optimizationTypeLabel = (type: OptimizedRequestLog["optimization_type"]): string => { + if (type === "both") return "Both"; + if (type === "caching") return "Caching"; + return "Compression"; +}; + +const optimizationTypeClass = (type: OptimizedRequestLog["optimization_type"]): string => { + if (type === "both") return "bg-purple-50 text-purple-700 border-purple-200"; + if (type === "caching") return "bg-green-50 text-green-700 border-green-200"; + return "bg-blue-50 text-blue-700 border-blue-200"; +}; + +const optimizedLogsDescription = ( + loading: boolean, + fetching: boolean, + error: unknown, + data: OptimizedRequestLogsResponse | undefined, +): string => { + if (loading || fetching) return "Loading..."; + if (error) return "Failed to load optimized requests"; + if (data?.total) return `Showing ${data.logs.length} of ${data.total} requests`; + return "No optimized requests for this period"; +}; + +const OptimizedRequestsTable = ({ + data, + logsPage, + onPrevious, + onNext, + onRowClick, +}: { + data: OptimizedRequestLogsResponse; + logsPage: number; + onPrevious: () => void; + onNext: () => void; + onRowClick: (log: OptimizedRequestLog) => void; +}) => ( + <> +
+ + + + + + + + + + + + + + + {data.logs.map((log) => ( + onRowClick(log)} + onKeyDown={(event) => { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + onRowClick(log); + } + }} + role="button" + tabIndex={0} + > + + + + + + + + + + ))} + +
Request IDTimestampModelTokensTypeOriginal CostOptimized CostSavings
+ {log.request_id} + + {new Date(log.timestamp).toLocaleString()} + + {log.model} + {formatNumberWithCommas(log.total_tokens)} + + {optimizationTypeLabel(log.optimization_type)} + + {usd(log.original_cost)}{usd(log.spend)}{usd(log.savings)}
+
+ {data.total_pages > 1 && ( +
+

+ Page {data.page} of {data.total_pages} +

+
+ + +
+
+ )} + +); + +const UsageTab: React.FC = ({ accessToken, userId, userRole }) => { + const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []); + const initialTo = useMemo(() => new Date(), []); + const [dateValue, setDateValue] = useState({ from: initialFrom, to: initialTo }); + + const startTime = dateValue.from ?? null; + const endTime = dateValue.to ?? null; + const isAdmin = all_admin_roles.includes(userRole); + const effectiveUserId = isAdmin ? null : userId; + + const { data, loading, isFetchingMore } = usePaginatedDailyActivity({ + fetchFn: userDailyActivityCall, + args: [accessToken, startTime, endTime, effectiveUserId], + enabled: !!accessToken && !!startTime && !!endTime, + }); + + const results = data.results as DailyData[]; + + const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]); + const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]); + const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]); + const totalSaved = compressionTotal + cachingTotal; + const [logsPage, setLogsPage] = useState(1); + const startDate = startTime ? startTime.toISOString().slice(0, 10) : ""; + const endDate = endTime ? endTime.toISOString().slice(0, 10) : ""; + const [selectedRequestId, setSelectedRequestId] = useState(null); + const [drawerOpen, setDrawerOpen] = useState(false); + const { + data: optimizedLogsData, + isLoading: optimizedLogsLoading, + isFetching: optimizedLogsFetching, + error: optimizedLogsError, + } = useQuery({ + queryKey: ["cost-optimization-usage-logs", startDate, endDate, logsPage], + queryFn: () => + getCostOptimizationUsageLogs({ + accessToken: accessToken!, + startDate, + endDate, + page: logsPage, + pageSize: 50, + }), + enabled: !!accessToken && !!startDate && !!endDate, + }); + const logsDescription = optimizedLogsDescription( + optimizedLogsLoading, + optimizedLogsFetching, + optimizedLogsError, + optimizedLogsData, + ); + const drawerStartTime = startTime + ? moment(startTime).utc().format("YYYY-MM-DD HH:mm:ss") + : moment().subtract(24, "hours").utc().format("YYYY-MM-DD HH:mm:ss"); + const drawerEndTime = endTime + ? moment(endTime).utc().endOf("day").format("YYYY-MM-DD HH:mm:ss") + : moment().utc().format("YYYY-MM-DD HH:mm:ss"); + const { data: fullLogResponse } = useQuery({ + queryKey: ["cost-optimization-spend-log", selectedRequestId, drawerStartTime, drawerEndTime], + queryFn: async () => { + if (!accessToken || !selectedRequestId) return null; + const response = await uiSpendLogsCall({ + accessToken, + start_date: drawerStartTime, + end_date: drawerEndTime, + page: 1, + page_size: 10, + params: { request_id: selectedRequestId }, + }); + return response as { data: ViewLogsLogEntry[]; total: number }; + }, + enabled: Boolean(accessToken && selectedRequestId && drawerOpen), + }); + const selectedLog: ViewLogsLogEntry | null = fullLogResponse?.data?.[0] ?? null; + + const overTime = useMemo( + () => + results.map((d) => ({ + date: shortDate(d.date), + Compression: compressionOf(d.metrics), + "Prompt caching": cachingOf(d.metrics), + })), + [results], + ); + + const byDriver = useMemo( + () => + [ + { driver: "Compression", usd: compressionTotal }, + { driver: "Prompt caching", usd: cachingTotal }, + ].filter((d) => d.usd > 0), + [compressionTotal, cachingTotal], + ); + + return ( +
+
+ + { + setDateValue(v); + setLogsPage(1); + }} + /> +
+ +
+ + + +
+ +
+ + + Savings over time + + + + + + + + Savings by driver + + + + + +
+ + + + Recent Optimized Requests +

{logsDescription}

+
+ + {optimizedLogsLoading && ( +
Loading optimized requests...
+ )} + {!optimizedLogsLoading && optimizedLogsError && ( +
Failed to load optimized requests.
+ )} + {!optimizedLogsLoading && !optimizedLogsError && optimizedLogsData?.logs.length ? ( + setLogsPage((page) => Math.max(1, page - 1))} + onNext={() => setLogsPage((page) => Math.min(optimizedLogsData.total_pages, page + 1))} + onRowClick={(log) => { + setSelectedRequestId(log.request_id); + setDrawerOpen(true); + }} + /> + ) : null} + {!optimizedLogsLoading && !optimizedLogsError && !optimizedLogsData?.logs.length && ( +
No optimized requests to display.
+ )} +
+
+ { + setDrawerOpen(false); + setSelectedRequestId(null); + }} + logEntry={selectedLog} + accessToken={accessToken} + allLogs={selectedLog ? [selectedLog] : []} + startTime={drawerStartTime} + /> +
+ ); +}; + +export default UsageTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.test.ts new file mode 100644 index 00000000000..239cded0815 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; + +import { buildCompressionGuardrailPayload, compressionGuardrailsOf } from "./helpers"; + +describe("compressionGuardrailsOf", () => { + it("keeps only headroom-provider guardrails and drops others", () => { + const filtered = compressionGuardrailsOf({ + guardrails: [ + { guardrail_id: "1", guardrail_name: "headroom-compression", litellm_params: { guardrail: "headroom" } }, + { guardrail_id: "2", guardrail_name: "pii-masker", litellm_params: { guardrail: "presidio" } }, + { guardrail_id: "3", guardrail_name: "no-params", litellm_params: null }, + ], + }); + + expect(filtered.map((g) => g.guardrail_id)).toEqual(["1"]); + }); +}); + +describe("buildCompressionGuardrailPayload", () => { + it("builds a headroom guardrail payload with trimmed fields", () => { + const payload = buildCompressionGuardrailPayload({ + name: " headroom-compression ", + apiBase: " https://compress ", + defaultOn: false, + }); + + expect(payload).toEqual({ + guardrail_name: "headroom-compression", + litellm_params: { + guardrail: "headroom", + mode: "pre_call", + api_base: "https://compress", + default_on: false, + }, + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.ts new file mode 100644 index 00000000000..7c8c92f8890 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.ts @@ -0,0 +1,39 @@ +export interface GuardrailLitellmParams { + guardrail?: string | null; + api_base?: string | null; + default_on?: boolean | null; +} + +export interface GuardrailListItem { + guardrail_id: string; + guardrail_name: string | null; + litellm_params?: GuardrailLitellmParams | null; +} + +export interface GuardrailListResponse { + guardrails?: GuardrailListItem[]; +} + +export const COMPRESSION_GUARDRAIL_PROVIDER = "headroom"; + +export const isCompressionGuardrail = (guardrail: GuardrailListItem): boolean => + (guardrail.litellm_params?.guardrail ?? "").toLowerCase() === COMPRESSION_GUARDRAIL_PROVIDER; + +export const compressionGuardrailsOf = (response: GuardrailListResponse): GuardrailListItem[] => + (response.guardrails ?? []).filter(isCompressionGuardrail); + +export interface CompressionGuardrailInput { + name: string; + apiBase: string; + defaultOn: boolean; +} + +export const buildCompressionGuardrailPayload = (input: CompressionGuardrailInput): Record => ({ + guardrail_name: input.name.trim(), + litellm_params: { + guardrail: COMPRESSION_GUARDRAIL_PROVIDER, + mode: "pre_call", + api_base: input.apiBase.trim(), + default_on: input.defaultOn, + }, +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 1e8658d5104..dda7a23a8d4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -33,7 +33,7 @@ interface GeneralSettingsPageProps { userID: string | null; } -interface generalSettingsItem { +export interface generalSettingsItem { field_name: string; field_type: string; field_value: any; @@ -90,7 +90,7 @@ const SettingValueEditor: React.FC<{ return null; }; -const PromptCachingPanel: React.FC<{ +export const PromptCachingPanel: React.FC<{ accessToken: string; settings: generalSettingsItem[]; onChange: (fieldName: string, newValue: any) => void;