From bd44c9e305b89526d4c5d773ee39ca935561b9c8 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Sat, 18 Jul 2026 20:36:51 -0700 Subject: [PATCH 1/3] fix(langfuse): send v4 ingestion header for otel callback (#33907) * fix(langfuse): send v4 ingestion header for otel callback * refactor(langfuse): inline otel ingestion header literals * test(langfuse): assert v4 ingestion header on dynamic key config paths * style: apply ruff format to langfuse otel header changes * chore(langfuse): drop stale development annotation on json import --------- Co-authored-by: Hassieb Pakzad <68423100+hassiebp@users.noreply.github.com> --- .../integrations/langfuse/langfuse_otel.py | 32 +++++++++++++-- .../integrations/test_langfuse_otel.py | 12 ++++-- tests/test_service_logger_otel.py | 39 +++++++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 449457bd123..d464d55453d 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -1,8 +1,8 @@ import base64 -import json # <--- NEW +import json import os from datetime import datetime -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from litellm._logging import verbose_logger from litellm.integrations.arize import _utils @@ -25,6 +25,8 @@ else: LANGFUSE_CLOUD_EU_ENDPOINT = "https://cloud.langfuse.com/api/public/otel" LANGFUSE_CLOUD_US_ENDPOINT = "https://us.cloud.langfuse.com/api/public/otel" +LANGFUSE_INGESTION_VERSION_HEADER = "x-langfuse-ingestion-version" +LANGFUSE_INGESTION_VERSION = "4" class LangfuseOtelLogger(OpenTelemetry): @@ -326,7 +328,9 @@ class LangfuseOtelLogger(OpenTelemetry): return OpenTelemetryConfig( exporter="otlp_http", endpoint=endpoint, - headers=f"Authorization={auth_header}", + headers=LangfuseOtelLogger._format_otel_headers( + LangfuseOtelLogger._build_langfuse_otel_headers(auth_header) + ), ) @staticmethod @@ -338,6 +342,26 @@ class LangfuseOtelLogger(OpenTelemetry): auth_header = base64.b64encode(auth_string.encode()).decode() return f"Basic {auth_header}" + @staticmethod + def _build_langfuse_otel_headers(auth_header: str) -> Dict[str, str]: + """ + Build the OTLP header set Langfuse expects. + + `x-langfuse-ingestion-version: 4` selects Langfuse's v4 ingestion path; + without it spans fall back to the older transformation path. + """ + return { + "Authorization": auth_header, + LANGFUSE_INGESTION_VERSION_HEADER: LANGFUSE_INGESTION_VERSION, + } + + @staticmethod + def _format_otel_headers(headers: Dict[str, str]) -> str: + """ + Serialize a header mapping into the comma-separated OTLP header string + """ + return ",".join(f"{key}={value}" for key, value in headers.items()) + def construct_dynamic_otel_headers( self, standard_callback_dynamic_params: StandardCallbackDynamicParams ) -> Optional[dict]: @@ -358,7 +382,7 @@ class LangfuseOtelLogger(OpenTelemetry): public_key=dynamic_langfuse_public_key, secret_key=dynamic_langfuse_secret_key, ) - dynamic_headers["Authorization"] = auth_header + dynamic_headers.update(LangfuseOtelLogger._build_langfuse_otel_headers(auth_header)) return dynamic_headers diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 2f2675ca790..28f138c7acd 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -456,7 +456,7 @@ class TestLangfuseOtelKeyDynamicConfig: import base64 expected_auth = base64.b64encode(b"key_public:key_secret").decode() - assert config.headers == f"Authorization=Basic {expected_auth}" + assert config.headers == f"Authorization=Basic {expected_auth},x-langfuse-ingestion-version=4" def test_construct_dynamic_otel_config_host_without_protocol(self): with self._clean_env(): @@ -521,7 +521,10 @@ class TestLangfuseOtelKeyDynamicConfig: import base64 expected_auth = base64.b64encode(b"key_public:key_secret").decode() - assert exporter._headers == {"Authorization": f"Basic {expected_auth}"} + assert exporter._headers == { + "Authorization": f"Basic {expected_auth}", + "x-langfuse-ingestion-version": "4", + } def test_key_dynamic_params_reuse_cached_provider(self): with self._clean_env(): @@ -574,7 +577,10 @@ class TestLangfuseOtelKeyDynamicConfig: provider = next(iter(logger._tracer_provider_cache.values())) exporter = provider._active_span_processor._span_processors[0].span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._headers == {"Authorization": f"Basic {secret}"} + assert exporter._headers == { + "Authorization": f"Basic {secret}", + "x-langfuse-ingestion-version": "4", + } class TestLangfuseOtelResponsesAPI: diff --git a/tests/test_service_logger_otel.py b/tests/test_service_logger_otel.py index 35070d55546..044d37d6781 100644 --- a/tests/test_service_logger_otel.py +++ b/tests/test_service_logger_otel.py @@ -12,6 +12,7 @@ from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger from litellm.integrations.opentelemetry import OpenTelemetry from litellm.types.services import ServiceTypes from litellm._service_logger import ServiceLogging +from litellm.types.utils import StandardCallbackDynamicParams class TestServiceLoggerOTEL(unittest.IsolatedAsyncioTestCase): @@ -108,6 +109,44 @@ class TestServiceLoggerOTEL(unittest.IsolatedAsyncioTestCase): "Generic OTEL logger should have received the log exactly once.", ) + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_tracing") + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_metrics") + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_logs") + async def test_langfuse_otel_env_config_includes_v4_ingestion_header( + self, mock_logs, mock_metrics, mock_tracing + ): + logger = LangfuseOtelLogger() + + headers = OpenTelemetry._get_headers_dictionary(logger.config.headers) + + self.assertEqual( + headers["x-langfuse-ingestion-version"], + "4", + ) + self.assertTrue(headers["Authorization"].startswith("Basic ")) + + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_tracing") + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_metrics") + @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_logs") + async def test_langfuse_otel_dynamic_headers_include_v4_ingestion_header( + self, mock_logs, mock_metrics, mock_tracing + ): + logger = LangfuseOtelLogger() + + headers = logger.construct_dynamic_otel_headers( + StandardCallbackDynamicParams( + langfuse_public_key="pk-lf-dynamic", + langfuse_secret_key="sk-lf-dynamic", + ) + ) + + self.assertIsNotNone(headers) + self.assertEqual( + headers["x-langfuse-ingestion-version"], + "4", + ) + self.assertTrue(headers["Authorization"].startswith("Basic ")) + if __name__ == "__main__": unittest.main() From 34561482ed092d78c296cab7999486022af5a938 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 07:30:49 -0700 Subject: [PATCH 2/3] feat(ui): add configuration tabs to the Cost Optimization page (#33899) * feat(ui): add configuration tabs to Cost Optimization page Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(ui): reuse AutoRouter v2 and Router Settings prompt-caching panel in Cost Optimization; clarify Headroom compression Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(ui): add experimental dashboard banner with feedback discussion link to Cost Optimization Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(ui): add savings methodology note and per-key/team compression enterprise callout to Cost Optimization Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): assert active tab state in Cost Optimization tab-switch test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Krrish Dholakia Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../_components/AutorouterTab.tsx | 28 +++ .../_components/CostOptimizationView.test.tsx | 113 ++--------- .../_components/CostOptimizationView.tsx | 187 +++++------------- .../_components/PromptCachingTab.tsx | 52 +++++ .../_components/PromptCompressionTab.tsx | 176 +++++++++++++++++ .../_components/UsageTab.test.tsx | 108 ++++++++++ .../_components/UsageTab.tsx | 184 +++++++++++++++++ .../_components/helpers.test.ts | 37 ++++ .../cost-optimization/_components/helpers.ts | 39 ++++ .../_components/general_settings.tsx | 4 +- 10 files changed, 699 insertions(+), 229 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCompressionTab.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx new file mode 100644 index 00000000000..3474036528a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx @@ -0,0 +1,28 @@ +"use client"; + +import React from "react"; +import { Form } from "antd"; + +import AddAutoRouterTab from "@/components/add_model/add_auto_router_tab"; + +interface AutorouterTabProps { + accessToken: string | null; + userId: string | null; + userRole: string; +} + +const AutorouterTab: React.FC = ({ accessToken, userRole }) => { + const [form] = Form.useForm(); + + if (!accessToken) { + return null; + } + + return ( +
+ form.resetFields()} accessToken={accessToken} userRole={userRole} /> +
+ ); +}; + +export default AutorouterTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx index 92426d3ce04..46aa23fcfc0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx @@ -1,109 +1,34 @@ -import { render } from "@testing-library/react"; +import { fireEvent, render } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; -import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; - -const mockUsePaginatedDailyActivity = vi.fn(); - -vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({ - usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args), -})); - -vi.mock("@/components/networking", () => ({ - userDailyActivityCall: vi.fn(), -})); - -vi.mock("@/components/shared/advanced_date_picker", () => ({ - __esModule: true, - default: () =>
, -})); - -vi.mock("@/components/shared/charts", () => ({ - AreaChart: ({ data, categories }: { data: unknown; categories: string[] }) => ( -
- ), - DonutChart: ({ data, label }: { data: unknown; label: string }) => ( -
- ), -})); +vi.mock("./UsageTab", () => ({ __esModule: true, default: () =>
})); +vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () =>
})); +vi.mock("./AutorouterTab", () => ({ __esModule: true, default: () =>
})); +vi.mock("./PromptCachingTab", () => ({ __esModule: true, default: () =>
})); import CostOptimizationView from "./CostOptimizationView"; -const baseMetrics = (overrides: Partial): SpendMetrics => ({ - spend: 0, - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - api_requests: 0, - successful_requests: 0, - failed_requests: 0, - cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, - ...overrides, -}); - -const day = (date: string, metrics: Partial): DailyData => ({ - date, - metrics: baseMetrics(metrics), - breakdown: { - models: {}, - model_groups: {}, - mcp_servers: {}, - providers: {}, - api_keys: {}, - entities: {}, - }, -}); - -const renderWith = (results: DailyData[]) => { - mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false }); - return render(); -}; +const renderView = () => render(); describe("CostOptimizationView", () => { - it("sums compression and caching dollars across days into the summary cards", () => { - const { getByText } = renderWith([ - day("2026-07-12", { - compression_savings_spend: 0.04, - prompt_caching_savings_spend: 0.006, - compression_saved_tokens: 40000, - }), - day("2026-07-13", { - compression_savings_spend: 0.1, - prompt_caching_savings_spend: 0.01, - compression_saved_tokens: 100000, - }), - ]); + it("renders all four cost-optimization tabs", () => { + const { getByText } = renderView(); - // compression 0.14 + caching 0.016 = 0.156 - expect(getByText("$0.1560")).toBeInTheDocument(); - expect(getByText("$0.1400")).toBeInTheDocument(); - expect(getByText("$0.0160")).toBeInTheDocument(); - expect(getByText("140,000 tokens compressed")).toBeInTheDocument(); + expect(getByText("Usage")).toBeInTheDocument(); + expect(getByText("Prompt Compression")).toBeInTheDocument(); + expect(getByText("Autorouter")).toBeInTheDocument(); + expect(getByText("Prompt Caching")).toBeInTheDocument(); }); - it("builds a per-day time series and per-driver donut from the daily rows", () => { - const { getByTestId } = renderWith([ - day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }), - day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }), - ]); + it("defaults to the Usage tab and switches the active tab on click", () => { + const { getByRole } = renderView(); - const series = JSON.parse(getByTestId("area-chart").getAttribute("data-series") ?? "[]"); - expect(series).toHaveLength(2); - expect(series[0]).toMatchObject({ Compression: 0.04, "Prompt caching": 0.006 }); - expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.01 }); + expect(getByRole("tab", { name: "Usage" })).toHaveAttribute("aria-selected", "true"); + expect(getByRole("tab", { name: "Prompt Compression" })).toHaveAttribute("aria-selected", "false"); - const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); - expect(slices).toEqual([ - { driver: "Compression", usd: expect.closeTo(0.14, 5) }, - { driver: "Prompt caching", usd: expect.closeTo(0.016, 5) }, - ]); - }); + fireEvent.click(getByRole("tab", { name: "Prompt Compression" })); - it("omits a driver slice when that driver has no savings", () => { - const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); - - const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); - expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); + expect(getByRole("tab", { name: "Usage" })).toHaveAttribute("aria-selected", "false"); + expect(getByRole("tab", { name: "Prompt Compression" })).toHaveAttribute("aria-selected", "true"); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx index e45e3e23f1d..6e6830b8451 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx @@ -1,16 +1,13 @@ "use client"; -import React, { useMemo, useState } from "react"; +import React from "react"; import { PiggyBank } from "lucide-react"; +import { Alert, Tabs } from "antd"; -import { AreaChart, DonutChart } from "@/components/shared/charts"; -import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { userDailyActivityCall } from "@/components/networking"; -import { DailyData, SpendMetrics } from "@/components/UsagePage/types"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { all_admin_roles } from "@/utils/roles"; -import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity"; +import UsageTab from "./UsageTab"; +import PromptCompressionTab from "./PromptCompressionTab"; +import AutorouterTab from "./AutorouterTab"; +import PromptCachingTab from "./PromptCachingTab"; interface CostOptimizationViewProps { accessToken: string | null; @@ -18,138 +15,62 @@ interface CostOptimizationViewProps { userRole: string; } -type DateRange = { from?: Date; to?: Date }; - -const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; - -const usd = (value: number): string => { - const decimals = value > 0 && value < 1 ? 4 : 2; - return `$${formatNumberWithCommas(value, decimals)}`; -}; - -const shortDate = (iso: string): string => - new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); - -const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; -const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; -const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; - -const SummaryCard = ({ label, value, hint }: { label: string; value: string; hint?: string }) => ( - - - {label} - - -

{value}

- {hint &&

{hint}

} - - -); - const CostOptimizationView: React.FC = ({ accessToken, userId, userRole }) => { - const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []); - const initialTo = useMemo(() => new Date(), []); - const [dateValue, setDateValue] = useState({ from: initialFrom, to: initialTo }); - - const startTime = dateValue.from ?? null; - const endTime = dateValue.to ?? null; - const isAdmin = all_admin_roles.includes(userRole); - const effectiveUserId = isAdmin ? null : userId; - - const { data, loading, isFetchingMore } = usePaginatedDailyActivity({ - fetchFn: userDailyActivityCall, - args: [accessToken, startTime, endTime, effectiveUserId], - enabled: !!accessToken && !!startTime && !!endTime, - }); - - const results = data.results as DailyData[]; - - const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]); - const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]); - const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]); - const totalSaved = compressionTotal + cachingTotal; - - const overTime = useMemo( - () => - results.map((d) => ({ - date: shortDate(d.date), - Compression: compressionOf(d.metrics), - "Prompt caching": cachingOf(d.metrics), - })), - [results], - ); - - const byDriver = useMemo( - () => - [ - { driver: "Compression", usd: compressionTotal }, - { driver: "Prompt caching", usd: cachingTotal }, - ].filter((d) => d.usd > 0), - [compressionTotal, cachingTotal], - ); + const items = [ + { + key: "usage", + label: "Usage", + children: , + }, + { + key: "compression", + label: "Prompt Compression", + children: , + }, + { + key: "autorouter", + label: "Autorouter", + children: , + }, + { + key: "caching", + label: "Prompt Caching", + children: , + }, + ]; return (
-
-
-
- -

Cost Optimization

-
-

- Money saved by prompt compression and prompt caching across your requests -

+
+
+ +

Cost Optimization

- setDateValue(v)} /> +

+ Track and configure the mechanisms that save you money: prompt compression, prompt caching, and auto routing +

-
- - - -
+ + Have feedback? Join the discussion{" "} + + here + + + } + /> -
- - - Savings over time - - - - - - - - Savings by driver - - - - - -
+
); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx new file mode 100644 index 00000000000..e6f73088824 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx @@ -0,0 +1,52 @@ +"use client"; + +import React, { useCallback, useEffect, useState } from "react"; + +import { getGeneralSettingsCall } from "@/components/networking"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { + PromptCachingPanel, + generalSettingsItem, +} from "@/app/(dashboard)/router-settings/_components/general_settings"; + +interface PromptCachingTabProps { + accessToken: string | null; +} + +const PromptCachingTab: React.FC = ({ accessToken }) => { + const [settings, setSettings] = useState([]); + + const loadSettings = useCallback(() => { + if (!accessToken) { + return; + } + getGeneralSettingsCall(accessToken) + .then((data: generalSettingsItem[]) => setSettings(data)) + .catch((error) => { + console.error("Failed to load prompt caching settings:", error); + NotificationsManager.fromBackend("Failed to load prompt caching settings"); + }); + }, [accessToken]); + + useEffect(() => { + loadSettings(); + }, [loadSettings]); + + const handleChange = (fieldName: string, newValue: unknown) => { + setSettings((prev) => + prev.map((setting) => (setting.field_name === fieldName ? { ...setting, field_value: newValue } : setting)), + ); + }; + + if (!accessToken) { + return null; + } + + return ( +
+ +
+ ); +}; + +export default PromptCachingTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCompressionTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCompressionTab.tsx new file mode 100644 index 00000000000..071ad1d6bd5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCompressionTab.tsx @@ -0,0 +1,176 @@ +"use client"; + +import React, { useCallback, useEffect, useState } from "react"; +import { Button, Form, Input, Switch } from "antd"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { createGuardrailCall, getGuardrailsList } from "@/components/networking"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { + buildCompressionGuardrailPayload, + compressionGuardrailsOf, + GuardrailListItem, + GuardrailListResponse, +} from "./helpers"; + +interface PromptCompressionTabProps { + accessToken: string | null; +} + +interface CompressionFormValues { + name: string; + apiBase: string; + defaultOn: boolean; +} + +const PromptCompressionTab: React.FC = ({ accessToken }) => { + const [form] = Form.useForm(); + const [guardrails, setGuardrails] = useState([]); + const [isLoading, setIsLoading] = useState(true); + const [isSaving, setIsSaving] = useState(false); + + const loadGuardrails = useCallback(() => { + if (!accessToken) { + return; + } + getGuardrailsList(accessToken) + .then((response) => setGuardrails(compressionGuardrailsOf(response as GuardrailListResponse))) + .catch((error) => { + console.error("Failed to load compression guardrails:", error); + NotificationsManager.fromBackend("Failed to load compression guardrails"); + }) + .finally(() => setIsLoading(false)); + }, [accessToken]); + + useEffect(() => { + loadGuardrails(); + }, [loadGuardrails]); + + const handleAdd = async (values: CompressionFormValues) => { + if (!accessToken) { + return; + } + setIsSaving(true); + try { + await createGuardrailCall( + accessToken, + buildCompressionGuardrailPayload({ + name: values.name, + apiBase: values.apiBase, + defaultOn: values.defaultOn ?? true, + }), + ); + NotificationsManager.success("Compression guardrail created"); + form.resetFields(); + await loadGuardrails(); + } catch (error) { + console.error("Failed to create compression guardrail:", error); + NotificationsManager.fromBackend("Failed to create compression guardrail"); + } finally { + setIsSaving(false); + } + }; + + return ( +
+ + + Headroom prompt compression + + +

+ Headroom is a native LiteLLM guardrail that compresses your prompts before they reach the model, so you pay + for fewer input tokens. The tokens it removes are priced and shown on the Usage tab as compression savings.{" "} + + Headroom setup docs + +

+ {isLoading &&

Loading...

} + {!isLoading && guardrails.length === 0 && ( +

+ No prompt compression guardrails configured yet. Add one below to start saving on input tokens +

+ )} + {!isLoading && guardrails.length > 0 && ( +
    + {guardrails.map((guardrail) => ( +
  • +
    +

    {guardrail.guardrail_name}

    +

    {guardrail.litellm_params?.api_base ?? ""}

    +
    + + {guardrail.litellm_params?.default_on ? "Always on" : "Opt-in"} + +
    • + ))} +
+ )} + + + + + + Add Headroom compression guardrail + + +
+ + + + + + + + + +
+

+ Applying compression to all requests is available to all users. Enabling it selectively per key or team + is a LiteLLM Enterprise feature. Get a trial key{" "} + + here + +

+
+
+ +
+
+ + +
+ ); +}; + +export default PromptCompressionTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx new file mode 100644 index 00000000000..048d9a34d31 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx @@ -0,0 +1,108 @@ +import { render } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; + +const mockUsePaginatedDailyActivity = vi.fn(); + +vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({ + usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args), +})); + +vi.mock("@/components/networking", () => ({ + userDailyActivityCall: vi.fn(), +})); + +vi.mock("@/components/shared/advanced_date_picker", () => ({ + __esModule: true, + default: () =>
, +})); + +vi.mock("@/components/shared/charts", () => ({ + AreaChart: ({ data, categories }: { data: unknown; categories: string[] }) => ( +
+ ), + DonutChart: ({ data, label }: { data: unknown; label: string }) => ( +
+ ), +})); + +import UsageTab from "./UsageTab"; + +const baseMetrics = (overrides: Partial): SpendMetrics => ({ + spend: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + api_requests: 0, + successful_requests: 0, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + ...overrides, +}); + +const day = (date: string, metrics: Partial): DailyData => ({ + date, + metrics: baseMetrics(metrics), + breakdown: { + models: {}, + model_groups: {}, + mcp_servers: {}, + providers: {}, + api_keys: {}, + entities: {}, + }, +}); + +const renderWith = (results: DailyData[]) => { + mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false }); + return render(); +}; + +describe("UsageTab", () => { + it("sums compression and caching dollars across days into the summary cards", () => { + const { getByText } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.04, + prompt_caching_savings_spend: 0.006, + compression_saved_tokens: 40000, + }), + day("2026-07-13", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.01, + compression_saved_tokens: 100000, + }), + ]); + + expect(getByText("$0.1560")).toBeInTheDocument(); + expect(getByText("$0.1400")).toBeInTheDocument(); + expect(getByText("$0.0160")).toBeInTheDocument(); + expect(getByText("140,000 tokens compressed")).toBeInTheDocument(); + }); + + it("builds a per-day time series and per-driver donut from the daily rows", () => { + const { getByTestId } = renderWith([ + day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }), + day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }), + ]); + + const series = JSON.parse(getByTestId("area-chart").getAttribute("data-series") ?? "[]"); + expect(series).toHaveLength(2); + expect(series[0]).toMatchObject({ Compression: 0.04, "Prompt caching": 0.006 }); + expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.01 }); + + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices).toEqual([ + { driver: "Compression", usd: expect.closeTo(0.14, 5) }, + { driver: "Prompt caching", usd: expect.closeTo(0.016, 5) }, + ]); + }); + + it("omits a driver slice when that driver has no savings", () => { + const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); + + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx new file mode 100644 index 00000000000..8e6fc40b5ad --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx @@ -0,0 +1,184 @@ +"use client"; + +import React, { useMemo, useState } from "react"; +import { Collapse } from "antd"; + +import { AreaChart, DonutChart } from "@/components/shared/charts"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { userDailyActivityCall } from "@/components/networking"; +import { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { all_admin_roles } from "@/utils/roles"; +import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity"; + +interface UsageTabProps { + accessToken: string | null; + userId: string | null; + userRole: string; +} + +type DateRange = { from?: Date; to?: Date }; + +const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; + +const usd = (value: number): string => { + const decimals = value > 0 && value < 1 ? 4 : 2; + return `$${formatNumberWithCommas(value, decimals)}`; +}; + +const shortDate = (iso: string): string => + new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); + +const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; +const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; +const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; + +const MethodologyNote = () => ( + How savings are calculated, + children: ( +
+

+ Savings are computed for each request when it is logged, using the provider's reported usage and the + model's pricing, then summed into a daily rollup. Totals below are read from that rollup over the + selected date range, so the numbers never require a scan of raw request logs. +

+

+ Compression savings are the tokens Headroom removed before the call, priced at the model's input + rate: compression_saved_tokens * input_cost_per_token +

+

+ Prompt caching savings are the tokens the provider served from cache (Anthropic{" "} + cache_read_input_tokens, or OpenAI-style prompt_tokens_details.cached_tokens), + priced at the discount between the normal input rate and the cache-read rate:{" "} + cache_read_input_tokens * max(input_cost_per_token - cache_read_input_token_cost, 0) +

+

+ Total saved is the sum of both drivers. Models without a separate cache-read price in the pricing map + contribute zero caching savings rather than erroring. +

+
+ ), + }, + ]} + /> +); + +const SummaryCard = ({ label, value, hint }: { label: string; value: string; hint?: string }) => ( + + + {label} + + +

{value}

+ {hint &&

{hint}

} + + +); + +const UsageTab: React.FC = ({ accessToken, userId, userRole }) => { + const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []); + const initialTo = useMemo(() => new Date(), []); + const [dateValue, setDateValue] = useState({ from: initialFrom, to: initialTo }); + + const startTime = dateValue.from ?? null; + const endTime = dateValue.to ?? null; + const isAdmin = all_admin_roles.includes(userRole); + const effectiveUserId = isAdmin ? null : userId; + + const { data, loading, isFetchingMore } = usePaginatedDailyActivity({ + fetchFn: userDailyActivityCall, + args: [accessToken, startTime, endTime, effectiveUserId], + enabled: !!accessToken && !!startTime && !!endTime, + }); + + const results = data.results as DailyData[]; + + const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]); + const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]); + const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]); + const totalSaved = compressionTotal + cachingTotal; + + const overTime = useMemo( + () => + results.map((d) => ({ + date: shortDate(d.date), + Compression: compressionOf(d.metrics), + "Prompt caching": cachingOf(d.metrics), + })), + [results], + ); + + const byDriver = useMemo( + () => + [ + { driver: "Compression", usd: compressionTotal }, + { driver: "Prompt caching", usd: cachingTotal }, + ].filter((d) => d.usd > 0), + [compressionTotal, cachingTotal], + ); + + return ( +
+
+ + setDateValue(v)} /> +
+ +
+ + + +
+ +
+ + + Savings over time + + + + + + + + Savings by driver + + + + + +
+
+ ); +}; + +export default UsageTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.test.ts new file mode 100644 index 00000000000..239cded0815 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; + +import { buildCompressionGuardrailPayload, compressionGuardrailsOf } from "./helpers"; + +describe("compressionGuardrailsOf", () => { + it("keeps only headroom-provider guardrails and drops others", () => { + const filtered = compressionGuardrailsOf({ + guardrails: [ + { guardrail_id: "1", guardrail_name: "headroom-compression", litellm_params: { guardrail: "headroom" } }, + { guardrail_id: "2", guardrail_name: "pii-masker", litellm_params: { guardrail: "presidio" } }, + { guardrail_id: "3", guardrail_name: "no-params", litellm_params: null }, + ], + }); + + expect(filtered.map((g) => g.guardrail_id)).toEqual(["1"]); + }); +}); + +describe("buildCompressionGuardrailPayload", () => { + it("builds a headroom guardrail payload with trimmed fields", () => { + const payload = buildCompressionGuardrailPayload({ + name: " headroom-compression ", + apiBase: " https://compress ", + defaultOn: false, + }); + + expect(payload).toEqual({ + guardrail_name: "headroom-compression", + litellm_params: { + guardrail: "headroom", + mode: "pre_call", + api_base: "https://compress", + default_on: false, + }, + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.ts new file mode 100644 index 00000000000..7c8c92f8890 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.ts @@ -0,0 +1,39 @@ +export interface GuardrailLitellmParams { + guardrail?: string | null; + api_base?: string | null; + default_on?: boolean | null; +} + +export interface GuardrailListItem { + guardrail_id: string; + guardrail_name: string | null; + litellm_params?: GuardrailLitellmParams | null; +} + +export interface GuardrailListResponse { + guardrails?: GuardrailListItem[]; +} + +export const COMPRESSION_GUARDRAIL_PROVIDER = "headroom"; + +export const isCompressionGuardrail = (guardrail: GuardrailListItem): boolean => + (guardrail.litellm_params?.guardrail ?? "").toLowerCase() === COMPRESSION_GUARDRAIL_PROVIDER; + +export const compressionGuardrailsOf = (response: GuardrailListResponse): GuardrailListItem[] => + (response.guardrails ?? []).filter(isCompressionGuardrail); + +export interface CompressionGuardrailInput { + name: string; + apiBase: string; + defaultOn: boolean; +} + +export const buildCompressionGuardrailPayload = (input: CompressionGuardrailInput): Record => ({ + guardrail_name: input.name.trim(), + litellm_params: { + guardrail: COMPRESSION_GUARDRAIL_PROVIDER, + mode: "pre_call", + api_base: input.apiBase.trim(), + default_on: input.defaultOn, + }, +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 1e8658d5104..dda7a23a8d4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -33,7 +33,7 @@ interface GeneralSettingsPageProps { userID: string | null; } -interface generalSettingsItem { +export interface generalSettingsItem { field_name: string; field_type: string; field_value: any; @@ -90,7 +90,7 @@ const SettingValueEditor: React.FC<{ return null; }; -const PromptCachingPanel: React.FC<{ +export const PromptCachingPanel: React.FC<{ accessToken: string; settings: generalSettingsItem[]; onChange: (fieldName: string, newValue: any) => void; From 8d96e959db0e251d6fbda9d74360617d18aff806 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 20 Jul 2026 08:47:39 -0700 Subject: [PATCH 3/3] test(e2e): guard destructive spend-log truncate behind an explicit opt-in (#33751) tests/e2e/conftest.py's pytest_sessionfinish truncated LiteLLM_SpendLogs against whatever DATABASE_URL resolved to, gated only by "an e2e test body ran". Pointed at a shared or staging DB, a routine local run wiped real spend data. It also reached the truncate helper through a sys.path.insert into quota_management/spend_tracking/spend_e2e_client.py, a cross-suite import-by-path hack it then unwound in a finally. The cleanup now routes through a new run_spend_log_cleanup in a top-level tests/e2e/e2e_db.py, which fires the destructive truncate only when the operator set E2E_RESET_SPEND_LOGS=1 and an e2e test actually ran. Any other value (unset, 0, true, empty) leaves the DB untouched, so presence of the variable alone or a test run alone never arms the truncate. The decision plus the injectable truncate callable live in that pure helper, and conftest is a thin adapter that supplies os.environ.get(...), the session stash, and reset_spend_logs. reset_spend_logs itself moved from spend_e2e_client.py into e2e_db.py (implementation unchanged), sitting next to e2e_config and lifecycle so both conftest and any suite import it by name; the sys.path munging is gone. Nothing else imported reset_spend_logs, so spend_e2e_client.py drops the definition, its __all__ entry, and the now-unused os import. --- tests/e2e/conftest.py | 35 +++++------- tests/e2e/e2e_db.py | 56 +++++++++++++++++++ .../spend_tracking/spend_e2e_client.py | 19 ------- 3 files changed, 69 insertions(+), 41 deletions(-) create mode 100644 tests/e2e/e2e_db.py diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 5347fffca4d..609da6a9b07 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -14,14 +14,14 @@ shared fixtures build on it. """ import functools -import sys +import os from collections.abc import Iterator -from pathlib import Path import pytest import requests from e2e_config import CONTROL_PLANE_BASE_URL, PROXY_BASE_URL +from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager from proxy_client import ProxyClient, build_proxy_client @@ -107,26 +107,17 @@ def pytest_runtest_call(item: pytest.Item) -> None: def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: - """Once the whole e2e session is done (all suites), truncate the spend logs so - the DB doesn't accumulate test rows. Sessions where no e2e test body ran leave - the DB alone so a `DATABASE_URL` pointing at a shared instance is never wiped - without an e2e run. Best-effort: a cleanup failure (no DB reachable) must not - fail the run. The spend_tracking dir goes on sys.path only for this import and - is removed after, so a broader `pytest tests/` run is not left with a mutated - path.""" - if not session.stash.get(_E2E_TEST_RAN, False): - return - spend_dir = str(Path(__file__).parent / "quota_management" / "spend_tracking") - sys.path.insert(0, spend_dir) - try: - from spend_e2e_client import reset_spend_logs # pyright: ignore - - reset_spend_logs() - except Exception as exc: # noqa: BLE001 - cleanup is best-effort - print(f"spend-log cleanup best-effort failed: {exc}") - finally: - if spend_dir in sys.path: - sys.path.remove(spend_dir) + """Once the whole e2e session is done (all suites), optionally truncate the + spend logs so the DB doesn't accumulate test rows. The truncate is destructive + and irreversible, so it runs only when the operator explicitly opts in + (`E2E_RESET_SPEND_LOGS=1`) and an e2e test body actually ran; otherwise a + `DATABASE_URL` pointing at a shared or staging instance is left untouched. + Best-effort: a cleanup failure (no DB reachable) must not fail the run.""" + run_spend_log_cleanup( + opt_in=os.environ.get(RESET_OPT_IN_ENV), + e2e_test_ran=session.stash.get(_E2E_TEST_RAN, False), + truncate=reset_spend_logs, + ) @pytest.fixture(scope="session") diff --git a/tests/e2e/e2e_db.py b/tests/e2e/e2e_db.py new file mode 100644 index 00000000000..439dd519c03 --- /dev/null +++ b/tests/e2e/e2e_db.py @@ -0,0 +1,56 @@ +"""Shared, destructive DB helpers for the e2e harness. + +Kept at the top level next to e2e_config and lifecycle so every suite imports it +by name (`from e2e_db import ...`); no suite reaches into another's directory by +mutating sys.path. + +reset_spend_logs truncates LiteLLM_SpendLogs and cannot be undone, so the +session-finish cleanup routes through run_spend_log_cleanup, which fires the +truncate only on an explicit operator opt-in. "An e2e test ran" is necessary but +never sufficient: a DATABASE_URL pointing at a shared or staging instance must +not be wiped by a routine local run that merely exercised a test. +""" + +import os +from collections.abc import Callable + +RESET_OPT_IN_ENV = "E2E_RESET_SPEND_LOGS" + + +def run_spend_log_cleanup( + *, opt_in: str | None, e2e_test_ran: bool, truncate: Callable[[], None] +) -> bool: + """Invoke `truncate` iff the destructive spend-log reset is both opted into + and warranted, returning whether the truncate was attempted. + + The truncate fires only when the opt-in value is exactly "1" AND an e2e test + body actually ran. Any other opt-in value (unset, "0", "true", "") leaves the + DB untouched, so the destructive path is never armed by the env var's mere + presence or by a test run on its own. Best-effort: a truncate failure is + swallowed so cleanup never fails the session, so the returned bool reports + that the reset was attempted, not that the DB call succeeded. + """ + if opt_in != "1" or not e2e_test_ran: + return False + try: + truncate() + except Exception as exc: # noqa: BLE001 - cleanup is best-effort + print(f"spend-log cleanup best-effort failed: {exc}") + return True + + +def reset_spend_logs() -> None: + """Truncate LiteLLM_SpendLogs for a clean slate. No proxy endpoint deletes + spend logs (/global/spend/reset keeps them), so go to the DB directly. Uses + DATABASE_URL (default: the local docker postgres on its mapped host port; the + in-container `@db` host isn't resolvable from the host, so default to + localhost). + """ + import psycopg + + url = os.environ.get( + "DATABASE_URL", + "postgresql://llmproxy:dbpassword9090@localhost:5432/litellm", + ) + with psycopg.connect(url) as conn: + _ = conn.execute('TRUNCATE TABLE "LiteLLM_SpendLogs"') diff --git a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py index 0d49869aa91..26860212fa3 100644 --- a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py +++ b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py @@ -11,7 +11,6 @@ helpers from one place. from __future__ import annotations -import os import time from collections.abc import Callable from dataclasses import dataclass @@ -50,7 +49,6 @@ from models import ( __all__ = [ "SpendClient", "build_client", - "reset_spend_logs", "unique_marker", "unwrap", "is_ok", @@ -59,23 +57,6 @@ __all__ = [ ] -def reset_spend_logs() -> None: - """Truncate LiteLLM_SpendLogs for a clean slate. No proxy endpoint deletes - spend logs (/global/spend/reset keeps them), so go to the DB directly. Uses - DATABASE_URL (default: the local docker postgres on its mapped host port; note - the in-container `@db` host isn't resolvable from the host, so default to - localhost). - """ - import psycopg - - url = os.environ.get( - "DATABASE_URL", - "postgresql://llmproxy:dbpassword9090@localhost:5432/litellm", - ) - with psycopg.connect(url) as conn: - _ = conn.execute('TRUNCATE TABLE "LiteLLM_SpendLogs"') - - def _chat_body( model: str, content: str,