Merge pull request #41294 from BerriAI/litellm_usage_export_gating

fix(ui): block usage export and flag the range when a spend page fails
This commit is contained in:
ryan-crabbe-berri 2026-09-15 17:19:38 -07:00 committed by GitHub
commit f34a6eda92
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 352 additions and 12 deletions

View file

@ -71,6 +71,7 @@ const renderWith = (results: DailyData[], overrides: Partial<DailyActivityRange>
isFetchingMore: false,
progress: { currentPage: 1, totalPages: 1 },
cancelled: false,
failed: false,
cancel: vi.fn(),
...overrides,
}}

View file

@ -87,6 +87,7 @@ const CostOptimizationView: React.FC<CostOptimizationViewProps> = ({ accessToken
<PaginationStatusAlerts
isFetchingMore={activity.isFetchingMore}
cancelled={activity.cancelled}
failed={activity.failed}
progress={activity.progress}
cancel={activity.cancel}
/>

View file

@ -35,6 +35,7 @@ describe("PromptCachingTab", () => {
isFetchingMore: false,
progress: { currentPage: 1, totalPages: 1 },
cancelled: false,
failed: false,
cancel: vi.fn(),
};
render(<PromptCachingTab accessToken="test-token" activity={activity} />);

View file

@ -123,6 +123,7 @@ const renderWith = (results: DailyData[], options: RenderOptions = {}) => {
isFetchingMore: false,
progress: { currentPage: 1, totalPages: 1 },
cancelled: false,
failed: false,
cancel: vi.fn(),
}}
/>,

View file

@ -20,6 +20,7 @@ export interface DailyActivityRange {
isFetchingMore: boolean;
progress: { currentPage: number; totalPages: number };
cancelled: boolean;
failed: boolean;
cancel: () => void;
}
@ -64,7 +65,7 @@ export const useScopedDailyActivityRange = (
args: [accessToken, startTime, endTime, userId, true, apiKey],
enabled: !!accessToken && !!startTime && !!endTime,
};
const { data, loading, isFetchingMore, progress, cancelled, cancel } =
const { data, loading, isFetchingMore, progress, cancelled, failed, cancel } =
usePaginatedDailyActivity(activityQueryOptions);
return {
@ -75,6 +76,7 @@ export const useScopedDailyActivityRange = (
isFetchingMore,
progress,
cancelled,
failed,
cancel,
};
};

View file

@ -25,6 +25,7 @@ import TeamMultiSelect from "@/components/common_components/team_multi_select";
import UserDropdown from "@/components/common_components/UserDropdown";
import { ActivityMetrics, processActivityData } from "@/components/activity_metrics";
import { UsageExportHeader } from "@/components/EntityUsageExport";
import { getExportBlockedReason } from "@/components/EntityUsageExport/exportBlockedReason";
import type { EntityType } from "@/components/EntityUsageExport/types";
import {
agentDailyActivityCall,
@ -148,6 +149,8 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
isFetchingMore,
progress,
cancelled,
failed,
coversRange,
cancel,
} = usePaginatedDailyActivity({
fetchFn,
@ -163,6 +166,7 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
isFetchingMore: agentIsFetchingMore,
progress: agentProgress,
cancelled: agentCancelled,
failed: agentFailed,
cancel: agentCancel,
} = usePaginatedDailyActivity({
fetchFn: agentDailyActivityCall,
@ -660,11 +664,14 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
{ key: "endpoints", label: "Endpoint Activity", content: <EndpointUsage userSpendData={spendData} /> },
];
const spendFetchState = { coversRange, cancelled, failed };
return (
<div style={{ width: "100%" }} className="relative">
<PaginationStatusAlerts
isFetchingMore={isFetchingMore}
cancelled={cancelled}
failed={failed}
progress={progress}
cancel={cancel}
/>
@ -672,6 +679,7 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
<PaginationStatusAlerts
isFetchingMore={agentIsFetchingMore}
cancelled={agentCancelled}
failed={agentFailed}
progress={agentProgress}
cancel={agentCancel}
subject="agent data"
@ -689,6 +697,7 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
onFiltersChange={setSelectedTags}
filterOptions={getAllTags() || undefined}
teams={teams || []}
exportBlockedReason={getExportBlockedReason(spendFetchState)}
/>
<Tabs defaultValue={tabs[0].key}>
<TabsList className="mt-1">

View file

@ -30,6 +30,7 @@ import { ActivityMetrics, processActivityData } from "@/components/activity_metr
import CloudZeroExportModal from "@/components/cloudzero_export_modal";
import UserDropdown from "@/components/common_components/UserDropdown";
import EntityUsageExportModal from "@/components/EntityUsageExport";
import { getExportBlockedReason } from "@/components/EntityUsageExport/exportBlockedReason";
import KeyActivityPanel from "@/components/UsagePage/components/KeyActivityPanel";
import { Team } from "@/components/key_team_helpers/key_list";
import {
@ -249,6 +250,15 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
const loading = aggregatedLoading || paginatedResult.loading;
// Read through the same range stamp as the tiles, so the export is blocked from the first
// render of a new range rather than from whenever the fetch effect gets around to running.
const spendFetchState = {
coversRange: activeAggregated !== null || paginatedResult.coversRange,
cancelled: paginatedResult.cancelled,
failed: paginatedResult.failed,
};
const exportBlockedReason = getExportBlockedReason(spendFetchState);
// Clear isDateChanging when paginated data starts arriving
useEffect(() => {
if (aggregatedFailed && !paginatedResult.loading && paginatedResult.data.results.length > 0) {
@ -489,6 +499,7 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
<PaginationStatusAlerts
isFetchingMore={paginatedResult.isFetchingMore}
cancelled={paginatedResult.cancelled}
failed={paginatedResult.failed}
progress={paginatedResult.progress}
cancel={paginatedResult.cancel}
/>
@ -525,10 +536,16 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
<Sparkles />
Ask AI
</Button>
<Button variant="outline" onClick={() => setIsGlobalExportModalOpen(true)}>
<Download />
Export Data
</Button>
<span title={exportBlockedReason}>
<Button
variant="outline"
disabled={exportBlockedReason !== undefined}
onClick={() => setIsGlobalExportModalOpen(true)}
>
<Download />
Export Data
</Button>
</span>
</div>
</div>
{/* Cost Panel */}

View file

@ -156,3 +156,164 @@ describe("usePaginatedDailyActivity page accumulation", () => {
expect(result.current.data.metadata.total_spend).toBe(5.5);
});
});
describe("usePaginatedDailyActivity failure reporting", () => {
const firstPage = { results: [dayOf("2026-08-16", 2)], metadata: { total_pages: 3, page: 1, total_spend: 2 } };
const start = new Date("2026-08-10");
const end = new Date("2026-08-17");
it("reports a failed range so partial totals cannot pass as the whole range", async () => {
const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
const fetchFn = vi.fn((_token: string, _start: Date, _end: Date, page: number) =>
page === 1 ? Promise.resolve(firstPage) : Promise.reject(new Error("page 2 never came back")),
);
const { result } = renderHook(() =>
usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled: true }),
);
await waitFor(() => expect(result.current.failed).toBe(true), { timeout: 5000 });
expect(result.current.isFetchingMore).toBe(false);
expect(result.current.loading).toBe(false);
expect(result.current.data.metadata.total_spend).toBe(2);
consoleError.mockRestore();
});
it("reports no pages loaded when the very first request is what failed", async () => {
const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
const fetchFn = vi.fn(() => Promise.reject(new Error("page 1 never came back")));
const { result } = renderHook(() =>
usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled: true }),
);
await waitFor(() => expect(result.current.failed).toBe(true), { timeout: 5000 });
expect(result.current.progress).toEqual({ currentPage: 0, totalPages: 0 });
consoleError.mockRestore();
});
it("stays unfailed when every page arrives", async () => {
const pages = [
firstPage,
{ results: [dayOf("2026-08-15", 1)], metadata: { total_pages: 2, page: 2, total_spend: 1 } },
];
const fetchFn = vi.fn((_token: string, _start: Date, _end: Date, page: number) =>
Promise.resolve({ ...pages[page - 1], metadata: { ...pages[page - 1].metadata, total_pages: 2 } }),
);
const { result } = renderHook(() =>
usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled: true }),
);
await waitFor(() => expect(result.current.data.metadata.page).toBe(2), { timeout: 5000 });
expect(result.current.failed).toBe(false);
});
it("clears the failure when a new range is requested, so the banner cannot outlive it", async () => {
const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
const fetchFn = vi.fn((...callArgs: unknown[]) => {
const [, , , page, filter] = callArgs as [string, Date, Date, number, string | null];
if (filter !== "broken")
return Promise.resolve({ ...firstPage, metadata: { ...firstPage.metadata, total_pages: 1 } });
if (page === 1) return Promise.resolve({ ...firstPage, metadata: { ...firstPage.metadata, total_pages: 2 } });
return Promise.reject(new Error("page 2 never came back"));
});
const { result, rerender } = renderHook(
({ filter }: { filter: string | null }) =>
usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, filter], enabled: true }),
{ initialProps: { filter: "broken" as string | null } },
);
await waitFor(() => expect(result.current.failed).toBe(true), { timeout: 5000 });
rerender({ filter: "healthy" });
await waitFor(() => expect(result.current.failed).toBe(false), { timeout: 5000 });
consoleError.mockRestore();
});
});
describe("usePaginatedDailyActivity range coverage", () => {
const start = new Date("2026-08-10");
const end = new Date("2026-08-17");
const singlePage = { results: [dayOf("2026-08-16", 2)], metadata: { total_pages: 1, page: 1, total_spend: 2 } };
it("does not cover the range while the hook is disabled", () => {
const fetchFn = vi.fn(() => Promise.resolve(singlePage));
const { result } = renderHook(() =>
usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled: false }),
);
expect(result.current.coversRange).toBe(false);
expect(fetchFn).not.toHaveBeenCalled();
});
it("covers the range only once every page of it has landed", async () => {
const pages = [
{ results: [dayOf("2026-08-16", 2)], metadata: { total_pages: 2, page: 1, total_spend: 2 } },
{ results: [dayOf("2026-08-15", 1)], metadata: { total_pages: 2, page: 2, total_spend: 1 } },
];
const fetchFn = vi.fn((_token: string, _start: Date, _end: Date, page: number) => Promise.resolve(pages[page - 1]));
const { result } = renderHook(() =>
usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled: true }),
);
expect(result.current.coversRange).toBe(false);
await waitFor(() => expect(result.current.coversRange).toBe(true), { timeout: 5000 });
});
it("never reports a range as covered while the data on screen is empty", async () => {
// Disabling the hook empties the data. Re-enabling it asks for the same args the last
// completed fetch used, so coverage that survives the disable would vouch for nothing.
const seen: Array<{ coversRange: boolean; rows: number }> = [];
const fetchFn = vi.fn(() => Promise.resolve(singlePage));
const { result, rerender } = renderHook(
({ enabled }: { enabled: boolean }) => {
const activity = usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled });
seen.push({ coversRange: activity.coversRange, rows: activity.data.results.length });
return activity;
},
{ initialProps: { enabled: true } },
);
await waitFor(() => expect(result.current.coversRange).toBe(true), { timeout: 5000 });
rerender({ enabled: false });
rerender({ enabled: true });
await waitFor(() => expect(result.current.coversRange).toBe(true), { timeout: 5000 });
expect(seen.filter((render) => render.coversRange && render.rows === 0)).toEqual([]);
});
it("stops covering the range on the very render the args change, not once an effect catches up", async () => {
// The render after a filter change still holds the previous filter's rows, so resetting
// coverage inside the fetch effect would leave a paint where the export reads them as the
// new range. That paint is the whole thing the gate exists to stop.
const seen: Array<{ filter: string; coversRange: boolean }> = [];
const fetchFn = vi.fn(() => Promise.resolve(singlePage));
const { result, rerender } = renderHook(
({ filter }: { filter: string }) => {
const activity = usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, filter], enabled: true });
seen.push({ filter, coversRange: activity.coversRange });
return activity;
},
{ initialProps: { filter: "team-a" } },
);
await waitFor(() => expect(result.current.coversRange).toBe(true), { timeout: 5000 });
rerender({ filter: "team-b" });
const rendersForNewFilter = seen.filter((render) => render.filter === "team-b");
expect(rendersForNewFilter.length).toBeGreaterThan(0);
expect(rendersForNewFilter.map((render) => render.coversRange)).not.toContain(true);
});
});

View file

@ -61,6 +61,8 @@ interface UsePaginatedDailyActivityReturn {
isFetchingMore: boolean;
progress: PaginationProgress;
cancelled: boolean;
failed: boolean;
coversRange: boolean;
cancel: () => void;
}
@ -200,6 +202,8 @@ export function usePaginatedDailyActivity({
totalPages: 0,
});
const [cancelled, setCancelled] = useState(false);
const [failed, setFailed] = useState(false);
const [completedKey, setCompletedKey] = useState<string | null>(null);
const fetchIdRef = useRef(0);
const cancelledRef = useRef(false);
@ -213,6 +217,11 @@ export function usePaginatedDailyActivity({
// Stable serialised key so the effect only re-runs when the arg *values* change.
const argsKey = JSON.stringify(args);
// Stamped like the data itself and compared during render, so the render that follows an arg
// change already reports the new range as uncovered. Clearing it inside the fetch effect would
// be one render too late, leaving a paint where an export reads the previous range's rows.
const coversRange = enabled && completedKey === argsKey;
const cancel = useCallback(() => {
cancelledRef.current = true;
setCancelled(true);
@ -230,12 +239,15 @@ export function usePaginatedDailyActivity({
setIsFetchingMore(false);
setProgress({ currentPage: 0, totalPages: 0 });
setCancelled(false);
setFailed(false);
setCompletedKey(null);
return;
}
const currentFetchId = ++fetchIdRef.current;
cancelledRef.current = false;
setCancelled(false);
setFailed(false);
const isStale = () => fetchIdRef.current !== currentFetchId || cancelledRef.current;
@ -252,7 +264,7 @@ export function usePaginatedDailyActivity({
const currentArgs = argsRef.current;
setLoading(true);
setIsFetchingMore(false);
setProgress({ currentPage: 1, totalPages: 1 });
setProgress({ currentPage: 0, totalPages: 0 });
if (aggregatedFetchFn) {
try {
@ -261,6 +273,7 @@ export function usePaginatedDailyActivity({
setData(aggregated);
setProgress({ currentPage: 1, totalPages: 1 });
setLoading(false);
setCompletedKey(argsKey);
return;
} catch (error) {
if (isStale()) return;
@ -283,6 +296,7 @@ export function usePaginatedDailyActivity({
if (totalPages <= 1) {
setLoading(false);
setCompletedKey(argsKey);
return;
}
@ -328,11 +342,13 @@ export function usePaginatedDailyActivity({
}
setIsFetchingMore(false);
setCompletedKey(argsKey);
} catch (error) {
if (!isStale()) {
console.error("Error fetching daily activity:", error);
setLoading(false);
setIsFetchingMore(false);
setFailed(true);
}
}
};
@ -350,5 +366,5 @@ export function usePaginatedDailyActivity({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [enabled, fetchFn, aggregatedFetchFn, argsKey]);
return { data, loading, isFetchingMore, progress, cancelled, cancel };
return { data, loading, isFetchingMore, progress, cancelled, failed, coversRange, cancel };
}

View file

@ -41,6 +41,27 @@ describe("UsageExportHeader", () => {
expect(screen.getByTestId("export-modal")).toBeInTheDocument();
});
it("blocks the export while the data on screen does not cover the range", async () => {
const user = userEvent.setup();
renderWithProviders(
<UsageExportHeader
{...defaultProps}
exportBlockedReason="Spend data is still loading, so an export would under-report. Wait for it to finish."
/>,
);
const exportButton = screen.getByRole("button", { name: /export data/i });
expect(exportButton).toBeDisabled();
await user.click(exportButton);
expect(screen.queryByTestId("export-modal")).not.toBeInTheDocument();
});
it("explains why the export is blocked on hover", () => {
renderWithProviders(<UsageExportHeader {...defaultProps} exportBlockedReason="Spend data is still loading" />);
expect(screen.getByTitle("Spend data is still loading")).toBeInTheDocument();
});
it("should close the export modal when onClose is called", async () => {
const user = userEvent.setup();
renderWithProviders(<UsageExportHeader {...defaultProps} />);

View file

@ -34,6 +34,7 @@ interface UsageExportHeaderProps {
customTitle?: string;
compactLayout?: boolean;
teams?: Team[];
exportBlockedReason?: string;
}
const UsageExportHeader: React.FC<UsageExportHeaderProps> = ({
@ -50,6 +51,7 @@ const UsageExportHeader: React.FC<UsageExportHeaderProps> = ({
customTitle,
compactLayout = false,
teams = [],
exportBlockedReason,
}) => {
const anchor = useComboboxAnchor();
const [isExportModalOpen, setIsExportModalOpen] = useState(false);
@ -121,10 +123,12 @@ const UsageExportHeader: React.FC<UsageExportHeaderProps> = ({
)}
<div className="justify-self-end">
<Button onClick={() => setIsExportModalOpen(true)}>
<Download />
Export Data
</Button>
<span title={exportBlockedReason}>
<Button disabled={exportBlockedReason !== undefined} onClick={() => setIsExportModalOpen(true)}>
<Download />
Export Data
</Button>
</span>
</div>
</div>
</div>

View file

@ -0,0 +1,34 @@
import { describe, expect, it } from "vitest";
import { getExportBlockedReason, type UsageFetchState } from "./exportBlockedReason";
const state = (overrides: Partial<UsageFetchState> = {}): UsageFetchState => ({
coversRange: true,
cancelled: false,
failed: false,
...overrides,
});
describe("getExportBlockedReason", () => {
it("lets the export through once the data on screen covers the range", () => {
expect(getExportBlockedReason(state())).toBeUndefined();
});
it("blocks whenever the data on screen does not cover the range, which is when a CSV silently under-reports", () => {
expect(getExportBlockedReason(state({ coversRange: false }))).toMatch(/still loading/i);
});
it("blocks after a stopped fetch and says a reload is what fixes it", () => {
const reason = getExportBlockedReason(state({ coversRange: false, cancelled: true }));
expect(reason).toMatch(/stopped/i);
expect(reason).toMatch(/reload/i);
});
it("blocks after a failed page and names the failure rather than the stop", () => {
const reason = getExportBlockedReason(state({ coversRange: false, failed: true, cancelled: true }));
expect(reason).toMatch(/failed to load/i);
expect(reason).not.toMatch(/stopped/i);
});
});

View file

@ -0,0 +1,13 @@
export interface UsageFetchState {
coversRange: boolean;
cancelled: boolean;
failed: boolean;
}
export const getExportBlockedReason = ({ coversRange, cancelled, failed }: UsageFetchState): string | undefined => {
if (failed) return "Some spend data failed to load, so an export would under-report. Reload the page to try again.";
if (cancelled)
return "Loading was stopped before the whole range arrived, so an export would under-report. Reload the page to load it all.";
if (!coversRange) return "Spend data is still loading, so an export would under-report. Wait for it to finish.";
return undefined;
};

View file

@ -33,6 +33,52 @@ describe("PaginationStatusAlerts", () => {
expect(screen.getByText("Showing partial spend data (7/42 pages loaded)")).toBeInTheDocument();
});
it("calls out a failed page as an error so partial totals do not read as final", () => {
render(
<PaginationStatusAlerts
isFetchingMore={false}
cancelled={false}
failed={true}
progress={{ currentPage: 7, totalPages: 42 }}
cancel={vi.fn()}
/>,
);
expect(
screen.getByText(/Fetching spend data failed, so the totals below cover only 7 of 42 pages of the range/),
).toBeInTheDocument();
});
it("does not claim a page loaded when the very first request is what failed", () => {
render(
<PaginationStatusAlerts
isFetchingMore={false}
cancelled={false}
failed={true}
progress={{ currentPage: 0, totalPages: 0 }}
cancel={vi.fn()}
/>,
);
expect(screen.getByText(/failed before any of it arrived/)).toBeInTheDocument();
expect(screen.queryByText(/pages of the range/)).not.toBeInTheDocument();
});
it("shows only the failure when a stopped fetch also failed", () => {
render(
<PaginationStatusAlerts
isFetchingMore={false}
cancelled={true}
failed={true}
progress={{ currentPage: 7, totalPages: 42 }}
cancel={vi.fn()}
/>,
);
expect(screen.getByText(/Fetching spend data failed/)).toBeInTheDocument();
expect(screen.queryByText(/Showing partial spend data/)).not.toBeInTheDocument();
});
it("names the subject it is fetching", () => {
render(
<PaginationStatusAlerts

View file

@ -9,14 +9,21 @@ interface PaginationStatusAlertsProps {
progress: { currentPage: number; totalPages: number };
cancel: () => void;
subject?: string;
failed?: boolean;
}
const failureMessage = (subject: string, progress: { currentPage: number; totalPages: number }) =>
progress.currentPage === 0
? `Fetching ${subject} failed before any of it arrived, so the totals below are empty rather than final. Reload the page to try again.`
: `Fetching ${subject} failed, so the totals below cover only ${progress.currentPage} of ${progress.totalPages} pages of the range. Reload the page to try again.`;
const PaginationStatusAlerts = ({
isFetchingMore,
cancelled,
progress,
cancel,
subject = "spend data",
failed = false,
}: PaginationStatusAlertsProps) => (
<>
{isFetchingMore && (
@ -38,7 +45,12 @@ const PaginationStatusAlerts = ({
</AlertDescription>
</Alert>
)}
{cancelled && (
{failed && (
<Alert variant="error" className="mb-2">
<AlertDescription className="text-inherit">{failureMessage(subject, progress)}</AlertDescription>
</Alert>
)}
{cancelled && !failed && (
<Alert variant="info" className="mb-2">
<AlertDescription className="text-inherit">
Showing partial {subject} ({progress.currentPage}/{progress.totalPages} pages loaded)

View file

@ -40,6 +40,7 @@ const mockActivity = (
isFetchingMore: false,
progress: { currentPage: 1, totalPages: 1 },
cancelled: false,
failed: false,
cancel: vi.fn(),
...overrides,
});