From f0021859a71f2178abe51b540762148c8823b41c Mon Sep 17 00:00:00 2001 From: jesus-berri Date: Thu, 12 Nov 2026 15:39:44 -0800 Subject: [PATCH 1/9] feat(ui): set per-threshold budget alert recipients on virtual keys (#51061) * feat(ui): set per-threshold budget alert recipients on virtual keys Create Key and Edit Key get the threshold and recipients rows the team member budget alerts already use, writing metadata.max_budget_alert_emails. The helpers and the row editor move to key_team_helpers so the team form and both key forms share them. * test(ui): cover the empty metadata box path and drop a redundant payload case * fix(ui): keep comma-separated string alert recipients stored through the API The backend accepts a string of recipients per threshold, so Edit Key and the team form now load them as rows instead of dropping them and deleting the alert on the next save * fix(ui): keep the stored key alert map on saves that do not edit its rows Edit Key rebuilt max_budget_alert_emails from the rows on every save, so API-written entries the rows cannot show (a null recipient list, a list with a null, thresholds outside 1 to 100) were deleted by unrelated edits. The map is now rebuilt only when the rows were edited and otherwise sent back as stored * fix(ui): load null and mixed alert recipient lists as rows and drop a typed map on Create Key A null recipient list loads as an owner-only row and a mixed list keeps its strings, matching what the backend sends. Create Key now ignores a max_budget_alert_emails typed into the Metadata box, as Edit Key does, and the hint says thresholds replace the default 80% alert * fix(ui): keep Create Key alert rows when the Metadata box holds null * fix(ui): move typed alert maps into rows, keep hidden alert entries, hide alert rows from team admins A max_budget_alert_emails map typed into the Metadata box on Edit Key or Create Key now moves into the threshold rows on blur with a toast, the same way a tags array moves to the Tags field. Entries the rows cannot show stay in the JSON box. When the rows are edited, the saved map keeps the stored entries the rows cannot show (thresholds outside 1 to 100, non-numeric keys, unreadable values). Create Key keeps such typed entries too. Team admins editing a member's key no longer see the alert rows, since that save only accepts budget fields * fix(ui): let the Metadata box own key alert entries the threshold rows cannot show Edit Key now shows max_budget_alert_emails entries outside whole numbers 1 to 100 in the Metadata JSON box and saves them from the box as the admin left it, so they can be seen, edited and removed. Rows still own every entry they can show, and an untouched save sends the stored map unchanged * fix(ui): leave zero-padded alert thresholds to the Metadata box instead of duplicating a row * fix(ui): keep loading zero-padded team member alert thresholds as rows The canonical-key rule applies only to key alert maps, where the Metadata box holds entries the rows cannot show. The team form has no such fallback, so it keeps loading "080" as row 80 as it did before * fix(ui): let the Metadata box own a non-object max_budget_alert_emails value A string, list, number or null stored under max_budget_alert_emails stays in the Edit Key Metadata box, is saved back as the box holds it, and is stored as typed on Create Key. Threshold rows only own an object map, so rows the admin adds replace a non-object value * fix(ui): move typed alert thresholds into the rows when Edit Key saves without a blur Submit now moves a max_budget_alert_emails map typed into the Metadata box into the rows, as it already does for tags, and reads the edited fields at submit time so the moved rows are saved * refactor(ui): name the Edit Key submit handler --- .../src/components/key_info_utils.test.tsx | 13 + .../src/components/key_info_utils.tsx | 13 + .../BudgetAlertEmailsEditor.tsx | 89 +++++++ .../budgetAlertEmails.test.ts | 228 ++++++++++++++++++ .../key_team_helpers/budgetAlertEmails.ts | 143 +++++++++++ .../organisms/createKeyPayload.test.ts | 72 ++++++ .../components/organisms/createKeyPayload.ts | 20 +- .../create_key_button.integration.test.tsx | 64 ++++- .../organisms/create_key_button.tsx | 64 ++++- .../src/components/team/TeamInfo.tsx | 126 +++------- .../team/teamMemberBudgetAlertEmails.test.ts | 94 -------- .../team/teamMemberBudgetAlertEmails.ts | 57 ----- .../templates/KeyEditViewControls.tsx | 41 +++- .../KeyInfoView.handleKeyUpdate.test.tsx | 182 +++++++++++++- .../templates/keyEditFieldNormalizers.test.ts | 44 +++- .../templates/keyEditFieldNormalizers.ts | 27 +++ .../components/templates/keyEditFormValues.ts | 18 +- .../key_edit_view.integration.test.tsx | 183 ++++++++++++++ .../components/templates/key_edit_view.tsx | 42 +++- .../templates/key_info_view.test.tsx | 18 +- .../components/templates/key_info_view.tsx | 37 ++- .../src/i18n/locales/en.json | 11 + .../keyBudgetAlertRecipients.spec.ts | 164 +++++++++++++ .../spend/test_key_budget_alerts.py | 74 ++++++ 24 files changed, 1537 insertions(+), 287 deletions(-) create mode 100644 litellm/ui/litellm-dashboard/src/components/key_team_helpers/BudgetAlertEmailsEditor.tsx create mode 100644 litellm/ui/litellm-dashboard/src/components/key_team_helpers/budgetAlertEmails.test.ts create mode 100644 litellm/ui/litellm-dashboard/src/components/key_team_helpers/budgetAlertEmails.ts delete mode 100644 litellm/ui/litellm-dashboard/src/components/team/teamMemberBudgetAlertEmails.test.ts delete mode 100644 litellm/ui/litellm-dashboard/src/components/team/teamMemberBudgetAlertEmails.ts create mode 100644 tests/e2e/ui/tests/proxy-admin/keyBudgetAlertRecipients.spec.ts create mode 100644 tests/integration/spend/test_key_budget_alerts.py diff --git a/litellm/ui/litellm-dashboard/src/components/key_info_utils.test.tsx b/litellm/ui/litellm-dashboard/src/components/key_info_utils.test.tsx index 490e188e21a..2f73eb4b3df 100644 --- a/litellm/ui/litellm-dashboard/src/components/key_info_utils.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/key_info_utils.test.tsx @@ -4,6 +4,7 @@ import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata, + stripFieldManagedKeysFromMetadata, } from "./key_info_utils"; describe("filterSensitiveMetadata", () => { @@ -57,3 +58,15 @@ describe("stripTagsFromMetadata", () => { expect(input).toEqual(originalCopy); }); }); + +describe("stripFieldManagedKeysFromMetadata", () => { + it.each(["ops@qa.test", ["ops@qa.test"], 80])( + "leaves a max_budget_alert_emails value that is not a threshold map in the Metadata box: %j", + (value) => { + expect(stripFieldManagedKeysFromMetadata({ cost_center: "ops", max_budget_alert_emails: value })).toEqual({ + cost_center: "ops", + max_budget_alert_emails: value, + }); + }, + ); +}); diff --git a/litellm/ui/litellm-dashboard/src/components/key_info_utils.tsx b/litellm/ui/litellm-dashboard/src/components/key_info_utils.tsx index e346878c24e..2f573a8e84e 100644 --- a/litellm/ui/litellm-dashboard/src/components/key_info_utils.tsx +++ b/litellm/ui/litellm-dashboard/src/components/key_info_utils.tsx @@ -2,6 +2,8 @@ * Utility functions for handling key information and metadata */ +import { budgetAlertEmailsMetadata, KEY_MAX_BUDGET_ALERT_EMAILS_KEY } from "./key_team_helpers/budgetAlertEmails"; + // Fields that should be filtered out from metadata display due to security concerns const SENSITIVE_METADATA_FIELDS = ["logging"] as const; @@ -61,3 +63,14 @@ export const stripTagsFromMetadata = (metadata: any) => { const { tags, ...rest } = metadata as Record; return rest; }; + +export const stripFieldManagedKeysFromMetadata = ( + metadata: Record | null | undefined, +): Record | null | undefined => { + const withoutTags: Record | null | undefined = stripTagsFromMetadata(metadata); + if (!withoutTags || typeof withoutTags !== "object") { + return withoutTags; + } + const { [KEY_MAX_BUDGET_ALERT_EMAILS_KEY]: _maxBudgetAlertEmails, ...rest } = withoutTags; + return { ...rest, ...budgetAlertEmailsMetadata({}, KEY_MAX_BUDGET_ALERT_EMAILS_KEY, withoutTags) }; +}; diff --git a/litellm/ui/litellm-dashboard/src/components/key_team_helpers/BudgetAlertEmailsEditor.tsx b/litellm/ui/litellm-dashboard/src/components/key_team_helpers/BudgetAlertEmailsEditor.tsx new file mode 100644 index 00000000000..87d40a55b4b --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/components/key_team_helpers/BudgetAlertEmailsEditor.tsx @@ -0,0 +1,89 @@ +import React from "react"; +import { type ArrayPath, type Control, type FieldPath, type FieldValues, useFieldArray } from "react-hook-form"; +import { CircleMinus, Plus } from "lucide-react"; + +import { Button } from "@/components/ui/button"; +import { Field, FieldLabel } from "@/components/ui/field"; +import { Input } from "@/components/ui/input"; +import { FormField } from "@/components/shared/form/FormField"; +import NumericalInput from "../shared/numerical_input"; +import type { BudgetAlertRow } from "./budgetAlertEmails"; + +export interface BudgetAlertEmailsEditorText { + readonly thresholdPlaceholder: string; + readonly recipientsPlaceholder: string; + readonly removeThreshold: string; + readonly addThreshold: string; +} + +type RowsPath = string extends keyof TFieldValues ? string : ArrayPath; + +interface BudgetAlertEmailsEditorProps { + readonly control: Control; + readonly name: RowsPath; + readonly label: React.ReactNode; + readonly text: BudgetAlertEmailsEditorText; +} + +const EMPTY_ROW: BudgetAlertRow = { threshold: null, emails: "" }; + +export const BudgetAlertEmailsEditor = ({ + control, + name, + label, + text, +}: BudgetAlertEmailsEditorProps) => { + const { fields, append, remove } = useFieldArray({ control, name: name as ArrayPath }); + const rowField = (index: number, column: keyof BudgetAlertRow) => + `${name}.${index}.${column}` as FieldPath; + + return ( + + {label} + {fields.map((row, index) => ( +
+ + {({ ref, value, onChange, ...field }) => ( + ) => + onChange(event.target.value === "" ? null : Number(event.target.value)) + } + placeholder={text.thresholdPlaceholder} + min={1} + max={100} + step={1} + /> + )} + + + {({ ref, value, ...field }) => ( + + )} + + +
+ ))} + + + ); +}; diff --git a/litellm/ui/litellm-dashboard/src/components/key_team_helpers/budgetAlertEmails.test.ts b/litellm/ui/litellm-dashboard/src/components/key_team_helpers/budgetAlertEmails.test.ts new file mode 100644 index 00000000000..23e372fff80 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/components/key_team_helpers/budgetAlertEmails.test.ts @@ -0,0 +1,228 @@ +import { describe, expect, it } from "vitest"; +import { + budgetAlertEmailsFromRows, + budgetAlertEmailsMetadata, + budgetAlertEmailsRowsCanShow, + budgetAlertRowsFromMetadata, + isValidThreshold, + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, + TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY, + teamMemberBudgetAlertSummary, +} from "./budgetAlertEmails"; + +describe("budgetAlertRowsFromMetadata", () => { + it("turns the stored threshold map into rows sorted by threshold", () => { + const metadata = { + team_member_max_budget_alert_emails: { "100": ["finance@example.com", "cto@example.com"], "50": [] }, + }; + expect(budgetAlertRowsFromMetadata(metadata, TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY)).toEqual([ + { threshold: 50, emails: "" }, + { threshold: 100, emails: "finance@example.com, cto@example.com" }, + ]); + }); + + it("drops non-numeric thresholds and non-email recipients instead of crashing", () => { + const metadata = { + team_member_max_budget_alert_emails: { fifty: [], "90": { a: 1 }, "100": ["a@b.c"] }, + }; + expect(budgetAlertRowsFromMetadata(metadata, TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY)).toEqual([ + { threshold: 100, emails: "a@b.c" }, + ]); + }); + + it("loads a null recipient list as an owner-only row and keeps the strings of a mixed list", () => { + const metadata = { max_budget_alert_emails: { "75": null, "80": ["b@example.com", null, 3] } }; + expect(budgetAlertRowsFromMetadata(metadata, KEY_MAX_BUDGET_ALERT_EMAILS_KEY)).toEqual([ + { threshold: 75, emails: "" }, + { threshold: 80, emails: "b@example.com" }, + ]); + }); + + it("lists API-stored string recipients in the team member summary", () => { + expect(teamMemberBudgetAlertSummary({ team_member_max_budget_alert_emails: { "80": "a@x.com, b@x.com" } })).toEqual( + ["80%: member, a@x.com, b@x.com"], + ); + }); + + it("leaves a zero-padded threshold to the Metadata box so it never duplicates a row", () => { + const metadata = { max_budget_alert_emails: { "080": ["padded@example.com"], "80": ["plain@example.com"] } }; + expect(budgetAlertRowsFromMetadata(metadata, KEY_MAX_BUDGET_ALERT_EMAILS_KEY)).toEqual([ + { threshold: 80, emails: "plain@example.com" }, + ]); + }); + + it("still loads a zero-padded team member threshold as a row, since the team form has no Metadata box fallback", () => { + const metadata = { team_member_max_budget_alert_emails: { "080": ["finance@example.com"] } }; + expect(budgetAlertRowsFromMetadata(metadata, TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY)).toEqual([ + { threshold: 80, emails: "finance@example.com" }, + ]); + }); + + it("keeps API-stored comma-separated string recipients, which the backend also accepts", () => { + const metadata = { + max_budget_alert_emails: { "80": "finance@example.com, cto@example.com", "90": ["ops@example.com"] }, + }; + const rows = budgetAlertRowsFromMetadata(metadata, KEY_MAX_BUDGET_ALERT_EMAILS_KEY); + expect(rows).toEqual([ + { threshold: 80, emails: "finance@example.com, cto@example.com" }, + { threshold: 90, emails: "ops@example.com" }, + ]); + expect(budgetAlertEmailsFromRows(rows)).toEqual({ + "80": ["finance@example.com", "cto@example.com"], + "90": ["ops@example.com"], + }); + }); + + it("drops API-stored thresholds outside 1 to 100 so they never block the form", () => { + const metadata = { + team_member_max_budget_alert_emails: { "0": ["a@b.c"], "50": [], "101": ["a@b.c"] }, + }; + expect(budgetAlertRowsFromMetadata(metadata, TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY)).toEqual([ + { threshold: 50, emails: "" }, + ]); + }); + + it.each([undefined, null, "50", { team_member_max_budget_alert_emails: "50" }, { soft_budget_alerting_emails: [] }])( + "returns no rows for unrelated or malformed metadata %j", + (metadata) => { + expect(budgetAlertRowsFromMetadata(metadata, TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY)).toEqual([]); + }, + ); + + it("reads only the metadata key it is asked for", () => { + const metadata = { + team_member_max_budget_alert_emails: { "50": ["team@example.com"] }, + max_budget_alert_emails: { "80": ["shared-mailbox@example.com"] }, + }; + expect(budgetAlertRowsFromMetadata(metadata, KEY_MAX_BUDGET_ALERT_EMAILS_KEY)).toEqual([ + { threshold: 80, emails: "shared-mailbox@example.com" }, + ]); + }); +}); + +describe("budgetAlertEmailsMetadata", () => { + it("writes the threshold map under the given metadata key", () => { + expect( + budgetAlertEmailsMetadata( + budgetAlertEmailsFromRows([{ threshold: 80, emails: "dl@example.com" }]), + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, + {}, + ), + ).toEqual({ max_budget_alert_emails: { "80": ["dl@example.com"] } }); + }); + + it.each([{ rows: [] }, { rows: [{ threshold: null, emails: "dl@example.com" }] }])( + "leaves the key out when no row has a valid threshold $rows", + ({ rows }) => { + expect(budgetAlertEmailsMetadata(budgetAlertEmailsFromRows(rows), KEY_MAX_BUDGET_ALERT_EMAILS_KEY, {})).toEqual( + {}, + ); + }, + ); + + it("keeps the entries the rows cannot show and lets the rows own every threshold they can", () => { + const boxMetadata = { + max_budget_alert_emails: { + "50": ["old@example.com"], + "80": ["removed@example.com"], + "90": null, + "150": ["over@example.com"], + "0": ["zero@example.com"], + "12.5": ["fraction@example.com"], + soon: "label@example.com", + }, + }; + + expect( + budgetAlertEmailsMetadata( + budgetAlertEmailsFromRows([{ threshold: 50, emails: "new@example.com" }]), + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, + boxMetadata, + ), + ).toEqual({ + max_budget_alert_emails: { + "50": ["new@example.com"], + "150": ["over@example.com"], + "0": ["zero@example.com"], + "12.5": ["fraction@example.com"], + soon: "label@example.com", + }, + }); + }); +}); + +describe("budgetAlertEmailsRowsCanShow", () => { + it("rebuilds the stored map together with the entries the rows cannot show, values untouched", () => { + const stored = { + max_budget_alert_emails: { + "50": "a@example.com, b@example.com", + "75": null, + "80": ["c@example.com", null], + "150": ["over@example.com"], + }, + }; + + expect( + budgetAlertEmailsMetadata( + budgetAlertEmailsRowsCanShow(stored, KEY_MAX_BUDGET_ALERT_EMAILS_KEY), + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, + stored, + ), + ).toEqual(stored); + expect(budgetAlertEmailsRowsCanShow(stored, KEY_MAX_BUDGET_ALERT_EMAILS_KEY)).not.toHaveProperty("150"); + }); +}); + +describe("budgetAlertEmailsFromRows", () => { + it("builds the threshold map, splitting, trimming and deduplicating recipients", () => { + expect( + budgetAlertEmailsFromRows([ + { threshold: 50, emails: "" }, + { threshold: 100, emails: " finance@example.com,cto@example.com , finance@example.com, " }, + ]), + ).toEqual({ "50": [], "100": ["finance@example.com", "cto@example.com"] }); + }); + + it("skips rows without a valid threshold", () => { + expect( + budgetAlertEmailsFromRows([ + { threshold: null, emails: "finance@example.com" }, + { threshold: 0, emails: "" }, + { threshold: 101, emails: "" }, + { threshold: 12.5, emails: "" }, + { threshold: 80, emails: "" }, + ]), + ).toEqual({ "80": [] }); + }); + + it("round-trips the stored config", () => { + const stored = { team_member_max_budget_alert_emails: { "50": [], "100": ["finance@example.com"] } }; + expect( + budgetAlertEmailsFromRows(budgetAlertRowsFromMetadata(stored, TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY)), + ).toEqual(stored.team_member_max_budget_alert_emails); + }); +}); + +describe("isValidThreshold", () => { + it.each([ + [1, true], + [50, true], + [100, true], + [0, false], + [101, false], + [33.3, false], + [null, false], + ])("treats %s as valid=%s", (threshold, valid) => { + expect(isValidThreshold(threshold)).toBe(valid); + }); +}); + +describe("teamMemberBudgetAlertSummary", () => { + it("states that the member is always notified and lists extra recipients", () => { + expect( + teamMemberBudgetAlertSummary({ + team_member_max_budget_alert_emails: { "100": ["finance@example.com"], "50": [] }, + }), + ).toEqual(["50%: member", "100%: member, finance@example.com"]); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/components/key_team_helpers/budgetAlertEmails.ts b/litellm/ui/litellm-dashboard/src/components/key_team_helpers/budgetAlertEmails.ts new file mode 100644 index 00000000000..b8f7ba4020e --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/components/key_team_helpers/budgetAlertEmails.ts @@ -0,0 +1,143 @@ +import { z } from "zod"; + +export const TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY = "team_member_max_budget_alert_emails" as const; +export const KEY_MAX_BUDGET_ALERT_EMAILS_KEY = "max_budget_alert_emails" as const; + +export type BudgetAlertEmailsMetadataKey = + | typeof TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY + | typeof KEY_MAX_BUDGET_ALERT_EMAILS_KEY; + +export interface BudgetAlertRow { + readonly threshold: number | null; + readonly emails: string; +} + +export type BudgetAlertEmails = Readonly>; + +export type BudgetAlertRowIssue = "thresholdRange" | "duplicateThreshold"; + +const splitEmails = (emails: string): readonly string[] => + Array.from( + new Set( + emails + .split(",") + .map((email) => email.trim()) + .filter((email) => email.length > 0), + ), + ); + +const storedRecipients = (value: unknown): readonly string[] | null => { + if (value === null) return []; + if (typeof value === "string") return splitEmails(value); + if (Array.isArray(value)) return value.filter((email): email is string => typeof email === "string"); + return null; +}; + +const THRESHOLD_MIN = 1; +const THRESHOLD_MAX = 100; + +export const isValidThreshold = (threshold: number | null): threshold is number => { + const isWholeNumber = threshold !== null && Number.isInteger(threshold); + return isWholeNumber && threshold >= THRESHOLD_MIN && threshold <= THRESHOLD_MAX; +}; + +const isBudgetAlertMap = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null && !Array.isArray(value); + +const storedBudgetAlertValue = (metadata: unknown, metadataKey: BudgetAlertEmailsMetadataKey): unknown => + typeof metadata === "object" && metadata !== null ? (metadata as Record)[metadataKey] : undefined; + +const storedBudgetAlertConfig = ( + metadata: unknown, + metadataKey: BudgetAlertEmailsMetadataKey, +): Readonly> => { + const config = storedBudgetAlertValue(metadata, metadataKey); + return isBudgetAlertMap(config) ? config : {}; +}; + +const ROW_THRESHOLD_KEY_PATTERN: Readonly> = { + [KEY_MAX_BUDGET_ALERT_EMAILS_KEY]: /^[1-9]\d*$/, + [TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY]: /^\d+$/, +}; + +const budgetAlertRowFromEntry = ( + [key, emails]: readonly [string, unknown], + metadataKey: BudgetAlertEmailsMetadataKey, +): BudgetAlertRow | null => { + const threshold = Number(key); + const recipients = storedRecipients(emails); + return ROW_THRESHOLD_KEY_PATTERN[metadataKey].test(key) && isValidThreshold(threshold) && recipients !== null + ? { threshold, emails: recipients.join(", ") } + : null; +}; + +export const budgetAlertRowsFromMetadata = ( + metadata: unknown, + metadataKey: BudgetAlertEmailsMetadataKey, +): readonly BudgetAlertRow[] => + Object.entries(storedBudgetAlertConfig(metadata, metadataKey)) + .flatMap((entry) => { + const row = budgetAlertRowFromEntry(entry, metadataKey); + return row === null ? [] : [row]; + }) + .sort((a, b) => (a.threshold ?? 0) - (b.threshold ?? 0)); + +const storedBudgetAlertEntries = ( + metadata: unknown, + metadataKey: BudgetAlertEmailsMetadataKey, + rowsCanShow: boolean, +): Readonly> => + Object.fromEntries( + Object.entries(storedBudgetAlertConfig(metadata, metadataKey)).filter( + (entry) => (budgetAlertRowFromEntry(entry, metadataKey) !== null) === rowsCanShow, + ), + ); + +export const budgetAlertEmailsRowsCanShow = ( + metadata: unknown, + metadataKey: BudgetAlertEmailsMetadataKey, +): Readonly> => storedBudgetAlertEntries(metadata, metadataKey, true); + +export const budgetAlertEmailsRowsCannotShow = ( + metadata: unknown, + metadataKey: BudgetAlertEmailsMetadataKey, +): Readonly> => storedBudgetAlertEntries(metadata, metadataKey, false); + +export const budgetAlertEmailsFromRows = (rows: readonly BudgetAlertRow[]): BudgetAlertEmails => + Object.fromEntries( + rows + .filter((row) => isValidThreshold(row.threshold)) + .map((row) => [String(row.threshold), splitEmails(row.emails)]), + ); + +export const budgetAlertEmailsMetadata = ( + rowEntries: Readonly>, + metadataKey: BudgetAlertEmailsMetadataKey, + metadataWithHiddenEntries: unknown, +): Readonly>> => { + const boxValue = storedBudgetAlertValue(metadataWithHiddenEntries, metadataKey); + if (Object.keys(rowEntries).length === 0 && boxValue !== undefined && !isBudgetAlertMap(boxValue)) { + return { [metadataKey]: boxValue }; + } + const emails = { + ...budgetAlertEmailsRowsCannotShow(metadataWithHiddenEntries, metadataKey), + ...rowEntries, + }; + return Object.keys(emails).length > 0 ? { [metadataKey]: emails } : {}; +}; + +export const budgetAlertRowsSchema = (message: (issue: BudgetAlertRowIssue) => string) => + z.array(z.object({ threshold: z.number().nullable(), emails: z.string() })).superRefine((rows, ctx) => { + rows.forEach((row, index) => { + if (!isValidThreshold(row.threshold)) { + ctx.addIssue({ code: "custom", message: message("thresholdRange"), path: [index, "threshold"] }); + } else if (rows.filter((other) => other.threshold === row.threshold).length > 1) { + ctx.addIssue({ code: "custom", message: message("duplicateThreshold"), path: [index, "threshold"] }); + } + }); + }); + +export const teamMemberBudgetAlertSummary = (metadata: unknown): readonly string[] => + budgetAlertRowsFromMetadata(metadata, TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY).map((row) => + row.emails.length > 0 ? `${row.threshold}%: member, ${row.emails}` : `${row.threshold}%: member`, + ); diff --git a/litellm/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts b/litellm/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts index c544a89930d..6104da1cf65 100644 --- a/litellm/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts +++ b/litellm/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts @@ -29,6 +29,78 @@ it.each([false, true])("preserves the prompt storage restriction %s in key metad expect(JSON.parse(String(result.payload.metadata))).toEqual({ note: "keep", store_prompts_in_spend_logs: store }); }); +it("merges the budget alert rows into the metadata JSON box content", () => { + const result = build( + { + metadata: JSON.stringify({ cost_center: "ops" }), + max_budget_alert_emails: [ + { threshold: 80, emails: "shared-mailbox@example.com" }, + { threshold: null, emails: "ignored@example.com" }, + ], + }, + { disabledCallbacks: ["langfuse"] }, + ); + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") throw new Error(result.kind); + expect(result.payload).not.toHaveProperty("max_budget_alert_emails"); + expect(JSON.parse(String(result.payload.metadata))).toEqual({ + cost_center: "ops", + litellm_disabled_callbacks: ["langfuse"], + max_budget_alert_emails: { "80": ["shared-mailbox@example.com"] }, + }); +}); + +it("keeps typed recipient entries the rows cannot show and lets the rows own every threshold they can", () => { + const result = build({ + metadata: JSON.stringify({ + cost_center: "ops", + max_budget_alert_emails: { "500": "typed@example.com", "80": ["stale@example.com"] }, + }), + max_budget_alert_emails: [{ threshold: 50, emails: "row@example.com" }], + }); + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") throw new Error(result.kind); + expect(JSON.parse(String(result.payload.metadata))).toEqual({ + cost_center: "ops", + max_budget_alert_emails: { "500": "typed@example.com", "50": ["row@example.com"] }, + }); +}); + +it("stores a typed max_budget_alert_emails string as typed when no row is set", () => { + const result = build({ metadata: JSON.stringify({ cost_center: "ops", max_budget_alert_emails: "ops@qa.test" }) }); + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") throw new Error(result.kind); + expect(JSON.parse(String(result.payload.metadata))).toEqual({ + cost_center: "ops", + max_budget_alert_emails: "ops@qa.test", + }); +}); + +it("replaces a typed max_budget_alert_emails string with the threshold rows", () => { + const result = build({ + metadata: JSON.stringify({ cost_center: "ops", max_budget_alert_emails: "ops@qa.test" }), + max_budget_alert_emails: [{ threshold: 80, emails: "row@example.com" }], + }); + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") throw new Error(result.kind); + expect(JSON.parse(String(result.payload.metadata))).toEqual({ + cost_center: "ops", + max_budget_alert_emails: { "80": ["row@example.com"] }, + }); +}); + +it("keeps the budget alert rows when the metadata JSON box holds null", () => { + const result = build({ + metadata: "null", + max_budget_alert_emails: [{ threshold: 80, emails: "shared-mailbox@example.com" }], + }); + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") throw new Error(result.kind); + expect(JSON.parse(String(result.payload.metadata))).toEqual({ + max_budget_alert_emails: { "80": ["shared-mailbox@example.com"] }, + }); +}); + const payloadOf = (result: KeyPayloadResult): Record => { expect(result.kind).toBe("ok"); if (result.kind !== "ok") throw new Error("unreachable"); diff --git a/litellm/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts b/litellm/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts index 311e9357825..4cd2e35a2fa 100644 --- a/litellm/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts +++ b/litellm/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts @@ -1,6 +1,12 @@ import { mapDisplayToInternalNames } from "../callback_info_helpers"; import { NEVER_RESETS_BUDGET_DURATION } from "../common_components/budget_duration_dropdown"; import type { RouterSettingsAccordionValue } from "../common_components/RouterSettingsAccordion"; +import { + type BudgetAlertRow, + budgetAlertEmailsFromRows, + budgetAlertEmailsMetadata, + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, +} from "../key_team_helpers/budgetAlertEmails"; import type { BudgetWindowEntry } from "../key_team_helpers/BudgetWindowsEditor"; import type { ModelMaxBudget } from "../key_team_helpers/ModelMaxBudgetEditor"; import { tagRowsToLimits, type TagRateLimitEntry } from "../key_team_helpers/TagRateLimitEditor"; @@ -103,7 +109,18 @@ const buildMetadataJson = (values: Record, input: KeyCreateInpu input.disabledCallbacks.length > 0 ? { ...(logged as object), litellm_disabled_callbacks: mapDisplayToInternalNames(input.disabledCallbacks) } : logged; - return JSON.stringify(disabled); + const metadataObject = disabled ?? {}; + const alertEmails = budgetAlertEmailsMetadata( + budgetAlertEmailsFromRows((values[KEY_MAX_BUDGET_ALERT_EMAILS_KEY] as readonly BudgetAlertRow[] | undefined) ?? []), + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, + metadataObject, + ); + if (typeof metadataObject !== "object" || Array.isArray(metadataObject)) return JSON.stringify(metadataObject); + const { [KEY_MAX_BUDGET_ALERT_EMAILS_KEY]: _typedAlertEmails, ...withoutTypedAlertEmails } = metadataObject as Record< + string, + unknown + >; + return JSON.stringify({ ...withoutTypedAlertEmails, ...alertEmails }); }; interface PermissionSources { @@ -153,6 +170,7 @@ const consumedSourceKeys = ( new Set([ "mcp_tool_permissions", "allowed_skills", + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, ...(values.disable_global_guardrails ? [] : ["disable_global_guardrails"]), ...(vectorStores ? ["allowed_vector_store_ids"] : []), ...(mcp ? ["allowed_mcp_servers_and_groups"] : []), diff --git a/litellm/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index bbc849167cc..89cdb51d234 100644 --- a/litellm/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -1,6 +1,6 @@ import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { act, renderWithProviders, screen, testQueryClient, waitFor } from "../../../tests/test-utils"; +import { act, fireEvent, renderWithProviders, screen, testQueryClient, waitFor } from "../../../tests/test-utils"; import type { Team } from "../key_team_helpers/key_list"; import { keyCreateCall, keyCreateServiceAccountCall, modelAvailableCall, userFilterUICall } from "../networking"; import { toast } from "@/lib/toast"; @@ -593,6 +593,68 @@ describe("CreateKey", () => { expect((await createdPayload()).metadata).toBe('{"team":"research"}'); }); + const addBudgetAlertRow = async (threshold: string, emails: string) => { + await userEvent.click(await screen.findByRole("button", { name: "Add Budget Alert Threshold" })); + const thresholds = screen.getAllByPlaceholderText("% of budget"); + fireEvent.change(thresholds[thresholds.length - 1], { target: { value: threshold } }); + const recipients = screen.getAllByPlaceholderText("Additional recipients, e.g. finance@example.com"); + fireEvent.change(recipients[recipients.length - 1], { target: { value: emails } }); + }; + + it("merges the budget alert rows into the metadata JSON box content", async () => { + await openModal(); + await nameTheKey(); + await openSection(/Optional Settings/i); + fireEvent.change(await screen.findByLabelText("Metadata"), { target: { value: '{"team":"research"}' } }); + await addBudgetAlertRow("80", "shared-mailbox@example.com, finance-dl@example.com"); + await submit(); + + const payload = await createdPayload(); + expect(JSON.parse(String(payload.metadata))).toEqual({ + team: "research", + max_budget_alert_emails: { "80": ["shared-mailbox@example.com", "finance-dl@example.com"] }, + }); + expect(payload).not.toHaveProperty("max_budget_alert_emails"); + }); + + it("moves a recipient map typed into the metadata JSON box into the rows on blur", async () => { + vi.mocked(toast.info).mockClear(); + await openModal(); + await nameTheKey(); + await openSection(/Optional Settings/i); + await addBudgetAlertRow("80", "row@example.com"); + const metadataBox = await screen.findByLabelText("Metadata"); + fireEvent.change(metadataBox, { + target: { value: JSON.stringify({ team: "research", max_budget_alert_emails: { "80": "typed@example.com" } }) }, + }); + fireEvent.blur(metadataBox); + + expect(toast.info).toHaveBeenCalledWith( + "Moved alert thresholds 80% from metadata to the Budget Alert Thresholds rows", + ); + expect(screen.getByPlaceholderText("Additional recipients, e.g. finance@example.com")).toHaveValue( + "typed@example.com", + ); + await submit(); + + expect(JSON.parse(String((await createdPayload()).metadata))).toEqual({ + team: "research", + max_budget_alert_emails: { "80": ["typed@example.com"] }, + }); + }); + + it("blocks the create on a duplicate budget alert threshold", async () => { + await openModal(); + await nameTheKey(); + await openSection(/Optional Settings/i); + await addBudgetAlertRow("80", "a@example.com"); + await addBudgetAlertRow("80", "b@example.com"); + await submit(); + + expect(await screen.findAllByText("Duplicate threshold")).toHaveLength(2); + expect(vi.mocked(keyCreateCall)).not.toHaveBeenCalled(); + }); + it("carries the typed key alias and the chosen team onto the wire", async () => { state.teams = [{ team_id: "team-1", team_alias: "Team One", models: [] }]; await openModal({ teams: state.teams as unknown as Team[] }); diff --git a/litellm/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/litellm/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 2e82e1ca4f5..cc2b75d10a0 100644 --- a/litellm/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/litellm/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -42,6 +42,7 @@ import { MountedFormField, MountedFormProvider, projectMountedValues, + useMountedName, useMountRegistry, type MountedFormValues, } from "../common_components/MountedFormField"; @@ -56,6 +57,13 @@ import TeamDropdown from "../common_components/team_dropdown"; import OrganizationDropdown from "../common_components/OrganizationDropdown"; import ProjectDropdown from "../common_components/ProjectDropdown"; import { CreateUserButton } from "../CreateUserButton"; +import { moveMetadataBudgetAlertsToRows } from "../templates/KeyEditViewControls"; +import { BudgetAlertEmailsEditor, type BudgetAlertEmailsEditorText } from "../key_team_helpers/BudgetAlertEmailsEditor"; +import { + type BudgetAlertRow, + budgetAlertRowsSchema, + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, +} from "../key_team_helpers/budgetAlertEmails"; import { BudgetFallbacksEditor } from "../key_team_helpers/BudgetFallbacksEditor"; import { END_USER_BUDGET_HINT, EndUserBudgetSelect } from "../key_team_helpers/EndUserBudgetSelect"; import { BudgetWindowEntry, BudgetWindowsEditor } from "../key_team_helpers/BudgetWindowsEditor"; @@ -132,6 +140,15 @@ type McpSelectorValue = { servers: string[]; accessGroups: string[]; toolsets?: type AgentSelectorValue = { agents: string[]; accessGroups: string[] }; +const MountedBudgetAlertEmailsEditor = (props: { + readonly control: Control; + readonly label: React.ReactNode; + readonly text: BudgetAlertEmailsEditorText; +}) => { + useMountedName(KEY_MAX_BUDGET_ALERT_EMAILS_KEY); + return ; +}; + const isBlank = (value: unknown): boolean => value === undefined || value === null || value === ""; const requiredRule = (required: boolean, message: string) => ({ @@ -532,7 +549,21 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp }; const handleSubmit = (event: React.FormEvent) => - void form.handleSubmit(() => handleCreate(projectMountedValues(registry, form.getValues)))(event); + void form.handleSubmit(() => { + const values = projectMountedValues(registry, form.getValues); + const alertRows = budgetAlertRowsSchema((issue) => + issue === "thresholdRange" + ? t("keyDetail.maxBudgetAlertEmails.thresholdRange") + : t("keyDetail.maxBudgetAlertEmails.duplicateThreshold"), + ).safeParse(values[KEY_MAX_BUDGET_ALERT_EMAILS_KEY] ?? []); + if (!alertRows.success) { + alertRows.error.issues.forEach((issue) => + form.setError([KEY_MAX_BUDGET_ALERT_EMAILS_KEY, ...issue.path].join("."), { message: issue.message }), + ); + return; + } + return handleCreate(values); + })(event); // Fetch available models when team or auth changes. // Note: Model prefill from URL params is handled by the useEffect below, which @@ -1093,6 +1124,25 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp /> )} +
+ + {t("keyDetail.maxBudgetAlertEmails.budgetAlertThresholds")}{" "} + + + + + } + text={{ + thresholdPlaceholder: t("keyDetail.maxBudgetAlertEmails.thresholdPlaceholder"), + recipientsPlaceholder: t("keyDetail.maxBudgetAlertEmails.recipientsPlaceholder"), + removeThreshold: t("keyDetail.maxBudgetAlertEmails.removeThreshold"), + addThreshold: t("keyDetail.maxBudgetAlertEmails.addThreshold"), + }} + /> +
= ({ team, teams, data, addKey, autoOp value={(control.value as string | undefined) ?? ""} rows={4} placeholder={t("createKey.metadata.placeholder")} + onBlur={() => { + control.onBlur(); + moveMetadataBudgetAlertsToRows( + form.getValues("metadata") as string | undefined, + form.getValues(KEY_MAX_BUDGET_ALERT_EMAILS_KEY) as BudgetAlertRow[] | undefined, + (metadata, rows) => { + form.setValue("metadata", metadata, { shouldDirty: true }); + form.setValue(KEY_MAX_BUDGET_ALERT_EMAILS_KEY, rows, { shouldDirty: true }); + }, + t, + ); + }} /> )} diff --git a/litellm/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/litellm/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 24e6dd75abf..ad4c88dd75f 100644 --- a/litellm/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/litellm/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -130,12 +130,13 @@ import { import TeamMembersComponent from "./TeamMemberTab"; import { useModelAccessGroupNames } from "@/app/(dashboard)/hooks/models/useModels"; import { - isValidThreshold, + budgetAlertEmailsFromRows, + budgetAlertRowsFromMetadata, + budgetAlertRowsSchema, TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY, - teamMemberBudgetAlertEmailsFromRows, - teamMemberBudgetAlertRowsFromMetadata, teamMemberBudgetAlertSummary, -} from "./teamMemberBudgetAlertEmails"; +} from "../key_team_helpers/budgetAlertEmails"; +import { BudgetAlertEmailsEditor } from "../key_team_helpers/BudgetAlertEmailsEditor"; import { TeamVirtualKeysTable } from "./TeamVirtualKeysTable"; import ResetMemberBudgetsDialog from "./ResetMemberBudgetsDialog"; import { customBudgetMemberUserIds, shouldPromptMemberBudgetReset } from "./memberBudgetReset"; @@ -387,26 +388,11 @@ const buildTeamUpdateFieldsSchema = (t: TFunction) => team_member_key_duration: z.string().optional(), team_member_tpm_limit: numericInputSchema, team_member_rpm_limit: numericInputSchema, - team_member_max_budget_alert_emails: z - .array(z.object({ threshold: z.number().nullable(), emails: z.string() })) - .superRefine((rows, ctx) => { - rows.forEach((row, index) => { - if (!isValidThreshold(row.threshold)) { - ctx.addIssue({ - code: "custom", - message: t("teams.teamInfo.validation.thresholdRange"), - path: [index, "threshold"], - }); - } else if (rows.filter((other) => other.threshold === row.threshold).length > 1) { - ctx.addIssue({ - code: "custom", - message: t("teams.teamInfo.validation.duplicateThreshold"), - path: [index, "threshold"], - }); - } - }); - }) - .optional(), + team_member_max_budget_alert_emails: budgetAlertRowsSchema((issue) => + issue === "thresholdRange" + ? t("teams.teamInfo.validation.thresholdRange") + : t("teams.teamInfo.validation.duplicateThreshold"), + ).optional(), budget_duration: z.string().nullish(), tpm_limit: numericInputSchema, rpm_limit: numericInputSchema, @@ -551,7 +537,9 @@ const toTeamFormValues = (info: TeamInfoRecord, effectiveGuardrails: string[]): team_member_key_duration: info.metadata?.team_member_key_duration, team_member_tpm_limit: info.team_member_budget_table?.tpm_limit, team_member_rpm_limit: info.team_member_budget_table?.rpm_limit, - team_member_max_budget_alert_emails: [...teamMemberBudgetAlertRowsFromMetadata(info.metadata)], + team_member_max_budget_alert_emails: [ + ...budgetAlertRowsFromMetadata(info.metadata, TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY), + ], budget_duration: info.budget_duration, tpm_limit: info.tpm_limit, rpm_limit: info.rpm_limit, @@ -639,11 +627,6 @@ const TeamInfoView: React.FC = ({ append: appendModelLimit, remove: removeModelLimit, } = useFieldArray({ control: form.control, name: "modelLimits" }); - const { - fields: memberBudgetAlertRows, - append: appendMemberBudgetAlertRow, - remove: removeMemberBudgetAlertRow, - } = useFieldArray({ control: form.control, name: "team_member_max_budget_alert_emails" }); const [teamMemberSettingsOpen, setTeamMemberSettingsOpen] = useState(false); const [searchToolSettingsOpen, setSearchToolSettingsOpen] = useState(false); const [isEditMemberModalVisible, setIsEditMemberModalVisible] = useState(false); @@ -1081,7 +1064,7 @@ const TeamInfoView: React.FC = ({ const memberBudgetAlertEmails = values.team_member_max_budget_alert_emails !== undefined - ? teamMemberBudgetAlertEmailsFromRows(values.team_member_max_budget_alert_emails) + ? budgetAlertEmailsFromRows(values.team_member_max_budget_alert_emails) : info.metadata?.[TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY]; const memberBudgetAlertEmailsMetadata = memberBudgetAlertEmails !== undefined && Object.keys(memberBudgetAlertEmails).length > 0 @@ -1877,71 +1860,22 @@ const TeamInfoView: React.FC = ({ /> )} - - - {labelWithHint( - t("teams.teamInfo.editForm.teamMemberSettings.budgetAlertThresholds"), - t("teams.teamInfo.editForm.teamMemberSettings.budgetAlertThresholdsHint"), - )} - - {memberBudgetAlertRows.map((row, index) => ( -
- - {({ ref, value, onChange, ...field }) => ( - ) => - onChange(event.target.value === "" ? null : Number(event.target.value)) - } - placeholder={t("teams.teamInfo.editForm.teamMemberSettings.thresholdPlaceholder")} - min={1} - max={100} - step={1} - /> - )} - - - {({ ref, value, ...field }) => ( - - )} - - -
- ))} - - + diff --git a/litellm/ui/litellm-dashboard/src/components/team/teamMemberBudgetAlertEmails.test.ts b/litellm/ui/litellm-dashboard/src/components/team/teamMemberBudgetAlertEmails.test.ts deleted file mode 100644 index 3faa051047b..00000000000 --- a/litellm/ui/litellm-dashboard/src/components/team/teamMemberBudgetAlertEmails.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - isValidThreshold, - teamMemberBudgetAlertEmailsFromRows, - teamMemberBudgetAlertRowsFromMetadata, - teamMemberBudgetAlertSummary, -} from "./teamMemberBudgetAlertEmails"; - -describe("teamMemberBudgetAlertRowsFromMetadata", () => { - it("turns the stored threshold map into rows sorted by threshold", () => { - const metadata = { - team_member_max_budget_alert_emails: { "100": ["finance@example.com", "cto@example.com"], "50": [] }, - }; - expect(teamMemberBudgetAlertRowsFromMetadata(metadata)).toEqual([ - { threshold: 50, emails: "" }, - { threshold: 100, emails: "finance@example.com, cto@example.com" }, - ]); - }); - - it("drops non-numeric thresholds and non-list recipients instead of crashing", () => { - const metadata = { - team_member_max_budget_alert_emails: { fifty: [], "75": "finance@example.com", "90": [1], "100": ["a@b.c"] }, - }; - expect(teamMemberBudgetAlertRowsFromMetadata(metadata)).toEqual([{ threshold: 100, emails: "a@b.c" }]); - }); - - it("drops API-stored thresholds outside 1 to 100 so they never block the form", () => { - const metadata = { - team_member_max_budget_alert_emails: { "0": ["a@b.c"], "50": [], "101": ["a@b.c"] }, - }; - expect(teamMemberBudgetAlertRowsFromMetadata(metadata)).toEqual([{ threshold: 50, emails: "" }]); - }); - - it.each([undefined, null, "50", { team_member_max_budget_alert_emails: "50" }, { soft_budget_alerting_emails: [] }])( - "returns no rows for unrelated or malformed metadata %j", - (metadata) => { - expect(teamMemberBudgetAlertRowsFromMetadata(metadata)).toEqual([]); - }, - ); -}); - -describe("teamMemberBudgetAlertEmailsFromRows", () => { - it("builds the threshold map, splitting, trimming and deduplicating recipients", () => { - expect( - teamMemberBudgetAlertEmailsFromRows([ - { threshold: 50, emails: "" }, - { threshold: 100, emails: " finance@example.com,cto@example.com , finance@example.com, " }, - ]), - ).toEqual({ "50": [], "100": ["finance@example.com", "cto@example.com"] }); - }); - - it("skips rows without a valid threshold", () => { - expect( - teamMemberBudgetAlertEmailsFromRows([ - { threshold: null, emails: "finance@example.com" }, - { threshold: 0, emails: "" }, - { threshold: 101, emails: "" }, - { threshold: 12.5, emails: "" }, - { threshold: 80, emails: "" }, - ]), - ).toEqual({ "80": [] }); - }); - - it("round-trips the stored config", () => { - const stored = { team_member_max_budget_alert_emails: { "50": [], "100": ["finance@example.com"] } }; - expect(teamMemberBudgetAlertEmailsFromRows(teamMemberBudgetAlertRowsFromMetadata(stored))).toEqual( - stored.team_member_max_budget_alert_emails, - ); - }); -}); - -describe("isValidThreshold", () => { - it.each([ - [1, true], - [50, true], - [100, true], - [0, false], - [101, false], - [33.3, false], - [null, false], - ])("treats %s as valid=%s", (threshold, valid) => { - expect(isValidThreshold(threshold)).toBe(valid); - }); -}); - -describe("teamMemberBudgetAlertSummary", () => { - it("states that the member is always notified and lists extra recipients", () => { - expect( - teamMemberBudgetAlertSummary({ - team_member_max_budget_alert_emails: { "100": ["finance@example.com"], "50": [] }, - }), - ).toEqual(["50%: member", "100%: member, finance@example.com"]); - }); -}); diff --git a/litellm/ui/litellm-dashboard/src/components/team/teamMemberBudgetAlertEmails.ts b/litellm/ui/litellm-dashboard/src/components/team/teamMemberBudgetAlertEmails.ts deleted file mode 100644 index 36d5ddbac02..00000000000 --- a/litellm/ui/litellm-dashboard/src/components/team/teamMemberBudgetAlertEmails.ts +++ /dev/null @@ -1,57 +0,0 @@ -export const TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY = "team_member_max_budget_alert_emails" as const; - -export interface TeamMemberBudgetAlertRow { - readonly threshold: number | null; - readonly emails: string; -} - -export type TeamMemberBudgetAlertEmails = Readonly>; - -const isEmailList = (value: unknown): value is readonly string[] => - Array.isArray(value) && value.every((email) => typeof email === "string"); - -const splitEmails = (emails: string): readonly string[] => - Array.from( - new Set( - emails - .split(",") - .map((email) => email.trim()) - .filter((email) => email.length > 0), - ), - ); - -const THRESHOLD_MIN = 1; -const THRESHOLD_MAX = 100; - -export const isValidThreshold = (threshold: number | null): threshold is number => { - const isWholeNumber = threshold !== null && Number.isInteger(threshold); - return isWholeNumber && threshold >= THRESHOLD_MIN && threshold <= THRESHOLD_MAX; -}; - -export const teamMemberBudgetAlertRowsFromMetadata = (metadata: unknown): readonly TeamMemberBudgetAlertRow[] => { - if (typeof metadata !== "object" || metadata === null) return []; - const config: unknown = (metadata as Record)[TEAM_MEMBER_MAX_BUDGET_ALERT_EMAILS_KEY]; - if (typeof config !== "object" || config === null || Array.isArray(config)) return []; - return Object.entries(config as Record) - .flatMap(([key, emails]) => { - const threshold = Number(key); - return /^\d+$/.test(key) && isValidThreshold(threshold) && isEmailList(emails) - ? [{ threshold, emails: emails.join(", ") }] - : []; - }) - .sort((a, b) => (a.threshold ?? 0) - (b.threshold ?? 0)); -}; - -export const teamMemberBudgetAlertEmailsFromRows = ( - rows: readonly TeamMemberBudgetAlertRow[], -): TeamMemberBudgetAlertEmails => - Object.fromEntries( - rows - .filter((row) => isValidThreshold(row.threshold)) - .map((row) => [String(row.threshold), splitEmails(row.emails)]), - ); - -export const teamMemberBudgetAlertSummary = (metadata: unknown): readonly string[] => - teamMemberBudgetAlertRowsFromMetadata(metadata).map((row) => - row.emails.length > 0 ? `${row.threshold}%: member, ${row.emails}` : `${row.threshold}%: member`, - ); diff --git a/litellm/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx b/litellm/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx index 3b96134f746..d44877a39e8 100644 --- a/litellm/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx +++ b/litellm/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx @@ -14,7 +14,8 @@ import RateLimitTypeFormItem from "../common_components/RateLimitTypeFormItem"; import { StorePromptsMetadataToggle } from "../common_components/StorePromptsToggle"; import NumericalInput from "../shared/numerical_input"; import SkillSelector from "../skills/SkillSelector"; -import { moveTagsOutOfMetadataJson } from "./keyEditFieldNormalizers"; +import type { BudgetAlertRow } from "../key_team_helpers/budgetAlertEmails"; +import { moveBudgetAlertEmailsOutOfMetadataJson, moveTagsOutOfMetadataJson } from "./keyEditFieldNormalizers"; import { AgentsAndGroups, KeyEditFormValues } from "./keyEditFormValues"; export const labelWithHint = (label: React.ReactNode, hint: string): React.ReactNode => ( @@ -186,8 +187,42 @@ export const moveMetadataTagsToTagsField = (form: KeyEditForm, t: TFunction): vo } }; -export const KeyMetadataField = ({ form }: { form: KeyEditForm }) => { +export const moveMetadataBudgetAlertsToRows = ( + metadata: string | undefined, + rows: readonly BudgetAlertRow[] | undefined, + write: (metadata: string, rows: BudgetAlertRow[]) => void, + t: TFunction, +): void => { + const moved = moveBudgetAlertEmailsOutOfMetadataJson(metadata, rows); + if (moved === null) return; + write(moved.metadata, [...moved.rows]); + const thresholds = moved.movedThresholds.map((threshold) => `${threshold}%`).join(", "); + toast.info(t("keyDetail.maxBudgetAlertEmails.movedThresholds", { thresholds })); +}; + +export const moveMetadataBudgetAlertsToKeyRows = (form: KeyEditForm, t: TFunction): void => + moveMetadataBudgetAlertsToRows( + form.getValues("metadata"), + form.getValues("max_budget_alert_emails"), + (metadata, rows) => { + form.setValue("metadata", metadata, { shouldDirty: true }); + form.setValue("max_budget_alert_emails", rows, { shouldDirty: true }); + }, + t, + ); + +export const KeyMetadataField = ({ + form, + budgetAlertRowsShown, +}: { + form: KeyEditForm; + budgetAlertRowsShown: boolean; +}) => { const { t } = useTranslation(); + const moveTypedFieldsOutOfMetadata = () => { + moveMetadataTagsToTagsField(form, t); + if (budgetAlertRowsShown) moveMetadataBudgetAlertsToKeyRows(form, t); + }; return ( { rows={10} onBlur={() => { field.onBlur(); - moveMetadataTagsToTagsField(form, t); + moveTypedFieldsOutOfMetadata(); }} /> diff --git a/litellm/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx b/litellm/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx index 2c57445104f..4ddb5001a06 100644 --- a/litellm/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx @@ -196,7 +196,8 @@ vi.mock("./key_edit_view", async () => { React.createElement( "button", { - onClick: () => props.onSubmit((globalThis as any).__TEST_FORM_VALUES ?? {}), + onClick: () => + props.onSubmit((globalThis as any).__TEST_FORM_VALUES ?? {}, (globalThis as any).__TEST_DIRTY_FIELDS ?? []), }, "Mock Submit", ), @@ -231,7 +232,7 @@ const baseKeyData = { next_rotation_at: null as any, }; -const renderView = (premiumUser: boolean) => { +const renderView = (premiumUser: boolean, keyData: Record = baseKeyData) => { // Configure the mock for this test mockUseAuthorized.mockReturnValue({ accessToken: "access_abc", @@ -245,19 +246,14 @@ const renderView = (premiumUser: boolean) => { }); return render( - {}} - keyData={baseKeyData as any} - onKeyDataUpdate={() => {}} - teams={[]} - />, + {}} keyData={keyData as any} onKeyDataUpdate={() => {}} teams={[]} />, ); }; beforeEach(() => { vi.clearAllMocks(); (globalThis as any).__TEST_FORM_VALUES = undefined; + (globalThis as any).__TEST_DIRTY_FIELDS = undefined; }); // ---- Tests ---- @@ -715,3 +711,171 @@ describe("KeyInfoView handleKeyUpdate organization", () => { expect(shownOrganization()).toHaveTextContent("org-1"); }); }); + +describe("KeyInfoView handleKeyUpdate max_budget_alert_emails", () => { + const submit = async ( + formValues: Record, + dirtyFields: readonly string[] = ["max_budget_alert_emails"], + keyData: Record = baseKeyData, + ) => { + renderView(true, keyData); + fireEvent.click(screen.getByText("Settings")); + fireEvent.click(screen.getByText("Edit Settings")); + Object.assign(globalThis, { + __TEST_FORM_VALUES: { token: "tok_123", ...formValues }, + __TEST_DIRTY_FIELDS: dirtyFields, + }); + fireEvent.click(screen.getByText("Mock Submit")); + await waitFor(() => expect(keyUpdateCallMock).toHaveBeenCalled()); + return keyUpdateCallMock.mock.calls[0][1]; + }; + + it("writes the alert rows into metadata next to the JSON box content and the dedicated metadata fields", async () => { + const formValues = { + metadata: JSON.stringify({ cost_center: "ops", nested: { owner: "platform" } }), + tags: ["prod"], + guardrails: ["gr-1"], + logging_settings: [{ callback_name: "langfuse" }], + disabled_callbacks: ["langsmith"], + max_budget_alert_emails: [ + { threshold: 80, emails: "shared-mailbox@example.com, finance-dl@example.com" }, + { threshold: 50, emails: "" }, + ], + }; + const expectedMetadata = { + cost_center: "ops", + nested: { owner: "platform" }, + tags: ["prod"], + guardrails: ["gr-1"], + logging: [{ callback_name: "langfuse" }], + litellm_disabled_callbacks: ["langsmith"], + max_budget_alert_emails: { "80": ["shared-mailbox@example.com", "finance-dl@example.com"], "50": [] }, + }; + + const sentPayload = await submit(formValues); + + expect(sentPayload.metadata).toEqual(expectedMetadata); + expect(sentPayload).not.toHaveProperty("max_budget_alert_emails"); + }); + + it("clears the stored recipients when every row is removed, even if the JSON box still names them", async () => { + const sentPayload = await submit({ + metadata: JSON.stringify({ cost_center: "ops", max_budget_alert_emails: { "80": ["old@example.com"] } }), + max_budget_alert_emails: [], + }); + + expect(sentPayload.metadata).toEqual({ cost_center: "ops" }); + }); + + it("writes the alert rows when the metadata JSON box is left empty", async () => { + const sentPayload = await submit({ + metadata: "", + max_budget_alert_emails: [{ threshold: 90, emails: "dl@example.com" }], + }); + + expect(sentPayload.metadata).toEqual({ max_budget_alert_emails: { "90": ["dl@example.com"] } }); + }); + + it("sends the stored map verbatim on a save that does not edit the rows", async () => { + const storedAlertEmails = { + "50": ["a@example.com"], + "75": null, + "80": ["b@example.com", null], + "150": ["c@example.com"], + }; + const sentPayload = await submit( + { + key_alias: "renamed", + metadata: JSON.stringify({ cost_center: "ops", max_budget_alert_emails: { "150": ["c@example.com"] } }), + max_budget_alert_emails: [{ threshold: 50, emails: "a@example.com" }], + }, + ["key_alias"], + { ...baseKeyData, metadata: { cost_center: "ops", max_budget_alert_emails: storedAlertEmails } }, + ); + + expect(sentPayload.metadata).toEqual({ cost_center: "ops", max_budget_alert_emails: storedAlertEmails }); + }); + const KEY_WITH_HIDDEN_ALERT = { + ...baseKeyData, + metadata: { + cost_center: "ops", + max_budget_alert_emails: { + "50": ["old@example.com"], + "80": ["removed@example.com"], + "150": ["c@example.com"], + }, + }, + }; + + it("keeps the entry the rows cannot show that the JSON box still holds when the rows were edited", async () => { + const sentPayload = await submit( + { + metadata: JSON.stringify({ cost_center: "ops", max_budget_alert_emails: { "150": ["c@example.com"] } }), + max_budget_alert_emails: [{ threshold: 50, emails: "new@example.com" }], + }, + ["max_budget_alert_emails"], + KEY_WITH_HIDDEN_ALERT, + ); + + expect(sentPayload.metadata).toEqual({ + cost_center: "ops", + max_budget_alert_emails: { "50": ["new@example.com"], "150": ["c@example.com"] }, + }); + }); + + it("keeps only the box's hidden entry when every row is removed", async () => { + const sentPayload = await submit( + { + metadata: JSON.stringify({ cost_center: "ops", max_budget_alert_emails: { "150": ["c@example.com"] } }), + max_budget_alert_emails: [], + }, + ["max_budget_alert_emails"], + KEY_WITH_HIDDEN_ALERT, + ); + + expect(sentPayload.metadata).toEqual({ + cost_center: "ops", + max_budget_alert_emails: { "150": ["c@example.com"] }, + }); + }); + + it("clears the map when every row is removed and the hidden entry is deleted from the JSON box", async () => { + const sentPayload = await submit( + { metadata: JSON.stringify({ cost_center: "ops" }), max_budget_alert_emails: [] }, + ["max_budget_alert_emails"], + KEY_WITH_HIDDEN_ALERT, + ); + + expect(sentPayload.metadata).toEqual({ cost_center: "ops" }); + }); + + it("saves an entry the rows cannot show typed into the JSON box on an untouched-rows edit", async () => { + const sentPayload = await submit( + { + metadata: JSON.stringify({ cost_center: "ops", max_budget_alert_emails: { "0": ["zero@example.com"] } }), + max_budget_alert_emails: [], + }, + ["metadata"], + { ...baseKeyData, metadata: { cost_center: "ops" } }, + ); + + expect(sentPayload.metadata).toEqual({ + cost_center: "ops", + max_budget_alert_emails: { "0": ["zero@example.com"] }, + }); + }); + + it("saves a stored max_budget_alert_emails string back unchanged on an unrelated edit", async () => { + const sentPayload = await submit( + { + key_alias: "renamed", + metadata: JSON.stringify({ cost_center: "ops", max_budget_alert_emails: "ops@qa.test" }), + max_budget_alert_emails: [], + }, + ["key_alias"], + { ...baseKeyData, metadata: { cost_center: "ops", max_budget_alert_emails: "ops@qa.test" } }, + ); + + expect(sentPayload.metadata).toEqual({ cost_center: "ops", max_budget_alert_emails: "ops@qa.test" }); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/components/templates/keyEditFieldNormalizers.test.ts b/litellm/ui/litellm-dashboard/src/components/templates/keyEditFieldNormalizers.test.ts index 9223cc6dbc8..cbae3e67540 100644 --- a/litellm/ui/litellm-dashboard/src/components/templates/keyEditFieldNormalizers.test.ts +++ b/litellm/ui/litellm-dashboard/src/components/templates/keyEditFieldNormalizers.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { moveTagsOutOfMetadataJson } from "./keyEditFieldNormalizers"; +import { moveBudgetAlertEmailsOutOfMetadataJson, moveTagsOutOfMetadataJson } from "./keyEditFieldNormalizers"; describe("moveTagsOutOfMetadataJson", () => { it("moves a tags array out of the JSON and appends it to the current tags", () => { @@ -48,3 +48,45 @@ describe("moveTagsOutOfMetadataJson", () => { expect(moveTagsOutOfMetadataJson("null", ["a"])).toBeNull(); }); }); + +describe("moveBudgetAlertEmailsOutOfMetadataJson", () => { + it("moves the typed thresholds into the rows, letting a typed entry replace the row with its threshold", () => { + const typed = JSON.stringify({ + env: "prod", + max_budget_alert_emails: { "80": "typed@example.com, dl@example.com", "90": null }, + }); + const rows = [ + { threshold: 50, emails: "kept@example.com" }, + { threshold: 80, emails: "replaced@example.com" }, + ]; + + expect(moveBudgetAlertEmailsOutOfMetadataJson(typed, rows)).toEqual({ + metadata: '{\n "env": "prod"\n}', + rows: [ + { threshold: 50, emails: "kept@example.com" }, + { threshold: 80, emails: "typed@example.com, dl@example.com" }, + { threshold: 90, emails: "" }, + ], + movedThresholds: [80, 90], + }); + }); + + it("leaves the typed entries the rows cannot show in the JSON", () => { + const typed = JSON.stringify({ max_budget_alert_emails: { "80": ["a@example.com"], "150": ["b@example.com"] } }); + + expect(moveBudgetAlertEmailsOutOfMetadataJson(typed, [])).toEqual({ + metadata: JSON.stringify({ max_budget_alert_emails: { "150": ["b@example.com"] } }, null, 2), + rows: [{ threshold: 80, emails: "a@example.com" }], + movedThresholds: [80], + }); + }); + + it("returns null when nothing typed can become a row", () => { + expect( + moveBudgetAlertEmailsOutOfMetadataJson('{"max_budget_alert_emails": {"150": ["b@example.com"]}}', []), + ).toBeNull(); + expect(moveBudgetAlertEmailsOutOfMetadataJson('{"env": "prod"}', [])).toBeNull(); + expect(moveBudgetAlertEmailsOutOfMetadataJson('{"max_budget_alert_emails": {', [])).toBeNull(); + expect(moveBudgetAlertEmailsOutOfMetadataJson(undefined, [])).toBeNull(); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/components/templates/keyEditFieldNormalizers.ts b/litellm/ui/litellm-dashboard/src/components/templates/keyEditFieldNormalizers.ts index 529bffb9a84..8dec643b747 100644 --- a/litellm/ui/litellm-dashboard/src/components/templates/keyEditFieldNormalizers.ts +++ b/litellm/ui/litellm-dashboard/src/components/templates/keyEditFieldNormalizers.ts @@ -1,5 +1,11 @@ import type { TFunction } from "i18next"; import i18n from "@/i18n/i18n"; +import { + type BudgetAlertRow, + budgetAlertEmailsRowsCannotShow, + budgetAlertRowsFromMetadata, + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, +} from "../key_team_helpers/budgetAlertEmails"; const WORD_FORM_BUDGET_DURATIONS: Record = { hourly: "1h", @@ -75,6 +81,27 @@ export const moveTagsOutOfMetadataJson = ( return { metadata: JSON.stringify(rest, null, 2), tags: [...existing, ...movedTags], movedTags }; }; +export type MovedMetadataBudgetAlerts = { + readonly metadata: string; + readonly rows: readonly BudgetAlertRow[]; + readonly movedThresholds: readonly (number | null)[]; +}; + +export const moveBudgetAlertEmailsOutOfMetadataJson = ( + metadataJson: string | undefined, + currentRows: readonly BudgetAlertRow[] | undefined, +): MovedMetadataBudgetAlerts | null => { + const parsed = metadataJson === undefined ? null : parseJsonObject(metadataJson); + const typedRows = budgetAlertRowsFromMetadata(parsed, KEY_MAX_BUDGET_ALERT_EMAILS_KEY); + if (parsed === null || typedRows.length === 0) return null; + const movedThresholds = typedRows.map((row) => row.threshold); + const unmoved = budgetAlertEmailsRowsCannotShow(parsed, KEY_MAX_BUDGET_ALERT_EMAILS_KEY); + const { [KEY_MAX_BUDGET_ALERT_EMAILS_KEY]: _moved, ...rest } = parsed; + const metadata = Object.keys(unmoved).length > 0 ? { ...rest, [KEY_MAX_BUDGET_ALERT_EMAILS_KEY]: unmoved } : rest; + const keptRows = (currentRows ?? []).filter((row) => !movedThresholds.includes(row.threshold)); + return { metadata: JSON.stringify(metadata, null, 2), rows: [...keptRows, ...typedRows], movedThresholds }; +}; + export const currentValuePlaceholder = ( premiumUser: boolean, current: unknown, diff --git a/litellm/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts b/litellm/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts index 9ad3efe84ce..800a5bdf079 100644 --- a/litellm/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts +++ b/litellm/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts @@ -4,7 +4,13 @@ import { UseFormReturn, UseFormSetValue } from "react-hook-form"; import { KeyResponse } from "../key_team_helpers/key_list"; import { ToolPermissionsChangeOrigin } from "../mcp_server_management/MCPToolPermissions"; -import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata } from "../key_info_utils"; +import { extractLoggingSettings, formatMetadataForDisplay, stripFieldManagedKeysFromMetadata } from "../key_info_utils"; +import { + type BudgetAlertRow, + budgetAlertRowsFromMetadata, + budgetAlertRowsSchema, + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, +} from "../key_team_helpers/budgetAlertEmails"; import { mapInternalToDisplayNames } from "../callback_info_helpers"; import { estimateChecks, estimateFields } from "./estimatedOutputTokens"; import { canonicalBudgetDuration, keyOrganizationId } from "./keyEditFieldNormalizers"; @@ -26,6 +32,7 @@ export interface KeyEditFormValues { allowed_routes?: string; max_budget?: number | string | null; soft_budget?: number | string | null; + max_budget_alert_emails?: BudgetAlertRow[]; budget_duration?: string | null; tpm_limit?: number | string | null; tpm_limit_type?: string | null; @@ -76,6 +83,7 @@ export const toKeyEditFormValues = (keyData: KeyResponse): KeyEditFormValues => max_budget: keyData.max_budget, soft_budget: (keyData.litellm_budget_table as { soft_budget?: number | null } | null | undefined)?.soft_budget ?? null, + max_budget_alert_emails: [...budgetAlertRowsFromMetadata(keyData.metadata, KEY_MAX_BUDGET_ALERT_EMAILS_KEY)], budget_duration: canonicalBudgetDuration(keyData.budget_duration), tpm_limit: keyData.tpm_limit, tpm_limit_type: (keyData as { tpm_limit_type?: string | null }).tpm_limit_type ?? null, @@ -114,7 +122,7 @@ export const toKeyEditFormValues = (keyData: KeyResponse): KeyEditFormValues => team_id: keyData.team_id, project_id: keyData.project_id, logging_settings: extractLoggingSettings(keyData.metadata), - metadata: formatMetadataForDisplay(stripTagsFromMetadata(keyData.metadata)), + metadata: formatMetadataForDisplay(stripFieldManagedKeysFromMetadata(keyData.metadata)), duration: (keyData as { duration?: string }).duration ?? "", token: keyData.token || keyData.token_id, disabled_callbacks: Array.isArray(readMetadata(keyData, "litellm_disabled_callbacks")) @@ -130,6 +138,11 @@ export const keyEditFormSchema = z.object({ allowed_routes: z.custom().optional(), max_budget: z.custom().optional(), soft_budget: z.custom().optional(), + max_budget_alert_emails: budgetAlertRowsSchema((issue) => + issue === "thresholdRange" + ? i18n.t("keyDetail.maxBudgetAlertEmails.thresholdRange") + : i18n.t("keyDetail.maxBudgetAlertEmails.duplicateThreshold"), + ).optional(), budget_duration: z.custom().optional(), tpm_limit: z.custom().optional(), tpm_limit_type: z.custom().optional(), @@ -187,6 +200,7 @@ export const toSubmittedValues = ( allowed_routes: values.allowed_routes, max_budget: values.max_budget, soft_budget: values.soft_budget, + max_budget_alert_emails: values.max_budget_alert_emails, budget_duration: values.budget_duration, tpm_limit: values.tpm_limit, tpm_limit_type: values.tpm_limit_type, diff --git a/litellm/ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx index 88d063358ca..ae21dfe7100 100644 --- a/litellm/ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx @@ -15,6 +15,7 @@ import { modelAvailableCall, vectorStoreListCall, } from "../networking"; +import { toast } from "@/lib/toast"; import { KeyEditView } from "./key_edit_view"; import { teamAdminMemberKeyPayload } from "./teamAdminMemberKeyPayload"; @@ -2577,6 +2578,7 @@ describe("KeyEditView", () => { models: [], max_budget: 0, soft_budget: null, + max_budget_alert_emails: [], budget_duration: "30d", tpm_limit: 10, tpm_limit_type: null, @@ -3308,4 +3310,185 @@ describe("KeyEditView", () => { }); }); }); + + describe("budget alert recipients", () => { + const KEY_WITH_ALERTS: KeyResponse = { + ...MOCK_KEY_DATA, + max_budget: 100, + metadata: { + ...MOCK_KEY_DATA.metadata, + cost_center: "ops", + max_budget_alert_emails: { "80": ["shared-mailbox@example.com", "finance-dl@example.com"], "50": [] }, + }, + }; + + const renderAlerts = ( + onSubmit: (values: Record) => Promise, + editingMemberKeyAsTeamAdmin = false, + keyData: KeyResponse = KEY_WITH_ALERTS, + ) => + renderWithProviders( + {}} + onSubmit={onSubmit} + accessToken={"test-token"} + userID={"test-user"} + userRole={"Admin"} + premiumUser={true} + editingMemberKeyAsTeamAdmin={editingMemberKeyAsTeamAdmin} + />, + ); + + const save = async (onSubmit: ReturnType) => { + await userEvent.click(screen.getByRole("button", { name: /save changes/i })); + await waitFor(() => expect(onSubmit).toHaveBeenCalledTimes(1)); + return onSubmit.mock.calls[0][0]; + }; + + it("shows the stored thresholds as rows and keeps them out of the metadata JSON box", async () => { + renderAlerts(vi.fn()); + + const thresholds = await screen.findAllByPlaceholderText("% of budget"); + expect(thresholds.map((input) => (input as HTMLInputElement).value)).toEqual(["50", "80"]); + expect(screen.getAllByPlaceholderText("Additional recipients, e.g. finance@example.com")[1]).toHaveValue( + "shared-mailbox@example.com, finance-dl@example.com", + ); + expect(JSON.parse((screen.getByLabelText("Metadata") as HTMLTextAreaElement).value)).toEqual({ + cost_center: "ops", + }); + }); + + it("shows only the stored entries the rows cannot show in the metadata JSON box", async () => { + renderAlerts(vi.fn(), false, { + ...KEY_WITH_ALERTS, + metadata: { + cost_center: "ops", + max_budget_alert_emails: { "80": ["row@example.com"], "150": ["over@example.com"], "0": [] }, + }, + }); + + const thresholds = await screen.findAllByPlaceholderText("% of budget"); + expect(thresholds.map((input) => (input as HTMLInputElement).value)).toEqual(["80"]); + expect(JSON.parse((screen.getByLabelText("Metadata") as HTMLTextAreaElement).value)).toEqual({ + cost_center: "ops", + max_budget_alert_emails: { "150": ["over@example.com"], "0": [] }, + }); + }); + + it("submits an untouched key's rows unchanged", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderAlerts(onSubmit); + await screen.findAllByPlaceholderText("% of budget"); + + expect((await save(onSubmit)).max_budget_alert_emails).toEqual([ + { threshold: 50, emails: "" }, + { threshold: 80, emails: "shared-mailbox@example.com, finance-dl@example.com" }, + ]); + expect(onSubmit.mock.calls[0][1]).not.toContain("max_budget_alert_emails"); + }); + + it("submits an added row and drops a removed one", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderAlerts(onSubmit); + await screen.findAllByPlaceholderText("% of budget"); + + await userEvent.click(screen.getAllByRole("button", { name: "Remove budget alert threshold" })[0]); + await userEvent.click(screen.getByRole("button", { name: "Add Budget Alert Threshold" })); + const thresholds = screen.getAllByPlaceholderText("% of budget"); + fireEvent.change(thresholds[thresholds.length - 1], { target: { value: "100" } }); + const recipients = screen.getAllByPlaceholderText("Additional recipients, e.g. finance@example.com"); + fireEvent.change(recipients[recipients.length - 1], { target: { value: "ap-team@example.com" } }); + + expect((await save(onSubmit)).max_budget_alert_emails).toEqual([ + { threshold: 80, emails: "shared-mailbox@example.com, finance-dl@example.com" }, + { threshold: 100, emails: "ap-team@example.com" }, + ]); + expect(onSubmit.mock.calls[0][1]).toContain("max_budget_alert_emails"); + }); + + it("reports the rows as edited when every row is removed, so the save clears them", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderAlerts(onSubmit); + await screen.findAllByPlaceholderText("% of budget"); + + for (const button of screen.getAllByRole("button", { name: "Remove budget alert threshold" }).reverse()) { + await userEvent.click(button); + } + + expect((await save(onSubmit)).max_budget_alert_emails).toEqual([]); + expect(onSubmit.mock.calls[0][1]).toContain("max_budget_alert_emails"); + }); + + it("moves a recipient map typed into the metadata JSON box into the rows on blur", async () => { + const info = vi.spyOn(toast, "info"); + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderAlerts(onSubmit); + await screen.findAllByPlaceholderText("% of budget"); + + const metadataBox = screen.getByLabelText("Metadata"); + fireEvent.change(metadataBox, { + target: { + value: JSON.stringify({ + cost_center: "ops", + max_budget_alert_emails: { "80": ["typed@example.com"], "95": [] }, + }), + }, + }); + fireEvent.blur(metadataBox); + + expect(JSON.parse((metadataBox as HTMLTextAreaElement).value)).toEqual({ cost_center: "ops" }); + expect(info).toHaveBeenCalledWith( + "Moved alert thresholds 80%, 95% from metadata to the Budget Alert Thresholds rows", + ); + expect((await save(onSubmit)).max_budget_alert_emails).toEqual([ + { threshold: 50, emails: "" }, + { threshold: 80, emails: "typed@example.com" }, + { threshold: 95, emails: "" }, + ]); + expect(onSubmit.mock.calls[0][1]).toContain("max_budget_alert_emails"); + }); + + it("moves a recipient map typed into the metadata JSON box into the rows even without a blur", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderAlerts(onSubmit); + await screen.findAllByPlaceholderText("% of budget"); + + fireEvent.change(screen.getByLabelText("Metadata"), { + target: { + value: JSON.stringify({ cost_center: "ops", max_budget_alert_emails: { "80": ["typed@example.com"] } }), + }, + }); + fireEvent.submit(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(onSubmit).toHaveBeenCalled()); + expect(onSubmit.mock.calls[0][0].max_budget_alert_emails).toEqual([ + { threshold: 50, emails: "" }, + { threshold: 80, emails: "typed@example.com" }, + ]); + expect(onSubmit.mock.calls[0][1]).toContain("max_budget_alert_emails"); + }); + + it("hides the rows from a team admin editing a member's key", async () => { + renderAlerts(vi.fn(), true); + + expect(await screen.findByLabelText("Metadata")).toBeInTheDocument(); + expect(screen.queryByPlaceholderText("% of budget")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Add Budget Alert Threshold" })).not.toBeInTheDocument(); + }); + + it("blocks the save on a duplicate threshold", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderAlerts(onSubmit); + await screen.findAllByPlaceholderText("% of budget"); + + await userEvent.click(screen.getByRole("button", { name: "Add Budget Alert Threshold" })); + const thresholds = screen.getAllByPlaceholderText("% of budget"); + fireEvent.change(thresholds[thresholds.length - 1], { target: { value: "80" } }); + await userEvent.click(screen.getByRole("button", { name: /save changes/i })); + + expect(await screen.findAllByText("Duplicate threshold")).toHaveLength(2); + expect(onSubmit).not.toHaveBeenCalled(); + }); + }); }); diff --git a/litellm/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/litellm/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index bafeedd5be7..1bb6676470c 100644 --- a/litellm/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/litellm/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -38,6 +38,7 @@ import { KeyRateLimitFields, KeyTypeSelect, labelWithHint, + moveMetadataBudgetAlertsToKeyRows, moveMetadataTagsToTagsField, } from "./KeyEditViewControls"; import { @@ -49,6 +50,7 @@ import { clearMcpGrants, writeMcpToolPermissions, } from "./keyEditFormValues"; +import { BudgetAlertEmailsEditor } from "../key_team_helpers/BudgetAlertEmailsEditor"; import { BudgetFallbacksEditor } from "../key_team_helpers/BudgetFallbacksEditor"; import { END_USER_BUDGET_HINT, EndUserBudgetSelect } from "../key_team_helpers/EndUserBudgetSelect"; import { @@ -90,6 +92,7 @@ interface KeyEditViewProps { userID: string | null; userRole: string | null; premiumUser?: boolean; + editingMemberKeyAsTeamAdmin?: boolean; } const sessionKey = (keyData: KeyResponse) => keyData.token || keyData.token_id; @@ -105,6 +108,7 @@ function KeyEditSession({ userID, userRole, premiumUser = false, + editingMemberKeyAsTeamAdmin = false, }: KeyEditViewProps) { const { t } = useTranslation(); const canEditGuardrails = premiumUser || (userRole != null && rolesWithWriteAccess.includes(userRole)); @@ -323,7 +327,7 @@ function KeyEditSession({ ...values, ...(detachProject && enableProjectsUI && canDetachProject ? { project_id: null } : {}), }), - [...Object.keys(dirtyFields), ...(budgetLimitsUnchanged ? [] : ["budget_limits"])], + [...Object.keys(form.formState.dirtyFields), ...(budgetLimitsUnchanged ? [] : ["budget_limits"])], ); } finally { setIsKeySaving(false); @@ -373,16 +377,17 @@ function KeyEditSession({ ); }; + const submitForm = (event: React.FormEvent) => { + moveMetadataTagsToTagsField(form, t); + if (!editingMemberKeyAsTeamAdmin) moveMetadataBudgetAlertsToKeyRows(form, t); + return form.handleSubmit((values) => handleSubmit(toSubmittedValues(values, { canViewPolicies, canViewPrompts })))( + event, + ); + }; + return ( -
{ - moveMetadataTagsToTagsField(form, t); - return form.handleSubmit((values) => - handleSubmit(toSubmittedValues(values, { canViewPolicies, canViewPrompts })), - )(event); - }} - > + {(field) => } @@ -467,6 +472,23 @@ function KeyEditSession({ placeholder={t("keyDetail.keyEditView.fields.softBudgetPlaceholder")} /> + {!editingMemberKeyAsTeamAdmin && ( + + )} + - +
diff --git a/litellm/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx b/litellm/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx index 8a30c6ff7a7..78607a70058 100644 --- a/litellm/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx @@ -17,6 +17,7 @@ import { useModelAccessGroupNames } from "@/app/(dashboard)/hooks/models/useMode const editViewMocks = vi.hoisted(() => ({ onSubmit: undefined as ((v: Record) => Promise) | undefined, + editingMemberKeyAsTeamAdmin: undefined as boolean | undefined, })); vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }), usePathname: () => "/ui/alerts" })); @@ -26,8 +27,15 @@ vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ })); vi.mock("./key_edit_view", () => ({ - KeyEditView: ({ onSubmit }: { onSubmit: (v: Record) => Promise }) => { + KeyEditView: ({ + onSubmit, + editingMemberKeyAsTeamAdmin, + }: { + onSubmit: (v: Record) => Promise; + editingMemberKeyAsTeamAdmin?: boolean; + }) => { editViewMocks.onSubmit = onSubmit; + editViewMocks.editingMemberKeyAsTeamAdmin = editingMemberKeyAsTeamAdmin; return
; }, })); @@ -892,7 +900,9 @@ describe("KeyInfoView", () => { await renderAndOpenSettingsTab({ ...MOCK_KEY_DATA, user_id: "someone-else-id" }); - expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: /edit settings/i })); + expect(await screen.findByTestId("key-edit-view-stub")).toBeInTheDocument(); + expect(editViewMocks.editingMemberKeyAsTeamAdmin).toBe(false); }); it("should show the Edit Settings button when the user is the key owner", async () => { @@ -961,7 +971,9 @@ describe("KeyInfoView", () => { await renderAndOpenSettingsTab({ ...MOCK_KEY_DATA, team_id: teamId, user_id: "other-user-id" }); - expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: /edit settings/i })); + expect(await screen.findByTestId("key-edit-view-stub")).toBeInTheDocument(); + expect(editViewMocks.editingMemberKeyAsTeamAdmin).toBe(true); }); }); diff --git a/litellm/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/litellm/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 317a3c309ab..8e5f9262101 100644 --- a/litellm/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/litellm/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -39,6 +39,12 @@ import DeleteResourceModal from "../common_components/DeleteResourceModal"; import RouterSettingsSummary from "../common_components/RouterSettingsSummary"; import { hasRouterSettings } from "../common_components/routerSettingsPayload"; import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata } from "../key_info_utils"; +import { + budgetAlertEmailsFromRows, + budgetAlertEmailsMetadata, + budgetAlertEmailsRowsCanShow, + KEY_MAX_BUDGET_ALERT_EMAILS_KEY, +} from "../key_team_helpers/budgetAlertEmails"; import { KeyResponse } from "../key_team_helpers/key_list"; import LoggingSettingsView from "../logging_settings_view"; import { toast } from "@/lib/toast"; @@ -219,6 +225,15 @@ export default function KeyInfoView({ ); } + const memberKeyEditContext = { + userRole: userRole || "", + userId: userID || "", + keyUserId: currentKeyData.user_id, + keyTeamId: currentKeyData.team_id, + teamMembers: teamsData?.find((team) => team.team_id === currentKeyData.team_id)?.members_with_roles, + }; + const editingMemberKeyAsTeamAdmin = isTeamAdminEditingMemberKey(memberKeyEditContext); + const handleKeyUpdate = async (formValues: Record, dirtyFields: readonly string[] = []) => { try { if (!accessToken) return; @@ -334,6 +349,12 @@ export default function KeyInfoView({ formValues.tpd_limit = mapEmptyStringToNull(formValues.tpd_limit); formValues.max_parallel_requests = mapEmptyStringToNull(formValues.max_parallel_requests); + const budgetAlertRowEntries = dirtyFields.includes(KEY_MAX_BUDGET_ALERT_EMAILS_KEY) + ? budgetAlertEmailsFromRows(formValues.max_budget_alert_emails ?? []) + : budgetAlertEmailsRowsCanShow(currentKeyData.metadata, KEY_MAX_BUDGET_ALERT_EMAILS_KEY); + const budgetAlertEmailsWithBoxEntries = (boxMetadata: unknown) => + budgetAlertEmailsMetadata(budgetAlertRowEntries, KEY_MAX_BUDGET_ALERT_EMAILS_KEY, boxMetadata); + // Convert metadata back to an object if it exists and is a string if (formValues.metadata && typeof formValues.metadata === "string") { try { @@ -342,9 +363,12 @@ export default function KeyInfoView({ if ("tags" in parsedMetadata) { delete parsedMetadata["tags"]; } + const budgetAlertEmails = budgetAlertEmailsWithBoxEntries(parsedMetadata); + delete parsedMetadata[KEY_MAX_BUDGET_ALERT_EMAILS_KEY]; formValues.metadata = { ...parsedMetadata, ...(Array.isArray(formValues.tags) && formValues.tags.length > 0 ? { tags: formValues.tags } : {}), + ...budgetAlertEmails, ...(formValues.guardrails?.length > 0 ? { guardrails: formValues.guardrails } : {}), ...(Array.isArray(formValues.logging_settings) && formValues.logging_settings.length > 0 ? { logging: formValues.logging_settings } @@ -362,10 +386,11 @@ export default function KeyInfoView({ } } else { const baseMetadata = formValues.metadata || {}; - const { tags: _omitTags, ...rest } = baseMetadata; + const { tags: _omitTags, [KEY_MAX_BUDGET_ALERT_EMAILS_KEY]: _boxAlertEmails, ...rest } = baseMetadata; formValues.metadata = { ...rest, ...(Array.isArray(formValues.tags) && formValues.tags.length > 0 ? { tags: formValues.tags } : {}), + ...budgetAlertEmailsWithBoxEntries(baseMetadata), ...(formValues.guardrails?.length > 0 ? { guardrails: formValues.guardrails } : {}), ...(Array.isArray(formValues.logging_settings) && formValues.logging_settings.length > 0 ? { logging: formValues.logging_settings } @@ -383,6 +408,7 @@ export default function KeyInfoView({ delete formValues.tags; } delete formValues.logging_settings; + delete formValues.max_budget_alert_emails; // Normalize any legacy word-form budget_duration to the canonical API format if (formValues.budget_duration) { @@ -395,14 +421,6 @@ export default function KeyInfoView({ formValues.budget_duration = wordToCanonical[formValues.budget_duration] ?? formValues.budget_duration; } - const memberKeyEditContext = { - userRole: userRole || "", - userId: userID || "", - keyUserId: currentKeyData.user_id, - keyTeamId: currentKeyData.team_id, - teamMembers: teamsData?.find((team) => team.team_id === currentKeyData.team_id)?.members_with_roles, - }; - const editingMemberKeyAsTeamAdmin = isTeamAdminEditingMemberKey(memberKeyEditContext); if (editingMemberKeyAsTeamAdmin) { const trimmed = teamAdminMemberKeyPayload(formValues, dirtyFields); if (trimmed.kind === "blocked") { @@ -1010,6 +1028,7 @@ export default function KeyInfoView({ userID={userID} userRole={userRole} premiumUser={premiumUser} + editingMemberKeyAsTeamAdmin={editingMemberKeyAsTeamAdmin} /> ) : (
diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json index 6e09e3eb785..bdade69fef6 100644 --- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json @@ -2391,6 +2391,17 @@ "description": "Tags are managed by the Tags field above. A tags array typed here is moved to that field." } }, + "maxBudgetAlertEmails": { + "budgetAlertThresholds": "Budget Alert Thresholds", + "budgetAlertThresholdsHint": "Thresholds are a percentage of the key's max budget and replace this key's default 80% alert. Global alert thresholds configured on the proxy still apply. Thresholds outside 1 to 100 can be edited in the Metadata box. The key owner's email is always included. Add comma-separated addresses, such as a shared mailbox or distribution list, to notify others as well. Requires email alerting to be configured on the proxy.", + "thresholdPlaceholder": "% of budget", + "recipientsPlaceholder": "Additional recipients, e.g. finance@example.com", + "removeThreshold": "Remove budget alert threshold", + "addThreshold": "Add Budget Alert Threshold", + "thresholdRange": "Enter a whole number from 1 to 100", + "duplicateThreshold": "Duplicate threshold", + "movedThresholds": "Moved alert thresholds {{thresholds}} from metadata to the Budget Alert Thresholds rows" + }, "estimatedOutputTokens": { "invalidPerModel": "Enter a JSON object of positive integers, e.g. {\"gpt-4\": 4096}", "invalidPositive": "Enter a positive integer", diff --git a/tests/e2e/ui/tests/proxy-admin/keyBudgetAlertRecipients.spec.ts b/tests/e2e/ui/tests/proxy-admin/keyBudgetAlertRecipients.spec.ts new file mode 100644 index 00000000000..af9c0942331 --- /dev/null +++ b/tests/e2e/ui/tests/proxy-admin/keyBudgetAlertRecipients.spec.ts @@ -0,0 +1,164 @@ +import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; +import { + ADMIN_STORAGE_PATH, + E2E_TEAM_CRUD_ALIAS, + E2E_TEAM_CRUD_ID, +} from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { + dismissFeedbackPopup, + navigateToPage, + openKeyDetail, +} from "../../helpers/navigation"; +import { readBack } from "../../helpers/roundTrip"; +import { + createVirtualKey, + deleteVirtualKey, + uniqueSuffix, +} from "../../helpers/traffic"; + +const SHARED_MAILBOX = "shared-mailbox@example.com"; +const DISTRIBUTION_LIST = "finance-dl@example.com"; +const RECIPIENTS_PLACEHOLDER = + "Additional recipients, e.g. finance@example.com"; + +async function keyMetadata( + page: PlaywrightPage, + token: string, +): Promise> { + const body = await readBack<{ info: { metadata: Record } }>( + page, + `/key/info?key=${encodeURIComponent(token)}`, + ); + return body.info.metadata; +} + +async function tokenForAlias( + page: PlaywrightPage, + alias: string, +): Promise { + const body = await readBack<{ keys: { key_alias: string; token: string }[] }>( + page, + `/key/list?key_alias=${encodeURIComponent(alias)}&return_full_object=true&size=100`, + ); + return body.keys.find((row) => row.key_alias === alias)?.token; +} + +async function addAlertRow( + page: PlaywrightPage, + threshold: string, + emails: string, +): Promise { + await page + .getByRole("button", { name: "Add Budget Alert Threshold" }) + .click(); + await page.getByPlaceholder("% of budget").last().fill(threshold); + await page.getByPlaceholder(RECIPIENTS_PLACEHOLDER).last().fill(emails); +} + +test.describe("Proxy Admin - Key budget alert recipients", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Create Key stores the alert recipients next to the metadata JSON", async ({ + page, + }) => { + const alias = `e2e-alert-create-${uniqueSuffix()}`; + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + await page.getByRole("button", { name: /Create New Key/i }).click(); + await page.getByLabel(/Key Name/).fill(alias); + await page.getByTestId("team-dropdown").getByRole("combobox").click(); + await page.keyboard.type(E2E_TEAM_CRUD_ALIAS); + await page + .getByRole("option", { name: E2E_TEAM_CRUD_ALIAS }) + .first() + .click(); + await page.getByRole("combobox", { name: "Select models" }).click(); + await page + .getByRole("option", { name: "All Team Models", exact: true }) + .click(); + await page.keyboard.press("Escape"); + + await page.getByRole("button", { name: /Optional Settings/i }).click(); + await page.getByLabel(/Max Budget \(USD\)/).fill("10"); + await addAlertRow(page, "80", `${SHARED_MAILBOX}, ${DISTRIBUTION_LIST}`); + await page.getByLabel("Metadata").fill('{"cost_center": "ops"}'); + await page.getByRole("button", { name: "Create Key", exact: true }).click(); + await expect(page.getByText("Save your Key")).toBeVisible({ + timeout: 10_000, + }); + + const token = await tokenForAlias(page, alias); + expect(token, `key ${alias} readable from /key/list`).toBeTruthy(); + try { + expect(await keyMetadata(page, token!)).toMatchObject({ + cost_center: "ops", + max_budget_alert_emails: { "80": [SHARED_MAILBOX, DISTRIBUTION_LIST] }, + }); + } finally { + await deleteVirtualKey(page.request, token!); + } + }); + + test("Edit Key round-trips the rows, keeps the rest of the metadata, and clears them when every row is removed", async ({ + page, + }) => { + const alias = `e2e-alert-edit-${uniqueSuffix()}`; + const created = await createVirtualKey(page.request, { + key_alias: alias, + team_id: E2E_TEAM_CRUD_ID, + max_budget: 10, + metadata: { + cost_center: "ops", + max_budget_alert_emails: { "80": [SHARED_MAILBOX] }, + }, + }); + try { + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + await openKeyDetail(page, alias); + await page.getByRole("tab", { name: "Settings" }).click(); + await page.getByRole("button", { name: "Edit Settings" }).click(); + + await expect(page.getByPlaceholder("% of budget")).toHaveValue("80"); + await expect(page.getByPlaceholder(RECIPIENTS_PLACEHOLDER)).toHaveValue( + SHARED_MAILBOX, + ); + expect( + JSON.parse(await page.getByLabel("Metadata").inputValue()), + ).toEqual({ cost_center: "ops" }); + + await page + .getByRole("button", { name: "Remove budget alert threshold" }) + .click(); + await addAlertRow(page, "50", DISTRIBUTION_LIST); + await page.getByRole("button", { name: "Save Changes" }).click(); + await expect + .poll(() => keyMetadata(page, created.token), { timeout: 20_000 }) + .toMatchObject({ + cost_center: "ops", + max_budget_alert_emails: { "50": [DISTRIBUTION_LIST] }, + }); + + await page.getByRole("button", { name: "Edit Settings" }).click(); + await expect(page.getByPlaceholder("% of budget")).toHaveValue("50"); + await page + .getByRole("button", { name: "Remove budget alert threshold" }) + .click(); + await page.getByRole("button", { name: "Save Changes" }).click(); + await expect + .poll( + async () => + "max_budget_alert_emails" in + (await keyMetadata(page, created.token)), + { timeout: 20_000 }, + ) + .toBe(false); + expect(await keyMetadata(page, created.token)).toMatchObject({ + cost_center: "ops", + }); + } finally { + await deleteVirtualKey(page.request, created.token); + } + }); +}); diff --git a/tests/integration/spend/test_key_budget_alerts.py b/tests/integration/spend/test_key_budget_alerts.py new file mode 100644 index 00000000000..a904d2ba03a --- /dev/null +++ b/tests/integration/spend/test_key_budget_alerts.py @@ -0,0 +1,74 @@ +import uuid +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.mail import smtp_sink +from integration._support.process import owned_proxy + +pytestmark: Final = pytest.mark.shares_worker_state + +KEY_BUDGET: Final = 0.10 +CALL_COST: Final = 20 * 0.001 + 20 * 0.002 + + +def _key_spend(key: str) -> float: + rows: Final = read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE key_alias = %s', (key,)) + return float(str(rows[0]["spend"])) if rows else 0.0 + + +def test_key_threshold_emails_owner_and_shared_mailbox_from_admin_ui_metadata(gateway: Gateway, tmp_path: Path) -> None: + owner_email: Final = f"owner-{uuid.uuid4().hex}@integration.test" + shared_mailbox: Final = f"shared-mailbox-{uuid.uuid4().hex}@integration.test" + alias: Final = f"alert-key-{uuid.uuid4().hex}" + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["general_settings"]["alerting"] = ["email"] + configuration["general_settings"]["user_api_key_cache_ttl"] = 1 + path: Final = tmp_path / "email-alerting.yaml" + path.write_text(yaml.safe_dump(configuration)) + with smtp_sink() as mailbox: + overrides: Final = { + "SMTP_HOST": mailbox.host, + "SMTP_PORT": str(mailbox.port), + "SMTP_TLS": "False", + "SMTP_SENDER_EMAIL": "alerts@integration.test", + } + with owned_proxy(gateway, tmp_path, overrides, config=path) as candidate, candidate.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + user_id: Final = scenario.user(user_email=owner_email) + key: Final = scenario.key( + key_alias=alias, + user_id=user_id, + models=[model], + max_budget=KEY_BUDGET, + metadata={"cost_center": "ops", "max_budget_alert_emails": {"50": [shared_mailbox]}}, + ) + + first: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "first call"}]}, + key=key, + ) + assert first.status_code == 200, first.text + assert float(first.headers["x-litellm-response-cost"]) == pytest.approx(CALL_COST) + eventually(lambda: _key_spend(alias), lambda spend: spend == pytest.approx(CALL_COST), seconds=70) + assert mailbox.deliveries() == (), "no threshold is reached before the first call is recorded" + + second: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "second call"}]}, + key=key, + ) + assert second.status_code == 200, second.text + halfway: Final = eventually(mailbox.deliveries, lambda found: len(found) >= 2, seconds=30) + assert {recipient for delivery in halfway for recipient in delivery.recipients} == { + owner_email, + shared_mailbox, + }, halfway + assert all("50%" in delivery.subject for delivery in halfway), halfway + assert len(halfway) == 2, halfway From effcb0705fbb2216ab36211b5ea074f457673910 Mon Sep 17 00:00:00 2001 From: kenneth-berri Date: Thu, 12 Nov 2026 15:41:50 -0800 Subject: [PATCH 2/9] fix(embeddings): unwrap extra_body for providers with their own embedding handlers (#51335) * fix(embeddings): unwrap extra_body for providers with their own embedding handlers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(embeddings): unwrap extra_body in get_optional_params_embeddings and keep lint gates flat Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(embeddings): let mapped params win over extra_body and drop the unused merge helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(embeddings): merge extra_body into the provider body after each native embedding transform Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(embeddings): let the transformed body win over extra_body so input and mapped params cannot be replaced Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(embeddings): sign the Bedrock body that is sent and align the shared handler so extra_body never replaces the input Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(embeddings): keep the shared handler's extra_body precedence, the router batch limit relies on extra_body.input Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/base_llm/embedding/transformation.py | 22 +++- litellm/llms/bedrock/embed/embedding.py | 25 ++-- litellm/llms/nvidia_nim/embed.py | 10 +- litellm/llms/ollama/completion/handler.py | 6 +- litellm/llms/sagemaker/completion/handler.py | 8 +- .../batch_embed_content_transformation.py | 9 +- .../multimodal_embeddings/transformation.py | 7 +- .../vertex_embeddings/transformation.py | 17 ++- litellm/utils.py | 10 +- .../test_provider_embedding_translation.py | 1 + .../bedrock/embed/test_bedrock_embedding.py | 119 ++++++++++++++++++ tests/unit/llms/nvidia_nim/test_nvidia_nim.py | 25 ++++ .../unit/llms/ollama/test_ollama_embedding.py | 18 +++ .../test_sagemaker_embedding_voyage.py | 37 ++++++ ...test_batch_embed_content_transformation.py | 29 +++++ ..._ai_multimodal_embedding_transformation.py | 32 +++++ .../test_embedding_handler.py | 17 +++ 17 files changed, 365 insertions(+), 27 deletions(-) diff --git a/litellm/llms/base_llm/embedding/transformation.py b/litellm/llms/base_llm/embedding/transformation.py index 46ac3ccf433..d25a0c3d524 100644 --- a/litellm/llms/base_llm/embedding/transformation.py +++ b/litellm/llms/base_llm/embedding/transformation.py @@ -1,7 +1,9 @@ from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues @@ -16,6 +18,24 @@ else: LiteLLMLoggingObj = Any +_BODY_FIELDS: Final = TypeAdapter(dict[str, object]) + + +def extra_body_fields(optional_params: Mapping[str, object] | None) -> Mapping[str, object]: + """The caller's ``extra_body`` fields, merged under the provider body after the transform so they add fields but never + replace the input or a mapped param.""" + extra_body: Final = (optional_params or {}).get("extra_body") + return _BODY_FIELDS.validate_python(extra_body) if isinstance(extra_body, Mapping) else {} + + +def split_extra_body( + optional_params: Mapping[str, object] | None, +) -> tuple[Mapping[str, object], Mapping[str, object]]: + """The params without ``extra_body``, and the ``extra_body`` fields to merge into the provider body.""" + params: Final = _BODY_FIELDS.validate_python(optional_params or {}) + return {k: v for k, v in params.items() if k != "extra_body"}, extra_body_fields(params) + + class BaseEmbeddingConfig(BaseConfig, ABC): @abstractmethod def transform_embedding_request( diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index e28e39ef2d4..0743bd6e7e7 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -6,6 +6,7 @@ import copy import json import urllib.parse from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, Final, get_args, overload import httpx @@ -15,6 +16,7 @@ import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.embedding.transformation import extra_body_fields from litellm.llms.cohere.embed.handler import embedding as cohere_embedding from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -103,7 +105,7 @@ class BedrockEmbedding(BaseAWSLLM): region_from_model: str | None = None, ) -> tuple[Credentials | None, str]: auth_params: Final = pop_aws_auth_params(optional_params) - explicit_region_name: Final = optional_params.pop("aws_region_name", None) + explicit_region_name: Final = _OPTIONAL_STR.validate_python(optional_params.pop("aws_region_name", None)) aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name ### SET REGION NAME ### @@ -291,9 +293,10 @@ class BedrockEmbedding(BaseAWSLLM): provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL, api_key: str | None = None, is_async_invoke: bool | None = False, + extra_body: Mapping[str, object] = MappingProxyType({}), ): responses: Final[list[dict]] = [] - for data in batch_data: + for data in ({**extra_body, **item} for item in batch_data): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} @@ -360,9 +363,10 @@ class BedrockEmbedding(BaseAWSLLM): provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL, api_key: str | None = None, is_async_invoke: bool | None = False, + extra_body: Mapping[str, object] = MappingProxyType({}), ): responses: Final[list[dict]] = [] - for data in batch_data: + for data in ({**extra_body, **item} for item in batch_data): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} @@ -462,9 +466,12 @@ class BedrockEmbedding(BaseAWSLLM): ) inference_params = copy.deepcopy(optional_params) inference_params = { - k: v for k, v in inference_params.items() if k.lower() not in self.aws_authentication_params + k: v + for k, v in inference_params.items() + if k.lower() not in self.aws_authentication_params and k != "extra_body" } inference_params.pop("user", None) # make sure user is not passed in for bedrock call + extra_body: Final = extra_body_fields(optional_params) data: CohereEmbeddingRequest | None = None batch_data: list | None = None @@ -535,6 +542,8 @@ class BedrockEmbedding(BaseAWSLLM): ) batch_data.append(nova_request) + request_data: Final = None if data is None else {**extra_body, **data} + ### SET RUNTIME ENDPOINT ### endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( api_base=api_base, @@ -552,6 +561,7 @@ class BedrockEmbedding(BaseAWSLLM): client=(client if client is not None and isinstance(client, AsyncHTTPHandler) else None), timeout=timeout, batch_data=batch_data, + extra_body=extra_body, credentials=credentials, extra_headers=extra_headers, endpoint_url=endpoint_url, @@ -566,6 +576,7 @@ class BedrockEmbedding(BaseAWSLLM): client=(client if client is not None and isinstance(client, HTTPHandler) else None), timeout=timeout, batch_data=batch_data, + extra_body=extra_body, credentials=credentials, extra_headers=extra_headers, endpoint_url=endpoint_url, @@ -579,7 +590,7 @@ class BedrockEmbedding(BaseAWSLLM): if returned_response is None: raise Exception("Unable to map Bedrock request to provider") return returned_response - elif data is None: + elif request_data is None: raise Exception("Unable to map Bedrock request to provider") headers = {"Content-Type": "application/json"} @@ -591,7 +602,7 @@ class BedrockEmbedding(BaseAWSLLM): aws_region_name=aws_region_name, extra_headers=extra_headers, endpoint_url=endpoint_url, - data=json.dumps(data), + data=json.dumps(request_data), headers=headers, api_key=api_key, ) @@ -607,7 +618,7 @@ class BedrockEmbedding(BaseAWSLLM): logging_obj=logging_obj, optional_params=optional_params, encoding=encoding, - data=data, + data=request_data, complete_api_base=prepped.url, api_key=None, aembedding=aembedding, diff --git a/litellm/llms/nvidia_nim/embed.py b/litellm/llms/nvidia_nim/embed.py index d2ef446299e..7368831f7a8 100644 --- a/litellm/llms/nvidia_nim/embed.py +++ b/litellm/llms/nvidia_nim/embed.py @@ -66,7 +66,7 @@ class NvidiaNimEmbeddingConfig: self, non_default_params: dict, optional_params: dict, - kwargs: dict | None = None, + kwargs: Mapping[str, object] | None = None, ): if "extra_body" not in optional_params: optional_params["extra_body"] = {} @@ -80,5 +80,11 @@ class NvidiaNimEmbeddingConfig: if kwargs is not None: # pass kwargs in extra_body - optional_params["extra_body"].update(kwargs) + nested_extra_body: Final = kwargs.get("extra_body") + optional_params["extra_body"].update( + { + **{k: v for k, v in kwargs.items() if k != "extra_body"}, + **(nested_extra_body if isinstance(nested_extra_body, Mapping) else {}), + } + ) return optional_params diff --git a/litellm/llms/ollama/completion/handler.py b/litellm/llms/ollama/completion/handler.py index e1fc9a8d262..fd874246980 100644 --- a/litellm/llms/ollama/completion/handler.py +++ b/litellm/llms/ollama/completion/handler.py @@ -14,6 +14,7 @@ from typing_extensions import NotRequired, ReadOnly import litellm from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.litellm_core_utils.url_utils import ensure_api_route +from litellm.llms.base_llm.embedding.transformation import split_extra_body from litellm.types.mcp import has_header from litellm.types.utils import EmbeddingResponse @@ -45,14 +46,15 @@ def _prepare_ollama_embedding_payload( data: Final[dict[str, object]] = {"model": model, "input": prompts} special_optional_params: Final = ["truncate", "options", "keep_alive", "dimensions"] - for k, v in optional_params.items(): + body_params, extra_body = split_extra_body(optional_params) + for k, v in body_params.items(): if k in special_optional_params: data[k] = v else: data.setdefault("options", {}) if isinstance(data["options"], dict): data["options"].update({k: v}) - return data + return {**extra_body, **data} def _process_ollama_embedding_response( diff --git a/litellm/llms/sagemaker/completion/handler.py b/litellm/llms/sagemaker/completion/handler.py index a852ae8b1a1..c20f0df6768 100644 --- a/litellm/llms/sagemaker/completion/handler.py +++ b/litellm/llms/sagemaker/completion/handler.py @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.optional_imports import ensure_optional_import +from litellm.llms.base_llm.embedding.transformation import extra_body_fields from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, pop_aws_auth_params from litellm.llms.bedrock.web_identity_session_policy import SAGEMAKER_SESSION_POLICY, WebIdentitySessionPolicy from litellm.llms.custom_httpx.http_handler import ( @@ -594,8 +595,11 @@ class SagemakerLLM(BaseAWSLLM): inference_component_name: Final = embedding_params.get("model_id") if inference_component_name is not None and not isinstance(inference_component_name, str): raise SagemakerError(status_code=400, message="model_id must be the inference component name as a string") - body_params: Final = {k: v for k, v in embedding_params.items() if k != "model_id"} - request_data: Final = provider_config.transform_embedding_request(endpoint_name, input, body_params, {}) + body_params: Final = {k: v for k, v in embedding_params.items() if k not in ("model_id", "extra_body")} + request_data: Final = { + **extra_body_fields(embedding_params), + **provider_config.transform_embedding_request(endpoint_name, input, body_params, {}), + } data: Final = json.dumps(request_data).encode("utf-8") invoke_args: Final = _InvokeEndpointArgs( EndpointName=endpoint_name, diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 96a785c06c9..b27003369f7 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -10,6 +10,7 @@ from typing import Annotated, Final, Literal, cast from pydantic import ConfigDict, Field, TypeAdapter, ValidationError from litellm.exceptions import BadRequestError +from litellm.llms.base_llm.embedding.transformation import extra_body_fields from litellm.llms.vertex_ai.common_utils import GEMINI_FILES_API_URI_PREFIX, gemini_video_metadata_from_openai from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.vertex_ai import ( @@ -414,6 +415,7 @@ def transform_openai_input_gemini_content( gemini_model_name: Final = f"models/{model}" gemini_params: Final = _filter_embed_params(optional_params) + extra_body: Final = extra_body_fields(optional_params) input_list: Final = _top_level_elements(input) requests: Final[list[EmbedContentRequest]] = [] @@ -432,7 +434,11 @@ def transform_openai_input_gemini_content( content=ContentType(parts=parts), **gemini_params, ) - requests.append(request) + requests.append( + cast( # cast-ok: extra_body carries raw Gemini fields the TypedDict does not list + EmbedContentRequest, {**extra_body, **request} + ) + ) return VertexAIBatchEmbeddingsRequestBody(requests=requests) @@ -472,6 +478,7 @@ def transform_openai_input_gemini_embed_content( parts.append(_build_part_for_input(element, resolved_files=resolved_files, drop_params=drop_params)) request_body: Final[dict] = { + **extra_body_fields(optional_params), "content": ContentType(parts=parts), **gemini_params, } diff --git a/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py b/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py index c282ae22ea6..7c276b43dc7 100644 --- a/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py @@ -21,7 +21,7 @@ from litellm.types.utils import ( ) from litellm.utils import count_characters, is_base64_encoded -from ...base_llm.embedding.transformation import BaseEmbeddingConfig +from ...base_llm.embedding.transformation import BaseEmbeddingConfig, extra_body_fields from ..common_utils import VertexAIError @@ -189,8 +189,11 @@ class VertexAIMultimodalEmbeddingConfig(BaseEmbeddingConfig): request_data["instances"] = [vertex_request_instance] + extra_body: Final = extra_body_fields(optional_params) if "outputDimensionality" in optional_params: - request_data["parameters"] = {"dimension": optional_params["outputDimensionality"]} + request_data["parameters"] = {**extra_body, "dimension": optional_params["outputDimensionality"]} + elif extra_body: + request_data["parameters"] = dict(extra_body) return cast(dict, request_data) diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index 96fccf007a5..5d99ce6d90f 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -1,10 +1,11 @@ import types from collections.abc import Mapping -from typing import Final, Literal +from typing import Final, Literal, cast # noqa: TID251 # extra_body carries raw fields the TypedDict lacks from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict +from litellm.llms.base_llm.embedding.transformation import split_extra_body from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import EmbeddingResponse, PromptTokensDetailsWrapper, Usage @@ -164,7 +165,12 @@ class VertexAITextEmbeddingConfig(LiteLLMBaseModel): vertex_text_embedding_input_list.append(embedding_input) vertex_request["instances"] = vertex_text_embedding_input_list - vertex_request["parameters"] = EmbeddingParameters(**optional_params) + body_params, extra_body = split_extra_body(optional_params) + vertex_request["parameters"] = ( + cast( # cast-ok: extra_body carries raw Vertex fields the TypedDict does not list + EmbeddingParameters, {**extra_body, **body_params} + ) + ) if labels: vertex_request["labels"] = labels @@ -205,7 +211,12 @@ class VertexAITextEmbeddingConfig(LiteLLMBaseModel): vertex_text_embedding_input_list.append(embedding_input) vertex_request["instances"] = vertex_text_embedding_input_list - vertex_request["parameters"] = TextEmbeddingFineTunedParameters(**optional_params) + body_params, extra_body = split_extra_body(optional_params) + vertex_request["parameters"] = ( + cast( # cast-ok: extra_body carries raw Vertex fields the TypedDict does not list + TextEmbeddingFineTunedParameters, {**extra_body, **body_params} + ) + ) # Remove 'shared_session' from parameters if present if vertex_request["parameters"] is not None and "shared_session" in vertex_request["parameters"]: del vertex_request["parameters"]["shared_session"] diff --git a/litellm/utils.py b/litellm/utils.py index 67b7b4f3d1e..118f89766a9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5024,9 +5024,7 @@ def get_optional_params_embeddings( optional_params["dimensions"] = non_default_params.pop("dimensions") if len(non_default_params.keys()) > 0: if litellm.drop_params is True or drop_params is True: # drop the unsupported non-default values - keys = list(non_default_params.keys()) - for k in keys: - non_default_params.pop(k, None) + non_default_params.clear() else: raise UnsupportedParamsError( status_code=500, @@ -5039,9 +5037,7 @@ def get_optional_params_embeddings( ): if len(non_default_params.keys()) > 0: if litellm.drop_params is True or drop_params is True: # drop the unsupported non-default values - keys = list(non_default_params.keys()) - for k in keys: - non_default_params.pop(k, None) + non_default_params.clear() else: raise UnsupportedParamsError( status_code=500, @@ -5060,7 +5056,7 @@ def get_optional_params_embeddings( additional_drop_params=kwargs.get("additional_drop_params", None), ) - if "extra_body" in final_params and len(final_params["extra_body"]) == 0: + if "extra_body" in final_params and not final_params["extra_body"]: final_params.pop("extra_body", None) return final_params diff --git a/tests/unit/llms/base_llm/embedding/test_provider_embedding_translation.py b/tests/unit/llms/base_llm/embedding/test_provider_embedding_translation.py index 24157263daa..316fdad5493 100644 --- a/tests/unit/llms/base_llm/embedding/test_provider_embedding_translation.py +++ b/tests/unit/llms/base_llm/embedding/test_provider_embedding_translation.py @@ -106,3 +106,4 @@ def test_image_embedding(case: _Case, respx_mock: MockRouter) -> None: else: assert body["inputImage"] == png_b64 assert response.data[0]["embedding"] == [0.11, 0.22, 0.33] + diff --git a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index 904ca3bb8ca..86e56307210 100644 --- a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -2130,3 +2130,122 @@ def test_bedrock_embedding_region_bug_reproduction(): img_base_64 = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkBAMAAACCzIhnAAAAG1BMVEURAAD///+ln5/h39/Dv79qX18uHx+If39MPz9oMSdmAAAACXBIWXMAAA7EAAAOxAGVKw4bAAABB0lEQVRYhe2SzWrEIBCAh2A0jxEs4j6GLDS9hqWmV5Flt0cJS+lRwv742DXpEjY1kOZW6HwHFZnPmVEBEARBEARB/jd0KYA/bcUYbPrRLh6amXHJ/K+ypMoyUaGthILzw0l+xI0jsO7ZcmCcm4ILd+QuVYgpHOmDmz6jBeJImdcUCmeBqQpuqRIbVmQsLCrAalrGpfoEqEogqbLTWuXCPCo+Ki1XGqgQ+jVVuhB8bOaHkvmYuzm/b0KYLWwoK58oFqi6XfxQ4Uz7d6WeKpna6ytUs5e8betMcqAv5YPC5EZB2Lm9FIn0/VP6R58+/GEY1X1egVoZ/3bt/EqF6malgSAIgiDIH+QL41409QMY0LMAAAAASUVORK5CYII=" + + +@pytest.mark.parametrize( + ("model_id", "embed_response", "extra_body", "expected_body"), + [ + ( + "amazon.titan-embed-text-v2:0", + titan_embedding_response, + {"normalize": False, "dimensions": 256}, + {"inputText": test_input, "normalize": False, "dimensions": 256}, + ), + ( + "cohere.embed-english-v3", + cohere_embedding_response, + {"truncate": "END", "embedding_types": ["float"]}, + {"texts": [test_input], "truncate": "END", "embedding_types": ["float"]}, + ), + ], +) +def test_bedrock_embedding_sends_extra_body_fields_in_the_request_body( + respx_mock: respx.MockRouter, + model_id: str, + embed_response: dict[str, object], + extra_body: dict[str, object], + expected_body: dict[str, object], +) -> None: + route = respx_mock.post(f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/invoke").mock( + return_value=httpx.Response(200, json=embed_response) + ) + + litellm.embedding( + model=f"bedrock/{model_id}", + input=[test_input], + api_key="test-bearer-token", + aws_region_name="us-east-1", + extra_body=extra_body, + ) + + body = json.loads(route.calls.last.request.read()) + assert "extra_body" not in body + assert expected_body.items() <= body.items() + + +def test_bedrock_nova_embedding_sends_extra_body_fields_in_the_request_body(respx_mock: respx.MockRouter) -> None: + model_id: Final = "amazon.nova-2-multimodal-embeddings-v1:0" + route = respx_mock.post(f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/invoke").mock( + return_value=httpx.Response(200, json={"embeddings": [{"embeddingType": "TEXT", "embedding": [0.1, 0.2]}]}) + ) + + litellm.embedding( + model=f"bedrock/{model_id}", + input=[test_input], + api_key="test-bearer-token", + aws_region_name="us-east-1", + extra_body={"segmentationConfig": {"durationSeconds": 10}}, + ) + + body = json.loads(route.calls.last.request.read()) + assert "extra_body" not in body + assert body["segmentationConfig"] == {"durationSeconds": 10} + + +@pytest.mark.parametrize( + ("extra_body", "field", "expected"), + [ + ({"dimensions": 512}, "dimensions", 256), + ({"inputText": "text the guardrails never saw"}, "inputText", test_input), + ], + ids=["mapped_param", "input"], +) +def test_bedrock_embedding_extra_body_cannot_replace_the_input_or_a_mapped_param( + respx_mock: respx.MockRouter, extra_body: dict[str, object], field: str, expected: object +) -> None: + route = respx_mock.post( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke" + ).mock(return_value=httpx.Response(200, json=titan_embedding_response)) + + litellm.embedding( + model="bedrock/amazon.titan-embed-text-v2:0", + input=[test_input], + api_key="test-bearer-token", + aws_region_name="us-east-1", + dimensions=256, + extra_body=extra_body, + ) + + assert json.loads(route.calls.last.request.read())[field] == expected + + +def test_bedrock_embedding_signs_the_body_it_sends(respx_mock: respx.MockRouter) -> None: + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + + route = respx_mock.post( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke" + ).mock(return_value=httpx.Response(200, json=titan_embedding_response)) + + litellm.embedding( + model="bedrock/amazon.titan-embed-text-v2:0", + input=[test_input], + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + aws_region_name="us-east-1", + dimensions=256, + extra_body={"dimensions": 512}, + ) + + sent: Final = route.calls.last.request + replay: Final = AWSRequest( + method="POST", url=str(sent.url), data=sent.content, headers={"Content-Type": "application/json"} + ) + replay.context["timestamp"] = sent.headers["x-amz-date"] + SigV4Auth( + Credentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), "bedrock", "us-east-1" + ).add_auth(replay) + + assert json.loads(sent.content)["dimensions"] == 256 + assert replay.headers["Authorization"] == sent.headers["authorization"] diff --git a/tests/unit/llms/nvidia_nim/test_nvidia_nim.py b/tests/unit/llms/nvidia_nim/test_nvidia_nim.py index 3633ac97265..04bd26d53ef 100644 --- a/tests/unit/llms/nvidia_nim/test_nvidia_nim.py +++ b/tests/unit/llms/nvidia_nim/test_nvidia_nim.py @@ -237,3 +237,28 @@ def test_completion_nvidia_nim(): assert request_body["model"] == "databricks/dbrx-instruct" assert request_body["frequency_penalty"] == 0.1 assert request_body["presence_penalty"] == 0.5 + + +def test_embedding_nvidia_nim_sends_extra_body_fields_without_an_extra_body_key(): + from openai import OpenAI + + transport: Final = CapturingTransport( + CreateEmbeddingResponse( + object="list", + data=(Embedding(object="embedding", index=0, embedding=(0.1, 0.2, 0.3)),), + model="nvidia/nv-embedqa-e5-v5", + usage=EmbeddingUsage(prompt_tokens=6, total_tokens=6), + ) + ) + client: Final = OpenAI(api_key="fake-api-key", http_client=httpx.Client(transport=transport)) + + litellm.embedding( + model="nvidia_nim/nvidia/nv-embedqa-e5-v5", + input="What is the meaning of life?", + extra_body={"input_type": "query", "truncate": "END"}, + client=client, + ) + + request_body: Final = transport.request_bodies[0] + assert "extra_body" not in request_body + assert (request_body["input_type"], request_body["truncate"]) == ("query", "END") diff --git a/tests/unit/llms/ollama/test_ollama_embedding.py b/tests/unit/llms/ollama/test_ollama_embedding.py index b32bb9c4415..abbe449bf4a 100644 --- a/tests/unit/llms/ollama/test_ollama_embedding.py +++ b/tests/unit/llms/ollama/test_ollama_embedding.py @@ -348,3 +348,21 @@ async def test_sdk_embedding_rejects_non_string_batch(embedding_route: respx.Rou litellm.embedding(model="ollama/nomic-embed-text", input=[123], api_base="http://ollama.test") assert not embedding_route.called + + +def test_ollama_embeddings_sends_extra_body_fields_at_the_documented_places(embedding_route: respx.Route) -> None: + response: Final = litellm.embedding( + model="ollama/nomic-embed-text", + input=["hello", "world"], + api_base="http://ollama.test", + extra_body={"truncate": False, "keep_alive": "5m", "options": {"num_ctx": 2048}}, + ) + + assert json.loads(embedding_route.calls.last.request.content) == { + "model": "nomic-embed-text", + "input": ["hello", "world"], + "truncate": False, + "keep_alive": "5m", + "options": {"num_ctx": 2048}, + } + assert response.data[0]["embedding"] == [0.1, 0.2] diff --git a/tests/unit/llms/sagemaker/test_sagemaker_embedding_voyage.py b/tests/unit/llms/sagemaker/test_sagemaker_embedding_voyage.py index 3951b17db92..088718b9c60 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_embedding_voyage.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_embedding_voyage.py @@ -626,3 +626,40 @@ class TestTEIEmbeddingResponse: if __name__ == "__main__": pytest.main([__file__]) + + +def test_voyage_endpoint_sends_extra_body_fields_in_the_invoke_body(): + client = MagicMock() + client.invoke_endpoint.return_value = { + "Body": MagicMock( + read=MagicMock( + return_value=json.dumps( + { + "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}], + "usage": {"total_tokens": 3}, + } + ).encode() + ) + ) + } + session = MagicMock() + session.client.return_value = client + + with patch("boto3.Session", return_value=session): + response = embedding( + model="sagemaker/voyage-3-5-endpoint", + input=["hello"], + aws_region_name="us-east-1", + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + extra_body={"input_type": "query", "truncation": False, "output_dimension": 512}, + ) + + assert json.loads(client.invoke_endpoint.call_args.kwargs["Body"]) == { + "input": ["hello"], + "model": "voyage-3-5-endpoint", + "input_type": "query", + "truncation": False, + "output_dimension": 512, + } + assert response.data[0]["embedding"] == [0.1, 0.2] diff --git a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index d7d921ab35c..06306981daf 100644 --- a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -652,3 +652,32 @@ def test_gemini_embedding(respx_mock: respx.MockRouter) -> None: expected_prompt_tokens: Final = litellm.token_counter(model="gemini/gemini-embedding-001", text="Hello, world!") assert response.usage.prompt_tokens == expected_prompt_tokens assert response.usage.total_tokens == expected_prompt_tokens + + +def test_gemini_embedding_sends_extra_body_fields_on_each_request(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post( + "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents" + ).mock(return_value=httpx.Response(200, json={"embeddings": [{"values": [0.1, 0.2]}]})) + + litellm.embedding( + model="gemini/gemini-embedding-001", + input="Hello, world!", + api_key="gemini-test-key", + extra_body={ + "title": "Greeting", + "taskType": "RETRIEVAL_DOCUMENT", + "outputDimensionality": 256, + "documentOcr": True, + }, + ) + + assert json.loads(route.calls.last.request.content)["requests"] == [ + { + "model": "models/gemini-embedding-001", + "content": {"parts": [{"text": "Hello, world!"}]}, + "title": "Greeting", + "taskType": "RETRIEVAL_DOCUMENT", + "outputDimensionality": 256, + "documentOcr": True, + } + ] diff --git a/tests/unit/llms/vertex_ai/multimodal_embeddings/test_vertex_ai_multimodal_embedding_transformation.py b/tests/unit/llms/vertex_ai/multimodal_embeddings/test_vertex_ai_multimodal_embedding_transformation.py index edb6e889814..123d4128679 100644 --- a/tests/unit/llms/vertex_ai/multimodal_embeddings/test_vertex_ai_multimodal_embedding_transformation.py +++ b/tests/unit/llms/vertex_ai/multimodal_embeddings/test_vertex_ai_multimodal_embedding_transformation.py @@ -1,8 +1,13 @@ import json from unittest.mock import AsyncMock, MagicMock, patch +import google.auth +import google.auth.credentials import httpx import pytest +import respx + +import litellm from litellm.llms.vertex_ai.multimodal_embeddings.transformation import ( @@ -174,3 +179,30 @@ class TestVertexMultimodalEmbedding: headers={}, ) assert "parameters" not in request + + +class _StaticGoogleCredentials(google.auth.credentials.Credentials): + def refresh(self, request: object) -> None: + self.token = "test-token" + + +def test_extra_body_fields_reach_the_multimodal_predict_body( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(google.auth, "default", lambda scopes: (_StaticGoogleCredentials(), "test-project")) + route = respx_mock.post(host="us-central1-aiplatform.googleapis.com").mock( + return_value=httpx.Response(200, json={"predictions": [{"textEmbedding": [0.1, 0.2]}]}) + ) + + litellm.embedding( + model="vertex_ai/multimodalembedding@001", + input="Hello world", + vertex_project="test-project", + vertex_location="us-central1", + extra_body={"dimension": 128}, + ) + + assert json.loads(route.calls.last.request.content) == { + "instances": [{"text": "Hello world"}], + "parameters": {"dimension": 128}, + } diff --git a/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py b/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py index 1058d4fb82d..0e8d49b01e5 100644 --- a/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py +++ b/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py @@ -305,3 +305,20 @@ async def test_vertex_embedding_usage_carries_the_billable_character_count_googl 8, character_count, ) + + +def test_vertex_embedding_call_sends_extra_body_fields_as_predict_parameters(predict_endpoint: respx.Route) -> None: + predict_endpoint.mock(return_value=_predict_response({"predictions": [_gecko_prediction([0.1, 2], 3)]})) + + litellm.embedding( + model=f"vertex_ai/{GECKO}", + input=["hello"], + vertex_project=PUBLIC_CALL_PROJECT, + vertex_location="us-central1", + extra_body={"autoTruncate": False}, + ) + + body: Final = json.loads(predict_endpoint.calls.last.request.content) + assert body["instances"] == [{"content": "hello"}] + assert "extra_body" not in body["parameters"] + assert body["parameters"]["autoTruncate"] is False From 3021312f601fac7a219748a0a0603c85eb2fee6a Mon Sep 17 00:00:00 2001 From: kenneth-berri Date: Thu, 12 Nov 2026 15:44:08 -0800 Subject: [PATCH 3/9] fix(bedrock): send max_tokens and priority in Cohere embedding requests (#51435) * fix(bedrock): send max_tokens and priority in Cohere embedding requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(sagemaker): mark Cohere transform result Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: rerun after unrelated checkout timeout and langsmith flake Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/types/llms/bedrock.py | 2 ++ .../bedrock/embed/test_bedrock_embedding.py | 28 +++++++++++++++++++ .../test_sagemaker_embedding_voyage.py | 16 +++++++++++ 3 files changed, 46 insertions(+) diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index e42a1a8f057..170f90c1adb 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -505,6 +505,8 @@ class CohereEmbeddingRequest(TypedDict, total=False): truncate: Literal["NONE", "START", "END"] embedding_types: Literal["float", "int8", "uint8", "binary", "ubinary"] output_dimension: int + max_tokens: ReadOnly[int] + priority: ReadOnly[int] class CohereEmbeddingRequestWithModel(CohereEmbeddingRequest): diff --git a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index 86e56307210..63526e917a6 100644 --- a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -190,6 +190,34 @@ def test_bedrock_cohere_embedding_returns_the_scripted_vector(respx_mock: respx. assert response.data == [{"object": "embedding", "index": 0, "embedding": [0.3, 0.2, 0.1]}] +def test_bedrock_cohere_embedding_sends_max_tokens_and_priority(respx_mock: respx.MockRouter) -> None: + route = respx_mock.post("https://bedrock-runtime.us-east-1.amazonaws.com/model/cohere.embed-v4:0/invoke").mock( + return_value=httpx.Response( + 200, + json={"embeddings": {"float": [[0.3, 0.2, 0.1]]}, "response_type": "embeddings_by_type"}, + ) + ) + + litellm.embedding( + model="bedrock/cohere.embed-v4:0", + input=["hello"], + api_key="test-bearer-token", + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com", + max_tokens=16, + priority=5, + truncate="END", + ) + + assert json.loads(route.calls.last.request.read()) == { + "input_type": "search_document", + "texts": ["hello"], + "max_tokens": 16, + "priority": 5, + "truncate": "END", + } + + GOVCLOUD_WEST_RUNTIME: Final = "https://bedrock-runtime.us-gov-west-1.amazonaws.com" diff --git a/tests/unit/llms/sagemaker/test_sagemaker_embedding_voyage.py b/tests/unit/llms/sagemaker/test_sagemaker_embedding_voyage.py index 088718b9c60..3d8c5ccfa21 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_embedding_voyage.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_embedding_voyage.py @@ -7,6 +7,7 @@ transformation, and model type detection. """ import json +from typing import Final from unittest.mock import MagicMock, patch import httpx @@ -105,6 +106,21 @@ class TestSagemakerCohereEmbeddingConfig: ) assert result["texts"] == ["hello"] + def test_transform_request_keeps_max_tokens_and_priority(self): + result: Final = self.config.transform_embedding_request( + model="cohere-embed-v4", + input=["hello"], + optional_params={"max_tokens": 16, "priority": 5, "truncate": "END"}, + headers={}, + ) + assert result == { + "texts": ["hello"], + "input_type": "search_document", + "max_tokens": 16, + "priority": 5, + "truncate": "END", + } + def test_map_openai_params_dimensions_to_output_dimension(self): params = self.config.map_openai_params( non_default_params={"dimensions": 512, "encoding_format": "float"}, From 6cb64568f16262a5086913e90e9d104f5f37ae05 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Thu, 12 Nov 2026 15:44:49 -0800 Subject: [PATCH 4/9] bump: litellm-enterprise 0.1.83 -> 0.1.84, litellm-proxy-extras 0.4.114 -> 0.4.115, litellm 1.110.0 -> 1.111.0 (#51486) The 1.110.0 line has rc tags, so the next nightly opens 1.111.0. Both subpackages changed since v1.110.0-dev.2 while still at their published versions --- litellm/enterprise/pyproject.toml | 4 ++-- litellm/proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 8 ++++---- uv.lock | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/litellm/enterprise/pyproject.toml b/litellm/enterprise/pyproject.toml index 7a8056e8f15..0fdc674aea7 100644 --- a/litellm/enterprise/pyproject.toml +++ b/litellm/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.83" +version = "0.1.84" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.83" +version = "0.1.84" version_files = [ "pyproject.toml:^version", "../../pyproject.toml:litellm-enterprise==", diff --git a/litellm/proxy-extras/pyproject.toml b/litellm/proxy-extras/pyproject.toml index 06d77e751c7..f749ee18cac 100644 --- a/litellm/proxy-extras/pyproject.toml +++ b/litellm/proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.114" +version = "0.4.115" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.114" +version = "0.4.115" version_files = [ "pyproject.toml:^version", "../../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index 39c760ef943..3b7e3e154b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.110.0" +version = "1.111.0" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.16" @@ -87,8 +87,8 @@ proxy = [ "mcp>=2.3.0,<3", "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3", - "litellm-proxy-extras==0.4.114", - "litellm-enterprise==0.1.83", + "litellm-proxy-extras==0.4.115", + "litellm-enterprise==0.1.84", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", @@ -441,7 +441,7 @@ litellm-enterprise = { workspace = true } members = ["litellm/enterprise", "litellm/proxy-extras", "litellm/packaging/litellm-proxy"] [tool.commitizen] -version = "1.110.0" +version = "1.111.0" version_files = [ "pyproject.toml:^version", ] diff --git a/uv.lock b/uv.lock index 01516d54939..9c23d77ea3b 100644 --- a/uv.lock +++ b/uv.lock @@ -5032,7 +5032,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.110.0" +version = "1.111.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -5571,7 +5571,7 @@ provides-extras = ["test"] [[package]] name = "litellm-enterprise" -version = "0.1.83" +version = "0.1.84" source = { editable = "litellm/enterprise" } [[package]] @@ -5581,7 +5581,7 @@ source = { editable = "litellm/packaging/litellm-proxy" } [[package]] name = "litellm-proxy-extras" -version = "0.4.114" +version = "0.4.115" source = { editable = "litellm/proxy-extras" } dependencies = [ { name = "psycopg" }, From 0df4d43297ef695f638a7e51aff08524b4cdcecb Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 12 Nov 2026 15:54:13 -0800 Subject: [PATCH 5/9] chore(cost-map): point the fireworks deepseek v4 flash successor at deepseek-v4p1-flash (#51478) deepseek-v4-flash-0731, the successor the three deprecated deepseek-v4-flash rows named, is itself retired (deprecation_date 2026-09-25) and no longer served, so the rows now name the live deepseek-v4p1-flash. Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4429fffe652..7c8a48f5aae 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28650,7 +28650,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", - "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", + "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -29124,7 +29124,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", - "successor_model": "fireworks_ai/deepseek-v4-flash-0731", + "successor_model": "fireworks_ai/deepseek-v4p1-flash", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -83697,7 +83697,7 @@ "mode": "chat", "output_cost_per_token": 2.1e-07, "source": "https://api.fireworks.ai/v1/serverless/models", - "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", + "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4429fffe652..7c8a48f5aae 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28650,7 +28650,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", - "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", + "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -29124,7 +29124,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", - "successor_model": "fireworks_ai/deepseek-v4-flash-0731", + "successor_model": "fireworks_ai/deepseek-v4p1-flash", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -83697,7 +83697,7 @@ "mode": "chat", "output_cost_per_token": 2.1e-07, "source": "https://api.fireworks.ai/v1/serverless/models", - "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", + "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, From a7052ea8a80296ca3db064e438904f562f176933 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Thu, 12 Nov 2026 15:54:56 -0800 Subject: [PATCH 6/9] fix(cost): preserve Bedrock cross-region pricing for Messages (#51479) * fix(cost): bill Bedrock cross-region profiles at the profile rate on /v1/messages Bedrock reports the region-stripped model id for a cross-region inference profile, and the cost selector preferred that served id, so streaming and non-streaming Messages spend (input, output, cache write and cache read) used the global row. A served id carrying a different cross-region prefix keeps its own price. Fixes #46286 * test(langfuse): derive the Bedrock profile export cost from the profile's model info The regional us. deployment is now billed at its profile rate, so the expected Langfuse cost comes from that profile's rates and the mock usage instead of a hardcoded global price. --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 18 ++ .../completion_with_bedrock_call.json | 3 - .../integrations/langfuse/test_langfuse.py | 23 +- tests/unit/test_cost_calculator.py | 200 +++++++++++++++++- 4 files changed, 237 insertions(+), 7 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 91529ae9ff5..03feb8dc1e6 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1144,11 +1144,29 @@ def _served_or_priced_requested_model( if served in (requested, requested.partition("/")[2]): return served pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider) + profile_of_served: Final = _is_bedrock_cross_region_profile_of(requested_model, served_model, pricing_provider) + if profile_of_served and _resolves_to_rates(requested, pricing_provider): + return requested if _resolves_to_rates(served, pricing_provider): return served return requested if _resolves_to_rates(requested, pricing_provider) else served +def _is_bedrock_cross_region_profile_of(profile: str, served_model: str, custom_llm_provider: str | None) -> bool: + if custom_llm_provider != "bedrock": + return False + from litellm.llms.bedrock.common_utils import ( + get_bedrock_cross_region_inference_regions, + strip_bedrock_routing_prefix, + ) + + region, _, profiled_model = strip_bedrock_routing_prefix(profile).partition(".") + return ( + region in get_bedrock_cross_region_inference_regions() + and strip_bedrock_routing_prefix(served_model) == profiled_model + ) + + def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str: """Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json index 6f359380245..96548edc60f 100644 --- a/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/completion_with_bedrock_call.json @@ -2,9 +2,6 @@ "name": "litellm-acompletion", "parent_span_id": null, "attributes": { - "langfuse.observation.cost_details": { - "total": 6e-05 - }, "langfuse.observation.input": { "messages": [ { diff --git a/tests/unit/integrations/langfuse/test_langfuse.py b/tests/unit/integrations/langfuse/test_langfuse.py index 7a374c1affe..e7fcad5c748 100644 --- a/tests/unit/integrations/langfuse/test_langfuse.py +++ b/tests/unit/integrations/langfuse/test_langfuse.py @@ -97,7 +97,9 @@ def _comparable(span: Mapping[str, object]) -> dict[str, object]: } -def _expected_generation(route: respx.Route, fixture_name: str, trace_id: str) -> dict[str, object]: +def _expected_generation( + route: respx.Route, fixture_name: str, trace_id: str, cost_total: float | None = None +) -> dict[str, object]: otel_trace_id: Final = resolve_trace_id(trace_id) generations: Final = tuple( span @@ -108,7 +110,14 @@ def _expected_generation(route: respx.Route, fixture_name: str, trace_id: str) - ) assert len(generations) == 1 expected_path: Final = Path(__file__).with_name("langfuse_expected_request_body") / fixture_name - expected: Final = TypeAdapter(dict[str, object]).validate_python(json.loads(expected_path.read_text())) + fixture: Final = TypeAdapter(dict[str, object]).validate_python(json.loads(expected_path.read_text())) + fixture_attributes: Final = TypeAdapter(dict[str, object]).validate_python(fixture["attributes"]) + priced_attributes: Final = ( + fixture_attributes + if cost_total is None + else {**fixture_attributes, "langfuse.observation.cost_details": {"total": cost_total}} + ) + expected: Final = {**fixture, "attributes": priced_attributes} actual: Final = _comparable(generations[0]) expected_comparable: Final = _comparable(expected) assert actual == expected_comparable, json.dumps( @@ -355,6 +364,12 @@ async def test_langfuse_logging_completion_with_bedrock_llm_response( langfuse_export: tuple[respx.Route, str], ) -> None: route, trace_id = langfuse_export + profile: Final = litellm.get_model_info( + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", custom_llm_provider="bedrock" + ) + input_rate: Final = profile["input_cost_per_token"] + output_rate: Final = profile["output_cost_per_token"] + assert isinstance(input_rate, float) and isinstance(output_rate, float) response: Final = litellm.ModelResponse( choices=[], usage=litellm.Usage(prompt_tokens=10, completion_tokens=10, total_tokens=20), @@ -373,7 +388,9 @@ async def test_langfuse_logging_completion_with_bedrock_llm_response( ) await _flush_langfuse() assert route.call_count == 1 - _expected_generation(route, "completion_with_bedrock_call.json", trace_id) + _expected_generation( + route, "completion_with_bedrock_call.json", trace_id, cost_total=10 * input_rate + 10 * output_rate + ) @pytest.mark.asyncio diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 71efa1352e4..23748790869 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -2,7 +2,7 @@ import json import asyncio import datetime import time -from collections.abc import Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping from pathlib import Path from types import MappingProxyType, SimpleNamespace from typing import Final, Literal, cast @@ -526,6 +526,204 @@ def test_completion_cost_bills_priced_response_model_over_priced_requested_model assert cost == pytest.approx(_token_rates_cost(litellm.model_cost["openai/sol-requested-test-2099-01-01"], usage)) +_BEDROCK_PROFILE_GLOBAL_RATES: Final = MappingProxyType( + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 5e-6, + "cache_read_input_token_cost": 1e-7, + "cache_creation_input_token_cost": 1.25e-6, + } +) +_BEDROCK_PROFILE_MODEL: Final = "anthropic.bedrock-profile-test-v1:0" +_BEDROCK_PROFILE_RATE_MULTIPLIERS: Final = MappingProxyType( + { + _BEDROCK_PROFILE_MODEL: 1.0, + f"eu.{_BEDROCK_PROFILE_MODEL}": 1.1, + f"us.{_BEDROCK_PROFILE_MODEL}": 1.2, + f"jp.{_BEDROCK_PROFILE_MODEL}": 1.3, + f"global.{_BEDROCK_PROFILE_MODEL}": 0.9, + "anthropic.bedrock-profile-other-test-v1:0": 3.0, + "anthropic.bedrock-profile-unpriced-test-v1:0": 1.0, + "bedrock-profile-router-test": 5.0, + } +) +_BEDROCK_PROFILE_CACHE_USAGE: Final = MappingProxyType( + {"cache_creation_input_tokens": 2000, "cache_read_input_tokens": 24000} +) + + +@pytest.fixture +def _bedrock_profile_rows(_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> None: + for key, multiplier in _BEDROCK_PROFILE_RATE_MULTIPLIERS.items(): + monkeypatch.setitem( + litellm.model_cost, + key, + { + "litellm_provider": "bedrock_converse", + "mode": "chat", + **{field: rate * multiplier for field, rate in _BEDROCK_PROFILE_GLOBAL_RATES.items()}, + }, + ) + monkeypatch.setitem( + litellm.model_cost, + "eu.anthropic.bedrock-profile-unpriced-test-v1:0", + {"litellm_provider": "bedrock_converse", "mode": "chat"}, + ) + + +def _bedrock_messages_body(model: str) -> dict[str, object]: + return { + "id": "msg_bdrk_profile", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 18, "output_tokens": 4, **_BEDROCK_PROFILE_CACHE_USAGE}, + } + + +def _bedrock_stream_response(model: str) -> ModelResponse: + cached: Final = _BEDROCK_PROFILE_CACHE_USAGE["cache_read_input_tokens"] + written: Final = _BEDROCK_PROFILE_CACHE_USAGE["cache_creation_input_tokens"] + return ModelResponse( + model=model, + choices=[], + usage=Usage( + prompt_tokens=18 + cached + written, + completion_tokens=4, + total_tokens=22 + cached + written, + cache_creation_input_tokens=written, + cache_read_input_tokens=cached, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached), + ), + ) + + +def _bedrock_private_route_response(model: str) -> ModelResponse: + response: Final = _bedrock_stream_response("bedrock-profile-alias") + response._hidden_params = {"custom_llm_provider": "bedrock", "provider_response_model": model} + return response + + +def _bedrock_global_cost(served_shape: Callable[[str], object]) -> float: + return completion_cost( + completion_response=served_shape(_BEDROCK_PROFILE_MODEL), + model=f"bedrock/{_BEDROCK_PROFILE_MODEL}", + custom_llm_provider="bedrock", + ) + + +_BEDROCK_SERVED_SHAPES: Final = ( + pytest.param(_bedrock_messages_body, id="non-streaming"), + pytest.param(_bedrock_stream_response, id="streaming"), + pytest.param(_bedrock_private_route_response, id="private-provider-response-model"), +) + + +@pytest.mark.parametrize("served_shape", _BEDROCK_SERVED_SHAPES) +@pytest.mark.parametrize("route", ["bedrock/", "bedrock/invoke/"]) +@pytest.mark.parametrize("region", ["eu", "us", "jp"]) +def test_completion_cost_bills_bedrock_cross_region_profile_when_bedrock_serves_the_region_stripped_id( + _bedrock_profile_rows: None, region: str, route: str, served_shape: Callable[[str], object] +) -> None: + profile: Final = f"{region}.{_BEDROCK_PROFILE_MODEL}" + + cost: Final = completion_cost( + completion_response=served_shape(_BEDROCK_PROFILE_MODEL), + model=f"{route}{profile}", + custom_llm_provider="bedrock", + ) + + assert cost == pytest.approx(_bedrock_global_cost(served_shape) * _BEDROCK_PROFILE_RATE_MULTIPLIERS[profile]) + + +@pytest.mark.parametrize("served_shape", _BEDROCK_SERVED_SHAPES) +@pytest.mark.parametrize( + ("deployment", "served_model"), + [ + pytest.param(f"bedrock/eu.{_BEDROCK_PROFILE_MODEL}", f"us.{_BEDROCK_PROFILE_MODEL}", id="other-region"), + pytest.param(f"bedrock/eu.{_BEDROCK_PROFILE_MODEL}", f"global.{_BEDROCK_PROFILE_MODEL}", id="global-profile"), + pytest.param( + f"bedrock/jp.{_BEDROCK_PROFILE_MODEL}", "anthropic.bedrock-profile-other-test-v1:0", id="other-model" + ), + pytest.param( + "bedrock/eu.anthropic.bedrock-profile-unpriced-test-v1:0", + "anthropic.bedrock-profile-unpriced-test-v1:0", + id="unpriced-profile", + ), + ], +) +def test_completion_cost_keeps_the_served_bedrock_row_unless_it_is_the_region_stripped_profile( + _bedrock_profile_rows: None, deployment: str, served_model: str, served_shape: Callable[[str], object] +) -> None: + cost: Final = completion_cost( + completion_response=served_shape(served_model), + model=deployment, + custom_llm_provider="bedrock", + ) + + assert cost == pytest.approx(_bedrock_global_cost(served_shape) * _BEDROCK_PROFILE_RATE_MULTIPLIERS[served_model]) + + +@pytest.mark.parametrize( + ("base_model", "custom_pricing", "router_model_id", "priced_row"), + [ + pytest.param( + "bedrock/anthropic.bedrock-profile-other-test-v1:0", + None, + None, + "anthropic.bedrock-profile-other-test-v1:0", + id="base-model", + ), + pytest.param(None, True, "bedrock-profile-router-test", "bedrock-profile-router-test", id="custom-pricing"), + ], +) +def test_completion_cost_explicit_pricing_precedes_the_bedrock_cross_region_profile( + _bedrock_profile_rows: None, + base_model: str | None, + custom_pricing: bool | None, + router_model_id: str | None, + priced_row: str, +) -> None: + cost: Final = completion_cost( + completion_response=_bedrock_messages_body(_BEDROCK_PROFILE_MODEL), + model=f"bedrock/eu.{_BEDROCK_PROFILE_MODEL}", + custom_llm_provider="bedrock", + base_model=base_model, + custom_pricing=custom_pricing, + router_model_id=router_model_id, + ) + + assert cost == pytest.approx( + _bedrock_global_cost(_bedrock_messages_body) * _BEDROCK_PROFILE_RATE_MULTIPLIERS[priced_row] + ) + + +def test_completion_cost_keeps_the_served_row_for_a_region_dotted_name_outside_bedrock( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "openai/eu.profile-dotted-test", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + monkeypatch.setitem( + litellm.model_cost, + "openai/profile-dotted-test", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 5e-6, "output_cost_per_token": 11e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="profile-dotted-test", choices=[], usage=usage), + model="openai/eu.profile-dotted-test", + custom_llm_provider="openai", + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost["openai/profile-dotted-test"], usage)) + + def test_completion_cost_keeps_response_pricing_when_requested_alias_has_no_rates( _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch ) -> None: From b304dcfe94541b6c6fbbecde94dbb8bb97d94c6c Mon Sep 17 00:00:00 2001 From: maciej-berri Date: Thu, 12 Nov 2026 16:18:01 -0800 Subject: [PATCH 7/9] test(image): point admin MCP schema push at /app/litellm/schema.prisma (#51446) The --target builder image copies the repo root to /app, so after the schema moved under litellm/ it lives at /app/litellm/schema.prisma there. Only the runtime stage adds the /app/schema.prisma alias. Claude-Session: https://claude.ai/code/session_01Bne5zS8tnYUDfKJYENWcYA Co-authored-by: Claude --- tests/integration/database/image/test_image_admin_mcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/database/image/test_image_admin_mcp.py b/tests/integration/database/image/test_image_admin_mcp.py index aab22f354e4..a7fd33f1417 100644 --- a/tests/integration/database/image/test_image_admin_mcp.py +++ b/tests/integration/database/image/test_image_admin_mcp.py @@ -186,7 +186,7 @@ def test_image_admin_mcp_personal_admin_manages_team(offline_postgres: tuple[str "--env", "HOME=/opt/prisma", "--env", "XDG_CACHE_HOME=/opt/prisma/.cache", "--env", "PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries", "--entrypoint", "prisma", SCHEMA_IMAGE, - "db", "push", "--schema", "/app/schema.prisma", "--skip-generate", "--accept-data-loss", + "db", "push", "--schema", "/app/litellm/schema.prisma", "--skip-generate", "--accept-data-loss", ], capture_output=True, text=True, timeout=180, check=False, ) From df8b3b498e33b369dea893490c901fd3611e1f5c Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Fri, 13 Nov 2026 08:32:13 +0800 Subject: [PATCH 8/9] feat(proxy): add customer hierarchy usage analytics (#51495) --- .../before-main-6685525.png | Bin 0 -> 8473 bytes .../customer-usage-detail/final-overview.png | Bin 0 -> 67408 bytes .../customer-usage-detail/final-report.png | Bin 0 -> 62798 bytes litellm/accounting/application.py | 42 ++ litellm/accounting/projection.py | 44 ++ litellm/accounting/snapshot.py | 1 + litellm/integrations/otel/model/spans.py | 1 + .../migration.sql | 47 +++ .../litellm_proxy_extras/schema.prisma | 39 ++ litellm/proxy/_types.py | 4 + litellm/proxy/auth/login_utils.py | 1 + litellm/proxy/common_utils/customer_usage.py | 187 +++++++++ litellm/proxy/litellm_pre_call_utils.py | 40 +- .../internal_user_endpoints.py | 17 +- .../key_management_endpoints.py | 59 ++- .../management_v1/__init__.py | 4 + .../management_v1/customer_usage.py | 220 ++++++++++ litellm/proxy/proxy_server.py | 1 + litellm/proxy/schema.prisma | 39 ++ .../accounting_delivery_repository.py | 12 + .../repositories/daily_activity_repository.py | 5 + litellm/repositories/daily_activity_sql.py | 1 + litellm/schema.prisma | 39 ++ litellm/types/repositories/daily_activity.py | 2 + .../_components/CustomerMemberSpend.tsx | 166 ++++++++ .../_components/CustomerUsageHeader.tsx | 112 ++++++ .../_components/CustomerUsageRankings.tsx | 200 +++++++++ .../_components/CustomerUsageTrend.tsx | 152 +++++++ .../_components/customerUsageApi.test.ts | 336 ++++++++++++++++ .../customers/_components/customerUsageApi.ts | 265 ++++++++++++ .../customerUsageChartData.test.ts | 185 +++++++++ .../_components/customerUsageChartData.ts | 71 ++++ .../_components/customerUsageData.test.ts | 104 +++++ .../_components/customerUsageData.ts | 262 ++++++++++++ .../_components/useCustomerHierarchyUsage.ts | 122 ++++++ .../customers/usage/page.integration.test.tsx | 379 ++++++++++++++++++ .../app/(dashboard)/customers/usage/page.tsx | 136 +++++++ .../components/overview/BreakdownChart.tsx | 2 +- .../src/i18n/locales/en.json | 66 +++ .../src/lib/http/schema.d.ts | 174 ++++++++ tests/unit/accounting/test_application.py | 103 +++++ tests/unit/accounting/test_projection.py | 100 +++++ tests/unit/accounting/test_worker.py | 129 +++++- .../gcs_pubsub/spend_logs_payload.json | 2 +- .../proxy/common_utils/test_customer_usage.py | 225 +++++++++++ .../hooks/test_dynamic_rate_limiter_v3.py | 20 +- .../management_v1/test_customer_usage.py | 278 +++++++++++++ .../test_internal_user_endpoints.py | 25 ++ .../test_key_management_endpoints.py | 88 +++- .../test_spend_management_endpoints.py | 2 +- 50 files changed, 4484 insertions(+), 25 deletions(-) create mode 100644 .github/pr-assets/customer-usage-detail/before-main-6685525.png create mode 100644 .github/pr-assets/customer-usage-detail/final-overview.png create mode 100644 .github/pr-assets/customer-usage-detail/final-report.png create mode 100644 litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql create mode 100644 litellm/proxy/common_utils/customer_usage.py create mode 100644 litellm/proxy/management_endpoints/management_v1/customer_usage.py create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerMemberSpend.tsx create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageHeader.tsx create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageRankings.tsx create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageTrend.tsx create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.test.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.test.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.test.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/useCustomerHierarchyUsage.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.integration.test.tsx create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.tsx create mode 100644 tests/unit/accounting/test_application.py create mode 100644 tests/unit/proxy/common_utils/test_customer_usage.py create mode 100644 tests/unit/proxy/management_endpoints/management_v1/test_customer_usage.py diff --git a/.github/pr-assets/customer-usage-detail/before-main-6685525.png b/.github/pr-assets/customer-usage-detail/before-main-6685525.png new file mode 100644 index 0000000000000000000000000000000000000000..563c231cfbe90d21fac0a1aba2af84e2fa247963 GIT binary patch literal 8473 zcmeI!X;hO}8UWxx3RtSv3>YeF2S!#wLWm*>Fm^y&B}5D%D4Ql_3lIb($d=UUSXqSB zqC$v33rCg&LL*C>M8wv_AO;vfwh*9AG^`-#nS#OQR)=lOE$l48;E#`dY-jJ_Fp^?Ez2n+)i z_@9P)jQ2hfKdG>To7kBrmLwF7%-8b#g7Dq2lNAcL0KVrG7pFW%kazGJ%ImVUpSpjs z_nlv*Xsd@(3RiP&xY)Oswli(koVEj1eXq!`qr2Vz*JZw+-1u$WQl;p6&D^ohK^1ppZU=BqvtLq2FXtzwMAf^v%ut+PGpFkPnvlNaQYdO8 z!cUtrxdD$I{Q;Wk8x;g*-j2PWbc7DJSBAzjoK=kKuW-&Yty2yYb1#}!sKB~A%90uk z2ew6le`tqfs&dBf$=t%*J);ynzx^ke1*}}P!tTk5=@oVHm+jiTN4&db1#w5=XYz$Z zbDfq$#6_VmML`TN!S`Ekg%ZC>3Ebx3h{(HLcngo$VsEv3+8i6nV=oleR9mrCCXrAR zl>lQRK8tz~rgfVevpPFBGTh6}J{q|}x_hG3G@`rCw&cXJWh#bYn|s=7;Aq#JDsFz# zqm#>d{O%|2gG)artd4O~Y;hGTEDVi(YV$4tvRWZ8Ir%)>^?T=Oi$O@bA}OAi8x+(? zZd4YL8rSOiuVJa8Im`a%Qtt`gn>Fgna?ZKI)LGGB(?ZDQUj~pqU0)LIE8?hSh;&hp z7xkOQp!=BCP(9SeXa}Llt_Vw7Rv$FPaJOdt^Utdmbj=m6?eszhC z-ty}6sZ0!KCO-QPLwZ;z&0hUHOGcxjHnX<;UfF#8g<-BupM3kZ-}B0KDOJi`cc;Lq z8+Ud9_L(zjerYYQJ~Dyh6xUPS7MF4(Q4Z$uvPo}p03RmI711hYS@vEtq61&$Sijjn zk#jj|q4V;#XfUZqf)o`UtvDwP~miU^5NLoL&q=6t-wLaDjWiIFR? z-s9(Lj}9_}UN){QuzD+)EE*q#O%B=IRjDrPIx>2Ey9^ti;Ufn_A20UY=hlqIShr>W zFzSjHXQs*+vN)M$W_o<}d++NHl#^zEc-kxKe~P48cjZh^Qqv&V+r98pFN+c_^HjxV+E3dwYVutno z*p-fa#=cRU=%M+J;!&7t+dJjsEK{}Z-c8bJn)ce=7aufipe>Hwf_q7doA&Oroub#x zXI^2yeb|gOt5d!xjeVs#P00({0o>XBXIa0cvAHX(8iGVa!<)Hnmz6!ZbdfQ+*p!Qq zsBmHdHj)2^u>4?wE3;krh@XsOoYQhs#%! zM@bQ9Ao2WaLN8-p#$Z%Awu6wS>X2m=4AoVQ_%Zt;jQm(kId$ca=%xdP`f zU4pMNfp?LTwlafzCqv|k(UTW$6K2L;ic)XlZL6jhZbx8Xqo}8dU{6sOO+zGFt+%r< z{ETKZo3M=?K)M`Le7pEAva1yr&@;E(qup5wc?8O3Xt(D}K%e$_sTML^e$`q}Ky4>5sL;$<26zDw^v) z%P6Qf`3+-^^$IJLEDm5Orq|@WMokitcz0=E=gr3G9{z(@kwe7bQ$#oW7_$_)FeAVd zg8?aQ8!K?WWPJHo95lz5&gBRQ5}#Xq?>sOhX&}XgGo0T}D0HZ@$5BMnGb_}Xn|_Vv zc{ybb-Yk3HIPEHlb!muHNPfT8esWqUYXJ#OY(Dg*X!ayyA5?6Y-5_lvx15Q%VFGly zSuqf~xQ7b?VYK*7TYH>+?sEUV`jC!{&QdOUk=-DlnV^GN!qBAv%n(dc!!4}wc{Gg3 zOOTNhgMOK|IL|mvVty-MfU=h8n2X))GUN{;K~Y%(Kc|C7s9>dIK!3Y8t@Gr$(YDI| zmayTK@;(k(bGRsrlons(#7P@wxRR|xhJ9X&gZrT%-@NPbQ(K371U>wml*k`hLho4L z?}7}n+JzHV$u&n zh@hZ*Ygl&=($w!p%bI_gAJc{;v=$T<70pUn^mL+I_NqWyN2}j#D2$r{L99=rXQu{+ zj-RS+ni^4!-Ewt7am&2cJmlsn7;7iRV~39X>bDF#5o8s|9#rP|8QvG8`0?CQf+EVA z9-8cz_?(T(6cMEBtSTo4UP8!M*anB2U6I9`5bP<2mVR3OQj1Ak`XHrWp1eX2jnQENl+;yvW5oXY$Hrc;0TORKBeG zxK@^w9CtMA&nquH$2LOQH#D1J^!xr2*{e z+Ui0Zy7V@&YL59(x=zcli`zz|3z{v$DayTp&5cb9;P$Z*Pgjpl=;GK&G6uJpn(f4$ z#-xT^<14CsWQZ+X8}ZwOYw)Ky0w+~nZ*y`G545j{uYJs0?^5J1$r3g%<|!Nm@3dc$ z@2yn3TuIC!D%vyk`zjkp=g~JplfkV=LGJTvJ1=@w76hzGAzjykl45d^0Z4yNE>t zr)xoz*3}P0LS=tuW`BvE`G2)V{CX&(5`9)A77f9UNa2S%wvN!Rt<7_gTCZ{?%NFNc z;#@AvTB~Lt4Ew4p%jy&o@zBE*SY9g_8@4*67gY1%j^E_yV>>XTls zqA2UoL99P&v?H#B5`7h*|4ByW`K$4R4(EF-o!zq8Hx`Cx11PK6xf-?h4HW=5^yXi> zl^-{k{Fcz0wCT$3t)8y0Fa1{u{%vq;ffxmmeMX}xa$h|8x4U!;Isu)4PCzH16VM6h O1at!bCjpzC@xK7`fiLC& literal 0 HcmV?d00001 diff --git a/.github/pr-assets/customer-usage-detail/final-overview.png b/.github/pr-assets/customer-usage-detail/final-overview.png new file mode 100644 index 0000000000000000000000000000000000000000..fb9438653a535bb794cea1155baf75a20f95911c GIT binary patch literal 67408 zcmeFZ2UJt-wkR5;suWQ=2_5OE^e#vV9qC=cfRxZfkEnnmQUU}Fz4y>N0cp~E@4eU1 zi-PFo|Nni?*!%4B-Z}T3cgG#?-LtaBH?r3J)|~U3YkqB(^?T~~65yVyl8O=l2L}Ma zx%mNp&jRED1bFxa_;>^a_ymN61Vnf4-MMr7_8kgRGU9vG6f`u{6jW4nAa+JNdR8D6 z6%#KL>q8DMZZ29zzQ=r=kJ&l7IR8L`Lr6$?hv?3OJ9i#%(oxZI{!bsjn*roRw}$Wm zxHv3;TjV&n+!;{a~q z;^C7M+<(AKNby8Y8_A-hYZ)~xAmr-iUq;C)_$-}>iaNTyg-uvQPydyBz?YFRc7u!x z8Zmj8$G7E&9HJ1b$|FV3zz;Du=;?57jvIddmGq6!{y&a&<+$RVp6!Ly^^GXUZnl()!n$pKFR zm)Yd^$jNd3gTniQfH;OBBNVYn$N_33mE-2D-8p8h%o6OpZI9^`Hinti!584P#)=1S5e%9#Vwfh;N!T)ujL8lYL zDJ2WbEf%K#>w=fWscbYxKL}ZTCPyi+Ir(IiOyN5tS-VQllLOT0X~kH=w}*ax$K1>) z@fEAmhH|ANOo?Te%Mrdo0qEP`5c5C%nO^laYvgb$b7Xw1@=<~NhTTJVrpN(?A+fJz z+eu5_p8U-iu;yiXfx1Z7j49c+zkcU}sPY2C0)xya#Omx){jac2FUk*&Vf9xJ(Qu9u zNQB{Z%8Uy-1?BrZfJ4?Yl+0= zP|QN?ipV@VbJiekn_*a3(J$~+cqP6wl-|f_k1^xC*Pj{G=Ct;9sIoDpPH^m9+>$z86ZxACwGV^m$)+d3bPuROe~^ z%xkjEA;y{535k$9zCOt>-9iv@k6lTit)E%(U~4tmoxcGwUf&~>e*=zYoc9_$u3Ot4 zMNf=&GBB!%A*Scw-TnlljqkB67>1g@nG9XWx6(EJm{%#0QQ_dKNyHY{E!Eulw31Uo zaN-E+C`rtL*>Pw}rN!dA=7r1t&rl!Kj) z%kSR@KLthrkK+-^#hco0={fm`l$`1Wy3aBnbW|rWixwR0_7b~B26k9TCLcxR>9HI< z4@JAku3|s}zzqd28XT5XS#MdPH0cvOd0Vw&( zEioiMr8|P=!$x1H*LBTkf#Ozy(Znv1^I%DC?uXfZ9c^$6>1o2yV@;LKerL%~!DjD% zh~MK9PPq|2!{876^{wsISh9koP6x-5FpOLL{JrTq>j~jYR#kae<;du`Ubi9WF~VhK zPE73L9&GF5yJ62Fx=F?73uav_dC~431)%yv(EVtxZIJivj!0cXl^ zncay*uSS(&kJhYEJmx+`BeP3o(68Mbm+;xQ`};4ivKNPzTvoiLAVj|b-GWa!<33+4 zwsfMhR+AQC&O?6uhW3py+=WlHhpT&tuU}-R&#BPUHhaRt7#7;6flCueA3&q$)`dnm z*C<1Dm3X~9>!^}y#P#mlm%;B3B1-X3g?{m0^N{7_g}68^?$RPEtBHuccCTg>Tv@Vn?*k+C_xzO@JG zbi$5jP4LRdMC76J%<}B?{8Z%@ew_yW+gzNVG#kmof}^x^T4W1uZ(s+?{tf?#1Cx`x zOIcha5hE4&Y#^UAELOy`Dbcuh*i}j@RlnG@VFYt_?}aTVtyoDw3)x@mgz(VM&jhqX zQ&znTf)GIj`$#+Usv=P8eIJ|!{o@UD35S4C)wyEwIyF$;4}6VI5`HKPiT8aeuc{J0 zb|yy&Ewk-@Q=!8t{-|*+T=}(+!;M^Yqf^)joLRr`-=?X##_AH)c2s){gfP^OidvQh z?p{354`DfwIM0r4x`V7ui>gcxrW7^vvR!V|_T7vgsQzz2p?WW$Zt*Qz^=#xFqeE+3 zsRiI<5`zz6y9wAM#b_x=B^M3}5s7Ws7bCHyWlLGg%RuKAX9g69*zR!ISRlDxxxikG z2mp!JG;gxpw=Mu0LiSpz@rk6o zieLU)3zIHQgK}ADN9TlM+hXU>5w#*>zX958vnMpR{47$HRuJHqj72q;A3g2TtvXui zd>%FMuhsUURSOAsa`aUws>ZYxR_~^qN17+s&d-3`buEI*=T&m~u?-8kMWIIYG^>-! z=MrJ6-@iLqN@_g}L<*+-YUivO-fDbVVSX^~n{a0)aUMjK@ymtI$8N_N_L=yZ&O#xl zf_3`GxYmlAy6Z_|(Y#I7(o2CWx9_r4*~7y@gXZ8% znG7!9Y>Ke>Js0Nn!PgTJpn4Uty{;ouRdr%%&Fo5DgF2M&gvjgKC!5evn$;=bPe;4W zDb+rH4_I+dRi_tf-X$yV!?@Jg^lk?Jh_d-M!P|>0pRiB(j(PUz^{+3~&$PAG)M>{- z!Y!v8RH$3PshJVB-f}YM*Wx@?e4|lrSZlYa^0{n>4Z0o9jMlYU38qQZr?)v&5o+Qa z<_CwyAt*f+w{hV^6*XJ7|NznC~NA*s|A5m#mPuD+CB<*1&Xq}7whT^A5B zTUw4KbdZw91NN|e$P93Fh3wVlZ@{^{gK*BgcRm`OFe-&?vj;tn9@@AFOh$Bf?w>~? zR4;qgF1b%Etfdvzrp@J9a~&$gmj^7uJ^?gwPJ`F|KV-~i@X2d8rZ{Smuy2&Bs##RfZ$LX| z^$&eRO2H+E_N76N+lfZz+ZHL_z)!4w$GU}7c_`(b!ds#k|BHuJvznr$6iE@mYg84A zUY1pMJA&qk@9SJPLPO&LWYNs6;+@0T3ogYvI(|D1CF}Hxaf$M#a8q{_>J@hi&Zjqx z?`hYhar$7`jjx4YDu?93+AnprnWhce9lu@qpsa5B`9Ig z%EPdgTrNABjA#~vt=Fv!4S0RW2gDU4+#V0d?Ph!Ly4cHOmRx3< zoG($NPzX6%~Gw141#)g=ENWyTffA0vE0|nUZjrA>~ zW~(06)@0-%9wuy?9})HLCH#DNrjd>8j#?oM*ogTiLShIq<<=Ok)K{o8 zX7!w)s*$Yr@4KX7vv}`1W$+G6Ju+tr+7`gw;z*Wr= zLBm~JKmO=aQJ>nkCrd$r+(2VIpCd`>hq^lKyK5L`inf`7-M)I>8RYnFQTfV4ffT*A z%G^oP)7auN_5t~tS+q^~8MJqiQh$)hISYA9Y zyCXr@y*|xL`Y5J8I=-RS%8|{oioerkZ?oI!NyuyKnW^)x=I?XB5dk7`eBY{TeDzD| zM=E$RNFDKyzX1i2$hYy582_;IRa{!rTWyuRl)nK6#)XuRN9TF)bbD}!iDAKZVskFy zq~pmiWnR9nne#YLi+z4JP;?t5c3JOcvH>a_eBb;%vMS6(eY9wJ7b|M)8WNJS0^+SD zekdwj>?%gm4UCvwss4C87d1-|7vN1|^i?G7v*uGh+Q2YIhQZbSmug06tHQEbbS@3b z*LKZYhToRs#83)~VR3vD{9)1!jnN!Y}v+*H0>sq}|4e`KGO$f|viOxKE!0AaX zPpIPA*@fNK>l(db2{BVi-xjCM%G#Q;tkpQ_UBcXhDBXocr)=Wf4~0}W00P89o=*h@oQL@DX2V$Tf*4h-n^V9;l+hT63&c^c)`07zFkK7pM(V- zDVSQ9qm;huq?09dorLGJZp-|foDfmx6@{k#JHnVS$ROIji5syOxEJIXyMq|}pZBN~RZ)77WLvt2KO)Cuw=aA?RUu&orMrbc zpv)2|~PT{v4Z9VSu{;Y@e3+~$z(ZFK)^o8Gwnx#)r}Xw)?eVi6OD zjY|^5ZDL!x`fL^;N$|B|2yFQ}wZ?F$ohJ*DilE~Ad*E_X8 z8cT+LDn+0*!>nmGy+kG95^dinKX_|FnSUl7e9b{=W8WU^rWzyJF91@EeGJzGcDN0n{SanYw5nBe*$3EYl zH6OuWAn2WF-k*lDxIZF$QQRlrUGrosTIV#`b=$r?RCjc|dQ7arUfk|nUgN#@8&M0J zlp*ot=IBA%m-5_9(V@g{1ekOz>C91pfF#k8e!id;QE*0R;%To|@kY?tuLxs21SFf% zwQK4&6Ltzrk0JmiF&xd#kC4M$F6Y9P$7UbActCv{C7F>bqDZYZR9nYwM6>b)RlV@E zLC(GM6#Fjg8sr)DKv8hEZNb(vOzhrj4gZ>~6y|uXF+p(~ob!;w-FAENSJQfX_iupv z&zXt3Vb5e|yWE;Zh5g(BOG$452gyJ+_?yQom99TiPQ{*9VBY-(7~br6CbiuwYxh~f zpah>ja7>Ma_qyogW~0jN>Q27~^DTvWr~Px8T=?-^jo6H2W0c8tMTR+_{kTKZtOYu! zz$(X*0+vGLN+=WYNwBxni|?9>HDsb=mkS$`Dz`Z~Q@iuTZHHA>z@T>10;K_{&YQZ1>?c4ym~-8qd_ z$<2!Tn&orN7t+Au$sx_~YO`&gZDW^RiV6Ym3%xZbXkd_%)U%AHG;?Zmb8SOOektQM z_mXx^f=ytO+eLA4=)1Ps`uV=Zi(DR3gpI~f4l<`)ZzjX~onBsENcNAs)0z$-eIQ~` zqxaRgFF`BX*|VT%Aw`87J3ty-b!D;y1mboc5F-YCOm+CBY8O~}YX)F`jnZ&-PYAhQ zbAL8p`L43CeA{BSW%iCE^RG{2Y}bs8$|uH?LgLxEaXe$@Zz_5v=0(lJ6%<$rl-&3h zB!gMG+Puo%wKoZ~@SX#WkM`Z*RH@XvPywn5#6vj?t>a|IB2djG+; zl(U_`Y*3usOo<6dPQ|n0hn8C9rL$+s(#{ zc4XEzGb?SZ4t>Y=q+G=8F@XvPYVp1ycuBOqyV*JPOJxpSi)~iv^tK~vLp-LkZ>o8d zTI{5oEoDT7E!DTb<;I%iO?<@t{PE*|$EK&PxH>agS%oEq1`)9`ecWNHNl(kHX@jI? z=Vpf`x0iO%+&*f)6UVPD@BrTiLq3FdnT*7o?-Ft%N{FLH)>;2pQu zv1hH^W8%Gv5|MiViNTBrJq8@_kAQuocdX=`<>gi6ZU+UR?$~~wFQ;jxPX*$;1iuds zCi;I|X1i$pxU8BoA`Ep_i#~^@)?*m0?OcL7!TE$9sCKglaC8kgz~XK@ zPr<(o5`WJn{+?EufK0Fxi`H#ri<`yt;jOmH+%f3rzzjg?fO}3%u&4HkL2q<}-aI$y zRMB9h{-kwRtf_LQS@gP*TayAFhYg==2}<%KUy_o%^|y+qdb)Bt%{_6D__Vd#bU>V= zje_M0_{*~T1d%|5h4(vr(Z2|{PtMYszm(Gz8VS<=+S6qkUi|j1d<5??-o-^hae-F5 zz#+@Z{o8L_kYLN6y5&@go0DpVD9~j10RUPf%U(`#J@xFP=Sg zow7T1$`2n)x|=BxI><1aXWaDhyW&NaBXjw?bA5WvK)Q`38<)z|rrxK4f`%%Dw3odShCg^NO0dU#Y@OHkQ+0O5NagW&f?`xJVUbShj>ONSxl#~YF- z41^2fA61yU`_!K|z8SmTY`ouM-oeH1ri-~V5M zOewZu_i@K$Ye@n6A|<264t z?Jzs>AitDtmAeSyBHn1{uVH}^6=j;+-w$l9ikw{OH;?D{J;lDy?9Awc2A-2f`9RiAy>TMPGjJ_kLSJ z{00BVel1weGYvIw$oUX7cpof6pbYKw1|G`(flZ;K-LiC8c)-sUokek#`02(}_hLm- ziH0$?%CS3a&^_Thgq(Jj((Cjn^zizX$wu15jF;vdt-Md2qME_hhi@Fh1MD9nL0t^S zc8mc7&&OL{Dry<3Fd-AXXL+3g;r%XMQrGRS4RCMA)SS<3Z}T%o1Jv+HR`qpne`s2M z;LAi!%y6cZ9)1~cmbdA6x$>P$s9oBFt&-ABx8p}}+~-!Ky|-Ip;=mu9IBr-EQAxM* zbj*mM-o)rg*GoQ%4B8o^4XmrJ3oR>Q^rt70wI3I-?}AOYxORC!GMK92J3g)RAtvu> zW@1Gd<T7kJZDhzR3 zff#Pmho!yG;N8VU{*fvjU2j}uc%|RCY{XmXi=^R5$fb=ITDesh$#r&))5!!LRC77N%Z^orvT3Q(^VS#xa{8-DP{1-LcKcJAksL&bHQ^3b^Qz`B-WQ8QLwwdAZqrnd9YJjG|v@-L!?_0$L~cS<1UimJ6rE46lI4doUuYSX2j4E%OtKk@Osh7 zVlyH2W$$5MN_RMm^`oFy;no}5l;8THH*UC{DxH=G96Rj?@5qL+e0wf?9Fbg=3s^56 zZ27F#OJQ>>xk_S(FB%a7GF0_ju}?5wGT!ThciD)T$ftZtlByhH&U=0#3yh8R!EkiN zi2ew+;7j&$zm(x)rxu(k#+^-ETJ9H7RYF5ho)4A!;}l+Z?uG-&+l5CgE=ud9J0uJ$ z!QWhb@CP|Kq>}<1MeEn`M?#inD!|XX71;)JQ=R;7(}(xV97Uq5%V>%EoY+lA@+xOGQ89FpfcW2p)on-0_;wkC__bV(i&SREDZ&w{e?f`Gy4s z`}*5NirFAb&iLG=qVe5i=C2YHY|`Gc-0E-;g4Wf^3*u}mRgvnp?1-7OaZ2@47QMUP zfG8C)(N*4l%<5=t&+^s2-TXo+E|xCAC4=AxOyB zotJIXrFoG5jLS>1oFA6XKR!Nxe|*Atxls5>(O~9@zK(rZgq;_0xgo$REQuhNcr{jL zyl0QvG^+H4P1YETiGNPP>sr~~kLK$ANs6ehrE`MOuE{k=Qrdk&T8*5|QPlygTV|!{ zdND?`$5*+m(QuQb8{i4=x*JNDkj$a?gSn(}qSY|##*p4V$;fsSJ`(22`e=^TlTA7h zfYXZ{+1(#i%c-g&;p@^j?ax8v7vGG|q*$g*#uWX)5{lFm%8m?rz~WKD*7`A`2=0gG3mR< zD+5lpdtz&n`0eMLzMf6>0vTtZx+BU3PV7*xsotbSBJT{X8qmqA(0d-!$p+k{(a5=Y zQ*X9~rFc|?(Gy7df`EWnPu{;+>Oj2UcVP_q`cfD=vt#d{4{0#p*BI+-YNeoBl?vd` z;%0fmKs_{N`pLP&Yc*AhFnd5O^2m#CoFaFMH)PuUGQVJlD=f93ODfYH8pGj1wRRr2YjwXtcj zdn~nq2;$j^ie!FHFL1sba2wb>$UzEWU8tSkhhNKN{<4)QFYQLZ1RI0Ywcx%Lbc=^W z-&~BI+++S)*1mU_Hs5K|^KO_{F_MSlq!^u2ZCAjai{NY{kIQr~3;4eJNczUb#Ad@) zM}<@IksAZSE5M*~@58%@aEidR&S5$&C7 ztT$8qr7F5To69Qg4YKRtFyA_U00D-b2K$F$2k!PHD)(H0K9W<|{#P8@|H)_NP}=po z#8<^XzeOn>6bkOwpWJmqD|RlAPV7fYXiT{5Y;>DySo;a{k>DinS@FG*+mD4R_a>>d zekG8MwJ-_F+q&$}bNaT;8%ysu_^}y@o;7CYjRx;Xm0rqYHwsC7J7KXJ3#YGRt6$Q> ztMvz6f=uFHu%n*b6)k5u)XMtinRL>Oi|jD-IM4S$Y->Aahp!~+jGJVmgMw5mRMcq2 zBENMKN^JrnUH47Hot5g`;FLMz8u%KG^m;)iTzU2{iH9)FpU9acKYMDrX}YC_W?-eY zY#bXuPjV}Dnp}O*zh}_Zy%N6)4UQIPLMQ;&CCSrz^Qsuq8{J*C9mlD@_0;urJ@lXy z`bMknD!o`3egO5K0Zbia94J5N;@PfK7$>P5+kGf9`ExvAcZ{UA?N{ZN_X_T#2cEbW zNqe(lUKd4!mx80%YSX;!-vDLZZaIU?tpb)p?|i`~DR7_ZycRB6sxb3#Btekc=*t3s>2VAE=x!Lt%aQcb^ zWdh4J^f|7dql@eAw|k(j+5M!Db*D_bPYYQI?kPA~ z0;iUf*1WQtH$0@-?Po}M^4XD5730rU6vvz>sS-kH&qGiLqec~7_-Z9e!|HpjR zy3tepJX61d95m*xwP`9J??am3fY-;kCT3d9{z)et`sAljE0M&HBxV;T`%P^3_4Eu3 z8tqbByx%}z^#DHL`^|;w`YJqXQ+?~kLa|Z(Z{OKT#Mn#h>{|BMR8cZMTN@!_=t_-I zRlBR}k^*hlgQ{`zZc1S@^Ote9@G{nQgWLd5IQ;m_zyspOVIPbI`1jF(0f~oPH`iV@QaK5voblyP#8ZEllBbI1?5alX?)|7d~qd?0=ec z42PqR`$-^9A^9W40V=#_KwUPc$o)PfEl|E5oV5b}#2!+zRo(eQ9u0MNiaHGlLCd*D z6&D6m-IgsXg_+O(K3gEdc;W$7{1bC3#2P#{)D}o9*O{oq)YpqYLGQ$775sEUd)Pe3 zs0iy8nCx7Ns9Ppak-qVvadHv&bOXk+tMMUoWv9@5GY=66Xp$E^D&Ns6N{8xV5O#{`K30gz9i6c*0ac_(8vhOU~|-kl7G`I=s8mZ!T_VfR1eFq! zq%=`k8#y#_e)8153{v%uwE(N@TvLymvZ*bTSR$Nt44FAEjXM7nc0JUXXD0tc(B08A zi|6^TtC1fHWV4b-VSslY7^%T`%qzu?KvxCUg8UG)?cCNpFND6-MU-?CQt6#{AQJ2p z7F?&2zw=5wB!kc~Djo~?PTmy}X()0j0rz8buZ#*_4zny`Y>QsdvXkEVe)r?eX4bjk zH=u_iM65As5q=ZF%Sp*-7`va^4KYyt+Ya}sN$&ZP&$j_hN6z><^)hNTe0D#h0Ene?>%qbry|`e@2|@e<4-=1g-2p)B7+K9x8#_tJSlhn5lMf=u)yI74LAS4t)2#WOaOR>gVpx=_PIRG%kj-QRP>J1c!Zl2ibv^VOOBME>>H7cR;6 zsk7~yw_i_h#o~$|s1tAEO2_{@luN0>haC{Lv7Zf0kt}?ST0A5_{-Im{ zUFp9?@c&&cNYd`!YKUlIxRn!V-EvwnUYpuzs{fr;UU}&Miq>69zq0WTZ%ta;^Q7AZWfG#` zU;IayqoNhMU%9wD+rrEwTlToK`%km?FNn{ttcxQ~=MmDS)vGOj{9T`zu!C%B{tH8b z2(>IuB222<1(k-bQ`w#FZP89XE2Qisrk@XzNOX!LCII{ms4Gq%_kyqlnAyr`+E17=?Fu3fF>N?ZTFlUj{M?VdOd&036v zD!*f{NAf-vmXRDohMY9i`%dhT9z(;5-+!O>q-<`s&jKQ|lJHJIFrL5O#4bdZQnBx;^k^oke=q^YzDqGX2F-h>Q*`JhgS=Bz| z6nY+it@)HQ^De(Ox&KoAGw08w&CqlnX3yj3KBn`$U2p|K7<)UH*cF-tikvth1O!g^ zeYADhDnm4k2qSAhKLJ5Flec~Yus{>S8d&N1TGJ@?5!zxXcD-f3&-6ls&t=iLD?K0 zK^S?TA6UR0&EW?BYh=ZQ-9d{OYo*%}ZN02wzPSg9qD(rvrh4^t8iHm8 zPU0e&^=LlH*Kx1;dtuY|r5x-Us4x!?ZC!TQ46mQ^yv$K{!BuNmiwGW|ph!s038Eg;?HATV>b;^NsU377^~M!iufxj^L!v%lotxjLG0j3ph76#4EnB$oc>@k?^FI=DmZS9-a6@aJ|i(5)=IGs|IyTYtAo4p3p%wak$k8_tF@NYAm2L;^jW>3S3=xot3 zg(&*pc&@BSU`vG#9celV5VBoUW;+)=n^OPTq<56a$~ntA9gh%KR#|u^oXu(S304fI ze1yk-w@u;sf1FJFzhc;j`u^%Pep`CT2kX>w4KnE0;{6a*d1BWMNX^*9FK4+6cLaJ~ zdGJx-lItpT89g1h1-w;VCdW?_=@E0PqUKy~cT{i)QBnKcOP>U|-3%OjEx1s`c{-ni zD3?InYeaYQj+V`VBPV+i%zz|V^LiNR9&a5}^Y}JBpMiQzx$s8%nVB-0Eqn$1okX{>7qi4F6XPrZU z`h3)_+0oFyT=z6hj!>$!@_b8tZLHK#byn}Oy!oVe%y1WbWPq~M$V$fvD?f06`k6vm zK<&PC+ig-V%biEvh?FBJ@UDSbMlTiBv@>pIh>^XXf5kzK$HW4$JeiNk99bA zWJ(^*KSf5}}d`8M( zI_DYTI}>FZlY_C>|& z{dBm?5iX({V=Xqi*2CL!<1fEojsUPX@W`|LwgBW zBkzAIAah@TIrljK2IP?V>?YQ}$vl;4d}o8fbB7S&(ZXkAKm35FZwz!CqkhT~-UmCg@6%k6mA!VN^C}kEl9(81 z&>(rbcTy4b0yaA+c_r@2d--r%)$VlS&Lz4an%n+^$aJx^^$?BQ4_q~()|2k6N}DLD zdY^2A5Ua9Ls$Lb|!}t#St+ruQH&^u5J`GHzS^Q|d--#$OD~BbgtGn{2LDvSk1UW6@ zwZa4z4g^j*(9p~ccR^Z=J99aZWnZes`%o2yos#oFdKy!Dvx!oYwul@}6&{E}Rn8~x z>NzWnmZCyW$B5QT65YTI%i!fb*nHc3r1X8yRdELV7cH6Qpbd#l{pNjOg)-y#JF_F! z7xN{N^>I$WWC7xDS)CQ;xinr>sGKG0voo5{j$=z|oR@7pmuVKmj;~W~nre_3ME{F^ zMd`uQaLH*-^58{PwannRtDq)aj(AC*)KcC{_MTC-^r4@GU;B-51M?qdJwn7$Cy zwt;~`wMmQ7Og^y7^Knqe)=WouoBt;pwp5hPsPi?$Oj-Mi>yB04rPyrkCg|(YN=X!- z!vC&u^*{J7!oLjVc>nHl$e?^K?^U5STUBLW!Mov)BbCyN5EwFG`vxh6-z$c4uJ2`M z0RtRlF7q{F^N2Yc)N|$F{5N41><3U;{J8;f(vg5HJvkwdd!0h}MU=B_VGOO2OK>9T zWrP1xr1Jk=lyiR66<|$8>OE*D>CYe-Wi~|y>oeNTr)+-(!2~F1Z~R^S&Hs5|P1Xc3 z`Fq@6rcENYzxmRr4B^?JjOfC-hsz3PG;>0j0oOgK?1ok?kE)-VeO?a2dk0i?#+Nm- zH8&|@2qeH{S-s_6G6qIpp;mimD$-+b zp9fto2#F0|U@HnkXnCJ{F?jFe-qpSr9)P}b%JP8Q+vE8uhWW|y$ppXkynu0l0=dT% zawaat(MFfCBGkd_VLIJyVNp?-L@yk~PfLsJ%A0g&d+wfIb$CI$Bx^b|4IrXy5FD|SV$u3p#R{@#zp)9wOXHyo2lOSv{dDNvly$aRc{6HULrT)y% zZ1)9yyeuqO`3LJc=M8N{(DnZLFuU!$rebfjpsQ@}1-^;Gy7rAA5DYl;bj@|N?BcFj zAAki*!Ug?Pn!gZ2 zp?PBlbzU>G?8^OvAVU(4&qm}*8pEWjYmPq{N%l%K) zunDh_BP}8v8n;8DR!m3<<~!aHOl`PLVn^(<6g zJB}Kt2->4LvB#z#g^(?+b0=-Cyr=%GQ3J9+butDfG&l3HBBOx3e17-gqd4a9krc*a z&%mIo$uD&oqe?SimFGX`^=T7W@gBv(2m}rkfTWhMUOJVOzDU-6UHvvP3mL*~(7^!( z-Hu@XIzKZ)w6_^qrfo;x_SiTr2kpmuv(@4AQy_JsXl)n~*m~$a884=L(vQO{>JLFRBR|qbP5*}{PvS~t-+z1~lnf4*( zJPTP1#}UIQZW>$l2cvqrGDgHjlDG_a2lkdOQWnmj1@7as&&~G*!EAbCha=T@%=gFy z>RFyG$p^J!3TavEE83rX);ydXt=nX8D3P`oOB9T+Jho)RO&J){9~ou27T3t5clSfu z2TI?p)*N8p7J%sk{J=Q@@$-+E{3og`EdM+rvi@yns`xLfus_mjHpT@_SxOiACTXU2 ze~yk(wHe3fL=AZ1_pKwJRN%SMT<{~~Xa{}Egu?HqUnt4v+2cD}=4H7*t`-~E=pcR^ z&p?l`A{1|3pbRY``8bKvF)u*pnQ%^13?@BRda*6~+E8dY7eH+hH9g;(KA~G)j51;u zAW)vq>NxHsD@9nE%wUDiF9ul|rAD=CKJy+^V_GoHn{xs_@SbX2)pCOgb8|W6tC$sB zb6s*|O4`>(kq+Q+8y|iv$2LSv=ec>(42*A~F1_+_c5_9&)6r-8xe*cTMKAz*LraTukHe|cKn7hG_Xb*kGpt{Do7aVsQ2~;{I zUu^UYb*%ZQb1~jhEPc&ZjMtv#YCS2r5o6eWUIvF_t5hVsvfE z->L@xH2VdKPH8Bg+b5_)i;m1~TR4in+X;NZL}o{`nZnvQFjP~H!^&Frz*U_ztv1wp zW|&U{Y?ubmwkm;@_Rk6otxnImRQ43CD_5KDtkg-6P$)6#Gb^ku2dof@=n!1x>fSj5>Y#m6``!`(LG%j5O1J2qt=s!67F&Fux4+&*N3 zm%9|TXc)?yL3j3D?u2~baE}@AGZ)WyFR_YuhKf0o-*Xe$+#3+6Ilb|QcxFV4l>U3# z_Mc1~!gQ*8NzT_AeQbm$Q-&C?QQKdcbLQpeTG;HIS5{5gdAco)C=o_H-rB-Y7G%*> z{`&0rW+nh+#$9Z#G|e=v!4n#!;6chH{4q@|_MSKdr5dn2Vy(D=80+;G>HQ*;TQlo) zuU*Q6*IlPxCM#1OtJ})@ie(-^BY5J-1U7MG?Wv*iN)y_9*+ghKR zPzR*m#wKqgccBNA8Wr32S06)=UuSJL5W+RD}2 z0fEiY8BM0?xj~B_2}tTrN%IP*k+%Xs@crJw!8~I9!=%U+_K~vhtAJ4BMZ3I%as5|D znM<#d3qOtsmu(34vNE82*h#;JM%$BExC<{_+UL}p_ouY0JwMmdrZ;}#J`nMCn5OMh zWqoNFF}}W+YHXcEVgp+pRiVKwmtu!^_%|keX2Q|=!O>GQ5#s9y{hS`|W;M^bR`^fV%%VgmY9@}HP_0`MLuB9P@OSWlZD?7 z!(<^KtS7hSy%fK>la`^$gK)7DpSaS8Exsxv=e)%RLmlQnJtPPES@pPR8^qMOZiKw0NeBQ2_H zU$eOMrc&5VUqE(KK!!D81K~(7$CR>Q7UX04{Io{b@bXT@k9=dwkRPsyML8EgU4iR; zKC(9!POm+Ygi@=OZarBZR%RLe@mp*S?e}UThRWyODS=;3zFLX*a#$Sy6&K;fw*Wk> ztBYa%TlhCi_@a@`yw?UrFy8yy8~|=e6jIdP89KW%8d;T+%aGYZ?x!h3nWDTgA#?tzC3Jn&d1#D&lH{A`LFFQCUA<%ld4MiJ}Zhkx5Sj&y-g14F0t%h@jF_r5_rrYTWzNNnb3M%|7Q_Y=%W$A63Wv%U; zHc|G{9Oh7LwJIH4Y6Vrd?V4NNsbBi^J=?o%Cc8CdTTj7WF%$DEotHv_Rr*dlu*9U{hxz;EQOb9YU@Vv5CprLr9co5NoQmW>tR zHS;^kriQQU)SP#npeEZ>blo(Y1WQh{OoRX!?YHN1+7^xpb-l0#4ZDo|__LwdPi14} zWtn!Z4WdFqCx=p3sHzcZrL>wIkf5QWmm^<_X*N4B#`W3VO#$iI-UV0?766!ou|-d1 zlLm#N?Cm^2vCn61BoOwY$vyj1cD$uvMnezMqc(p0#MVTHn|abg5s9&e}qt0i-MCXI`b>}}e`=pRR#J2owFm2P z{eqW=2}(v?Ic(nw>gCVOZ&JT{^W}e#_nuKrc3ZzUNKja7n7K zK&anR(oCFT=eTs-Oej+G}F?StXth^9dcQ*ZiK`8NG5+@~pnLZTd zf{y5^>2T!SR@_N2YW`}Bu842R+T=skBpEF%us1(@#{$1$PwfL~ko9Kay4L*XVJT*M z*OZTmDow&z*Tk_yLL+{&X-iePkSaT1)p75PgAsmdl_?2vDpv$MMD0Ef{t--c#9!Ac z;F9`Ruq*#6DSYM60)X~k(a?X&{`=d4nfrxaqF)4RRhoL#2oDy%tLPNzj>PQCja=tX zcgNrEZD92?c{pHfrC8{5quzUgd#zTB=HVXvB+gYa;73|?%^7u+)ozZumD_&6XU|V1 z;(bq1C6EQS>!d53BaUlWrd|a(4mOi%o2{K>MX#?2HrWH%Y6%rl(#xyaQ{8YF0%DW@ z%b?b9rvi8n)qorSOlhiRida}2cB+D_>)qg|7KQI~wu^^wrH$WF=iX?b4zBRzV6&>6 zvI1U+>cZgG8eU;fn1pF%GMXC0!zATla+_gh*HVgdJW-gUm|>htHg??jNg*A9{~~x# z_BQ$DldM-AFxXt*hHH(W?dENg7Qfc=jopU^JB#;GJ7Q^m+2|WS32wwiqiNkM%<;4B z^FxS`-aEyY;F+0Y%VN(UndA^p4UsT3A|(n$<-tX-o=bDk%d#AfS7PueYKzn8s1F4N z1h<7vW6~8teD{hb4AX%$cRB(GBPn$ou2BN6FRl(dcwR}I<~$!?H=LF7#9BMEb3$Ab zTh1a1(^)dQXzp05K{2R-5tewhity;^(_^^JWW>QsjA`yWn7hcaaB!bJGzESNO^w1VUC3 zdrr)*J&sLioN3*}4A(2xpZ&<7&P#NbCzqK2EfLj484x-U8}9~9J-fb3uFQl}Mj)r2 zp<`o#$~^rs&`0Z8Ie51HTZiLu&VHOp(jhdFIO&OE98H4M(Oe(*g95ImKP+0 zQ;R&J+o0*FC&>rrH$83r$Kk7mt}q)aUpYM4rERTV)2UoAbXD%slt%$1lOP5bFC{x352igR4s-z*VhWNzBWK{^b3NAi|<;S>8^(Pw-) zmHVT-^L%YcjRDNB#_7rC$e8Z;e&L5q41e59UbpJSCKbjZqnz`S2368q$SnLIILy`& zx~f^TDVrk3_WR9A?%!c5L)KR(FULx zu-b<46A_I8<&`_0e=llw-@R`E#+O!z7F0~7Ng2sa>F-`wmkFDeh*aHvxXTb0=(#y; zw-96%YZ1!dviS?!4J|3lFNr~-!qAn;IUD_)oZ{*)J8R_$(zr5==yLgmkm7J9a?>LR z!)R)VwQz>225a{zeNU(^!}Nq^E8nGNDku$7#KCn+70PUk^%)B3By+k8sNzv(mvTQ| zOhxQe6}9@5UOt9rG|%oIPZ%CQ)E?Hf`9(gRD1OY734AeqG8+1-t0;shHp({Wk6oQj zcnkI2_uP>)P#=J9ryV1P$Tf@2W@Jo~XmN2g6W6S{D|UqAS7pMSw}7M={tvbFke`9 z*{p(8i9a=ia1~=EI_#@1+@}!J3u5MyzlDjHC;EDty-g5e73MeaAPr9Se)u1sSRd9K zTE2S%8J0r6bw-o6!15VdODwf#9&>I} zX!ks}-KiM?^x#x^T{Cw|kkSNcw7cKIZz6*!EttpoAjZ0^n+N^no2$2*&l>bJQ=OcA zJb}rlL~3+5#1wU3*(>~#-g0y0%Y{#44a;#05A;E9XYPouqXvcIj4c60s8A+N^_!@u zxoS7qhIz5o`pqYTab@|mXMdYz6R~h1lC!MwBC%~xIDMDq&Tg<={g-d@tB!9jhYn9a zf=|5_6$2}}nQHn!EAS*G8%$aro*J0e&XCoj3C1;3tRbaLt(o))Kz3bugVT}%aG}_P zyiJqfep&~Sdz$}hQW^4c=}{f{MbClPM-q~KYazaP^0I=rivgGg0WrPzne5gFObY@p zT8LfPV2vIy+g*yFCzLi5t<<<8h9S>PqU^XuW!!GJn`F zT%B(dPZL$pGhwu&eaDBsPPgaV2>5uqGf(_-DGC}I+_}``T&8Dt#~%m3$#SMzKj)P+ zRg2t&I*H>t*mHr>qP&m2y~iegEd023sZ8H1ia9m-VY*wH*F4_qdx#(4KaT#a}T?jO-d2ED-i_{h!DTTF)HrPT)lYi(ojkt!%OU>8SW*4 zl>CzU=I7$eUlr9$m!q_&HvSS{o&~NQhm!}mB4wTdwwqiHHSEj$E-a^!!)nv;Ga-c- zy{gWUU&00OgSkz_9TytdVnmVQsiOKLX(y=$;A+7Kl}szS43=Tc7wzxX)ye8vU4eHS zSpZGvm4z|0xC%D0Qr-BT^|H#OClFx)P@RWVs}-{`z3zM+?2Q;dZ^{A@v7?BU-og9& zB0g-t)sLK!%5CEEM$mPd73XpvQQj*SM+k+=((~HpKltL9>nF-5r=Jv07v`77JS#=J z7r4Rla%Cwbe&};cimM%vMpV)umSIJcVp9QLICJ)4DL-fls(5c06(hMV{%CQ8E2{A$ zZ+3P7*%+NJn?{zVP3Ysc-BPcu1FM1C^Se3vpea&`58*uDm3yNPDo$Uq^m5%!SEjRM z@~r%*OQ3o+jvM@n_roNzjT@4yLBBk~J(;13a{aJz%j`?c>~_V@CPC&CeoD4F*-)ll z41<({yAIF~awgt5(m>U7qNwETUt&cWiXUc~HAlOYhxq$Do1|gVp!Nyz4~~=BB-w}v z+)FuXN_!0vW=b>gZ=!7yPtniHb|p6UjW0$+86tsmc!})uuf4AWwGI4nix{nAF>oxVjTvuVYX|~|+y&#X+3V&#kZ&BlR)W=YA)vKHW z)}9BWoMf&^)@&Su1wle>$$}Ugf4ztgH@M5+uWqL4`MF5k^Ob3oS~-y?ZhdQZu96_U zvt46sb9SSude(`4zT`3wp}JLHD<<;3_eD?RtH=g^(sQxsH;(TNccQ?-BjuCWrj3fS z(&Du9OU_%@G6?D;>0*_1dV?7_l|4;JnP-ER-;1||B0Ea%NUs3moc)N>THaOGqmPsV z))O*k`N#maNcR4y*%sGpjs}Y{PDui#9p^p#q)jD1+yPDazW3aR_LZLeIK8Qn^%rDLKY4f&sw!W#+>?2WLiE>AlcD-rocsCOM58IiuQQbiBf+)h)N7HaLihcd`@xG; z@zEy6{f+Wzx01Fi48ZerP1ySz=D1ysr1SOudQ$FfAC*|q&E zkqyLBcawlfWdp=NpLx))5>*ht4Y5^v|Ejm$6=P#Gscxu8YXWooX3x-f zPRB6wwncm~3lr()F1(688Sj20riPk-AxaYgS3sqbg#d~^gKzY z5TvnF3W<1z^YfEIzb^;@Js!7a*O3$HfNzt>y-M_MFzS)t9 zB!>6Kh&UhKX|0#eYJsK0PR-|PW-I6@q7O`{XrVKn#r6wIv|^5l<_3lnYPwVQ#gPgg zbe|lFTxi|eH%Iv_GPQ=4rTyMn2ajKJZx&bjW8;aSzX`cS)|^ z{3e<&Ha!(4AdR(j?u0CVFKg>o2Qt~D84@^BX!1J`9ElGJzr);)FWLfNt`cPaO$(k} zJG^#ZJM0%uK24;!4+~x1q`g29P~i+B#Tu=W;4A*IgdN3tQ;Efu$ULKtsRc_j-rzwS zj`&6zA8`NGMBKU9Zz9~i^s5K&F?`rTT_APO_Tz7z=h5F1ln=c*dB3U%vyQ7odPWrl zQ)d~ES?^WK9E$-RU8uw|j@WIq!owO!HnWB!v4cO;E9w@wpWoX=TbomIia)cp(v+UP z*ynu~$AtC!=v+{L+TqgXKC=j}Z^cEj4Er~AS4A8|KY*$#XQ^?)?)I|w+@4OWH^Gd{ z2;tp?xWNmoQVuJeLn{|wh4H02?#-^orW2jYijh=C)$)ZQQOQujuM}F0%AB^%wwOBL zM{_$N0o4-E5%8-pdJ+UDXHTPS%aY!;-H!vIdNX?}F+%GYb>1^Zf4XTFM^@PELX>^L zid+ll=0pW{fYL@UW==y_ivgi4xaML14nV|&PsdlKo|!+sgrAKmDL`MP(Yq^T%84Gn zzokgm)S~7Mmq~zZ;AmPe&dF6F{Ly@h4h=ZOhAyP$qKJXyusaHYTDHSWO;AWu$pV}V|-v%YFJ z`B&pdz#P6wd9-nDUoKd#I+lvT-L&!L3X$Vz zjq|diVPo`1ke0#DN)3*crC&aTXbi^|;M@`daLkb3gpCjLo`v=Y6k34So+o3mf|P8M z>J|<#gmOpWFUtMfmKBJ`*zk&a%@1&sx9r__jvR%86xehGmIf3o1ZTfC4abPLudk_s zx>YnMo0~^Nf9Xmgv;5WHdDpceg!*#Wv#QgTsOZI4;JHITrp~LLc6Z8eBKd$p&&p*;{P?^8*ll(_T_%^g=f*|k%fhh8NUl0B4|bkeZ|`Q!k9{(612UuNjPmr&C3fKo7fh8 z`)|S#_dNgqD4i$}Zr+_Q?;3V^#r{0}jwTp0n7@4-Im-05Ui(d?*yjF7?G9Pn{jV)s z+5Z|Rcs2fYY4N{?3m?&a`PZf3|2hAn)dK{n3)6qY`^&xludX%B>F(E|$mZso$;V_S zdXb+p!+$cov8z(UW{C#nS^ybn(k$ILKx`y!!CDS!)%`<{Xm7&V7iR zD+ZN$v*eqLsGhNqpsZ?CMglW<$BS$YrlzF*T0o!>67T{ zI>a&8MdH}xWv4g~)sxe1Sr4iMMSLW!EI9it8Z|C4?XRRyxs?N6N9rEnrf*l$rb0Asoimhwu2z>*RFOons|}u&m#yaoa5m}nyEJ>DYxI6xdnTt z?Mn12*Vz((W?TwBOfj{FIC}B{U&T9F=kSY@sFGe_i$y@VmiP+8gI zei+<%Q19DEtKNs~A-D9@NI4&sW|fv`FnOFwm`_fN)p+Ckn<$TnbUOfk(*!x1 zKfR9+v(81K^Y96-Jb^G{DF*5g6CX_PJjvvP9OxM#N-jhLo z-JbtR_}dZgsWZ+JL=3WI?C^Q&NYV3zxuzDrL;?(4Y(8afiD}Xb^?Jg5(3r`Ty(Ii` zjfqCo>Tg2xEQXxk#NHs2qPjlJjQbu-$!FKH;)2IooC>Fgfv)3ys^B6?rgi zs8~0pm*4Ao;x#E7ZohGQa<=Juxx?+jB%??W^UZlQ8iL>DDXK!y+$ceS1qr-UMO~ki zC3yhDN@`GkOn@H#t@{p@$yUi(BNiGB|6rdxVw0c}5;KV$$)9nl&YCo8K)=Ii6alI;{7*+lxEnfnd&n+9hM@s^NJDor~53leen8R5`?@5 z!QSwOnd19|s^_su*Fc6d(%cxEl=R|b^m5^+r8T1d{vO4dC{u$ofo&&guZ*vmhT5|q zZTLv|iV9ckVIw}T(W37(Af6S@pW@T#dZpC5!U01f!P;_M;YXBq(*ZG;LM_hJSOsKT zr4fgcLG5EH=`PIsF6xE4(I#&4$yhtK%GWBZZ9-Tz)$Q7#bdAgEtwfpdT&E3t;-wGJ z@_fd#T9XNLBOs8%(vOLbyl>~qC2l3|r{BrAloX4n z3Dr8-KD3O&yZD zdDFrnT4OE`mgjfL_>wQ!VR>&-X-wDV+k z14~^qDaXW3497(tRT3N(qLWRN|(tLyZo6v zV#PnLBlO_GwGUqU=A*;XJ`TzL3sjpgKC574av(BuW%4NE1TXkmK>&0X+DE=PO`L=*VWrGPL~bvqb$LK)Nm52J3|TQ|K!pOK&K;;n*Q zdbXJx*#2bzQDrV=I=fL2mYCRk9-A+g;hSSHKR8nfe+p& zio=c42DfUWt|*El07mMM zg-YE^XSXr0M(&nFtTcgmJ>b3>Q&8zw_uPK6t=Vy9le1eO4)(_=`yQy{8fzbo+{R%? z_BzIjeLxhd?4Y4sio_z4H%+H`pZ{?k{3Z$EaCddRrjh5vRfeIgjWwT44^G|AJ@lv+014_M91auB&l}#g2AyrslQ?F3l+e{@*IytyM@hqq3Z~E3A-SbRIOcI9EoF~!t+xe9 z!xWJ;zILP4uB@zr>K+<#HA}2;#8bS^o>te&jF$P0xUa0F@GzCm^YYBUYd(1WFXzx- zZ5IDQD97;Qt0w6eR_of9E_HU_$%Uxr&a`h|iK*UN&2@Zh?Jmrq{-h3kfj=~n)DpG> z>3T9+%G1KaUo9ere2*;#s1@P$k~4;?d#q)GyBkHB@=vyEhWxTRAmKpLTZ&B;c{&nYH)y1PX|~OK1?xy0Aw34 zYO*vbYi)uh^;EjC+|cAq{~5gn_o}Vhw+0goQeI5e{k$-8Sysol%C|SJN$NZ7>kK~( z@Vl+!4hFwwV)|6`HsiQiGg-FlWE>fe7}K9^USlKJ?PICw^NoLzMO4*HKXZ87jwT^3-(m`0QRClR=AoYC`}m$3pDlyK2!4 z`T?-a6^|vTm^VK&{|3a)oD^lGM`frWKE2I4()UwQf2|Tjlb(5G2oGhxCs}$E-kF`P z!r$~O@lLZZ1yAnOKPN!|=kGga!#d@e3LEJwaJ^~iv@u7L({#v@-9)t9{bVmhR@(ni zF-L?kO;75|_6uRIUv|c3u56Kf281!=33u?c;ykwsH4k|fDQWMgw;~mJZ3R|OuP3Pw zw$OD>{)QiXP*&I=L+M1XIzbnlbI*bephgrxV|l#QK&orymda*!L{i#FKaOv}_0tGA zC)NUOPn~8}>DJ$kZ;xWEI}7&MF25hr5xfUL4`kAnnWZ})>P4^wvn_mRq^?w{g5Mkh=iE1jHW~trbv+<%vs*FI zg=r@`@BThg@UCJMx0N277EfPg*qe|DDXWS*=E_Q@#%#btWnZ0s zdsuFtZR`7pSDiQ(YAB7Omc+1*U$;SRjE`5H@?ohoy>Yq@dwe6i^VZ7oAg=w(MuUm5 zU+T1oN0a4n>5P#E@a?EbtygdtT^w_@{KBZLl=EVHc^ZCE)OmW?n7x2*eB?VX4L>iz zutlKWaZ(Dsai7|pfQ>qS80%N^NUp6pGiYt{_Cn3ndAA?A^_it}Jbd<{c2&A-OwE{& zWG4@CyLD)=Pb@gwik1*Il0n_l1s^j`$3!i;3w5ro_Iong)5AW(5W$M_zll!Gnei@G z^%zmn{$uZ5fjPk8h-G9o1!u@Y%s{?X({#o}vdwAt5YXW0OPsO6&^aLHneAHytrA1Z z5sPI|EE(=ApG`yY_(ba@{rDevbNO{iN7#Z5#mmkYOAx3=fR6EF8R1QjE#26Ip?QzY zi1>&j`QT3PHC?UyT6HbUxz?tYIg`i%oj^+6)rh3VhU&yI8m0lk2M~~0f}X8oaRc5Z zIpI`;TG!QSSZrN;S`o%#9Y>Pwv|W843Xd8CTUFK>^c6+6&Rj2Yf~Hj{dG0R|eEp7c z`&+@ttv6sjPtAE`QPUGtVMWoJN|*qvvG$X@kk<&GgitWAD=!>+Asp$3zmVB^WOk|7 zKqXNtu$BCnS+_A=ez>1E;1}oFn4#_BO?Uh|y{SdlW|drRHld3A_^bs@PR?zo6;6wX zS4^_(o|m@zmzr`1X^fNmb3nk?zD`!2i9q7lw>gKl6MvXZ$5?{Me-jl}1_*{WzP`q6lzo0n_h3)jN> z4uZjQ*Y_B_KY%wB(|PL8QEQ!>jm~?u?{uO$*85N~;M(;O`Ss*(kmpBrJ9Aqv&xUyK zh`17jeEPC2a+03H)MUsL%p5@_DhnugB#_IFE^nAxO7gb%7cH?-)!#&92d1TgUSDUVkE074G7WAhHXQLQ=VVWV>8H>Qx}$td zG01$2*M9KtZl#nr`tQfb4a&nf-yHP%)>ZE+lnTYb40S!+P;ZX0`SN^z$@Giw=k@RT z@;}sFXm}06rA}8M*-DL3&yxQL)ii3mflb+!tEpfQgt3ATKJ|7NdohaDjXt`(aqqpq z!*k!()!G|{3uD*4xGBKBUJ34th9amph6QE!!q}RIIW)RVhs{N3g6Ak3ux4To|0m)8!V0TL;Lb?exCXfkrr7Ur*I%edmB zPZFlT|J@ND8r)Eq{LYN-F}X4y6*OSp^*CAFRo>Bxwvomr9rVZma=V3T z@O%5B(_@L#0EQu6kSz zGO|Y!j%fn4K|CJSj6(cG{o>-Z%(bMRHdmrgF(QGHY7ZKu8l~cz+J0&A*F4#!W>=03 zR@^L&G$gdRd%V)ad?@*tmy~9OsE2tkW_JD5M3&C4dQOnq3Hif$rd&p_qUz1KCgq5o z1RsJE$=;ALsxl;U7FDngZZVesn7^zJGT4jW^p}v<>*uQw8yOFKk;uwL(><0X0{g)n zOUU*+<~1&qY+~@m(mv4+Fz4sF$T6TCp0aF&F3NU2O^=}^xYYf;nbH`6-@CkE}@#dlAfvsiX-@Uv^df=y&v zX^mYm)AOQfexDZ2b=r4??ygIu5fN<@>B}h5cq;$=v`i=Bj%No=f>)oo-+QTLbgrPF z<}7nG6y7i?vkTI?>wMeF)$G2!Pp}1Bu%6`M z`20#msWjPPKDi{zdT|=2e*Ab@SIQ%;R*~#|Em0ud=z^wrmJ{|omKF__M&bo{3ybNS zP9k;rsZV5>SCUVrgC02%4|YFgFM+1PY84@($;7Jo-ae;52x7Y)Lns#=fdNh1f*ROU zD`mZUbTIyhKgrgcM0Lgd6%fzHn$$yxO%Jj@GU>EMKgzxBm$!|z&4og4`f0G3XhSY^ zEoYYa7a%t3em~(p&^JixSHYnuu`~=)C{CTMxwLiP(J{C&qQWVU;^G#3o1=?ns>JVb za59||Sd_Ucn#@1ZoS!^lGFv(?SQag5TTHgj3Q<+cRa1VWv+vfpm7iOCFn-YX)v0Zd zy>ktluEEsoVmQq6CG?XED`_OsQ-mmPn_Ad**SOFUHI^zfXjXzMXjT~Hz*nv^uH^zp z+8q%@-UqmcI4-KluZY%+7Nm^?dVF0^ z9fDb1hXnXKwsHnqR{<|0+hPeylosBTL1W)sL|U3umc_|j?wBp-AxMlgGG*3=*4JPs zy2&Lva@RX=^3pf|AQ-zKD0G+h!DMZ%`?K7hmNQdQ=Z+OcvQ81+^(GGvsu@$8OVJ6_ z^qX(oOScLjo0+cZ%G>WG^(BoWw=!6JgF3;xcT5f8<*6>pBrm>pZe5evq9joOtd8&H*5Qg<(AT%b{q*n5MX41HtcUeWu^(Fjw z=RgV8H)Iu#!ECA{>VZV6N>dfTtk9*>eCon`;2dzV<8B6HMEPQ;|U%|)AY2y?u(Y21DSybD3vO`eoDuy6yCeT`|!0B`5)DVmFTiLSPQnfD8+Cg1H>Vh z7~SZw4+gtBh|%yXf9Mgf#LeSGgD!c?Cz!T9|^fjkYwdF$2bFr!$nk{SiQT1aG8gQ=;UO5k0x zBNFxmm!n?RpJtXC|2=e%KUFaQ!I%FPCdMfI*$SDkwU9+6fUGji+#k_r$n_LJN_qXB z{&CRV5izgUKl*fto9c0p?+ix=rX1y6;!&AV9CK9>{EFso6aGO;uFa$b7-IV%hfh>x z?zLBpitsY852iCqowAz!S4q2Nm)QoY$O$KcDnTc&3~2R=%9@IIaawI{^Cn-=E0Pm! zFq<|*X9Zn~B_5v-Y$=b1JVK<1$UD%AEl4be_q))E=~&u$`ZcgTMWAbQDH}AuZ9ew` zcd*icDXNq3)i}2w&l~7?-foR!lj_T38-=%Tk>$>4Wk}TQEB&> z*0iLFMi8JfBRuXZF1W4{lR>!SY#IoAg&#+(MO&tIw^EKv&d{6^g9@vRjOK?PUF|`j zm&5*uCe2L(VuLMhu%JTJyEs1#Vs}$P;RX77QLdFQ*SVfVQ_;{7ejnl5Rh~1bYjJ2M zJGXa`I)ZH`*6o16pK^(_;TD)*%2UxrWNL`GUmTF3jxeIR>t6TtuA@T9^DRhgsFDC- zvQo+g{3*5aWu?P+pO%Mhl9{$9t9i!=xO3ls@oDy>3){kKR&Xpq@Nq!wnnYfs``&{Q z8C2#hEp1|QX7r{wDQyB9_(URXH~d>vBFQe?*E~~S*UFF3f7Zf2GdDq=4rnT}nq3ol z3^*uQ5NGzhWo6}N>4BXX@7h+bktHt|FnZ;`H2C8X;}0-6P+AX$|=jED3hx35Ix6uAKB zBniH#Vm+%;K`?sxu_e<3Dj#pB=g#FkWKCdFLL;;D@@Bri{9KmZ6$A^*>{|T$I7dwq zRTd(lKQ(Iq_axkZ>!*L~nEB6_hW}vo_;0!7Z%;yoO^0Hl39KB2=VaW7cjuzYhwKS! zuR24YSN$aV&QHBG_nV0PC!rUM3VUnaFY5mZBP{p}=up+hnj%HNX=ea7wBsXh)Wxr3 zBd&zVBM$;l{sr(~qkIII;s4_!I*BtGHcP@^*l6vKMq3t8h7F%-e^mS&p!oB>Ncr>r zbqIoo|CvLSnP1LjV}1fIXp_tSoD6M(T2Jd6`j*z8FVAYr{KJ*ye{_HPe`+Aquj;4j zcb+FJD{~Slvsw-W_SVCv>r3N}Fa8&=&coTOdCoA0Y`ms&^9%+lJoBC3Wa3^m{zDke4!{|+2)RJlTi+5F&9Zc~ zkLa9KA}UU6{ci@9l;8UxrzWjbr6^~?|C;iNW!HAYaSt=LsM#n#(2`VAR~XyNuDkG@ zh*kJey(Mo&30B?L*=Z6~T*mVTkQljTaV{0MrX@PBj_{@s)4iiCZ8H5f6f|xtc0_i(G`|+EIa#?R7sVN^ZM*I zGgPXTGwpw$=LDW zM;?rY=AzQEaU^S@hr)!U`LwO4k&^-s^l&g z403cu2slG9@k+;rlYgu}+3M3!Ivi58A;}R&3>PO+ktBWj)S?STw2Wa#FAYY25my7d z@kjZ=$8o=jNU^p%P-t2z>^6{@a{W0p{(zzy*+_!W%AB^SbwuDBB#g%&h{6{eeKz~PQt}vsESWBs zc@w8+Od@eL3l_ZgEIk*sW^y&Hg-w+Q`~74x$Yu0)%=?Hh+5{2P7X3sb0H=266{+Cu z2g%a|5Qk!C4kxk5RfFw1w62I`ebZ0nj@NSWg$~1g6d14O=K~&o98YsjabGN(Z+~bV zlH2F~%73s`*i9#V`ftfHGWHh)AAa-l5Sr2Rvh&yHP|2`IkJKWd#hB(s!#C~-LHQSK0`N$Vw!F+lEm-W{d^_YwLV zlS8+!p8_M>vcdHOg}jk#^LCI1{diR^ebc6Jsxr5 zR)8!A8x8TYRMn$Fl0D})gqO~tVD_wyU*)bJ-#x*<6E2c^ zUDb!iBx(M+^)MCr4|8qx+oD^V#~w)q?>nj&=M@{?~+<0DAsU6}RmC)lj+JZtYWF*BKG zaksA&_)!&z4R;|jr<2>>TLCxq?6+wy2%xBq%E<<8jpDs$JR!$N&4vp?CBs~v0)>u? zwVor3r!Qio=b`{KTA9_&6DA6qR<^o6{&N$5curudU_8;%8D#AT(CaDQ{JFs=jQR6l z7p5gf0*NbFc`wKw)AuS#3{&F26}kljCK6XP)hJaN=?Brm`BMI-qG*5BWZ)|D0lTi= z_{RVmO@^1__y@UbwdVt`+;;Zl1$Q^Yo(YnW8foq<_V!3b(GWmMoeLGrvpd3ftBO1| z9>nJRc$3Dt(s2D^y}1erDDy~dW;(~q41DJVfdI!w zH{mf1WFk{E2+)X_sA#bIon#ujzHcjx(Y1>lx1}e26j&ke?8so1F4*i2lb#QcxE5J@ zR(D>Vz5;Kjd@GOTuo&ob1MvmWrK5#7_{n-yC}S69yn3#mzU&FShnpWj1g;fBs7_?W zFR(J{0Ozh1+WNE+5k}d2Ch}8am56%d$(wc5fOtOy*ZNsiisNE7^T8dWyvSK+OjC-r z&^;8Yv@mr*us9w^v=6*Yrw<@LjfG|ZNC86&V|(mtTGXg}xioK=3H@&{t-8-87xHJFo4oc*a*ML2w+_%nR~WqUgE}*1SmofMET**|1G~ zszRwKN#PUZDe5IUCi%i#1cReqG)oEtvMg=OIguw&W#_Im1p5;@J6*c>$zCtz_9$Nv zhZ87~#|Ae4+k@x+2d|5p3vp-pN4IvtzhWKk<6ekvf<5a-` ze)_p%IQ@&6G9yKynA}`8AFeQ&YVEH)HB0Mw zvxKT zbADgi`87S&J8l*);)B%c{GvR{Dyt~MhCwp5hWvlLEC2TXTwng-_M3=_JN`EjHh~Z;bBj_G>7|N!Y9vkciwy0y(~o+zg2S?uZsa4k+}RP8g~)sV(keS~CuMBZI%xAI1ck z)|`fe+6$9TFG{$aFeWff3r*F9EttHEJ`^_Oig6JBn93Pl*gUM;ysgYD`Ng@hL$@sr zF`|O>)%qZ{-6AxJXpRxmc?%J|i!Mw{F&^sa2gJ!k@S18AA8wh;OozyTLBcOe>P=)e zy~~TBd+)u9>9KD^D|KIPCS|*F>>2D1_Dab)wA0OpF+b2? zQ>3}h^_BFN8ku{NUkz$&SHvwSz;rlZVP~kOXM*I(^zEjI`MN|lwaxi{0k55=%DUZj zy_k@8-rUAd>;uT$+D)nvDr}p zSMA^2r3rPfmtiY)b6*vo?3v@{mBTD%kkb8B6*A3p*F*YEg+H11i!dz9JhJAMps8ZF z1&F0}l+)9%sL4|#Dn$#28vJ{@>Hkb$6V8BtTO0ka>Z*TBTxRt9O=KQ>n9#3XP<;8p z&hLdbuhl3Ju44;)@5LOnw`ILj9u+nHMAZY@C(6(31SLTRo{MHU_#?COrtnYOy5W#v z4o4ZK>eXScDCzcZQAAp^;hT{rcFoG=l~_oKEObf%o;#&MroOWzBbf>k4AXPWSg?_Q zHM@|M{Vs1K4>LI{iqBG?tMCt)J*dWxxrbcC`;t$)%DYj)nqE+*cK>{LMh)7_pMXu9 z-;k;hOw3dV0}BAZTdvOOHYw*kYRAVr9){pWOFH zV6vWkZex=*2AMF$w)EY}dLMQab=*berP~If*jdPvb9u~|q4c2} z1+P8xIl>n0<(fQl#OZ; zboKPU2Sa@$wkh*Vu^~dekc@)*S-T{|Lc(Vz} zwiveUlP)xpCSiX(l2x#%rR!dtqSvkK8+Tv-IR42@Yh${1Anb#aUgRFz&1!p>iIrp3 z0)a5oGf9K5*GCfpO^NXJbna$GWqP(>36+_47Vf$cKc`GH8zHWwx{vp%TLR=4-^#Og zENEWqnh#=ZoLuraQ<%4!aaRm^zow%gDA)yBkO=$QG~Ytu^FL2Sic?eO7q*Gr2ajy5lzya%v>VV5ivNb92jU{h1U) z3wMHJ3UURQV`X*oIyz?4FT@>nd*-l#A=9bYVXfy=^n2t%)TsM~xBTH!j&7Yf*(cuw zS7aIgEfyn?%K;BRUc@A6(H>o`6{^}8f7f~f5fiCB4X_Wrsw&xX%TBmT6IV$|XiNnD zCNfQ+#8gn|p9V}S=(R$uD<)3U{GFcO)NcMrU2hnacL-m9Ii4+h(G;Z9J}$a5F-S6zu(l zdv6^USF){*H)$jxXprC%EWv_H@WDNJa7g2gyAvQdG!h`egS*qX26wl{JxJqDLVi6n zXYQFZXXc*sJu=l-61`JwQEU-3iSWl#DMm-uEXU%?$GUM_i6SW zc|ciKVtQ4URYXLV1@)K&5xk6+%$D>^#nck6mTYm^s|X1zSPQ$h6Xn`?uM$1P+_T-5 zNGQK8i5A58h5BA`Hc`*!RX8MX*bcf3pG6b1=VCvAc<@JEllk}A^KA{%)Bzu<`~WPM zpT08zUNjm?-YOrNOuxVMI^O>FOt|VR33m3U8sVsV9qW%T6la>etCp;2)&)nlS7obd zs9b}0dKr?mgzzjF4WhZ~z7{Y>xhPZ%XXRCnA2=U5zF1Sw7eN~H83S@z z+1YY3hn@kX7UBDt7MOWa^|lXvSKUA%^D7&>NFVMM9pTfBw*q&jc{5Ogw_ZUE7(9fW zPN8ctN^2ieKddjB*T6~IvcemOZFow@8`JX@m-pV{Y7kD=X1!R-4utc7Xyl6Q76M#&& z$#sPO03>)1;T$E_9X`sI^Mspc+c|5LW+KSEh~8$}2(Qtw2fu@N+5*VCb^XsyrU`C4 zv-C;FC-p837;Sr|-(XLe;BV!axB)0BFO>8dZs^6xx%0Kffa1@>FKCFy7sti`x=I;e zeV5F3J-5n0CCgUtqr*e|{gJBxl=R;h&vs8?!WnjSiC>Gee*m^Ip7#9!$X+0#uw9Ny zM}a||i^96_+qXv=h_#^wv}}2R?LY;D_`Em44XSeX5VQ6> zl`IC&;|A-RdY|$6k7zb3@ZEKQ9j<;Fah4yutAXoj!L^(UnpSA*s-goqrLp3K{PU;- z;B!D*M^w(ibWM8OcVJLnNtF(CcpyT0dblTUiVB!Hrs|l(!)^5GFFZd0n;Xn$ z&M$J8)MaDDX;6KnhK^UhQL#whuu2kWur?Q$fD9c9TNMUMH|U0YM1>C;mDlSUmylG* zN2Sw53Ye@f!2k@OTCzlq3J}tc{irc=%;DsmM=9v_Iky2^-o4496r`P(SeW9N{>M2}~_ zQ)c;w*8%Zd$C9s|>k6T)@y%~6K=mE?%yIU+xu7>an7JwTi-{)bRx3*9v$|rRxm`3X z^AOo#b_c3Ox7AJ|d2jV*e9}KqraE>y3KDh{-O`m8%(ygWNbU`kay!(saX=&U@p^dx zr0-aX1Vf^QLB!fZ1mlG1lecg4CY>V|1c?LHt+Hh{I z&f^Set-Ih%=pr{=?`vtt7Nvpum1*s+s9{haA4rE6Zw{&N0LiCNUIkO1O9LO_TjTMG z79z7qHd_-J@<2e0?6g6u>??!(a#^Eq(Y&;6jhJmQ-n3e`1R3mWXR?CQc`yPDjg8az50wj3;gAULH>)QVnltCo zKO@azH7==7&|28Mz{pNaUXa8;4%_ykAQ2C9rpq69pdr0E!+gwA%tAQ0WK$dYx@$kk zciuehfb31-e89T010oP>S@<#O-EQx2It;RX8j%S8>B{h>OC*G$*LJUIk2b#6)1a3E z=!GRlp4-4r@$v04lm!8u*&M4X44BVoJj4`-kH+n~lL~J)bq6w7ahYX!~wO zvEDE_;VvjDN09l1{|jofQYRBQfeV;a`9e zo!wWWe7if2=ULwlLH&r?WrA^LP>NL@CRV^r(0}1q78QTozM(KrHbUL0v)l=DfFUt3 z@EMHhgBkZNb`LSI6Vu`CSjl|X{&@*UI~9o`h};90j_`pdc{_`Spw=7LfI%^38!WIT z%WHL$T9&G(!1E;|h?V6hsJb?D(Y zZbO||dW1`;{`=wZ7m&ipZVC>U7S7R~qx|IVmg|v1b@=wD(c3IGECK!pwYp3CpmKFr zm|V|m9vM1>(_N%sld%O1Y3puYc(oJ}qgBJSol~A`P7+?4z>*j6h@R`X`rY~&%wT5+ zi`@TR?2$G?dA3m;$fRBK*{uz9>-hAppx;Q(i{D3<|BRUHdpWDmz266RIO#CHxMwDF2WliI@n! zog#1ZwWbNWzALxzT~7nknahQJqVr)`E9A2K|nHq#@C(yR$J1NkHu2M}(5HjW&_1;p)ybL3 zR`kAS>1f}dZ&^4g!_bom7^!VgP{=TXDpWg2jZi94r2vr>V=EX}Yx0W7TooK77TLjs z7(9G1l9})CUf|*j_Hu*8Y_saZP*BK)*kTU-Q0c23PWZ^CC#dSugX3IB=665GR==AU z-#`mRGjf9n?Y+{~XpJqgVL@SL0bNJoewkTYa6KdX5}=Oe24D1|6vf7(0`;jeRmIn+H*NrXEk*qxKQX;%hDP2GWkH=*+pZMUlI5|0&HwC#M%j!-L^qu|LLR}~Sp~~38^XIgQawLpa<^xVN zSz#rtclP`RaIFZRnMEe}g@dA$-nN5S`=Fi-10J=VP-yj-o zF$e>(T0CL<7+GUp;~w-ikypgXOTOV0ErWH6P*vN;aiS_w-UhGm6C57eKGDS%UJTC2PlP>!htgm3zDbwk0yp}X-=Emo!br>6E4^AK3 zfJmH)sWNKQP&R>J@O(5hK4J7PJD!=fRYB7nx7-N|+3zcCQKff)AI*)o1D}OMnB9d_ z_yz3j{6b(Q%vz14TglWDnty#Ze`_U59swWX~eyodyd0bsH>T@ZM?7m@7 z-Revfx&P_W?-lGn7}yB2OQ85O#Zk=a)!n32(WTSS>%sNq8Mv+2Ny}o3vgbk~%M7VE zGy%%}x;9U)C3DdXxb^sM6DK_D)n^71mb>iWKy%kO3wid-EKM z9426*=KJW|Gz2WrJU@IQRC7VEnbuL_RHX~!ARSyGfY6|gJKkyCHr-^n-c?q zW)BE%m;E=p&7-l{hx|C?&CcH)E{RnNx5#wj5qahz4a_(Yh!`0W@^Vgnj z?2O<&;q&HTud|(4e$(s6?5NZUjn0`>*WeB&+K8-NS9@eCPj*EF({p2UGNR5$zULcA zC)XhbZk5*!D@;l;86&|xV&D5i^dW;OgUe;R=TmG?t6sql3v_9;19=|u~+^|L8a$Xw&~8n2`UibsfJ=)iBpJ9S_n)vGNIN}uE8DhT=tVXd+t*l zvUgTaZ~BQvcf|%}Ls2&!&v`CQ$}`7#@mJk!7@X_D#9)-7HJ}wkR`LQD*XzT~peQp$ zF2mK13g`_;iacF=VDR)Yl`KZrC-8T%M_liM^2ogMID zSCp01XJ2VAeqb-Jzt8I=L_OMmrk{)FpqyTF|Dk!|K{An~ZuVL{#*C4N|)^+;Tu6;v?fcQqZ8?AG*z`*Lm4lNeZX&Jzdbh&x5z9Xqww9J)by+;@uTyj^FvQO+dzd0b-JMjz2asR@= z`CIy?-)5?RSO9;wbN(w&=s))_p8Rg`vQdN%;avQS&P(+h)9CM4*FRK_Y=4h{gBc}% z#m4_2{QaT(4rB4sr*+5CX&u8gGx%al@CRT$g8OhQ zBb(+027AU;E{opvF>GG4htu%+O?4wpcYW66vlc5DEVN5g*QOD()1$bUQ?O3(cy8gH z!+q7_j0;!3Qk{Y~{rw=`+rOc)>ft=!jeh`K+LH9-{>2X>q~Aqwe3nEcEC(cYUYVq2 zu3&7IFg1!-%il8v@4ATGhX+gZ4y}@Y)8Zi$E^~0-;65~S3_Y=Zn3ezRn#`+>W*^=}Jz=m=?N2Tp@{GL$~$zdcBDj6XU9^}@n7>}P$OrnAf ziyGmzrmZ$zNH#s{6@>UT%aDa1fP1<4bISmMFm%zIjBFkagIlp(Z1r;o+qe}_K~2B1 zsNqv|{(aY)%3zJ$IAR1eZjNv7scSEHCb`4hHv5y-?W4Ud2015 z-+#+Zo9m6Rlgw+G@GMv$G^g+G6|U!}P&fRs0A_=P*C&8Y0pw7x>X+Us;H*II)1 z$;*7RZPiB#N5Sh6a-92~LuZvbbISH{MN-k3bIT<*rPk;Od|2$rI;ZMku&THH8XVD$sgqf=X2BGMp+2)d&@!rAd75F4x2P9=9V)gKXcYHTf&ZS!{(34mSZ zO_ys=MVo<_R4%hk+$(49BpYGuwx4ryiF)^_!6H=U_BpT=EsQOG#i7#YM&}EcB}Khl zp5kzAg6`z@KWcND1ke76l2q3(*F_7ldN z&rGM4Bc7HIBH)|Gmo7>rjo-DHYKhx)+3>;=*|`)!>9n-c4W7;tDp?nHx(vTfX%u%& zwpfkq=ZcI9gpS$+eCfbY*pd8re`n&AG;Qlh+M=5`aa^UtSD{yQPo~CPc0;TaCR#sd zcr43k@4Z6TA-Z-}uTZP5wpIzoey3#?EX*nL{QJb|Tm3SLZ>2h(cRI@B9(njD8NR#n z9=_^iStPBQq*U^bzMR{^d3+5J3i^2oy+J5mZDcp z*w(JM^wQB+=>~puxZ^yxk-iDVP5KWHOUh!0Piuk^ar)3THt0ToZ#DN}XmpizG$F}C z-oea}%VP~pfsi9fU4O(Uo_L4#B%wY8(9~7+c4CuRSG39>=w(RCs6l`f~ z)0wHk5qVJJ5%C5pp9zs=J;%y2&USHr6E9;(>QGN1j!l{}m|Z7QLL;U)87h%TcHD3Y8aK$zjg%Cl!)MKRN^2NhNWv2B9ka_GZA)&> z%WP?Exniue+pnBNw*Z+$vp2oO@>+Ba;6$MC4v=Xt-2Drtg*T_qWoxB1pJ!-v6F+%E zn)d0Ix{H?|eyAR${n(b5(PMRPfX$;yanu0|6O+03+4@>t($eAUexv$E?SjBvKZ#)j zrZj4&`s~RkM3)<6%fg9X*l*ifboN;}2QHP#Sc1`91Wnd6hpA)^4V%)%Gc1K4&2gNU zC(2o5sNe*fb3N@LZ!$h2K6>-jg)Yl=0AXr#dJEGNsH#c*1abg%}y5x$4SN|cB=yypg8+;eD%2Z+skS)pj`WkHMvu)oI(Pl-} z+~L`@x*pvgX7N6odMD7tD_f8Uu|~U&d4xQ{5^xxdYfKI}-b-eR310+%=H!|TX{k(< zt-HF6)8oPNl5&)$vqJqTUq$CdR(_(@wS-f37-_KHV<8ZvJxh~NN7<8yxT47p3#^?LMCMFnd z0&()k!f@}>zrmz^qJE+2dCL9x>)x|q&#RuV6pseQK_D52US_ypk3(wVYsRAK+JYBZ zX;rP7!kIOj9K5!mv}N5$h_q1ZT~&F|AcvlR98RYM>(iv9`|`CPt4>;a7L-eA^EpnC>}a7iR(aRN>DTfGc~Q&*L*X zlJq`V>{)pc4c%qZzZF>EV3FA0H)&hnd%V={KB;Tk+uf0UU8GdhJUX(HeJI&7Qd|&> zz1Gls&8nn-^Kf&TwX0aJ_cULU&C_y}I5P6C{12o{VX%HsT{MDSKZ^ zUPI=~A+!?LT52%kKuLNhftaD4l}4s>$(gi8UVYanpm%`d3=Oz`qp_e_FSs|dyukv$ zC<3P~?+rBMjjry1u9*fVqQN5ZzGsh|#;azn9&@(|dBW}TXlmqN_fW2Dk|@P;73=8| z%qP4gtDWek-Mk@DukqCEkc1SP)=5Yo@gyZ(Z!od;Es%=88JmH$qXOjCHoll|;NHr9 zSM=Vm&%I5|n9>lvRb>__}qgLNu7*{*1pf=CEbT;DUg+P+m~qgZq_0Xd-A;Jj5J z5Yp+$&{M}SR~iQ{lX4wWcKfT^yh`^gwb~_`gu&z7l$C_HHxv8XoSy|pwQ(fD+&?M+>z8DfDJkuFEy)HXz^NnZ&USw zH6&7>JaJwel53ll-)oMFT1^Qkzbtgko9^TvQFCIFsm+^aor2{Su@vbThb>&+%R*%V z-@)oh2Tx@}>l6OLM@Zt0S9(DOtcsO4!bc#4B|0s0S#JwW#xVjWs!0kxGQ+iEFw|wB zyi_Du-(R(SZdg5XJcIDel-^y^&>jd;5SN*pXX$CkrUcVqiF7-8=9T{dSd@Er`AmBA z;_HA4xTEY|m5PbkS#DJYgm|0{Yn?gXHA^kYeo5e!#q-U0kIK61LO4uBSyzV3 zRp;T#{wJNC8;J6WcKa9rV1xn|r+kzV?<3&5`(4reX4J3v2Y}F$F_G<37C|6@;ZtgR z<%c2`t*aSF-=J-ShzUBc87*-CW`H2W`BGa#Pp|``=)t8lZ0Q{YL2QB}6fG(AzIh%Wi5_4hJ6n) zhILe6WeE}RSYu80{g<`c*fqACFUIn?zkD%EzyI>#(fdjYJah&6CtdCeFW5fGiLJ86o_OiCYH)j!PLteY}nZ_xPPqeEIsi&5X! zX|vb}wtZowEp)$H^UuC6Gt?LnDvI{*U1E6TgB1+XlT|D7 zWJEsz9t%s~G(?7&n0H(z%k>TSwPxgB-amo2b@0DZft#O(WwbmMB2Gy!pRBEt^x0d% z>pdif8E&D8bbzp0>xSkP^Qt6jK(ot#%WFp=;9-4e}WqX$biS-HKH{v{&q%)!%s z$Dk!V5t3W2y^N_D%sY+gnCH%2_)$V}dQ33(%PO{wZyNL;-pn`^wduk)TH^5qMw7KD z4@&c`Y_u|P57uyx+gegQ2i0D2C-8rN>D{_0-WqRL!Y<{v6k(yIPQt15y3I!@IU;PK zD~_G!+erUC(_2`Is(YGt_!9SNbUZ02i(YDmjXMg{ViQ1wq!DB=1*^BoB~oC#%$ur| z!b5KVI~d{s&v*9|JIC5N1a7zePXhsQikf&^}?dxqqA>NlI*9 z9vQhk*^2!)2$5Du}#)RlWkdXD+WmVFf9-0P1#LR`*Z;%(U6hh^pJh`F2&Sr_v zU@7WNYF2L|tZY=_iiL0gmAI&WAtXq1dS5pnkx4=__V=@%n2l3be#2`;XXBNW|IgeOyjWUgJ8bkd%J^!T3?m|@vpj1e#M zo9IVD^mvcn{&JEo^!O7(^G>AscSY~`gE;chb7FprfsU10DG@oWJ7Yw^WT@GSpPh%E zgy~xb=OH{9Yf7Ao$oz;O=@PKe;>U*W8if<}9`>}4AEWPB+2}RwTUmK|DBNw)5J5H( zQwUGqbg$LVO`C%K(wd)p5YaXpM2{g_nu6%S{{vxo%DMDhi`ZQFKRM+8soVbV#@`=D z-@jv${=MnoKM((rBwkA=ZNRu?=yIn@_$vVM@6g`AjR4~O0weww$(Zxo!})ih%RlU_ z{#B#?10BkWIl;zk?(v~n6~vMI_Z8n5W1~?hUBCoaqr#E|ELrRW=dOiN9C5$i=8$1=672#T~>%MpMoV3>KFn0Z<@${%eKd zALougoj?EJvhx4O!#}PoKWjCQ)yI;{6cx1!GvgBC*GF9wVH2|-9mIi3XM?c;V%1OsgVd*1`G9)~$2o70PL%UJ~_)w-6YM{RN& zg{Y|x9u|>=(=OH~LL=vH8%^W`6T<#k)3>fo%ELZ6Ew5$GLj-jyIo%y%Axc8MpEfTjm>nBNqUByjWU(XDEU1Brg2oNU`b)acNhOFSU+6RogHo&;`^_;cjhw)#tL0$d`T*Ch^Ykt&Z{OP%>m}ggI2tl#uGyOH@Rk|wWHeEdqxo@s>xc@f->X%`DjGc zDbe$DPa@okvFavC0=BqY`+e)QG`k)-vni1sU`+^0H=8gmNf(sdn!VRFJXEi}eG8BB zK1Drrdqn2B9Rvw*8=?iCYYToHk7r<_l9vrV7@+T!k*e&ehRF{5tbt5O~M`uR}b z%E`;dWEsnsm4?hUqwl+J-YY&Ze|QozkGc`*)V}$-ZFg{ZA$y~QQuYljr7RC}q7TT$ zQzXFf+6c?T8&HhutWB4o@Cg%d9^W5k(S2LoqH0(`51qhaWF&sh^8SN=Q?m(zLAD4} z+qhn*jQ163Df8u>Q;w;Igx%mXW(%iNFa8JEbZ4$Pb@!hpTBw#;T(W{~#9*dKVL)&XEd)b#1zt&xkJL;4#7$SCF|bovN!-qF zN!`9v{fQ>SmY?-$p3dbi%Om4G?pS?m*a(A`#I06!@NB>8c9R2U zSZ;R}?D5{#w3#%T+SZ|+4(8H)s&(IJ02J6Z=K@Kc!kCwM4lUa@DA13Z`SWCa}?tY~^`BWe{Z??wMEI>+DBX9Sy zt}@{`XOimUIW{&ukR=gZD11NL!tzLnX7?_LZs1Ipt}5JOlXCb{cQqfBR3paJvlwHq zLqf(eIme+IGWIaED(jDihrie$0YfclQvuo7v7IcDNcYtheso*P2g<9tVicwqG-nxc zW`?RF{5Q-ly(&5^(6`C08;Z*ONDH*?mK_I73}lgsJpFjE>t$f_c4`2&F$xO*$EoNM zU)dxdnQuFrh*V75zg$lLYh9;_c_exK=clz&(8I{Y)U{zgnq#UU#)oGw3n^2-ec-r_ zy&3KIKvCevPO2D~ZG3vXi76sEhlaKiAk3)#k5wTUe}273(l0lZ@82`_Kd!RW=~cKV zGs?(0!1KR;;D_>us%4T_Fx?}OWIQ;v}TL%cB`pB zf0UML>H4gHEP;V{nF_0PyYrH&s)dOyX&6sX$o`{HvHBnlb=axgWfjkZ9sXXV&#s7g zW^BHnZ^bus(H3x;Zc${pk)AFkk6emv5^3*bFv0PVCKlPPz}y0SKB-X~(6m=V>Fsn*wiU`5tUUuDYKh7e0y5|BtHC$8rAR;4y)&eO4wj0W|)+`)0l; zp=1fNXmSQeU+1lr;y!682L+2KJ3c&lBw6nM^UwL^eiosu*MC1~b_&#y4+Xf!#lQTu zN!t5N5@O!`5Sh<;44)3w3bGif9h{BQP3dW+9AF+)uQY)Hn;X)IcB|=-us!Hj-$QIH z#}*jvH2Z9+YDlyhH>|Vm2YntOqEa2V0~vavXYL>U{DBTN#@wD4QY$pBA8*Z3*aP92 zS^5!_E^mYQ7fHlJ;lSj_BV4|9$4WncXcQT=U&g%l=rJ$-h8H=id(Ff?^ju#{VMO>g zRCl_^Y_NST(tP50Yc#ngvH#~E_3INLV>Egh7T_MSZRu@;P3$GbAsxCc%xID|>uu)w zw+C9`JkDXU?j31e_2A8yRkBq9dx4YK6EQBOq_Hp}Gq8AS`&vCP^@$)qWq=C!o>Rzf zwFr!${&6h%SV~uh>_*ihA&LKculYa3<^R*+pN1ym5;h}vh3P&U5hc&~2NX3O8i>#FjeksDMs^^RU{ULDC#=8!!3!s9`j^hJ(IC6>)7v!751<2@d(E z43*Ky)o@@X^4ChJ+k!p|aZzGp>SP!xu9pcG(g+{$TwDrkqi9>_Qo4wuxHouMuaF^ZI>dTIL>*?eNB zALW!uOV8p>kI;1dFnICJgsBCVJsg<9d8p2_qhM7`<#jC;Um zXAvdPikh>BSD5XjA-fAQ7C#*iRdTW{GEN!>)Jq*70OwPw(g=l5+_P%bz>+M~2Ur;R6=!xXfxLD;)s*NMY6*&3V)_u+PiPvkB6 z4Jdoh4Yia864hGWQ(m7Wb{POug*s~rWBxN|C=HP+k&x|N74`eBdvtXYx{eF@j8@OW zUTMwNl=IqlLr&nG0tngbEcJ>>=+*3OwStP;jFzg(eRR^O38zutwoK1=Dgn;8hrKTy zbh~X5XAMOt8W$whcmV5)K#BvbgP1a1d8K}u*cgfVXU0(w3_xXb3yxO^_2435>5Jj7 ztXkp8eZ|8Pv2wQbXuT{>q=X&Rb`O8@2(clHwiKL-)8A`T9M@ExnkDf#%=&O~p@8A! zmdGFV&J_tV88|4k@57VgQ15akCO_*p!l$_I?!#=Vc1t)rORjax6H&_Fv26=<`v{D2 zirf)AP5GP`k;lZS?~BjvC@*chKW&%qY};NOudx1R2df=TsI4V%EvRyDe$!=S(2o1V z>j{m1s$>hpP6rkOY zhp6L_7^EA05r28Zl>HgHl;&I(r(M>QFxINq{e`ntsL2l_aN(NerY3$k=>tzXE|>Kh z`&yur_Fmv3KbBmqxD2()Pl-r)wCj}t?9lQNc)J@}oYTfIFaC_D7$%41c5C5OluLIh zCwoWk=bn?k^e;}4cQY?8pfw`7l=Ak4C0W&);xD{5mv_3h?;Cjqu4)6DGT7<{1{ig| zb~0IhE&$0=SM^XJb&%h$ORrbXKBl2GjhnZ0=INswH2mV9V{K2}jwOnhzHIqKm7GpR z=nLb1YE{da#*{!&Mbo%7^Erv=;Znd+7xPoaj7_|@>!>D#m*1B<9Y%|Sg4Nu7+c{$p z2;@zI`rxz{lGvKzudj%yK$?6{HXc&vq(_DrOjPg04yg!ULiHkYywVPXBj$^tY2Qhhx}4`Itp{=h^uVeeVJ{A!)uKuMw6 za4MLC6p#w+02p~jL@8>r){=ThpaQcQzfdAap?Q@mm7ZeFLiVr^P?_377&8-zWe|ES zPu7;jlm_B{2wg5dggzyP2UNeT@70%99lNX<#e0VpP1lJ*g+cMn)TKeYmABd{r5?8- zAKB|mk2cpsv5_$AAmXwVr}&VE0OPcKLB`+a76NW$M4aLmo3ZW z`_^G8G$3C71yRyeN9E?CSb7P&+b%zsmBkztk$v3nvwn^VVq}ze%B`AdlLXwY#bVRl z?ga%o*&Cu)>YaeppLCh@X8VaZzjgrah30hw$%tuShuZuU7mU~`d)xw9(8avZrJr*m z&X;P_i&ZSI_3o&q=QjP*GdxO&Mkg$Xj2ODyAjaXCxTiVF2oa@LdX2^J*E$T8Nk3%g4 zFR$n&EU`^tf$>5S{KD$_p<{Tlsd%*jO)`C#0wgSD&<(ex6lrdhZ6lSypscKM*9&|h zl~V?-%9aaH=2Q-j2r}m25j~PPA@`@;>n&5BKhMI!;t&vc($g@Gu&jpYX#D^LsUkv^ zp=L#`DZbr~r3~)g>CtSc9O1!l=9wPTfebofvlsyl-vt^WNpV(A(?oYAmFiPnW0d&x z<|cs<@fhMmeN2L|JiXSc#w`YQpDbxT1@1GR0Lu}D(!CvobE_9}J#Z!RC976)RD3|< zKC^RPL>Pgjq-@w|h66Sh5@c~Z1`YdTgA0I-O#tVe*H2C1PJ;>hLp&Ec`%wx`uhFHe zo-c*g@khQKl<7xfh}Ghnm7OmmSEO^EM!I+zRnf`Hr--pR+qHd(wX}Ms3p)eyAIZPi zd&XR>G(A2oTUfFlj4l_}%`J;N4Xj+LK0eZE*0{h=j8Mm!b&(TSm9(?LqobID~y z*P-Mk?cC5`jG;PH%OB*m?!vU=Fa+f5`bpRTLPptycJifrP*=s!x@AJqP)+wmRx=D+5vH=9#5M8hvg)ac<5gU7jO&DX*GEXh&y2jV zh2I?NQ+xOietdf&n}VnMoxvx3O-uosb1`4rl3e6KCxNp1+fSE-i9JZ0s`d)znOL7VdicD!N3(2twJ)$*$xLgn1%!z0!*4#9O%58NVi!{-OJj*q# zFnLXCCzM@EGuw1Z&au6@3D)Tcsc*fgaT})5fZ)MYl0{0WH9RqIcS4bcTT*>^zKGR% zT&CA}torM5fc8A7cS}R%xk$vvcxN8 zlQ2vhJgs%|&^ctL%b~LK2f(}WOP6Joka!;aRCVF~$djSHu)K6FdFF|xU7;g44kf@H z;4u?z08gB<@Y;G@G5sY<=H9Jal+3_$m^<&>dcaSDZ)2WP?k!5M)^*H1E$W(NRY9I5 zvMAmP@iMLcY0wa_rF4V*2n~(07|g%AQP-f~3U@S3mXne-*%Np4V4+#{ z_tR_@)KAYdZYPZ`6k0utY5u4{azmzbMD|XMf+>>;97MxZu$31l3o;4Ba#m(`732`W zb5PKXD^A`~)^d(LCGl(s%Zply3~LSLJERTp!>L9rgzkk1N2`1z2+=2^9!NW&9o;No z2Ouz&V376fulZPiZzlUKbo5UxLE~F>OM5I@0?%uO`SzI@97^IWE_oORc5E@RSXp5n z574-qpH*^k$=W>Cc9Gyl4I2sy|^ ztk1SSimivZ%G1ESyuyt2AE_~0Cj`Y+mK(#t!D3pDYAD$>NtYpdR&&JKjvhs^`B!29 z_O?Kvbhni=%=JPb2V9sYO#{E+8ga*w#xwDq03!P)E(Q+IigKKukZnih+X9du$c%VBxXhY#v?iE7SNF2TU6eGXed&dINd$@-rX|DD z?X|O@*lK*Ht%y}8uTizc!i-bv$P&PE=p_Bbm;TE2@`)_}!FEyOL~@Iwq@+>@1}{-| zV(X`drYvh@{-!`#pzG?0G|^*E_0YL_aw?Dn1i;_!#CO7Y_#)CqrTNW1r7~VhvoeNp zm#3F}Qbt8=zh&F-UF0RC95by3Ps}1Q>N-y3IEifL9THjv;fIf)l-B9-Ue+#E`f$su zVu7_Xr`?ju*$G*(X_}y5o>0n$t~fGLOnVwylUt$i9&&z{oF=tdndRmI6@^J)`l5m^ z4k*`F<{XBApO1Pu6WS25KBJ{D$2EfCtSplJSHyet1W4!bsI|50ZPIQBfp;tp?I)Xr zx=M4owNr0Q@MAkVpp6LEp1RA1(02|CLxMXQe&MijM7#^O#yq$v!v{ht4$d%m<{~6@ z-HQemWf2IaL4Jy1LcVx1jEDq=&Jt%5uI|z{a6s#X4$f(?z_pi0OTmdCq7FD*w+ivf z@Rm%>CL(k|8?eHYSIH3vgd^x0&lzZbBSalHT|=>eFTjx)CS1c|q=_+RJ} z5De57Wu-IfuYOv~s0YQ)i5ocdIq6fYf)FmN5RgI?GIy5~F}wrO%|=knY~KXJeIEhS zReN`pC#Vv3mhB9Q_7nW2o_QUHLWdrPirQZa3xo4`d-lvib;d>|N2r(AEh=+qt>NWyrO}s~`B(V$oQe zC2U~a_MpA2<3Dc_+V_@plT{>`0Y|)4cM`XAaNz3kpmXNiDMKmor=t8vZ*I`ai;%*c4;iiRoXmFq5(|on$B~ z`qiF2^fTO`jQ=)yanF9EJgg8({oOCpSM5xB_PTdVgqwpZ2?+|JD0f7{d&%j{iw8+K;y-`6SonzMN#I-{zS49kU%!Y&7Seh7 znz5cedg-1~LQEPZz5GP!v5?_`)o_TCQw{%ZB~J!8%PEy)_^v_=+Slm8c+Sk$r9 z*+({0wLfoP3;qdKj~@XeRd&Fr)^+~REz`(C=VQL`3;(5M1Ui2rDJ%i|hJLyD)xqFD z*`NR2Y2lau{xk^ccktCu_U|HwqWwBJgb>8m6j^69E5{0mL$reCzN>F|h$FGRm%qpm zhd7@9Tl9Z>;Ez3E@cIAH4E(35Z4pLTm0c3JJ#@jC3ybTxPmjJBr%?%uy~c<_Tcp_o z!)s=-!n15x%{>SFyzI&N(3r0O2tQJfILynzKM=-$9Z$p*jNpJKRb=LluTr1xWgaQY z%JWe(=;1>>h|1^Z@9%Git$pa%6LV|Q&k&clDX#10Eb!cfVunpfQ?s&wbsnL7j#)=Q z)=c#^pL4@w&z}TfelPS zY4vdrd*Jg_kfC_|!Ga=&37&;0T9sa1duN=Uc@AyTO?yj*X8)nPQ)^~ zR{AOu_o8bFUXw_tEB>TgQK^&2)l+R*=OHC%)e?c;iK;jmODkP7J?KL98ST&$A$GrjP?|GEE3 zjtc+%pT=+diN9>=pKdh%QB(O#73RNQ5dGx#;FL1?t)YIp_?g(V=O-`AMyYS|=q~0r zMYx_k`)6@EWz4t9C$}dz*u%)7*01_zY2aW_3zaF zFK+(-N-X-DD>3rA0dkTTd(Hjs@tykDU!L(})m_G_9cg1Hc%H=?%ZCzWWUtLgRPr3G zRV9wKA?)x4O;Yu_piBR;sxnX6PKp^>yawap4n(>IC*|Ju_e9y=h4*%9Rl$wuce$Vy zOfGn|1pqIJVr0GNGMWqnqxg~SY{S8=Z63p-yVICUctb(q+t`tqS`L0bBiH6t-^YT( zG-fU|#w9tXPMzsz+^?}?M?418QbDxW0Dea53Fn&HJra2v8yBGETegtNmdpvR4F`J` z7TVafc3|qp!xxSEIkri4qu`<^gL=2_)Wu$C_v7sjk3BF}KSUy>Z$%U|=@)|Osz}yY z)QXJ)7rLj^!m56r$P_-`g|SY zoKjIJ&oq;!syVD0`>nRp66m2L76>Ns;Mm~ccWBHuJ&#VrSd)s2<8*T|y_eovp~vE6 z7+J8*b1rW*nulB=j#{(3z<`?jv0Hpm?Eeb>w@dyX42DFKi^79TokX?8ePleLRH>St zE7JxVLH{EZJ4`_+gDtMpW+k3b%-|6P)@l5*Ao&)P_PyV3#(B11;fH(mzyGT-yyq=_ z>*1YSX5d3tZTBxqt6vDyd>3q0bv)Z;N%?tv=bfw`z0esw`vFk+0mvo&u8?tByw6im zyew9#-Dy}7GsWNd*82)kIiC- zTH#-AT(c`-R%Kc*B-G;%makkh0DvRfRitH^$2&F}#dfa-JYEe|spvQbzwY_Q75xLi z77Y{V^1)ri;&8v<{{OnW(x|4AZ5`W2Z4e?&1A+nyVy-f&Ad^5;q(KpaOaURrHfY_cfIxg)T*=2 zUVGP%T24B-LF<-c6>{?s26SRj+v{Vl-{%M=5tzYw zIEH%`9~5E82c6p?3LZEfeNHo3!}knbZqLKKqFnT&GcYB@w4;fmY^}a*4ORrwW<}#6 zjd3Q@T09nw588eP-GDG3{l$&@)zb?8sQEZ;pQrDwI@8CxC`twmypNFeEe;;G&Q51v z9BkVnz;N$!_@Gp1zA(Vp5P;d@T$9^L>Cy9Vd6{)5<7=DMz#Hb>;)4=_h*rNxWc*z~ zr&z7p#9pg^wI88lI5k`slMhlWZu{@f6WN!(#^r;)27XZeb!eyl)h^@d(3G}$HZFw( z@XCq+!bg?A#Rx2LVxg3e?+$Ct4oZDxR71gvrDx5dPKUustR2ET2Z#7KEm|_qJH(|4bbuLDxvC2+T#gY;<0;dq5 z4MuseCY!F!P5|t<&zQIsHy-(84i(mC+~_#2HLEi#uU3tfmp`V$2jw8wtMa(O3?Sbb zkl=#`Dt7E|@H+S)dL^bsaR&iI$2xpUal2yl+2bEyPcKd#_zA%v)&pCT)*AV74aOXDDZqhJL$dy(HF(S%{8*a2`O&yV9C?~j0?euP!-BcT^ zrVHg%XceoyWd3qCRw%mjDR+6Lf2mIeUU(w*=?U*cddU&FD`1&2gZ)QGROc?<%gK%F z9@XuY|8)=3%jC_r%H)>+i&*=~GJ=-pueBBT{GI#1+dUHDz>wh`cV)So zWtgqCG^>X#%1uPvRLj;|uM+8Q^!4z0`9&@UMOU`}trHWP)DxXq+wF#FjX^ zm`uN+qi*#yZjb<-5qSMrHbC2tZCJA|vm=ajF!jq)s@0n}v(PmAaRn&yPJRgbX=P@P z8Z+4I`*VrrRFq@yPLnGe%fl}=wYOA!vxl_dw0dqJaFYN2|F3 z<6e$ZkKe-lXBtusbdc&5*|1<1I=+`<+O2J}6R4wevgm#-D(-^r*k~ z!W=HISb%CRV998`vGkYZs;!x+y{- zJ405_2CRmuLh1dOvF^#(8!yv-Ks`MC|45#_kf~t{T=d?2qG}8#%I_OIT2=I1#)9=#0OX^+61950-oE3c+qkHxpLuBvbB3o=2au8B>8b!(oVnaw zy2q{#k4Pz9ZIRr$4rN4<1~}THqQ2v~^2a81W%(dw2VOs9OO;xWG@M$9!i~=J7A7P+ z*}fal1KeA_;`FE}44G#IEcBIxT@OTor+YfC8w+ItE3DhHYzT)GkU!HV+Ak^^m_7&` z6MJw4fmdQlaw{G@vcy9I?+LEEJC+47(YQqw(A2t@0y40Z@0tK{>0tlMUnYv+wVdv% zA}`^+FNDu(W)H2A$aCi$=^royuY9`Sp4skfet3WnvM+NqcwD7hly;M0y3*lE?y297 z8GsMN9ETh+NTUa3$#c%dnoZBbFa_gcN#jbm;4J(2phJLv7@S0Hd%q23uwg5MH$0Go zzZ7mNPpy52qYbf@NxZTY-f=aqF1EGMaJy|kAZdA(n|4}-!B+aOUrdb6A49-w2AYSh z1%Y?Jj8wycO3kyDo2UY?kRl&cXn(}Xa4RC;gC@P}eIk$|yr~NPh4KpgPOj2!rdIdl zt>-iHs^{85cpn0t>V_xWF(gfDNNth`Sqyx8Eui`EBoQY<{Dxg&YCk>15P<*Yki*MD082*N;_OO)-B#0MUbCLYH3qmfwU)O zr@~}yqr*u|)UMOZqEs-nm`a;|q;xm)oupxu?Xw9Xv@2!(&wZu?SX^B=)Q;lX=KHf=pXZyT_1^T=pgo}F*xRh zFFMG;Yn4j1$?%P~qF9 z4(Y&QoV~XJ(0J1b*I+5_RqdPc&9lC>a2}>7p8i}h}Ca$<%&Q((~RdZ z$nAo@vdtjKMzr-{p$!n0;L_SuGHDAo`V@HwK^K*Lt(PK0Yv-Wd_#h|fnBy>}JOkRP zImJSA3;N&_She-%xGBlvobCJa@{9&RgtHl=&$AnO{NAG;D z6exdBdGFnBE-i$JnuuHs&CrmTiOhbleW&Q`w`ML$Pi^*{k|ymFP`|vEV>|FTyX2OD z#h+yNfAs!V^B1VWl_((UkIcB%oF$2zF+RDbV)umj`TjT|POr+}UGK4Qtwf;}rv9nE z8Io=LQAXWx;yeZ@pZY8!Rr_$gpMlm^*sGr_ZQ4g!R~fMAGB&nSY^WX5@13n_=ybI^ zvwAUZ*8J+q0j(1^n;Ro@0~OrriAG1g3HvV{bW2o>Z#CYR5b)d4{x-o6t_x#nxegbA z=OMGM#|tAXLJWOOI-%%PeEzo{hd+Q%9LZJI%qTA@3ha)VrOuaR7fmg+PqnoEmGpr4Iw zgx{p9^b>6Qt}k{^&mF0Ze@LLy*Btu`_wRMDof~Ug7pkT9qEwX*bgM$&5pt}GMFKf= zk{k^?wR{bC!FxaZ7J*HCWzsq{%*0h0STO5QgPA;^B`~waW3vF7*%-`KbKpALoq)jy zdB5#ks+NsrY!EDP1q0xoA$ky35z;Jj4sZP|XJ9xZqLShVbLo=Q~S&{Jn;2v8{J z>39?dE|AvgMac}!$Vl}c^yx=Y|Ck-yTF(|*bb~(huz*eTw+7uMu1@{omHM@GJ}BPn z32QQplO-qD++ra2dHRYGts&&frCQ8SJj6}jD0F?i^)b=_bKA4r?dR<-K+I}^DZ<*W zNC1uwC};VgkDDomQLMS~VlVRy9^u|@LXaE6Dc}KIk@hYaey_6_LKz-mLMdz8+=3Xn zwc|a5qJh;!0Oz>ti{X-)I4{_Wc%e#ruGc<79)y&*E8y+N=y3@sT1<76EB8j`gWVpbevZaoAgm4^js{0SZ7)d0|6m X*@`JazA)f{jp_8H#g|<)m_PbY?x+6Z literal 0 HcmV?d00001 diff --git a/.github/pr-assets/customer-usage-detail/final-report.png b/.github/pr-assets/customer-usage-detail/final-report.png new file mode 100644 index 0000000000000000000000000000000000000000..7118992dfdc8300bf9fb80fe09c365ec76d88743 GIT binary patch literal 62798 zcmeFZ2UJtr)-W26Vs8i$rFjlb2u+ZHfavi+2pvNbY5*nFgn)Di#bZMW5)2TKP?S(Y z5;_D3MUNsa2nZzfBGS7cML^+k&bjyf_uTt`WBlKH_xca z>ASq!4>-PmuO0t>e!z9WR@VQ3`>%KVmlySA5T`yEM+`Tf*bWutyP>$PEyz%U%QYS0 zMue?2e zKac-c=GJZ(}ZYRE={#~`cJUp*LR)`IF_X-pCITY7G!6H+tF;Yl0Q%xmxHneLhHow zQ%Cxm*7tL+Rxh|%zE^MRw>vd(1o^s~lI68zwsgU-!s5%k&Mx41R}zq_65~)S!@!$I z5JO*A>v)VJuUUrW-Vn&P83aJun1uBS2skR6?MQatS5Sc&lzqP$IXW#EIo*F9WbON z82KrRc%5>3#jSi9SKe0V8!oT-g|m_AA*N-phP4uAgYUd1O^RGB?mm&fQaT~6P8j42 zN5hZS(U-hp48DEmnc?9deRl!+11A`Y_u@S~WF=GFZwHu|lEo){M^&F%PKgEwxJ50E zeEf55z{MYQwYiF@#4!zT9aeVx4mhmLd}-F-3t#jrh2-%-j;7CB)}G7 z!+y3=k5G)|Pu^PbtWy_u0Z?hZtbwbHyQ}lNfVUTv-Mq~TE(D<(ts7qT1@JB>=(?`7 zl!LeVOP&obsFkfDRgsM*q&A}EXw$?j0=1bN^eqX8!4=fs97Z*L%8f?^@IzBo^$UOk zSAClAO6wZ#0*sKmfUN-Nqg}vu*45X>t?El*@y!8s9LRk^Kgq%lzzDD2cc=s8P*1J2N{9;*v|9bd$BEI6H8T+;VNKu59e0d?~5i$~_rQQc~n_g3d zdP{j?9|rn8RlWu-(QN8w6dur>^1fKDo+399>%3s*MR~IB;heh*SaOsqxckQHLflWQ zYP)~{_5}U%^3@UGK;OVnR}-O2<$|}(+&YV;m2>f@Gtgx#ce5TgEm`Ib(F!G6}t@PE_f|-cvz%3fu7ymo@6Cyc2BX>6StMd zzx{kNt>bdL$t-ec+e&`mh5Hkc;f0|uJ14eI3{+2j6Is35<*^09=C@@-8H&fWr0BxT z6m7@)ywORvRSKgS}E8qT$MqlNS?*kv{GgIrkvhh8#wu%XsO~Z;2g=o_C zaX&O^fpZt(kZH2wBlvXt#=KL#q2z5pCw+N4QHnjCo?yZj2%_VM;_t+MAFrH9`1&zk zkO4Rh02~0luKsTPjIpz{w%@KlCMTTYZCi7)qJ|mrCGEGYSMjb^85evBRQkLU)=Xs4i_S~J$V1!)W4$7FfoV&YI;KkUe z6*0r>ubXPmFAsUN?eXmH`B}Btn??3B%H4RoH1=Y+qA|D_#lyxEfk2@+DwRqrYJIaa zLwm6)ybGA^Z{M3;?#^9ZJ$!vgElzhO^Yh9MdFQA=?zgM2v+v%2cyoGqk$YAiwINcE zQuDv0frvL+P{jCtWqj6~gV%&4C7GUvdFFFQMY4(q3ZEec@9_L=&VZ-mbhP?=m`J@7 zH0SjxH}!Il(IR_EM*6|b@kbr6ee4MbL)aCiA)6(9iWHt%6ha9g; zo6KsKlzTp@^n6Itej{~W-ZmLUE!=Nkyg*j`v^HKs;oY4PP;EiX-PZ+i@D+J@BLgb^481J8w7u-0J z_2RZ-Kt*UxIVPGlOgWl^G!$fUVR{pf+C!1RzOEP)@7hI?1K`A2)70wn4SagJVv(J& z_gq$}n10+Voq^}f>|n4ED7oEH0_L6hX5~9&DT3MWE|2l{_qT)W6e(UUThm~7LZYNC zSDiBqc)9him*&5GVTk~NqAQ->3D|Ebrgtg^C96>G9InW<3xfYD z|4Q?Q;J8oxqAb6PhRC=-D;jEQjC8c_&Kv<=h6TCTklTDS;W~z?$!ME0OL+g172f!1 zaTgbX$Tu-3!{}<5ZmI`zGXE`=W~q~Xk6bqVBm0_ris8zOK0(~8S zwsoVjl;3hTvoVhT{OatKy(yNsq*yg-2s_Ht!FaB9NK_dDhkY+^5Bn~gWImh2oj{^0 zqB-qIpkP^3+mo4sQ#5PaJP1@>on>KCEnBmRzUO0%zjL+Pv%o^#E+Iedb^(>3aLxDu zRlnAjxO$0ob$*IaJ<#h4vVlprV6Y$Ka*Q~#&u0zdpzi4IG$Gk{D9zv%1S|^I%?PG@ zsB%==4kW>7*DlYq*v_Do` zMBYNRU_Pbh9+ht}b)&*aRn2kr?Tu}yoM7_c3l|YfV42CBq z7frEc1z?$3vt#|Kr@P$?=It!oVflexNBoCc&u*wnNRD$fFQmH)OVR>ch zc$%fv64NWL^F1B03VDor=lq=e!diqa3jVNnTh6fmW66{=^=Y7Q zQ|AL!nQyL3e6s60Qg`*TmSvs1_;eao4>P4(6#Zx(1qw6uM!UF~*1}VnuXm^h=R>)6v>fJ_Ux`~J!JSIm29=VRhLYp*s` z#}!T*KC19P$k5E4WPm^}K&>~2X8q2z9M{t5go;;4isUy@$oP2HFMhf=SJFCm+GyB5 z)$Z$da)XUvIvsW;WRMqpTOjtNn9CEkba_5R>zbuPhvX+Ox=6n2y9I}Ln%+MRgDHLa z&GdY3GGak*{%U8kq!SJm;7Pa3bZ3z56-d>_!2Gs+*;K3dZyC2?LZS)n^S-YV#}qVT zl&ZMDH2;)#ZTy+U#pmx=J}~RGR)}+>Zbo*VzPHVM0#Y3}5+;Of$TtB8m@?Dc_%_lj zWE3%CQLQEIVH>|Tf`_a27Q?+AG1v~!JIVMsLGr2g&3iI@^a&%nDo9)mYR*A|cvgLA z1FU!td)_b8>DrZk#vr>Xckg9dXmub+z$_|{S#eW~7+DhHZ@izp!mQM!3`FYl5Oo-P zV%9|Cy~-}Z_bvUs>rL;%yr5vXY%Uve-SqA)`U_~u>N7(_Y02k>tG*z{_cePIOeH<| zEn4O`C)?w|T>w_0`{;9zMRJtj3KgVsZVCjp5>7huMl-od0yP_5ggr6#Z7 zvl=*IJE@fDR($CNP}kO}(>1*US}~wu<3k{%eMe{q`!0kYm&$R>kmtSK0-Y>fiWK z5`}KJBPnBn6ssPq8+?hQa51&HgX?K%Kh(HBvy|T20RE}B%+b*?Fg+3*U@DOLsI($W zK0t>t;Vr`$BPiQE=-VDgY*Y-;VPq96M5{LJ0u>YF<2NAX^(i$(nRPs}RV!!hj@Jjo zFC2G!ld{#%l55wGeq*yx$=A);*GjJ2NBS z$94gm2}S&QbOJHamXhQiDZ?C|2aO6)Mz5sZEwk&qHnn|9e@5rBh1YVr&9};_q6vTh z`A2bea8px3uO4k$3@TbUe7r5IagwcOM*KcyFwF|Tp6$KU$*mQkVtMUXI;y_-qhGu$ zKD;Wd^_(f^M9&GVbiwEl-{!zVl!T(tG_3#9qm@>;DO`})v;Sp`ZItkLZd-Qy9~-t3<(~0s{$2NWjIKvTr%I6#J?%1R^to9LV>FyV zdj`EV(4uo7@T}+es#lS6_L8DsCCIV9jLq2shcF-V+p&9JzNQJYlaY;^5rrSW>9=#t z-eY347AL2~(>{gI_3O}>KkX2MI5f*ag}B=GY)>7K6tZmr*3=^rDrBLg^CEfiae9bE zuco?w9F;LXd2I^uqz4?G`0m{pO$WnlXWoy%PP=7Bb}`Kc^=ZvtyVu)x0p|-H8fhcf zR%V74FZJg-B5%g}N@UtLbt#9YR$^__FpOL@U70LrXKor)Zs=`G?%Ln&+SH?cvCjpH(6smiu~v#Y-bj&86;NbIMkp(ZGqd`-wTPlB@Y2ev&>;ka*( zYWG0bj`UVa->AWbcsEu9IPJ-);}eX=&y>aoOSETqodEoJE2)A_W9 zDWrAxplpieLVy}Kz*z9IvO~@;p!rfz;EQI<796mk!;6ufm`>pKlq7;|)FfeB0dDD5 ztq{&~Td3*0$mXKVYMWnIS7G1AMP+k0a{sr7^SF{JLr0^Q#|GKS@r?|c9K8T40~gD^ zQ@C=hIDp3k`jw2;rbCUsAx6QPzLld3Iw+Cv()Gir=GTj<`C{pArsHsDdLCIlH`%cR z1d^afV6HCsWRfaTzF!7C?W~H_#pt#D1r_)ZDQ1PH*csNA?Ko7DB>rO9WKWGxxv4oj z793++n&uZCzP(DWGl9VOt8?sJ1`O0QW;FVXI#<;h*WV9KSS_w^w!CX&=T0)M2y$De zuI1xd-SB{p+$Rq^oZa}m%4QLWreSUN-TK!SzjA0bL3>*p(Gi)!ixazmv)_#X7yl(k zrhqgk++!|b3){f%o! zDRSL%HuNe~@;qYl`7C(OJ6_8UC+MFbV$8Y;hoZ}z76Ge0sO9mTN6O$?Dh`#KWMi7J9ibNu<`dvTn0{pv(qiGDJz>$aJ2WGFIQl4@A zn4uM0*Ro~xRTgrpkyVIcEkMD|k02)0j4i<2hYzrUPuA0N9y4epH$PI%3A8?XdX%p` zG93|@PN2x~x8XHOU?zP~l&v>e8&31w*PT_-0c%w&OYr|}tU3v;7qBgK=mA~4Fl)FT zik=ZO+71w*JA5C@uoljG}Uk&-kzN$1V=SpCLH6PO^m= zw)m!vfIE<@A#KUlMDzP;`6FdsJ!QxJ%Nge{80-)mv`GCgt-rON|Csx{k#Ek-D9C89 zc<(&Pq$R2aDCY^stPBoG%q zE|JK8wmNrF!#~eIq2sq{ zN!p`l)u}iqNvkc<7YCGk3S@_B^sp;jdxJFN{y+_E|K54vjp?7nd83)S_T!&VI$;3* z+np%Tm#tafy;si<>;Lk?OTcw0v@`noY72@$NXx|v+MRIAN+ehh+?+DS&<@EINx(iz*ZthCEd+^AD3Bvvs>#Bvh#-O zZwoN*bHop|xq-g{0Q+g+a*7>UCp4MxoWH3@<4BqzQK#l)Am>~@SH+@hb_kk^K@|e` zw(yes_N34!DoO;&@0TS13dGz6Tz&?)Ly?_|S~eAtikf+>GQEc%Rl{mVB`1ek0bTNrSBdA z1pc@__@lZY;Qxosj!Z_~BAw9(F|8uA<~5f-E!lj!0s9(q*Ju}TP%U1qJ55v7?rU{- z@H6k7BS#93+nqx8n>=vud^80|5N4Jf9UcbVffD= zS!oeXeBcBY0K>8hSxWe!}WuErf=@M`oAgJt-j%vr+o~E zbJ?MzZGaZ%B)#aah{+i5Z#{?8GP#~1NnE#>hzF`$PnQcnZ0?w^zAVlhB+Y+se&5dL zp9gWAY23WB3+Rt*-4Uz2J9VjBxLUeyTHzm&58K`aU_>;&29E9mQtMc_U4UV0N;;j? z)3=!LjD4qsWh3(flLPbFbiS<<{kX=}5ta_#I{pU)T_(EvgJV8oLmq-a`h!pKM2idg zb^#QSFTz2V7R;gjNv+Gux$VN=j{+|Kw~`Epg%=&WfSpElP4Pk68+>f1Px$6tt8dTv zpraGt<-eWllKwwu{Wh9?;hi@`P7H`MBe67=S zcv^{jW`HpPu@2CYB6`Xog;(eF#vbXxuVA$G8tQKsb%5K#eR&qr*K$U;o=-9d>8+RE zMclpYCF~k5HPVqI0iGdLZ-v%x->pn!mx5! zEcn)Wj&P;?G<|kP%219LQ_`>sF*Wsd5sED~7a16+^W3qOB5lJVrBTxN4=yq2an4EK zsor3I)N_Z@$PW$QU)JJ87ha>?PPT9{?8pneUKRa_<1?1}Q;$&m z#R@TyNSbtjDNaX$u<4#{;D#w~?VP}93V@zFoHUb5xcbXd;OlpZTSxJnM)L~wDk)l; z_`C&ILWQN|b#00o@#@?so<#-C(ljWar8?txKRNIf@sZdGc>ns_GJ^sqZB0FgyD!AU z^kvM04z-6mT9z}uXJ;oahh+6@htfy;D6+HJ2#OS%pU~}M5fHkT$=gbLd}iB>Jacel zByrCF{+JuvV%6YN9;xUZQ%tK?!@UYc zoD{cQA@_^9EF?~}2rB)mD!ccdT>8{6@wNH>Pr1>LB?_-fmZWO&{JBq=rHWgx*xduy zx_JYAa2?OwqBm?S{T>~$7d5$0CQxIMcNh-@n^&#XP87ZH@=(WmxMgq4EfR@VH?sw# zY#&CzjYc{7Nc~xqMELxzFWan!X7M)*BI|obB3@iJP(CMgb|RT%Fw;+2dktzP1AX5i zOz!%V&XzLLcZ~jKXuv~{o7jl2jdxRBtUd-`NO&^qsck5hWBhLQtU*4JLZDVHA&)j& zlxRBR4c_^-fI=$KN#`7nyd!c^sMUhUxTOkVe9ruW_JYn?LwcW$c4oxpjJQbt?84zZ z{2i!|-fU%#_e)vO$HEL+UM8ue3!&dfCDo)i%D~`XAiqq#7t)U5Xo;BW;D*X62IZK0 z>&a4ux!DlX?JpxDx6u`yl4UuM1JH-r%|#1>R-mTxoiUq`D#)Eu+i3=UOA%3|*Sw`# zyEL>}!>dV6B?n79H5u;leKnw1w)O?ttrznW?ac5DA$?|z1fUuw27|{Qi=LN4ARuPO z?>1Yo&a`r?O{07l7o^xCqutQTIH$mp+0kl0_ncCcOMDy3ev1ZKan!p^5B8y?sRg;2 zl{ZoDABl1W@sh*!{`f>KOP1u2rg&}vzQF2g<&W=LmlA}Gtn5c?FH6b z=$L+MPMcI&i3QittWRDenyKUpY{#m}+tyqu3OzK+NuV}XosQ%mT`kdJ%Jqn#x4wZ*-{_j*WKHAB@u`CDpM zIcAGJWyl0;Udg%Hv96zJER?X_8khcR6z&~fK&gh{;IqI|g_7bn2%eFXh1v7TXr|#z z!{K!G{8GoX8wc!Qq9M8I!5Lxp!r(zI_mXLA2Alib~e+XVf1_=bBRTDWm(9fB~1) z{i790P3w{#iezhS<%qj|{&_EC?U#gL)29vOMYjMS|N50-pUFPishk|y1FU_32)p>1{Hb+syW zO{{7eI4ryHp~Vi-qm`o2`81oTp=zlE#`sRa{8Dc@xo)(u+K3&$Yz>fg(!;Y=RI}PE-~bc(nB)8f^pS01y~ta z`R#X*Aps-PUq9y^=+NvRHxyXYWnrlwV^xBps)*Gym``QG8J@*W&1g@OW?{@U5ldl1 zAmh2TN4V*)d{ML(C-b(n4`nYL@tNQ2nXcT&@<|8L3DTMcN z?~tFK_S>7T=9 zbozGzu6^3}v&03(PPtk6yw$nmU7UKul%4lgSy)TcGo?m`k2so_7&r$b%jIi*i2edH zE55!Yq2%T@L+7AvM9KRpsAT0P=b1|Cmq+;cGhjz((XWR;{Go#X`~LPT{B7O&=t%YQ z@$X7IHwQQwW1r{OsyT-ehDsk}Q>1ZV@+!AX?U;6NlkHd1Vv;dZbqZ7<*86-`=U1+J zX7gH#yH~S+qH)Ztsj5LkBrY#5(J*sOJkL?{`Zei$h1QF{^FvE$b%(w%m&HCSoww&P zE_h=Fk7M8xX*_82J+hNM_E7I5T1h*crBplwQYpn41xzLBHk#Cq z8U@09<~4*zWU4iF2wj+*`#B>g7?4B)MmVWXP01IeX`yFtRKS@6v6~OqBzdmsq?K=7 zMA1dWu#5bN8+MWx32r$_Aj4D!=neuk?yvtqsRJkeVdQEm!Y+{7o5#${yYB{rn>^zb zzuUkY46n0=j3nHHhJsQFHK&(MCj>Ae9`pej^`0HZn}F$KUe+%<(@uW>(I$ve6x+@2 zpq-nW)3`q0?l$0A0qgpnVJ^*QH%HAQ6wg?czSeH6GMV7aGn7J7omU)7;~edM{p5qZ zm@|tN+3+n9Mkw#G7P4=wFP%DgLi|fbHP$TUhTqF@a`%Achwo4OZC*yKIO7|-*$%|f zY!`2-vBqA@r3pz7b*Fd?uE)7z(EIEbN7&Ygg10#@QUyoa7Dg5LCFzU3VJ6+t`$T)$ zUKD7UtK{!Z46kpP9H^O&E3nKEZIqy{$)LSYy`Ko^H3oLut&&$)jOIFTbcRdhoE1dG zyhnU$l<`ZL6WafNHY^*5JIGm(DzQ+akGZwb z44hPu?OkcRUj`QJ!N_JsvR7DHQJ%`4Edx&lGbS94JZuXdyLk8;rSELloeXCBoPF2$ zeIo*a)6;r+LmApYNo(e}P36C`mSW$3|5!ewWWY)xkjs0^bl_n|@s*&s#`bGw z?wJIplUIsd>BhFBK6PXS%aDu%VJfSx3u1J7No^5826ugBqP#co3nuMkGmdXaQMqUV z>?ivvgBCNRBfchDtm$k8GJNyO*Tcl^-rU*fJ)>v`1eXPsR+KtUixOwsAIE@IYs2_Y zn6-@8xhZT=FkAYT?1Z>$7%e$W`j)N6Yppavqbl1PtCjZSXB(3i)H`vlb4@uPaFT_B z>eG^O1{p3u?LMt?Epm)~P0u7_Oq4(PVnt;8#3|49Sl29<6N-~Oorbu)c5|!Tjf7|-qX6shwM@HQS$3%BQ{kK zj*4>_5wVl}=F5d7>;iI|llFbh+_!|7y?V3Q>cM#da;14I%zl&c!ef5D(bg0A0XdL- z*)?*2kIvWx%X|`@Hg6|Pm{YEIS~Wl7jFhob;BMJ*IXu+Df{`B2x63$I6BYHnM_J6? zh!IUZ-yIcgYGM`FqLZF^J9oA~%?>QZ66)z_1y6cN*mEF9XZF;h+kJjXu^yb1rd6S1 zk6Kp{-TgC;0l62!psf(!Ee#|W6b*g9co*K={ zAmpVjS&Yfqi8nFYh*uVc%T;t<+7A;_JhOZ(bGlEpPP>R07Gk1t#Z0vF;{I%7#W>n6 zPvqH!sLV{tl-Te%OEE7F1j&VZWswRkS%t`tlah81^(;5rESGhySIEWm;BsGLy zAUGtWmK#&tB9uDG6p!&sNw8c~b}af<7NYmcb~=BxDEs)+4F48^oE&T#)j4qeqB6?M ziop!^>3yv0p&mEI>ax3)o}R2F6)wDINlTN&+7=-7Gws?5J1)`Uyq(8Z{W&AvfroV3 zsNouBSY)uuLZ~(41x4rb*3GS#%coXdS8#SEXE`mrEg-E5W!()m8~^h1k+ftdBSWJ{`2qD#c1d1tXYh(Z|9p+7;B6#$T{Vrd4cGq)~(myWpAcBMtJjf0W-h+s|I}`8@{m| z{GpA8%T2DberK;BNM7@>jLpr;*PosdEXnis3l-yzf9o)bHKHwjT7Ef$`2Nj7O+`gu zcGW!%FSELe&JR3lbhHA=oCVDxtF#~preyzc!y5Ryl&%u=PvHKHh#h6&!%t#$QFDVS zJl?IphMx^zvd5;fRPpM$nBsf74K)Z|>x>$u^8XJjb#fmsM&@S1j+f z5Z1CzIS!*TW-v1Rsx?+$EQbCOnC)$y-=6}5!DX}lf-!54h!GMJZil1NLB{_`_G2pP zEf=0ViF&A~w|?fI`?BXWLOT6>XF?7CWYYg&07}M>s6(3(|C1ws5>=rg^b(5|ir}#A zbX`kKO^67hVZlnnh6c+vp-s=xHTzpzRX2{naD7wCBsy*kj;u!3|06q&-Fc3|4rzG4 z{(7?gZo}lBYy2^raw9_Yt-cA!tljU`>;ADi-lsNu zG)T_do!SMQVC@3Np{vS{x~MIf_hOz-E2cNWVZ>e zG=3xA`u7HB{ohvZ_C@tGzst1RxUhT5t=@=GhEB6YH{{6z_dztTkhSwiXJjG?A#%57 zz{g|Sh+^)oW27q@7N1QDkFSXbL2c~HDk#FHMYL@>Qq$(`iu}u~ZL2ZO4FUEi>rfM& z&mGRsDx5Ni)HfB7j%u17=K7)1+-nlXLtb$lmhQBBm)lP73=}u!oE;2O&h9vK2a3Tn zj33TnWPE;Gy$`tB&iP&+j~K`^w9%2NoXCqa5Z=&abZZvTo)aozEP zdzQf7fcuYhQ;nXGX9PZB18(Qi8!fXM$8E(Qo>Ft!p%S#8s&8~!@XSN$QT%1`h-MHt zAL-}hv>c?sV1Q5emy2&JesPgo3_1us&_|BH4a|q22YDzEgErI#3_dDy_^P*)dU7eQ z2E2kan_gNm7s(*SVt?#>J^vMH3Su;{uLLGng)?8bilxsm`UhOvW)l>1hO|Qkz-+b^ z<}KS_34Vn5!{e+6t>rrgEf}T5F z3u4Z8Ixrh)mfVuD+Huc2+K5wF@q9evbsMCkzUI1$Zs(2MO?k=~yf`61$d;E+fg0qK zS`ofrar$ErV-`hS9<4jVi}(8RFY@7E7$;?EJ99Rho7*d1yCH(*AzWkt4Q$cmpOrsx4hL-jE!~B=txQ;A_?EeKhP?l zpr<$1I6ypD_Rhq#W$5j^rx$LHS6z^!bA73r>#cDvUwuaYBGKktJ=S0ENr5lBiL1w< z`XT1DRyF!4F*U zW9RCE1J1rIU+jM(R^6qBtwHMFZmJ?{QUg~U?{($Z4!BRxVqEhd1eoru*{=E)hXMr( zdokQ@=G&@1y)v)+TAH9zn_4FpY@T#(lB!g=B;y-Y|paOiz+dVN7!(o0))EfM}k{@~uj|sBn^o#NaP_wCF@< zBd;AraI&r7C#2QLK#LMtw5fS$&*3Nj3Mn@*ue{InST_${7{*Aj-~*5JP@!f8gEl#@ zJ%>zBC_HIS%?|}D@pGb6R&QJs_+jTF9XR-!jnm7W;d9d3Cdc zJq4`BEUlTY9DkGdYu-KBJWC$UsZIw1(Ipq(*bj5~CSygRN=O7SGK>Q_N2eu2VV`Sf z#|PgJosnSO8Sk2RC>yOm9>MLbVrV&T7q2ZB&rW%{4w;`BdypBJvtvAU@iMk5@%+0Z$;Js|hD4A7UktH_S}vS#0msC<8N*?VkJzLgOd!WmU* z?0vmIfyI6HzSo6uFOCh%M`!8zGFJ3-+oq>MbnFbBF&|TjDCwvUll|HgIAyd`0g@5Q zhsPFuN~Dy9clVZeqp)IL*;zB7bqQUW1-J#1EOdS>aUyJ_sq&4tLAd>8kd4{a6T6$M zElyTc&F227RO3QA@6V zQx+M;Ow_odw-=q_<6zxx!_DlgHWhOYKyWF0^8pQA5sSc1G%Uk_K8;7}&Y9(%zjN?l zp|dB;#zxK=+nupy*Tj&DnUU<^pCw{RFd#oaOh~+r1ub3YXbUD{Y;nAD{ieoW`ox-N z8RJklxisVE;DGZ#A5^C{dgWGh+-xz4HrD$(PD49PE)HE7L?BR~-_R;e9%BJD;SSfLJ&BX+jcJD` zILms?c>6R~=Nnw)QG1mEhZ#9|@Y)a?*2Jt2{_Mw|N=^iz=gTo73aWLUk3Q5XM1vY@ z{3t{RJPheiE4~!@^Iv#yXpBO{u$l-w6OgjB?&?07uAl2;Y>wlsdpV%8{GeZ-M=`$?vO0guH60+DA@Rd#pK&nqIGAY={Sa-z-`qcsq{QOCx_FJe`p7>vGIsS;^8G)=P~@E zll7?ud?#y04G5^3|8fv47#J8(7d*&Zxf6WtXiIy4zx-7Q!%$C&^AZ1)64=$IG=7 zLNAn(8qFF96@7KJXY3_pSXd-5chTlcNp#CSF1t@@1CL#EE?><_XURDcNG|Bqe4?IT z>I9^#<%-G5jhg|u01*h57M@>%$ngrEuEZ34xzZ>#x)J}v<88EuBCp~QtV_{&!ZS7s zmC)(c(b@)gNYgRy_VBO%DIjtNDn(k7Ti|s-4XPU^$0DVeJJ5rxEfgl0b#TSu#Hv{uCD4Km%jg z{9>`U+m_18TOe-w1&Nl%RU-vYE=Yir5nKirB?Akg6f}AAx_j(yC4P)IKQ7o=>=c$) zFUahEzTK2R+MVm-ZFk+yzrR@qhlGf|GWLZVDu~pY+mDfLrX(15P!j@MglMgjARSuS zfaQT)x|Npu#(M*_KeKwR$4E(J;ktYvwM)MrYIt)~1S#Bl=FPDQ?z;y2N$Tj8jHvLc zS8wnErj)+*M^Xtei*mKJJb!s{SdD!vgOFM|aU5YM860^`O^|1vO`#C73v=(a)=7tU zw7LorDhN3+E%wV?B|B5EZd>=!9ExF17*G-WDG?MRg=Q)7t9CP7teD9j>%=~@#Kri_ zqjnEA{1<-r>OxU{ASM+A+Eu?x#8OLbv;yMNhdhBU1pJ!yS<8Zl=njXf(noU+;ciyg zK+Hm!Q0UW+W+3>gTK@ecJN`NOk~n!1tT`h997Q%OuxC7A`r^X13w-VR*Jfbv`chlv z-w+s2*b}XS3&PHhUU1xguh^P?c!0CT@-R|A~ zeZBDa?*jb#*W&-LrT%B=UKQb-tcP`m81zc}zvz_oG38f^7*(XnYXWAd zL!zA-@4=YU-@>>Z%2Owjm-rlS4D5p9*k#cNEdO-o+z_QLJLQD-W zXLltw#=81#5=iqec{MLayn5yW^eQMv4~q#_Nzg00%NZ1UC8WFYV(Rnp+;LrxAO9bf z8viA5w0D`6CF{us{Ebu-4<$*?M>aN{{0L^~P}7AIBG-UwrU8;n5otAI{BaBmC^{sY z`^p!6zFnZa+T0U&Ift~yKa*UfrO{;&CpnmmaqnUhd`I80q)UJPP1|F-+vsA$Do&vw zxztcq0eiGzkfAC1B)3M3V_vr2BqER<7sy)9YIBH{SbWZW=%0G^9ZGrlH@afV!RoMn zhOuw6pzP)>I#D+5$w)OeeKLoPmq`drayzeWH87wEO#C_#vqYq~duO84zh(M9>1F|~ zB(lnHovSkbUe5g93O!NBP@R;&)4;Z!pSS`7-G)-2SVo!{0 zj2?$EaRn`vF>>$>j#geTrpm^U*F@eC&vrJIfJ%kqf6ypM@wkiT|niOroiFoeun-M zODzF;#PkP4@GpoulZm10fyB^$!ccG*U3h*>=a;PVw}3ymEu0b%Gq1iYCp%6Wo>fn~1j zZ4XIM*J)Ri{Y=YIp3C+xtGuyUapScI&r3&}_%nLYnG6Y;$MU5crDU_Pn5GBW0o8W- zr_1fK1Hhu*_KN;1nfxui$&-O{n4g$5BHkvGazp?w71K1y)`r3FQ^5V90jCmnNPFK< zASADMmqcQ+^`Z)))xl2+J`&$n3_Qp-4=x*+E03VI$S-Vb!|vpHwv*;W&(xw(^1hY& z@*VywCe+bmx}9y>4c-7g@+ z5Tj3}&Y)cQP6QeDAJh3H-11=7rO0f{6=@U`p!Yr$CW-Lk?=MchcvXTR{`S<5t?@IBAEvw`XW-IR>88{gkNl!=nq-_a#4sg8 zz-a72*p6)K=j5i0xnGx!7P4wB>v1z0(o{^}PfVg0rC%@3O=1o|!K`^p4T&O#Ri=Q` zv1#odAgTOn#4ibnRYqh&+VH*3+CuJxmumiKOC)~t$BgNhUzBC62vJMM5If)8N`xV# z^Q`sUdAQj(9RXXJ1b^ekMUVtq#~ttSuk*zJOabJU7rMCC;)qQSPIRJdUOd8B$_vBb z#6dFng_xqPW&0*7((d=;jdwaMJ?+;=bfIOk#5_WIa_6gbd#$t9$O1D^>Yc^&n;7}x z4P5$4eraF~p&&Bd&ay=_hEjFN@C0<*Otepp0dlb~y#EV$U7dZdlovXR$ObUr`NdlEk>fWnXiq}7sR@@bmN2N=ZXOYG25#7gi$Rj zkIblY6bIVHdWAnOy@zgcS#iYIcrJEyI25RBoiFf@$$pe*hrUG=)fC@z1M1YaVDZ~A z&&|%ySZqqyw)oGm)~Qh2~44)@R{u)9XJlPR$R1 ziwd`x$EYbM9L<3p%*S71f(dC!V|PV@AcoFz5CYjJ&MMHX;(Oa*^25_vh5v19-ECE(|OWzxSd zM=OW{MJcRtJ$+Sprwxy4hDhvb1Dk2T9ei2+V_DSAM8n%JjVStW)Mp4I20V@quoA(M zC;1TFYd(A{pU;Nq=;p-Px?-4-w4gIOurumxl_U^@F{KH=!^g3p#5AI9e3QQ`dH*uY zX28xT_#x-@S+GzTT<0OS+^oDj$Q%wnz6Lz4di1eDv)TmP6o*AUVM9LgM$=3E_shxA ztvq?516GDmq)|(NsY!qtv|)n8;5jj2eovJiqQ02vTYUG*W9FCh1}5)o<_>RoBs20O za%LJ&%C#pa|G;q%!lkDz+sVQ@n{8llHFA8tK%C-xMnQ5FTSA%gB$+d+lf#Rss2atD zODX1{n8@?m!1o9Uq>&F|h5~Ay|4|`_H~{_>Ip$*Ik$knNJfAZY2dx)D70<+4Fsdt# zGSiRxrl4It{R;^gk695xQtN9di!jwxB2S6&1>J57;+uRvuW?H?j#5^wvm>Y-~)63~V7ZId$y37wm2ZYoG5; zfW)tR9M?+19QN9?o*X6PZy*IS;(rttQ!lI?n}N?oFL>h;T)0B7hq_Zd1v2o+OtzZ`wZ`0H^|dvUf0BCRFDn^n;`>=F#N9*skyB#gp?3w{dH(MCR)J#`{eIzUr>x1J-%UjM2{ZZqBkUjje40o| zyl65_&p_`5!3XEZ@H=i*-=#N~eFod^g+yDH2J-Z22e;_+A5)M1sOPEC@w{pV)+peX}Y8nnv~E%T@ZQ;NeCr?bV=yFh=sL4 z2m%raN+?P~0!bi92@sl!0ulrS5_%CSp`$bv+sRu0zxH1Hzw4Z{?|Js!cR%;;JkQLW zb7agh$Na`OM#lWc_r6~}+U;hEue)Jc4{u7f9mNx(d`%nQnN3pLe7Yd{I-s$@%O(_P6U8-^Xgn8zT zYl^nAr^A)hXwU-FTsJ^fv_zrr#Wc)|QxWKzya4SmX!|F1GOVi0-=}guH*3d)WMcG0DCA-JMe~nOEUV5P?Lx~PF!nlJq%t|k za-nh>h8)$_@HiUA1?R6N`fCP-f!r;VnM;N`)Fp%jyCQaBD%q|e3-R4r&^fP6GoWyE zdLi}Wmi3$hR#5Ayw^B@s8zJA-!p;0+VMb(BXa%Z69bGs8+46YRee0ie>;D1!??2t) z-<+x*I+#B1ZS;A3rZ+U~cMS2J1wGm2s&Csv4rF_|-+{D;e@EHM^A!IHZTsklgnvcj z{(S4`-vPV7XXiq~{!=b9&XM;1HaEW69Ud-YpVk!dou}mm3(b4kR{r-VJg003Qhx6J zQ!Q&MB$^>LJSHMk?VAw>Xm+7=QpoJw@G!$t!MMD>o)9-uHN>TDDBl}H!;|8f@E26+ zyVe{f6gZI!BRGdUaJV|58u4WKu=`$oP2RCC2vO-B`vs>%k)OF(g)1+ghk5i%m+$$t zHt@*=wJ*`>nh=(qejyvF>zm<{k)ht?GFf(tTz|m39~FJ3$<#EPJowLGNqm%~r1#+!KJTuj@?eF? z$72d-YOE4#RtLodheuv45kT<^+W@L~akN+&nvB#?I8!rYY-7KYx)7ya z38|q}BAfLf$DYIq08$IHe*U5B;b^_erWJ2y(p)$pZvxp>O9@2OmBgcmlTqS)F!J%c7F=>)sK*t>})uUE?AX0drMn^;zF}?xgRO zy^9Lb+F^i8EfgQ&wCkPxkiOmtaQVhnTcT|pOuoa>Xw@eTOV>U;GDu$!G6}w2O?9E9 zkp|YZ7c$(-kAFjrvE&{{Y4^zO=*ASfcF7u9z z4W!}&N;)w?JqesSo`!n!c-X?hlS?S=?~ZY9O(RnmZPPgO^uQ#ZFks zI`cR7z|RhDC_IZzuozlLQ@I0si4SCGz`|v8j^e{W>C#6()7}>9e90QO$;nq(`KhAn z_CUAgzGEd&;KI$q2g4j+m|Nzn3&-5j^V8LPJuFi{U#~miYk1mZnDCxMk-D$=JnKu- zOifhuL>;>domNq+Pa5&FjE^cu#!Djsts9+UAF|atY+KtuMOqc5)$J{h0gaKa%7PeR zInA#S@B|%T3qTic6&}Y%q5Z$+8@AEwc3J!t%GvKxK6jxe5s1k$O8T?ds|z+hMf7@R zQ&0LzS*gXqM&z}rx7}rV!o4O#&vd?dm#`-uyPaMSS?1#-FZg7oI3daxyhj&MIST#g z2w+ChIBs!5-jDH`xvz_Kv!?ebMv#3R>9As2=p<)f%n&oNhkyl6Z^rCGpd8yG{VrsFEC8EQ-k{n#ssSx#CF~^k$t8-C5C`T#TnPW>as)Uwd=F{DxhG((C|S%S%TU!sUOMPL-Iz2RFn!h zF3h83w7>^jZFk~C{1Aj_tf(MmPM1-7Nug#MI59l5sfx*}##Hj*A=uz#NR1oWH<>%9 zF<(BrlLO6)7|_&JE3zW$%$?nnXv(O-rRb5Sl!Hm+vW%RH06jxma=$5KY2G%dU+`H+ zv}OE5x(~zc3=+5aoMisUeDKJNg$N-smgx&g+EYDepnoxNeyH2kaDz^#3u#hK3mRjr zeXRPAVwsfpuzm-D685pynxtdtlgh+|nUJZ{*c*Drh2>Mqv(HaLLp(GxqKEq}fZa1b zZI=3hm6@dIK32P;3wMYn!9^R(P8ACztGoamb3S7^Vj*)V3*%oA>q3ke_!e{SY`o;| zmOi^#doXBZ-~E2sy0g%JL)=tcMzGpL{S6NF!BSMrNZGV}qCL*b;`3__PV~RJKB+}lgzNX;i+oKPtWS>X1@a;%Q z8jZvXFTszFUM#w0RcbcA`NP*E=2`S<-hH9==;Lx%PcQTDH9viFWl(B8=Jt)`(4t?C zjvo3)Ns+0a-QIYVAmpz8S1oW(I!#e1n{Xe_3j+Mi3h3fz_$MzU94TD}O^5 zTnX0=P0fcL+!0q;%*6k=czYcqBAVqr&iI`wGL%M1Ny$V~$~N94qd%Qbc2l-R{dOtW zL4s?1EW^a7i3zIF5g!<#875qAPCs8mgIWu!Nf5l*3fA+hHH}}VYW|k}`McqU83_@0 zWFZR49~Jh0m!3dzEP1@6*t9b%!5hg4yU(Rb+=l54vW;P_OPeE2*#no7NaBSz6 z0aBFPwZ>J(STL~KsYO^E%O8)v#g+DKvBI!PifOBNeCJoiA{EUOR2#=C)Kpn&M(VtPAL>J}DHTCUONMsFs-E<^ z6Z*Ge+K+Oi6!>yp3G-z^@-|kK2K^{W4&@OjD+Z)6UYn1Jjx=d2!gV6x#~$JUfCHER zxTzV@c*r5gd#Txxc;eE<;0Dwzf9t7ril*XG*DZ#_so!dk0knb)9l%Yw| zdBvvPq7=F~o9QbAzHtnF>tCPxkDC0)?Egxk2gU{!s!pIM?83de-D$;{_=j%lK#%*@ z-C-%a+_lR7loM~x$liH4p$t3ou@aWcib_=59+olY!p#;H{T!uQp>U~-t&}1X0F&8f z>H2G^!?2kkD-0PJInP5cV+8-Gc}q)g|S8G^)+qV(#gBZ?2t-M<;t0s}6=O!^OL z{mn{&v!fN97U4YuVc$C8|#mqm6;k>TB`5L>XcS!qznw3$?n0W5amWN&?n2t_7o z_DXo-v>8ha?~57~R*MSg!A3IBIIw6w6cK6-D0nR*xOLxk#bM-HIss;EA;FMzR6f3? zi?#T`7VQFpd!C0q;R3 z*2Tq#K^OI%O;&bm+pDdVBy_3j8ae~bIh)>}8ST+wK&~C$O>D`xB#ER)Cs*Ef-;Kwc zqyc`;?3eviJx9GCu*8!GVP2?pggFO?RsikthdHH>E5)k`91sXeE#5VI$u*TBJtO)t zBHinOYh!NmC6vi6i2l@9O~uGm~3lEwzTkA&OUQg*KwnuQ*mJL3aqB_*$fm-=HfdEzUs!}=sj!ndR*63JZbY}w7?xBzn|+z>_66L()8ln@>U*9 zoD-i@ zM|}-C+c4F6pA+UwX~lRQqUn%cF$`5$!+@Of!gqf0J{j&U;WtGe=vEdwyeFD=4MU_= zq#TMUu3nCT#4`Bzg}}OaF`hFJ>*V!E6?L$|*KQ8u!!S^Dq+P-|EfU!=(9w%rj@G#A ztMSVZb@YLOF&_-NExvJC3JI||)D*P}b5mDK6#EQYzUw_6pQ-{f*m6ZWv~fi3VMKje zC9d-Ei|;%())j7gFr$5`ca#j~tgyUe(%{Mpt$e+TAg1DKW^zzIp(!RV;%Am(L><`N4f5 zNJ)+e@a2UFvhqXZ1YmHmizQW(q+5vIb=;V)rnwK#xr?@7A*1nUr7i4?nud^FmXBI2 zPEU+ZM1aqIx=^6fM@Q1`bM7zbZWG#{P@n6>s zNt3G*4t91WH6d>t&8#bef%Fv|qSb;J%PAjOS_%#ud(fBeGORWAb2q_Te1uVn17R~3 zIM>_}@5tkKh(CTXgg9cHUG@O2p3!-VuyqsYXG$Z&SLa+Ix1*^w z4h~o1l5og)#`{C7f;A4dAEdC~yiaRAc@-|J$ym*`3%2dBAQdWNC0i9-O0ftS)N}ns zr83BD9AcPdYKiXh-YI00;av_Xx4-km^^C_~77^S|PnPHlD!HChX$6o_I!Vma z)AezzwE{{Sp1bRX93hJrq59?1Mw6aBsVc>gFyxpIdY!BlI9~5Kb5W|bBU}wa4lg2o zkbwQC8n>-ggt6$(}OlCTlVZ-w@H3K z*kk{3(Z=O*MFO}caYlDiSo|Z`8sGy#j7^~OW1EME%>b-QV`uR!kcnfk7n81WSCP9Y z{3_}!U^RSbc*X%B+A08lZqRcm9D6@iR|X{+SRtcPa7H2uE8Xd9P%2Je?a+qBo^jcj9vHgEFUO5;?*ju8SzI zh!2Qgpk&6J6d%e=6y$1=sy(zjxVDq5G^j;6u|WGMJ0>LY5*Q@hV{ z8Vn}OQ`^`VV@!KPttwV|nWD~$kFH*Ha(u6A%c@MYE8)Op(qq4CrO%Y@RAV=kW*VULxSP2%fJ zdXKZ%Py{dv#Z)yY(-^uN1`e^uYZzknA|z5%FJg%x7s6(I@Na-`z$`aKp&8<=KtzJkY%@XrpQGy&J>qj_9(K`!I;YuwN-gZG}=OAg=k& znr6lCy<&nOLGBrWw|WxrpREFrl{6xrG2wufm&MiKEg5GsybENMf}V?<5l*iuZ-lw~ z_0-n`^ozQBl!q(XR;~e=#qmdK@TM}F8F=qqx@dVm%w%r#VSa$}MxFM!=<4opznxT( zTVS9n*{d2XiXmVhxLf5Sx8{9M-Pgj9mbFHMcoW_C-LOl0&h;5}G)I<^(CroUgYP_R z15C<}HK0sI#HgUb!b-U<&nf~RC4fggiqv^sO_43qnZ4_M=m9bMgjB$FlCxI$>Rw_W zZYM{nGD*855NE!a-OI|zSei5pD?VCmH5$w*{N(i|CTHe5kKiB|rf>aI$Qi3bcsfF8 z=RM3=@Fis!c6rH%d%waiNkI6jam)4Ig|NZyg;Pk8I_+bM@mjc3j~6R2#Hx(uC%QZ2 zQ%B4(9C-!;FDFG^h@T(xYrGO2r5V$lh%S1^9%eynoEM$g%zbk)qUe0MN|`>UOt4)c zKzaA9(X}?$lJUbqyTi*e*7V`8_Z#3t=3~>nQ)Xt^&yzr~&=@kzoxQHUqcs}X41)32 z*Uw(=p*`<$a|ZJ*lw*-m)vXuAuyFI#58>^5r|-Pxd6%_wy@C)t0H3I=9$g+_Lf!a!@c z6b_xbz$%JcY$a9CXZYPm)q9yVgP+%-lx9#v4aByZo4|=q7Jbb7;u-)jtDctR(Dvxh zPM=Z1_m5e@nn-PP29Z0F0~IwrlCu@u_%qPqJ#VJOFV{9Q8o(B=ygu1V@624yl+{@# z4|?W?S*$`x_W0I4=25NFdp)gfO>-ep8kl(G8A4ft9y^s%2-%ESS)GqvCj?9>C#-83 zmbyKz`I_bQ0m=dskU{37EiWQp4s$%FpinbkzP}lz9O;?1^`|S_A-k_njb4bQ z{)+T$ba7*u0lR{H{{TV!aMy1G67``Yh^j>Wkd|Q zfl^uKqCh!QvPwC^>gxg#;kJc99q(dzr8V!Cl~SI|9e04DFjH>7^F?LGmK9mjvhFo{ zrkOF&JlH}H=(5Mpxbhq?|GP3q(W34)Ost?}G6STNfZ4CO+lGntY-NbBYW2yN2Tf|@ z9ID+_FZm7(dgN8t)QGrzhOU{5e^HDC`h zi2S;5E>oUn%43ow7ZarK#KtY0+Iru`fC3Tag7wyXn-p>dj%MZNu&!rh;gLKU>OAn} zw9SV@UFQ-_$>*=EeVxX9=kdWeySj4*98Ge5<^WwN?1fx+)61Vcx?4WrT$Q!QUoN!v z*V=vOffR|={-Ed;T@R!_F9j^aV11AzPaRcNx&1TIJps+HOIN)rTq~6oDf2l0;6}Z& zxV#$2?%_sQPqvQdVVO44k$!!FfU?KKpClVyeJaqj4}nDuBHDwN^dECwhw*e6IFcb? zDH+;+;gqCMWMp&P6=Mq}ZCu(i@(h02!q7@KSUmmP)S-Nn_^LyQcZhc~XZAIDK!5af zq?p~pv7Xxd{WXdNMlT|_ml`3z%?yNrTDK27uy-$XcGYB4`g+sNt07LS(8H3p+QFD z2HWzZQCHs>bu&-3!Z4uu{?qdsohy#61`o{| z8Umned(D%a;1C#g*%srVVYUlw}U)1+nZ@nhphN4_G;CxFX?{cS#0)^+cfoHaey)7*$K zi1-An%(vR4&4yMPJOu8z`Z=1n%-oBLnP!=OdZ!0^-f%Kn2$4CM{=9-4M=bb0Qp~ly zAe8LG=WIW285x}@O#?Yr8&WbnZciCGQ!_gvN8Ji&jPbW+wG+|wpYlqu5Igg_KCjL5 zd=Kwv35Bca^MgZMpf9JW*W?W;TK-AE|46bL{ZrZ{{&Fs|Am1G|9{QIOwioz6O+HC)M*~(9Vd7(f zFts*0s*Z{{8ckk*3;&lEmoO}@gE$37e}#lPvq-+w&%e=GTh6j%tx&ksk>Rm}dT zizj~9(DSJ*5RA)A__l56Pq*coHGa$buY%&~5k6v-p!h{<5q7bK+l3@c%i_f2H%ks_$W0+)npi63WGa zrRJSBSURX}<#9&}Zs$Tm6vjoUcQ!52Q1ti1JBK5+p02nh?DPq(N4I!ax1=BR`fUz% zO20Me>*7OXRei#l?zW8y&5Je=#o*(~OiXp32}*ogak06IIv#&v9xKM{dSmNhJvQpT z*g9zhW@juO($F{R>X;@rupUl@!R-#sX0;G+*Soe5*641OAvfjeysUK{sq(TUifIj8 zYKkae8%ry7eAh1sk+Y*u=tJsGJCQe|msR;Ggz^b}M=10yxy)dwQn#dhd{0Zv1DhVn zawOGsg^bmekg&b1(NfF1%OH2Xx`l{-^d3$*%eShcZms@hcP;E9V(mlcBp{@3U|>E2 z!`Q%L{9TLpDB6}~oM*g08>$9gC#`+!E8TfFVm~rfQhZjrp$j_9oSLu9fI+gR*jVf9 zCo&TWwO2r46*!J1(>0QM!iSd4|JYD1uQmRDUr5KPLsetc9ew$)ShtJ{3gb%M(a@+_ zvS^2r6{IRYlTG#@j>8(?un<+pCB8P-u4`&tQqb4EQY$R>OBMU85vHCuE37%fi#6i= zaz0!MnVOnjxGQ&>b;`GS{YKvyt!}S}zA8RukB9B8tkKW_N=F%7ac!AJbJ>l{9TST%o3Tu1f_B4sX#3vu(HE z(V#FtN`zsO5tU@`(P<094?g zD^1)j`m2&Wy=vb4Z#v7x0vmds#C;!?alT5K!=GSM77vqe=+5d=?YVE*qJK)`J5S9n=L;y98!alX zF#|a4u(0uPeEQinXlJ<&6E)aa`EYl#2WE*UB~FkNBCl z9*_RD2JfDHR&~Gm!g2G<{j7gEx*PdH#lt*sdW1`>qVDe$%))-gnq*(N0n1;=T})fo zNa&1z8OzD`kb=Y+HH{#s!2`ok)|G)VaW$&bFg2S=#lB@Ay=$`!-tA6jw$+@)wV~_Z z7uTGaGxU)f7#Y68-F{Ylj7H~!mBC;v~Xf~JAhn*as*xv3dX`b z)IJWDqcL3Xh{xKBnPWSA;|~3PjYNjxg@&fLO=9Q1xyipk!Nw=2>`<_m4?ylMfU<}{ zEv!~l-f)+(Ocbo07*LX;>wI1iZ6UZW;I*^k z!_{JvBngO=kU9q_1g@MfgyR;}#o$-MS{$CPv|?-A3as2@ah7AQlHDhig6(4b^^>KoE3st@hO&Z8WR)$pi=3AXmZU(a-HGv(fOaFR>U&i z&Kg??W@X+nmL<;;;^&z!ija$R7b**vt-B!LR1-KgI5fYgXbm0yFgX|?U4{lt551Po z(S2pQV$HbV{Vwg-zPqygv3CCT(cgKvJnKJ|k`wB^)dnN~td&v3n!5 zyz$^`ozbQsYJ$-^Vn@v2 z$l)e`b7$xh0vSVt5(jq|_^}C~TCvM{6G65cLx)1uUQ3Pia-$1Zs;>c-tpPyw7FcXd z{nnhvwA7X;Q&(tBKh#}oB8Ns`K>>y%L&37JPagVoa zB~1Akf(zyfA_ayi77la85Ty&toP8#66#F*t3~EUtQknd?(oFhz3pR;XM?WlLQ@4Pf{NHYUKc2<pQ z>jWbrR|+&{DOKsFi$|7CrG$?aM@z6CY{L&Av~DtmY})HzuIBak2Gz~fq+KVn|pzY)YZBKU$ z?9*r^^K}G3sUzNx&2;kh(gTfTobV!u4&%Lh_ZC@FlRUqRNi3(3>R8>ux7k9t$mgO_t+A3-)f(8npKcG0HM2qEuD?cU zDid9F9e5-Brsm5)iP)yHS^QonQ!r9olHZya_W=ykgm` zU;Frq-63{OSH@ejjPTaJ?U@cpCG4h@>-Z|~R*c&U*~1Jn=%6!~G0^Ipsl^bsg6r7o zhQFJ82Nb$C3edvHYqj$oe+*FC`MhXHO{r*p)v14Wp)dXyEsTOz$KH=`;CedRRS&rl zQ+WsOM(a`wcOU3~#Vx?YDHVY>Q9*`6ny)QHJu}NQEyOE>;Raxs&aMupwmyD5VWGZ_ zc0>!AiY%kXK%}z=s7_CET++Vt{0fzDZRBCz`a^2m{Y5cn?o)n=>~4&*G1)XR$F1u* z=YryC!at(t?U0lUd5mGR;0BmuU#1iJZDf@x2?i!(yPgh4)1D>fk02?|@|1W;3F_?9 zbd8yqG?L)b^f;}M5gRtQrp4E@fz_hyA46V;E1z+-Pfg>She9120F;L#baiBj0DWD2 zS18TKBgnd_B3 z?oWQovf8-L zd!w5UQ!#8^36vFZIITtut_=IeuHCFide^O_)Rb5)5EJ0g%*xWQV`?jOJv2HCY_`Md zuYYRR;LY;Cwn%HWw{uNfJ)^Vy!C`5;@GcArU9(<*PyzDs9}Td@uwd-twOwG9zgObM zoSSoCIRRH?98gwOww|!Ey60D9^4U2<#ts}pn1)LYutyZos-9uWb>^Zh2;VAMxplVaQC0DfAgsG& z2Ta4IG7P~me({b~sK>@G=Z?7Kv%CYs^5~+(NB^6aGDzsTnWu2QXsTQ}SBvDxhF%%; zkRDWY8dfZ1nY4!?=n-Sj>$Wd_wx7QFyngcw2bUpW=rFr`&B%dkoIk<&qR)NL=!;{j z?%hqh%2n)57eUT;62_uC&v7Gk@zl0y zxtLN_N{Z0v5fASE7w&c|`oPFC-^!!kHi=tdY5Nm}SLb;SzS`TWdG9%{n1~7o zlG4tO`^mKB6}fksk4+_qGsdFt`x-+q>WW zZ*|%~#RV}*UlT^f61luBRL{JlTz^l@oD$ehuwq(cNZ*p~Do*A|MxbH=yBw`c0kf?h zl!t~)`6-B)TSz^C;4v_y_#`}nEgK~j|KRQ$G)++Z>Vbc0kuPTSz6h6jxHeAmz1^#f zo8~T}#Yq*GtrHE3a#JJ8Ll6BzW92YS6lP*8=HWHfk%{0zD7YEl9IP0!cXOwiLV8S`3WNUmS{_}!ckW3JsLe5Q3xni3^POc_66;5V^<6FZi zl$M;^zizdECiQEP-k;dK=Geq_>W_y2y&LR-lo#&UyL1-{4Hof{qQWom!=371dV@80 z3%naXOt^TcL^p{Uo&Y?}8&6L5(EQw`R0aL~GPM&Aygq&X_;;Qhs|DZB-E7>tdz$j~ zy0Ltx)HLY~7?Khd$DI7m12#q-Nsm)4uWE49x=|1~QLow1wB&v_vAtD>P@aF5mcXBp zJq{B9u)<{{Ir%f=`qBgId;Q~FlES(~^|;{$L>7bjv4oH@=~@Cmm%y$tS!0jN$G_&e z^@l_cWQ>bF9CPGYzx=>f*Oe?w*6mSj1U}hze*v;rzdWqPi*i77PW07SHa$SiXAJ7X zARE;%vC&D&>~X)DDU?qp9yb7%(U;ebfc1-Eg;m0(B)RK(AXeK8lU3!Y()yhfqPtNQ z4ROY#@P4_mgjCOBWvddZ;ylWyOL`mkqTee$Xt<`Csfj~MTe*NV`Ast2PDWb3hhz5i~>0bj{KYfv=OJ}BMUjf^Ino%l7vuq z#k?|=LwkDfVRh6%d=wX@tC?HfufgsMq*u*1#u)y{4T`;6^~2k|ozMf=Kcx4++W!R? z%WXd8)CVzH_EaD>EF*jP zihvYOo8U?me~~R5y%ySY)5z}~&vsdy5)mJjJpoK$ILp+(uB`UCEtNeqM9p3qK_T(_ zjxh1GXRuh&S2wIx@+wJVmWB_T;jahAm4)nUR$ji1FU`uQnHpw-xIB=3D=VNr`@P4m z=QN_Xe4UNMMqTuI6FZ7d*0j~vzbk0499u|(NyqDIQglBe8^jwEtXBcP{1>W-b zlV~`6GS{g5d{Q8pvd`;fLWVK zK1`w5@gO9y0tpo0#zXqs*Qr{z@x7nsn*AxiMEAo^UOuT`A78E=xNASpBPl7%fAJR2 z;Yzp1_2`KTdTra27|k_GaT?vuGJxb)q#7C5UiE<8CF4i}+ms&2B-J`VW>6s~%$k~L z@^2ER*9hKzL~dAmi~G88I^2>sdT}bZz5#h-+GgJ|Vrml} zRTa}b9k5#$*ZN?kU%=o`3Z3E04c);ZVcoNP#R%cm%n?rrU$U)yOI&eK?X)#uX%9;u zX(iB_UUS)DA;yQwUxasy>yz62rND?v5Y*Nnjt1R%r*y3*5H|K1b#8#hW|dZV?)zPN zT{4)vnO&pg+Se%-==0@{f_xO7t#b==R$tn#1Ls60(XQu z)k7}VYxZnBZ=)>lB`1Aw^(5zI)CnnaBMV!Uc+oOXfd=+%l%5|=oO`F!)~|pGB;7wrgfpd_jcEY>sR7Sy@C;o zRAB46NhHk_Ko^>#w0z8;Q0eOkdNRGKZYN_C)%zabx|DjaLoD=V!3LgMvoegL;t414%G}KiMnC3y{P2~{jkdc}pjU^(bh#Q)j1@i`jBW1%MylIoU zq}nXv*IYv!95j=FZkXwnkU9SHj89j`4g_0XRn&4p6BR}7o1Fj&$QOt~ehuFqe@25PidvD|jB#Q})v;%b*q4}^i9Sy$==%oJRLZ{6jsKI+IDTqVE+21I& z=ND6+@^)YT*MFKnsn$Oq{?(-SpY!||I{*Jbb?p#EGaFp+IYa-+B(Qtx+uK`5S96-b zKL79J|CY&r4Tbpk_4mKB=luJw{6EQcFo~H;NzDu|Gmw+oPWa~fjmKVK`a4e^H{8lU zc3e-4Cks#BX#e)b^|sjj6!P)JZ(m$*jo`6lO2cYPe zu+NmTtcjMw>2gY7{b<071D6jgSczKYG*yD7p9?3<+?A?Xi=HiwKJ(nw8T@#!ph$L# z=!PMQl&0YGe!O;V>U5Z6Pjk=chJ0!!EL_!qWM!Y_oYiyqnGs3n(y*SN;r@VxbhuiE zqrZbLlZAr!x*sd2J{Mv73)64FE01qQ)#)2oVk z81rTzK+B!;HGS>t@jSih2UF!i)~t=v==V~~cCRvn)iVaNbYs0oZ+!9{>*;NuT>u6R zCnQ^;D)TxoKZY1C>yO#9!?llc6kYNW;aBVpZ~C?114%<}W%W**7h&+9+3_zj5`aMA zV~dK}{Ofyk*pFjhNNZ7xU-A-y_KM>_=)TG*?;^-V#1W(&3Fq26DaELGQd8XJ@vlXh z1GOI5h$67cuZ7dcv97-& z@b6%Az3O))NLv{-32G5mQ5)}tfPT3N1oQcvX!d-SBM_Y0N_&>PbqYzTTPy0*+ZS1) z$hJ6ISK@>+`jOp>&A)lyq&{$eZoWmf3Ts#=etY5{^x4@Wrm{5cZeB_nc1IY55)$ox zPzm)jTA@@a(e!$HkI)X{jx(+%Xdqi0*({uh^EZ&&_O6YlM3?UVj4l;wK#ThF#V-(_)aLD>H&GIi^|j zoPc1pLM~;lwy2$AeZj_bAo0F0V3{Tu1EgAZwJAvDNwDqJpsnk z^NVA%>x3u43fn}gmQG%q?gCmZ1R29#vO#8)Pm{3#&xlOg;X(Vua>!*QD*6Tuhc2nx zJ&gxuMX*!b!hIRG3S{-kvH;3}qy5_4hQL=W995Z;ysdoZ(|O*Xfsj5_j&3*r@1Ha~ znQP+W`uaer^IqmsZL8u=_d&9UUf;e5*}E(OENoS{vyI~h)#$n-RcS_#5p>zJQiAE! z+qfMkgJksi*Sc_1C{6=9Y<62kom)zoSyTcR0lQ~9+|X;fvOP(e zpUYi+W4SUhvDvIEz`>`ysi2}r-r^s3NOyJ5QVL;e&viiVNK57BgZwok$I>(E%^XV; z0mXQi07q#ks<4Y1C8?_PW>|hR!5G3zi6l4B8x-bA-i}4g$IEb9%gXls z-$FVFV>nTakq(AJ-NkEr1sl7A-iRs*H?NuJumgdjT0m@W%$MR| zVa$8I$PvC)TFQ5xh$I&?d+_9n(9UKCY^x!Tm9HC0t45-?>I;-&`KxF4sVCm0TNE7b0T zu6z_YJz)+E%4%Ux0{oI@VF$|sZL(RbRx+Be8E~{Bf=X^_nTG@j2tNAz`-^Sa*B9xd zY^eM{j5<8{@XKP|f32VYl=@dpXvZ}hK6+PA2BVIrO9L4L&xB${JNCSJevn4EUwcX)!~Da z0Wv-|@Ry60Y*U~v<^7vGUNfNt#+9}P)^o^d73_68Gf7qdlhwvG$Mv<)QxZzylZp!R zQ<4nPF~C`dNqmDmqkZe!!`Ex>A%czx@s1;w8MobsrLrMtM(BfYH>Dk3zdU%tqArh# z{!~6p`;ik!77V;z2`W%R4Y=gLO|LpXV7sbPcdD`JHnwh~66t%@Lck@ITk|yIN{iaP z6&pfT+cSZJa2i@iF58`w=@U1Ii<1M+`*LG_3?@x~%=d7$v;G7>ZsB7}eWH}6Qpb-F zxgry!k8;lsc!OKok+o5{HYwr2Om1xSw-iq#{ z6iO3L#J~N;2s5<`w2<_A^WZ6)Ixf-WoB7HVE&J}KqCrE^40rLkE#}-dNdNTodYHYL zt8#mGT{tbzx~9SSezmYcoa$5imj?;{jOUT{T1~3z!g?m4@G{v}7iw|8ww!H2KqU-@ z>9BLQQ&PKJ+igGQQeDkI+D?m&MvFZ11Vod14++*PUq5XO)|$=ASSru#gd4^S*G538*wifkk!0dLSGMDW{$0r4*syrq`*GgKPrJvgL{(U!u^?*Y#CV3&73oc!g= z6~jbVZ;R4ETa=iX5KKUY3rL0uA6)|*Up#IVjUF8bxS*TLmoNl9DN4BM4io_Z0CrV* zSCkD`#_XKNEj9$J*}5-TXQMebxOncqt$9`I^<}`B2#gj4*LaHui24B(?)W5pi5&0L$2o=1VH zRd&m-O6hfvcQV#Zte4g-{8;lg6L6jIJvrPH-h+=%Vy1%D8?};A0T_CB?k};pk(9WI zg%^;iO12PW=!%<-jERV|cOaL*NOwW+v3$%GRs&)w*g9c-H~_+uu=3NeDKwhfO;u+8YDoE-P$VxWR%g(Ovf7)6}ZbQYqz(|6k*lW`y%0- zq^NJ8l7d6>r~vBnV7zE5b~U*()y+lvai1cTewz)D?dC3^KG@<}e1j^fhSOuUHYG`3O=1@9}#Vv95MV zQX*d|^%p$5#;m3+c53E9CCGYjVMazD8&lvb{Wlv}C=o64@x=!wD==+5y;$Kzo&KB7 z5f5cnFh&>oX8Cy9;D&?A@FJbE0JZBD0-ab%p*fKUNr&M^n%OL>l%uIHKTJp;B3*(x z;(+HE+c^tK?}nPeR5X9);0?$6E*6epz8oEN55i{m?Ykx`y(+V2az`xlAv)YZAnE9JLplMbaQUShw8!&8>M`o;hwXx)J zL&pKvlE!COB&;=H-Flao2-%b+whQ&XoI&WPk_l3?>k7rcj6@^l#cH%>U?dq2-Z1RE z^6o^QO~3JaVgnAsH*md{Qyt+J$5R(c;%UiU9^$Fd=t(ZBwhj5Aii$LH*4tF$zF(z$=BsoR7OS?Js)sl4q71zCmx(E~t#0f+snj7n=~w2!B!O zUD%ZKo#$BDqNT_!?`75KrtYZ|GM$(JS2E(|0*cEGR>eD?uZMMo`Ia?UxZFP$N1n^B z*7eM$JuhsuebReV8m4UC`%5Jxlt9E=HsKz45uqRp_Z?GOT-aWb^X48p)(e+VBpzlx z71lZgo&e5c2khMQnWsBG&;vSW}I- zYwiY++674By^CQ&Mv`}~pN;>DZ*YKvK!cun^lf#a+Zfq;B*GU`ETHgSBiU3!U%~#SD}GRe9vG)Lc;RpCAzSRp}|k$!%=rd9Hmf`;`7il6ErQ6IcJd(-*rD<8)y^SRMe7S@#Q@9$NXp?I=1L zctFqQopjmFEMW+v4XmF=BEzWWi~bY>{K4Dp-GMI_e!o@0PWTLR1$($@sIAC8ZDSaMU8B6mY0jhv|fh)ksS87 z=F0p_R&AF>9Ye?|Avj4qE;va9}QL4S7@`^T#n z0tm(SF6EBv|B>vw^Uq9oeEt?Y6Y&=b`81UY87wgqHuoLW>M!C+f0x()^83$h9zW!H z1U9jf<^s@!LQV|#*ehI`-U&UcjVP?!16md!_1^Q3lVR7Yb%Cm(l6nFis>+0)?f^t# zpm9)s0;?O`Bk;)Sd?u~|u{(PY)iYM)cx550UOluXK2Gti{0N2JwkGcjwl(3tPc7Qs zG0?Pj)U}%7vzkceHYc<$CF#bfe}2V%o$C&~T6T__`oR18!(MM@2!lz)xbNT_N-^%a zI=h0)c2gGhr-Y#w4mTEr8=%jWY=Son3Di5=6pZ3G5nsx0c&$@PFybv&h%p02?bK7s z8GY$X^G+_5pnj1rl&WUuZO%bDtMB*&4Ht1S$A>K|v}=ge5B4rEVRAJ0?w%W)<@djd zd;!zKdF(e@+ROZHRbpjMb*+Yt{LQIIQXa{uc~MKo_ae6GJFGnn845eZZtq2;9*ZHl zI*0B6#nlR|gn%=P3m-c228sd}#^MUShP!P|{bCZ@Ii;Zig(kc*;X0!ETtmB|bi)bo zrL2I;>t!bWjDbdJsQ5U#SPo*X9GxyifR+#rCE(TR*x(ArX4_3&&hePX9Oq{alLRM(CDskXV<%;@-!+rRZ zATyUYWfaSn?vQv%&F4zD;OCsVwJ~+WH;l~uG6{z|5&1@(a@^hvMDMEp zH8|QEp~3x;FNL4YRIt3FXcB%~=AuU~8Wmv%33x5!9;DciKegUbjZhFosts5)ULRtJ zRXo@HY5j_=YI&$e%Q_djrOzS=Q9>-$jzJ;5W6}j^3ds(q3WMPQht}0aFeS)M6R5904RB)GaziqJ3JPk$zS)J5suthh7TT{PY+9n zMi!0P+cZZ#kkblougRIZbc@_DqFOABZ#-eC?wB7vZHTXHlstq+W1f>9zxhtX-J;JO zmDKt}^}^w*ob->K{gYPiJQ>4b!pJ9^Q#r#gx@}zja5v2UypzYpHHE7n7pHB%Kk6#1?W+=WVQ+=-0 zd3^{q!PvL~3d8ZZ!v$CF8AWw53m(;d zBdR3JorLFDkeubtzHQ`JN&MJQBW0zpe{hakFynYH)fHQk_aNe`<+CV^(zFnT$YCtF zVKDT3uZ(@toj`oBQ-Jl!-JpgjDLSVEKn1LHT!=Fe0I72(!=?3?_V$m?de~KFdWwqY zH#@k(i6x&Bu8iDKFDHHmb>>^~(XRPNjA#66O#>`DUj{!O;4a%x6SRevX)`u2#$6nSy##z!x=KM-;%L+-UsT%NBi;9?5hq|aikJ=D_& z(uz!_(Dd<#Aoki+mP9y@cF(j&7c2AWkr0d8a11={zr!^>T`<4)u@N5iCH=jq>df_! zmT8JoXu!kKg^+A~kq*2K>(vogfpn6Vn)8zR7(k_Oi4rgGr1{Ir)ll>1)3K+wYV`ei z5OX`Fo18D((pOF$Z{w->6Ddut7K~G1*V>M<+pOyLx29>7%5yn|g+hSy9-hyAD3BTN zc8=VNoo};4k@iPyeztEGb&s2DQjMHly!SeKk^G2Mo=2xz@jEQWG%e$(mErEbn5gZ% zjK=`|jHr@SUh{c|1XLn6sJ;C5g+rm&{8jDM7LkRrd_0SQIh37T&h+H7A9$W`WI<^U z0+VKT3N1rg*aRud_Awy?x~qTEp}VQ_uvc^E&~$ih&p>nQ{OMmsv35Un^Gcae2+KlS zKE_n{Fa%Y$9s+@|%Ok!N8X@i5Wl3%yQXluRiq~xOGA*poZRRf39+Zrp^JMbFUXy%R z#q6i>kU!Pe)vKrppEJD!pRXPd-+leD0+6tur;b*JDZ#HHG49G>AD~t zT~hS@|P;j5HkdvE0e`HV4*`dF>*#ksnyR$r0J#8Jb*P*LW4R_m_SHsA! zOXt&O%5QAMY($e$mp}Bej$Cfc#~aT1Qrrvep$XAMM#`uV$QE=pL#OXfWG22XCwaW- zGJA)A2w{+SIXmS|nI*OHbQ&i$dF|OigIib>*NZYyTJD~JgE%WXcilH1Vx&;%Lic~f z+(hr=vG3H&2@4=DgRZ8Bxj>B!PD%UItbt2YmUDJMG4W*awR!!g!Vy$26GNOpmX8h4sPee1pim3J z3RP~!_c4W{GyDv!RBykJkKPWm4w_q_d4!XP+7jIEMt4Kjuw*VuLX0hBA*h;pl4PE} z)`_y2*ehoKnyY7;26N^ zjc6CYf6OU7YJc+3wp-I!Y0We5)VgC3w+U9ymixTssRXY38|>p1WaO~_0{@|zM!<+h z72m$>@kns+>|z?&KeUGhn0J#^cMMq&Qa(@_U;{hEWE#O_pW5*M#vaGvAKneR2#I*6 z@w^BO&jYAal3Z!6ADX)GOu}>b?Q7EUo-2nxnaKN{E#|3xxx#S`8_}v$3nfQKC?cR& z#!l$T7t9y~sGAeciQMOvP~mbee}rC%$?^#|zWmvIv#=DKO>?x~!_zPpYI_|eI5wu| z9_OK*5IVrWc=72+LHcS*UG7L(CZq00LVrw!tE}}AB86soh=$FiJC_SSLBcAO3_eaa zFfzjRp4(^;TH$AcGJ7LhHS##sgDti4p2p)Ltc?j3x}&~KmLdD2Q#euxa1m3N;8ZuV zKdQNO=#^f~@I&%=X-K$DwqMKxgjEfqy)5cRy@PI*&aAigFMs935{bys>k^;90<~x9 zktpJwSEo!)uqo>Ry_G&jOms!InQx^bDKVAg_m9l4^)|oyy{oQ12s2YGa7@)C?J}?VF(w1~M z4ET1*oYu=L`l`5bd|{`duacS1O_D%kbHSGuBTA|pqq-;$ya=L(UrLP$!Xj|`k&*VU z<_;!?n#Vmxo{cwY)#rPg{_yIm=5n2Na4URhXx=sWEzcJkP3FbWxP~2V4O5s_&(EjF zLiZA`92>rslf;!_41&$VXpacWw&GDunNkH6k%rgPNC$`Sm<>JsM-bpYdghONM@IJK z*Nxc6^NsCMeQv`x&m!wabvZozt1r_`iAXs$m=-&bpH?7*-f8)*^OUgFZ}4N?^z`OJ zJZ|dQcsu|OtH`P|q&DPZf?amk&7mlRhz}jfN8ZA!B*}UxQFFU`t_weB272BDps9kf z_B>|sYJ5B$rp>N|zDF0{$qwlSUA1j8D3nB$Kc1DG?NXxkvZ;_Vl@!hb)P_r zWJA1~# z_gDPs8lsX7Csx-|4iGr5g8Gxre@*WzzNXe^S$C-G0ZPaUUMK^Mv~CJ<^g2LYUfT3G zD384Vd&owsY!OSb)fhVHiy|h3nt6NS235i|eZ}A#o^G)EW#v&wuY?yU<@A>ykMMS> ze9^0!W9jaN#l;@!Ts%*AT|xrqka{w$?Dux+oEN$^itC<&E;w3vVkp#wJNl)NYZi$f zR^QlIk2zN+li+}P%|xP43T;Q zIM`^zY-`hLSGDRy<1cf$p>eK2Qhpzbit3qEZ1Bqeg{-6t5};UYM}=sjXf z<8C#)=yqbLNhV#;*Gd}zcz>*xmVQWS!q!6(Rpl}5T(|}n~UwA`EXnCtAF|#;Jt%_4oLVx-2bqJC_W}0@tDfJ#}Ta~wf z^9O;aScBbQM+t=mTlgnuSKolU>94rY8Q@c(#d-Zc_28}B$~ngOc}t|ZAqqLzIr$VF z=SaG3>ELAO-WqkoUo1}R`1up*zt_2IW-~TF`JoZWcn>)c7M}Y3MK4K1D13uD`R2QS z@gV*4rVJB|J+%FBJ=>p8`9<$QbU<9+w9;!PG_?+p~ zWo@m&X4zmK4A&oev!N-K_DrYuhl`!PnN7EPuY?HfI}tN14^^!6o&|-R973%Au?s5P z>u4ImV1X@n`k`o~Zr9bU1c{pEBcbw@8p(uM$Y5V|r;)<3Cc`MXp6t&(A3D!Bm>XYN zRLqf9WO823%;tPXd;Schml+@aj&{< zZQh9IG@&&QN0U|WC6^XwQkF~0aJ;VZRG6(0hNRZyT-Z}r&d;l{-lR}U@B|s1;P1z% z8?1p^OU=P6vDB83$i_(Sha}W(rzGW2*to=|G?jcNFmqwK*q$Cpv^dWO_x-@2V<lPfo%8d!X&Gyu!UwP$79Bp|Yhn0>|j~2bm$}gM4WTssLpK zu{0x9(zY-gQH3A|DrkDyY{q}^fSL3*GNtf!RhgMa!_1;mgr#~@^Mo@sVt%~*lc+B{ zs5x^09xm+&tIMFdYQkWUGN4AtgXHqPNL{@EC9yY=rnjqfxUNIh=}0lDp>i>?b1C;{ z75}Ff^I!F`bY$R{0-pM?9$dfPI30e4Mc6+|^P|kJG@hHbz`%hL2B`O=ze*jyV6zC> z{G6zv;W=NlIH!%#tMtshMY0b`sS>H!@uYdXIRF z=sI6C`uJta$q6bUX=oO^uTe_2x6`E?9tMnwQfy0wLils4C3NiL=(~}cTd{>k!!j^g z1PW=A|AwZO(de*{mZu0tmKM)tM!PA8Y0DFPBqCE<3dHX;-tm3>e?J@_8`XKYQpF*E z?~$DF6}XfB!t|Sn)UM61iVcBJCa#kIbM-&1@gLdvZ|e*Hm?HkmgFmy^2!Xf(4Zp+$ z-Xi%@L`q2Jlj!1F;u^^8lx0LQ-R53NMLF3fPWE`$ar&(}!St~A4L}TW#c1(~+hkcO z1+x$y1BB<}rT?;LW>I!*dTI z4#oLfol;n~0QN$l7E|#b(>_WrY|sMdh~$=0Rr0kQNV0yK-sP38!^L_eC$?8;Sq z6zt3LSKm-eK{AsL1sZd?hv$ms3ciV$b)eXD5hM24R84w<;v05+JVM~MR;fkz6D7O@ zNf#Ea9Wi&mz}nwq>|d7v1v8|5frcqvei!PNvL$TqH6Kl)&`!3HvkSrO0}KVbAMLqm{T^ zDJwvEW{qc1hmjt&MEqb`b6m_C6e@naVz)j>L(&!((>fAaN~IS=Y#w@JPtQW5?xGB} z_I)~~!)AWp8~%9Po8J{fz(^UXghn5DMGO-&=)!q%y%=62$pet{##{YyN<9@xD3fV2iQ2ACd1_+sl`zKEDy}@mv#EYCY1-eL#y? z@WmLuKm1;s0b777PGkDP5oH)5Jzey13|7BJPL4{|C!9;`cti3CV%9vXjr^r+cRcy+ z_G&eg=;zZYj{gv!RsZ{`#}Q}?TWN(!o?k`KfeOH_BUz=|W)-X+416?O5g`a+< zV5#O&;GdXxJ~plFNDj{_q6rGJHro?pt6MoRR36cG8)*LZQS5i=IlBmS>zJA1^E_Pa z3B&d4w0>D*_c5Cahh}B*LjUOzP;@bqJM>H<5!IUJutmHOdN4Owcs4}xOidw!Jnfizl;%s#4#3`}aPN{|{NtuGe;BJw{@5Nle zT`*I1u*J?pvC14>6#OMBddjHw-X~cp*^=7fw=|98u>iM9NJ^Tiu_7ozhfnBf?K!mv za5PtG^IVeZ z4Dt+MQ(Gc(>dUXozZHANF-8`G$qP#m2sg&I{D4wn#m%la?bNAveapZ8+q+6PpF8F) zuIcrtT5|TWb{5=OF>KSdgwqgKX3pMJvWcc7fZy8{x4MvS${xjOA#U!s$z@#lz6aj) zzC*8i*{-@fGLju(_?K+zQ(dixW9@FW80FJWb$A{11bA^ii5f-<%QH|aaJ&;AUl=y9 zZXWBhuXOeyN@U+PA@$ITR%P$^yB3NYy!Thh$N!frF-&^k=d^l0Ro!w76>Z zq`=Bw!$mFhVwOfG6_W|1l<)(b!R*Pdk~9G!M zy~j(7C7|-Y_QZ>V&BA$EA}yz!U`-*$$a>755|v}W=B9e~QeuL=4n6=7BP0}S zC5DXc6Xwr&A|Fu_&Fzx{?HxKQuna!F!{2}*iI)2a8jq+8*(G!Xg{yN~{`T0L_eCj3)HP>N!r6cLcet3>owx#^3hg zbQphzQO)|HtF4jU2=v9H(3vAge&}2jG$KzVy+D+^w^zg%%5#idcN_xq7g6E*DLezYh*JR-h@3{Cg zy*-BgSCPEPib7{q6@|*O`>3tPtEV?lf?v;@TN)d5$Ta18QZ$$yX4VqX+-m2iY#x5d z_R5=n1uVTMJF1Z?G$b_FkgjnzV}mp$-QSFUj<0#Aw`QKwp_HzcJP&;@bhtKuzKN^x z(gf<=(2U_!L<0{a^Nz^NY)cau?1?iSb=e-iKhPLZhR$NQr@sPjb7nA(4}a-fIyHjp zI9(xqrR+~Ej=~XXPg#PEf6Z`N)H_+Z!OqqM9ZJOj3wI9P*dBLwOyuU=V~J>{hH(E` z7MLF-IK~|q(Ryv(|pRn{syZe6gzB^lLr2w8$GDD}sg_%o`;5}wkkXB!}6itdz} zRQdio9Uv>0Af|nHs)K)Rb1Do-09RG;_@9Bya#P_xe`EORD5dmeT(_&xRiYI|0?=+w zPtkrGsLYqF8Ai+@!}5WC^om^R&iNoYAy1x|xMma-b|9rFQ#OtH`c3$e@=$(PNTrS6 z%Y$H>PJQb)V^$3gvX{B!keGHt$*XQ6fju+m((Uc_o2~Av+RBtVB1sV?=~$4G+Ggm- z6-4n#)o~}$Pd~|e%B-k)94cpC&dyYHsEBYhKCql18(5QA2pasEHXA(#AOO*)>g|Vx zQPW7z2SiQrB-B}M{h}aeB$C|VAvm$)zEiMFy;kn4kfny>OxjhQHUVYqed1MGF+w}t z_f9?EY+n>A$td2ykMTM^IUVbeGNf5xWGzFH$Lc&2Gd#Crat|sC^q`DE?GG=JuuAv* zd<=d6ItZa;DA^PDyavYo{w^!u$AH^x?KMDl>F5IwYpk|3#pvd2YL(B~%fLDzEo+vu zP$>bazjm=b{Mxcw;B^Rt_z3Z8+hevGMuFPLj&u#?+bpyz+(Ag3%pUr*)(C>WA0@IZ z6Rs}m_G(G0YF0FPEQIE!>fK7}2g1RFeVrPnP>&C{1U*0TESoCKRC{Wkru4(4en6BhCT zrolp{37Fk*ox{;!7VMkIIBqFi7`1u7N$;HnKLsFd?U42C*o)(-(lM4&6Y7Ww74E>> zgD<)rp6d>IK-D2mU=I~H&|L4!9`DXoyvwH z$8wd&{PwlR$|b zFx|l_B@%-_Fdsn1DCxyvDf0aWPaGm}3+)is6?zscWV%1pILl8>pc}!b0A>Y{YKNXI znA*o0yXhHc3-sGl<5OM3$vPO`wyNgw1bMUS*`ceUn^Lbd52xyO*~>j2TAE(4te8co zECJ|IEu@yaLfQkvw9^{ZFg4L->9fE)eIQxlGh;1L>1eu|nA}V7$M@?Ar&+KJ{M3Rx zb4yx^D;O9XJ3i=D!I|^pj5*SL+IQ7|ONNdN%RepvgJJxHcsZr(1=|ngChluq?)Xh` zGdtQTlzWsTf97_hyQ&XUdoB;*^}Cw;D@P{Dpk%bn1UiEWmv6JpKe3G!OP*3zxru8gH2i)Sk%|aYYk2vmm zN-cWMN;i((?HRujC=-`(I7Z{`Owi*I&q_CGPn#Wtl1+ifg_YP-y=M$DAoxyE-YoaT zCPytEq#HR4ulEpp*&{ZdLSP)JfUqm%3rM{?9_>DcBPnh8w~K| z*xbm&{5;K1KmxC^l#lgKJ z-nht8Xfe!zx(#(|L-e6M>d$s+h3X-8@>l~>0s7&Gq41TF5=`h~bTtzWKqAFt@4+W7 z=IyY5I1j>RzYalDsY3W6Y89~a;RsjEWr3E?<5FWBGMaibF1{;yUE1hqe?6xXy|!7g zSkvl>*Z5252vr&`dK^n4`392BM9W}O`98K=i~h?NAVIA6;Sy2JKox_Lj3Tj7b9=Uicaw{9l_K_c$pPNl}BHiZb?ZgM9}Vf+{S$(;jV@Z2mVIy!ejg3X6=(~>=!#~vrJ{iCAO=RgpbW+j*UkeMaFPM^6cH!V% zWhe6)xe9UiFiH(~<-;17w^8yw;zCoAWA}RO=AG&%<6}kNL{iPZi8R~}l9v9e!CU^6 zeerWf&SWN0pLRweWpWyyJ@F*JCF^DRt*C9!i?An@^)K(h|C;&!U!Ozx+jF)9Z|f&0 zr&@ds+jKmZQScSI>-Cj1xRz+Y`zYj}1wF(6q5Ydk8YdW1C_XuK#M*g><~#Zz6yEc?j@>Nok-OYcw5(O|79r%R*gn1yG%Pj# z;o~QQbX4+i@?=cpo}WjA;_xq@KQ()bfe+>VB9it9G)cYlSS||c{PcS{IdAa;B75l9 zAC)yce?#*QsX$@{p!f917$WJom)eR^X2>pO{o^lnP6}7#mfN}qORDpjqm@=9<>*14IMe!#MjInZ&i-)|d2hdeNmU&u zBaflv`!lSi2_)s?VB*4&YEI|({V%clml@(JbeDQtVR&r0)WZ)Pzo#cRd7<4FL}$xn zhE&0}oT0c7-0iEP!TA?ReH8So zy=1#lba+tehNsTyt*znFUO_+?ba4{9T#6y?r!th%GEEH`36e5)p6z#niiueNkV2*7 z`3H((0hD{6q+6EO>`(LF((CoZg-C7)Q}@QPi)ImOiIib}a#5Xdb9=ZTVz|4{H~dUa zUfdpz?=U^tc8_ie(8uUxvm+c5EvnPvW)k4Rc_}QyTh8E^wKM-NDaNGJXf@rHmAeyV zV06l@d02X+34T!nGSK?=#h}Zw<-VbGw;T5cSm+xHAq*`Xc0LjSWE|imKu2L}G)Jvv zS=%xwt0k9uU|HJ;fk#;Cxl^FsmKHRs={yklLE_w$U8VNR+Im^uVB11o+Thk!KIiSiZ9ePvk+n{)~R8goEExbGnxyAlVYZ%uX@k={b)RpqDO>b-_eg#taYzAOj^{qfi--n69x-j zF9pvYFx`gwJ){PPt!9-$@7k_xB$Q>HdJBwt)-{wW1W7&j!}UMqu_tVe-hTS%`-FE9 z-$cG^**f!42nO}Xsqn;(FNZ{w|2QN4$F=W&J-949^}~O%|EFdCUkbq=Gn4`4SqIR- z;K-h?A70znNnez;Zv|$~ghiF#OYLz1c(0HcPGaD0Z$z-&!r?>Wu*azj|s{^-x$c?1UotM+u9f?awBw4+?Skpq@2zjz{@b+A9?_zE6 zu7}o_%U5@z#9u5q*7lz{@qQct+*SS<^y}LvHQl6*kavIN?AUax`&yR&^{m#e;f1SF zHD7bze-p`=yZ9UEhmGBw(2JCSZz3r|4b9==U+($K(>uDBP#0}K-tJg0dPe*$;VXNy zh$|#fcqt4CBwYk|x^Uu|S3li-vOY1jWqhXW)5VD$$M=2DT1lDd?1opD+k4*UeMhJ`_NTcL3$?Kx%c!Vf{tSTn8PXnd>lL5r zUbt9qKEtt1-+**c-6FcwI>n6eLd8G%_25V?Lpow=yDDX6tsR+~Lf4+9%JjV-#LE^L z?)*b>`5u`4*kz#09wzReHea|lajs*Ygs2im(0w@k z>V$~MpE0wtP`E>HUQo=Z5Njg9@t;pa`|c<{--Ywo8W%OWeIg z=->mXCf-80pRPM%^m$4?AS5=GUVo@FU&E*xP>#V{|6B&!eVJpyaBqW?fD02zlDz;A z4Zmg=51}uRb~i}f3Hpj}eMaKK3O!0zA%%-{XJw(QOg2@DoA!EOdU~s*o%>R$DlUv&~IVJM$ zRF2IJi z)A5MFm%Q_NT@tVGSv1ub3dMx|xv z7tTTr`E4!dmw&v$|Netp$nc7ypDte7rnlDKlJ`gq`0~r{{%Ph;_a`9(c>cAEH zLGK?AXp}Ko3F(lmA$6pDS(wX7F)3C-o`YsP^i_(lib+N*0SSH=k>2mjce6D)&?dCu zUT-})^YsC)b65HeYMv<>yjJt^#r0Zb!BAv zIt)^Zc^>B8DYb~IA{6|n7K`m#L>(zFc?;&4R+5UVfddnI7mp!-AV1j!p43N<)E6V+QlHGtOF} zo`pAf1e14K%sDW8jFg`Zndp?nX7jHq*Q2{Ev-vhE@1d<_@718>3;eN#Kt3WNF-K)F=0wT~OmU!FY5pnP&7rw;-SU&oH`Yw=29&GyOUc~iYiL>^e`#|;(v8I7R zD{PD8ec2%~7Q|RhjtisR zysYn<-u+?;gwrkM8V$>DJVNrPAV@6KfGT%q=O}JqW*_50j2XZS!yZ3y20&B>da5)y zzFu_ZfM7yfusA_j5EiM>Fo7 z&VzZMxwq!^5M&AXaPFaAf(z#=X|>oiX)2_&gkh9Sq~hiPohGTr``a^6CPR!@8<21z z(lF&!KPuc$k5J3@@mz8-4$Udgq^i<1qqB@99@r#mq4nnM?GkV@EL=3z7o87Z|Lpor z1OORsUli!``ugodxSLrqh8YC?<>6v)h} zS*Xpf%Pni{0}n#VppKT&F+Y{`*Cui~!}4Dy9ZbacO1pFj4jWt7=#8w&$CyB)=X6Tq zr+naX-Oz>S#PY=CjLDMlFpoe*nZdD5r|z61UxxU=_@T?_09cNlUtvSprtf%>{=l2d^^uoe8xA+Mf*}xgJSi@L&xgIzOhZ7YmEaIIjzZ zw(DSI9Ib(Bp=f3a`$$(%_>pwEq#yd+iI#}|V;=_CEtrJIg)G;?a=yafZQI{Np+5~c zcY~p7a!DuDGp#mWxN_{M)D5|_;Hx9u`QJo5P+ct!nH7p ze3>X~jpO|;Z>P|1!Ia6KyBH<)O!gOmW-`MV44^yIO4lZ6FcOUGJUnA#`n5D^<@Z$H z7h=IoqKoC-p^Ct~4A#T<UwR=-_CWRCOV+?gbYdYv02)ZD$U0Q?>0Bf zGuT7Di@1I2#|F5-uB%tR(2ApoxE1XNxE%HJh7ht5GU>bUg#a7l1reJ$Wl)oZi?9$1 zB@$Q_Qez~M?mk3R&ToQiu?k5rf2L&6wF(}sZJRUr?$`n_k?&%iZL#jAI*x?V#dNO= zV3LnEp8h*#{*NZB6ef2>$1R}Vy`8rxaCpOD$v9TpBz%~b^brY1zg3y0lJ-5j7RfiQXmt~uaPQQM zdk|v}G6Bvh`aU$mm8$hvvK=omhD)XU-f7 zwRq1C4gD=Byv~mF?oBy~kQ)WI_OO^EV!{mZifqFhhG5$!adbr2qx6Z8@S_zmdOf^i zGCqeVJ5*!TIV1_JLG+7W=dPzUWuZTa9~^n{q|U?t$dgW%z+YO(P3HUpya#cn2~kQN zGs?w?_u4%~x1_dQsR-jWbsH<_(TB&lev%Zh1)GcKRC_FIfy1AZxVeP;hb%-8yFQmLQ zwHgR-jH>SL@u~+*LF2iQqjby)aA4~;$!fwzEqZ03LoN_JIi_c=!@c`f`=V|{*|zD( z=N_)WG$f?{3cV0xVOY&2a{8!34~}&A0;p>W9Rx)*HV*>w!=8Bn==laiqYts8R1%&^ zAy1<66tTT7QwEyZk^1)!1CQu+J%%j{j;RSYnIq?&)Zpy-cu%gqPuOIJ`9nCcHC_!X ze$++(z{5TIml#{F;T%o^2vPPZsRmE1JkjP{)4lgPEwWbMM$TWRz@3`XY_MjOllY|S zYAs?jd3fKEN`V>hIoc}FO(7jytAv~*OL5MwTqYSH^xUsQGU! zNeZcCe!e%CgS0mdl}*OsVEb*aRKHq}G+dBTNr%AXrD>WF5L+2R59bP5PEanl(e+Lb)C9(?Vt6M5u zjI2zSn16A5(zDRN$2_xVpE8;rgfm zCC}5+Gm$RD&p%#DQo6MF`b)fTI=P+XBj>#rbM0T~A&%B9Ln9f6XJ6 zY+^aoz+WQjZuF@8YD?2vT6w@7tZHM?^f>0EZ1>s zihsVf1hJsOJG>>%56c1kh0t9xVC>9oUd!+zCHO?bEl=~h&Y33;@rd5})xL%=pCZS@ zU1%oj32wJoR)RTTOYh)MHIu_8t8Ifa*2)N8w z^Qx2BK{l_XuTz4`v7kALYglTXZY}N$SwHO*qAxl)r$RfMnW=WO{vdU9D!sZo!#9#V zX~%GyCog#+CX`WWVN65~o_RyFFN{nK;Q}@r)5K+{=};k%rp$E* None: + for item in contributions: + if item.table == "LiteLLM_DailyCustomerMemberSpend": + await _upsert_customer_daily(transaction, item) + if item.table == "LiteLLM_CustomerOrganizationTable": + await transaction.litellm_customerorganizationtable.update_many( + where=dict(item.key), data={"spend": {"increment": dict(item.increments)["spend"]}} + ) + if item.table == "LiteLLM_CustomerTeamTable": + await transaction.litellm_customerteamtable.update_many( + where=dict(item.key), data={"spend": {"increment": dict(item.increments)["spend"]}} + ) + if item.table == "LiteLLM_CustomerMemberTable": + await transaction.litellm_customermembertable.update_many( + where=dict(item.key), data={"spend": {"increment": dict(item.increments)["spend"]}} + ) + + +async def _upsert_customer_daily(transaction: LedgerTransaction, contribution: Contribution) -> None: + row_id: Final = _customer_daily_row_id(contribution.key) + await transaction.litellm_dailycustomermemberspend.upsert( + where={"id": row_id}, + data={ + "create": { + "id": row_id, + **dict(contribution.key), + **dict(contribution.values), + **dict(contribution.increments), + }, + "update": {name: {"increment": value} for name, value in contribution.increments}, + }, + ) + + +def _customer_daily_row_id(key: tuple[tuple[str, str], ...]) -> str: + encoded: Final = dumps(sorted(key), separators=(",", ":")).encode("utf-8") + return sha256(encoded).hexdigest() + + def _window(item: Contribution) -> WindowSpendTransaction: key: Final = dict(item.key) values: Final = dict(item.values) diff --git a/litellm/accounting/projection.py b/litellm/accounting/projection.py index d1e461efca9..2c11372d7cb 100644 --- a/litellm/accounting/projection.py +++ b/litellm/accounting/projection.py @@ -14,6 +14,7 @@ from litellm.constants import ( MODEL_INSIGHTS_TASK_TAG_PREFIX, ) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import GuardrailCostByUnitEntry +from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, daily_spend_entity_ids from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks from litellm.proxy.db.model_usage_rollup import date_from_start_time @@ -202,10 +203,53 @@ def project(row: Mapping[str, object]) -> tuple[Contribution, ...]: *_window_contributions(row, metadata), *_model_contributions(row, metadata), *_guardrail_contributions(row, metadata), + *_customer_usage_contributions(row, metadata), ) return tuple(_capped(contribution) for contribution in contributions if _indexable(contribution)) +def _customer_usage_contributions( + row: Mapping[str, object], metadata: Mapping[str, object] +) -> tuple[Contribution, ...]: + serialized: Final = metadata.get(CUSTOMER_USAGE_METADATA_KEY) + date: Final = _date(row) + if not isinstance(serialized, str) or date is None: + return () + try: + attribution: Final = CustomerUsageAttribution.model_validate_json(serialized) + except ValidationError: + return () + counters: Final = _daily_counters(row, metadata) + identities: Final = ( + ("LiteLLM_CustomerOrganizationTable", "customer_org_id", attribution.customer_org_id), + ("LiteLLM_CustomerTeamTable", "customer_team_id", attribution.customer_team_id), + ("LiteLLM_CustomerMemberTable", "customer_member_id", attribution.customer_member_id), + ) + return ( + Contribution( + "LiteLLM_DailyCustomerMemberSpend", + ( + ("date", date), + ("customer_org_id", attribution.customer_org_id), + ("customer_team_id", attribution.customer_team_id or ""), + ("customer_member_id", attribution.customer_member_id or ""), + ("model", _text(row.get("model"))), + ("model_group", _text(row.get("model_group"))), + ("custom_llm_provider", _text(row.get("custom_llm_provider"))), + ("api_key", _text(row.get("api_key"))), + ("mcp_namespaced_tool_name", _text(row.get("mcp_namespaced_tool_name"))), + ("endpoint", _text(ROUTE_ENDPOINT_MAPPING.get(_text(row.get("call_type"))))), + ), + counters, + ), + *( + _spend(table, ((column, identity),), float(_number(row.get("spend")))) + for table, column, identity in identities + if identity is not None + ), + ) + + def _indexable(contribution: Contribution) -> bool: texts: Final = (value for _, value in (*contribution.key, *contribution.values) if value is not None) return all(len(text.encode("utf-8", "surrogatepass")) <= _MAX_INDEXED_TEXT_BYTES for text in texts) diff --git a/litellm/accounting/snapshot.py b/litellm/accounting/snapshot.py index 874b90110c1..b86f37c3f5e 100644 --- a/litellm/accounting/snapshot.py +++ b/litellm/accounting/snapshot.py @@ -31,6 +31,7 @@ _METADATA_FIELDS: Final = frozenset( "compression_savings", "internal_call_origin", "billed_retry_attempt", + "user_api_key_customer_usage", } ) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index d1ed9f7b62b..2c9d64d50dd 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -308,6 +308,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_DailyUserSpend", "LiteLLM_DailyGlobalSpend", "LiteLLM_DailyOrganizationSpend", + "LiteLLM_DailyCustomerMemberSpend", "LiteLLM_DailyEndUserSpend", "LiteLLM_DailyAgentSpend", "LiteLLM_DailyTeamSpend", diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql new file mode 100644 index 00000000000..f2746f41854 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql @@ -0,0 +1,47 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend" ( + "id" TEXT NOT NULL, + "date" TEXT NOT NULL, + "customer_org_id" TEXT NOT NULL, + "customer_team_id" TEXT NOT NULL DEFAULT '', + "customer_member_id" TEXT NOT NULL DEFAULT '', + "model" TEXT NOT NULL DEFAULT '', + "model_group" TEXT NOT NULL DEFAULT '', + "custom_llm_provider" TEXT NOT NULL DEFAULT '', + "mcp_namespaced_tool_name" TEXT NOT NULL DEFAULT '', + "endpoint" TEXT NOT NULL DEFAULT '', + "api_key" TEXT NOT NULL DEFAULT '', + "prompt_tokens" BIGINT NOT NULL DEFAULT 0, + "completion_tokens" BIGINT NOT NULL DEFAULT 0, + "cache_read_input_tokens" BIGINT NOT NULL DEFAULT 0, + "cache_creation_input_tokens" BIGINT NOT NULL DEFAULT 0, + "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0, + "api_requests" BIGINT NOT NULL DEFAULT 0, + "successful_requests" BIGINT NOT NULL DEFAULT 0, + "failed_requests" BIGINT NOT NULL DEFAULT 0, + "total_response_time_ms" BIGINT NOT NULL DEFAULT 0, + "timed_requests" BIGINT NOT NULL DEFAULT 0, + "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "LiteLLM_DailyCustomerMemberSpend_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_customer_org_id_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("customer_org_id", "date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_customer_team_id_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("customer_team_id", "date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_customer_member_id_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("customer_member_id", "date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_api_key_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("api_key"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_model_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("model"); diff --git a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma index 2c0246bb94e..bac27daa5de 100644 --- a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma @@ -1035,6 +1035,45 @@ model LiteLLM_DailyOrganizationSpend { } // Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyCustomerMemberSpend { + id String @id + date String + customer_org_id String + customer_team_id String @default("") + customer_member_id String @default("") + model String @default("") + model_group String @default("") + custom_llm_provider String @default("") + mcp_namespaced_tool_name String @default("") + endpoint String @default("") + api_key String @default("") + spend Float @default(0.0) + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@index([date]) + @@index([customer_org_id, date]) + @@index([customer_team_id, date]) + @@index([customer_member_id, date]) + @@index([api_key]) + @@index([model]) +} + model LiteLLM_DailyEndUserSpend { id String @id @default(uuid()) end_user_id String? diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3ee3ea92d7b..7a791ff7876 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -726,12 +726,15 @@ class LiteLLMRoutes(enum.Enum): "/jwt/key/mapping/info", "/management/v1/customer_organizations", "/management/v1/customer_organizations/{customer_org_id}", + "/management/v1/customer_organizations/daily/activity", "/management/v1/customer_organizations/{customer_org_id}/default_budgets", "/management/v1/customer_organizations/{customer_org_id}/default_budgets/{budget_id}", "/management/v1/customer_teams", "/management/v1/customer_teams/{customer_team_id}", + "/management/v1/customer_teams/daily/activity", "/management/v1/customer_members", "/management/v1/customer_members/{customer_member_id}", + "/management/v1/customer_members/daily/activity", ] + key_management_routes + mcp_management_routes @@ -3855,6 +3858,7 @@ class SpendLogsRouterMetadata(TypedDict): class SpendLogsMetadata(TypedDict): + user_api_key_customer_usage: ReadOnly[NotRequired[str | None]] actor_agent_id: ReadOnly[NotRequired[str | None]] target_agent_id: ReadOnly[NotRequired[str | None]] billing_agent_id: ReadOnly[NotRequired[str | None]] diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 7d413121b56..16b59657d80 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -354,6 +354,7 @@ async def _sign_in( response = await generate_key_helper_fn( llm_router=None, request_type="key", + actor=None, **{ "user_role": LitellmUserRoles.PROXY_ADMIN, "duration": LITELLM_UI_SESSION_DURATION, diff --git a/litellm/proxy/common_utils/customer_usage.py b/litellm/proxy/common_utils/customer_usage.py new file mode 100644 index 00000000000..f356fc0f2bf --- /dev/null +++ b/litellm/proxy/common_utils/customer_usage.py @@ -0,0 +1,187 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum +from typing import Annotated, Final, Protocol, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + +from litellm.repositories.prisma_protocols import TableActions + +CustomerIdentifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, max_length=256)] +_IDENTITY_FIELDS: Final = ("customer_org_id", "customer_team_id", "customer_member_id") +CUSTOMER_USAGE_METADATA_KEY: Final = "user_api_key_customer_usage" +_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +class CustomerUsageInput(BaseModel): + model_config = ConfigDict(frozen=True) + + customer_org_id: CustomerIdentifier | None = None + customer_team_id: CustomerIdentifier | None = None + customer_member_id: CustomerIdentifier | None = None + + +class CustomerUsageAttribution(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + customer_org_id: CustomerIdentifier + customer_team_id: CustomerIdentifier | None = None + customer_member_id: CustomerIdentifier | None = None + + +class _Team(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + parent_org_id: str + + +class _Member(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + parent_team_id: str + + +class _KeyMetadata(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + metadata: object = None + + +class CustomerUsageDatabase(Protocol): + @property + def litellm_customerorganizationtable(self) -> TableActions[object]: ... + + @property + def litellm_customerteamtable(self) -> TableActions[object]: ... + + @property + def litellm_customermembertable(self) -> TableActions[object]: ... + + +class CustomerUsageError(str, Enum): + INVALID = "Customer usage metadata contains invalid identifiers." + MISMATCH = "Customer usage identifiers must belong to the same organization and team." + FORBIDDEN = "Customer usage must match the customer organization assigned to this API key." + UNBOUND = "Assign a customer organization to the API key before tracking customer usage." + NOT_FOUND = "The customer usage organization, team, or member does not exist." + UNAVAILABLE = "Customer usage tracking requires a database and spend accounting ledger." + KEY_BINDING_FORBIDDEN = "Only proxy admins can change a key's customer organization, team, or member binding." + + +@dataclass(frozen=True, slots=True) +class CustomerUsageFailure: + error: CustomerUsageError + + +def normalize_customer_key_metadata( + metadata: Mapping[str, object] | None, + existing_metadata: Mapping[str, object] | None, + *, + administrator: bool, +) -> Mapping[str, object] | CustomerUsageFailure | None: + if administrator: + return metadata + stored: Final = existing_metadata or {} + requested: Final = metadata or {} + if any(name in requested and requested[name] != stored.get(name) for name in _IDENTITY_FIELDS): + return CustomerUsageFailure(CustomerUsageError.KEY_BINDING_FORBIDDEN) + bindings: Final = {name: stored[name] for name in _IDENTITY_FIELDS if stored.get(name) is not None} + if not bindings: + return metadata + return {**requested, **bindings} + + +def _input(metadata: Mapping[str, object], binding: Mapping[str, object]) -> CustomerUsageInput | CustomerUsageFailure: + if any( + name in metadata and name in binding and metadata[name] != binding[name] + for name in _IDENTITY_FIELDS + if binding.get(name) is not None + ): + return CustomerUsageFailure(CustomerUsageError.FORBIDDEN) + try: + return CustomerUsageInput.model_validate( + {**metadata, **{name: binding[name] for name in _IDENTITY_FIELDS if binding.get(name) is not None}} + ) + except ValidationError: + return CustomerUsageFailure(CustomerUsageError.INVALID) + + +def _hierarchy_failure(error: CustomerUsageError, *, administrator: bool) -> CustomerUsageFailure: + if administrator: + return CustomerUsageFailure(error) + return CustomerUsageFailure(CustomerUsageError.FORBIDDEN) + + +async def resolve_customer_usage( + *, + metadata: Mapping[str, object], + key_metadata: Mapping[str, object], + administrator: bool, + database: CustomerUsageDatabase | None, + ledger_enabled: bool, +) -> CustomerUsageAttribution | CustomerUsageFailure | None: + if not customer_usage_requested(metadata, key_metadata): + return None + candidate: Final = _input(metadata, key_metadata) + if isinstance(candidate, CustomerUsageFailure): + return candidate + if not administrator and key_metadata.get("customer_org_id") is None: + return CustomerUsageFailure(CustomerUsageError.UNBOUND) + if database is None or not ledger_enabled: + return CustomerUsageFailure(CustomerUsageError.UNAVAILABLE) + member: Final = ( + await database.litellm_customermembertable.find_unique( + where={"customer_member_id": candidate.customer_member_id} + ) + if candidate.customer_member_id is not None + else None + ) + if candidate.customer_member_id is not None and member is None: + return _hierarchy_failure(CustomerUsageError.NOT_FOUND, administrator=administrator) + member_team: Final = _Member.model_validate(member).parent_team_id if member is not None else None + if candidate.customer_team_id is not None and member_team is not None and candidate.customer_team_id != member_team: + return _hierarchy_failure(CustomerUsageError.MISMATCH, administrator=administrator) + team_id: Final = candidate.customer_team_id or member_team + team: Final = ( + await database.litellm_customerteamtable.find_unique(where={"customer_team_id": team_id}) + if team_id is not None + else None + ) + if team_id is not None and team is None: + return _hierarchy_failure(CustomerUsageError.NOT_FOUND, administrator=administrator) + team_org: Final = _Team.model_validate(team).parent_org_id if team is not None else None + if candidate.customer_org_id is not None and team_org is not None and candidate.customer_org_id != team_org: + return _hierarchy_failure(CustomerUsageError.MISMATCH, administrator=administrator) + organization_id: Final = candidate.customer_org_id or team_org + if organization_id is None: + return CustomerUsageFailure(CustomerUsageError.INVALID) + organization: Final = await database.litellm_customerorganizationtable.find_unique( + where={"customer_org_id": organization_id} + ) + if organization is None: + return _hierarchy_failure(CustomerUsageError.NOT_FOUND, administrator=administrator) + return CustomerUsageAttribution( + customer_org_id=organization_id, + customer_team_id=team_id, + customer_member_id=candidate.customer_member_id, + ) + + +def customer_usage_requested(metadata: Mapping[str, object], key_metadata: Mapping[str, object]) -> bool: + return any(metadata.get(name) is not None or key_metadata.get(name) is not None for name in _IDENTITY_FIELDS) + + +def customer_usage_fields(value: object) -> Mapping[str, object]: + return _FIELDS.validate_python(value) if isinstance(value, Mapping) else {} + + +def customer_usage_key_fields(key: object) -> Mapping[str, object]: + return customer_usage_fields(_KeyMetadata.model_validate(key).metadata) + + +def customer_usage_http_status(error: CustomerUsageError) -> int: + if error == CustomerUsageError.UNAVAILABLE: + return 503 + if error in (CustomerUsageError.UNBOUND, CustomerUsageError.FORBIDDEN, CustomerUsageError.KEY_BINDING_FORBIDDEN): + return 403 + return 400 diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8abc91a72ed..5ba324e7154 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -59,6 +59,7 @@ from litellm.litellm_core_utils.core_helpers import ( iter_items, keyword_arguments, normalize_drop_params, + require_attributes, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -93,6 +94,16 @@ from litellm.proxy.common_utils.callback_utils import ( key_logging_entries, strip_callback_config, ) +from litellm.proxy.common_utils.customer_usage import ( + CUSTOMER_USAGE_METADATA_KEY, + CustomerUsageDatabase, + CustomerUsageFailure, + customer_usage_fields, + customer_usage_http_status, + customer_usage_key_fields, + customer_usage_requested, + resolve_customer_usage, +) from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export detach_request_parsed_body, @@ -2712,7 +2723,7 @@ async def add_litellm_data_to_request( """ - from litellm.proxy.proxy_server import llm_router, premium_user + from litellm.proxy.proxy_server import llm_router, premium_user, prisma_client from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields # Strip internal-only keys from user input before the proxy sets its own. @@ -2989,6 +3000,33 @@ async def add_litellm_data_to_request( if _metadata_variable_name == "metadata": data.pop("litellm_metadata", None) + _customer_metadata: Final = customer_usage_fields(_REQUEST_FIELDS.validate_python(data)[_metadata_variable_name]) + _customer_key_metadata: Final = customer_usage_key_fields(user_api_key_dict) + _customer_usage: Final = await resolve_customer_usage( + metadata=_customer_metadata, + key_metadata=_customer_key_metadata, + administrator=user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN, + database=require_attributes( + prisma_client.db, + CustomerUsageDatabase, + "litellm_customerorganizationtable", + "litellm_customerteamtable", + "litellm_customermembertable", + ) + if prisma_client is not None and customer_usage_requested(_customer_metadata, _customer_key_metadata) + else None, + ledger_enabled=general_settings_view is None + or general_settings_view.get("use_spend_logs_ledger", True) is not False, + ) + if isinstance(_customer_usage, CustomerUsageFailure): + raise HTTPException( + status_code=customer_usage_http_status(_customer_usage.error), detail=_customer_usage.error.value + ) + if _customer_usage is not None: + data[_metadata_variable_name][CUSTOMER_USAGE_METADATA_KEY] = ( # rebind-ok: data is an out-param + _customer_usage.model_dump_json() + ) + data = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( data=data, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index f973fadb53c..a2c32cdb008 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -79,9 +79,11 @@ from litellm.proxy.management_endpoints.common_utils import ( # noqa: F401 # l from litellm.proxy.management_endpoints.key_management_endpoints import ( # noqa: F401 # legacy module exports _check_permissions_caller_permission, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export check_permissions_caller_permission, + customer_key_actor, generate_key_helper_fn, prepare_metadata_fields, raise_on_logging_credential_change, + validated_customer_key_metadata, ) from litellm.proxy.management_helpers.bulk_user_deletion import delete_users_from_current_teams from litellm.proxy.management_helpers.object_permission_utils import ( # noqa: F401 # legacy module exports @@ -560,17 +562,15 @@ async def new_user( }' ``` """ + actor: Final = customer_key_actor(user_api_key_dict) + if data.auto_create_key and actor is not None: + validated_customer_key_metadata(data.metadata, None, actor) try: from litellm.proxy.proxy_server import _license_check, prisma_client if prisma_client is None: raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) - if prisma_client is None: - raise HTTPException( - status_code=500, - detail=CommonProxyErrors.db_not_connected_error.value, - ) validate_budget_duration(data.budget_duration) # Check for duplicate user_id or email @@ -629,7 +629,12 @@ async def new_user( teams = check_if_default_team_set() organization_ids: Final = cast(list[str] | None, data_json.pop("organizations", None)) - response: Final = await generate_key_helper_fn(request_type="user", **data_json, llm_router=None) + response: Final = await generate_key_helper_fn( + request_type="user", + **data_json, + llm_router=None, + actor=actor, + ) created_user_id: Final = cast(str | None, response.get("user_id", None)) if created_user_id is not None: forget_missing_user(created_user_id) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 3bd17c4b5d8..b6f9dd0ef55 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -82,6 +82,12 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( coordination_redis_cache, publish_config_change, ) +from litellm.proxy.common_utils.customer_usage import ( + CustomerUsageFailure, + customer_usage_fields, + customer_usage_http_status, + normalize_customer_key_metadata, +) from litellm.proxy.common_utils.openai_error_payload import openai_error_param, proxy_exception_for from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time, keeps_due_budget_reset @@ -1479,6 +1485,25 @@ async def _stored_user_role(prisma_client: PrismaClient | None, user_id: str | N return user_row.user_role if user_row is not None else None +def validated_customer_key_metadata( + metadata: object, + existing_metadata: object, + actor: UserAPIKeyAuth | None, +) -> Mapping[str, object] | None: + binding: Final = normalize_customer_key_metadata( + customer_usage_fields(metadata) if metadata is not None else None, + customer_usage_fields(existing_metadata) if existing_metadata is not None else None, + administrator=actor is not None and actor.user_role == LitellmUserRoles.PROXY_ADMIN.value, + ) + if isinstance(binding, CustomerUsageFailure): + raise HTTPException(status_code=customer_usage_http_status(binding.error), detail=binding.error.value) + return binding + + +def customer_key_actor(value: object) -> UserAPIKeyAuth | None: + return value if isinstance(value, UserAPIKeyAuth) else None + + async def _common_key_generation_helper( data: GenerateKeyRequest, user_api_key_dict: UserAPIKeyAuth, @@ -1493,6 +1518,8 @@ async def _common_key_generation_helper( prisma_client, ) + validated_customer_key_metadata(data.metadata, None, user_api_key_dict) + acting_as_team_admin: Final = _team_admin_requests_member_key( data=data, team_table=team_table, @@ -1857,7 +1884,7 @@ async def _common_key_generation_helper( try: response = await generate_key_helper_fn( - request_type="key", **data_json, table_name="key", llm_router=llm_router + request_type="key", **data_json, table_name="key", llm_router=llm_router, actor=user_api_key_dict ) except KeyProjectBindingError: if prisma_client is not None: @@ -3062,6 +3089,7 @@ async def prepare_key_update_data( *, prisma_client: PrismaClient | None = None, llm_router: Router | None = None, + user_api_key_dict: UserAPIKeyAuth | None = None, ): if data.router_settings is not None or ( "router_settings" not in data.model_fields_set @@ -3080,6 +3108,13 @@ async def prepare_key_update_data( llm_router=llm_router, ) data_json: Final[dict] = data.model_dump(exclude_unset=True) + if "metadata" in data_json: + customer_binding: Final = validated_customer_key_metadata( + data.metadata, + _StoredKeyMetadata.model_validate(existing_key_row, from_attributes=True).metadata, + user_api_key_dict, + ) + data_json["metadata"] = dict(customer_binding) if customer_binding is not None else None data_json.pop("key", None) data_json.pop("new_key", None) data_json.pop("grace_period", None) # Request-only param, not a DB column @@ -3477,7 +3512,11 @@ async def _process_single_key_update( # Prepare update data raise_on_logging_credential_change(key_request.metadata, existing_key_row, user_api_key_dict) non_default_values = await prepare_key_update_data( - data=key_request, existing_key_row=existing_key_row, prisma_client=prisma_client, llm_router=llm_router + data=key_request, + existing_key_row=existing_key_row, + prisma_client=prisma_client, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, ) await _enforce_custom_key_policy( @@ -4231,7 +4270,11 @@ async def update_key_fn( _enforce_upperbound_key_params(data, fill_defaults=False) raise_on_logging_credential_change(data.metadata, existing_key_row, user_api_key_dict) non_default_values: Final = await prepare_key_update_data( - data=data, existing_key_row=existing_key_row, prisma_client=prisma_client, llm_router=llm_router + data=data, + existing_key_row=existing_key_row, + prisma_client=prisma_client, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, ) # Only validate key_alias format if it's actually being changed @@ -5384,7 +5427,9 @@ async def generate_key_helper_fn( budget_limits: list | None = None, # multiple concurrent budget windows *, llm_router: Router | None = None, + actor: UserAPIKeyAuth | None = None, ): + customer_metadata: Final = validated_customer_key_metadata(_DECODED_JSON.validate_python(metadata), None, actor) from litellm.proxy.proxy_server import premium_user, prisma_client if prisma_client is None: @@ -5436,7 +5481,7 @@ async def generate_key_helper_fn( router_settings_json: Final = safe_dumps(router_settings) if router_settings is not None else safe_dumps({}) metadata_json: Final = metadata_json_with_limits( - metadata, + customer_metadata, model_rpm_limit=model_rpm_limit, model_tpm_limit=model_tpm_limit, mcp_rpm_limit=mcp_rpm_limit, @@ -6419,7 +6464,11 @@ async def _execute_virtual_key_regeneration( _enforce_upperbound_key_params(data, fill_defaults=False) raise_on_logging_credential_change(data.metadata, key_in_db, user_api_key_dict) non_default_values = await prepare_key_update_data( - data=data, existing_key_row=key_in_db, prisma_client=prisma_client, llm_router=llm_router + data=data, + existing_key_row=key_in_db, + prisma_client=prisma_client, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, ) # Only validate key_alias format if it's actually being changed new_key_alias: Final = non_default_values.get("key_alias") diff --git a/litellm/proxy/management_endpoints/management_v1/__init__.py b/litellm/proxy/management_endpoints/management_v1/__init__.py index c42a786db0d..c4483ef7ff1 100644 --- a/litellm/proxy/management_endpoints/management_v1/__init__.py +++ b/litellm/proxy/management_endpoints/management_v1/__init__.py @@ -8,6 +8,9 @@ from litellm.proxy.management.users.endpoints import router as current_user_rout from litellm.proxy.management_endpoints.management_v1.budgets import ( router as budgets_router, ) +from litellm.proxy.management_endpoints.management_v1.customer_usage import ( + router as customer_usage_router, +) from litellm.proxy.management_endpoints.management_v1.customers import ( router as customers_router, ) @@ -24,6 +27,7 @@ from litellm.proxy.management_endpoints.management_v1.users import ( router: Final = APIRouter() router.include_router(budgets_router) router.include_router(customers_router) +router.include_router(customer_usage_router) router.include_router(spend_logs_router) router.include_router(teams_router) router.include_router(users_router) diff --git a/litellm/proxy/management_endpoints/management_v1/customer_usage.py b/litellm/proxy/management_endpoints/management_v1/customer_usage.py new file mode 100644 index 00000000000..4f0a7fc154e --- /dev/null +++ b/litellm/proxy/management_endpoints/management_v1/customer_usage.py @@ -0,0 +1,220 @@ +from collections.abc import Sequence +from enum import Enum +from functools import partial +from typing import Annotated, Final, Protocol + +from fastapi import APIRouter, Depends +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import require_attributes +from litellm.proxy.list_api.common import reject_unknown_query_params +from litellm.proxy.management_endpoints.common_daily_activity import DailySpendRecord, get_daily_activity +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.management_endpoints.management_v1.customers import ( + raise_public, + reject_duplicate_list_params, + require_customer_admin, +) +from litellm.proxy.management_helpers.customer_hierarchy import ErrorCode, Failure, PrismaRecord +from litellm.proxy.management_helpers.utils import ( + management_endpoint_wrapper, # pyright: ignore[reportUnknownVariableType] # legacy decorator is untyped +) +from litellm.proxy.utils import PrismaClient +from litellm.repositories.chunked_in import find_many_in +from litellm.repositories.prisma_protocols import DatabaseClient, TableActions +from litellm.types.proxy.management_endpoints.common_daily_activity import SpendAnalyticsPaginatedResponse + + +class CustomerEntity(str, Enum): + ORGANIZATION = "customer_org_id" + TEAM = "customer_team_id" + MEMBER = "customer_member_id" + + +class CustomerMetadataDatabase(Protocol): + @property + def litellm_customerorganizationtable(self) -> TableActions[PrismaRecord]: ... + + @property + def litellm_customerteamtable(self) -> TableActions[PrismaRecord]: ... + + @property + def litellm_customermembertable(self) -> TableActions[PrismaRecord]: ... + + +_TEXT: Final = TypeAdapter(str) +_OPTIONAL_TEXT: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) + + +def get_customer_usage_prisma() -> PrismaClient: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise_public(Failure(ErrorCode.DATABASE_UNAVAILABLE)) + return prisma_client + + +def metadata_table(prisma_client: PrismaClient, entity: CustomerEntity) -> TableActions[PrismaRecord]: + database: Final = require_attributes(prisma_client, DatabaseClient, "db").db + tables: Final = require_attributes( + database, + CustomerMetadataDatabase, + "litellm_customerorganizationtable", + "litellm_customerteamtable", + "litellm_customermembertable", + ) + if entity is CustomerEntity.ORGANIZATION: + return tables.litellm_customerorganizationtable + if entity is CustomerEntity.TEAM: + return tables.litellm_customerteamtable + return tables.litellm_customermembertable + + +async def page_aliases( + table: TableActions[PrismaRecord], entity: CustomerEntity, records: Sequence[DailySpendRecord] +) -> dict[str, dict[str, object]]: # mutable-ok: get_daily_activity requires dictionary-returning metadata callbacks + identifiers: Final = tuple(_OPTIONAL_TEXT.validate_python(getattr(row, entity.value, None)) for row in records) + rows: Final = await find_many_in(table, entity.value, tuple(value for value in identifiers if value)) + values: Final = tuple(row.model_dump() for row in rows) + return { + _TEXT.validate_python(row[entity.value]): { + "alias": _OPTIONAL_TEXT.validate_python(row.get("alias")) or _TEXT.validate_python(row[entity.value]) + } + for row in values + } + + +async def customer_activity( + prisma_client: PrismaClient, + entity: CustomerEntity, + entity_ids: str | None, + excluded_ids: str | None, + start_date: str | None, + end_date: str | None, + model: str | None, + api_key: str | None, + page: int, + page_size: int, +) -> SpendAnalyticsPaginatedResponse: + return await get_daily_activity( + prisma_client=prisma_client, + table_name="litellm_dailycustomermemberspend", + entity_id_field=entity.value, + entity_id=entity_ids.split(",") if entity_ids else None, + entity_metadata_field=None, + exclude_entity_ids=excluded_ids.split(",") if excluded_ids else None, + start_date=start_date, + end_date=end_date, + model=model, + api_key=api_key, + page=page, + page_size=page_size, + resolve_entity_metadata=partial(page_aliases, metadata_table(prisma_client, entity), entity), + ) + + +router: Final = APIRouter( + prefix=MANAGEMENT_V1_PREFIX, + tags=["customer usage"], + dependencies=( + Depends(require_customer_admin), + Depends(reject_unknown_query_params), + Depends(reject_duplicate_list_params), + ), +) + + +@router.get( + "/customer_organizations/daily/activity", + response_model=SpendAnalyticsPaginatedResponse, + description="Get paginated daily customer organization activity with model, provider, key and organization breakdowns. " + "Filters accept comma-separated organization IDs. Proxy admins only.", +) +@management_endpoint_wrapper +async def customer_organization_daily_activity( + prisma_client: Annotated[PrismaClient, Depends(get_customer_usage_prisma)], + customer_org_ids: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + model: str | None = None, + api_key: str | None = None, + page: int = 1, + page_size: int = 10, + exclude_customer_org_ids: str | None = None, +) -> SpendAnalyticsPaginatedResponse: + return await customer_activity( + prisma_client, + CustomerEntity.ORGANIZATION, + customer_org_ids, + exclude_customer_org_ids, + start_date, + end_date, + model, + api_key, + page, + page_size, + ) + + +@router.get( + "/customer_teams/daily/activity", + response_model=SpendAnalyticsPaginatedResponse, + description="Get paginated daily customer team activity with model, provider, key and team breakdowns. " + "Filters accept comma-separated team IDs. Proxy admins only.", +) +@management_endpoint_wrapper +async def customer_team_daily_activity( + prisma_client: Annotated[PrismaClient, Depends(get_customer_usage_prisma)], + customer_team_ids: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + model: str | None = None, + api_key: str | None = None, + page: int = 1, + page_size: int = 10, + exclude_customer_team_ids: str | None = None, +) -> SpendAnalyticsPaginatedResponse: + return await customer_activity( + prisma_client, + CustomerEntity.TEAM, + customer_team_ids, + exclude_customer_team_ids, + start_date, + end_date, + model, + api_key, + page, + page_size, + ) + + +@router.get( + "/customer_members/daily/activity", + response_model=SpendAnalyticsPaginatedResponse, + description="Get paginated daily customer member activity with model, provider, key and member breakdowns. " + "Filters accept comma-separated member IDs. Proxy admins only.", +) +@management_endpoint_wrapper +async def customer_member_daily_activity( + prisma_client: Annotated[PrismaClient, Depends(get_customer_usage_prisma)], + customer_member_ids: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + model: str | None = None, + api_key: str | None = None, + page: int = 1, + page_size: int = 10, + exclude_customer_member_ids: str | None = None, +) -> SpendAnalyticsPaginatedResponse: + return await customer_activity( + prisma_client, + CustomerEntity.MEMBER, + customer_member_ids, + exclude_customer_member_ids, + start_date, + end_date, + model, + api_key, + page, + page_size, + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b37eede1745..f9d4d5704d8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -19970,6 +19970,7 @@ async def _generate_onboarding_ui_session_token(user_obj: _UserTableRow) -> str: response: Final = await generate_key_helper_fn( llm_router=llm_router, request_type="key", + actor=None, **{ "user_role": user_obj.user_role, "duration": LITELLM_UI_SESSION_DURATION, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 2c0246bb94e..bac27daa5de 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1035,6 +1035,45 @@ model LiteLLM_DailyOrganizationSpend { } // Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyCustomerMemberSpend { + id String @id + date String + customer_org_id String + customer_team_id String @default("") + customer_member_id String @default("") + model String @default("") + model_group String @default("") + custom_llm_provider String @default("") + mcp_namespaced_tool_name String @default("") + endpoint String @default("") + api_key String @default("") + spend Float @default(0.0) + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@index([date]) + @@index([customer_org_id, date]) + @@index([customer_team_id, date]) + @@index([customer_member_id, date]) + @@index([api_key]) + @@index([model]) +} + model LiteLLM_DailyEndUserSpend { id String @id @default(uuid()) end_user_id String? diff --git a/litellm/repositories/accounting_delivery_repository.py b/litellm/repositories/accounting_delivery_repository.py index 624c2b8851a..e172e863b63 100644 --- a/litellm/repositories/accounting_delivery_repository.py +++ b/litellm/repositories/accounting_delivery_repository.py @@ -14,6 +14,18 @@ if TYPE_CHECKING: class LedgerTransaction(Protocol): + @property + def litellm_dailycustomermemberspend(self) -> TableActions[object]: ... + + @property + def litellm_customerorganizationtable(self) -> TableActions[object]: ... + + @property + def litellm_customerteamtable(self) -> TableActions[object]: ... + + @property + def litellm_customermembertable(self) -> TableActions[object]: ... + @property def litellm_dailyguardrailusageunits(self) -> TableActions[object]: ... diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 9aa0950785b..d8b4d1eb8d5 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -96,6 +96,9 @@ class _DailyActivityDatabase(Protocol): @property def litellm_dailyenduserspend(self) -> TableActions[DailyActivityRow]: ... + @property + def litellm_dailycustomermemberspend(self) -> TableActions[DailyActivityRow]: ... + @property def litellm_dailyagentspend(self) -> TableActions[DailyActivityRow]: ... @@ -178,6 +181,8 @@ def _daily_rows_table( return prisma_client.db.litellm_dailyorganizationspend if table is DailyActivityTable.CUSTOMER: return prisma_client.db.litellm_dailyenduserspend + if table is DailyActivityTable.CUSTOMER_MEMBER: + return prisma_client.db.litellm_dailycustomermemberspend if table is DailyActivityTable.AGENT: return prisma_client.db.litellm_dailyagentspend assert_never(table) diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index dc3ca19bb30..4b575761e19 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -43,6 +43,7 @@ PRISMA_TO_PG_TABLE: Final[Mapping[DailyActivityTable, str]] = MappingProxyType( DailyActivityTable.TAG: "LiteLLM_DailyTagSpend", DailyActivityTable.ORGANIZATION: "LiteLLM_DailyOrganizationSpend", DailyActivityTable.CUSTOMER: "LiteLLM_DailyEndUserSpend", + DailyActivityTable.CUSTOMER_MEMBER: "LiteLLM_DailyCustomerMemberSpend", DailyActivityTable.AGENT: "LiteLLM_DailyAgentSpend", } ) diff --git a/litellm/schema.prisma b/litellm/schema.prisma index 2c0246bb94e..bac27daa5de 100644 --- a/litellm/schema.prisma +++ b/litellm/schema.prisma @@ -1035,6 +1035,45 @@ model LiteLLM_DailyOrganizationSpend { } // Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyCustomerMemberSpend { + id String @id + date String + customer_org_id String + customer_team_id String @default("") + customer_member_id String @default("") + model String @default("") + model_group String @default("") + custom_llm_provider String @default("") + mcp_namespaced_tool_name String @default("") + endpoint String @default("") + api_key String @default("") + spend Float @default(0.0) + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@index([date]) + @@index([customer_org_id, date]) + @@index([customer_team_id, date]) + @@index([customer_member_id, date]) + @@index([api_key]) + @@index([model]) +} + model LiteLLM_DailyEndUserSpend { id String @id @default(uuid()) end_user_id String? diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index 8d68d92a336..babf4a6c8fd 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -12,6 +12,7 @@ class DailyActivityTable(str, Enum): TAG = "litellm_dailytagspend" ORGANIZATION = "litellm_dailyorganizationspend" CUSTOMER = "litellm_dailyenduserspend" + CUSTOMER_MEMBER = "litellm_dailycustomermemberspend" AGENT = "litellm_dailyagentspend" @@ -22,6 +23,7 @@ _ENTITY_FIELDS: Mapping[DailyActivityTable, frozenset[str]] = MappingProxyType( DailyActivityTable.TAG: frozenset(("tag",)), DailyActivityTable.ORGANIZATION: frozenset(("organization_id",)), DailyActivityTable.CUSTOMER: frozenset(("end_user_id",)), + DailyActivityTable.CUSTOMER_MEMBER: frozenset(("customer_org_id", "customer_team_id", "customer_member_id")), DailyActivityTable.AGENT: frozenset(("agent_id",)), } ) diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerMemberSpend.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerMemberSpend.tsx new file mode 100644 index 00000000000..dfcd9470764 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerMemberSpend.tsx @@ -0,0 +1,166 @@ +"use client"; + +import { useMemo } from "react"; +import { parseAsString, parseAsInteger, useQueryStates } from "nuqs"; +import { Button } from "@/components/ui/button"; +import { useTranslation } from "react-i18next"; +import { Search } from "lucide-react"; +import { formatCompact, formatUsd } from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { Input } from "@/components/ui/input"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { cn } from "@/lib/cva.config"; +import type { CustomerHierarchyUsage, CustomerUsageEntity } from "./customerUsageApi"; + +const MEMBER_TABLE_PARSERS = { memberSearch: parseAsString.withDefault(""), memberPage: parseAsInteger.withDefault(1) }; +const PAGE_SIZE = 20; +const memberStatus = (member: CustomerUsageEntity) => { + if (member.budgetLimit === null) return "customerPortal.analytics.noCap" as const; + if (member.spend > member.budgetLimit) return "customerPortal.analytics.overBudget" as const; + if (member.spend === member.budgetLimit) return "customerPortal.analytics.atBudget" as const; + return "customerPortal.analytics.withinBudget" as const; +}; + +export default function CustomerMemberSpend({ + data, + onMemberClick, +}: { + data: CustomerHierarchyUsage; + onMemberClick: (id: string) => void; +}) { + const { t } = useTranslation(); + const [table, setTable] = useQueryStates(MEMBER_TABLE_PARSERS); + const search = table.memberSearch; + const usage = useMemo(() => new Map(data.members.map((member) => [member.id, member])), [data.members]); + const members = useMemo( + () => + data.memberBudgets + .filter((member) => member.label.toLocaleLowerCase().includes(search.toLocaleLowerCase())) + .map((member) => ({ ...member, rangeUsage: usage.get(member.id) })) + .sort((a, b) => (b.rangeUsage?.spend ?? 0) - (a.rangeUsage?.spend ?? 0) || a.label.localeCompare(b.label)), + [data.memberBudgets, search, usage], + ); + + const lastPage = Math.max(1, Math.ceil(members.length / PAGE_SIZE)); + const page = Math.max(1, Math.min(table.memberPage, lastPage)); + const start = (page - 1) * PAGE_SIZE; + return ( +
+
+
+

{t("customerPortal.analytics.memberSpend")}

+

{t("customerPortal.analytics.memberSpendHint")}

+
+
+
+
+ + + + {t("customerPortal.analytics.member")} + {t("customerPortal.analytics.spend")} + {t("customerPortal.analytics.tokens")} + {t("customerPortal.analytics.currentBudget")} + {t("customerPortal.analytics.status")} + + + + {members.slice(start, start + PAGE_SIZE).map((member) => { + const atLimit = member.budgetLimit !== null && member.spend >= member.budgetLimit; + return ( + + +
+ +
+ + {member.id} +
+
+ + {formatUsd(member.rangeUsage?.spend ?? 0)} + + {formatCompact(member.rangeUsage?.tokens ?? 0)} + + + {member.budgetLimit === null + ? t("customerPortal.analytics.noBudget") + : `${formatUsd(member.spend)} / ${formatUsd(member.budgetLimit)}`} + + + + {t(memberStatus(member))} + + + + ); + })} + {members.length === 0 && ( + + + {t(search ? "customerPortal.analytics.noMatches" : "customerPortal.analytics.noMembers")} + + + )} + +
+ {members.length > PAGE_SIZE && ( +
+ + {t("customerPortal.analytics.pagination", { + from: start + 1, + to: Math.min(start + PAGE_SIZE, members.length), + total: members.length, + })} + +
+ + +
+
+ )} +
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageHeader.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageHeader.tsx new file mode 100644 index 00000000000..618aeb0b80b --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageHeader.tsx @@ -0,0 +1,112 @@ +import type { ParseKeys } from "i18next"; +import { ChevronRight } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import { formatUsd } from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; +import { uiHref } from "@/utils/uiHref"; +import type { CustomerHierarchyUsage, CustomerUsageEntity, CustomerUsageScope } from "./customerUsageApi"; + +interface CustomerUsageHeaderProps { + scope: CustomerUsageScope; + data: CustomerHierarchyUsage | undefined; + dateValue: DateRangePickerValue; + preset: string | null; + onOrganizationClick: () => void; + onTeamClick: () => void; + onDateRangeChange: (value: DateRangePickerValue, preset: string | null) => void; +} + +const scopeLabel = (scope: CustomerUsageScope): ParseKeys => { + if (scope.member) return "customerPortal.analytics.memberUsage"; + if (scope.team) return "customerPortal.analytics.teamUsage"; + return "customerPortal.analytics.organizationUsage"; +}; + +function CustomerUsageBreadcrumbs({ + scope, + data, + onOrganizationClick, + onTeamClick, +}: Pick) { + const { t } = useTranslation(); + return ( + + ); +} + +function CustomerCurrentSpend({ entity }: { entity: CustomerUsageEntity }) { + const { t } = useTranslation(); + return ( +

+ {t("customerPortal.analytics.currentSpend")}{" "} + {formatUsd(entity.spend)} + {entity.budgetLimit === null + ? ` · ${t("customerPortal.analytics.noBudget")}` + : ` / ${formatUsd(entity.budgetLimit)}`} +

+ ); +} + +export default function CustomerUsageHeader({ + scope, + data, + dateValue, + preset, + onOrganizationClick, + onTeamClick, + onDateRangeChange, +}: CustomerUsageHeaderProps) { + const { t } = useTranslation(); + const entity = data?.member ?? data?.team ?? data?.organization; + const label = entity?.label ?? scope.member ?? scope.team ?? scope.org; + return ( +
+ +
+
+

{t(scopeLabel(scope))}

+

{label}

+

{t("customerPortal.analytics.subtitle")}

+ {entity && } +
+ +
+
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageRankings.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageRankings.tsx new file mode 100644 index 00000000000..aa881cb997f --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageRankings.tsx @@ -0,0 +1,200 @@ +"use client"; + +import type { ParseKeys } from "i18next"; +import { useTranslation } from "react-i18next"; +import { ChevronRight, Search } from "lucide-react"; +import { + type UsageMetric, + formatCompact, + formatUsd, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { cn } from "@/lib/cva.config"; +import type { CustomerUsageRow } from "./customerUsageData"; +import { ModelMark } from "@/app/(dashboard)/usage/_components/components/overview/BreakdownChart"; +import { colorForModel } from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { providerForModel } from "@/app/(dashboard)/usage/_components/components/overview/modelProvider"; + +export const CUSTOMER_USAGE_DIMENSIONS = ["models", "api_keys", "teams"] as const; +export type CustomerUsageDimension = (typeof CUSTOMER_USAGE_DIMENSIONS)[number]; + +const DIMENSION_LABELS: Record = { + models: "customerPortal.analytics.models", + api_keys: "customerPortal.analytics.keys", + teams: "customerPortal.analytics.teams", +}; +const PAGE_SIZE = 10; + +interface CustomerUsageRankingsProps { + rows: readonly CustomerUsageRow[]; + dimension: CustomerUsageDimension; + metric: UsageMetric; + search: string; + page: number; + colorByModel: ReadonlyMap; + onSearchChange: (search: string) => void; + onPageChange: (page: number) => void; + onDrillDown: (dimension: "teams", id: string) => void; +} + +export default function CustomerUsageRankings({ + rows, + dimension, + metric, + search, + page, + colorByModel, + onSearchChange, + onPageChange, + onDrillDown, +}: CustomerUsageRankingsProps) { + const { t } = useTranslation(); + const filtered = rows.filter((row) => row.label.toLocaleLowerCase().includes(search.toLocaleLowerCase())); + const lastPage = Math.max(1, Math.ceil(filtered.length / PAGE_SIZE)); + const currentPage = Math.min(Math.max(1, page), lastPage); + const start = (currentPage - 1) * PAGE_SIZE; + const canDrillDown = dimension === "teams"; + + return ( +
+
+
+
+
+
+

{t("customerPortal.analytics.rankingHint")}

+ + + + {t("customerPortal.analytics.rank")} + {t(DIMENSION_LABELS[dimension])} + {t("customerPortal.analytics.share")} + + {t("customerPortal.analytics.spend")} + + + {t("customerPortal.analytics.tokens")} + + + {t("customerPortal.analytics.requests")} + + + + + {filtered.slice(start, start + PAGE_SIZE).map((row, index) => ( + + {start + index + 1} + + {canDrillDown && row.id ? ( + + ) : ( + + {dimension === "models" && ( + + )} + + {row.label} + {dimension === "models" && providerForModel(row.id) && ( + + {t("usage.breakdown.byProvider", { provider: providerForModel(row.id) })} + + )} + + + )} + + +
+ + + + {formatUsd(row.spend)} + + + {formatCompact(row.tokens)} + + + {formatCompact(row.requests)} + + + ))} + {filtered.length === 0 && ( + + + {t(search ? "customerPortal.analytics.noMatches" : "customerPortal.analytics.noBreakdown")} + + + )} + +
+ {filtered.length > PAGE_SIZE && ( +
+ + {t("customerPortal.analytics.pagination", { + from: start + 1, + to: Math.min(start + PAGE_SIZE, filtered.length), + total: filtered.length, + })} + +
+ + +
+
+ )} +
+
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageTrend.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageTrend.tsx new file mode 100644 index 00000000000..0ba88ea4d6d --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageTrend.tsx @@ -0,0 +1,152 @@ +"use client"; + +import { useMemo } from "react"; +import { useTranslation } from "react-i18next"; +import { + bucketSeries, + bucketTotals, + formatMetricValue, + labelForDate, + type UsageMetric, + type Granularity, + type OverviewTotals, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { ChartSkeleton, Segmented } from "@/app/(dashboard)/usage/_components/components/overview/Primitives"; +import { StackedUsageChart } from "@/components/shared/charts"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import type { CustomerHierarchyUsage } from "./customerUsageApi"; +import { customerUsageSeries } from "./customerUsageChartData"; +import { CUSTOMER_USAGE_DIMENSIONS, type CustomerUsageDimension } from "./CustomerUsageRankings"; +import { ModelMark } from "@/app/(dashboard)/usage/_components/components/overview/BreakdownChart"; + +const DIMENSION_LABELS = { + models: "customerPortal.analytics.models", + api_keys: "customerPortal.analytics.keys", + teams: "customerPortal.analytics.teams", +} as const; + +interface CustomerUsageTrendProps { + data: CustomerHierarchyUsage | undefined; + dimension: CustomerUsageDimension; + onDimensionChange: (dimension: CustomerUsageDimension) => void; + totals: OverviewTotals; + metric: UsageMetric; + bucket: Granularity; + loading: boolean; + colorByModel: ReadonlyMap; + onMetricChange: (metric: UsageMetric) => void; + onBucketChange: (bucket: Granularity) => void; +} + +export default function CustomerUsageTrend({ + data, + dimension, + onDimensionChange, + totals, + metric, + bucket, + loading, + colorByModel, + onMetricChange, + onBucketChange, +}: CustomerUsageTrendProps) { + const { t } = useTranslation(); + const dailySeries = useMemo( + () => + data + ? customerUsageSeries(data, dimension, metric, colorByModel) + : { data: [], keys: [], labels: [], colors: [] }, + [data, dimension, metric, colorByModel], + ); + const series = useMemo(() => bucketSeries(dailySeries, bucket), [dailySeries, bucket]); + const bucketValues = useMemo(() => bucketTotals(series), [series]); + const empty = totals.requests === 0 && totals.tokens === 0 && totals.spend === 0; + return ( +
+
+
+

{t("customerPortal.analytics.trend")}

+

+ {t("customerPortal.analytics.trendHint", { dimension: t(DIMENSION_LABELS[dimension]).toLocaleLowerCase() })} +

+
+
+ + + label={t("customerPortal.analytics.metric")} + value={metric} + options={[ + { value: "spend", label: t("customerPortal.analytics.spend") }, + { value: "tokens", label: t("customerPortal.analytics.tokens") }, + { value: "requests", label: t("customerPortal.analytics.requests") }, + ]} + onChange={(metric) => onMetricChange(metric)} + /> + +
+ {loading && ( +
+ +
+ )} + {!loading && empty && ( +
+ {t("customerPortal.analytics.empty")} +
+ )} + {!loading && !empty && ( + <> + labelForDate(series, date)} + format={(value) => formatMetricValue(value, metric)} + totalFor={(date) => bucketValues.get(date)} + className="h-72 sm:h-80" + /> +
+ {series.labels.map((label, index) => ( + + {dimension === "models" ? ( + + ) : ( + + ))} +
+ + )} +
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.test.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.test.ts new file mode 100644 index 00000000000..72443540bab --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.test.ts @@ -0,0 +1,336 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { registerBaseUrlGetter } from "@/lib/http/runtime"; +import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import { customerHierarchyUsageCall } from "./customerUsageApi"; + +const audit = { + created_at: "2026-11-01T00:00:00Z", + created_by: "admin", + updated_at: "2026-11-01T00:00:00Z", + updated_by: "admin", + spend: 40, +}; +const organization = { + ...audit, + customer_org_id: "org-1", + alias: "Customer organization", + budget: { max_budget: 100 }, + budget_reset_at: "2026-12-01T00:00:00Z", + default_budgets: [], +}; +const teams = [ + { ...audit, customer_team_id: "team-a", parent_org_id: "org-1", alias: "Product", budget: { max_budget: 50 } }, + { ...audit, customer_team_id: "team-b", parent_org_id: "org-1", alias: "Operations", budget: null }, +]; +const members = [ + { + ...audit, + customer_member_id: "member-a", + parent_team_id: "team-a", + alias: "Member One", + effective_budget: { max_budget: 10 }, + }, + { + ...audit, + customer_member_id: "member-b", + parent_team_id: "team-b", + alias: "Member Two", + effective_budget: { max_budget: 20 }, + }, +]; +const metrics = (spend: number, tokens: number, requests: number): SpendMetrics => ({ + spend, + prompt_tokens: tokens, + completion_tokens: 0, + total_tokens: tokens, + api_requests: requests, + successful_requests: requests, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}); +const records = [ + { + organization: "org-1", + team: "team-a", + member: "member-a", + model: "model-a", + key: "hash-a", + alias: "Assistant key", + metrics: metrics(3, 100, 8), + }, + { + organization: "org-1", + team: "team-b", + member: "member-b", + model: "model-b", + key: "hash-b", + alias: "Search key", + metrics: metrics(1, 500, 2), + }, +]; +type UsageRecord = (typeof records)[number]; +const daily = (record: UsageRecord, entityId: string): DailyData => { + const key = { metrics: record.metrics, metadata: { key_alias: record.alias, team_id: null } }; + const entity = { metrics: record.metrics, metadata: {}, api_key_breakdown: { [record.key]: key } }; + return { + date: "2026-11-01", + metrics: record.metrics, + breakdown: { + models: { [record.model]: entity }, + model_groups: { [record.model]: entity }, + providers: {}, + mcp_servers: {}, + api_keys: { [record.key]: key }, + entities: { [entityId]: entity }, + }, + }; +}; +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } }); +const requestUrl = (input: Parameters[0]) => + new URL(input instanceof Request ? input.url : String(input), "http://localhost"); +const fetchMock = vi.fn(); +const urls = () => fetchMock.mock.calls.map(([input]) => requestUrl(input)); +const dailyUrls = () => urls().filter((url) => url.pathname.endsWith("/daily/activity")); +const request = { + scope: { org: "org-1", team: null, member: null }, + accessToken: "test-session", + startDate: "2026-11-01", + endDate: "2026-11-03", +}; +const list = (data: T[], page = 1, totalPages = 1) => ({ + data, + meta: { total_count: data.length, page, page_size: 100, total_pages: totalPages }, +}); + +const activityDimension = (resource: string | undefined): "organization" | "team" | "member" => { + if (resource === "customer_organizations") return "organization"; + if (resource === "customer_teams") return "team"; + return "member"; +}; +const filterFor = (resource: string | undefined) => + ({ + organization: "customer_org_ids", + team: "customer_team_ids", + member: "customer_member_ids", + })[activityDimension(resource)]; +const activityResponse = (url: URL) => { + const collection = url.pathname.split("/").at(-3); + const dimension = activityDimension(collection); + const filter = filterFor(collection); + const ids = url.searchParams.get(filter)?.split(",") ?? []; + const selected = records.filter((row) => ids.includes(row[dimension])); + const page = Number(url.searchParams.get("page")); + const current = selected.slice(page - 1, page); + const values = current[0]?.metrics ?? metrics(0, 0, 0); + return json({ + results: current.map((row) => daily(row, row[dimension])), + metadata: { + ...EMPTY_DAILY_ACTIVITY_METADATA, + total_spend: values.spend, + total_tokens: values.total_tokens, + total_api_requests: values.api_requests, + page, + total_pages: selected.length, + has_more: page < selected.length, + }, + }); +}; + +const serve = (input: Parameters[0]): Response => { + const url = requestUrl(input); + if (url.pathname.endsWith("/daily/activity")) return activityResponse(url); + if (url.pathname === "/management/v1/customer_organizations/org-1") return json(organization); + if (url.pathname === "/management/v1/customer_teams") { + const page = Number(url.searchParams.get("page")); + return json(list([teams[page - 1]], page, 2)); + } + if (url.pathname === "/management/v1/customer_members") + return json(list(members.filter((member) => member.parent_team_id === url.searchParams.get("parent_team_id")))); + const team = teams.find((team) => url.pathname.endsWith(`/customer_teams/${team.customer_team_id}`)); + if (team) return json(team); + const member = members.find((member) => url.pathname.endsWith(`/customer_members/${member.customer_member_id}`)); + if (member) return json(member); + throw new Error(`Unexpected usage request ${url.pathname}`); +}; + +beforeEach(() => { + registerBaseUrlGetter(() => "http://localhost"); + fetchMock.mockReset(); + fetchMock.mockImplementation((input) => Promise.resolve(serve(input))); + vi.stubGlobal("fetch", fetchMock); +}); +afterEach(() => vi.unstubAllGlobals()); + +describe("customerHierarchyUsageCall", () => { + it("should read exact daily activity routes, all pages, and scoped descendant lists", async () => { + fetchMock.mockImplementation((input) => { + const url = requestUrl(input); + if (url.pathname === "/management/v1/customer_members" && url.searchParams.get("parent_team_id") === "team-a") { + const page = Number(url.searchParams.get("page")); + const member = page === 1 ? members[0] : { ...members[0], customer_member_id: "member-c" }; + return Promise.resolve(json(list([member], page, 2))); + } + return Promise.resolve(serve(input)); + }); + const result = await customerHierarchyUsageCall(request); + expect(result.metadata.total_spend).toBe(4); + expect(result.metadata.total_tokens).toBe(600); + expect(result.metadata.total_api_requests).toBe(10); + expect(result.results).toHaveLength(1); + expect(result.results[0].metrics.spend).toBe(4); + expect(result.results[0].breakdown.model_groups["model-b"].metrics.total_tokens).toBe(500); + expect(result.api_keys.map((row) => row.label).sort()).toEqual(["Assistant key", "Search key"]); + expect(result.teams.map((row) => [row.id, row.spend])).toEqual([ + ["team-a", 3], + ["team-b", 1], + ]); + expect(result.members.map((row) => [row.id, row.tokens])).toEqual([ + ["member-a", 100], + ["member-b", 500], + ]); + expect(result.teamResults).toHaveLength(1); + expect(result.teamResults[0].metrics.spend).toBe(4); + expect(result.teamResults[0].breakdown.entities["team-a"]).toMatchObject({ + metadata: { alias: "Product" }, + metrics: { spend: 3 }, + }); + expect(result.memberResults).toHaveLength(1); + expect(result.memberResults[0].breakdown.entities["member-b"].metrics.total_tokens).toBe(500); + expect(result.memberBudgets[0]).toMatchObject({ id: "member-a", spend: 40, budgetLimit: 10 }); + for (const url of dailyUrls()) { + expect(url.pathname).toMatch(/^\/management\/v1\/customer_(organizations|teams|members)\/daily\/activity$/); + expect(url.searchParams.get("start_date")).toBe(request.startDate); + expect(url.searchParams.get("end_date")).toBe(request.endDate); + expect(url.searchParams.get("page_size")).toBe("100"); + expect([...url.searchParams.keys()].filter((name) => name.endsWith("_ids"))).toHaveLength(1); + } + const organizationPages = dailyUrls().filter((url) => url.searchParams.get("customer_org_ids") === "org-1"); + expect(organizationPages.map((url) => url.searchParams.get("page"))).toEqual(["1", "2"]); + expect( + urls() + .filter((url) => url.pathname === "/management/v1/customer_teams") + .map((url) => [url.searchParams.get("parent_org_id"), url.searchParams.get("page")]), + ).toEqual([ + ["org-1", "1"], + ["org-1", "2"], + ]); + expect( + dailyUrls() + .find((url) => url.searchParams.has("customer_team_ids")) + ?.searchParams.get("customer_team_ids"), + ).toBe("team-a,team-b"); + expect( + dailyUrls() + .find((url) => url.searchParams.has("customer_member_ids")) + ?.searchParams.get("customer_member_ids"), + ).toBe("member-a,member-c,member-b"); + expect( + urls().some( + (url) => + url.pathname === "/management/v1/customer_members" && + url.searchParams.get("parent_team_id") === "team-a" && + url.searchParams.get("page") === "2", + ), + ).toBe(true); + }); + + it("should infer member ancestry and retain live spend and effective budget separately from range totals", async () => { + const result = await customerHierarchyUsageCall({ + ...request, + scope: { org: "org-1", team: null, member: "member-a" }, + }); + expect(result.team?.id).toBe("team-a"); + expect(result.member).toMatchObject({ id: "member-a", spend: 40, budgetLimit: 10 }); + expect(result.organization).toMatchObject({ spend: 40, budgetLimit: 100, budgetResetAt: "2026-12-01T00:00:00Z" }); + expect(result.metadata.total_spend).toBe(3); + expect(result.teams).toEqual([{ id: "team-a", label: "Product", spend: 3, tokens: 100, requests: 8 }]); + expect(Object.keys(result.teamResults[0].breakdown.entities)).toEqual(["team-a"]); + expect(result.teamResults[0].breakdown.entities["team-a"].metrics.spend).toBe(3); + expect(result.memberResults[0].breakdown.entities["member-a"].metrics.total_tokens).toBe(100); + expect(dailyUrls()).toHaveLength(1); + expect(dailyUrls()[0].searchParams.get("customer_member_ids")).toBe("member-a"); + expect(urls().some((url) => url.pathname === "/management/v1/customer_members")).toBe(false); + }); + + it("should reject mismatched ancestry before requesting any usage", async () => { + fetchMock.mockImplementation((input) => + Promise.resolve( + requestUrl(input).pathname.endsWith("/customer_teams/team-a") + ? json({ ...teams[0], parent_org_id: "another-org" }) + : serve(input), + ), + ); + await expect( + customerHierarchyUsageCall({ ...request, scope: { org: "org-1", team: "team-a", member: null } }), + ).rejects.toThrow("does not belong"); + expect(dailyUrls()).toHaveLength(0); + }); + + it("should never send an empty descendant filter and should retain unassigned usage", async () => { + fetchMock.mockImplementation((input) => + Promise.resolve(requestUrl(input).pathname === "/management/v1/customer_teams" ? json(list([])) : serve(input)), + ); + const result = await customerHierarchyUsageCall(request); + expect(dailyUrls().every((url) => url.searchParams.get("customer_org_ids") === "org-1")).toBe(true); + expect(result.teams).toEqual([ + { id: "", label: "Unassigned or removed team", spend: 4, tokens: 600, requests: 10 }, + ]); + expect(result.members).toEqual([ + { id: "", label: "Unassigned or removed member", spend: 4, tokens: 600, requests: 10 }, + ]); + }); + + it("should fail a snapshot when a later page fails instead of presenting partial totals", async () => { + fetchMock.mockImplementation((input) => { + const url = requestUrl(input); + return Promise.resolve( + url.searchParams.get("customer_org_ids") === "org-1" && url.searchParams.get("page") === "2" + ? json({ error: "Second page unavailable" }, 503) + : serve(input), + ); + }); + await expect(customerHierarchyUsageCall(request)).rejects.toThrow("Second page unavailable"); + }); + + it("should reject identifiers the comma-separated API cannot represent instead of broadening filters", async () => { + await expect( + customerHierarchyUsageCall({ ...request, scope: { org: "org-1,org-2", team: null, member: null } }), + ).rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("should chunk descendant filters and keep every request phase within four concurrent reads", async () => { + const manyTeams = Array.from({ length: 205 }, (_, index) => ({ ...teams[0], customer_team_id: `team-${index}` })); + let active = 0; + let maximumActive = 0; + fetchMock.mockImplementation(async (input) => { + active++; + maximumActive = Math.max(maximumActive, active); + await Promise.resolve(); + const url = requestUrl(input); + active--; + if (url.pathname === "/management/v1/customer_teams") { + const page = Number(url.searchParams.get("page")); + return json(list(manyTeams.slice((page - 1) * 100, page * 100), page, 3)); + } + if (url.pathname === "/management/v1/customer_members") return json(list([])); + if (url.pathname === "/management/v1/customer_teams/daily/activity") { + return json({ results: [], metadata: EMPTY_DAILY_ACTIVITY_METADATA }); + } + return serve(input); + }); + + await customerHierarchyUsageCall(request); + + expect(maximumActive).toBeLessThanOrEqual(4); + const teamFilters = dailyUrls() + .filter((url) => url.searchParams.has("customer_team_ids")) + .map((url) => url.searchParams.get("customer_team_ids")!.split(",")); + expect(teamFilters.map((ids) => ids.length)).toEqual([100, 100, 5]); + expect(new Set(teamFilters.flat()).size).toBe(manyTeams.length); + expect(dailyUrls().some((url) => url.searchParams.has("customer_member_ids"))).toBe(false); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.ts new file mode 100644 index 00000000000..3bfeeaebef5 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.ts @@ -0,0 +1,265 @@ +import { apiClient } from "@/components/networking"; +import { type DailyActivityAggregatedResponse } from "@/components/UsagePage/dailyActivityApi"; +import type { components } from "@/lib/http/schema"; +import i18n from "@/i18n/i18n"; +import { + mergeCustomerActivityPages, + customerEntityRankings, + customerKeyRankings, + customerEntityDailyData, + type CustomerUsageRow, +} from "./customerUsageData"; + +type Organization = components["schemas"]["Organization"]; +type Team = components["schemas"]["Team"]; +type Member = components["schemas"]["CustomerMember"]; +type Collection = "customer_organizations" | "customer_teams" | "customer_members"; +type UsageRow = Omit; + +export interface CustomerUsageEntity { + id: string; + label: string; + spend: number; + budgetLimit: number | null; + budgetResetAt: string | null; +} + +export interface CustomerHierarchyUsage extends ReturnType { + organization: CustomerUsageEntity; + team: CustomerUsageEntity | null; + member: CustomerUsageEntity | null; + teams: UsageRow[]; + members: UsageRow[]; + api_keys: UsageRow[]; + teamResults: ReturnType["results"]; + memberResults: ReturnType["results"]; + memberBudgets: CustomerUsageEntity[]; +} + +export interface CustomerUsageScope { + org: string; + team: string | null; + member: string | null; +} + +interface CustomerUsageRequest { + scope: CustomerUsageScope; + accessToken: string; + startDate: string; + endDate: string; + signal?: AbortSignal; +} + +interface ResolvedScope { + organization: Organization; + team: Team | null; + member: Member | null; +} + +const PREFIX = "/management/v1"; +const PAGE_SIZE = 100; +const MAX_PAGES = 1_000; +const MAX_CONCURRENT = 4; +const ID_FILTER: Record = { + customer_organizations: "customer_org_ids", + customer_teams: "customer_team_ids", + customer_members: "customer_member_ids", +}; + +const assertIdentifiers = (ids: readonly string[]) => { + if (ids.some((id) => !id || id.includes(","))) throw new Error(i18n.t("customerPortal.analytics.invalidIdentifier")); +}; + +const assertScope = (valid: boolean) => { + if (!valid) throw new Error(i18n.t("customerPortal.analytics.scopeMismatch")); +}; + +const batches = async (items: readonly T[], task: (item: T) => Promise): Promise => { + const results: R[] = []; + for (let start = 0; start < items.length; start += MAX_CONCURRENT) { + results.push(...(await Promise.all(items.slice(start, start + MAX_CONCURRENT).map(task)))); + } + return results; +}; + +const info = (resource: Collection, id: string, request: CustomerUsageRequest) => + apiClient.get(`${PREFIX}/${resource}/${encodeURIComponent(id)}`, { + accessToken: request.accessToken, + signal: request.signal, + }); + +const resolveScope = async (request: CustomerUsageRequest): Promise => { + const { scope } = request; + assertIdentifiers([scope.org, ...[scope.team, scope.member].filter((id): id is string => id !== null)]); + const [organization, requestedTeam, member] = await Promise.all([ + info("customer_organizations", scope.org, request), + scope.team ? info("customer_teams", scope.team, request) : null, + scope.member ? info("customer_members", scope.member, request) : null, + ]); + const team = requestedTeam ?? (member ? await info("customer_teams", member.parent_team_id, request) : null); + assertScope(organization.customer_org_id === scope.org); + assertScope(!team || team.parent_org_id === scope.org); + assertScope(!scope.team || team?.customer_team_id === scope.team); + assertScope(!member || member.customer_member_id === scope.member); + assertScope(!member || member.parent_team_id === team?.customer_team_id); + return { organization, team, member }; +}; + +const listAll = async ( + resource: Collection, + parent: Record, + request: CustomerUsageRequest, +): Promise => { + const rows: T[] = []; + for (let page = 1; page <= MAX_PAGES; page++) { + const result = await apiClient.get<{ data: T[]; meta: components["schemas"]["ListMeta"] }>( + `${PREFIX}/${resource}`, + { + accessToken: request.accessToken, + signal: request.signal, + query: { ...parent, page, page_size: PAGE_SIZE }, + }, + ); + rows.push(...result.data); + if (page >= result.meta.total_pages) return rows; + } + throw new Error(i18n.t("customerPortal.analytics.incompletePages")); +}; + +const descendants = async (scope: ResolvedScope, request: CustomerUsageRequest) => { + const teams = scope.team + ? [scope.team] + : await listAll("customer_teams", { parent_org_id: request.scope.org }, request); + if (teams.some((team) => team.parent_org_id !== request.scope.org)) + throw new Error(i18n.t("customerPortal.analytics.scopeMismatch")); + const memberLists = scope.member + ? [[scope.member]] + : await batches(teams, async (team) => { + const members = await listAll("customer_members", { parent_team_id: team.customer_team_id }, request); + if (members.some((member) => member.parent_team_id !== team.customer_team_id)) + throw new Error(i18n.t("customerPortal.analytics.scopeMismatch")); + return members; + }); + return { + teams: [...new Map(teams.map((team) => [team.customer_team_id, team])).values()], + members: [...new Map(memberLists.flat().map((member) => [member.customer_member_id, member])).values()], + }; +}; + +const activityPages = async (resource: Collection, ids: readonly string[], request: CustomerUsageRequest) => { + assertIdentifiers(ids); + const pages: DailyActivityAggregatedResponse[] = []; + for (let page = 1; page <= MAX_PAGES; page++) { + const result = await apiClient.get(`${PREFIX}/${resource}/daily/activity`, { + accessToken: request.accessToken, + signal: request.signal, + query: { + start_date: request.startDate, + end_date: request.endDate, + [ID_FILTER[resource]]: ids.join(","), + page, + page_size: PAGE_SIZE, + }, + }); + if (!result.metadata) throw new Error(i18n.t("customerPortal.analytics.incompletePages")); + pages.push(result); + if (!result.metadata.has_more && page >= result.metadata.total_pages) return pages; + } + throw new Error(i18n.t("customerPortal.analytics.incompletePages")); +}; + +interface ActivityJob { + dimension: "scope" | "teams" | "members"; + resource: Collection; + ids: readonly string[]; +} + +const chunkJobs = (resource: Collection, dimension: ActivityJob["dimension"], ids: readonly string[]): ActivityJob[] => + Array.from({ length: Math.ceil(ids.length / PAGE_SIZE) }, (_, index) => ({ + resource, + dimension, + ids: ids.slice(index * PAGE_SIZE, (index + 1) * PAGE_SIZE), + })); + +const entity = (id: string, record: Organization | Team | Member): CustomerUsageEntity => ({ + id, + label: record.alias || id, + spend: record.spend, + budgetLimit: ("effective_budget" in record ? record.effective_budget : record.budget)?.max_budget ?? null, + budgetResetAt: record.budget_reset_at ?? null, +}); + +const selectedResource = (scope: ResolvedScope): Collection => { + if (scope.member) return "customer_members"; + if (scope.team) return "customer_teams"; + return "customer_organizations"; +}; + +export const customerHierarchyUsageCall = async (request: CustomerUsageRequest): Promise => { + const selected = await resolveScope(request); + const children = await descendants(selected, request); + const resource = selectedResource(selected); + const selectedId = selected.member?.customer_member_id ?? selected.team?.customer_team_id ?? request.scope.org; + const jobs: ActivityJob[] = [ + { dimension: "scope", resource, ids: [selectedId] }, + ...(!selected.team + ? chunkJobs( + "customer_teams", + "teams", + children.teams.map((team) => team.customer_team_id), + ) + : []), + ...(!selected.member + ? chunkJobs( + "customer_members", + "members", + children.members.map((member) => member.customer_member_id), + ) + : []), + ]; + const activity = await batches(jobs, async (job) => ({ + dimension: job.dimension, + pages: await activityPages(job.resource, job.ids, request), + })); + const primary = mergeCustomerActivityPages( + activity.filter((result) => result.dimension === "scope").flatMap((result) => result.pages), + ); + const teamDays = selected.team + ? primary.results + : mergeCustomerActivityPages( + activity.filter((result) => result.dimension === "teams").flatMap((result) => result.pages), + ).results; + const memberDays = selected.member + ? primary.results + : mergeCustomerActivityPages( + activity.filter((result) => result.dimension === "members").flatMap((result) => result.pages), + ).results; + const teamLabels = new Map( + children.teams.map((team) => [team.customer_team_id, team.alias || team.customer_team_id]), + ); + const memberLabels = new Map( + children.members.map((member) => [member.customer_member_id, member.alias || member.customer_member_id]), + ); + return { + ...primary, + organization: entity(selected.organization.customer_org_id, selected.organization), + team: selected.team ? entity(selected.team.customer_team_id, selected.team) : null, + member: selected.member ? entity(selected.member.customer_member_id, selected.member) : null, + teams: customerEntityRankings(teamDays, teamLabels, primary.metadata, { + unassignedLabel: i18n.t("customerPortal.analytics.unassignedTeam"), + onlyEntityId: selected.member ? selected.team?.customer_team_id : undefined, + }), + members: customerEntityRankings(memberDays, memberLabels, primary.metadata, { + unassignedLabel: i18n.t("customerPortal.analytics.unassignedMember"), + }), + api_keys: customerKeyRankings(primary.results, i18n.t("customerPortal.analytics.unnamedKey")), + teamResults: customerEntityDailyData( + primary.results, + teamDays, + teamLabels, + selected.member ? selected.team?.customer_team_id : undefined, + ), + memberResults: customerEntityDailyData(primary.results, memberDays, memberLabels), + memberBudgets: children.members.map((member) => entity(member.customer_member_id, member)), + }; +}; diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.test.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.test.ts new file mode 100644 index 00000000000..2d2a46178e4 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, it } from "vitest"; +import { + bucketTotals, + OTHER_COLOR, + type Series, + type UsageMetric, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { stackedUsageColor } from "@/components/shared/charts"; +import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; +import type { DailyData, MetricWithMetadata, SpendMetrics } from "@/components/UsagePage/types"; +import i18n from "@/i18n/i18n"; +import type { CustomerHierarchyUsage } from "./customerUsageApi"; +import { customerUsageSeries } from "./customerUsageChartData"; +import type { CustomerUsageDimension } from "./CustomerUsageRankings"; + +const metrics = (spend: number, tokens: number, requests: number): SpendMetrics => ({ + spend, + prompt_tokens: tokens, + completion_tokens: 0, + total_tokens: tokens, + api_requests: requests, + successful_requests: requests, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}); + +const entry = (spend: number, tokens: number, requests: number): MetricWithMetadata => ({ + metrics: metrics(spend, tokens, requests), + metadata: {}, + api_key_breakdown: {}, +}); + +const day = (date: string, total: SpendMetrics, breakdown: Partial = {}): DailyData => ({ + date, + metrics: total, + breakdown: { models: {}, model_groups: {}, providers: {}, mcp_servers: {}, api_keys: {}, entities: {}, ...breakdown }, +}); + +const snapshot = (): CustomerHierarchyUsage => ({ + organization: { id: "org", label: "Organization", spend: 0, budgetLimit: null, budgetResetAt: null }, + team: null, + member: null, + memberBudgets: [], + metadata: { ...EMPTY_DAILY_ACTIVITY_METADATA }, + results: [ + day("2026-11-01", metrics(100, 1_000, 100), { + models: { deployed: entry(100, 1_000, 100) }, + model_groups: { alpha: entry(60, 300, 20), beta: entry(40, 700, 80) }, + api_keys: { + "hash-a": { metrics: metrics(30, 800, 70), metadata: { key_alias: "Support", team_id: null } }, + "hash-b": { metrics: metrics(70, 200, 30), metadata: { key_alias: "Search", team_id: null } }, + }, + }), + day("2026-11-02", metrics(60, 900, 30)), + day("2026-11-03", metrics(50, 500, 40)), + ], + teamResults: [ + day("2026-11-01", metrics(75, 800, 60), { + entities: { "team-a": entry(25, 100, 40), "team-b": entry(50, 700, 20) }, + }), + day("2026-11-02", metrics(10, 90, 5), { entities: { "team-a": entry(10, 90, 5) } }), + day("2026-11-09", metrics(999, 999, 999), { entities: { "team-a": entry(999, 999, 999) } }), + ], + memberResults: [ + day("2026-11-01", metrics(80, 700, 80), { + entities: { "member-a": entry(20, 200, 30), "member-b": entry(60, 500, 50) }, + }), + ], + teams: [ + { id: "team-a", label: "Product", spend: 35, tokens: 190, requests: 45 }, + { id: "team-b", label: "Operations", spend: 50, tokens: 700, requests: 20 }, + ], + members: [ + { id: "member-a", label: "Member A", spend: 20, tokens: 200, requests: 30 }, + { id: "member-b", label: "Member B", spend: 60, tokens: 500, requests: 50 }, + ], + api_keys: [ + { id: "hash-a", label: "Support", spend: 30, tokens: 800, requests: 70 }, + { id: "hash-b", label: "Search", spend: 70, tokens: 200, requests: 30 }, + ], +}); + +const modelColors = new Map([ + ["alpha", stackedUsageColor(3)], + ["beta", stackedUsageColor(4)], +]); +const valueAt = (series: Series, label: string, date = "2026-11-01"): number => { + const index = series.labels.indexOf(label); + return Number(series.data.find((row) => row.date === date)?.[series.keys[index]] ?? 0); +}; +const colorOf = (series: Series, label: string): string => series.colors[series.labels.indexOf(label)]; +const other = () => i18n.t("usage.overview.otherSeries"); + +describe("customerUsageSeries", () => { + it.each<[CustomerUsageDimension, string, number]>([ + ["models", "alpha", 60], + ["api_keys", "Support", 30], + ["teams", "Product", 25], + ])("should use %s values without adding the other breakdowns", (dimension, label, value) => { + const series = customerUsageSeries(snapshot(), dimension, "spend", modelColors); + + expect(valueAt(series, label)).toBe(value); + expect(bucketTotals(series).get("2026-11-01")).toBe(100); + expect(series.labels).not.toContain("deployed"); + }); + + it.each<[UsageMetric, number, number, number]>([ + ["spend", 25, 50, 25], + ["tokens", 100, 700, 200], + ["requests", 40, 20, 40], + ])("should use %s for both named teams and the unattributed remainder", (metric, first, second, remainder) => { + const series = customerUsageSeries(snapshot(), "teams", metric, modelColors); + + expect(valueAt(series, "Product")).toBe(first); + expect(valueAt(series, "Operations")).toBe(second); + expect(valueAt(series, other())).toBe(remainder); + expect(colorOf(series, other())).toBe(OTHER_COLOR); + }); + + it("should preserve all selected-scope dates and totals when companion records are missing", () => { + const series = customerUsageSeries(snapshot(), "teams", "spend", modelColors); + + expect(series.data.map((row) => row.date)).toEqual(["2026-11-01", "2026-11-02", "2026-11-03"]); + expect(valueAt(series, "Product", "2026-11-02")).toBe(10); + expect(valueAt(series, other(), "2026-11-02")).toBe(50); + expect(valueAt(series, "Product", "2026-11-03")).toBe(0); + expect(valueAt(series, other(), "2026-11-03")).toBe(50); + expect([...bucketTotals(series).values()]).toEqual([100, 60, 50]); + }); + + it("should include removed and unnamed descendants in Other rather than a misleading named bucket", () => { + const data = snapshot(); + const removed = day("2026-11-01", metrics(100, 1_000, 100), { + entities: { "team-a": entry(25, 100, 40), removed: entry(10, 150, 10), "": entry(5, 50, 5) }, + }); + const series = customerUsageSeries({ ...data, teamResults: [removed] }, "teams", "tokens", modelColors); + + expect(series.labels).toEqual(["Product", other()]); + expect(valueAt(series, "Product")).toBe(100); + expect(valueAt(series, other())).toBe(900); + }); + + it("should keep spend-ranked palette colors attached to API keys when the selected metric changes", () => { + const spend = customerUsageSeries(snapshot(), "api_keys", "spend", modelColors); + const tokens = customerUsageSeries(snapshot(), "api_keys", "tokens", modelColors); + + expect(spend.labels.slice(0, 2)).toEqual(["Search", "Support"]); + expect(tokens.labels.slice(0, 2)).toEqual(["Support", "Search"]); + expect(colorOf(spend, "Search")).toBe(stackedUsageColor(0)); + expect(colorOf(tokens, "Search")).toBe(stackedUsageColor(0)); + expect(colorOf(spend, "Support")).toBe(stackedUsageColor(1)); + expect(colorOf(tokens, "Support")).toBe(stackedUsageColor(1)); + }); + + it("should use key aliases and hide hashes even when a label is missing or repeats the hash", () => { + const data = snapshot(); + const series = customerUsageSeries( + { ...data, api_keys: [{ ...data.api_keys[0], label: " hash-a " }] }, + "api_keys", + "spend", + modelColors, + ); + + expect(series.labels).not.toContain("hash-a"); + expect(series.labels).not.toContain("hash-b"); + expect(series.labels.filter((label) => label === i18n.t("customerPortal.analytics.unnamedKey"))).toHaveLength(2); + expect(bucketTotals(series).get("2026-11-01")).toBe(100); + }); + + it("should preserve model colors and fall back to the models breakdown when groups are absent", () => { + const data = snapshot(); + const color = stackedUsageColor(5); + const series = customerUsageSeries( + { ...data, results: [day("2026-11-01", metrics(7, 70, 2), { models: { deployed: entry(7, 70, 2) } })] }, + "models", + "tokens", + new Map([["deployed", color]]), + ); + + expect(series.labels).toEqual(["deployed"]); + expect(valueAt(series, "deployed")).toBe(70); + expect(colorOf(series, "deployed")).toBe(color); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.ts new file mode 100644 index 00000000000..804d50bfbe8 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.ts @@ -0,0 +1,71 @@ +import { + rankValue, + rollUpBreakdown, + seriesBy, + type Series, + type UsageMetric, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { stackedUsageColor } from "@/components/shared/charts"; +import type { DailyData } from "@/components/UsagePage/types"; +import i18n from "@/i18n/i18n"; +import type { CustomerHierarchyUsage } from "./customerUsageApi"; +import type { CustomerUsageDimension } from "./CustomerUsageRankings"; + +const TOP_SERIES = 8; + +const entityDays = ( + primary: readonly DailyData[], + entities: readonly DailyData[], + labels: ReadonlyMap, +): DailyData[] => { + const byDate = new Map(entities.map((day) => [day.date, day])); + return primary.map((day) => ({ + ...day, + breakdown: { + ...day.breakdown, + model_groups: Object.fromEntries( + Object.entries(byDate.get(day.date)?.breakdown.entities ?? {}).filter(([id]) => id && labels.has(id)), + ), + }, + })); +}; + +export const customerUsageSeries = ( + data: CustomerHierarchyUsage, + dimension: CustomerUsageDimension, + metric: UsageMetric, + colorByModel: ReadonlyMap, +): Series => { + if (dimension === "models") { + const days = data.results.map((day) => ({ + ...day, + breakdown: { + ...day.breakdown, + model_groups: + Object.keys(day.breakdown.model_groups).length > 0 ? day.breakdown.model_groups : day.breakdown.models, + }, + })); + return seriesBy(days, "model_groups", metric, { top: TOP_SERIES, colorByKey: colorByModel }); + } + + const unnamedKey = i18n.t("customerPortal.analytics.unnamedKey"); + const rows = data[dimension]; + const labels = new Map( + rows + .filter((row) => row.id) + .map((row) => { + const label = row.label.trim(); + return [row.id, dimension === "api_keys" && (!label || label === row.id) ? unnamedKey : label]; + }), + ); + const days = dimension === "api_keys" ? data.results : entityDays(data.results, data.teamResults, labels); + const breakdown = dimension === "api_keys" ? "api_keys" : "model_groups"; + const ranking = rollUpBreakdown(days, breakdown).sort((a, b) => b.spend - a.spend || a.key.localeCompare(b.key)); + const colors = new Map(ranking.map((row, index) => [row.key, stackedUsageColor(index)])); + const series = seriesBy(days, breakdown, metric, { top: TOP_SERIES, colorByKey: colors }); + const namedCount = Math.min(TOP_SERIES, ranking.filter((row) => rankValue(row, metric) > 0).length); + return { + ...series, + labels: series.labels.map((id, index) => (index < namedCount ? labels.get(id) ?? unnamedKey : id)), + }; +}; diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.test.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.test.ts new file mode 100644 index 00000000000..c8e6d638c79 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "vitest"; +import type { DailyData, MetricWithMetadata, SpendMetrics } from "@/components/UsagePage/types"; +import { rankCustomerModels, rankCustomerUsage, type CustomerUsageRow } from "./customerUsageData"; + +const metrics = (spend: number, tokens: number, requests: number): SpendMetrics => ({ + spend, + prompt_tokens: tokens, + completion_tokens: 0, + total_tokens: tokens, + api_requests: requests, + successful_requests: requests, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}); + +const model = (spend: number, tokens: number, requests: number): MetricWithMetadata => ({ + metrics: metrics(spend, tokens, requests), + metadata: {}, + api_key_breakdown: {}, +}); + +const day = ( + date: string, + modelGroups: Record, + models: Record = {}, +): DailyData => ({ + date, + metrics: metrics(0, 0, 0), + breakdown: { + models, + model_groups: modelGroups, + providers: {}, + mcp_servers: {}, + api_keys: {}, + entities: {}, + }, +}); + +const totals = { spend: 10, tokens: 1_000, requests: 20 }; + +describe("rankCustomerUsage", () => { + it("should sort by the selected metric and calculate shares against overall customer totals", () => { + const rows = [ + { id: "assistant", label: "Assistant", spend: 3, tokens: 100, requests: 8 }, + { id: "search", label: "Search", spend: 1, tokens: 500, requests: 2 }, + ]; + + expect(rankCustomerUsage(rows, "spend", totals).map((row) => [row.label, row.share])).toEqual([ + ["Assistant", 0.3], + ["Search", 0.1], + ]); + expect(rankCustomerUsage(rows, "tokens", totals).map((row) => [row.label, row.share])).toEqual([ + ["Search", 0.5], + ["Assistant", 0.1], + ]); + expect(rankCustomerUsage(rows, "requests", totals).map((row) => [row.label, row.share])).toEqual([ + ["Assistant", 0.4], + ["Search", 0.1], + ]); + }); + + it("should preserve zero-cost token usage while removing rows with no activity", () => { + const freeUsage = { id: "assistant", label: "Assistant", spend: 0, tokens: 200, requests: 3 }; + const inactive = { id: "inactive", label: "Inactive", spend: 0, tokens: 0, requests: 0 }; + + expect(rankCustomerUsage([inactive, freeUsage], "tokens", totals)).toEqual([{ ...freeUsage, share: 0.2 }]); + expect(rankCustomerUsage([freeUsage], "spend", totals)).toEqual([{ ...freeUsage, share: 0 }]); + }); + + it("should use stable label and id order for ties without modifying the source rows", () => { + const rows: readonly Omit[] = Object.freeze([ + { id: "b", label: "Search", spend: 1, tokens: 100, requests: 1 }, + { id: "c", label: "Assistant", spend: 1, tokens: 100, requests: 1 }, + { id: "a", label: "Assistant", spend: 1, tokens: 100, requests: 1 }, + ]); + expect(rankCustomerUsage(rows, "tokens", totals).map((row) => row.id)).toEqual(["a", "c", "b"]); + expect(rows.map((row) => row.id)).toEqual(["b", "c", "a"]); + }); + + it("should return finite zero shares when the overall metric is zero", () => { + const rows = [{ id: "assistant", label: "Assistant", spend: 0, tokens: 200, requests: 1 }]; + expect(rankCustomerUsage(rows, "spend", { spend: 0, tokens: 200, requests: 1 })[0].share).toBe(0); + expect(rankCustomerUsage([], "spend", totals)).toEqual([]); + }); +}); + +describe("rankCustomerModels", () => { + it("should sum models across days without adding model and model-group totals together", () => { + const days = [ + day("2026-11-01", { assistant: model(2, 100, 3) }, { assistant: model(2, 100, 3) }), + day("2026-11-02", { assistant: model(1, 200, 2), search: model(0, 400, 1) }), + day("2026-11-03", {}, { assistant: model(1, 100, 1) }), + ]; + expect(rankCustomerModels(days, "tokens", totals)).toEqual([ + { id: "assistant", label: "assistant", spend: 4, tokens: 400, requests: 6, share: 0.4 }, + { id: "search", label: "search", spend: 0, tokens: 400, requests: 1, share: 0.4 }, + ]); + }); + + it("should return no model rows for an empty date range", () => { + expect(rankCustomerModels([], "spend", totals)).toEqual([]); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.ts new file mode 100644 index 00000000000..afa0d3c7767 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.ts @@ -0,0 +1,262 @@ +import type { DailyData, SpendMetrics, BreakdownMetrics, MetricWithMetadata } from "@/components/UsagePage/types"; +import { + EMPTY_DAILY_ACTIVITY_METADATA, + toDailyData, + type DailyActivityAggregatedResponse, + type DailyActivityMetadata, +} from "@/components/UsagePage/dailyActivityApi"; +import { + rollUpBreakdown, + type UsageMetric, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; + +export interface CustomerUsageTotals { + spend: number; + tokens: number; + requests: number; +} + +export interface CustomerUsageRow extends CustomerUsageTotals { + id: string; + label: string; + share: number; +} + +export const rankCustomerUsage = ( + rows: readonly Omit[], + metric: UsageMetric, + totals: CustomerUsageTotals, +): CustomerUsageRow[] => + rows + .filter((row) => row.spend > 0 || row.tokens > 0 || row.requests > 0) + .map((row) => ({ ...row, share: totals[metric] > 0 ? row[metric] / totals[metric] : 0 })) + .sort((a, b) => b[metric] - a[metric] || a.label.localeCompare(b.label) || a.id.localeCompare(b.id)); + +export const rankCustomerModels = ( + days: readonly DailyData[], + metric: UsageMetric, + totals: CustomerUsageTotals, +): CustomerUsageRow[] => { + const withModelGroups = days.map((day) => ({ + ...day, + breakdown: { + ...day.breakdown, + model_groups: + Object.keys(day.breakdown.model_groups).length > 0 ? day.breakdown.model_groups : day.breakdown.models, + }, + })); + return rankCustomerUsage( + rollUpBreakdown(withModelGroups, "model_groups").map((model) => ({ + id: model.key, + label: model.key, + spend: model.spend, + tokens: model.tokens, + requests: model.requests, + })), + metric, + totals, + ); +}; + +const ZERO_METRICS: SpendMetrics = { + spend: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + api_requests: 0, + successful_requests: 0, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}; + +const sumMetrics = (values: readonly SpendMetrics[]): SpendMetrics => + values.reduce( + (total, value) => ({ + ...total, + ...Object.fromEntries( + (Object.keys(value) as (keyof SpendMetrics)[]).map((field) => [ + field, + (total[field] ?? 0) + (value[field] ?? 0), + ]), + ), + }), + ZERO_METRICS, + ); + +const mergeKeys = ( + left: BreakdownMetrics["api_keys"], + right: BreakdownMetrics["api_keys"], +): BreakdownMetrics["api_keys"] => { + const result = { ...left }; + for (const [id, entry] of Object.entries(right)) { + const previous = result[id]; + result[id] = { + ...entry, + metrics: sumMetrics([...(previous ? [previous.metrics] : []), entry.metrics]), + metadata: { + ...previous?.metadata, + ...entry.metadata, + key_alias: entry.metadata.key_alias ?? previous?.metadata.key_alias ?? null, + }, + }; + } + return result; +}; + +const mergeMetrics = ( + left: Record, + right: Record, +): Record => { + const result = { ...left }; + for (const [id, entry] of Object.entries(right)) { + const previous = result[id]; + result[id] = { + ...entry, + metrics: sumMetrics([...(previous ? [previous.metrics] : []), entry.metrics]), + metadata: { ...previous?.metadata, ...entry.metadata }, + api_key_breakdown: mergeKeys(previous?.api_key_breakdown ?? {}, entry.api_key_breakdown), + }; + } + return result; +}; + +const mergeBreakdowns = (left: BreakdownMetrics, right: BreakdownMetrics): BreakdownMetrics => ({ + models: mergeMetrics(left.models, right.models), + model_groups: mergeMetrics(left.model_groups, right.model_groups), + providers: mergeMetrics(left.providers, right.providers), + mcp_servers: mergeMetrics(left.mcp_servers, right.mcp_servers), + endpoints: mergeMetrics(left.endpoints ?? {}, right.endpoints ?? {}), + entities: mergeMetrics(left.entities, right.entities), + api_keys: mergeKeys(left.api_keys, right.api_keys), +}); + +export const mergeCustomerActivityPages = ( + pages: readonly DailyActivityAggregatedResponse[], +): { results: DailyData[]; metadata: DailyActivityMetadata } => { + const days = new Map(); + for (const day of pages.flatMap(toDailyData)) { + const previous = days.get(day.date); + days.set( + day.date, + previous + ? { + date: day.date, + metrics: sumMetrics([previous.metrics, day.metrics]), + breakdown: mergeBreakdowns(previous.breakdown, day.breakdown), + } + : day, + ); + } + const results = [...days.values()].sort((a, b) => a.date.localeCompare(b.date)); + const total = sumMetrics(results.map((day) => day.metrics)); + return { + results, + metadata: { + ...EMPTY_DAILY_ACTIVITY_METADATA, + total_spend: total.spend, + total_flat_cost: total.flat_cost ?? 0, + total_ptu_hours: total.ptu_hours ?? 0, + total_prompt_tokens: total.prompt_tokens, + total_completion_tokens: total.completion_tokens, + total_tokens: total.total_tokens, + total_api_requests: total.api_requests, + total_successful_requests: total.successful_requests, + total_failed_requests: total.failed_requests, + total_cache_read_input_tokens: total.cache_read_input_tokens, + total_cache_creation_input_tokens: total.cache_creation_input_tokens, + total_compression_saved_tokens: total.compression_saved_tokens ?? 0, + total_compression_savings_spend: total.compression_savings_spend ?? 0, + total_prompt_caching_savings_spend: total.prompt_caching_savings_spend ?? 0, + total_gateway_injected_caching_savings_spend: total.gateway_injected_caching_savings_spend ?? 0, + total_autorouter_savings_spend: total.autorouter_savings_spend ?? 0, + total_response_time_ms: total.total_response_time_ms ?? 0, + total_timed_requests: total.timed_requests ?? 0, + total_api_keys: new Set(results.flatMap((day) => Object.keys(day.breakdown.api_keys))).size, + }, + }; +}; + +const usageTotals = (value: SpendMetrics): CustomerUsageTotals => ({ + spend: value.spend, + tokens: value.total_tokens, + requests: value.api_requests, +}); +const addTotals = (left: CustomerUsageTotals, right: CustomerUsageTotals): CustomerUsageTotals => ({ + spend: left.spend + right.spend, + tokens: left.tokens + right.tokens, + requests: left.requests + right.requests, +}); + +export const customerKeyRankings = ( + days: readonly DailyData[], + unnamedKey: string, +): Omit[] => { + const keys = new Map>(); + for (const [id, entry] of days.flatMap((day) => Object.entries(day.breakdown.api_keys))) { + const previous = keys.get(id); + keys.set(id, { + id, + label: entry.metadata.key_alias?.trim() || previous?.label || unnamedKey, + ...addTotals(previous ?? { spend: 0, tokens: 0, requests: 0 }, usageTotals(entry.metrics)), + }); + } + return [...keys.values()]; +}; + +interface EntityRankingOptions { + unassignedLabel: string; + onlyEntityId?: string; +} + +export const customerEntityDailyData = ( + primary: readonly DailyData[], + days: readonly DailyData[], + labels: ReadonlyMap, + onlyEntityId?: string, +): DailyData[] => { + const byDate = new Map(days.map((day) => [day.date, day])); + return primary.map((day) => { + const entities = onlyEntityId + ? { + [onlyEntityId]: { + metrics: day.metrics, + metadata: { alias: labels.get(onlyEntityId) }, + api_key_breakdown: day.breakdown.api_keys, + }, + } + : Object.fromEntries( + Object.entries(byDate.get(day.date)?.breakdown.entities ?? {}) + .filter(([id]) => labels.has(id)) + .map(([id, entry]) => [id, { ...entry, metadata: { ...entry.metadata, alias: labels.get(id) } }]), + ); + return { ...day, breakdown: { ...day.breakdown, entities } }; + }); +}; + +export const customerEntityRankings = ( + days: readonly DailyData[], + labels: ReadonlyMap, + metadata: DailyActivityMetadata, + options: EntityRankingOptions, +): Omit[] => { + const entries = new Map(); + for (const [id, entry] of days.flatMap((day) => Object.entries(day.breakdown.entities))) { + const identity = options.onlyEntityId ?? id; + if (labels.has(identity)) + entries.set( + identity, + addTotals(entries.get(identity) ?? { spend: 0, tokens: 0, requests: 0 }, usageTotals(entry.metrics)), + ); + } + const rows = [...entries].map(([id, total]) => ({ id, label: labels.get(id)!, ...total })); + const assigned = rows.reduce(addTotals, { spend: 0, tokens: 0, requests: 0 }); + const unassigned = { + spend: Math.max(0, metadata.total_spend - assigned.spend), + tokens: Math.max(0, metadata.total_tokens - assigned.tokens), + requests: Math.max(0, metadata.total_api_requests - assigned.requests), + }; + return unassigned.spend > 0.000001 || unassigned.tokens > 0 || unassigned.requests > 0 + ? [...rows, { id: "", label: options.unassignedLabel, ...unassigned }] + : rows; +}; diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/useCustomerHierarchyUsage.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/useCustomerHierarchyUsage.ts new file mode 100644 index 00000000000..05fa467d0e5 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/useCustomerHierarchyUsage.ts @@ -0,0 +1,122 @@ +import { useMemo } from "react"; +import { skipToken, useQuery } from "@tanstack/react-query"; +import { parseAsInteger, parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { + dateRangeFromParams, + dateRangePatch, + defaultUsageDateRange, + usageUrlParsers, +} from "@/app/(dashboard)/usage/_components/usageUrlState"; +import { + modelColorMap, + overviewTotals, + type Granularity, + type UsageMetric, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { formatDate } from "@/components/networking"; +import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; +import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; +import type { DailyData } from "@/components/UsagePage/types"; +import { customerHierarchyUsageCall, type CustomerUsageScope } from "./customerUsageApi"; +import { rankCustomerModels, rankCustomerUsage } from "./customerUsageData"; +import { CUSTOMER_USAGE_DIMENSIONS, type CustomerUsageDimension } from "./CustomerUsageRankings"; + +const usageParsers = { + org: parseAsString, + team: parseAsString, + member: parseAsString, + metric: parseAsStringLiteral(["spend", "tokens", "requests"] as const).withDefault("spend"), + breakdown: parseAsStringLiteral(CUSTOMER_USAGE_DIMENSIONS).withDefault("models"), + bucket: parseAsStringLiteral(["day", "week"] as const).withDefault("day"), + search: parseAsString.withDefault(""), + usagePage: parseAsInteger.withDefault(1), + range: usageUrlParsers.range, + from: usageUrlParsers.from, + to: usageUrlParsers.to, +}; +const EMPTY_RESULTS: readonly DailyData[] = []; +const ORGANIZATION_PATCH = { team: null, member: null, search: null, usagePage: null }; +const TEAM_PATCH = { member: null, search: null, usagePage: null }; + +const usageRequest = ( + scope: CustomerUsageScope | null, + accessToken: string | null, + startDate: string | undefined, + endDate: string | undefined, +) => { + if (!scope || !accessToken) return null; + if (!startDate || !endDate) return null; + return { scope, accessToken, startDate, endDate }; +}; + +export default function useCustomerHierarchyUsage() { + const { accessToken } = useAuthorized(); + const [params, setParams] = useQueryStates(usageParsers); + const defaultRange = useMemo(() => defaultUsageDateRange("month_to_date"), []); + const dateValue = useMemo(() => dateRangeFromParams(params) ?? defaultRange.value, [params, defaultRange]); + const preset = params.range ?? (params.from === null && params.to === null ? defaultRange.preset : null); + const startDate = dateValue.from ? formatDate(dateValue.from) : undefined; + const endDate = dateValue.to ? formatDate(dateValue.to) : undefined; + const scope = params.org ? { org: params.org, team: params.team, member: params.member } : null; + const request = usageRequest(scope, accessToken, startDate, endDate); + const usage = useQuery({ + queryKey: ["customer-hierarchy", "usage", params.org, params.team, params.member, startDate, endDate], + queryFn: request ? ({ signal }) => customerHierarchyUsageCall({ ...request, signal }) : skipToken, + }); + const data = usage.data; + const results = data?.results ?? EMPTY_RESULTS; + const totals = useMemo(() => overviewTotals(data?.metadata ?? EMPTY_DAILY_ACTIVITY_METADATA, null), [data]); + const modelColors = useMemo(() => modelColorMap(results), [results]); + const rows = useMemo( + () => + params.breakdown === "models" + ? rankCustomerModels(results, params.metric, totals) + : rankCustomerUsage(data?.[params.breakdown] ?? [], params.metric, totals), + [data, results, params.breakdown, params.metric, totals], + ); + const drillDown = (dimension: "teams" | "members", id: string) => { + const patch = { + ...(dimension === "teams" ? { team: id, member: null } : { member: id }), + search: null, + usagePage: null, + breakdown: null, + }; + void setParams(patch, { history: "push" }); + }; + const setDateRange = (value: DateRangePickerValue, selectedPreset: string | null) => { + const { range, from, to } = dateRangePatch(value, selectedPreset); + const patch = { range, from, to, usagePage: null }; + void setParams(patch); + }; + const setDimension = (breakdown: CustomerUsageDimension) => + void setParams({ breakdown, search: null, usagePage: null }); + const selectOrganization = () => void setParams(ORGANIZATION_PATCH, { history: "push" }); + const selectTeam = () => void setParams(TEAM_PATCH, { history: "push" }); + const setMetric = (metric: UsageMetric) => void setParams({ metric, usagePage: null }); + const setBucket = (bucket: Granularity) => void setParams({ bucket }); + const setSearch = (search: string) => void setParams({ search, usagePage: null }); + const setPage = (usagePage: number) => void setParams({ usagePage }); + + return { + params, + scope, + dateValue, + preset, + usage, + data, + results, + totals, + modelColors, + rows, + drillDown, + setDateRange, + setDimension, + selectOrganization, + selectTeam, + setMetric, + setBucket, + setSearch, + setPage, + }; +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.integration.test.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.integration.test.tsx new file mode 100644 index 00000000000..852d85f3320 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.integration.test.tsx @@ -0,0 +1,379 @@ +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { registerBaseUrlGetter } from "@/lib/http/runtime"; +import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; +import type { SpendMetrics } from "@/components/UsagePage/types"; +import { + act, + chooseSelectOption, + fireEvent, + renderWithProviders, + screen, + testQueryClient, + waitFor, + within, +} from "../../../../../tests/test-utils"; +import CustomerHierarchyUsagePage from "./page"; + +const metrics = (spend: number, tokens: number, requests: number): SpendMetrics => ({ + spend, + api_requests: requests, + successful_requests: requests, + failed_requests: 0, + total_tokens: tokens, + prompt_tokens: tokens, + completion_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}); +const audit = { + created_at: "2026-11-01T00:00:00Z", + created_by: "admin", + updated_at: "2026-11-01T00:00:00Z", + updated_by: "admin", + spend: 40, +}; +const organization = { + ...audit, + customer_org_id: "org-1", + alias: "Customer organization", + budget: { max_budget: 100 }, + default_budgets: [], +}; +const teams = [ + { ...audit, customer_team_id: "team-a", parent_org_id: "org-1", alias: "Product", budget: null }, + { ...audit, customer_team_id: "team-b", parent_org_id: "org-1", alias: "Operations", budget: null }, +]; +const members = [ + { + ...audit, + customer_member_id: "member-a", + parent_team_id: "team-a", + alias: "Member One", + effective_budget: { max_budget: 10 }, + }, + { + ...audit, + customer_member_id: "member-b", + parent_team_id: "team-b", + alias: "Member Two", + effective_budget: { max_budget: 20 }, + }, +]; +const records = [ + { + organization: "org-1", + team: "team-a", + member: "member-a", + model: "model-a", + key: "hash-a", + alias: "Assistant key", + metrics: metrics(12, 100, 2), + date: "2026-11-01", + }, + { + organization: "org-1", + team: "team-b", + member: "member-b", + model: "model-b", + key: "hash-b", + alias: "Search key", + metrics: metrics(3, 600, 5), + date: "2026-11-02", + }, +]; +const fetchMock = vi.fn(); +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } }); +const requestUrl = (input: Parameters[0]) => + new URL(input instanceof Request ? input.url : String(input), "http://localhost"); +const requestedUrls = () => fetchMock.mock.calls.map(([input]) => requestUrl(input)); +const initialUrl = "?org=org-1&from=2026-11-01&to=2026-11-03&source=portal"; +const tableRows = (name: string) => within(screen.getByRole("table", { name })).getAllByRole("row").slice(1); +const list = (data: T[]) => ({ data, meta: { total_count: data.length, page: 1, page_size: 100, total_pages: 1 } }); +const selectBreakdown = (user: ReturnType, name: string) => + chooseSelectOption(user, screen.getByRole("combobox", { name: "Break down by" }), name); +const activityDimension = (resource: string | undefined): "organization" | "team" | "member" => { + if (resource === "customer_organizations") return "organization"; + if (resource === "customer_teams") return "team"; + return "member"; +}; +const filterFor = (resource: string | undefined) => + ({ + organization: "customer_org_ids", + team: "customer_team_ids", + member: "customer_member_ids", + })[activityDimension(resource)]; +const activityUrl = (resource: string, id: string) => { + const filter = filterFor(resource); + return requestedUrls().findLast( + (url) => url.pathname === `/management/v1/${resource}/daily/activity` && url.searchParams.get(filter) === id, + ); +}; +const serve = (input: Parameters[0]): Response => { + const url = requestUrl(input); + if (url.pathname.endsWith("/daily/activity")) { + const resource = url.pathname.split("/").at(-3); + const dimension = activityDimension(resource); + const filter = filterFor(resource); + const ids = url.searchParams.get(filter)?.split(",") ?? []; + const selected = records.filter((record) => ids.includes(record[dimension])); + const results = selected.map((record) => { + const key = { metrics: record.metrics, metadata: { key_alias: record.alias, team_id: null } }; + const entry = { metrics: record.metrics, metadata: {}, api_key_breakdown: { [record.key]: key } }; + return { + date: record.date, + metrics: record.metrics, + breakdown: { + model_groups: { [record.model]: entry }, + api_keys: { [record.key]: key }, + entities: { [record[dimension]]: entry }, + }, + }; + }); + return json({ results, metadata: EMPTY_DAILY_ACTIVITY_METADATA }); + } + if (url.pathname === "/management/v1/customer_organizations/org-1") return json(organization); + if (url.pathname === "/management/v1/customer_teams") return json(list(teams)); + if (url.pathname === "/management/v1/customer_members") + return json(list(members.filter((member) => member.parent_team_id === url.searchParams.get("parent_team_id")))); + const team = teams.find((team) => url.pathname.endsWith(`/customer_teams/${team.customer_team_id}`)); + if (team) return json(team); + const member = members.find((member) => url.pathname.endsWith(`/customer_members/${member.customer_member_id}`)); + if (member) return json(member); + throw new Error(`Unexpected usage request ${url.pathname}`); +}; + +describe("CustomerHierarchyUsagePage", () => { + beforeEach(() => { + testQueryClient.clear(); + registerBaseUrlGetter(() => "http://localhost"); + fetchMock.mockReset(); + fetchMock.mockImplementation((input) => Promise.resolve(serve(input))); + vi.stubGlobal("fetch", fetchMock); + vi.stubGlobal("requestIdleCallback", (callback: IdleRequestCallback) => { + callback({ didTimeout: false, timeRemaining: () => 50 }); + return 1; + }); + }); + + afterEach(() => { + testQueryClient.clear(); + vi.unstubAllGlobals(); + }); + + it("should render organization totals and model rankings from the selected UTC date range", async () => { + renderWithProviders(, { searchParams: initialUrl }); + + expect(await screen.findByRole("heading", { name: "Customer organization" })).toBeInTheDocument(); + expect(screen.getByText("$15.00")).toBeInTheDocument(); + expect(screen.getByTitle("700 tokens")).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "Usage over time" })).toBeInTheDocument(); + expect(tableRows("Models")[0]).toHaveTextContent("model-a"); + expect(within(tableRows("Models")[0]).getByText("80%")).toBeInTheDocument(); + expect(tableRows("Models")[1]).toHaveTextContent("model-b"); + expect(activityUrl("customer_organizations", "org-1")?.pathname).toBe( + "/management/v1/customer_organizations/daily/activity", + ); + expect(activityUrl("customer_organizations", "org-1")?.searchParams.get("start_date")).toBe("2026-11-01"); + expect(activityUrl("customer_organizations", "org-1")?.searchParams.get("end_date")).toBe("2026-11-03"); + }); + + it("should apply the global metric to sorting and shares across models, keys and teams", async () => { + const user = userEvent.setup(); + renderWithProviders(, { searchParams: initialUrl }); + await screen.findByRole("table", { name: "Models" }); + + await user.click(screen.getByRole("radio", { name: "Tokens" })); + expect(tableRows("Models")[0]).toHaveTextContent("model-b"); + expect(within(tableRows("Models")[0]).getByText("85.7%")).toBeInTheDocument(); + + for (const [tab, leader] of [ + ["API keys", "Search key"], + ["Teams", "Operations"], + ]) { + await selectBreakdown(user, tab); + expect(tableRows(tab)[0]).toHaveTextContent(leader); + expect(within(tableRows(tab)[0]).getByText("85.7%")).toBeInTheDocument(); + expect(within(screen.getByRole("region", { name: "Usage over time" })).getByText(leader)).toBeInTheDocument(); + } + + await user.click(screen.getByRole("radio", { name: "Requests" })); + expect(tableRows("Teams")[0]).toHaveTextContent("Operations"); + expect(within(tableRows("Teams")[0]).getByText("71.4%")).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Requests" })).toHaveAttribute("aria-sort", "descending"); + }); + + it("should drill into customer teams and members and retain dates and unrelated URL state", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { searchParams: initialUrl, onUrlUpdate }); + await screen.findByRole("heading", { name: "Customer organization" }); + + await selectBreakdown(user, "Teams"); + await user.click(within(screen.getByRole("table", { name: "Teams" })).getByRole("button", { name: "Product" })); + expect(await screen.findByRole("heading", { name: "Product" })).toBeInTheDocument(); + expect(activityUrl("customer_teams", "team-a")?.pathname).toBe("/management/v1/customer_teams/daily/activity"); + expect(activityUrl("customer_teams", "team-a")?.searchParams.get("customer_team_ids")).toBe("team-a"); + + await user.click( + within(screen.getByRole("table", { name: "Member spend" })).getByRole("button", { name: "Member One" }), + ); + expect(await screen.findByRole("heading", { name: "Member One" })).toBeInTheDocument(); + const memberUrl = activityUrl("customer_members", "member-a")!; + expect(memberUrl.pathname).toBe("/management/v1/customer_members/daily/activity"); + expect(memberUrl.searchParams.get("customer_member_ids")).toBe("member-a"); + expect(memberUrl.searchParams.has("customer_org_ids")).toBe(false); + expect(memberUrl.searchParams.has("customer_team_ids")).toBe(false); + expect(memberUrl.searchParams.get("start_date")).toBe("2026-11-01"); + expect(memberUrl.searchParams.get("end_date")).toBe("2026-11-03"); + expect(onUrlUpdate).toHaveBeenLastCalledWith( + expect.objectContaining({ + queryString: "?org=org-1&from=2026-11-01&to=2026-11-03&source=portal&team=team-a&member=member-a", + }), + ); + + await user.click(screen.getByRole("button", { name: "Customer organization" })); + expect(await screen.findByRole("heading", { name: "Customer organization" })).toBeInTheDocument(); + expect(onUrlUpdate).toHaveBeenLastCalledWith(expect.objectContaining({ queryString: initialUrl })); + }); + + it("should fall back to Models for a removed Members breakdown and offer only chart breakdowns", async () => { + const user = userEvent.setup(); + renderWithProviders(, { searchParams: `${initialUrl}&breakdown=members` }); + + expect(await screen.findByRole("table", { name: "Models" })).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "Top models" })).toBeInTheDocument(); + expect(screen.getByRole("table", { name: "Member spend" })).toBeInTheDocument(); + await user.click(screen.getByRole("combobox", { name: "Break down by" })); + expect((await screen.findAllByRole("option")).map((option) => option.textContent)).toEqual([ + "Models", + "API keys", + "Teams", + ]); + }); + + it("should retain a separate member spend table with range usage and current budget status across chart breakdowns", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { searchParams: initialUrl, onUrlUpdate }); + const table = await screen.findByRole("table", { name: "Member spend" }); + const firstMember = within(table).getAllByRole("row")[1]; + expect(firstMember).toHaveTextContent("Member One"); + expect(within(firstMember).getByText("$12.00")).toBeInTheDocument(); + expect(within(firstMember).getByText("$40.00 / $10.00")).toBeInTheDocument(); + expect(within(firstMember).getByText("Over budget")).toBeInTheDocument(); + + await selectBreakdown(user, "Teams"); + await user.click(screen.getByRole("radio", { name: "Tokens" })); + + expect(tableRows("Teams")[0]).toHaveTextContent("Operations"); + expect(within(table).getAllByRole("row")[1]).toHaveTextContent("Member One"); + const requestCount = fetchMock.mock.calls.length; + act(() => + fireEvent.change(screen.getByRole("textbox", { name: "Search members" }), { target: { value: "Member One" } }), + ); + expect(within(table).queryByText("Member Two")).not.toBeInTheDocument(); + expect(within(table).getByText("Member One")).toBeInTheDocument(); + await waitFor(() => + expect(new URLSearchParams(onUrlUpdate.mock.lastCall?.[0].queryString).get("memberSearch")).toBe("Member One"), + ); + const updatedUrl = new URLSearchParams(onUrlUpdate.mock.lastCall?.[0].queryString); + expect(updatedUrl.get("org")).toBe("org-1"); + expect(updatedUrl.get("breakdown")).toBe("teams"); + expect(updatedUrl.get("metric")).toBe("tokens"); + expect(updatedUrl.get("source")).toBe("portal"); + expect(fetchMock.mock.calls).toHaveLength(requestCount); + }); + + it("should clear the previous scope values while customer team usage loads", async () => { + const user = userEvent.setup(); + renderWithProviders(, { searchParams: initialUrl }); + await screen.findByRole("heading", { name: "Customer organization" }); + await selectBreakdown(user, "Teams"); + fetchMock.mockImplementation(() => new Promise(() => {})); + + await user.click(within(screen.getByRole("table", { name: "Teams" })).getByRole("button", { name: "Product" })); + + expect(screen.getByRole("status", { name: "Loading customer usage" })).toBeInTheDocument(); + expect(screen.queryByText("$15.00")).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Customer organization" })).not.toBeInTheDocument(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + await waitFor(() => + expect(requestedUrls().some((url) => url.pathname === "/management/v1/customer_teams/team-a")).toBe(true), + ); + }); + + it("should request a changed date range without showing stale totals or losing the customer scope", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { searchParams: initialUrl, onUrlUpdate }); + await screen.findByRole("heading", { name: "Customer organization" }); + fetchMock.mockImplementation((input) => + requestUrl(input).pathname.endsWith("/daily/activity") && + requestUrl(input).searchParams.get("start_date") === "2026-10-01" + ? new Promise(() => {}) + : Promise.resolve(serve(input)), + ); + + await user.click(screen.getByRole("button", { name: / - / })); + act(() => { + fireEvent.change(screen.getByLabelText("Start date"), { target: { value: "2026-10-01" } }); + fireEvent.change(screen.getByLabelText("End date"), { target: { value: "2026-10-07" } }); + }); + await user.click(screen.getByRole("button", { name: "Apply" })); + + await waitFor(() => + expect(activityUrl("customer_organizations", "org-1")?.searchParams.get("start_date")).toBe("2026-10-01"), + ); + expect(activityUrl("customer_organizations", "org-1")?.searchParams.get("end_date")).toBe("2026-10-07"); + expect(activityUrl("customer_organizations", "org-1")?.pathname).toBe( + "/management/v1/customer_organizations/daily/activity", + ); + expect(screen.getByRole("status", { name: "Loading customer usage" })).toBeInTheDocument(); + expect(screen.queryByText("$15.00")).not.toBeInTheDocument(); + expect(onUrlUpdate).toHaveBeenLastCalledWith( + expect.objectContaining({ queryString: "?org=org-1&from=2026-10-01&to=2026-10-07&source=portal" }), + ); + }); + + it("should show a failed request as an error and load usage when retried", async () => { + const user = userEvent.setup(); + fetchMock.mockResolvedValueOnce(json({ error: "Usage temporarily unavailable" }, 503)); + renderWithProviders(, { searchParams: initialUrl }); + + expect(await screen.findByRole("alert")).toHaveTextContent("Could not load customer usage"); + expect(screen.getByRole("alert")).toHaveTextContent("Usage temporarily unavailable"); + expect(screen.queryByText("$0.00")).not.toBeInTheDocument(); + expect(screen.queryByText("No usage in this range")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Retry" })); + + expect(await screen.findByText("$15.00")).toBeInTheDocument(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + }); + + it("should show an empty date range after the scoped request succeeds without activity", async () => { + fetchMock.mockImplementation((input) => + Promise.resolve( + requestUrl(input).pathname.endsWith("/daily/activity") + ? json({ results: [], metadata: EMPTY_DAILY_ACTIVITY_METADATA }) + : serve(input), + ), + ); + renderWithProviders(, { searchParams: initialUrl }); + + expect(await screen.findByText("No usage in this range")).toBeInTheDocument(); + expect(screen.getByTitle("0 tokens")).toBeInTheDocument(); + expect(screen.queryByRole("status", { name: "Loading customer usage" })).not.toBeInTheDocument(); + expect(screen.getByText("No usage recorded for this breakdown.")).toBeInTheDocument(); + }); + + it("should request an organization selection without issuing an unscoped usage request", () => { + renderWithProviders(, { searchParams: "?source=portal" }); + + expect(screen.getByText("Open a customer organization to explore its usage.")).toBeInTheDocument(); + expect(screen.getByRole("link", { name: "Back to customers" })).toHaveAttribute("href", "/ui/customers"); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.tsx new file mode 100644 index 00000000000..3fa99627bee --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.tsx @@ -0,0 +1,136 @@ +"use client"; + +import { useTranslation } from "react-i18next"; +import UsageStatStrip from "@/app/(dashboard)/usage/_components/components/overview/UsageStatStrip"; +import { ErrorState } from "@/components/shared/ErrorState"; +import { Button } from "@/components/ui/button"; +import { extractProxyErrorMessage } from "@/lib/http/client"; +import { uiHref } from "@/utils/uiHref"; +import CustomerMemberSpend from "../_components/CustomerMemberSpend"; +import CustomerUsageHeader from "../_components/CustomerUsageHeader"; +import CustomerUsageRankings from "../_components/CustomerUsageRankings"; +import CustomerUsageTrend from "../_components/CustomerUsageTrend"; +import useCustomerHierarchyUsage from "../_components/useCustomerHierarchyUsage"; + +const METRIC_LABELS = { + spend: "customerPortal.analytics.spend", + tokens: "customerPortal.analytics.tokens", + requests: "customerPortal.analytics.requests", +} as const; +const TOP_DIMENSION_LABELS = { + models: "customerPortal.analytics.topModels", + api_keys: "customerPortal.analytics.topKeys", + teams: "customerPortal.analytics.topTeams", +} as const; + +export default function CustomerHierarchyUsagePage() { + const { t } = useTranslation(); + const { + params, + scope, + dateValue, + preset, + usage, + data, + results, + totals, + modelColors, + rows, + drillDown, + setDateRange, + setDimension, + selectOrganization, + selectTeam, + setMetric, + setBucket, + setSearch, + setPage, + } = useCustomerHierarchyUsage(); + const metricLabel = t(METRIC_LABELS[params.metric]); + + if (!scope) + return ( +
+

{t("customerPortal.analytics.title")}

+

{t("customerPortal.analytics.selectOrganization")}

+ + {t("customerPortal.analytics.backToCustomers")} + +
+ ); + + return ( +
+ + {usage.isError ? ( + void usage.refetch()} + /> + ) : ( + <> + + + {!usage.isPending && ( + <> +
+

{t(TOP_DIMENSION_LABELS[params.breakdown])}

+

+ {t("customerPortal.analytics.sortedBy", { metric: metricLabel.toLocaleLowerCase() })} +

+
+ + + )} + {data && drillDown("members", id)} />} + {(params.team || params.member) && ( + + )} +

{t("customerPortal.analytics.freshness")}

+ + )} +
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx index 79c64e6079b..04b6ddbc2b0 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx @@ -84,7 +84,7 @@ export function BreakdownControls({ } /** Provider logo on a soft tile with a series-colored dot, so mixed logo styles still read as a set. */ -function ModelMark({ model, color }: { model: string; color: string }) { +export function ModelMark({ model, color }: { model: string; color: string }) { const provider = providerForModel(model); return ( diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json index bdade69fef6..a2369c9f077 100644 --- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json @@ -20681,6 +20681,72 @@ "maxBudget": "Max budget (USD)", "maxBudgetPlaceholder": "Leave empty for no budget", "sharedBudgetWarning": "This customer is on budget {{budgetId}}. Changing the budget here changes it for every customer on that budget." + }, + "analytics": { + "title": "Customer usage", + "customers": "Customers", + "breadcrumb": "Customer hierarchy", + "selectOrganization": "Open a customer organization to explore its usage.", + "backToCustomers": "Back to customers", + "organizationUsage": "Organization usage", + "teamUsage": "Team usage", + "memberUsage": "Member usage", + "subtitle": "Understand where usage goes, from your organization to each member.", + "trend": "Usage over time", + "trendHint": "Daily activity, broken down by {{dimension}}.", + "metric": "Usage metric", + "spend": "Spend", + "tokens": "Tokens", + "requests": "Requests", + "bucket": "Time interval", + "daily": "Daily", + "weekly": "Weekly", + "loading": "Loading customer usage", + "empty": "No usage in this range", + "error": "Could not load customer usage", + "breakdown": "Usage breakdown", + "models": "Models", + "keys": "API keys", + "useCases": "Use cases", + "teams": "Teams", + "members": "Members", + "sortedBy": "Ranked by {{metric}}", + "search": "Search breakdown", + "rank": "Rank", + "share": "Share", + "rankingHint": "Share of the selected metric across this customer scope. Click a team or member to explore its usage.", + "useCaseHint": "Use cases recorded with requests. Unattributed activity appears as Unassigned.", + "noMatches": "No results match your search.", + "noBreakdown": "No usage recorded for this breakdown.", + "pagination": "{{from}}–{{to}} of {{total}}", + "previous": "Previous", + "next": "Next", + "allOrganizationUsage": "View all organization usage", + "freshness": "Usage appears after accounting completes. Date ranges use UTC. Customer budgets are managed separately.", + "scopeMismatch": "This team or member does not belong to the selected organization.", + "invalidIdentifier": "Select a valid customer identifier.", + "incompletePages": "Could not load the complete usage range. Please choose fewer days.", + "unassignedTeam": "Unassigned or removed team", + "unassignedMember": "Unassigned or removed member", + "unnamedKey": "Unnamed key", + "currentSpend": "Budget spend", + "noBudget": "No budget set", + "breakDownBy": "Break down by", + "topModels": "Top models", + "topKeys": "Top API keys", + "topTeams": "Top teams", + "topMembers": "Top members", + "memberSpend": "Member spend", + "memberSpendHint": "Usage in the selected date range. Cap status uses each member’s recorded budget spend.", + "searchMembers": "Search members", + "member": "Member", + "currentBudget": "Spend / cap", + "status": "Status", + "noCap": "No cap", + "atBudget": "At budget", + "withinBudget": "Within budget", + "noMembers": "No members in this customer scope.", + "overBudget": "Over budget" } } } diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index 97f169bf9f5..ff97a43c136 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9608,6 +9608,26 @@ export interface paths { patch?: never; trace?: never; }; + "/management/v1/customer_members/daily/activity": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Customer Member Daily Activity + * @description Get paginated daily customer member activity with model, provider, key and member breakdowns. Filters accept comma-separated member IDs. Proxy admins only. + */ + get: operations["customer_member_daily_activity_management_v1_customer_members_daily_activity_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/management/v1/customer_members/{customer_member_id}": { parameters: { query?: never; @@ -9660,6 +9680,26 @@ export interface paths { patch?: never; trace?: never; }; + "/management/v1/customer_organizations/daily/activity": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Customer Organization Daily Activity + * @description Get paginated daily customer organization activity with model, provider, key and organization breakdowns. Filters accept comma-separated organization IDs. Proxy admins only. + */ + get: operations["customer_organization_daily_activity_management_v1_customer_organizations_daily_activity_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/management/v1/customer_organizations/{customer_org_id}": { parameters: { query?: never; @@ -9756,6 +9796,26 @@ export interface paths { patch?: never; trace?: never; }; + "/management/v1/customer_teams/daily/activity": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Customer Team Daily Activity + * @description Get paginated daily customer team activity with model, provider, key and team breakdowns. Filters accept comma-separated team IDs. Proxy admins only. + */ + get: operations["customer_team_daily_activity_management_v1_customer_teams_daily_activity_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/management/v1/customer_teams/{customer_team_id}": { parameters: { query?: never; @@ -68072,6 +68132,44 @@ export interface operations { }; }; }; + customer_member_daily_activity_management_v1_customer_members_daily_activity_get: { + parameters: { + query?: { + customer_member_ids?: string | null; + start_date?: string | null; + end_date?: string | null; + model?: string | null; + api_key?: string | null; + page?: number; + page_size?: number; + exclude_customer_member_ids?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_customer_member_management_v1_customer_members__customer_member_id__get: { parameters: { query?: never; @@ -68232,6 +68330,44 @@ export interface operations { }; }; }; + customer_organization_daily_activity_management_v1_customer_organizations_daily_activity_get: { + parameters: { + query?: { + customer_org_ids?: string | null; + start_date?: string | null; + end_date?: string | null; + model?: string | null; + api_key?: string | null; + page?: number; + page_size?: number; + exclude_customer_org_ids?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_customer_organization_management_v1_customer_organizations__customer_org_id__get: { parameters: { query?: never; @@ -68494,6 +68630,44 @@ export interface operations { }; }; }; + customer_team_daily_activity_management_v1_customer_teams_daily_activity_get: { + parameters: { + query?: { + customer_team_ids?: string | null; + start_date?: string | null; + end_date?: string | null; + model?: string | null; + api_key?: string | null; + page?: number; + page_size?: number; + exclude_customer_team_ids?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_customer_team_management_v1_customer_teams__customer_team_id__get: { parameters: { query?: never; diff --git a/tests/unit/accounting/test_application.py b/tests/unit/accounting/test_application.py new file mode 100644 index 00000000000..d60893a147d --- /dev/null +++ b/tests/unit/accounting/test_application.py @@ -0,0 +1,103 @@ +from collections.abc import Mapping +from dataclasses import dataclass, replace +from datetime import datetime, timezone +from typing import Final, cast + +import pytest +from pydantic import TypeAdapter + +from litellm.accounting.application import _apply_customer_usage +from litellm.accounting.delivery_projection import project_deliveries +from litellm.accounting.projection import Contribution +from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution +from litellm.repositories.accounting_delivery_repository import DeliveryProjection, LedgerTransaction + +_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_TEXT: Final = TypeAdapter(str) +_UPDATES: Final = TypeAdapter(Mapping[str, Mapping[str, float]]) +_IDENTIFIER: Final = "\U0001f680" * 256 +_ATTRIBUTION: Final = CustomerUsageAttribution( + customer_org_id=_IDENTIFIER, customer_team_id=_IDENTIFIER, customer_member_id=_IDENTIFIER +) + + +@dataclass(frozen=True, slots=True) +class Bucket: + values: Mapping[str, object] + counters: Mapping[str, float] + + +class Buckets: + def __init__(self) -> None: + self.rows: Mapping[str, Bucket] = {} + + async def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: + row_id: Final = _TEXT.validate_python(where["id"]) + create: Final = _FIELDS.validate_python(data["create"]) + previous: Final = self.rows.get(row_id) + increments: Final = ( + {name: fields["increment"] for name, fields in _UPDATES.validate_python(data["update"]).items()} + if previous is not None + else {name: float(value) for name, value in create.items() if isinstance(value, (int, float))} + ) + self.rows = { + **self.rows, + row_id: Bucket( + values=previous.values if previous is not None else create, + counters={ + name: (previous.counters.get(name, 0) if previous is not None else 0) + value + for name, value in increments.items() + }, + ), + } + + +@dataclass(frozen=True, slots=True) +class Transaction: + litellm_dailycustomermemberspend: Buckets + + +def contribution(alias: str, provider: str, spend: float) -> Contribution: + delivery: Final = DeliveryProjection( + { + "request_id": "long-customer-id-request", + "startTime": datetime(2026, 1, 1, tzinfo=timezone.utc).isoformat(), + "api_key": "hashed-key", + "model": "shared-model", + "model_group": alias, + "custom_llm_provider": provider, + "spend": spend, + "prompt_tokens": 10, + "completion_tokens": 5, + "metadata": {CUSTOMER_USAGE_METADATA_KEY: _ATTRIBUTION.model_dump_json()}, + } + ) + return next(item for item in project_deliveries((delivery,)) if item.table == "LiteLLM_DailyCustomerMemberSpend") + + +@pytest.mark.asyncio +async def test_long_customer_ids_roll_up_without_truncation_and_keep_alias_provider_buckets_distinct() -> None: + buckets: Final = Buckets() + transaction: Final = cast(LedgerTransaction, Transaction(buckets)) # cast-ok: in-memory Prisma upsert boundary + first: Final = contribution("alias-a", "provider-a", 1) + second: Final = replace(contribution("alias-a", "provider-a", 2), key=tuple(reversed(first.key))) + + await _apply_customer_usage(transaction, (first,)) + await _apply_customer_usage(transaction, (second, contribution("alias-b", "provider-a", 4))) + await _apply_customer_usage(transaction, (contribution("alias-a", "provider-b", 8),)) + + assert len(buckets.rows) == 3 + assert all(len(row_id.encode("utf-8")) == 64 for row_id in buckets.rows) + assert all( + row.values["customer_org_id"] + == row.values["customer_team_id"] + == row.values["customer_member_id"] + == _IDENTIFIER + for row in buckets.rows.values() + ) + assert { + (row.values["model_group"], row.values["custom_llm_provider"]): row.counters["spend"] + for row in buckets.rows.values() + } == {("alias-a", "provider-a"): 3, ("alias-b", "provider-a"): 4, ("alias-a", "provider-b"): 8} + assert sum(row.counters["api_requests"] for row in buckets.rows.values()) == 4 + assert sum(row.counters["prompt_tokens"] + row.counters["completion_tokens"] for row in buckets.rows.values()) == 60 diff --git a/tests/unit/accounting/test_projection.py b/tests/unit/accounting/test_projection.py index 090825dca75..75b3f76de26 100644 --- a/tests/unit/accounting/test_projection.py +++ b/tests/unit/accounting/test_projection.py @@ -6,20 +6,120 @@ from typing import Final import pytest from pydantic import TypeAdapter +from litellm.accounting.delivery_projection import project_deliveries from litellm.accounting.projection import project from litellm.accounting.snapshot import build_accounting_payload from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.proxy._types import SpendLogsPayload +from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.proxy_server import proxy_logging_obj from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.proxy.utils import PrismaClient +from litellm.repositories.accounting_delivery_repository import DeliveryProjection from litellm.types.utils import ModelResponse, Usage _NOW: Final = datetime(2026, 11, 2, tzinfo=timezone.utc) _FIELDS: Final = TypeAdapter(dict[str, object]) +def test_customer_model_aliases_and_providers_keep_separate_daily_totals() -> None: + stamp: Final = CustomerUsageAttribution(customer_org_id="org-1", customer_member_id="member-1").model_dump_json() + base: Final = { + "startTime": _NOW, + "api_key": "hashed-key", + "model": "shared-model", + "metadata": {CUSTOMER_USAGE_METADATA_KEY: stamp}, + } + deliveries: Final = tuple( + DeliveryProjection({**base, "model_group": alias, "custom_llm_provider": provider, "spend": spend}) + for alias, provider, spend in ( + ("alias-a", "provider-a", 1), + ("alias-b", "provider-a", 2), + ("alias-a", "provider-b", 3), + ) + ) + totals: Final = { + (dict(item.key)["model_group"], dict(item.key)["custom_llm_provider"]): dict(item.increments)["spend"] + for item in project_deliveries(deliveries) + if item.table == "LiteLLM_DailyCustomerMemberSpend" + } + assert totals == {("alias-a", "provider-a"): 1, ("alias-b", "provider-a"): 2, ("alias-a", "provider-b"): 3} + + +@pytest.mark.parametrize("retry", [False, True]) +def test_customer_usage_survives_logging_and_accounting_with_correct_request_counts(retry: bool) -> None: + attribution: Final = CustomerUsageAttribution( + customer_org_id="org-1", customer_team_id="team-1", customer_member_id="member-1" + ) + payload: Final = get_logging_payload( + kwargs={ + "model": "usage-model", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "usage-request", + "litellm_params": { + "metadata": { + "user_api_key": "hashed-key", + "status": "success", + CUSTOMER_USAGE_METADATA_KEY: attribution.model_dump_json(), + **({BILLED_RETRY_ATTEMPT_METADATA_KEY: True} if retry else {}), + } + }, + }, + response_obj=ModelResponse( + id="usage-request", model="usage-model", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + ) + accounting: Final = build_accounting_payload( + {**payload, "spend": 0.75}, entities={}, model_access_groups=(), windows=(), router=lambda: None + ) + contributions: Final = project(accounting) + usage: Final = next(item for item in contributions if item.table == "LiteLLM_DailyCustomerMemberSpend") + assert dict(usage.key) == { + "date": _NOW.date().isoformat(), + "customer_org_id": attribution.customer_org_id, + "customer_team_id": attribution.customer_team_id, + "customer_member_id": attribution.customer_member_id, + "model": payload["model"], + "model_group": payload["model_group"], + "custom_llm_provider": payload["custom_llm_provider"], + "api_key": payload["api_key"], + "mcp_namespaced_tool_name": "", + "endpoint": "/chat/completions", + } + assert dict(usage.increments) == { + "spend": 0.75, + "prompt_tokens": 10, + "completion_tokens": 5, + "api_requests": 0 if retry else 1, + "successful_requests": 0 if retry else 1, + "failed_requests": 0, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "compression_saved_tokens": 0, + "compression_savings_spend": 0, + "prompt_caching_savings_spend": 0, + "gateway_injected_caching_savings_spend": 0, + "autorouter_savings_spend": 0, + "guardrail_spend": 0, + "total_response_time_ms": 0, + "timed_requests": 1, + } + assert { + item.table: dict(item.increments)["spend"] + for item in contributions + if item.table + in ("LiteLLM_CustomerOrganizationTable", "LiteLLM_CustomerTeamTable", "LiteLLM_CustomerMemberTable") + } == { + "LiteLLM_CustomerOrganizationTable": 0.75, + "LiteLLM_CustomerTeamTable": 0.75, + "LiteLLM_CustomerMemberTable": 0.75, + } + + @pytest.mark.parametrize("status", ["success", "failure"]) @pytest.mark.parametrize("origin", [None, "auto_router", "billed_retry"]) @pytest.mark.parametrize("spend", [0.0, 0.25]) diff --git a/tests/unit/accounting/test_worker.py b/tests/unit/accounting/test_worker.py index d3b5d71cff9..372ae90b158 100644 --- a/tests/unit/accounting/test_worker.py +++ b/tests/unit/accounting/test_worker.py @@ -1,5 +1,9 @@ import json +from collections.abc import AsyncIterator, Mapping +from contextlib import asynccontextmanager from dataclasses import dataclass +from datetime import datetime, timezone +from functools import partial from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock @@ -10,11 +14,18 @@ from fakeredis.aioredis import FakeRedis from pydantic import TypeAdapter from litellm import in_memory_llm_clients_cache +from litellm.accounting.application import apply_deliveries from litellm.accounting.worker import flush_accounting_deliveries from litellm.caching.redis_cache import RedisCache +from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.accounting_delivery_repository import ( + AccountingDeliveryRepository, + DeliveryProjection, + LedgerTransaction, +) _FIELDS: Final = TypeAdapter(dict[str, object]) _INCREMENTS: Final = TypeAdapter(dict[str, dict[str, float]]) @@ -30,7 +41,7 @@ class _Ledger: spend: float = 0.0 attempts: int = 0 - async def query(self, query: str, *args: object) -> tuple[dict[str, object], ...]: + async def query(self, query: str, *args: object) -> tuple[Mapping[str, object], ...]: if query.startswith("SELECT delivery_id, attempts"): if not self.pending or self.attempts or args[1] is True: return () @@ -96,3 +107,119 @@ async def test_delivery_charges_the_creating_key_despite_an_orphaned_maintenance in_memory_llm_clients_cache.delete_cache(key=cache_key) await redis.aclose() sync_redis.close() + + +class _CustomerLedger: + def __init__(self, fail_acknowledgement: bool) -> None: + self.pending = True + self.attempts = 0 + self.fail_acknowledgement = fail_acknowledgement + self.daily: Mapping[str, int | float] = {} + self.entities: Mapping[str, float] = {} + + async def query(self, query: str, *args: object) -> tuple[Mapping[str, object], ...]: + if query.startswith("SELECT delivery_id, attempts"): + if not self.pending or (self.attempts > 0) is not args[1]: + return () + return ( + { + "delivery_id": "customer-delivery", + "attempts": self.attempts, + "accounting_payload": _CUSTOMER_PAYLOAD, + }, + ) + if "SET attempts = attempts + 1" in query: + self.attempts += 1 + return ({"delivery_id": "customer-delivery", "attempts": self.attempts},) + return () + + async def execute(self, query: str, *args: object) -> int: + if "SET state = 'applied'" not in query: + return 0 + if self.fail_acknowledgement: + self.fail_acknowledgement = False + raise OSError("database connection lost before delivery acknowledgement") + self.pending = False + return 1 + + def upsert_daily(self, *, where: object, data: object) -> None: + values: Final = _FIELDS.validate_python(_FIELDS.validate_python(data)["create"]) + counters: Final = {name: value for name, value in values.items() if isinstance(value, (int, float))} + self.daily = {name: self.daily.get(name, 0) + value for name, value in counters.items()} + + def update_entity(self, entity: str, *, where: object, data: object) -> int: + spend: Final = _INCREMENTS.validate_python(data)["spend"]["increment"] + self.entities = {**self.entities, entity: self.entities.get(entity, 0.0) + spend} + return 1 + + +_CUSTOMER_PAYLOAD: Final = json.dumps( + { + "request_id": "customer-completed-request", + "startTime": datetime(2026, 1, 1, tzinfo=timezone.utc).isoformat(), + "api_key": "hashed-key", + "spend": 0.75, + "prompt_tokens": 10, + "completion_tokens": 5, + "metadata": { + CUSTOMER_USAGE_METADATA_KEY: CustomerUsageAttribution( + customer_org_id="customer-org", customer_team_id="customer-team", customer_member_id="customer-member" + ).model_dump_json() + }, + } +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_acknowledgement", (False, True)) +async def test_customer_daily_and_lifetime_totals_commit_once_after_delivery_replay( + fail_acknowledgement: bool, +) -> None: + ledger: Final = _CustomerLedger(fail_acknowledgement) + transaction: Final = MagicMock() + transaction.query_raw = AsyncMock(side_effect=ledger.query) + transaction.execute_raw = AsyncMock(side_effect=ledger.execute) + transaction.litellm_dailycustomermemberspend.upsert = AsyncMock(side_effect=ledger.upsert_daily) + transaction.litellm_customerorganizationtable.update_many = AsyncMock( + side_effect=partial(ledger.update_entity, "organization") + ) + transaction.litellm_customerteamtable.update_many = AsyncMock(side_effect=partial(ledger.update_entity, "team")) + transaction.litellm_customermembertable.update_many = AsyncMock(side_effect=partial(ledger.update_entity, "member")) + transaction.batch_.return_value.__aenter__ = AsyncMock(return_value=MagicMock()) + transaction.batch_.return_value.__aexit__ = AsyncMock(return_value=False) + + @asynccontextmanager + async def database_transaction(*, timeout: object) -> AsyncIterator[LedgerTransaction]: + before: Final = (ledger.pending, ledger.daily, ledger.entities) + try: + yield transaction + except Exception: + ledger.pending, ledger.daily, ledger.entities = before + raise + + database: Final = MagicMock(spec=PrismaClient) + database.tx.side_effect = database_transaction + repository: Final = AccountingDeliveryRepository(database) + writer: Final = DBSpendUpdateWriter(accounting_clock=lambda: 0.0) + logger: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache()) + + async def apply(tx: LedgerTransaction, deliveries: tuple[DeliveryProjection, ...]) -> None: + await apply_deliveries(tx, deliveries, prisma_client=database, writer=writer, proxy_logging_obj=logger) + + first: Final = await repository.apply_pending(apply) + if fail_acknowledgement: + assert first.failed == 1 + assert ledger.daily == {} + assert ledger.entities == {} + retried: Final = await repository.apply_pending(apply) + assert retried.applied == 1 + else: + assert first.applied == 1 + await repository.apply_pending(apply) + await repository.apply_pending(apply) + assert ledger.daily["spend"] == 0.75 + assert ledger.daily["prompt_tokens"] == 10 + assert ledger.daily["completion_tokens"] == 5 + assert ledger.daily["api_requests"] == 1 + assert ledger.entities == {"organization": 0.75, "team": 0.75, "member": 0.75} + assert ledger.pending is False diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json index 6e5461c3b60..40674758371 100644 --- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json +++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", + "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_customer_usage\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/proxy/common_utils/test_customer_usage.py b/tests/unit/proxy/common_utils/test_customer_usage.py new file mode 100644 index 00000000000..79b16cafa82 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_customer_usage.py @@ -0,0 +1,225 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Final, cast # noqa: TID251 # the fake exposes only the Prisma lookup methods exercised here + +import pytest + +from litellm.proxy.common_utils.customer_usage import ( + CustomerUsageAttribution, + CustomerUsageDatabase, + CustomerUsageError, + CustomerUsageFailure, + resolve_customer_usage, +) + + +@dataclass(frozen=True, slots=True) +class Table: + identifier: str + rows: Mapping[str, Mapping[str, object]] + + async def find_unique(self, where: Mapping[str, object]) -> Mapping[str, object] | None: + return self.rows.get(str(where[self.identifier])) + + +@dataclass(frozen=True, slots=True) +class Database: + litellm_customerorganizationtable: Table + litellm_customerteamtable: Table + litellm_customermembertable: Table + + +DATABASE: Final = cast( # cast-ok: the external Prisma fake implements the lookup subset used by attribution + CustomerUsageDatabase, + Database( + Table("customer_org_id", {"org-1": {}, "org-2": {}}), + Table("customer_team_id", {"team-1": {"parent_org_id": "org-1"}, "team-2": {"parent_org_id": "org-2"}}), + Table("customer_member_id", {"member-1": {"parent_team_id": "team-1"}}), + ), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata", "expected"), + ( + ( + {"customer_member_id": "member-1"}, + CustomerUsageAttribution(customer_org_id="org-1", customer_team_id="team-1", customer_member_id="member-1"), + ), + ({"customer_team_id": "team-1"}, CustomerUsageAttribution(customer_org_id="org-1", customer_team_id="team-1")), + ({"customer_org_id": "org-1"}, CustomerUsageAttribution(customer_org_id="org-1")), + ), +) +async def test_resolves_ancestors_without_requiring_redundant_request_ids( + metadata: Mapping[str, object], expected: CustomerUsageAttribution +) -> None: + actual: Final = await resolve_customer_usage( + metadata=metadata, key_metadata={}, administrator=True, database=DATABASE, ledger_enabled=True + ) + assert actual == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata", "key_metadata", "administrator", "expected"), + ( + ({"customer_member_id": "member-1", "customer_team_id": "team-2"}, {}, True, CustomerUsageError.MISMATCH), + ({"customer_member_id": "member-1"}, {"customer_org_id": "org-2"}, False, CustomerUsageError.FORBIDDEN), + ({"customer_team_id": "team-2", "customer_org_id": "org-1"}, {}, True, CustomerUsageError.MISMATCH), + ({"customer_org_id": "org-2"}, {"customer_org_id": "org-1"}, False, CustomerUsageError.FORBIDDEN), + ({"customer_org_id": "org-1"}, {}, False, CustomerUsageError.UNBOUND), + ({"customer_member_id": "missing"}, {}, True, CustomerUsageError.NOT_FOUND), + ({"customer_team_id": "missing"}, {}, True, CustomerUsageError.NOT_FOUND), + ({"customer_org_id": "missing"}, {}, True, CustomerUsageError.NOT_FOUND), + ({"customer_org_id": 123}, {}, True, CustomerUsageError.INVALID), + ), +) +async def test_rejects_cross_organization_attribution_and_invalid_identifiers( + metadata: Mapping[str, object], + key_metadata: Mapping[str, object], + administrator: bool, + expected: CustomerUsageError, +) -> None: + actual: Final = await resolve_customer_usage( + metadata=metadata, + key_metadata=key_metadata, + administrator=administrator, + database=DATABASE, + ledger_enabled=True, + ) + assert actual == CustomerUsageFailure(expected) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata", "key_metadata"), + ( + ({"customer_member_id": "missing"}, {"customer_org_id": "org-1"}), + ({"customer_member_id": "member-1"}, {"customer_org_id": "org-2"}), + ( + {"customer_member_id": "member-1"}, + {"customer_org_id": "org-2", "customer_team_id": "team-2"}, + ), + ({"customer_team_id": "missing"}, {"customer_org_id": "org-1"}), + ({"customer_team_id": "team-2"}, {"customer_org_id": "org-1"}), + ({}, {"customer_org_id": "missing"}), + ), +) +async def test_non_admin_cannot_distinguish_missing_and_other_organization_identifiers( + metadata: Mapping[str, object], key_metadata: Mapping[str, object] +) -> None: + actual: Final = await resolve_customer_usage( + metadata=metadata, + key_metadata=key_metadata, + administrator=False, + database=DATABASE, + ledger_enabled=True, + ) + assert actual == CustomerUsageFailure(CustomerUsageError.FORBIDDEN) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("administrator", (False, True)) +async def test_invalid_identifier_types_keep_validation_errors(administrator: bool) -> None: + actual: Final = await resolve_customer_usage( + metadata={"customer_member_id": 123}, + key_metadata={"customer_org_id": "org-1"}, + administrator=administrator, + database=DATABASE, + ledger_enabled=True, + ) + assert actual == CustomerUsageFailure(CustomerUsageError.INVALID) + + +@pytest.mark.asyncio +async def test_key_binding_tracks_requests_with_no_repeated_customer_metadata() -> None: + actual: Final = await resolve_customer_usage( + metadata={}, + key_metadata={"customer_org_id": "org-1"}, + administrator=False, + database=DATABASE, + ledger_enabled=True, + ) + assert actual == CustomerUsageAttribution(customer_org_id="org-1") + + +@pytest.mark.asyncio +async def test_requires_accounting_before_accepting_a_tracked_request() -> None: + actual: Final = await resolve_customer_usage( + metadata={"customer_org_id": "org-1"}, + key_metadata={}, + administrator=True, + database=DATABASE, + ledger_enabled=False, + ) + assert actual == CustomerUsageFailure(CustomerUsageError.UNAVAILABLE) + + +@pytest.mark.asyncio +async def test_untracked_requests_do_not_require_a_database_or_ledger() -> None: + actual: Final = await resolve_customer_usage( + metadata={}, key_metadata={}, administrator=False, database=None, ledger_enabled=False + ) + assert actual is None + + +@pytest.mark.asyncio +async def test_untracked_request_enrichment_does_not_touch_customer_database_tables( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from fastapi import Request + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=object())) + request: Final = Request( + {"type": "http", "path": "/v1/chat/completions", "method": "POST", "headers": (), "query_string": b""} + ) + enriched: Final = await add_litellm_data_to_request( + data={"model": "usage-model", "metadata": {"application": "support"}}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=ProxyConfig(), + general_settings={}, + ) + assert enriched["metadata"]["application"] == "support" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/chat/completions", "/v1/responses"]) +async def test_request_enrichment_replaces_forged_stamps_and_preserves_valid_attribution( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from fastapi import Request + from pydantic import TypeAdapter + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request, get_metadata_variable_name + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=DATABASE)) + request: Final = Request({"type": "http", "path": path, "method": "POST", "headers": (), "query_string": b""}) + metadata_name: Final = get_metadata_variable_name(request) + forged: Final = CustomerUsageAttribution(customer_org_id="org-2").model_dump_json() + metadata: Final = {"customer_member_id": "member-1", CUSTOMER_USAGE_METADATA_KEY: forged} + enriched: Final = await add_litellm_data_to_request( + data={ + "model": "usage-model", + "metadata": {CUSTOMER_USAGE_METADATA_KEY: forged}, + "litellm_metadata": {CUSTOMER_USAGE_METADATA_KEY: forged}, + metadata_name: metadata, + }, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", metadata={"customer_org_id": "org-1"}), + proxy_config=ProxyConfig(), + general_settings={}, + ) + stamp: Final = TypeAdapter(str).validate_python(enriched[metadata_name][CUSTOMER_USAGE_METADATA_KEY]) + assert CustomerUsageAttribution.model_validate_json(stamp) == CustomerUsageAttribution( + customer_org_id="org-1", customer_team_id="team-1", customer_member_id="member-1" + ) diff --git a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py index 554a8bb3215..a54912c44ba 100644 --- a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -3488,7 +3488,7 @@ async def test_a_failed_request_settles_the_team_share_gauges_too(monkeypatch: p registry = _prometheus_sink(monkeypatch) await _admit_then_settle(handler, _team_key(), tokens=90) claim_request_stash_for_data({"litellm_call_id": "failed-call"}) - await _check(handler, _team_key(), saturation=0.0, data=_prompt(words=40), priority=None) + await _check(handler, _team_key(), saturation=0.5) await handler.async_log_failure_event( kwargs={ "litellm_params": {"metadata": {"model_group": "shared-model"}}, @@ -3504,10 +3504,10 @@ async def test_a_failed_request_settles_the_team_share_gauges_too(monkeypatch: p @pytest.mark.asyncio @pytest.mark.parametrize( - "auth, burns, saturation_threshold, reason", + "auth, burns, saturation_threshold, reason, enforced_share_tokens", [ - (_team_key(), (60, 70), 0.1, "model_per_team_saturated"), - (_capped_team_key(2), (150, 60), 0.5, "model_per_team_max_borrow_multiple"), + (_team_key(), (60, 70), 0.0, "model_per_team_saturated", 100), + (_capped_team_key(2), (150, 60), 0.5, "model_per_team_max_borrow_multiple", 200), ], ids=["saturated", "borrow_cap"], ) @@ -3517,6 +3517,7 @@ async def test_a_refused_request_is_counted_under_the_reason_its_429_header_carr burns: tuple[int, ...], saturation_threshold: float, reason: str, + enforced_share_tokens: int, ): handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=saturation_threshold) registry = _prometheus_sink(monkeypatch) @@ -3532,7 +3533,10 @@ async def test_a_refused_request_is_counted_under_the_reason_its_429_header_carr ) == 1 ) - assert _sample(registry, "litellm_dynamic_limiter_borrowed_tokens", **TEAM_LABELS) == sum(burns) - 100 + assert _sample(registry, "litellm_dynamic_limiter_share_tokens", **TEAM_LABELS) == enforced_share_tokens + assert ( + _sample(registry, "litellm_dynamic_limiter_borrowed_tokens", **TEAM_LABELS) == sum(burns) - enforced_share_tokens + ) @pytest.mark.asyncio @@ -3596,7 +3600,7 @@ async def test_an_alternates_refusal_is_not_counted_while_the_preferred_model_ad @pytest.mark.asyncio async def test_share_gauges_and_throttles_carry_the_teams_priority_so_the_series_join(monkeypatch: pytest.MonkeyPatch): - handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.1) + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) registry = _prometheus_sink(monkeypatch) team: Final = UserAPIKeyAuth( api_key="sk-interactive", @@ -3619,7 +3623,7 @@ async def test_share_gauges_and_throttles_carry_the_teams_priority_so_the_series async def test_a_fallback_served_request_publishes_the_teams_share_on_the_fallback_model( monkeypatch: pytest.MonkeyPatch, ): - handler = _borrowing_handler(monkeypatch, borrows=True) + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) registry = _prometheus_sink(monkeypatch) team: Final = _team_key(team_limits={"model_tpm_limit": {"other-model": 40, "shared-model": 100}}) assert await handler.async_pre_call_hook(team, DualCache(), {"model": "other-model"}, "completion") is None @@ -3642,7 +3646,7 @@ async def test_a_failing_metrics_sink_never_changes_admission(monkeypatch: pytes def increment_dynamic_limiter_throttled(self, *args: object, **kwargs: object) -> None: raise RuntimeError("metrics backend down") - handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.1) + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) _prometheus_sink(monkeypatch, BrokenSink) await _admit_at_the_hook(handler, _team_key(), tokens=60) await _admit_at_the_hook(handler, _team_key(), tokens=70) diff --git a/tests/unit/proxy/management_endpoints/management_v1/test_customer_usage.py b/tests/unit/proxy/management_endpoints/management_v1/test_customer_usage.py new file mode 100644 index 00000000000..af16d11373b --- /dev/null +++ b/tests/unit/proxy/management_endpoints/management_v1/test_customer_usage.py @@ -0,0 +1,278 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from functools import reduce +from typing import Final, cast + +import pytest +from fastapi import FastAPI, Request +from fastapi.testclient import TestClient +from pydantic import BaseModel, ConfigDict + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.list_api.common import ManagementProblem, problem_response +from litellm.proxy.management_endpoints.management_v1.customer_usage import get_customer_usage_prisma, router +from litellm.proxy.utils import PrismaClient +from litellm.types.proxy.management_endpoints.common_daily_activity import SpendMetrics + + +class Record(BaseModel): + model_config = ConfigDict(frozen=True, extra="allow") + + +class DailyRecord(SpendMetrics): + model_config = ConfigDict(frozen=True) + + id: str + date: str + customer_org_id: str + customer_team_id: str + customer_member_id: str + api_key: str + model: str | None = None + model_group: str | None = None + custom_llm_provider: str | None = None + mcp_namespaced_tool_name: str | None = None + endpoint: str | None = None + + +def matches(values: Mapping[str, object], where: Mapping[str, object]) -> bool: + for key, condition in where.items(): + if key == "OR": + if not any(matches(values, clause) for clause in condition): + return False + elif key == "AND": + if not all(matches(values, clause) for clause in condition): + return False + elif isinstance(condition, Mapping): + if "in" in condition and values.get(key) not in condition["in"]: + return False + if "not" in condition and matches(values, {key: condition["not"]}): + return False + if "gte" in condition and str(values.get(key)) < str(condition["gte"]): + return False + if "lte" in condition and str(values.get(key)) > str(condition["lte"]): + return False + elif values.get(key) != condition: + return False + return True + + +@dataclass(frozen=True, slots=True) +class Table: + rows: tuple[Record | DailyRecord, ...] = () + + async def find_many( + self, + where: Mapping[str, object] | None = None, + take: int | None = None, + skip: int = 0, + order: Mapping[str, object] | Sequence[Mapping[str, object]] | None = None, + ) -> tuple[Record | DailyRecord, ...]: + selected: Final = tuple(row for row in self.rows if matches(row.model_dump(), where or {})) + orders: Final = (order,) if isinstance(order, Mapping) else tuple(order or ()) + ordered: Final = reduce( + lambda rows, item: tuple( + sorted( + rows, + key=lambda row: str(row.model_dump()[next(iter(item))]), + reverse=next(iter(item.values())) == "desc", + ) + ), + reversed(orders), + selected, + ) + return ordered[skip:] if take is None else ordered[skip : skip + take] + + async def count(self, where: Mapping[str, object] | None = None) -> int: + return len(await self.find_many(where=where)) + + +@dataclass(frozen=True, slots=True) +class Database: + litellm_dailycustomermemberspend: Table + litellm_customerorganizationtable: Table = Table( + ( + Record(customer_org_id="org-a", alias="Organization A"), + Record(customer_org_id="org-b", alias="Organization B"), + ) + ) + litellm_customerteamtable: Table = Table( + (Record(customer_team_id="team-a", alias="Product"), Record(customer_team_id="team-b", alias="Operations")) + ) + litellm_customermembertable: Table = Table( + ( + Record(customer_member_id="member-a", alias="Member A"), + Record(customer_member_id="member-b", alias="Member B"), + ) + ) + litellm_verificationtoken: Table = Table( + ( + Record(token="hash-a", key_alias="Support", team_id=None, user_id=None, metadata=None), + Record(token="hash-b", key_alias="Search", team_id=None, user_id=None, metadata=None), + ) + ) + litellm_deletedverificationtoken: Table = Table() + litellm_usertable: Table = Table() + + +@dataclass(frozen=True, slots=True) +class Client: + db: Database + + +def usage_row( + *, + id: str = "usage-a", + date: str = "2026-01-01", + customer_org_id: str = "org-a", + customer_team_id: str = "team-a", + customer_member_id: str = "member-a", + api_key: str = "hash-a", + model: str = "model-a", + model_group: str = "alias-a", + provider: str = "provider-a", + spend: float = 2.0, +) -> DailyRecord: + return DailyRecord( + id=id, + date=date, + customer_org_id=customer_org_id, + customer_team_id=customer_team_id, + customer_member_id=customer_member_id, + api_key=api_key, + model=model, + model_group=model_group, + custom_llm_provider=provider, + spend=spend, + prompt_tokens=10, + completion_tokens=5, + total_tokens=15, + api_requests=3, + successful_requests=2, + failed_requests=1, + ) + + +def client(rows: tuple[DailyRecord, ...] = (), role: LitellmUserRoles = LitellmUserRoles.PROXY_ADMIN) -> TestClient: + app: Final = FastAPI() + app.include_router(router) + + @app.exception_handler(ManagementProblem) + async def problem(request: Request, exc: ManagementProblem): + return problem_response(exc.problem) + + auth: Final = UserAPIKeyAuth(user_id="admin", user_role=role) + prisma: Final = cast(PrismaClient, Client(Database(Table(rows)))) # cast-ok: in-memory Prisma boundary + app.dependency_overrides[user_api_key_auth] = lambda: auth + app.dependency_overrides[get_customer_usage_prisma] = lambda: prisma + return TestClient(app) + + +DATES: Final = {"start_date": "2026-01-01", "end_date": "2026-01-03"} +SCOPES: Final = ( + ("customer_organizations", "customer_org_ids", "exclude_customer_org_ids", "org-a", "org-b", "Organization A"), + ("customer_teams", "customer_team_ids", "exclude_customer_team_ids", "team-a", "team-b", "Product"), + ("customer_members", "customer_member_ids", "exclude_customer_member_ids", "member-a", "member-b", "Member A"), +) + + +@pytest.mark.parametrize("resource,parameter,excluded,first,second,label", SCOPES) +def test_standard_daily_activity_filters_entities_and_preserves_aliases_and_dimensions( + resource: str, parameter: str, excluded: str, first: str, second: str, label: str +) -> None: + rows: Final = ( + usage_row(), + usage_row(id="usage-c", date="2026-01-03", model_group="alias-b", provider="provider-b", spend=4), + usage_row( + id="usage-b", customer_org_id="org-b", customer_team_id="team-b", customer_member_id="member-b", spend=10 + ), + usage_row(id="outside-range", date="2025-12-31", spend=50), + ) + response: Final = client(rows).get(f"/management/v1/{resource}/daily/activity", params={**DATES, parameter: first}) + assert response.status_code == 200, response.text + body: Final = response.json() + assert set(body) == {"results", "metadata"} + assert body["metadata"]["total_spend"] == 6 + assert body["metadata"]["total_tokens"] == 30 + assert body["metadata"]["total_api_requests"] == 6 + assert body["metadata"]["has_more"] is False + day: Final = body["results"][0] + assert day["breakdown"]["entities"][first]["metadata"]["alias"] == label + assert day["breakdown"]["models"]["model-a"]["metrics"]["spend"] == 4 + assert day["breakdown"]["model_groups"]["alias-b"]["metrics"]["spend"] == 4 + assert day["breakdown"]["providers"]["provider-b"]["metrics"]["spend"] == 4 + assert day["breakdown"]["api_keys"]["hash-a"]["metadata"]["key_alias"] == "Support" + + +@pytest.mark.parametrize("resource,parameter,excluded,first,second,label", SCOPES) +def test_inclusion_and_exclusion_lists_are_comma_separated( + resource: str, parameter: str, excluded: str, first: str, second: str, label: str +) -> None: + rows: Final = ( + usage_row(), + usage_row( + id="usage-b", customer_org_id="org-b", customer_team_id="team-b", customer_member_id="member-b", spend=10 + ), + ) + response: Final = client(rows).get( + f"/management/v1/{resource}/daily/activity", params={**DATES, parameter: f"{first},{second}", excluded: second} + ) + assert response.status_code == 200, response.text + assert response.json()["metadata"]["total_spend"] == 2 + + +@pytest.mark.parametrize("resource,parameter,excluded,first,second,label", SCOPES) +def test_standard_model_key_and_page_filters_use_the_shared_response_contract( + resource: str, parameter: str, excluded: str, first: str, second: str, label: str +) -> None: + rows: Final = ( + usage_row(id="page-a"), + usage_row(id="page-b", date="2026-01-02", spend=4), + usage_row(id="other-model", model="model-b", spend=20), + usage_row(id="other-key", api_key="hash-b", spend=30), + ) + response: Final = client(rows).get( + f"/management/v1/{resource}/daily/activity", + params={**DATES, parameter: first, "model": "model-a", "api_key": "hash-a", "page": 1, "page_size": 1}, + ) + assert response.status_code == 200, response.text + body: Final = response.json() + assert body["metadata"]["page"] == 1 + assert body["metadata"]["total_pages"] == 2 + assert body["metadata"]["has_more"] is True + assert body["metadata"]["total_spend"] == 4 + assert body["results"][0]["date"] == "2026-01-02" + next_page: Final = client(rows).get( + f"/management/v1/{resource}/daily/activity", + params={**DATES, parameter: first, "model": "model-a", "api_key": "hash-a", "page": 2, "page_size": 1}, + ) + assert next_page.status_code == 200, next_page.text + assert next_page.json()["metadata"]["total_spend"] == 2 + assert next_page.json()["metadata"]["has_more"] is False + + +@pytest.mark.parametrize("resource,parameter,excluded,first,second,label", SCOPES) +@pytest.mark.parametrize("role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)) +def test_all_hierarchy_daily_activity_routes_are_admin_only( + resource: str, parameter: str, excluded: str, first: str, second: str, label: str, role: LitellmUserRoles +) -> None: + response: Final = client(role=role).get(f"/management/v1/{resource}/daily/activity", params=DATES) + assert response.status_code == 403, response.text + + +@pytest.mark.parametrize("params", ({}, {"start_date": "invalid", "end_date": "2026-01-03"}, {**DATES, "page": 0})) +def test_shared_helper_rejects_missing_dates_invalid_dates_and_nonpositive_pagination( + params: Mapping[str, object], +) -> None: + response: Final = client().get("/management/v1/customer_members/daily/activity", params=params) + assert response.status_code == 400, response.text + + +def test_unmatched_entities_return_the_standard_empty_response() -> None: + response: Final = client((usage_row(),)).get( + "/management/v1/customer_members/daily/activity", params={**DATES, "customer_member_ids": "missing"} + ) + assert response.status_code == 200, response.text + assert response.json()["results"] == [] + assert response.json()["metadata"]["total_spend"] == 0 diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index ea36ffad044..e7bc5675e06 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -41,6 +41,7 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import ( ) from litellm.proxy.proxy_server import app from litellm.types.proxy.auth.auth_checks import UserNotFoundError +from litellm.types.proxy.common_proxy_errors import CommonProxyErrors from litellm.types.proxy.management_endpoints.internal_user_endpoints import InsensitiveContains from tests.unit.proxy.management_endpoints.jwt_key_mapping_doubles import ( CascadingJWTMappingTable, @@ -50,6 +51,30 @@ from tests.unit.proxy.management_endpoints.jwt_key_mapping_doubles import ( client = TestClient(app) +@pytest.mark.asyncio +@pytest.mark.parametrize("binding", ["customer_org_id", "customer_team_id", "customer_member_id"]) +async def test_new_user_cannot_mint_customer_bound_keys_without_proxy_admin(binding: str) -> None: + with pytest.raises(HTTPException) as caught: + await new_user( + data=NewUserRequest(metadata={binding: "another-customer"}), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER), + ) + assert caught.value.status_code == 403 + assert caught.value.detail == "Only proxy admins can change a key's customer organization, team, or member binding." + + +@pytest.mark.asyncio +async def test_new_user_reports_missing_database_after_customer_binding_validation(mocker: MockerFixture) -> None: + mocker.patch("litellm.proxy.proxy_server.prisma_client", None) + with pytest.raises(ProxyException) as caught: + await new_user( + data=NewUserRequest(metadata={"customer_org_id": "org-1"}), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert str(caught.value.code) == "400" + assert caught.value.message == CommonProxyErrors.db_not_connected_error.value + + @pytest.mark.parametrize("user_id", ["", " \t\n", "user\x00id", 123, True, [], {}]) def test_new_user_rejects_invalid_custom_id(user_id: object) -> None: with pytest.raises(ValidationError) as exc: diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index c67cb974583..9e0e123436a 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from contextlib import ExitStack from datetime import datetime, timedelta, timezone from types import SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, call, patch from uuid import UUID, uuid4 @@ -24618,3 +24618,89 @@ def test_user_new_without_a_key_does_not_check_guardrail_names(spend_permission_ assert response.status_code == 200, response.text find_many.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ("customer_org_id", "customer_team_id", "customer_member_id")) +async def test_customer_binding_requires_admin_on_key_creation(field: str) -> None: + with pytest.raises(HTTPException) as rejected: + await _common_key_generation_helper( + data=GenerateKeyRequest(metadata={field: "customer-id"}), + user_api_key_dict=UserAPIKeyAuth(user_id="member", user_role=LitellmUserRoles.INTERNAL_USER), + litellm_changed_by=None, + team_table=None, + ) + assert rejected.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_type", ("key", "user")) +async def test_direct_key_mint_cannot_bypass_customer_binding_authorization(request_type: Literal["key", "user"]) -> None: + with pytest.raises(HTTPException) as rejected: + await generate_key_helper_fn(request_type=request_type, metadata={"customer_org_id": "another-customer"}) + assert rejected.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ("update", "regenerate")) +@pytest.mark.parametrize("field", ("customer_org_id", "customer_team_id", "customer_member_id")) +@pytest.mark.parametrize("requested", ("another-customer-id", None)) +async def test_customer_binding_cannot_be_reassigned_or_cleared_by_key_holder( + operation: Literal["update", "regenerate"], field: str, requested: str | None +) -> None: + existing: Final = LiteLLM_VerificationToken(token="hashed-key", metadata={field: "customer-id"}) + data: Final = ( + UpdateKeyRequest(key="hashed-key", metadata={field: requested}) + if operation == "update" + else RegenerateKeyRequest(metadata={field: requested}) + ) + with pytest.raises(HTTPException) as rejected: + await prepare_key_update_data( + data=data, + existing_key_row=existing, + user_api_key_dict=UserAPIKeyAuth(user_id="member", user_role=LitellmUserRoles.INTERNAL_USER), + ) + assert rejected.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ("update", "regenerate")) +@pytest.mark.parametrize("metadata", ({"application": "support"}, {}, None)) +async def test_unrelated_key_metadata_changes_preserve_existing_customer_binding( + operation: Literal["update", "regenerate"], metadata: dict[str, object] | None +) -> None: + bindings: Final = {"customer_org_id": "org-id", "customer_team_id": "team-id", "customer_member_id": "member-id"} + existing: Final = LiteLLM_VerificationToken(token="hashed-key", metadata={**bindings, "old": "discard"}) + data: Final = ( + UpdateKeyRequest(key="hashed-key", metadata=metadata) + if operation == "update" + else RegenerateKeyRequest(metadata=metadata) + ) + actual: Final = await prepare_key_update_data( + data=data, + existing_key_row=existing, + user_api_key_dict=UserAPIKeyAuth(user_id="member", user_role=LitellmUserRoles.INTERNAL_USER), + ) + assert actual["metadata"] == {**(metadata or {}), **bindings} + assert existing.metadata == {**bindings, "old": "discard"} + assert data.metadata == metadata + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ("update", "regenerate")) +@pytest.mark.parametrize("metadata", ({"customer_org_id": "another-org"}, {}, None)) +async def test_proxy_admin_can_replace_or_clear_customer_key_binding( + operation: Literal["update", "regenerate"], metadata: dict[str, object] | None +) -> None: + existing: Final = LiteLLM_VerificationToken(token="hashed-key", metadata={"customer_org_id": "org-id"}) + data: Final = ( + UpdateKeyRequest(key="hashed-key", metadata=metadata) + if operation == "update" + else RegenerateKeyRequest(metadata=metadata) + ) + actual: Final = await prepare_key_update_data( + data=data, + existing_key_row=existing, + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert actual["metadata"] == metadata diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 337aa2939dd..f4138260e28 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -4097,7 +4097,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"user_api_key_customer_usage": null, "user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, From 0324b9d915bdeee8fb4910b3b90259ba8636c1e5 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 12 Nov 2026 17:08:02 -0800 Subject: [PATCH 9/9] fix(dynamic_rate_limiter_v3): a capped borrower keeps its cap on its own counter and gets a fresh share at saturation (#51494) * fix(dynamic_rate_limiter_v3): give a capped borrower its own cap counter so it also gets a fresh share at saturation A team with team_model_limit_max_borrow_multiple enforced its cap on the same counter as its share, so its idle-time usage kept counting and it got no fresh share once the model saturated. The cap now runs on its own counter (model_per_team_max_borrow_multiple) that counts all of the team's usage this window, while the share counter only counts usage once the model is saturated. Also repairs the six share-metric unit tests that main broke when #51415 and #51417 landed together, by saturating the model from a separate priority class before the measured calls. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): size a capped team's token reservation by its share, idle or saturated The reservation for a prompt without max_tokens was bounded by whichever descriptor was being held, the cap while idle and the share once saturated, so the same request was refused at one moment and admitted the next with the same counters. Reserve against the share, the tighter of the two, in both states. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): track the cap's reservation apart from the share's The cap and the share shared one reserved-token figure and the fallback carry keyed on the share's window, so an idle admission's hold on the cap was refunded from a share that never carried it once a WebSocket turn saturated, stacked idle fallback holds could carry across cap windows, and an idle fallback served for a capped team published no share gauges. Each counter now carries its own reservation and window, and a fallback hold keeps the target's share so the gauges read the cap counter against it. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): keep a WebSocket's share hold until it settles and read capped idle borrowing against the team share A frame checked after the model went idle again replaced the connection's share descriptor with None while the admission's hold on that share was still outstanding, so settlement never gave the hold back. The share is now kept while its reservation is open. With the class share split on, the share gauges picked the wider class share over a capped idle team's own share, understating its borrowing, so the tighter of the two is published. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 171 ++++++++++++++---- .../hooks/parallel_request_limiter_v3.py | 8 + .../hooks/test_dynamic_rate_limiter_v3.py | 147 ++++++++++++++- 3 files changed, 290 insertions(+), 36 deletions(-) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index a58c6feb99a..cadc07a35df 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -390,6 +390,20 @@ def _tightest_team_share(descriptors: Iterable[RateLimitDescriptor | None]) -> t return min(shares, key=lambda share: share[1], default=None) +def _tightest_of(*shares: tuple[RateLimitDescriptor, int] | None) -> tuple[RateLimitDescriptor, int] | None: + return min((share for share in shares if share is not None), key=lambda share: share[1], default=None) + + +def _capped_idle_share( + share: RateLimitDescriptor | None, cap: RateLimitDescriptor | None +) -> tuple[RateLimitDescriptor, int] | None: + """A capped team's usage while the model is idle sits on its cap counter, read against its share.""" + tightest: Final = _tightest_team_share((share,)) + if tightest is None or cap is None: + return None + return cap, tightest[1] + + def _usage_of(kwargs: Mapping[str, object], response_obj: object) -> Usage | None: """Usage of a modeled response, a Responses WebSocket turn's combined usage, or its raw event list.""" usage: Final = getattr(response_obj, "usage", None) @@ -428,7 +442,7 @@ class _BorrowingTeamShare: ceiling: RateLimitDescriptor | None def enforced(self, saturated: bool) -> RateLimitDescriptor | None: - return self.share if saturated else self.ceiling + return self.share if saturated else None def _team_limit_name(rate_limit_type: str) -> str: @@ -480,15 +494,25 @@ def _team_model_settlement( return () served: Final = stash.borrowing_team_model_fallback_limits.get(model_group) if model_group is not None else None admitted: Final = stash.borrowing_team_model_limit + cap: Final = stash.borrowing_team_model_cap reserved_tokens: Final = stash.borrowing_team_model_reserved_tokens + cap_reserved_tokens: Final = stash.borrowing_team_model_cap_reserved_tokens window: Final = stash.borrowing_team_model_reserved_window + cap_window: Final = stash.borrowing_team_model_cap_reserved_window if served is None: - return _window_guarded(_team_model_token_increment(admitted, total_tokens - reserved_tokens, ttl), window) + return ( + *_window_guarded(_team_model_token_increment(admitted, total_tokens - reserved_tokens, ttl), window), + *_window_guarded(_team_model_token_increment(cap, total_tokens - cap_reserved_tokens, ttl), cap_window), + ) return ( *_window_guarded(_team_model_token_increment(admitted, -reserved_tokens, ttl), window), + *_window_guarded(_team_model_token_increment(cap, -cap_reserved_tokens, ttl), cap_window), *_window_guarded( _team_model_token_increment(served.descriptor, total_tokens - served.reserved_tokens, ttl), served.window ), + *_window_guarded( + _team_model_token_increment(served.cap, total_tokens - served.cap_reserved_tokens, ttl), served.cap_window + ), ) @@ -858,12 +882,16 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return _BorrowingTeamShare( limits=limits, share=self._team_share_descriptor(value, limits), - ceiling=None if ceiling is None else self._team_share_descriptor(value, ceiling), + ceiling=None + if ceiling is None + else self._team_share_descriptor(value, ceiling, TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE), ) - def _team_share_descriptor(self, value: str, limits: BorrowingTeamModelLimits) -> RateLimitDescriptor: + def _team_share_descriptor( + self, value: str, limits: BorrowingTeamModelLimits, key: str = TEAM_MODEL_LIMIT_AT_SATURATION + ) -> RateLimitDescriptor: return RateLimitDescriptor( - key=TEAM_MODEL_LIMIT_AT_SATURATION, + key=key, value=value, rate_limit={ "tokens_per_unit": limits.tpm, @@ -881,7 +909,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict: UserAPIKeyAuth, team_share: _BorrowingTeamShare, ) -> ProxyRateLimitError: - if saturation >= _get_priority_settings().saturation_threshold: + if status["descriptor_key"] != TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: return self._team_limit_error(model, status, saturation, priority, user_api_key_dict) return self._team_borrow_ceiling_error( model, status, saturation, priority, user_api_key_dict, team_share.limits @@ -922,8 +950,13 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): loggers: Final = _prometheus_loggers() if not loggers or stash is None or model_group is None or settlement.user_api_key_team_id is None: return - share: Final = _tightest_team_share( - (stash.borrowing_team_model_limit, stash.priority_class_share) if hold is None else (hold.descriptor,) + share: Final = ( + _tightest_of(_tightest_team_share((hold.descriptor,)), _capped_idle_share(hold.share, hold.cap)) + if hold is not None + else _tightest_of( + _tightest_team_share((stash.borrowing_team_model_limit, stash.priority_class_share)), + _capped_idle_share(stash.borrowing_team_model_share, stash.borrowing_team_model_cap), + ) ) if share is None: return @@ -1078,10 +1111,19 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): saturation: Final = await self._check_model_saturation(requested_model, model_group_info) saturated: Final = saturation >= _get_priority_settings().saturation_threshold enforced_team: Final = None if borrowing is None else borrowing.enforced(saturated) + team_cap: Final = None if borrowing is None else borrowing.ceiling if connection_stash is not None and borrowing is not None: - connection_stash.borrowing_team_model_limit = enforced_team - descriptors: Final = (share_descriptors if saturated else ()) + ( - () if enforced_team is None else (enforced_team,) + connection_stash.borrowing_team_model_limit = ( + connection_stash.borrowing_team_model_limit + if enforced_team is None and connection_stash.borrowing_team_model_reserved_tokens + else enforced_team + ) + connection_stash.borrowing_team_model_share = borrowing.share + connection_stash.borrowing_team_model_cap = team_cap + descriptors: Final = ( + (share_descriptors if saturated else ()) + + (() if enforced_team is None else (enforced_team,)) + + (() if team_cap is None else (team_cap,)) ) if not descriptors: return @@ -1226,28 +1268,34 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if team_share is not None: descriptors_to_check.append(team_share.share) enforced_team_share: Final = None if team_share is None else team_share.enforced(should_enforce_priority) + team_cap: Final = None if team_share is None else team_share.ceiling + held_team_share: Final = enforced_team_share if enforced_team_share is not None else team_cap get_or_create_request_stash().borrowing_team_model_limit = enforced_team_share + get_or_create_request_stash().borrowing_team_model_share = None if team_share is None else team_share.share + get_or_create_request_stash().borrowing_team_model_cap = team_cap enforced_descriptors: Final[list[RateLimitDescriptor]] = [model_wide_descriptor] if priority_descriptors and should_enforce_priority: enforced_descriptors.extend(priority_descriptors) if enforced_team_share is not None: enforced_descriptors.append(enforced_team_share) + if team_cap is not None: + enforced_descriptors.append(team_cap) per_request_increment: Final[dict[Literal["requests", "tokens"], int]] = { "requests": 1, "tokens": 0, } team_reservation: Final = ( - await self._team_token_reservation(enforced_team_share, data, model, call_type, user_api_key_dict) - if enforced_team_share is not None + await self._team_token_reservation(team_share.share, data, model, call_type, user_api_key_dict) + if team_share is not None and held_team_share is not None else 0 ) atomic_response: Final = await self.v3_limiter.atomic_check_and_increment_by_n( descriptors=enforced_descriptors, increments=[ {"requests": 1, "tokens": team_reservation} - if descriptor is enforced_team_share + if descriptor is enforced_team_share or descriptor is team_cap else per_request_increment for descriptor in enforced_descriptors ], @@ -1307,7 +1355,10 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): llm_provider=llm_provider, ) - if descriptor_key == TEAM_MODEL_LIMIT_AT_SATURATION and team_share is not None: + if ( + descriptor_key in (TEAM_MODEL_LIMIT_AT_SATURATION, TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE) + and team_share is not None + ): raise self._team_share_error(model, status, saturation, priority, user_api_key_dict, team_share) if descriptor_key == PRIORITY_CLASS_SHARE_AT_SATURATION and share is not None: raise self._class_share_error(model, status, saturation, priority, user_api_key_dict, share) @@ -1352,11 +1403,15 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # enforced. The increment-then-check semantics of should_rate_limit # are fine because we ignore the OVER_LIMIT response. stash: Final = get_or_create_request_stash() - stash.borrowing_team_model_reserved_tokens = team_reservation + stash.borrowing_team_model_reserved_tokens = team_reservation if enforced_team_share is not None else 0 + stash.borrowing_team_model_cap_reserved_tokens = team_reservation if team_cap is not None else 0 stash.borrowing_team_model_reserved_window = ( _reserved_window(atomic_response, enforced_team_share) if team_reservation else None ) - self._cap_admitted_output(enforced_team_share, team_reservation, data, call_type) + stash.borrowing_team_model_cap_reserved_window = ( + _reserved_window(atomic_response, team_cap) if team_reservation else None + ) + self._cap_admitted_output(None if team_share is None else team_share.share, team_reservation, data, call_type) if priority_descriptors and share is not None and not should_enforce_priority: get_or_create_request_stash().rate_limit_response = atomic_response elif priority_descriptors and not should_enforce_priority: @@ -1431,33 +1486,48 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return True saturation: Final = await self._check_model_saturation(model, model_group_info) enforced: Final = team_share.enforced(saturation >= _get_priority_settings().saturation_threshold) + cap: Final = team_share.ceiling + held: Final = enforced if enforced is not None else cap reservation: Final = ( await self._team_token_reservation( - enforced, + team_share.share, {**request_data, "model": model}, model, get_or_create_request_stash().call_type, user_api_key_dict, ) - if enforced is not None + if held is not None else 0 ) + team_descriptors: Final = tuple(d for d in (enforced, cap) if d is not None) if not commit: - return enforced is None or await self._share_has_room(enforced, reservation, user_api_key_dict) + for descriptor in team_descriptors: + if not await self._share_has_room(descriptor, reservation, user_api_key_dict): + return False + return True model_wide_descriptor: Final = self._create_model_tracking_descriptor( model=model, model_group_info=model_group_info, high_limit_multiplier=1 ) response: Final = await self.v3_limiter.atomic_check_and_increment_by_n( - descriptors=[model_wide_descriptor, *(() if enforced is None else (enforced,))], + descriptors=[model_wide_descriptor, *team_descriptors], increments=[ {"requests": 1, "tokens": 0}, - *(() if enforced is None else ({"requests": 1, "tokens": reservation},)), + *({"requests": 1, "tokens": reservation} for _ in team_descriptors), ], parent_otel_span=user_api_key_dict.parent_otel_span, ) if response["overall_code"] == "OVER_LIMIT": return False - self._hold_fallback_share(model, team_share.share, enforced, reservation, _reserved_window(response, enforced)) + self._hold_fallback_share( + model, + team_share.share, + enforced, + reservation if enforced is not None else 0, + _reserved_window(response, enforced), + cap, + reservation if cap is not None else 0, + _reserved_window(response, cap), + ) return True async def _share_has_room( @@ -1479,16 +1549,34 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): charged: RateLimitDescriptor | None, reserved_tokens: int, window: ReservationWindowIdentity | None, + cap: RateLimitDescriptor | None = None, + cap_reserved_tokens: int = 0, + cap_window: ReservationWindowIdentity | None = None, ) -> None: """Remember the hold under the requested name and the resolved group, so settlement finds it either way. ``charged`` is the share the target's usage settles to, or None for an idle target whose usage is forgiven. + ``cap`` is the team's borrow ceiling on the target, charged whether or not the target is saturated. Each + counter carries an earlier hold on the same target only while that hold sits in the counter's current window. """ stash: Final = get_or_create_request_stash() earlier: Final = stash.borrowing_team_model_fallback_limits.get(model) - carried: Final = earlier.reserved_tokens if earlier is not None and earlier.window == window else 0 + carried: Final = ( + earlier.reserved_tokens if earlier is not None and window is not None and earlier.window == window else 0 + ) + carried_cap: Final = ( + earlier.cap_reserved_tokens + if earlier is not None and cap_window is not None and earlier.cap_window == cap_window + else 0 + ) hold: Final = BorrowingTeamModelHold( - descriptor=charged, reserved_tokens=reserved_tokens + carried, window=window + descriptor=charged, + reserved_tokens=reserved_tokens + carried, + window=window, + cap=cap, + cap_reserved_tokens=cap_reserved_tokens + carried_cap, + cap_window=cap_window, + share=share, ) stash.borrowing_team_model_fallback_limits = MappingProxyType( {**stash.borrowing_team_model_fallback_limits, model: hold, share["value"].rpartition(":")[2]: hold} @@ -1811,7 +1899,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ) if stash is not None: stash.borrowing_team_model_reserved_tokens = 0 + stash.borrowing_team_model_cap_reserved_tokens = 0 stash.borrowing_team_model_reserved_window = None + stash.borrowing_team_model_cap_reserved_window = None stash.borrowing_team_model_fallback_limits = MappingProxyType( {k: h for k, h in stash.borrowing_team_model_fallback_limits.items() if h is not served} ) @@ -1915,28 +2005,41 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): _usage_of(callback_kwargs, None if isinstance(response_obj, BaseException) else response_obj), self.v3_limiter.get_rate_limit_type(), ) + own_usage: Final = used if failed_hold is None else 0 + own_delta: Final = own_usage - stash.borrowing_team_model_reserved_tokens + own_cap_delta: Final = own_usage - stash.borrowing_team_model_cap_reserved_tokens refund: Final = ( *_window_guarded( - _team_model_token_increment( - stash.borrowing_team_model_limit, - (used if failed_hold is None else 0) - stash.borrowing_team_model_reserved_tokens, - self.v3_limiter.window_size, - ), + _team_model_token_increment(stash.borrowing_team_model_limit, own_delta, self.v3_limiter.window_size), stash.borrowing_team_model_reserved_window, ), + *_window_guarded( + _team_model_token_increment(stash.borrowing_team_model_cap, own_cap_delta, self.v3_limiter.window_size), + stash.borrowing_team_model_cap_reserved_window, + ), *( () if failed_hold is None - else _window_guarded( - _team_model_token_increment( - failed_hold.descriptor, used - failed_hold.reserved_tokens, self.v3_limiter.window_size + else ( + *_window_guarded( + _team_model_token_increment( + failed_hold.descriptor, used - failed_hold.reserved_tokens, self.v3_limiter.window_size + ), + failed_hold.window, + ), + *_window_guarded( + _team_model_token_increment( + failed_hold.cap, used - failed_hold.cap_reserved_tokens, self.v3_limiter.window_size + ), + failed_hold.cap_window, ), - failed_hold.window, ) ), ) stash.borrowing_team_model_reserved_tokens = 0 + stash.borrowing_team_model_cap_reserved_tokens = 0 stash.borrowing_team_model_reserved_window = None + stash.borrowing_team_model_cap_reserved_window = None if failed_hold is not None: stash.borrowing_team_model_fallback_limits = MappingProxyType( {k: h for k, h in stash.borrowing_team_model_fallback_limits.items() if h is not failed_hold} diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index e520a8d7d92..bf6703443c2 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -581,6 +581,10 @@ class BorrowingTeamModelHold: descriptor: RateLimitDescriptor | None reserved_tokens: int = 0 window: ReservationWindowIdentity | None = None + cap: RateLimitDescriptor | None = None + cap_reserved_tokens: int = 0 + cap_window: ReservationWindowIdentity | None = None + share: RateLimitDescriptor | None = None @dataclass(slots=True) @@ -631,6 +635,10 @@ class RequestRateLimiterStash: borrowing_team_model_limit: RateLimitDescriptor | None = None borrowing_team_model_reserved_tokens: int = 0 borrowing_team_model_reserved_window: ReservationWindowIdentity | None = None + borrowing_team_model_share: RateLimitDescriptor | None = None + borrowing_team_model_cap: RateLimitDescriptor | None = None + borrowing_team_model_cap_reserved_tokens: int = 0 + borrowing_team_model_cap_reserved_window: ReservationWindowIdentity | None = None borrowing_team_model_fallback_limits: Mapping[str, BorrowingTeamModelHold] = field( default_factory=lambda: MappingProxyType({}) ) diff --git a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py index a54912c44ba..44af7ddc08f 100644 --- a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -3140,6 +3140,62 @@ def _held_to_borrow_cap(exc: ProxyRateLimitError, rate_limit_type: str, limit_se ), exc.detail +async def _cap_tokens_remaining(handler: _PROXY_DynamicRateLimitHandlerV3, auth: UserAPIKeyAuth) -> int: + share: Final = handler._borrowing_team_share("shared-model", auth) + assert share is not None and share.ceiling is not None + response: Final = await handler.v3_limiter.should_rate_limit( + [share.ceiling], read_only=True, skip_parallel_check=True + ) + return next(s["limit_remaining"] for s in response["statuses"] if s["rate_limit_type"] == "tokens") + + +@pytest.mark.asyncio +async def test_a_capped_borrowers_idle_usage_counts_toward_its_cap_but_not_toward_its_share( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _check(handler, _capped_team_key(2), saturation=0.0) + await _settle_tokens(handler, 150) + assert await _cap_tokens_remaining(handler, _capped_team_key(2)) == 50 + assert await _team_tokens_remaining(handler, _capped_team_key(2)) == 100 + await _check(handler, _capped_team_key(2), saturation=0.5) + await _settle_tokens(handler, 30) + assert await _cap_tokens_remaining(handler, _capped_team_key(2)) == 20 + assert await _team_tokens_remaining(handler, _capped_team_key(2)) == 70 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("saturation", [0.0, 0.5], ids=["idle", "saturated"]) +async def test_a_capped_borrowers_reservation_is_sized_by_its_share_not_its_cap( + monkeypatch: pytest.MonkeyPatch, saturation: float +): + """A prompt without max_tokens reserves a floor that scales with the limit it is sized against. Sizing it + against the cap would refuse a request its own share would let through, and would admit different + requests depending on whether the model happened to be saturated.""" + handler: Final = _borrowing_handler(monkeypatch, borrows=True) + await _admit_at_the_hook(handler, _capped_team_key(2), tokens=160) + short_prompt_without_max_tokens: Final[dict[str, object]] = {"messages": [{"role": "user", "content": "hi"}]} + await _check(handler, _capped_team_key(2), saturation=saturation, data=short_prompt_without_max_tokens) + assert short_prompt_without_max_tokens["max_tokens"] == handler.v3_limiter.no_max_tokens_output_floor(100) + + +@pytest.mark.asyncio +async def test_a_capped_borrower_is_held_to_its_cap_while_idle_after_a_saturated_stretch( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _check(handler, _capped_team_key(2), saturation=0.0) + await _settle_tokens(handler, 150) + await _check(handler, _capped_team_key(2), saturation=0.5) + await _settle_tokens(handler, 60) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _capped_team_key(2), saturation=0.49) + _held_to_borrow_cap(exc.value, "tokens", "model_tpm_limit") + with pytest.raises(ProxyRateLimitError) as saturated: + await _check(handler, _capped_team_key(2), saturation=0.5) + _held_to_borrow_cap(saturated.value, "tokens", "model_tpm_limit") + + @pytest.mark.asyncio async def test_team_borrows_past_its_share_while_it_stays_under_its_max_borrow_multiple( monkeypatch: pytest.MonkeyPatch, @@ -3248,6 +3304,44 @@ async def test_a_websocket_turn_is_refused_at_the_teams_max_borrow_multiple_whil _held_to_borrow_cap(exc.value, "tokens", "model_tpm_limit") +@pytest.mark.asyncio +async def test_a_websocket_admitted_while_saturated_gives_its_share_hold_back_when_the_model_goes_idle( + monkeypatch: pytest.MonkeyPatch, +): + handler: Final = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.5) + await _check(handler, _capped_team_key(2), saturation=0.5, data=_prompt(20)) + assert await _team_tokens_remaining(handler, _capped_team_key(2)) < 100 + await handler.enforce_token_limits_for_frame(user_api_key_dict=_capped_team_key(2), requested_model="shared-model") + await _settle_tokens( + handler, + 90, + response=[], + combined_usage_object=litellm.Usage(prompt_tokens=60, completion_tokens=30, total_tokens=90), + ) + assert await _team_tokens_remaining(handler, _capped_team_key(2)) == 10 + assert await _cap_tokens_remaining(handler, _capped_team_key(2)) == 110 + + +@pytest.mark.asyncio +async def test_a_capped_websocket_admitted_while_idle_settles_its_first_saturated_turn_on_a_fresh_share( + monkeypatch: pytest.MonkeyPatch, +): + """The idle admission held tokens on the cap only, so the share the turn settles to once the model saturates + starts from zero rather than being refunded a hold it never carried.""" + handler: Final = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.1) + await _check(handler, _capped_team_key(2), saturation=0.0, data=_prompt(20)) + await asyncio.create_task(_saturate_model_from_another_request(handler, 200)) + await handler.enforce_token_limits_for_frame(user_api_key_dict=_capped_team_key(2), requested_model="shared-model") + await _settle_tokens( + handler, + 90, + response=[], + combined_usage_object=litellm.Usage(prompt_tokens=60, completion_tokens=30, total_tokens=90), + ) + assert await _team_tokens_remaining(handler, _capped_team_key(2)) == 10 + assert await _cap_tokens_remaining(handler, _capped_team_key(2)) == 110 + + @pytest.mark.asyncio async def test_a_websocket_turn_under_the_teams_max_borrow_multiple_still_borrows(monkeypatch: pytest.MonkeyPatch): handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.5) @@ -3444,6 +3538,22 @@ async def _admit_at_the_hook(handler: _PROXY_DynamicRateLimitHandlerV3, auth: Us await _settle_tokens(handler, tokens) +async def _saturate_model_from_another_request(handler: _PROXY_DynamicRateLimitHandlerV3, tokens: int) -> None: + """``_saturate_model`` on a request stash of its own, as a second request in the proxy would have, so the + stash of the request under test keeps its holds.""" + from litellm.proxy.hooks import parallel_request_limiter_v3 + + parallel_request_limiter_v3._request_stash.set(None) # pyright: ignore[reportPrivateUsage] # fresh request context + await _saturate_model(handler, tokens) + + +async def _saturate_model(handler: _PROXY_DynamicRateLimitHandlerV3, tokens: int) -> None: + """Idle usage that pushes the model past its saturation threshold without touching any team's share, from a + priority class of its own so the default pool the measured team sits in stays untouched.""" + warm_up: Final = UserAPIKeyAuth(api_key="sk-warm-up", team_id="team-warm-up", team_metadata={"priority": "batch"}) + await _admit_at_the_hook(handler, warm_up, tokens=tokens) + + async def _refused_at_the_hook( handler: _PROXY_DynamicRateLimitHandlerV3, auth: UserAPIKeyAuth, model: str = "shared-model" ) -> ProxyRateLimitError: @@ -3507,7 +3617,7 @@ async def test_a_failed_request_settles_the_team_share_gauges_too(monkeypatch: p "auth, burns, saturation_threshold, reason, enforced_share_tokens", [ (_team_key(), (60, 70), 0.0, "model_per_team_saturated", 100), - (_capped_team_key(2), (150, 60), 0.5, "model_per_team_max_borrow_multiple", 200), + (_capped_team_key(2), (150, 60), 0.5, "model_per_team_max_borrow_multiple", 100), ], ids=["saturated", "borrow_cap"], ) @@ -3535,7 +3645,8 @@ async def test_a_refused_request_is_counted_under_the_reason_its_429_header_carr ) assert _sample(registry, "litellm_dynamic_limiter_share_tokens", **TEAM_LABELS) == enforced_share_tokens assert ( - _sample(registry, "litellm_dynamic_limiter_borrowed_tokens", **TEAM_LABELS) == sum(burns) - enforced_share_tokens + _sample(registry, "litellm_dynamic_limiter_borrowed_tokens", **TEAM_LABELS) + == sum(burns) - enforced_share_tokens ) @@ -3607,6 +3718,7 @@ async def test_share_gauges_and_throttles_carry_the_teams_priority_so_the_series team_id="team-a", team_metadata={"model_tpm_limit": {"shared-model": 100}, "priority": "interactive"}, ) + await _saturate_model(handler, 200) await _admit_at_the_hook(handler, team, tokens=60) await _admit_at_the_hook(handler, team, tokens=70) labels: Final = {"model_group": "shared-model", "team": "team-a", "priority": "interactive"} @@ -3626,6 +3738,7 @@ async def test_a_fallback_served_request_publishes_the_teams_share_on_the_fallba handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) registry = _prometheus_sink(monkeypatch) team: Final = _team_key(team_limits={"model_tpm_limit": {"other-model": 40, "shared-model": 100}}) + await _saturate_model(handler, 500) assert await handler.async_pre_call_hook(team, DualCache(), {"model": "other-model"}, "completion") is None assert await handler.async_fallback_target_allowed(team, "shared-model", {}) is True await _settle_tokens(handler, 150, model_group="shared-model") @@ -3635,6 +3748,35 @@ async def test_a_fallback_served_request_publishes_the_teams_share_on_the_fallba assert _sample(registry, "litellm_dynamic_limiter_share_tokens", model_group="other-model") is None +@pytest.mark.asyncio +async def test_a_capped_idle_teams_borrowing_is_read_against_its_share_not_the_wider_class_share( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True, splits=True) + registry = _prometheus_sink(monkeypatch) + await _admit_at_the_hook(handler, _capped_team_key(2), tokens=150) + assert _sample(registry, "litellm_dynamic_limiter_share_tokens", **TEAM_LABELS) == 100 + assert _sample(registry, "litellm_dynamic_limiter_borrowed_tokens", **TEAM_LABELS) == 50 + + +@pytest.mark.asyncio +async def test_an_idle_fallback_served_for_a_capped_team_still_publishes_its_borrowing(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True) + registry = _prometheus_sink(monkeypatch) + team: Final = _team_key( + team_limits={ + "model_tpm_limit": {"other-model": 40, "shared-model": 100}, + "team_model_limit_max_borrow_multiple": 2, + } + ) + assert await handler.async_pre_call_hook(team, DualCache(), {"model": "other-model"}, "completion") is None + assert await handler.async_fallback_target_allowed(team, "shared-model", {}) is True + await _settle_tokens(handler, 150, model_group="shared-model") + shared: Final = {"model_group": "shared-model", "team": "team-a", "priority": "default"} + assert _sample(registry, "litellm_dynamic_limiter_share_tokens", **shared) == 100 + assert _sample(registry, "litellm_dynamic_limiter_borrowed_tokens", **shared) == 50 + + @pytest.mark.asyncio async def test_a_failing_metrics_sink_never_changes_admission(monkeypatch: pytest.MonkeyPatch): from litellm.integrations.prometheus import PrometheusLogger @@ -3648,6 +3790,7 @@ async def test_a_failing_metrics_sink_never_changes_admission(monkeypatch: pytes handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) _prometheus_sink(monkeypatch, BrokenSink) + await _saturate_model(handler, 200) await _admit_at_the_hook(handler, _team_key(), tokens=60) await _admit_at_the_hook(handler, _team_key(), tokens=70) assert await _team_tokens_remaining(handler) == -30