mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
feat(ui): add per-model rate limits to team edit/info views
Exposes the backend's existing model_tpm_limit/model_rpm_limit fields (which lived in team.metadata) through a new "Model-Specific Rate Limits" form section on the team Settings tab. Limits round-trip through the team-update API and render on the Overview card and Settings view. Model picker is scoped to the team's currently-selected models (unfurls wildcards, falls back to userModels for all-proxy-models / all-team-models).
This commit is contained in:
Ryan Crabbe
parent
976e4c889d
commit
89f32e1d75
1 changed files with 138 additions and 3 deletions
|
|
@ -17,10 +17,10 @@ import {
|
|||
import { formatNumberWithCommas } from "@/utils/dataUtils";
|
||||
import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils";
|
||||
import { isProxyAdminRole } from "@/utils/roles";
|
||||
import { EditOutlined, InfoCircleOutlined, SaveOutlined } from "@ant-design/icons";
|
||||
import { EditOutlined, InfoCircleOutlined, MinusCircleOutlined, PlusOutlined, SaveOutlined } from "@ant-design/icons";
|
||||
import { ArrowLeftIcon } from "@heroicons/react/outline";
|
||||
import { Badge, Card, Grid, Text, TextInput, Title } from "@tremor/react";
|
||||
import { Button, Form, Input, message, Select, Switch, Tabs, Tooltip } from "antd";
|
||||
import { Button, Form, Input, InputNumber, message, Select, Space, Switch, Tabs, Tooltip } from "antd";
|
||||
import { CheckIcon, CopyIcon } from "lucide-react";
|
||||
import React, { useEffect, useMemo, useState } from "react";
|
||||
import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils";
|
||||
|
|
@ -194,6 +194,17 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
return org?.members?.some((m: any) => m.user_id === userId && m.user_role === "org_admin") ?? false;
|
||||
}, [teamData, userOrganizations, userId]);
|
||||
|
||||
// Models currently selected in the team edit form, used to scope the per-model
|
||||
// rate limit dropdown to models this team actually has access to.
|
||||
const selectedModelsInForm = Form.useWatch("models", form) as string[] | undefined;
|
||||
const availableRateLimitModels = useMemo(() => {
|
||||
const selected = selectedModelsInForm ?? teamData?.team_info?.models ?? [];
|
||||
if (selected.includes("all-proxy-models") || selected.includes("all-team-models")) {
|
||||
return userModels;
|
||||
}
|
||||
return unfurlWildcardModelsInList(selected, userModels);
|
||||
}, [selectedModelsInForm, teamData, userModels]);
|
||||
|
||||
const canEditTeam = is_team_admin || is_proxy_admin || is_org_admin || isOrgAdminForTeam;
|
||||
const visibleTabs = useMemo(() => getTeamInfoVisibleTabs(canEditTeam), [canEditTeam]);
|
||||
const defaultTabKey = useMemo(
|
||||
|
|
@ -466,12 +477,23 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
return v;
|
||||
};
|
||||
|
||||
const modelTpmLimit: Record<string, number> = {};
|
||||
const modelRpmLimit: Record<string, number> = {};
|
||||
for (const entry of (values.modelLimits ?? []) as { model?: string; tpm?: number; rpm?: number }[]) {
|
||||
if (entry?.model) {
|
||||
if (entry.tpm != null) modelTpmLimit[entry.model] = entry.tpm;
|
||||
if (entry.rpm != null) modelRpmLimit[entry.model] = entry.rpm;
|
||||
}
|
||||
}
|
||||
|
||||
const updateData: any = {
|
||||
team_id: teamId,
|
||||
team_alias: values.team_alias,
|
||||
models: values.models,
|
||||
tpm_limit: sanitizeNumeric(values.tpm_limit),
|
||||
rpm_limit: sanitizeNumeric(values.rpm_limit),
|
||||
model_tpm_limit: modelTpmLimit,
|
||||
model_rpm_limit: modelRpmLimit,
|
||||
max_budget: values.max_budget,
|
||||
soft_budget: sanitizeNumeric(values.soft_budget),
|
||||
budget_duration: values.budget_duration,
|
||||
|
|
@ -648,6 +670,22 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
<Text>TPM: {info.tpm_limit || "Unlimited"}</Text>
|
||||
<Text>RPM: {info.rpm_limit || "Unlimited"}</Text>
|
||||
{info.max_parallel_requests && <Text>Max Parallel Requests: {info.max_parallel_requests}</Text>}
|
||||
{(() => {
|
||||
const modelTpm = (info.metadata?.model_tpm_limit ?? {}) as Record<string, number>;
|
||||
const modelRpm = (info.metadata?.model_rpm_limit ?? {}) as Record<string, number>;
|
||||
const models = Array.from(new Set([...Object.keys(modelTpm), ...Object.keys(modelRpm)]));
|
||||
if (models.length === 0) return null;
|
||||
return (
|
||||
<div className="mt-3">
|
||||
<Text className="text-gray-500">Per-model limits:</Text>
|
||||
{models.map((m) => (
|
||||
<Text key={m} className="text-xs">
|
||||
{m}: TPM {modelTpm[m] ?? "—"}, RPM {modelRpm[m] ?? "—"}
|
||||
</Text>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
|
|
@ -793,6 +831,16 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
models: info.models,
|
||||
tpm_limit: info.tpm_limit,
|
||||
rpm_limit: info.rpm_limit,
|
||||
modelLimits: Array.from(
|
||||
new Set([
|
||||
...Object.keys(info.metadata?.model_tpm_limit ?? {}),
|
||||
...Object.keys(info.metadata?.model_rpm_limit ?? {}),
|
||||
]),
|
||||
).map((model) => ({
|
||||
model,
|
||||
tpm: info.metadata?.model_tpm_limit?.[model],
|
||||
rpm: info.metadata?.model_rpm_limit?.[model],
|
||||
})),
|
||||
max_budget: info.max_budget,
|
||||
soft_budget: info.soft_budget,
|
||||
budget_duration: info.budget_duration,
|
||||
|
|
@ -809,7 +857,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
: "",
|
||||
metadata: info.metadata
|
||||
? JSON.stringify(
|
||||
(({ logging, secret_manager_settings, soft_budget_alerting_emails, ...rest }) => rest)(info.metadata),
|
||||
(({ logging, secret_manager_settings, soft_budget_alerting_emails, model_tpm_limit, model_rpm_limit, ...rest }) => rest)(info.metadata),
|
||||
null,
|
||||
2,
|
||||
)
|
||||
|
|
@ -934,6 +982,77 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
<NumericalInput step={1} style={{ width: "100%" }} />
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
label="Model-Specific Rate Limits"
|
||||
tooltip="Set per-model TPM/RPM limits that apply across the whole team."
|
||||
>
|
||||
<Form.List name="modelLimits">
|
||||
{(fields, { add, remove }) => (
|
||||
<>
|
||||
{fields.map(({ key, name, ...restField }) => (
|
||||
<Space
|
||||
key={key}
|
||||
style={{ display: "flex", marginBottom: 8 }}
|
||||
align="baseline"
|
||||
>
|
||||
<Form.Item
|
||||
{...restField}
|
||||
name={[name, "model"]}
|
||||
rules={[
|
||||
{ required: true, message: "Missing model" },
|
||||
{
|
||||
validator: (_, value) => {
|
||||
if (!value) return Promise.resolve();
|
||||
const all = form.getFieldValue("modelLimits") ?? [];
|
||||
const dupes = all.filter(
|
||||
(entry: { model?: string }) => entry?.model === value,
|
||||
);
|
||||
if (dupes.length > 1) {
|
||||
return Promise.reject(new Error("Duplicate model"));
|
||||
}
|
||||
return Promise.resolve();
|
||||
},
|
||||
},
|
||||
]}
|
||||
style={{ minWidth: 240 }}
|
||||
>
|
||||
<Select
|
||||
showSearch
|
||||
placeholder="Select model"
|
||||
allowClear
|
||||
options={availableRateLimitModels.map((m) => ({
|
||||
value: m,
|
||||
label: m,
|
||||
}))}
|
||||
/>
|
||||
</Form.Item>
|
||||
<Form.Item {...restField} name={[name, "tpm"]}>
|
||||
<InputNumber placeholder="TPM Limit" min={0} />
|
||||
</Form.Item>
|
||||
<Form.Item {...restField} name={[name, "rpm"]}>
|
||||
<InputNumber placeholder="RPM Limit" min={0} />
|
||||
</Form.Item>
|
||||
<MinusCircleOutlined
|
||||
onClick={() => remove(name)}
|
||||
style={{ color: "#ef4444" }}
|
||||
/>
|
||||
</Space>
|
||||
))}
|
||||
<Form.Item>
|
||||
<Button
|
||||
type="dashed"
|
||||
onClick={() => add()}
|
||||
block
|
||||
icon={<PlusOutlined />}
|
||||
>
|
||||
Add Model Limit
|
||||
</Button>
|
||||
</Form.Item>
|
||||
</>
|
||||
)}
|
||||
</Form.List>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
label={
|
||||
<span>
|
||||
|
|
@ -1161,6 +1280,22 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
<Text className="font-medium">Rate Limits</Text>
|
||||
<div>TPM: {info.tpm_limit || "Unlimited"}</div>
|
||||
<div>RPM: {info.rpm_limit || "Unlimited"}</div>
|
||||
{(() => {
|
||||
const modelTpm = (info.metadata?.model_tpm_limit ?? {}) as Record<string, number>;
|
||||
const modelRpm = (info.metadata?.model_rpm_limit ?? {}) as Record<string, number>;
|
||||
const models = Array.from(new Set([...Object.keys(modelTpm), ...Object.keys(modelRpm)]));
|
||||
if (models.length === 0) return null;
|
||||
return (
|
||||
<div className="mt-2">
|
||||
<Text className="text-gray-500">Per-model limits:</Text>
|
||||
{models.map((m) => (
|
||||
<div key={m} className="text-xs ml-2">
|
||||
{m}: TPM {modelTpm[m] ?? "—"}, RPM {modelRpm[m] ?? "—"}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
</div>
|
||||
<div>
|
||||
<Text className="font-medium">Team Budget</Text>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue