mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-18 04:44:37 +00:00
feat(ui): add the 1 hour cache write cost to the add model and model edit forms (#46830)
* feat(ui): add the 1 hour cache write cost to the add model and model edit forms Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): cover both cache write tiers in the add model and edit forms Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): hoist the expected pricing payload out of the inline assertion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(ui): share the per-token conversion between both cache write tiers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(ui): derive the touched pricing field type from the pricing field list Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(ui): collapse the duplicated pricing source keys into one map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): convert add model pricing once for all selected models and describe the 1 hour fallback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): type the cache write tier deployment read back and fill the 1 hour field Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(e2e): keep the cache write tier spec on the dashboard prettier config Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
ryan-crabbe-berri
GitHub
parent
d6b54ae2b0
commit
c7b115b56a
13 changed files with 409 additions and 75 deletions
|
|
@ -3900,6 +3900,7 @@ class MirroredPricingParams(LiteLLMBaseModel):
|
|||
output_cost_per_character: float | None = None
|
||||
cache_read_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost_above_1hr: float | None = None
|
||||
tiered_pricing: list[dict[str, Any]] | None = None
|
||||
|
||||
|
||||
|
|
@ -3927,7 +3928,6 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
input_cost_per_token_priority: float | None = None
|
||||
input_cost_per_token_balanced: float | None = None
|
||||
input_cost_per_token_ultrafast: float | None = None
|
||||
cache_creation_input_token_cost_above_1hr: float | None = None
|
||||
cache_creation_input_token_cost_above_100k_tokens: float | None = None
|
||||
cache_creation_input_token_cost_above_1hr_above_100k_tokens: float | None = None
|
||||
cache_creation_input_token_cost_above_200k_tokens: float | None = None
|
||||
|
|
|
|||
146
tests/e2e/ui/tests/modelsPage/cacheWriteTiers.spec.ts
Normal file
146
tests/e2e/ui/tests/modelsPage/cacheWriteTiers.spec.ts
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { Role, users } from "../../fixtures/users";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { navigateToPage } from "../../helpers/navigation";
|
||||
import { captureRequestBody, readBack } from "../../helpers/roundTrip";
|
||||
|
||||
const FIVE_MINUTE_PER_MILLION = 3.75;
|
||||
const ONE_HOUR_PER_MILLION = 7.5;
|
||||
const perToken = (perMillion: number): number => perMillion / 1_000_000;
|
||||
const ONE_HOUR_LABEL = /^Cache Write Cost \(1 hour, per 1M tokens\)/;
|
||||
|
||||
type Deployment = {
|
||||
model_name?: string;
|
||||
model_info?: { id?: string };
|
||||
litellm_params: {
|
||||
cache_creation_input_token_cost?: number;
|
||||
cache_creation_input_token_cost_above_1hr?: number;
|
||||
};
|
||||
};
|
||||
|
||||
const masterKey = users[Role.ProxyAdmin].password;
|
||||
const auth = { Authorization: `Bearer ${masterKey}` };
|
||||
|
||||
async function deleteModel(page: PlaywrightPage, id: string) {
|
||||
await page.request.post("/model/delete", { headers: auth, data: { id } });
|
||||
}
|
||||
|
||||
async function readDeployment(page: PlaywrightPage, modelId: string): Promise<Deployment | undefined> {
|
||||
const body = await readBack<{ data: Deployment[] }>(page, `/model/info?litellm_model_id=${modelId}`);
|
||||
return body.data[0];
|
||||
}
|
||||
|
||||
async function selectProvider(page: PlaywrightPage, providerName: string) {
|
||||
const providerDropdown = page.getByRole("combobox", { name: "Provider", exact: true });
|
||||
await providerDropdown.click();
|
||||
await providerDropdown.fill(providerName);
|
||||
await page.getByRole("option").filter({ hasText: providerName }).first().click();
|
||||
await expect(providerDropdown).toHaveValue(providerName);
|
||||
}
|
||||
|
||||
test.describe("Cache write cost tiers", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
let createdModelId: string | null = null;
|
||||
|
||||
test.afterEach(async ({ page }) => {
|
||||
if (createdModelId) {
|
||||
await deleteModel(page, createdModelId);
|
||||
createdModelId = null;
|
||||
}
|
||||
});
|
||||
|
||||
test("the edit form saves the 1 hour rate next to the 5 minute rate and the detail view shows both", async ({
|
||||
page,
|
||||
}) => {
|
||||
const modelName = `e2e-cache-tiers-edit-${Date.now()}`;
|
||||
const created = await page.request.post("/model/new", {
|
||||
headers: auth,
|
||||
data: {
|
||||
model_name: modelName,
|
||||
litellm_params: {
|
||||
model: "anthropic/claude-sonnet-4-6",
|
||||
api_key: "sk-e2e-not-used",
|
||||
cache_creation_input_token_cost: perToken(FIVE_MINUTE_PER_MILLION),
|
||||
},
|
||||
model_info: {},
|
||||
},
|
||||
});
|
||||
expect(created.ok(), `POST /model/new for ${modelName}`).toBe(true);
|
||||
createdModelId = (await created.json()).model_info?.id;
|
||||
expect(createdModelId, "model id from /model/new").toBeTruthy();
|
||||
|
||||
await page.goto("/ui");
|
||||
await page.getByText("Models + Endpoints").click();
|
||||
const modelIdCell = page.getByTestId(`model-id-${createdModelId}`);
|
||||
await expect(modelIdCell).toBeVisible({ timeout: 15_000 });
|
||||
await modelIdCell.click();
|
||||
await expect(page.getByText("Back to Models").first()).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
await page.getByRole("button", { name: "Edit Settings" }).click();
|
||||
const oneHour = page.getByLabel(ONE_HOUR_LABEL);
|
||||
await oneHour.waitFor({ timeout: 15_000 });
|
||||
await oneHour.fill(String(ONE_HOUR_PER_MILLION));
|
||||
|
||||
const patchBody = await captureRequestBody(
|
||||
page,
|
||||
{ method: "PATCH", urlIncludes: `/model/${createdModelId}/update` },
|
||||
async () => {
|
||||
await page.getByRole("button", { name: "Save Changes" }).click();
|
||||
},
|
||||
);
|
||||
expect(patchBody.litellm_params.cache_creation_input_token_cost_above_1hr, "1 hour rate sent per token").toBe(
|
||||
perToken(ONE_HOUR_PER_MILLION),
|
||||
);
|
||||
await expect(page.getByText("Model settings updated successfully")).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
const stored = await readDeployment(page, createdModelId!);
|
||||
expect(stored?.litellm_params?.cache_creation_input_token_cost_above_1hr, "1 hour rate persisted").toBe(
|
||||
perToken(ONE_HOUR_PER_MILLION),
|
||||
);
|
||||
expect(stored?.litellm_params?.cache_creation_input_token_cost, "5 minute rate untouched").toBe(
|
||||
perToken(FIVE_MINUTE_PER_MILLION),
|
||||
);
|
||||
|
||||
await expect(page.getByText(ONE_HOUR_LABEL)).toBeVisible();
|
||||
await expect(page.getByText(ONE_HOUR_PER_MILLION.toFixed(4))).toBeVisible({ timeout: 10_000 });
|
||||
await expect(page.getByText(FIVE_MINUTE_PER_MILLION.toFixed(4))).toBeVisible();
|
||||
});
|
||||
|
||||
test("Add Model sends both cache write tiers per token", async ({ page }) => {
|
||||
const publicName = `e2e-cache-tiers-add-${Date.now()}`;
|
||||
await navigateToPage(page, Page.Models);
|
||||
await page.getByRole("tab", { name: "Add Model" }).click();
|
||||
await selectProvider(page, "Anthropic");
|
||||
|
||||
await page.getByRole("combobox", { name: "Select models" }).click();
|
||||
await page.getByRole("option", { name: "Custom Model Name (Enter below)" }).click();
|
||||
await page.keyboard.press("Escape");
|
||||
await page.getByPlaceholder("Enter custom model name").fill(publicName);
|
||||
await page.locator("#api_key").fill("sk-e2e-not-used");
|
||||
|
||||
await page.getByText("Advanced Settings").click();
|
||||
await page.getByRole("switch", { name: "Custom Pricing" }).click();
|
||||
await page.getByLabel(/^Input Cost \(per 1M tokens\)/).fill("3");
|
||||
await page.getByLabel(/^Output Cost \(per 1M tokens\)/).fill("15");
|
||||
await page.getByLabel(/^Cache Write Cost \(per 1M tokens\)/).fill(String(FIVE_MINUTE_PER_MILLION));
|
||||
await page.getByLabel(ONE_HOUR_LABEL).fill(String(ONE_HOUR_PER_MILLION));
|
||||
|
||||
const body = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => {
|
||||
await page.getByTestId("add-model-btn").click();
|
||||
});
|
||||
await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 });
|
||||
const listed = await readBack<{ data: Deployment[] }>(page, "/v2/model/info");
|
||||
createdModelId = listed.data.find((row) => row.model_name === publicName)?.model_info?.id ?? null;
|
||||
|
||||
expect(body.model_name).toBe(publicName);
|
||||
expect(body.litellm_params.cache_creation_input_token_cost, "5 minute rate sent per token").toBe(
|
||||
perToken(FIVE_MINUTE_PER_MILLION),
|
||||
);
|
||||
expect(body.litellm_params.cache_creation_input_token_cost_above_1hr, "1 hour rate sent per token").toBe(
|
||||
perToken(ONE_HOUR_PER_MILLION),
|
||||
);
|
||||
expect(createdModelId, "the model landed in the database").toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
|
@ -87,7 +87,7 @@ test.describe("Clear custom pricing on a deployment", () => {
|
|||
// Both cache fields share the same placeholder ("Defaults to Input Cost if blank"),
|
||||
// so disambiguate via their labels.
|
||||
const cacheReadCost = page.getByLabel(/Cache Read Cost/);
|
||||
const cacheWriteCost = page.getByLabel(/Cache Write Cost/);
|
||||
const cacheWriteCost = page.getByLabel("Cache Write Cost (per 1M tokens)", { exact: true });
|
||||
await inputCost.waitFor({ timeout: 15_000 });
|
||||
for (const field of [inputCost, outputCost, cacheReadCost, cacheWriteCost]) {
|
||||
await field.click({ clickCount: 3 });
|
||||
|
|
|
|||
|
|
@ -3758,6 +3758,43 @@ class TestUpdateDBModelClearPricing:
|
|||
assert "cache_creation_input_token_cost" not in params
|
||||
assert "cache_creation_input_token_cost" not in info
|
||||
|
||||
def test_clear_cache_write_1h_cost_removes_from_both_blobs(self):
|
||||
"""The dashboard clears the 1 hour cache-write tier with an explicit null, which only
|
||||
takes effect for fields in SPECIAL_MODEL_INFO_PARAMS."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
update_db_model,
|
||||
)
|
||||
from litellm.types.router import (
|
||||
Deployment,
|
||||
LiteLLM_Params,
|
||||
ModelInfo,
|
||||
updateLiteLLMParams,
|
||||
)
|
||||
|
||||
db_model = Deployment(
|
||||
model_name="anthropic/*",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="anthropic/*",
|
||||
cache_creation_input_token_cost=0.00000375,
|
||||
cache_creation_input_token_cost_above_1hr=0.000006,
|
||||
),
|
||||
model_info=ModelInfo(id="dep-cache-write-1h-0"),
|
||||
)
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(
|
||||
litellm_params=updateLiteLLMParams(cache_creation_input_token_cost_above_1hr=None)
|
||||
),
|
||||
)
|
||||
|
||||
params = json.loads(result["litellm_params"])
|
||||
info = json.loads(result["model_info"])
|
||||
assert "cache_creation_input_token_cost_above_1hr" not in params
|
||||
assert "cache_creation_input_token_cost_above_1hr" not in info
|
||||
assert params["cache_creation_input_token_cost"] == 0.00000375
|
||||
assert info["cache_creation_input_token_cost"] == 0.00000375
|
||||
|
||||
def test_clear_cache_read_preserves_other_pricing(self):
|
||||
"""Clearing cache_read must not touch input/output cost overrides."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ def test_custom_pricing_params_keeps_every_field_it_had():
|
|||
"output_cost_per_character",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
"cache_creation_input_token_cost_above_1hr",
|
||||
"cost_per_second",
|
||||
"input_cost_per_second",
|
||||
"cache_read_input_token_cost_flex",
|
||||
|
|
|
|||
|
|
@ -63,20 +63,21 @@ const PTU_EDIT_FIELDS: PtuEditField[] = [
|
|||
{ name: PTU_END_FIELD, label: "PTU Effective To (UTC)", input: "datetime" },
|
||||
];
|
||||
|
||||
export type TouchedPricingField = "input_cost" | "output_cost" | "cache_read_cost" | "cache_write_cost";
|
||||
|
||||
const PRICING_FIELDS: readonly TouchedPricingField[] = [
|
||||
const PRICING_FIELDS = [
|
||||
"input_cost",
|
||||
"output_cost",
|
||||
"cache_read_cost",
|
||||
"cache_write_cost",
|
||||
"cache_write_cost_1h",
|
||||
] as const;
|
||||
export type TouchedPricingField = (typeof PRICING_FIELDS)[number];
|
||||
|
||||
const COST_SOURCES: Record<TouchedPricingField, { param: string; info: string }> = {
|
||||
input_cost: { param: "input_cost_per_token", info: "input_cost_per_token" },
|
||||
output_cost: { param: "output_cost_per_token", info: "output_cost_per_token" },
|
||||
cache_read_cost: { param: "cache_read_input_token_cost", info: "cache_read_input_token_cost" },
|
||||
cache_write_cost: { param: "cache_creation_input_token_cost", info: "cache_creation_input_token_cost" },
|
||||
const COST_SOURCES: Record<TouchedPricingField, string> = {
|
||||
input_cost: "input_cost_per_token",
|
||||
output_cost: "output_cost_per_token",
|
||||
cache_read_cost: "cache_read_input_token_cost",
|
||||
cache_write_cost: "cache_creation_input_token_cost",
|
||||
cache_write_cost_1h: "cache_creation_input_token_cost_above_1hr",
|
||||
};
|
||||
|
||||
export interface ModelEditFormValues {
|
||||
|
|
@ -94,6 +95,7 @@ export interface ModelEditFormValues {
|
|||
output_cost?: string | number | null;
|
||||
cache_read_cost?: string | number | null;
|
||||
cache_write_cost?: string | number | null;
|
||||
cache_write_cost_1h?: string | number | null;
|
||||
ptu_count?: string | number | null;
|
||||
cost_per_ptu_per_hour?: string | number | null;
|
||||
ptu_effective_from?: Dayjs | null;
|
||||
|
|
@ -131,6 +133,7 @@ const modelEditShape = {
|
|||
output_cost: scalar,
|
||||
cache_read_cost: scalar,
|
||||
cache_write_cost: scalar,
|
||||
cache_write_cost_1h: scalar,
|
||||
ptu_count: scalar,
|
||||
cost_per_ptu_per_hour: scalar,
|
||||
ptu_effective_from: z.custom<Dayjs | null>().nullish(),
|
||||
|
|
@ -243,6 +246,10 @@ export const toModelEditFormValues = (localModelData: any, isWildcardModel: bool
|
|||
localModelData.litellm_params?.cache_creation_input_token_cost,
|
||||
localModelData.model_info?.cache_creation_input_token_cost,
|
||||
),
|
||||
cache_write_cost_1h: perMillionTokens(
|
||||
localModelData.litellm_params?.cache_creation_input_token_cost_above_1hr,
|
||||
localModelData.model_info?.cache_creation_input_token_cost_above_1hr,
|
||||
),
|
||||
cache_control: localModelData.litellm_params?.cache_control_injection_points ? true : false,
|
||||
cache_control_injection_points: localModelData.litellm_params?.cache_control_injection_points || [],
|
||||
model_access_group: Array.isArray(localModelData.model_info?.access_groups)
|
||||
|
|
@ -271,8 +278,8 @@ export const toModelEditFormValues = (localModelData: any, isWildcardModel: bool
|
|||
});
|
||||
|
||||
const displayCost = (localModelData: any, field: TouchedPricingField): string => {
|
||||
const { param, info } = COST_SOURCES[field];
|
||||
const rate = localModelData?.litellm_params?.[param] ?? localModelData?.model_info?.[info];
|
||||
const key = COST_SOURCES[field];
|
||||
const rate = localModelData?.litellm_params?.[key] ?? localModelData?.model_info?.[key];
|
||||
return rate != null ? (Number(rate) * 1_000_000).toFixed(4) : "Not Set";
|
||||
};
|
||||
|
||||
|
|
@ -527,6 +534,12 @@ const ModelInfoEditForm: React.FC<ModelInfoEditFormProps> = ({
|
|||
"Defaults to Input Cost if blank",
|
||||
"If left blank on save, defaults to Input Cost (backend falls back to input_cost_per_token).",
|
||||
)}
|
||||
{pricingField(
|
||||
"cache_write_cost_1h",
|
||||
"Cache Write Cost (1 hour, per 1M tokens)",
|
||||
"Defaults to the cost map 1 hour rate, else Cache Write Cost",
|
||||
"Rate for prompt cache writes with a 1 hour TTL. If left blank on save, known models use their cost map 1 hour rate, otherwise Cache Write Cost.",
|
||||
)}
|
||||
|
||||
{textField("api_base", "API Base", "Enter API base", localModelData.litellm_params?.api_base)}
|
||||
{textField(
|
||||
|
|
|
|||
|
|
@ -52,6 +52,17 @@ describe("AdvancedSettings", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("offers both cache write tiers once custom pricing is on", async () => {
|
||||
renderAdvancedSettings();
|
||||
act(() => {
|
||||
fireEvent.click(screen.getByText("Advanced Settings"));
|
||||
});
|
||||
fireEvent.click(await screen.findByRole("switch", { name: "Custom Pricing" }));
|
||||
|
||||
expect(await screen.findByLabelText(/^Cache Write Cost \(per 1M tokens\)/)).toBeInTheDocument();
|
||||
expect(screen.getByLabelText(/^Cache Write Cost \(1 hour, per 1M tokens\)/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("hides every PTU field when PTU cost attribution is disabled", async () => {
|
||||
renderAdvancedSettings();
|
||||
act(() => {
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ const USAGE_COST_FIELDS = [
|
|||
"output_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
"cache_creation_input_token_cost_above_1hr",
|
||||
"input_cost_per_second",
|
||||
];
|
||||
|
||||
|
|
@ -393,6 +394,25 @@ const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({
|
|||
/>
|
||||
)}
|
||||
</MountedFormField>
|
||||
<MountedFormField
|
||||
name="cache_creation_input_token_cost_above_1hr"
|
||||
label={labelWithHint(
|
||||
"Cache Write Cost (1 hour, per 1M tokens)",
|
||||
"Rate for prompt cache writes with a 1 hour TTL (Anthropic's cache_control ttl 1h). If left blank, known models use their cost map 1 hour rate, otherwise Cache Write Cost.",
|
||||
)}
|
||||
rules={usageCostRules}
|
||||
className="mb-4"
|
||||
>
|
||||
{(control) => (
|
||||
<Input
|
||||
id={control.id}
|
||||
value={(control.value as string | undefined) ?? ""}
|
||||
onChange={control.onChange}
|
||||
onBlur={control.onBlur}
|
||||
placeholder="Defaults to the cost map 1 hour rate, else Cache Write Cost"
|
||||
/>
|
||||
)}
|
||||
</MountedFormField>
|
||||
</>
|
||||
) : (
|
||||
<MountedFormField
|
||||
|
|
|
|||
|
|
@ -35,6 +35,61 @@ describe("prepareModelAddRequest", () => {
|
|||
expect(deployment.modelInfoObj.team_id).toBe("team-123");
|
||||
});
|
||||
|
||||
it("converts both cache write tiers from per-million to per-token", async () => {
|
||||
const formValues = {
|
||||
model_mappings: [{ public_name: "claude-sonnet", litellm_model: "anthropic/claude-sonnet-4-6" }],
|
||||
model_name: "anthropic/claude-sonnet-4-6",
|
||||
custom_pricing: true,
|
||||
input_cost_per_token: "3",
|
||||
output_cost_per_token: "15",
|
||||
cache_creation_input_token_cost: "3.75",
|
||||
cache_creation_input_token_cost_above_1hr: "6",
|
||||
};
|
||||
|
||||
const [deployment] = (await prepareModelAddRequest({ ...formValues }, "token", null))!;
|
||||
|
||||
expect(deployment.litellmParamsObj.cache_creation_input_token_cost).toBeCloseTo(0.00000375, 12);
|
||||
expect(deployment.litellmParamsObj.cache_creation_input_token_cost_above_1hr).toBeCloseTo(0.000006, 12);
|
||||
});
|
||||
|
||||
it("converts the pricing once when several models are added together", async () => {
|
||||
const formValues = {
|
||||
model_mappings: [
|
||||
{ public_name: "claude-sonnet", litellm_model: "anthropic/claude-sonnet-4-6" },
|
||||
{ public_name: "claude-opus", litellm_model: "anthropic/claude-opus-4-2" },
|
||||
],
|
||||
model_name: "anthropic/claude-sonnet-4-6",
|
||||
custom_pricing: true,
|
||||
input_cost_per_token: "3",
|
||||
cache_creation_input_token_cost: "3.75",
|
||||
cache_creation_input_token_cost_above_1hr: "6",
|
||||
};
|
||||
|
||||
const deployments = (await prepareModelAddRequest({ ...formValues }, "token", null))!;
|
||||
|
||||
expect(deployments).toHaveLength(2);
|
||||
for (const deployment of deployments) {
|
||||
expect(deployment.litellmParamsObj.input_cost_per_token).toBeCloseTo(0.000003, 12);
|
||||
expect(deployment.litellmParamsObj.cache_creation_input_token_cost).toBeCloseTo(0.00000375, 12);
|
||||
expect(deployment.litellmParamsObj.cache_creation_input_token_cost_above_1hr).toBeCloseTo(0.000006, 12);
|
||||
}
|
||||
});
|
||||
|
||||
it("leaves the 1 hour cache write tier unset when its field is blank", async () => {
|
||||
const formValues = {
|
||||
model_mappings: [{ public_name: "claude-sonnet", litellm_model: "anthropic/claude-sonnet-4-6" }],
|
||||
model_name: "anthropic/claude-sonnet-4-6",
|
||||
custom_pricing: true,
|
||||
cache_creation_input_token_cost: "3.75",
|
||||
cache_creation_input_token_cost_above_1hr: "",
|
||||
};
|
||||
|
||||
const [deployment] = (await prepareModelAddRequest({ ...formValues }, "token", null))!;
|
||||
|
||||
expect(deployment.litellmParamsObj).not.toHaveProperty("cache_creation_input_token_cost_above_1hr");
|
||||
expect(deployment.litellmParamsObj.cache_creation_input_token_cost).toBeCloseTo(0.00000375, 12);
|
||||
});
|
||||
|
||||
it("uses a lowercase fallback for unrecognized custom providers", async () => {
|
||||
const fallbackValues = {
|
||||
model_mappings: [
|
||||
|
|
|
|||
|
|
@ -24,6 +24,65 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
|
|||
formValues["model"] = wildcardModel;
|
||||
}
|
||||
|
||||
// Handle pricing conversion before processing other fields
|
||||
// Use explicit checks to allow 0 (zero cost models for budget bypass)
|
||||
if (
|
||||
formValues.input_cost_per_token !== undefined &&
|
||||
formValues.input_cost_per_token !== null &&
|
||||
formValues.input_cost_per_token !== ""
|
||||
) {
|
||||
formValues.input_cost_per_token = Number(formValues.input_cost_per_token) / 1000000;
|
||||
}
|
||||
if (
|
||||
formValues.output_cost_per_token !== undefined &&
|
||||
formValues.output_cost_per_token !== null &&
|
||||
formValues.output_cost_per_token !== ""
|
||||
) {
|
||||
formValues.output_cost_per_token = Number(formValues.output_cost_per_token) / 1000000;
|
||||
}
|
||||
|
||||
// Cache Read Cost: if blank, default to Input Cost (already token-unit converted above)
|
||||
if (
|
||||
formValues.cache_read_input_token_cost !== undefined &&
|
||||
formValues.cache_read_input_token_cost !== null &&
|
||||
formValues.cache_read_input_token_cost !== ""
|
||||
) {
|
||||
formValues.cache_read_input_token_cost = Number(formValues.cache_read_input_token_cost) / 1000000;
|
||||
} else if (
|
||||
formValues.input_cost_per_token !== undefined &&
|
||||
formValues.input_cost_per_token !== null &&
|
||||
formValues.input_cost_per_token !== ""
|
||||
) {
|
||||
formValues.cache_read_input_token_cost = Number(formValues.input_cost_per_token);
|
||||
} else {
|
||||
delete formValues.cache_read_input_token_cost;
|
||||
}
|
||||
|
||||
// Cache Write Cost: explicit value if provided, else leave unset so the
|
||||
// backend keeps the model-level default (per-second pricing, model_prices
|
||||
// entries, etc.). Sending 0 here would overwrite that default.
|
||||
// The backend falls back to input_cost_per_token when this key is absent.
|
||||
if (
|
||||
formValues.cache_creation_input_token_cost !== undefined &&
|
||||
formValues.cache_creation_input_token_cost !== null &&
|
||||
formValues.cache_creation_input_token_cost !== ""
|
||||
) {
|
||||
formValues.cache_creation_input_token_cost = Number(formValues.cache_creation_input_token_cost) / 1000000;
|
||||
} else {
|
||||
delete formValues.cache_creation_input_token_cost;
|
||||
}
|
||||
if (
|
||||
formValues.cache_creation_input_token_cost_above_1hr !== undefined &&
|
||||
formValues.cache_creation_input_token_cost_above_1hr !== null &&
|
||||
formValues.cache_creation_input_token_cost_above_1hr !== ""
|
||||
) {
|
||||
formValues.cache_creation_input_token_cost_above_1hr =
|
||||
Number(formValues.cache_creation_input_token_cost_above_1hr) / 1000000;
|
||||
} else {
|
||||
delete formValues.cache_creation_input_token_cost_above_1hr;
|
||||
}
|
||||
// Keep input_cost_per_second as is, no conversion needed
|
||||
|
||||
// Create a deployment for each mapping
|
||||
const deployments = [];
|
||||
for (const mapping of modelMappings) {
|
||||
|
|
@ -34,55 +93,6 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
|
|||
const modelName = mapping.public_name;
|
||||
litellmParamsObj["model"] = mapping.litellm_model;
|
||||
|
||||
// Handle pricing conversion before processing other fields
|
||||
// Use explicit checks to allow 0 (zero cost models for budget bypass)
|
||||
if (
|
||||
formValues.input_cost_per_token !== undefined &&
|
||||
formValues.input_cost_per_token !== null &&
|
||||
formValues.input_cost_per_token !== ""
|
||||
) {
|
||||
formValues.input_cost_per_token = Number(formValues.input_cost_per_token) / 1000000;
|
||||
}
|
||||
if (
|
||||
formValues.output_cost_per_token !== undefined &&
|
||||
formValues.output_cost_per_token !== null &&
|
||||
formValues.output_cost_per_token !== ""
|
||||
) {
|
||||
formValues.output_cost_per_token = Number(formValues.output_cost_per_token) / 1000000;
|
||||
}
|
||||
|
||||
// Cache Read Cost: if blank, default to Input Cost (already token-unit converted above)
|
||||
if (
|
||||
formValues.cache_read_input_token_cost !== undefined &&
|
||||
formValues.cache_read_input_token_cost !== null &&
|
||||
formValues.cache_read_input_token_cost !== ""
|
||||
) {
|
||||
formValues.cache_read_input_token_cost = Number(formValues.cache_read_input_token_cost) / 1000000;
|
||||
} else if (
|
||||
formValues.input_cost_per_token !== undefined &&
|
||||
formValues.input_cost_per_token !== null &&
|
||||
formValues.input_cost_per_token !== ""
|
||||
) {
|
||||
formValues.cache_read_input_token_cost = Number(formValues.input_cost_per_token);
|
||||
} else {
|
||||
delete formValues.cache_read_input_token_cost;
|
||||
}
|
||||
|
||||
// Cache Write Cost: explicit value if provided, else leave unset so the
|
||||
// backend keeps the model-level default (per-second pricing, model_prices
|
||||
// entries, etc.). Sending 0 here would overwrite that default.
|
||||
// The backend falls back to input_cost_per_token when this key is absent.
|
||||
if (
|
||||
formValues.cache_creation_input_token_cost !== undefined &&
|
||||
formValues.cache_creation_input_token_cost !== null &&
|
||||
formValues.cache_creation_input_token_cost !== ""
|
||||
) {
|
||||
formValues.cache_creation_input_token_cost = Number(formValues.cache_creation_input_token_cost) / 1000000;
|
||||
} else {
|
||||
delete formValues.cache_creation_input_token_cost;
|
||||
}
|
||||
// Keep input_cost_per_second as is, no conversion needed
|
||||
|
||||
// Iterate through the key-value pairs in formValues
|
||||
litellmParamsObj["model"] = mapping.litellm_model;
|
||||
for (const [key, value] of Object.entries(formValues)) {
|
||||
|
|
@ -155,7 +165,8 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
|
|||
key === "output_cost_per_token" ||
|
||||
key === "input_cost_per_second" ||
|
||||
key === "cache_read_input_token_cost" ||
|
||||
key === "cache_creation_input_token_cost"
|
||||
key === "cache_creation_input_token_cost" ||
|
||||
key === "cache_creation_input_token_cost_above_1hr"
|
||||
) {
|
||||
if (value !== undefined && value !== null && value !== "") {
|
||||
litellmParamsObj[key] = Number(value);
|
||||
|
|
|
|||
|
|
@ -1695,13 +1695,54 @@ describe("ModelInfoView", () => {
|
|||
const [cacheRead, cacheWrite] = screen.getAllByPlaceholderText("Defaults to Input Cost if blank");
|
||||
await user.type(cacheRead, "5");
|
||||
await user.type(cacheWrite, "9");
|
||||
await user.type(screen.getByPlaceholderText("Defaults to the cost map 1 hour rate, else Cache Write Cost"), "15");
|
||||
|
||||
const payload = await save(user);
|
||||
|
||||
expect(payload.litellm_params).toMatchObject({
|
||||
const expectedPricing = {
|
||||
output_cost_per_token: 0.000012,
|
||||
cache_read_input_token_cost: 0.000005,
|
||||
cache_creation_input_token_cost: 0.000009,
|
||||
cache_creation_input_token_cost_above_1hr: 0.000015,
|
||||
};
|
||||
expect(payload.litellm_params).toMatchObject(expectedPricing);
|
||||
});
|
||||
|
||||
describe("1 hour cache write tier", () => {
|
||||
const withCacheWriteTiers = {
|
||||
...defaultModelData,
|
||||
litellm_params: {
|
||||
...defaultModelData.litellm_params,
|
||||
cache_creation_input_token_cost: 0.00000375,
|
||||
cache_creation_input_token_cost_above_1hr: 0.000006,
|
||||
},
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheWriteTiers] }, isLoading: false, error: null });
|
||||
mockModelInfoV1Call.mockResolvedValue({ data: [withCacheWriteTiers] });
|
||||
});
|
||||
|
||||
it("shows both configured cache write rates per million tokens", async () => {
|
||||
render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
|
||||
|
||||
expect(await screen.findByText("Cache Write Cost (1 hour, per 1M tokens)")).toBeInTheDocument();
|
||||
expect(screen.getByText("3.7500")).toBeInTheDocument();
|
||||
expect(screen.getByText("6.0000")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("clearing the 1 hour rate sends an explicit null and keeps the 5 minute rate", async () => {
|
||||
const user = userEvent.setup();
|
||||
await enterEditMode(user);
|
||||
|
||||
const oneHour = screen.getByPlaceholderText("Defaults to the cost map 1 hour rate, else Cache Write Cost");
|
||||
expect(oneHour).toHaveValue(6);
|
||||
await user.clear(oneHour);
|
||||
|
||||
const payload = await save(user);
|
||||
|
||||
expect(payload.litellm_params.cache_creation_input_token_cost_above_1hr).toBeNull();
|
||||
expect(payload.litellm_params.cache_creation_input_token_cost).toBe(0.00000375);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ import { Tag } from "./tag_management/types";
|
|||
import { getDisplayModelName } from "./view_model/model_name_display";
|
||||
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
|
||||
|
||||
const perTokenOrNull = (perMillion: string | number | null | undefined): number | null =>
|
||||
perMillion === undefined || perMillion === null || perMillion === "" ? null : Number(perMillion) / 1_000_000;
|
||||
|
||||
interface ModelInfoViewProps {
|
||||
modelId: string;
|
||||
onClose: () => void;
|
||||
|
|
@ -327,19 +330,13 @@ export default function ModelInfoView({
|
|||
}
|
||||
}
|
||||
|
||||
// Cache Write Cost: explicit value if provided, else explicit null so the
|
||||
// backend removes the override and falls back to the model-level default.
|
||||
// Sending 0 here would persist a zero rate even when the user intended to unset it.
|
||||
// Cache write tiers: explicit value if provided, else explicit null so the backend removes the
|
||||
// override. Sending 0 would persist a zero rate even when the user intended to unset it.
|
||||
if (isFieldTouched("cache_write_cost")) {
|
||||
if (
|
||||
values.cache_write_cost !== undefined &&
|
||||
values.cache_write_cost !== null &&
|
||||
values.cache_write_cost !== ""
|
||||
) {
|
||||
updatedLitellmParams.cache_creation_input_token_cost = Number(values.cache_write_cost) / 1_000_000;
|
||||
} else {
|
||||
updatedLitellmParams.cache_creation_input_token_cost = null;
|
||||
}
|
||||
updatedLitellmParams.cache_creation_input_token_cost = perTokenOrNull(values.cache_write_cost);
|
||||
}
|
||||
if (isFieldTouched("cache_write_cost_1h")) {
|
||||
updatedLitellmParams.cache_creation_input_token_cost_above_1hr = perTokenOrNull(values.cache_write_cost_1h);
|
||||
}
|
||||
|
||||
const storedCredentialName: string | null = localModelData?.litellm_params?.litellm_credential_name ?? null;
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -49929,6 +49929,8 @@ export interface components {
|
|||
blocked?: boolean | null;
|
||||
/** Cache Creation Input Token Cost */
|
||||
cache_creation_input_token_cost?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 1Hr */
|
||||
cache_creation_input_token_cost_above_1hr?: number | null;
|
||||
/** Cache Read Input Token Cost */
|
||||
cache_read_input_token_cost?: number | null;
|
||||
/** Cost Per Ptu Per Hour */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue