mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(model_management): stop persisting model cost map pricing as a deployment override
/model/info fills model_info pricing in from the model cost map when a deployment has no override, and the Admin UI echoed that blob back on every save, so editing any unrelated setting froze the deployment at that day's price and Reload Price Data could no longer move it. Treat model_info pricing as a mirror of litellm_params, which is the only source of truth for a deployment override. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
milan
parent
0e9a624a97
commit
e62a8eed0e
4 changed files with 249 additions and 21 deletions
|
|
@ -169,25 +169,30 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr
|
|||
merged_deployment_dict["model_info"] = {}
|
||||
merged_deployment_dict["model_info"].update(updated_patch.model_info.model_dump(exclude_none=True))
|
||||
|
||||
# Honor explicit-null clears LAST, after both merges, so a model_info blob the UI
|
||||
# passes through (which today re-sends the OLD pricing on every save) cannot
|
||||
# silently undo a litellm_params clear via .update().
|
||||
#
|
||||
# Restricted to SPECIAL_MODEL_INFO_PARAMS (input/output cost per token/character
|
||||
# and cache read/write costs) so this path cannot be used to null out privileged
|
||||
# model_info fields like team_id or access groups. SPECIAL_MODEL_INFO_PARAMS are
|
||||
# mirrored between litellm_params and model_info by Deployment.__init__, so the
|
||||
# clear propagates to both blobs.
|
||||
if updated_patch.litellm_params:
|
||||
for field in updated_patch.litellm_params.model_fields_set:
|
||||
if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.litellm_params, field) is None:
|
||||
merged_deployment_dict["litellm_params"].pop(field, None) # type: ignore
|
||||
merged_deployment_dict.get("model_info", {}).pop(field, None)
|
||||
if updated_patch.model_info:
|
||||
for field in updated_patch.model_info.model_fields_set:
|
||||
if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.model_info, field) is None:
|
||||
merged_deployment_dict["model_info"].pop(field, None) # type: ignore
|
||||
merged_deployment_dict.get("litellm_params", {}).pop(field, None) # type: ignore
|
||||
# Resolved LAST, after both merges: litellm_params is the only source of truth for a
|
||||
# deployment's custom pricing and model_info only mirrors it (Deployment.__init__).
|
||||
# /model/info fills model_info pricing in from the model cost map when a deployment has
|
||||
# no override, so clients that round-trip that response (the Admin UI does on every
|
||||
# save) must not turn those prices into an override or resurrect a cleared one.
|
||||
cleared_pricing_fields = frozenset(
|
||||
field
|
||||
for patch in (updated_patch.litellm_params, updated_patch.model_info)
|
||||
if patch is not None
|
||||
for field in patch.model_fields_set
|
||||
if field in SPECIAL_MODEL_INFO_PARAMS and getattr(patch, field) is None
|
||||
)
|
||||
merged_params = merged_deployment_dict["litellm_params"]
|
||||
for field in cleared_pricing_fields:
|
||||
merged_params.pop(field, None) # pyright: ignore[reportUnknownMemberType] # dynamic key on a TypedDict
|
||||
if "model_info" in merged_deployment_dict:
|
||||
for field in SPECIAL_MODEL_INFO_PARAMS:
|
||||
override = merged_params.get(field) # pyright: ignore[reportUnknownMemberType] # dynamic key on a TypedDict
|
||||
if override is None:
|
||||
merged_deployment_dict["model_info"].pop( # pyright: ignore[reportUnknownMemberType] # untyped model_info blob
|
||||
field, None
|
||||
)
|
||||
else:
|
||||
merged_deployment_dict["model_info"][field] = override
|
||||
|
||||
# convert to prisma compatible format
|
||||
|
||||
|
|
|
|||
|
|
@ -3112,6 +3112,129 @@ class TestUpdateDBModelClearPricing:
|
|||
assert info["cache_creation_input_token_cost"] == 0.000003
|
||||
|
||||
|
||||
_ENCRYPT_VALUE_HELPER = "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper"
|
||||
|
||||
|
||||
class TestUpdateDBModelModelInfoPricingIsMirrorOnly:
|
||||
"""`/model/info` fills `model_info` pricing in from the model cost map for
|
||||
deployments that have no override, and the Admin UI sends that blob back on every
|
||||
save. Those model-cost-map prices must not become a deployment override, otherwise
|
||||
"Reload Price Data" can never move the deployment's price again.
|
||||
"""
|
||||
|
||||
def test_model_info_pricing_alone_does_not_create_an_override(self):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
update_db_model,
|
||||
)
|
||||
from litellm.types.router import (
|
||||
Deployment,
|
||||
LiteLLM_Params,
|
||||
ModelInfo,
|
||||
updateLiteLLMParams,
|
||||
)
|
||||
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6-sol",
|
||||
litellm_params=LiteLLM_Params(model="bedrock_mantle/openai.gpt-5.6-sol"),
|
||||
model_info=ModelInfo(id="dep-no-override-0"),
|
||||
)
|
||||
|
||||
with patch(_ENCRYPT_VALUE_HELPER, side_effect=lambda value: value):
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(
|
||||
litellm_params=updateLiteLLMParams(
|
||||
model="bedrock_mantle/openai.gpt-5.6-sol", tags=["prod"]
|
||||
),
|
||||
model_info=ModelInfo(
|
||||
id="dep-no-override-0",
|
||||
input_cost_per_token=0.0000055,
|
||||
output_cost_per_token=0.000033,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
params = json.loads(result["litellm_params"])
|
||||
info = json.loads(result["model_info"])
|
||||
assert params["tags"] == ["prod"]
|
||||
assert "input_cost_per_token" not in params
|
||||
assert "output_cost_per_token" not in params
|
||||
assert "input_cost_per_token" not in info
|
||||
assert "output_cost_per_token" not in info
|
||||
|
||||
def test_cleared_pricing_is_not_resurrected_by_a_later_save(self):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
update_db_model,
|
||||
)
|
||||
from litellm.types.router import (
|
||||
Deployment,
|
||||
LiteLLM_Params,
|
||||
ModelInfo,
|
||||
updateLiteLLMParams,
|
||||
)
|
||||
|
||||
cleared = update_db_model(
|
||||
db_model=_build_db_model_with_pricing(),
|
||||
updated_patch=updateDeployment(
|
||||
litellm_params=updateLiteLLMParams(
|
||||
input_cost_per_token=None, output_cost_per_token=None
|
||||
),
|
||||
model_info=ModelInfo(
|
||||
id="dep-pricing-0",
|
||||
input_cost_per_token=0.000001,
|
||||
output_cost_per_token=0.000002,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
cleared_model = Deployment(
|
||||
model_name="openai/*",
|
||||
litellm_params=LiteLLM_Params(**json.loads(cleared["litellm_params"])),
|
||||
model_info=ModelInfo(**json.loads(cleared["model_info"])),
|
||||
)
|
||||
|
||||
with patch(_ENCRYPT_VALUE_HELPER, side_effect=lambda value: value):
|
||||
result = update_db_model(
|
||||
db_model=cleared_model,
|
||||
updated_patch=updateDeployment(
|
||||
litellm_params=updateLiteLLMParams(model="openai/*", tags=["prod"]),
|
||||
model_info=ModelInfo(
|
||||
id="dep-pricing-0",
|
||||
input_cost_per_token=0.000001,
|
||||
output_cost_per_token=0.000002,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
params = json.loads(result["litellm_params"])
|
||||
info = json.loads(result["model_info"])
|
||||
assert "input_cost_per_token" not in params
|
||||
assert "output_cost_per_token" not in params
|
||||
assert "input_cost_per_token" not in info
|
||||
assert "output_cost_per_token" not in info
|
||||
|
||||
def test_litellm_params_pricing_overwrites_a_stale_model_info_mirror(self):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
update_db_model,
|
||||
)
|
||||
from litellm.types.router import ModelInfo, updateLiteLLMParams
|
||||
|
||||
result = update_db_model(
|
||||
db_model=_build_db_model_with_pricing(),
|
||||
updated_patch=updateDeployment(
|
||||
litellm_params=updateLiteLLMParams(input_cost_per_token=0.000009),
|
||||
model_info=ModelInfo(
|
||||
id="dep-pricing-0", input_cost_per_token=0.000001
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
params = json.loads(result["litellm_params"])
|
||||
info = json.loads(result["model_info"])
|
||||
assert params["input_cost_per_token"] == 0.000009
|
||||
assert info["input_cost_per_token"] == 0.000009
|
||||
|
||||
|
||||
class TestGetModelInfoWithIdBlocked:
|
||||
"""`ProxyConfig.get_model_info_with_id` must propagate the DB-level `blocked`
|
||||
column into the in-memory `model_info` dict so the router filter can read it."""
|
||||
|
|
|
|||
|
|
@ -636,6 +636,84 @@ describe("ModelInfoView", () => {
|
|||
expect(updatePayload.litellm_params).not.toHaveProperty("output_cost_per_token");
|
||||
});
|
||||
|
||||
it("should not send model cost map pricing back in model_info when user does not touch cost fields", async () => {
|
||||
// Regression: /model/info fills model_info pricing in from the model cost map for
|
||||
// deployments that have no override, and echoing it back froze the deployment at
|
||||
// that price, so "Reload Price Data" could never move it again.
|
||||
const user = userEvent.setup();
|
||||
render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /edit settings/i }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("button", { name: /save changes/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockModelPatchUpdateCall).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
const updatePayload = mockModelPatchUpdateCall.mock.calls[0][1];
|
||||
expect(updatePayload.model_info).not.toHaveProperty("input_cost_per_token");
|
||||
expect(updatePayload.model_info).not.toHaveProperty("output_cost_per_token");
|
||||
expect(updatePayload.model_info.id).toBe("123");
|
||||
});
|
||||
|
||||
it("should clear pricing with an explicit null and not re-send the old values in model_info", async () => {
|
||||
const pricedModelData = {
|
||||
...defaultModelData,
|
||||
litellm_params: {
|
||||
...defaultModelData.litellm_params,
|
||||
input_cost_per_token: 0.00009,
|
||||
output_cost_per_token: 0.0009,
|
||||
},
|
||||
model_info: {
|
||||
...defaultModelData.model_info,
|
||||
input_cost_per_token: 0.00009,
|
||||
output_cost_per_token: 0.0009,
|
||||
},
|
||||
};
|
||||
mockUseModelsInfo.mockReturnValue({
|
||||
data: { data: [pricedModelData] },
|
||||
isLoading: false,
|
||||
error: null,
|
||||
});
|
||||
mockModelInfoV1Call.mockResolvedValue({ data: [pricedModelData] });
|
||||
|
||||
const user = userEvent.setup();
|
||||
render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /edit settings/i }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("button", { name: /save changes/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
await user.clear(screen.getByPlaceholderText("Enter input cost"));
|
||||
await user.clear(screen.getByPlaceholderText("Enter output cost"));
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockModelPatchUpdateCall).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
const updatePayload = mockModelPatchUpdateCall.mock.calls[0][1];
|
||||
expect(updatePayload.litellm_params.input_cost_per_token).toBeNull();
|
||||
expect(updatePayload.litellm_params.output_cost_per_token).toBeNull();
|
||||
expect(updatePayload.model_info).not.toHaveProperty("input_cost_per_token");
|
||||
expect(updatePayload.model_info).not.toHaveProperty("output_cost_per_token");
|
||||
});
|
||||
|
||||
it("never re-sends a masked secret on save (regression: masked auth value must not overwrite the real secret)", async () => {
|
||||
// /model/info redacts secrets by masking (e.g. "azur****BBCC"), not removing them.
|
||||
// A plain save re-PATCHes the whole litellm_params blob; if the masked value were
|
||||
|
|
|
|||
|
|
@ -67,6 +67,21 @@ interface ModelInfoViewProps {
|
|||
modelAccessGroups: string[] | null;
|
||||
}
|
||||
|
||||
// Mirrors SPECIAL_MODEL_INFO_PARAMS in litellm/types/router.py. `/model/info` fills these
|
||||
// into model_info from the model cost map when a deployment has no override, so echoing
|
||||
// them back on save would freeze the deployment at whatever the price map said that day.
|
||||
const MIRRORED_PRICING_FIELDS = [
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"input_cost_per_character",
|
||||
"output_cost_per_character",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
];
|
||||
|
||||
const withoutMirroredPricing = (modelInfo: Record<string, unknown>): Record<string, unknown> =>
|
||||
Object.fromEntries(Object.entries(modelInfo).filter(([key]) => !MIRRORED_PRICING_FIELDS.includes(key)));
|
||||
|
||||
interface ComplexityRouterTierConfig {
|
||||
tiers?: {
|
||||
SIMPLE?: unknown;
|
||||
|
|
@ -438,10 +453,17 @@ export default function ModelInfoView({
|
|||
// Credential rotation has its own dedicated path (UpdateModelCredentialsModal).
|
||||
const safeLitellmParams = stripMaskedSecrets(updatedLitellmParams);
|
||||
|
||||
const pricingOverrides = Object.fromEntries(
|
||||
MIRRORED_PRICING_FIELDS.filter((field) => field in safeLitellmParams).map((field) => [
|
||||
field,
|
||||
safeLitellmParams[field],
|
||||
]),
|
||||
);
|
||||
|
||||
const updateData = {
|
||||
model_name: values.model_name,
|
||||
litellm_params: safeLitellmParams,
|
||||
model_info: updatedModelInfo,
|
||||
model_info: withoutMirroredPricing(updatedModelInfo),
|
||||
};
|
||||
|
||||
await modelPatchUpdateCall(accessToken, updateData, modelId);
|
||||
|
|
@ -451,7 +473,7 @@ export default function ModelInfoView({
|
|||
model_name: values.model_name,
|
||||
litellm_model_name: values.litellm_model_name,
|
||||
litellm_params: safeLitellmParams,
|
||||
model_info: updatedModelInfo,
|
||||
model_info: { ...updatedModelInfo, ...pricingOverrides },
|
||||
};
|
||||
|
||||
setLocalModelData(updatedModelData);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue