Merge b98b893bdd into f370235bb4

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -168,7 +168,9 @@ if TYPE_CHECKING:
     from prisma import types as prisma_types
 
 router: Final = APIRouter()
-CLEARABLE_LITELLM_PARAMS: Final = frozenset({"cache_control_injection_points", "litellm_credential_name"})
+CLEARABLE_LITELLM_PARAMS: Final = frozenset(
+    {"cache_control_injection_points", "litellm_credential_name", "weight", "order"}
+)
 NULL_CLEARABLE_LITELLM_PARAMS: Final = frozenset((*SPECIAL_MODEL_INFO_PARAMS, *CLEARABLE_LITELLM_PARAMS))
 
 

tests/unit/proxy/management_endpoints/test_model_management_endpoints.py

@@ -4621,6 +4621,44 @@ class TestUpdateDBModelClearCredentialName:
         assert params["tpm"] == 100
 
 
+class TestUpdateDBModelClearWeightAndOrder:
+    def test_explicit_null_removes_stored_weight_and_order(self):
+        from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
+
+        db_model: Final = Deployment(
+            model_name="gpt-4",
+            litellm_params=LiteLLM_Params(model="openai/gpt-4o", weight=0.7, order=1),
+            model_info=ModelInfo(id="dep-weight-1"),
+        )
+        update_patch: Final = updateDeployment(litellm_params=updateLiteLLMParams(weight=None, order=None))
+
+        with patch("litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper", side_effect=lambda value, **kwargs: value):
+            result: Final = update_db_model(db_model=db_model, updated_patch=update_patch)
+
+        params: Final = json.loads(result["litellm_params"])
+        assert "weight" not in params
+        assert "order" not in params
+        assert params["model"] == "openai/gpt-4o"
+
+    def test_omitted_weight_and_order_keep_stored_values(self):
+        from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
+
+        db_model: Final = Deployment(
+            model_name="gpt-4",
+            litellm_params=LiteLLM_Params(model="openai/gpt-4o", weight=0.7, order=1),
+            model_info=ModelInfo(id="dep-weight-1"),
+        )
+        update_patch: Final = updateDeployment(litellm_params=updateLiteLLMParams(tpm=10))
+
+        with patch("litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper", side_effect=lambda value, **kwargs: value):
+            result: Final = update_db_model(db_model=db_model, updated_patch=update_patch)
+
+        params: Final = json.loads(result["litellm_params"])
+        assert params["weight"] == 0.7
+        assert params["order"] == 1
+        assert params["tpm"] == 10
+
+
 class TestPatchModelCredentialName:
     @staticmethod
     async def _patch_model(

ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx

@@ -103,6 +103,8 @@ const modelEditShape = {
   max_retries: scalar,
   timeout: scalar,
   stream_timeout: scalar,
+  weight: scalar,
+  order: scalar,
   input_cost: perMillionToPerToken,
   output_cost: perMillionToPerToken,
   cache_read_cost: perMillionToPerToken,
@@ -206,6 +208,8 @@ export const toModelEditFormValues = (localModelData: any, isWildcardModel: bool
   max_retries: localModelData.litellm_params.max_retries,
   timeout: localModelData.litellm_params.timeout,
   stream_timeout: localModelData.litellm_params.stream_timeout,
+  weight: localModelData.litellm_params.weight,
+  order: localModelData.litellm_params.order,
   input_cost: perMillionTokens(
     localModelData.litellm_params.input_cost_per_token,
     localModelData.model_info?.input_cost_per_token,
@@ -248,7 +252,8 @@ export const toModelEditFormValues = (localModelData: any, isWildcardModel: bool
   litellm_extra_params: JSON.stringify(
     Object.fromEntries(
       Object.entries(localModelData.litellm_params || {}).filter(
-        ([key, value]) => key !== "litellm_credential_name" && !isMaskedSecret(value),
+        ([key, value]) =>
+          key !== "litellm_credential_name" && key !== "weight" && key !== "order" && !isMaskedSecret(value),
       ),
     ),
     null,
@@ -545,6 +550,8 @@ const ModelInfoEditForm: React.FC<ModelInfoEditFormProps> = ({
               "Enter stream timeout",
               localModelData.litellm_params?.stream_timeout,
             )}
+            {numberField("weight", "Weight", "e.g. 0.7", localModelData.litellm_params?.weight)}
+            {numberField("order", "Order", "e.g. 1", localModelData.litellm_params?.order)}
 
             <div>
               <FieldLabel>Model Access Groups</FieldLabel>

ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx

@@ -484,6 +484,48 @@ const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({
                 )}
               </MountedFormField>
             )}
+            <MountedFormField
+              name="weight"
+              label={labelWithHint(
+                "Weight",
+                "Load balancing share for this deployment when several deployments share the same model name. Only used by the simple-shuffle routing strategy.",
+              )}
+              className="mb-4"
+            >
+              {(control) => (
+                <Input
+                  id={control.id}
+                  type="number"
+                  step="any"
+                  min={0}
+                  value={(control.value as string | undefined) ?? ""}
+                  onChange={control.onChange}
+                  onBlur={control.onBlur}
+                  placeholder="e.g. 0.7"
+                />
+              )}
+            </MountedFormField>
+            <MountedFormField
+              name="order"
+              label={labelWithHint(
+                "Order",
+                "Failover priority for this deployment. Lower orders are tried first, higher orders only receive traffic when the lower ones are failing.",
+              )}
+              className="mb-4"
+            >
+              {(control) => (
+                <Input
+                  id={control.id}
+                  type="number"
+                  step={1}
+                  min={0}
+                  value={(control.value as string | undefined) ?? ""}
+                  onChange={control.onChange}
+                  onBlur={control.onBlur}
+                  placeholder="e.g. 0 for primary, 1 for fallback"
+                />
+              )}
+            </MountedFormField>
             <MountedFormField
               name="litellm_extra_params"
               label={labelWithHint(

ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx

@@ -215,4 +215,52 @@ describe("prepareModelAddRequest", () => {
     expect(deployment.modelName).toBe(wildcardModel);
     expect(deployment.litellmParamsObj.model).toBe(wildcardModel);
   });
+
+  it("sends weight and order as numbers in litellm_params", async () => {
+    const formValues = {
+      model_mappings: [{ public_name: "Public Model", litellm_model: "litellm/public" }],
+      model_name: "custom-model-name",
+      weight: "0.7",
+      order: "1",
+    };
+
+    const deployments = await prepareModelAddRequest({ ...formValues }, "token", null);
+
+    expect(deployments).toHaveLength(1);
+    const [deployment] = deployments!;
+    expect(deployment.litellmParamsObj.weight).toBe(0.7);
+    expect(deployment.litellmParamsObj.order).toBe(1);
+  });
+
+  it("omits weight and order from litellm_params when left blank", async () => {
+    const formValues = {
+      model_mappings: [{ public_name: "Public Model", litellm_model: "litellm/public" }],
+      model_name: "custom-model-name",
+      weight: "",
+      order: "",
+    };
+
+    const deployments = await prepareModelAddRequest({ ...formValues }, "token", null);
+
+    expect(deployments).toHaveLength(1);
+    const [deployment] = deployments!;
+    expect(deployment.litellmParamsObj).not.toHaveProperty("weight");
+    expect(deployment.litellmParamsObj).not.toHaveProperty("order");
+  });
+
+  it("omits weight and order when the mounted fields registered as undefined", async () => {
+    const formValues = {
+      model_mappings: [{ public_name: "Public Model", litellm_model: "litellm/public" }],
+      model_name: "custom-model-name",
+      weight: undefined,
+      order: undefined,
+    };
+
+    const deployments = await prepareModelAddRequest({ ...formValues }, "token", null);
+
+    expect(deployments).toHaveLength(1);
+    const [deployment] = deployments!;
+    expect(deployment.litellmParamsObj).not.toHaveProperty("weight");
+    expect(deployment.litellmParamsObj).not.toHaveProperty("order");
+  });
 });

ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx

@@ -157,6 +157,11 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
               modelInfoObj[key] = value;
             }
           }
+        } else if (key === "weight" || key === "order") {
+          if (value !== undefined && value !== null) {
+            litellmParamsObj[key] = Number(value);
+          }
+          continue;
         }
 
         // Handle the pricing fields

ui/litellm-dashboard/src/components/model_info_view.test.tsx

@@ -1686,6 +1686,73 @@ describe("ModelInfoView", () => {
       });
     });
 
+    const modelWithStoredRouting = () => ({
+      ...defaultModelData,
+      litellm_params: {
+        ...defaultModelData.litellm_params,
+        weight: 0.7,
+        order: 1,
+      },
+    });
+
+    const renderStoredRoutingModel = () => {
+      const stored = modelWithStoredRouting();
+      mockUseModelsInfo.mockReturnValue({
+        data: { data: [stored] },
+        isLoading: false,
+        error: null,
+      });
+      mockModelInfoV1Call.mockResolvedValue({ data: [stored] });
+    };
+
+    it("routes edited weight and order into the payload as numbers", async () => {
+      const user = userEvent.setup();
+      await enterEditMode(user);
+
+      await user.type(screen.getByPlaceholderText("e.g. 0.7"), "0.9");
+      await user.type(screen.getByPlaceholderText("e.g. 1"), "2");
+
+      const payload = await save(user);
+
+      expect(payload.litellm_params).toMatchObject({ weight: 0.9, order: 2 });
+    });
+
+    it("excludes weight and order from the LiteLLM Params JSON so the dedicated fields own them", async () => {
+      renderStoredRoutingModel();
+      const user = userEvent.setup();
+      await enterEditMode(user);
+
+      const paramsJson = (screen.getByPlaceholderText(/"rpm": 100/) as HTMLTextAreaElement).value;
+
+      expect(paramsJson).not.toContain("weight");
+      expect(paramsJson).not.toContain("order");
+      expect(screen.getByPlaceholderText("e.g. 0.7")).toHaveValue(0.7);
+    });
+
+    it("keeps a stored weight out of the payload on an untouched save", async () => {
+      renderStoredRoutingModel();
+      const user = userEvent.setup();
+      await enterEditMode(user);
+
+      const payload = await save(user);
+
+      expect(payload.litellm_params).not.toHaveProperty("weight");
+      expect(payload.litellm_params).not.toHaveProperty("order");
+    });
+
+    it("clears a stored weight by sending null when the field is emptied", async () => {
+      renderStoredRoutingModel();
+      const user = userEvent.setup();
+      await enterEditMode(user);
+
+      await user.clear(screen.getByPlaceholderText("e.g. 0.7"));
+
+      const payload = await save(user);
+
+      expect(payload.litellm_params.weight).toBeNull();
+      expect(payload.litellm_params).not.toHaveProperty("order");
+    });
+
     it("routes each edited pricing field into its own payload key", async () => {
       const user = userEvent.setup();
       await enterEditMode(user);

ui/litellm-dashboard/src/components/model_info_view.tsx

@@ -62,6 +62,9 @@ interface ModelInfoViewProps {
   modelAccessGroups: string[] | null;
 }
 
+const numericRoutingParam = (value: string | number | null | undefined): number | null =>
+  value === undefined || value === null || value === "" ? null : Number(value);
+
 export default function ModelInfoView({
   modelId,
   onClose,
@@ -294,6 +297,18 @@ export default function ModelInfoView({
         tags: values.tags,
       };
 
+      // Weight/Order are only sent when actually changed, so an untouched save
+      // never clears a stored value. A changed-but-empty value sends null, which
+      // the backend treats as an explicit clear.
+      const storedWeight: string | number | null = localModelData?.litellm_params?.weight ?? null;
+      const storedOrder: string | number | null = localModelData?.litellm_params?.order ?? null;
+      if (numericRoutingParam(values.weight) !== numericRoutingParam(storedWeight)) {
+        updatedLitellmParams.weight = numericRoutingParam(values.weight);
+      }
+      if (numericRoutingParam(values.order) !== numericRoutingParam(storedOrder)) {
+        updatedLitellmParams.order = numericRoutingParam(values.order);
+      }
+
       if (isFieldTouched("input_cost")) {
         updatedLitellmParams.input_cost_per_token = values.input_cost;
       }