fix(router): replay off-peak deployments in order since they can copy a base model's mode

litellm/router_python_core.py

@@ -1142,8 +1142,10 @@ class _SharedCostWrite:
         return self.model_id in writers or not writers.isdisjoint(self.backend_keys)
 
     @property
-    def sets_mode(self) -> bool:
-        return isinstance(self.model_info.get("mode"), str)
+    def may_write_mode(self) -> bool:
+        """Whether its write can carry ``mode``: set directly, or copied with the backend entry an off-peak schedule
+        inherits."""
+        return isinstance(self.model_info.get("mode"), str) or bool(self.model_info.get("off_peak_pricing"))
 
     def replay_key(self, targets: frozenset[str]) -> tuple[str | None, str]:
         """Equal for two entries only when replaying both writes the ``targets`` rows exactly as replaying one does.
@@ -12513,10 +12515,10 @@ class PythonCore(DeprecatedRouterMembers):
     ) -> tuple[int, ...]:
         """The position of the last of each set of ``landing`` writes that leave the ``targets`` rows as one of them
         does. Each field keeps the last value written to it, so only the last write of each set counts, unless a write
-        sets ``mode``: what a ``mode`` write does depends on the mode the row holds, so then only back-to-back runs
+        may carry ``mode``: what a ``mode`` write does depends on the mode the row holds, so then only back-to-back runs
         collapse."""
         keyed: Final = tuple((position, write.replay_key(targets)) for position, write in landing)
-        if any(write.sets_mode for _position, write in landing):
+        if any(write.may_write_mode for _position, write in landing):
             return tuple(tuple(run)[-1][0] for _replay_key, run in groupby(keyed, key=itemgetter(1)))
         return tuple(sorted({replay_key: position for position, replay_key in keyed}.values()))
 

tests/unit/test_router_model_cost_isolation.py

@@ -2995,6 +2995,39 @@ def test_deleting_a_deployment_keeps_the_mode_its_siblings_registered_in_model_l
     assert litellm.model_cost[backend_model].get("mode") == registered_mode
 
 
+def _openai_catalog_model_with_mode(mode: str) -> str:
+    return next(
+        key
+        for key, entry in GetModelCostMap.loaded_model_cost_map().items()
+        if (entry.get("mode"), entry.get("litellm_provider"), "/" in key) == (mode, "openai", False)
+    )
+
+
+def test_deleting_a_deployment_keeps_the_mode_off_peak_siblings_inherit_in_model_list_order() -> None:
+    backend_model: Final = "hosted_vllm/withdraw-test-inherited-mode-model"
+    off_peak: Final = {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-7}
+    responses_model: Final = _openai_catalog_model_with_mode("responses")
+    router: Final = Router(
+        model_list=[
+            _cache_point_deployment("doomed", model=backend_model, max_input_tokens=5),
+            *(
+                _cache_point_deployment(model_id, model=backend_model, base_model=base_model, off_peak_pricing=off_peak)
+                for model_id, base_model in (
+                    ("responses-first", responses_model),
+                    ("embedding", _openai_catalog_model_with_mode("embedding")),
+                    ("responses-last", responses_model),
+                )
+            ),
+        ]
+    )
+    registered_mode: Final = litellm.model_cost[backend_model].get("mode")
+    assert registered_mode is not None
+
+    assert router.delete_deployment(id="doomed") is not None
+
+    assert litellm.model_cost[backend_model].get("mode") == registered_mode
+
+
 class _CostMapWriteRecordingHost(PythonHost):
     __slots__ = ("cost_map_writes",)