fix(cost): inherit the backend's full price structure for off-peak-only deployments

Copying only the flat token rates dropped threshold, tiered, service-tier,
cache, character, and per-second rates from peak-hour billing once cost
lookup switched to the deployment entry, and get_model_info synthesizes
zero flat rates for backends without one, which would have marked
tiered-only backends explicitly priced free. Copy every price-bearing
field instead, deep-copied, rejecting the synthesized zeros the way
_inherit_builtin_tiered_output_rate already does.
This commit is contained in:
mateo-berri 2026-09-01 12:47:26 -07:00
parent 4875872fe5
commit 0ec3e936b7
2 changed files with 80 additions and 9 deletions

View file

@ -8169,16 +8169,23 @@ class Router:
backend_model: str,
custom_llm_provider: str | None,
) -> None:
"""Fill missing base token rates on a deployment entry that only sets
"""Fill missing pricing fields on a deployment entry that only sets
``off_peak_pricing``, from the backend model's built-in cost map entry.
Cost lookup selects the deployment-scoped entry over the shared backend
entry only when the deployment entry carries a base pricing field, and
``off_peak_pricing`` is deliberately kept off the shared entry, so a
deployment spelling out only its off-peak schedule would otherwise
never receive the discount. User-specified rates always win; no-op when
any base pricing field is already set or the backend model has no
canonical entry.
never receive the discount. Every price-bearing backend field is
copied, not just the flat token rates: threshold, tiered, service-tier,
cache, character, and per-second rates all carry over, so peak-hour
billing through the deployment entry matches the shared backend entry
exactly. Values are deep-copied to keep the builtin entry isolated, and
a flat token rate ``get_model_info`` synthesized as zero for a backend
without one is rejected, like ``_inherit_builtin_tiered_output_rate``
does, so a tiered-only backend is never marked explicitly priced free.
User-specified rates always win; no-op when any base pricing field is
already set or the backend model has no canonical entry.
"""
if not model_info.get("off_peak_pricing"):
return
@ -8191,11 +8198,14 @@ class Router:
backend_info: Final = litellm.get_model_info(model=backend_model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # get_model_info raises plain Exception for an unmapped backend model
return
for field in ("input_cost_per_token", "output_cost_per_token"):
if model_info.get(field) is None:
backend_value = backend_info.get(field)
if backend_value is not None:
model_info[field] = backend_value
for field, backend_value in backend_info.items():
if "cost" not in field and field != "tiered_pricing":
continue
if model_info.get(field) is not None or backend_value is None:
continue
if field in ("input_cost_per_token", "output_cost_per_token") and not backend_value:
continue
model_info[field] = copy.deepcopy(backend_value)
@staticmethod
def _inherit_builtin_tiered_output_rate(

View file

@ -564,6 +564,67 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
assert model_info["off_peak_pricing"] == off_peak_block
def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
"""A backend with above-threshold pricing hands the whole rate structure to
the deployment entry, so peak-hour billing of large prompts through that
entry matches the shared backend entry instead of flattening to the base
rate.
"""
backend_model = "gemini/gemini-2.5-pro"
builtin_info = litellm.get_model_info(model=backend_model)
assert builtin_info["input_cost_per_token_above_200k_tokens"] is not None
model_info = {
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
}
Router._inherit_builtin_base_rates_for_off_peak(
model_info=model_info,
backend_model=backend_model,
custom_llm_provider="gemini",
)
assert model_info["input_cost_per_token"] == builtin_info["input_cost_per_token"]
assert (
model_info["input_cost_per_token_above_200k_tokens"]
== builtin_info["input_cost_per_token_above_200k_tokens"]
)
assert (
model_info["output_cost_per_token_above_200k_tokens"]
== builtin_info["output_cost_per_token_above_200k_tokens"]
)
def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_zero():
"""A tiered-only backend has no flat token rates; get_model_info synthesizes
zeros for them, and storing those would mark the deployment explicitly
priced free. The tier table itself must carry over as an isolated copy so
mutating the deployment entry never touches the shared cost map.
"""
backend_model = "dashscope/qwen-flash"
raw_tiers = litellm.model_cost[backend_model]["tiered_pricing"]
model_info = {
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
}
Router._inherit_builtin_base_rates_for_off_peak(
model_info=model_info,
backend_model=backend_model,
custom_llm_provider="dashscope",
)
assert model_info.get("input_cost_per_token") != 0
assert model_info.get("output_cost_per_token") != 0
assert model_info["tiered_pricing"] == raw_tiers
assert model_info["tiered_pricing"] is not raw_tiers
assert model_info["tiered_pricing"][0] is not raw_tiers[0]
original_first_tier = copy.deepcopy(raw_tiers[0])
model_info["tiered_pricing"][0]["input_cost_per_token"] = 123.0
assert raw_tiers[0] == original_first_tier
def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
"""An entry that sets its own base rate beside the block already counts as
a full custom pricing entry; the helper must not mix builtin rates into it.