mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
fix(cost): inherit the backend's full price structure for off-peak-only deployments
Copying only the flat token rates dropped threshold, tiered, service-tier, cache, character, and per-second rates from peak-hour billing once cost lookup switched to the deployment entry, and get_model_info synthesizes zero flat rates for backends without one, which would have marked tiered-only backends explicitly priced free. Copy every price-bearing field instead, deep-copied, rejecting the synthesized zeros the way _inherit_builtin_tiered_output_rate already does.
This commit is contained in:
parent
4875872fe5
commit
0ec3e936b7
2 changed files with 80 additions and 9 deletions
|
|
@ -8169,16 +8169,23 @@ class Router:
|
|||
backend_model: str,
|
||||
custom_llm_provider: str | None,
|
||||
) -> None:
|
||||
"""Fill missing base token rates on a deployment entry that only sets
|
||||
"""Fill missing pricing fields on a deployment entry that only sets
|
||||
``off_peak_pricing``, from the backend model's built-in cost map entry.
|
||||
|
||||
Cost lookup selects the deployment-scoped entry over the shared backend
|
||||
entry only when the deployment entry carries a base pricing field, and
|
||||
``off_peak_pricing`` is deliberately kept off the shared entry, so a
|
||||
deployment spelling out only its off-peak schedule would otherwise
|
||||
never receive the discount. User-specified rates always win; no-op when
|
||||
any base pricing field is already set or the backend model has no
|
||||
canonical entry.
|
||||
never receive the discount. Every price-bearing backend field is
|
||||
copied, not just the flat token rates: threshold, tiered, service-tier,
|
||||
cache, character, and per-second rates all carry over, so peak-hour
|
||||
billing through the deployment entry matches the shared backend entry
|
||||
exactly. Values are deep-copied to keep the builtin entry isolated, and
|
||||
a flat token rate ``get_model_info`` synthesized as zero for a backend
|
||||
without one is rejected, like ``_inherit_builtin_tiered_output_rate``
|
||||
does, so a tiered-only backend is never marked explicitly priced free.
|
||||
User-specified rates always win; no-op when any base pricing field is
|
||||
already set or the backend model has no canonical entry.
|
||||
"""
|
||||
if not model_info.get("off_peak_pricing"):
|
||||
return
|
||||
|
|
@ -8191,11 +8198,14 @@ class Router:
|
|||
backend_info: Final = litellm.get_model_info(model=backend_model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception: # noqa: BLE001 # get_model_info raises plain Exception for an unmapped backend model
|
||||
return
|
||||
for field in ("input_cost_per_token", "output_cost_per_token"):
|
||||
if model_info.get(field) is None:
|
||||
backend_value = backend_info.get(field)
|
||||
if backend_value is not None:
|
||||
model_info[field] = backend_value
|
||||
for field, backend_value in backend_info.items():
|
||||
if "cost" not in field and field != "tiered_pricing":
|
||||
continue
|
||||
if model_info.get(field) is not None or backend_value is None:
|
||||
continue
|
||||
if field in ("input_cost_per_token", "output_cost_per_token") and not backend_value:
|
||||
continue
|
||||
model_info[field] = copy.deepcopy(backend_value)
|
||||
|
||||
@staticmethod
|
||||
def _inherit_builtin_tiered_output_rate(
|
||||
|
|
|
|||
|
|
@ -564,6 +564,67 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
|
|||
assert model_info["off_peak_pricing"] == off_peak_block
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
|
||||
"""A backend with above-threshold pricing hands the whole rate structure to
|
||||
the deployment entry, so peak-hour billing of large prompts through that
|
||||
entry matches the shared backend entry instead of flattening to the base
|
||||
rate.
|
||||
"""
|
||||
backend_model = "gemini/gemini-2.5-pro"
|
||||
builtin_info = litellm.get_model_info(model=backend_model)
|
||||
assert builtin_info["input_cost_per_token_above_200k_tokens"] is not None
|
||||
|
||||
model_info = {
|
||||
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
|
||||
}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider="gemini",
|
||||
)
|
||||
|
||||
assert model_info["input_cost_per_token"] == builtin_info["input_cost_per_token"]
|
||||
assert (
|
||||
model_info["input_cost_per_token_above_200k_tokens"]
|
||||
== builtin_info["input_cost_per_token_above_200k_tokens"]
|
||||
)
|
||||
assert (
|
||||
model_info["output_cost_per_token_above_200k_tokens"]
|
||||
== builtin_info["output_cost_per_token_above_200k_tokens"]
|
||||
)
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_zero():
|
||||
"""A tiered-only backend has no flat token rates; get_model_info synthesizes
|
||||
zeros for them, and storing those would mark the deployment explicitly
|
||||
priced free. The tier table itself must carry over as an isolated copy so
|
||||
mutating the deployment entry never touches the shared cost map.
|
||||
"""
|
||||
backend_model = "dashscope/qwen-flash"
|
||||
raw_tiers = litellm.model_cost[backend_model]["tiered_pricing"]
|
||||
|
||||
model_info = {
|
||||
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
|
||||
}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider="dashscope",
|
||||
)
|
||||
|
||||
assert model_info.get("input_cost_per_token") != 0
|
||||
assert model_info.get("output_cost_per_token") != 0
|
||||
assert model_info["tiered_pricing"] == raw_tiers
|
||||
assert model_info["tiered_pricing"] is not raw_tiers
|
||||
assert model_info["tiered_pricing"][0] is not raw_tiers[0]
|
||||
|
||||
original_first_tier = copy.deepcopy(raw_tiers[0])
|
||||
model_info["tiered_pricing"][0]["input_cost_per_token"] = 123.0
|
||||
assert raw_tiers[0] == original_first_tier
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
|
||||
"""An entry that sets its own base rate beside the block already counts as
|
||||
a full custom pricing entry; the helper must not mix builtin rates into it.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue