scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-12 23:01:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

fix(cost): prefer matching tier at equal threshold

Browse source
This commit is contained in:
LH-kevin 2026-08-22 15:54:21 +08:00
parent c04a8b7014
commit 6271e1e7bd
2 changed files with 130 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -347,8 +347,10 @@ def _get_token_base_cost(
base_key = match.group("base_key")
selected = selected_threshold_keys.get(base_key)
selected_specificity = 0 if selected is None or selected[2] is None else 1
candidate_specificity = 0 if threshold_key_tier is None else 1
if selected is None or threshold > selected[0]:
if selected is None or (threshold, candidate_specificity) > (selected[0], selected_specificity):
selected_threshold_keys[base_key] = (
threshold,
key,

127
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py
View file

@ -70,6 +70,133 @@ def test_cost_types_select_thresholds_independently() -> None:
assert rates[1] == pytest.approx(18e-6)
@pytest.mark.parametrize(
"model_info",
[
{
"input_cost_per_token": 1e-6,
"output_cost_per_token": 2e-6,
"output_cost_per_token_above_200k_tokens": 1e-6,
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
},
{
"input_cost_per_token": 1e-6,
"output_cost_per_token": 2e-6,
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
"output_cost_per_token_above_200k_tokens": 1e-6,
},
],
ids=["standard-first", "priority-first"],
)
def test_equal_threshold_matching_tier_wins_for_output(model_info):
"""At an equal threshold the matching service-tier rate must win regardless of the
order the standard and tier-qualified keys appear in model_info."""
usage = Usage(
prompt_tokens=250_000,
completion_tokens=1_000,
total_tokens=251_000,
)
rates = _get_token_base_cost(model_info=model_info, usage=usage, service_tier="priority")
assert rates[1] == pytest.approx(1.5e-6)
@pytest.mark.parametrize(
"model_info",
[
{
"input_cost_per_token": 1e-6,
"cache_read_input_token_cost": 2e-7,
"cache_read_input_token_cost_above_200k_tokens": 3e-7,
"cache_read_input_token_cost_above_200k_tokens_priority": 4e-7,
},
{
"input_cost_per_token": 1e-6,
"cache_read_input_token_cost": 2e-7,
"cache_read_input_token_cost_above_200k_tokens_priority": 4e-7,
"cache_read_input_token_cost_above_200k_tokens": 3e-7,
},
],
ids=["standard-first", "priority-first"],
)
def test_equal_threshold_matching_tier_wins_for_cache_read(model_info):
usage = Usage(
prompt_tokens=250_000,
completion_tokens=1_000,
total_tokens=251_000,
)
rates = _get_token_base_cost(model_info=model_info, usage=usage, service_tier="priority")
assert rates[4] == pytest.approx(4e-7)
def test_equal_threshold_default_tier_keeps_standard():
"""service_tier=None must ignore the tier-qualified key and keep the standard rate."""
model_info = {
"input_cost_per_token": 1e-6,
"output_cost_per_token": 2e-6,
"output_cost_per_token_above_200k_tokens": 1e-6,
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
}
usage = Usage(
prompt_tokens=250_000,
completion_tokens=1_000,
total_tokens=251_000,
)
rates = _get_token_base_cost(model_info=model_info, usage=usage)
assert rates[1] == pytest.approx(1e-6)
def test_equal_threshold_wrong_tier_keeps_standard():
"""A priority-qualified key must not win under a different service tier."""
model_info = {
"input_cost_per_token": 1e-6,
"output_cost_per_token": 2e-6,
"output_cost_per_token_above_200k_tokens": 1e-6,
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
}
usage = Usage(
prompt_tokens=250_000,
completion_tokens=1_000,
total_tokens=251_000,
)
rates = _get_token_base_cost(
model_info=model_info,
usage=usage,
service_tier="flex",
)
assert rates[1] == pytest.approx(1e-6)
def test_unequal_threshold_highest_standard_still_wins():
"""A higher standard threshold governs over a lower matching-tier threshold."""
model_info = {
"input_cost_per_token": 1e-6,
"output_cost_per_token": 2e-6,
"output_cost_per_token_above_200k_tokens": 1e-6,
"output_cost_per_token_above_128k_tokens_priority": 1.5e-6,
}
usage = Usage(
prompt_tokens=250_000,
completion_tokens=1_000,
total_tokens=251_000,
)
rates = _get_token_base_cost(
model_info=model_info,
usage=usage,
service_tier="priority",
)
assert rates[1] == pytest.approx(1e-6)
@pytest.mark.parametrize(
("tier_field", "tier_value", "rate_index"),
[