scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

refactor(cost-map): drop the legacy max_tokens field in favor of max_input_tokens and max_output_tokens (#48471)

Browse source 
* refactor(cost-map): drop the legacy max_tokens field in favor of max_input_tokens and max_output_tokens

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test: drop assertions that pinned the retired cost-map max_tokens field

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-10-26 17:03:05 -07:00 • committed by GitHub
parent 7fd8740ad5
commit db1c871c12
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 543 additions and 6718 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3573
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

2
litellm/proxy/_experimental/mcp_server/sampling_handler.py
View file

@ -222,7 +222,7 @@ def _select_model_by_priority(
input_cost = info.get("input_cost_per_token") or 0.0
output_cost = info.get("output_cost_per_token") or 0.0
total_cost = input_cost + output_cost
max_output = info.get("max_output_tokens") or info.get("max_tokens") or 0
max_output = info.get("max_output_tokens") or 0
output_tps = info.get("output_tokens_per_second") or 0.0
scored.append(
_ScoredModel(

1
litellm/router.py
View file

@ -12264,7 +12264,6 @@ class Router:
_fallback_entry = litellm.model_cost.get(_azure_fallback_key)
_fallback_resolves = _fallback_entry is not None and (
(_fallback_entry.get("max_input_tokens") or 0) > 0
or (_fallback_entry.get("max_tokens") or 0) > 0
or (_fallback_entry.get("input_cost_per_token") or 0) > 0
)
if _fallback_resolves:

2
litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py
View file

@ -148,7 +148,7 @@ def _model_max_output_tokens(model_name: str) -> int | None:
# this lookup is a fallback default and must never fail the request.
with contextlib.suppress(Exception):
info: Final = litellm.get_model_info(model=model_name)
model_max: Final = info.get("max_output_tokens") or info.get("max_tokens")
model_max: Final = info.get("max_output_tokens")
if model_max is not None:
return max(0, int(model_max))
return None

15
litellm/utils.py
View file

@ -6079,11 +6079,8 @@ def get_max_tokens(model: str) -> int | None:
return None
try:
if model in litellm.model_cost:
if "max_output_tokens" in litellm.model_cost[model]:
return litellm.model_cost[model]["max_output_tokens"]
elif "max_tokens" in litellm.model_cost[model]:
return litellm.model_cost[model]["max_tokens"]
if model in litellm.model_cost and "max_output_tokens" in litellm.model_cost[model]:
return litellm.model_cost[model]["max_output_tokens"]
get_llm_provider: Final = litellm_utils.get_llm_provider
model, custom_llm_provider, _, _ = get_llm_provider(model=model)
if custom_llm_provider == "huggingface":
@ -6092,8 +6089,6 @@ def get_max_tokens(model: str) -> int | None:
if model in litellm.model_cost: # check if extracted model is in model_list
if "max_output_tokens" in litellm.model_cost[model]:
return litellm.model_cost[model]["max_output_tokens"]
elif "max_tokens" in litellm.model_cost[model]:
return litellm.model_cost[model]["max_tokens"]
else:
raise Exception()
return None
@ -6742,7 +6737,7 @@ def get_model_info_helper(
returned_model_info: Final = ModelInfoBase(
key=key,
max_tokens=_model_info.get("max_tokens", None),
max_tokens=_model_info.get("max_output_tokens") or _model_info.get("max_input_tokens"),
max_input_tokens=_model_info.get("max_input_tokens", None),
max_output_tokens=_model_info.get("max_output_tokens", None),
input_cost_per_token=_input_cost_per_token,
@ -8242,9 +8237,7 @@ def trim_messages(
if max_tokens is None:
# Check if model is valid
if model in litellm.model_cost:
max_tokens_for_model: Final = litellm.model_cost[model].get(
"max_input_tokens", litellm.model_cost[model]["max_tokens"]
)
max_tokens_for_model: Final = litellm.model_cost[model]["max_input_tokens"]
max_tokens = int(max_tokens_for_model * trim_ratio)
else:
# if user did not specify max (input) tokens

3573
model_prices_and_context_window.json
View file

File diff suppressed because it is too large Load diff

8
tests/unit/litellm_core_utils/test_fallback_generalizations.py
View file

@ -369,7 +369,7 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp
"output_cost_per_token": 2e-6,
"litellm_provider": "openai",
"mode": "chat",
"max_tokens": 7,
"max_output_tokens": 7,
"supports_reasoning": False,
},
"acme-bare": {
@ -398,7 +398,7 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp
"name": "acme-backfill",
"pattern": r"^acme-",
"fill_missing_for_providers": ["openai"],
"model_info": {"supports_reasoning": True, "max_tokens": 5},
"model_info": {"supports_reasoning": True, "max_output_tokens": 5},
}
]
)
@ -406,10 +406,12 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp
full = litellm.get_model_info("acme-full", custom_llm_provider="openai")
assert full["supports_reasoning"] is False
assert full["max_output_tokens"] == 7
assert full["max_tokens"] == 7
bare = litellm.get_model_info("acme-bare", custom_llm_provider="openai")
assert bare["supports_reasoning"] is True
assert bare["max_output_tokens"] == 5
assert bare["max_tokens"] == 5
assert bare["input_cost_per_token"] == 3e-6
assert bare["key"] == "acme-bare"
@ -525,7 +527,7 @@ def test_shipped_gemini_chat_baseline_loses_to_exact_entries(shipped_cost_map):
model = "gemini-2.5-flash-lite"
info = litellm.get_model_info(model, custom_llm_provider="gemini")
entry = litellm.model_cost["gemini/gemini-2.5-flash-lite"]
assert info["max_tokens"] == entry["max_tokens"]
assert info["max_tokens"] == entry["max_output_tokens"]
assert info["input_cost_per_token"] == entry["input_cost_per_token"]
assert entry["input_cost_per_token"] > 0

9
tests/unit/llms/openai_like/test_openinfer_provider.py
View file

@ -11,9 +11,7 @@ from litellm.proxy.auth.model_checks import get_provider_models
_REPO_ROOT: Final = Path(__file__).resolve().parents[4]
_PRIMARY_PRICES: Final = json.loads((_REPO_ROOT / "model_prices_and_context_window.json").read_text())
_BACKUP_PRICES: Final = json.loads(
(_REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text()
)
_BACKUP_PRICES: Final = json.loads((_REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text())
_CATALOG_MODELS: Final = tuple(
key for key, row in _PRIMARY_PRICES.items() if isinstance(row, dict) and row.get("litellm_provider") == "openinfer"
)
@ -31,7 +29,9 @@ _CHAT_RESPONSE: Final = {
"object": "chat.completion",
"created": 1_792_000_000,
"model": "@oi/Qwen3.5-27B",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello from OpenInfer"}, "finish_reason": "stop"}],
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "Hello from OpenInfer"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
}
@ -228,7 +228,6 @@ def test_catalog_row_matches_its_backup_and_fits_its_context(model: str):
assert row == _BACKUP_PRICES[model]
assert row["mode"] == "chat"
assert row["max_tokens"] == row["max_output_tokens"]
assert 0 < row["max_output_tokens"] <= row["max_input_tokens"]
assert row["input_cost_per_token"] > 0
assert row["output_cost_per_token"] > 0

23
tests/unit/test_model_prices_and_context_window.py
View file

@ -50,7 +50,6 @@ class _CopilotEndpointRow(BaseModel):
mode: str
max_input_tokens: int
max_output_tokens: int
max_tokens: int
supported_endpoints: tuple[str, ...]
@ -96,18 +95,22 @@ def _anthropic_counterpart(key: str) -> str:
@pytest.mark.parametrize(("path",), [(PRICES_PATH,), (BACKUP_PRICES_PATH,)], ids=("main", "backup"))
def test_github_copilot_rows_derive_mode_and_max_tokens_from_their_endpoints(path: Path) -> None:
def test_github_copilot_rows_derive_mode_from_their_endpoints(path: Path) -> None:
rows: Final[Mapping[str, _CopilotEndpointRow]] = _copilot_endpoint_rows(path)
assert rows, f"No GitHub Copilot endpoint rows found in {path}"
assert {key: (row.mode, row.max_tokens) for key, row in rows.items()} == {
key: (
"chat" if "/v1/chat/completions" in row.supported_endpoints else "responses",
row.max_output_tokens,
)
for key, row in rows.items()
assert {key: row.mode for key, row in rows.items()} == {
key: "chat" if "/v1/chat/completions" in row.supported_endpoints else "responses" for key, row in rows.items()
}
def _rows_carrying_legacy_max_tokens(prices: Mapping[str, Mapping[str, object]]) -> tuple[str, ...]:
return tuple(key for key, row in prices.items() if "max_tokens" in row)
def test_no_row_carries_the_retired_max_tokens_field(prices: dict) -> None:
assert _rows_carrying_legacy_max_tokens(prices) == ()
def test_github_copilot_backup_rows_match_the_main_catalog() -> None:
assert _github_copilot_catalog_rows(PRICES_PATH) == _github_copilot_catalog_rows(BACKUP_PRICES_PATH)
@ -601,7 +604,6 @@ def test_tencent_tokenhub_glm_rows_match_published_metadata(path: Path):
or entry.get("litellm_provider") != "tencent"
or entry.get("max_input_tokens") != 1000000
or entry.get("max_output_tokens") != 128000
or entry.get("max_tokens") != 128000
or entry.get("input_cost_per_token_cache_hit") != entry.get("cache_read_input_token_cost")
)
assert actual == TENCENT_TOKENHUB_GLM_ROWS
@ -619,7 +621,7 @@ def test_tencent_tokenhub_kimi_rows_match_published_metadata(path: Path):
entry.get("input_cost_per_token"),
entry.get("output_cost_per_token"),
entry.get("cache_read_input_token_cost"),
entry.get("max_tokens"),
entry.get("max_output_tokens"),
entry.get("supports_vision", False),
entry.get("supports_video_input", False),
)
@ -793,7 +795,6 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa
assert entry["mode"] == "chat"
assert entry["max_input_tokens"] == 1024000
assert entry["max_output_tokens"] == 256000
assert entry["max_tokens"] == 256000
# Volcengine Ark pricing https://www.volcengine.com/docs/82379/1544106 checked 2026-10-20
assert entry["input_cost_per_token"] == pytest.approx(6.0 / 6.9565 / 1_000_000, rel=1e-4)
assert entry["output_cost_per_token"] == pytest.approx(30.0 / 6.9565 / 1_000_000, rel=1e-4)

14
tests/unit/test_together_ai_model_metadata.py
View file

@ -75,18 +75,6 @@ def cost_map() -> CostMap:
return COST_MAP_ADAPTER.validate_python(json.load(f))
def test_together_chat_entries_never_carry_context_length_as_output_ceiling(cost_map: CostMap):
inflated = sorted(
model
for model, info in cost_map.items()
if info.get("litellm_provider") == "together_ai"
and info.get("mode") == "chat"
and "max_output_tokens" in info
and info["max_output_tokens"] == info.get("max_input_tokens")
)
assert inflated == []
def _successor(info: dict[str, object]) -> str | None:
metadata = info.get("metadata")
if not isinstance(metadata, dict):
@ -95,8 +83,6 @@ def _successor(info: dict[str, object]) -> str | None:
return successor if isinstance(successor, str) else None
def test_together_backup_cost_map_in_sync(cost_map: CostMap):
with open(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") as f:
backup = COST_MAP_ADAPTER.validate_python(json.load(f))

41
tests/unit/test_utils.py
View file

@ -2260,6 +2260,45 @@ class TestProxyFunctionCalling:
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
def test_get_model_info_derives_max_tokens_from_the_output_limit() -> None:
model: Final = "derived-max-tokens-chat-model"
litellm.register_model(
{
model: {
"litellm_provider": "openai",
"mode": "chat",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
}
},
persist_across_reloads=False,
)
info: Final = litellm.get_model_info(model, custom_llm_provider="openai")
assert info["max_tokens"] == 128000
assert litellm.get_max_tokens(model) == 128000
def test_get_model_info_derives_max_tokens_from_the_input_limit_when_the_row_has_no_output_limit() -> None:
model: Final = "derived-max-tokens-embedding-model"
litellm.register_model(
{
model: {
"litellm_provider": "openai",
"mode": "embedding",
"max_input_tokens": 8191,
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
}
},
persist_across_reloads=False,
)
info: Final = litellm.get_model_info(model, custom_llm_provider="openai")
assert info["max_tokens"] == 8191
assert info["max_output_tokens"] is None
def test_register_model_price_is_used_for_completion_cost() -> None:
model: Final = "migration-custom-pricing-model"
price: Final = {
@ -8629,7 +8668,7 @@ def test_basic_trimming_no_max_tokens_specified():
trimmed_messages = trim_messages(messages, model="gpt-4")
print("trimmed messages for gpt-4")
print(trimmed_messages)
assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_tokens"]
assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_input_tokens"]
def test_multiple_messages_trimming():