mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 73b4cfcb2f into 3b8ad37296
This commit is contained in:
commit
6b47d0a318
5 changed files with 123 additions and 22 deletions
|
|
@ -3507,18 +3507,23 @@ def _builtin_cost_map() -> dict[str, Mapping[str, object]]:
|
|||
|
||||
|
||||
def _replace_cost_map_entry(
|
||||
cost_map: dict[str, Mapping[str, object]], key: str, entry: Mapping[str, object], registration: Mapping[str, object]
|
||||
cost_map: dict[str, Mapping[str, object]],
|
||||
key: str,
|
||||
entry: Mapping[str, object],
|
||||
registration: Mapping[str, object],
|
||||
aliases: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
current: Final = cost_map.get(key)
|
||||
merged: Final = {**(current or {}), **entry}
|
||||
names: Final = current.names if isinstance(current, AliasedCostEntry) else ()
|
||||
previous_names: Final = current.names if isinstance(current, AliasedCostEntry) else ()
|
||||
names: Final = tuple(dict.fromkeys((*previous_names, *aliases)))
|
||||
previous_fields: Final[frozenset[str]] = (
|
||||
current.registered_fields if isinstance(current, AliasedCostEntry) else frozenset()
|
||||
)
|
||||
registered_fields: Final = previous_fields | frozenset(
|
||||
name for name, value in registration.items() if value is not None
|
||||
)
|
||||
sharing_names: Final = tuple(name for name in names if cost_map.get(name) is current)
|
||||
sharing_names: Final = tuple(name for name in names if name in aliases or cost_map.get(name) is current)
|
||||
cost_map.update(dict.fromkeys((key, *sharing_names), AliasedCostEntry(merged, names, registered_fields)))
|
||||
|
||||
|
||||
|
|
@ -3751,15 +3756,26 @@ def _skips_builtin_registration_lookup(key: str, provider: object) -> bool:
|
|||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RegistrationWrite:
|
||||
class _RegistrationBase:
|
||||
key: str
|
||||
entry: Mapping[str, object]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RegistrationWrite:
|
||||
keys: tuple[str, ...]
|
||||
entry: Mapping[str, object]
|
||||
|
||||
|
||||
def _write_keys(caller_key: str, resolved_key: str) -> tuple[str, ...]:
|
||||
if resolved_key == caller_key:
|
||||
return (caller_key,)
|
||||
provider, _, bare = caller_key.partition("/")
|
||||
return (caller_key, resolved_key) if bare == resolved_key and provider in LlmProvidersSet else (caller_key,)
|
||||
|
||||
|
||||
def model_cost_registration_row(key: str, provider: object) -> Mapping[str, object]:
|
||||
"""The row a ``register_model`` write of a ``key`` entry with this ``litellm_provider`` merges onto: the
|
||||
``litellm.model_cost`` row it resolves to (``openai/o3-pro`` lands on ``o3-pro``), else the row it seeds ``key``
|
||||
with from the built-in entry it falls back to (an OpenAI fine-tune takes its base model's row)."""
|
||||
"""The inheritance base for a registration, independent of which keys it writes."""
|
||||
base: Final = _registration_base(
|
||||
key, {"litellm_provider": provider}, custom_llm_provider=None, warning_display_name=None
|
||||
)
|
||||
|
|
@ -3768,15 +3784,15 @@ def model_cost_registration_row(key: str, provider: object) -> Mapping[str, obje
|
|||
|
||||
def _registration_base(
|
||||
key: str, value: Mapping[str, object], custom_llm_provider: str | None, warning_display_name: str | None
|
||||
) -> _RegistrationWrite:
|
||||
) -> _RegistrationBase:
|
||||
provider: Final = str(value.get("litellm_provider") or "")
|
||||
if _skips_builtin_registration_lookup(key, provider):
|
||||
return _RegistrationWrite(key=key, entry=dict(_builtin_cost_map().get(key, {})))
|
||||
return _RegistrationBase(key=key, entry=dict(_builtin_cost_map().get(key, {})))
|
||||
builtin_model_info: Final = _get_builtin_model_info_for_registration(
|
||||
model=key, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
if not isinstance(builtin_model_info, ModelInfoLookupMiss):
|
||||
return _RegistrationWrite(
|
||||
return _RegistrationBase(
|
||||
key=builtin_model_info["key"],
|
||||
entry=dict(_without_synthesized_costs(builtin_model_info, key=key, value=value)),
|
||||
)
|
||||
|
|
@ -3791,7 +3807,7 @@ def _registration_base(
|
|||
"register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info",
|
||||
warning_display_name or key,
|
||||
)
|
||||
return _RegistrationWrite(key=key, entry=dict(fallback.entry))
|
||||
return _RegistrationBase(key=key, entry=dict(fallback.entry))
|
||||
|
||||
|
||||
def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]:
|
||||
|
|
@ -3818,7 +3834,7 @@ def _registration_write(
|
|||
if name != "litellm_provider" or field_value is not None
|
||||
}
|
||||
return _RegistrationWrite(
|
||||
key=base.key,
|
||||
keys=_write_keys(key, base.key),
|
||||
entry={**update_dictionary(existing_model, value), **dict.fromkeys(_base_fields_replaced_by(value))},
|
||||
)
|
||||
|
||||
|
|
@ -3832,7 +3848,10 @@ def _registration_attempt(
|
|||
) -> _RegistrationWrite | None:
|
||||
source: Final = _builtin_cost_map()
|
||||
write: Final = _registration_write(key, value, custom_llm_provider, warning_display_name)
|
||||
_replace_cost_map_entry(_builtin_cost_map() if final_attempt else source, write.key, write.entry, value)
|
||||
target: Final = _builtin_cost_map() if final_attempt else source
|
||||
aliases: Final = write.keys if len(write.keys) > 1 else ()
|
||||
for write_key in reversed(write.keys):
|
||||
_replace_cost_map_entry(target, write_key, write.entry, value, aliases)
|
||||
return write if final_attempt or litellm.model_cost is source else None
|
||||
|
||||
|
||||
|
|
@ -3999,7 +4018,7 @@ def register_model(
|
|||
# Invalidate case-insensitive lookup map since model_cost was modified
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", write.key, write.key)
|
||||
verbose_logger.debug("added/updated models=%s in litellm.model_cost", write.keys)
|
||||
if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter":
|
||||
litellm.openrouter_models.add(str(key).split("/", 1)[-1])
|
||||
if add_to_provider_model_sets:
|
||||
|
|
|
|||
|
|
@ -4826,7 +4826,6 @@ def test_completion_cost_legacy_mantle_route_prices_after_router_registration(lo
|
|||
}
|
||||
]
|
||||
)
|
||||
assert "bedrock/mantle/anthropic.claude-sonnet-5" not in litellm.model_cost
|
||||
|
||||
response = litellm.ModelResponse(
|
||||
id="msg_x",
|
||||
|
|
@ -4835,6 +4834,9 @@ def test_completion_cost_legacy_mantle_route_prices_after_router_registration(lo
|
|||
usage={"prompt_tokens": 16, "completion_tokens": 4, "total_tokens": 20},
|
||||
)
|
||||
row = litellm.model_cost["anthropic.claude-sonnet-5"]
|
||||
registered_row: Final = litellm.model_cost["bedrock/mantle/anthropic.claude-sonnet-5"]
|
||||
assert registered_row["input_cost_per_token"] == row["input_cost_per_token"]
|
||||
assert registered_row["output_cost_per_token"] == row["output_cost_per_token"]
|
||||
expected = 16 * row["input_cost_per_token"] + 4 * row["output_cost_per_token"]
|
||||
assert expected > 0
|
||||
|
||||
|
|
|
|||
|
|
@ -883,9 +883,9 @@ def test_register_model_keeps_a_catalog_rows_own_prices_when_the_lookup_fails(mo
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ["", "converse/", "invoke/"])
|
||||
def test_register_model_leaves_no_shadow_key_when_the_lookup_trips_once(
|
||||
monkeypatch: pytest.MonkeyPatch, route: str
|
||||
@pytest.mark.parametrize("route, expected_input_cost", [("", 9e-06), ("converse/", 3.3e-06), ("invoke/", 3.3e-06)])
|
||||
def test_register_model_keeps_exact_key_and_only_links_provider_twins_when_the_lookup_trips_once(
|
||||
monkeypatch: pytest.MonkeyPatch, route: str, expected_input_cost: float
|
||||
) -> None:
|
||||
catalog_key: Final = "au.anthropic.lit-5853-geo-model"
|
||||
alias: Final = f"bedrock/{route}{catalog_key}"
|
||||
|
|
@ -919,8 +919,9 @@ def test_register_model_leaves_no_shadow_key_when_the_lookup_trips_once(
|
|||
litellm.register_model({catalog_key: {"input_cost_per_token": repriced_input_cost}}, persist_across_reloads=False)
|
||||
|
||||
try:
|
||||
assert litellm.get_model_info(alias)["key"] == catalog_key
|
||||
assert litellm.get_model_info(alias)["input_cost_per_token"] == repriced_input_cost
|
||||
assert litellm.get_model_info(alias)["key"] == alias
|
||||
assert litellm.get_model_info(alias)["input_cost_per_token"] == expected_input_cost
|
||||
assert litellm.get_model_info(catalog_key)["input_cost_per_token"] == repriced_input_cost
|
||||
finally:
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
|
@ -1163,7 +1164,9 @@ def _token_prices(name: str) -> tuple[float | None, float | None]:
|
|||
return info["input_cost_per_token"], info["output_cost_per_token"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("registered_key", [_CANONICAL_KEY, _ALIAS_KEY])
|
||||
@pytest.mark.parametrize(
|
||||
"registered_key", [_CANONICAL_KEY, _ALIAS_KEY, f"openai/{_CANONICAL_KEY}", f"openai/{_ALIAS_KEY}"]
|
||||
)
|
||||
def test_register_model_prices_every_alias_of_the_registered_entry(
|
||||
monkeypatch: pytest.MonkeyPatch, registered_key: str
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -4026,8 +4026,9 @@ def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_
|
|||
registrations_landed: Final = writer_outcome.get(timeout=5)
|
||||
try:
|
||||
assert isinstance(registrations_landed, int) and registrations_landed > 0, registrations_landed
|
||||
assert tuple(backend for backend in backends if backend in litellm.model_cost) == ()
|
||||
assert tuple(backend for backend in backends if backend in litellm.model_cost) == backends
|
||||
for backend in backends:
|
||||
assert litellm.model_cost[backend]["input_cost_per_token"] == canonical_prices[backend]
|
||||
assert litellm.get_model_info(backend)["input_cost_per_token"] == canonical_prices[backend]
|
||||
assert tuple(router.get_model_names()) == tuple(backend.rsplit(".", 1)[-1] for backend in backends)
|
||||
finally:
|
||||
|
|
|
|||
|
|
@ -2259,6 +2259,82 @@ class TestProxyFunctionCalling:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"caller_key, base_key, provider",
|
||||
[
|
||||
("databricks/databricks-claude-opus-4-99", "databricks/databricks-claude-opus-4", "databricks"),
|
||||
("bedrock/us.anthropic.claude-sonnet-4-6", "anthropic.claude-sonnet-4-6", "bedrock"),
|
||||
("openai/gpt-4o-2026-01-01", "gpt-4o", "openai"),
|
||||
],
|
||||
)
|
||||
def test_register_model_preserves_exact_key_and_inherited_sibling(
|
||||
monkeypatch: pytest.MonkeyPatch, caller_key: str, base_key: str, provider: str
|
||||
) -> None:
|
||||
builtin: Final = {
|
||||
"max_tokens": 32000,
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
"cache_read_input_token_cost": 0.0001,
|
||||
"supports_function_calling": True,
|
||||
"litellm_provider": provider,
|
||||
"mode": "chat",
|
||||
}
|
||||
monkeypatch.setattr(litellm, "model_cost", {base_key: dict(builtin)})
|
||||
assert litellm.get_model_info(caller_key, custom_llm_provider=provider)["key"] == base_key
|
||||
|
||||
litellm.register_model(
|
||||
{caller_key: {"max_tokens": 123456}}, custom_llm_provider=provider, persist_across_reloads=False
|
||||
)
|
||||
|
||||
assert litellm.model_cost[base_key] == builtin
|
||||
assert litellm.model_cost[caller_key]["max_tokens"] == 123456
|
||||
info: Final = litellm.get_model_info(caller_key)
|
||||
assert info["key"] == caller_key
|
||||
assert info["max_tokens"] == 123456
|
||||
assert info["cache_read_input_token_cost"] == builtin["cache_read_input_token_cost"]
|
||||
assert info["supports_function_calling"] is True
|
||||
assert litellm.cost_per_token(model=caller_key, prompt_tokens=10, completion_tokens=5) == pytest.approx(
|
||||
(0.01, 0.01)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("update_key", ("openai/o3-pro", "o3-pro"))
|
||||
def test_register_model_provider_prefix_keeps_bare_price_and_responses_mode(
|
||||
monkeypatch: pytest.MonkeyPatch, update_key: str
|
||||
) -> None:
|
||||
from litellm.utils import model_cost_registration_row
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"model_cost",
|
||||
{
|
||||
"o3-pro": {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "responses",
|
||||
}
|
||||
},
|
||||
)
|
||||
assert model_cost_registration_row("openai/o3-pro", "openai")["mode"] == "responses"
|
||||
|
||||
litellm.register_model({"openai/o3-pro": {"input_cost_per_token": 0.003}}, persist_across_reloads=False)
|
||||
|
||||
assert litellm.model_cost["openai/o3-pro"]["input_cost_per_token"] == 0.003
|
||||
assert litellm.model_cost["o3-pro"]["input_cost_per_token"] == 0.003
|
||||
assert model_cost_registration_row("openai/o3-pro", "openai")["mode"] == "responses"
|
||||
assert litellm.cost_per_token(model="o3-pro", prompt_tokens=10, completion_tokens=5) == pytest.approx((0.03, 0.01))
|
||||
|
||||
litellm.register_model({update_key: {"input_cost_per_token": 0.004}}, persist_across_reloads=False)
|
||||
|
||||
assert litellm.model_cost["openai/o3-pro"]["input_cost_per_token"] == 0.004
|
||||
assert litellm.model_cost["o3-pro"]["input_cost_per_token"] == 0.004
|
||||
assert model_cost_registration_row("openai/o3-pro", "openai")["mode"] == "responses"
|
||||
assert litellm.cost_per_token(model="openai/o3-pro", prompt_tokens=10, completion_tokens=5) == pytest.approx(
|
||||
(0.04, 0.01)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
|
||||
def test_get_model_info_derives_max_tokens_from_the_output_limit() -> None:
|
||||
model: Final = "derived-max-tokens-chat-model"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue