mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
Merge pull request #41843 from BerriAI/litellm_lit8064_unpin_derived_pricing
fix(proxy): unpin cost-map pricing copied into model_info and report pricing overrides
This commit is contained in:
commit
4eb13a0b2a
9 changed files with 401 additions and 11 deletions
|
|
@ -137,7 +137,7 @@ from litellm.types.router import (
|
|||
updateDeployment,
|
||||
updateLiteLLMParams,
|
||||
)
|
||||
from litellm.types.utils import without_server_derived_pricing
|
||||
from litellm.types.utils import echoed_cost_map_pricing_fields, without_server_derived_pricing
|
||||
from litellm.utils import get_utc_datetime
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -876,7 +876,11 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr
|
|||
_raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True))
|
||||
merged_model_name: Final = updated_patch.model_name or db_model.model_name
|
||||
merged_litellm_params: Final = db_model.litellm_params.model_dump(exclude_none=True)
|
||||
merged_model_info: Final[dict[str, object]] = db_model.model_info.model_dump(exclude_none=True)
|
||||
stored_model_info: Final = db_model.model_info.model_dump(exclude_none=True)
|
||||
echoed_pricing: Final = echoed_cost_map_pricing_fields(stored_model_info)
|
||||
merged_model_info: Final[dict[str, object]] = {
|
||||
k: v for k, v in stored_model_info.items() if k not in echoed_pricing
|
||||
}
|
||||
|
||||
# update litellm params
|
||||
if updated_patch.litellm_params:
|
||||
|
|
|
|||
|
|
@ -147,11 +147,15 @@ from litellm.router_utils.auto_router_tuning_baseline import (
|
|||
from litellm.router_utils.routing_groups import parse_routing_groups
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.utils import (
|
||||
PRICING_OVERRIDES_KEY,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
TextCompletionResponse,
|
||||
TokenCountResponse,
|
||||
echoed_cost_map_pricing_fields,
|
||||
is_server_derived_pricing_key,
|
||||
pricing_override_fields,
|
||||
)
|
||||
from litellm.utils import cost_map_omits_token_price, load_credentials_from_list
|
||||
|
||||
|
|
@ -4867,6 +4871,16 @@ def _bind_general_settings_store(settings: SettingsStore) -> None:
|
|||
general_settings = settings # pyright: ignore[reportAssignmentType] # legacy global accepts mappings
|
||||
|
||||
|
||||
@lru_cache(maxsize=4096)
|
||||
def _log_ignored_cost_map_copy(model_id: str, fields: tuple[str, ...]) -> None:
|
||||
verbose_proxy_logger.warning(
|
||||
"Deployment %s stores a copy of the cost map in model_info (%s); ignoring it so the deployment follows the "
|
||||
"current cost map. Set the price on litellm_params to override the cost map on purpose.",
|
||||
model_id,
|
||||
", ".join(fields),
|
||||
)
|
||||
|
||||
|
||||
class ProxyConfig:
|
||||
"""
|
||||
Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic.
|
||||
|
|
@ -6692,7 +6706,12 @@ class ProxyConfig:
|
|||
model.model_info["id"] = model.model_id
|
||||
if "db_model" in model.model_info and model.model_info["db_model"] is False:
|
||||
model.model_info["db_model"] = db_model
|
||||
_model_info = RouterModelInfo(**model.model_info)
|
||||
echoed_pricing: Final = echoed_cost_map_pricing_fields(model.model_info)
|
||||
if echoed_pricing:
|
||||
_log_ignored_cost_map_copy(str(model.model_info["id"]), echoed_pricing)
|
||||
_model_info = RouterModelInfo(
|
||||
**MappingProxyType({k: v for k, v in model.model_info.items() if k not in echoed_pricing})
|
||||
)
|
||||
|
||||
else:
|
||||
_model_info = RouterModelInfo(id=model.model_id, db_model=db_model)
|
||||
|
|
@ -9382,6 +9401,15 @@ def select_data_generator(
|
|||
)
|
||||
|
||||
|
||||
def _pricing_override_stamps(
|
||||
model_info: Mapping[str, object], litellm_params: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
own_pricing: Final = MappingProxyType(
|
||||
{k: v for k, v in litellm_params.items() if v is not None and is_server_derived_pricing_key(k)}
|
||||
)
|
||||
return MappingProxyType({**own_pricing, PRICING_OVERRIDES_KEY: pricing_override_fields(model_info, own_pricing)})
|
||||
|
||||
|
||||
def get_litellm_model_info(model: dict = {}):
|
||||
model_info: Final = model.get("model_info", {})
|
||||
model_to_lookup = model.get("litellm_params", {}).get("model", None)
|
||||
|
|
@ -13722,10 +13750,17 @@ def _enrich_model_info_with_litellm_data(
|
|||
llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({})
|
||||
)
|
||||
unpriced: Final = cost_map_omits_token_price(model_info.get("id"), litellm_model_info.get("key"))
|
||||
for k, v in MappingProxyType({**litellm_model_info, **discovered_model_info}).items():
|
||||
if k not in model_info or (model_info[k] is None and k in discovered_model_info):
|
||||
model_info[k] = None if unpriced and k in ("input_cost_per_token", "output_cost_per_token") else v
|
||||
model["model_info"] = model_info
|
||||
stamped_model_info: Final = MappingProxyType(
|
||||
{**model_info, **_pricing_override_stamps(model_info, model.get("litellm_params") or MappingProxyType({}))}
|
||||
)
|
||||
model["model_info"] = {
|
||||
**stamped_model_info,
|
||||
**{
|
||||
k: None if unpriced and k in ("input_cost_per_token", "output_cost_per_token") else v
|
||||
for k, v in MappingProxyType({**litellm_model_info, **discovered_model_info}).items()
|
||||
if k not in stamped_model_info or (stamped_model_info[k] is None and k in discovered_model_info)
|
||||
},
|
||||
}
|
||||
# don't return the api key / vertex credentials
|
||||
# don't return the llm credentials
|
||||
model = remove_sensitive_info_from_deployment(model, excluded_keys={"litellm_credential_name"})
|
||||
|
|
|
|||
|
|
@ -3727,6 +3727,10 @@ def is_server_derived_pricing_key(key: str) -> bool:
|
|||
return key in SERVER_DERIVED_PRICING_FIELDS or ABOVE_THRESHOLD_COST_KEY_PATTERN.search(key) is not None
|
||||
|
||||
|
||||
PRICING_OVERRIDES_KEY: Final = "pricing_overrides"
|
||||
COST_MAP_LOOKUP_KEY: Final = "key"
|
||||
|
||||
|
||||
def without_server_derived_pricing(model_info: Mapping[str, Any]) -> Mapping[str, Any]:
|
||||
"""Drop the pricing ``/model/info`` derives for display, keeping everything else.
|
||||
|
||||
|
|
@ -3736,7 +3740,32 @@ def without_server_derived_pricing(model_info: Mapping[str, Any]) -> Mapping[str
|
|||
deployment at that day's price where no cost map refresh can reach it. A deployment's
|
||||
own pricing belongs on ``litellm_params``, which is unaffected.
|
||||
"""
|
||||
return MappingProxyType({k: v for k, v in model_info.items() if not is_server_derived_pricing_key(k)})
|
||||
return MappingProxyType(
|
||||
{k: v for k, v in model_info.items() if k != PRICING_OVERRIDES_KEY and not is_server_derived_pricing_key(k)}
|
||||
)
|
||||
|
||||
|
||||
def echoed_cost_map_pricing_fields(model_info: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
"""Pricing fields a stored ``model_info`` blob copied from a ``/model/info`` response.
|
||||
|
||||
Only ``litellm.get_model_info`` emits ``key`` (the resolved cost-map entry), so a stored
|
||||
blob carrying it alongside pricing fields holds the cost map as it stood on the day the
|
||||
row was saved, not a price anyone typed. Rows saved before 1.102 through the Admin UI
|
||||
edit form look exactly like this, and a price typed into ``litellm_params`` never does.
|
||||
"""
|
||||
if COST_MAP_LOOKUP_KEY not in model_info:
|
||||
return ()
|
||||
return tuple(sorted(k for k in model_info if is_server_derived_pricing_key(k)))
|
||||
|
||||
|
||||
def pricing_override_fields(*sources: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
frozenset(
|
||||
k for source in sources for k, v in source.items() if v is not None and is_server_derived_pricing_key(k)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# Server-controlled fields that bound or drive an interceptor's agentic loop
|
||||
|
|
|
|||
|
|
@ -3709,6 +3709,91 @@ class TestModelInfoServerDerivedPricingFilter:
|
|||
assert field not in info, f"{field} was persisted as a per-deployment override"
|
||||
assert field not in params
|
||||
|
||||
def test_echoed_pricing_overrides_report_is_not_persisted(self):
|
||||
"""LIT-8064. `/model/info` reports which pricing fields a deployment overrides; a
|
||||
client echoing that response back must not store the report as a field."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
update_db_model,
|
||||
)
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.6"),
|
||||
model_info=ModelInfo(id="dep-report-0"),
|
||||
)
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(
|
||||
model_info=ModelInfo(id="dep-report-0", access_groups=["prod"], pricing_overrides=[]),
|
||||
),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert info["access_groups"] == ["prod"]
|
||||
assert "pricing_overrides" not in info
|
||||
|
||||
def test_a_row_pinned_before_1_102_drops_its_cost_map_copy_on_its_next_save(self, monkeypatch: pytest.MonkeyPatch):
|
||||
"""LIT-8064. A stored ``model_info`` carrying ``key`` is a ``/model/info`` response an old
|
||||
UI wrote back, so its pricing is the cost map of that day. The next edit of the row, here
|
||||
only its reasoning level, leaves that copy behind and keeps everything the operator set."""
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
update_db_model,
|
||||
)
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-lit8064-heal-on-save")
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.6", reasoning_effort="medium"),
|
||||
model_info=ModelInfo(
|
||||
id="dep-pinned-0",
|
||||
key="gpt-5.6",
|
||||
mode="chat",
|
||||
access_groups=["prod"],
|
||||
input_cost_per_token=4e-06,
|
||||
output_cost_per_token=2e-05,
|
||||
cache_read_input_token_cost_above_272k_tokens=8e-07,
|
||||
),
|
||||
)
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(litellm_params=updateLiteLLMParams(reasoning_effort="low")),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
params = json.loads(result["litellm_params"])
|
||||
assert decrypt_value_helper(value=params["reasoning_effort"], key="reasoning_effort") == "low"
|
||||
assert (info["key"], info["mode"], info["access_groups"]) == ("gpt-5.6", "chat", ["prod"])
|
||||
for field in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost_above_272k_tokens"):
|
||||
assert field not in info, f"{field} still pins the row to the cost map of the day it was saved"
|
||||
assert field not in params
|
||||
|
||||
def test_a_litellm_params_price_survives_the_cost_map_copy_being_dropped(self):
|
||||
"""The price an operator typed on ``litellm_params`` is the override the customer asked
|
||||
for, so dropping the echoed ``model_info`` copy must leave it in place."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
update_db_model,
|
||||
)
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.6", input_cost_per_token=3e-06),
|
||||
model_info=ModelInfo(id="dep-typed-0", key="gpt-5.6", input_cost_per_token=3e-06),
|
||||
)
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_info=ModelInfo(id="dep-typed-0", access_groups=["prod"])),
|
||||
)
|
||||
|
||||
assert json.loads(result["litellm_params"])["input_cost_per_token"] == 3e-06
|
||||
assert json.loads(result["model_info"])["access_groups"] == ["prod"]
|
||||
|
||||
def test_tiered_above_threshold_pricing_is_dropped(self):
|
||||
"""Tiered rates ride `get_model_info` on a pattern match and are declared on no
|
||||
model, so a filter built only from the declared pricing fields would miss them."""
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import re
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Any, Dict, Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
|
|
@ -2633,6 +2633,113 @@ def test_ProxyConfig_get_model_info_with_id_returns_router_model_info():
|
|||
assert snapshot == {"id": "m-1", "db_model": True, "blocked": False}
|
||||
|
||||
|
||||
PINNED_MODEL_INFO: Final = MappingProxyType(
|
||||
{
|
||||
"id": "pinned-row",
|
||||
"key": "gpt-5.6",
|
||||
"mode": "chat",
|
||||
"access_groups": ["prod"],
|
||||
"input_cost_per_token": 4e-06,
|
||||
"output_cost_per_token": 2e-05,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 8e-07,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_ProxyConfig_get_model_info_with_id_ignores_cost_map_pricing_echoed_into_model_info():
|
||||
"""LIT-8064. A pre-1.102 Admin UI save wrote the whole ``/model/info`` response back into
|
||||
the row's ``model_info``, cost-map pricing included. Only that response carries ``key``, so
|
||||
a stored blob with it holds a copy of the map, not a price anyone typed, and the deployment
|
||||
must keep following the live cost map."""
|
||||
pc = ProxyConfig()
|
||||
model = SimpleNamespace(model_id="pinned-row", model_info=dict(PINNED_MODEL_INFO), blocked=False)
|
||||
out = pc.get_model_info_with_id(model=model, db_model=True).model_dump(exclude_none=True)
|
||||
assert out["access_groups"] == ["prod"]
|
||||
assert out["mode"] == "chat"
|
||||
for field in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost_above_272k_tokens"):
|
||||
assert field not in out, f"{field} still pins the deployment to the cost map of the day it was saved"
|
||||
|
||||
|
||||
def test_ProxyConfig_get_model_info_with_id_keeps_pricing_typed_into_model_info():
|
||||
"""A custom-priced deployment the cost map does not know never got ``key``, so its
|
||||
``model_info`` pricing is the operator's own and stays."""
|
||||
pc = ProxyConfig()
|
||||
model = SimpleNamespace(
|
||||
model_id="custom-row",
|
||||
model_info={"id": "custom-row", "input_cost_per_token": 7e-06, "output_cost_per_token": 9e-06},
|
||||
blocked=False,
|
||||
)
|
||||
out = pc.get_model_info_with_id(model=model, db_model=True).model_dump(exclude_none=True)
|
||||
assert (out["input_cost_per_token"], out["output_cost_per_token"]) == (7e-06, 9e-06)
|
||||
|
||||
|
||||
def test_ProxyConfig__add_deployment_pinned_row_follows_the_cost_map_across_reloads(monkeypatch, local_model_cost_map):
|
||||
"""The customer's symptom end to end: a row pinned before 1.102 must bill at the live cost
|
||||
map price on boot and again after Reload Price Data, while a price typed on
|
||||
``litellm_params`` keeps overriding it."""
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.decrypt_value_helper",
|
||||
lambda value, key, return_original_value: value,
|
||||
)
|
||||
router = litellm.Router(model_list=[])
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router)
|
||||
pinned = SimpleNamespace(
|
||||
model_id="pinned-row",
|
||||
model_name="gpt-5.6",
|
||||
model_info=dict(PINNED_MODEL_INFO),
|
||||
litellm_params={"model": "openai/gpt-5.6", "api_key": "sk-test"},
|
||||
blocked=False,
|
||||
)
|
||||
typed = SimpleNamespace(
|
||||
model_id="typed-row",
|
||||
model_name="gpt-5.6-typed",
|
||||
model_info={"id": "typed-row", "key": "gpt-5.6", "input_cost_per_token": 4e-06},
|
||||
litellm_params={"model": "openai/gpt-5.6", "api_key": "sk-test", "input_cost_per_token": 3e-06},
|
||||
blocked=False,
|
||||
)
|
||||
|
||||
assert ProxyConfig()._add_deployment(db_models=[pinned, typed]) == 2
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost["gpt-5.6"], "input_cost_per_token", 1e-06)
|
||||
router._replay_model_cost_registrations()
|
||||
|
||||
assert litellm.model_cost.get("pinned-row", {}).get("input_cost_per_token") is None
|
||||
assert router.get_deployment(model_id="pinned-row").model_info.input_cost_per_token is None
|
||||
assert litellm.get_model_info("openai/gpt-5.6")["input_cost_per_token"] == 1e-06
|
||||
assert litellm.model_cost["typed-row"]["input_cost_per_token"] == 3e-06
|
||||
|
||||
|
||||
def test_ProxyConfig__add_deployment_ptu_row_with_a_cost_map_copy_still_bills_zero(monkeypatch, local_model_cost_map):
|
||||
"""A PTU deployment bills nothing per token: the proxy writes zeros to both blobs. When such
|
||||
a row also carries the echoed cost map, dropping the ``model_info`` copy must not send it
|
||||
back to the per-token price, because the ``litellm_params`` zeros are the operator's."""
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.decrypt_value_helper",
|
||||
lambda value, key, return_original_value: value,
|
||||
)
|
||||
router = litellm.Router(model_list=[])
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router)
|
||||
ptu = SimpleNamespace(
|
||||
model_id="ptu-row",
|
||||
model_name="gpt-5.6-ptu",
|
||||
model_info={**PINNED_MODEL_INFO, "id": "ptu-row", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
|
||||
litellm_params={
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_key": "sk-test",
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
blocked=False,
|
||||
)
|
||||
|
||||
assert ProxyConfig()._add_deployment(db_models=[ptu]) == 1
|
||||
router._replay_model_cost_registrations()
|
||||
|
||||
assert litellm.model_cost["ptu-row"]["input_cost_per_token"] == 0.0
|
||||
assert litellm.model_cost["ptu-row"]["output_cost_per_token"] == 0.0
|
||||
assert router.get_deployment(model_id="ptu-row").model_info.input_cost_per_token == 0.0
|
||||
|
||||
|
||||
def test_ProxyConfig_get_model_info_with_id_missing_model_id_raises(monkeypatch):
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False)
|
||||
pc = ProxyConfig()
|
||||
|
|
|
|||
|
|
@ -286,6 +286,91 @@ def test_get_proxy_model_info_surfaces_supports_parallel_function_calling(local_
|
|||
assert enriched["model_info"]["supports_parallel_function_calling"] is True
|
||||
|
||||
|
||||
def _enriched_model_info(monkeypatch, litellm_params: dict, model_info: dict) -> dict:
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
enriched: Final = proxy_server._get_proxy_model_info(
|
||||
model={"model_name": "gpt-5.6", "litellm_params": litellm_params, "model_info": model_info}
|
||||
)
|
||||
return enriched["model_info"]
|
||||
|
||||
|
||||
def test_get_proxy_model_info_reports_no_pricing_overrides_for_a_cost_map_priced_deployment(
|
||||
monkeypatch, local_model_cost_map
|
||||
):
|
||||
"""LIT-8064. A deployment with no price of its own follows the cost map, and ``/model/info``
|
||||
says so with an empty ``pricing_overrides``."""
|
||||
info = _enriched_model_info(monkeypatch, {"model": "openai/gpt-5.6"}, {"id": "dep-synced", "db_model": True})
|
||||
assert info["pricing_overrides"] == ()
|
||||
assert info["input_cost_per_token"] == litellm.model_cost["gpt-5.6"]["input_cost_per_token"]
|
||||
|
||||
|
||||
def test_get_proxy_model_info_shows_litellm_params_pricing_and_names_it_as_an_override(
|
||||
monkeypatch, local_model_cost_map
|
||||
):
|
||||
"""A price on ``litellm_params`` is what the deployment bills at, so the model page shows that
|
||||
value rather than the cost map's and lists the field under ``pricing_overrides``."""
|
||||
info = _enriched_model_info(
|
||||
monkeypatch,
|
||||
{"model": "openai/gpt-5.6", "input_cost_per_token_batches": 1e-09},
|
||||
{"id": "dep-batches", "db_model": True},
|
||||
)
|
||||
assert info["pricing_overrides"] == ("input_cost_per_token_batches",)
|
||||
assert info["input_cost_per_token_batches"] == 1e-09
|
||||
assert info["input_cost_per_token"] == litellm.model_cost["gpt-5.6"]["input_cost_per_token"]
|
||||
|
||||
|
||||
def test_get_proxy_model_info_names_config_model_info_pricing_as_an_override(monkeypatch, local_model_cost_map):
|
||||
"""Pricing declared under ``model_info`` in config.yaml overrides the cost map too."""
|
||||
info = _enriched_model_info(
|
||||
monkeypatch, {"model": "openai/gpt-5.6"}, {"id": "dep-config", "db_model": False, "output_cost_per_token": 7e-06}
|
||||
)
|
||||
assert info["pricing_overrides"] == ("output_cost_per_token",)
|
||||
assert info["output_cost_per_token"] == 7e-06
|
||||
|
||||
|
||||
def test_v2_model_info_reports_pricing_overrides_to_the_admin_ui(client, auth_as, monkeypatch, local_model_cost_map):
|
||||
"""LIT-8064. The Admin UI model page reads ``GET /v2/model/info``, so the override report
|
||||
has to ride that route too, not only ``/model/info``."""
|
||||
model_list: Final = [
|
||||
{
|
||||
"model_name": "gpt-5.6",
|
||||
"litellm_params": {"model": "openai/gpt-5.6", "input_cost_per_token": 3e-06},
|
||||
"model_info": {"id": "dep-typed", "db_model": True},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-5.6",
|
||||
"litellm_params": {"model": "openai/gpt-5.6"},
|
||||
"model_info": {"id": "dep-synced", "db_model": True},
|
||||
},
|
||||
]
|
||||
router: Final = MagicMock()
|
||||
router.model_list = model_list
|
||||
router.get_discovered_model_info = MagicMock(return_value={})
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", model_list)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
monkeypatch.setattr(proxy_server.proxy_config, "get_config", AsyncMock(return_value={}))
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"_apply_search_filter_to_models",
|
||||
AsyncMock(side_effect=lambda all_models, **kw: (all_models, len(all_models))),
|
||||
)
|
||||
import litellm.proxy.agent_endpoints.model_list_helpers as mlh
|
||||
|
||||
monkeypatch.setattr(mlh, "append_agents_to_model_info", AsyncMock(side_effect=lambda models, **kw: models))
|
||||
|
||||
with auth_as():
|
||||
response = client.get("/v2/model/info")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
by_id: Final = {m["model_info"]["id"]: m["model_info"] for m in response.json()["data"]}
|
||||
assert by_id["dep-typed"]["pricing_overrides"] == ["input_cost_per_token"]
|
||||
assert by_id["dep-typed"]["input_cost_per_token"] == 3e-06
|
||||
assert by_id["dep-synced"]["pricing_overrides"] == []
|
||||
assert by_id["dep-synced"]["input_cost_per_token"] == litellm.model_cost["gpt-5.6"]["input_cost_per_token"]
|
||||
|
||||
|
||||
def test_model_info_reports_null_cost_for_unpriced_deployment_and_zero_for_declared_zero():
|
||||
"""A deployment configured with no cost fields must not surface the 0 that ``get_model_info``
|
||||
defaults to, since the zero-cost budget bypass only honours a declared zero. The declared zero
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ export interface ModelInfo {
|
|||
blocked?: boolean;
|
||||
team_public_model_name?: string;
|
||||
key?: string;
|
||||
pricing_overrides?: string[];
|
||||
}
|
||||
|
||||
export interface LiteLLMParams {
|
||||
|
|
|
|||
|
|
@ -52,4 +52,31 @@ describe("ModelPricingSummary", () => {
|
|||
expect(screen.getByText("-")).toBeInTheDocument();
|
||||
expect(screen.queryByText(/\$/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("names the fields a deployment prices itself", () => {
|
||||
render(
|
||||
<ModelPricingSummary
|
||||
model={{
|
||||
...tokenPriced,
|
||||
model_info: { pricing_overrides: ["input_cost_per_token", "output_cost_per_token"] },
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Custom pricing")).toBeInTheDocument();
|
||||
expect(
|
||||
screen.getByText("Overrides the model cost map for input_cost_per_token, output_cost_per_token"),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("says the price follows the cost map when nothing is overridden", () => {
|
||||
render(<ModelPricingSummary model={{ ...tokenPriced, model_info: { pricing_overrides: [] } }} />);
|
||||
expect(screen.getByText("Follows the model cost map")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("says nothing about the source when the proxy did not report it", () => {
|
||||
render(<ModelPricingSummary model={{ ...tokenPriced, model_info: {} }} />);
|
||||
expect(screen.queryByText(/cost map/)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,10 +1,26 @@
|
|||
import { ModelData } from "@/components/model_dashboard/types";
|
||||
import { ModelData, ModelInfo } from "@/components/model_dashboard/types";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { formatPerSecondCost } from "@/utils/dataUtils";
|
||||
|
||||
type PricingFields = Pick<
|
||||
ModelData,
|
||||
"input_cost" | "output_cost" | "output_cost_per_second" | "output_cost_per_second_tiers"
|
||||
>;
|
||||
> & { model_info?: Pick<ModelInfo, "pricing_overrides"> };
|
||||
|
||||
function PricingSource({ overrides }: { overrides: string[] | undefined }) {
|
||||
if (overrides === undefined) return null;
|
||||
if (overrides.length === 0) {
|
||||
return <p className="mt-2 text-xs text-muted-foreground">Follows the model cost map</p>;
|
||||
}
|
||||
return (
|
||||
<p className="mt-2 text-xs text-muted-foreground">
|
||||
<Badge variant="outline" className="mr-1">
|
||||
Custom pricing
|
||||
</Badge>
|
||||
Overrides the model cost map for {overrides.join(", ")}
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
export function ModelPricingSummary({ model }: { model: PricingFields }) {
|
||||
const perSecond = model.output_cost_per_second;
|
||||
|
|
@ -26,6 +42,7 @@ export function ModelPricingSummary({ model }: { model: PricingFields }) {
|
|||
Output ({resolution}): {formatPerSecondCost(cost)}
|
||||
</p>
|
||||
))}
|
||||
<PricingSource overrides={model.model_info?.pricing_overrides} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue