diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bb0e432af51..ad79f8e802c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -147,11 +147,15 @@ from litellm.router_utils.auto_router_tuning_baseline import ( ) from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.utils import ( + PRICING_OVERRIDES_KEY, ModelResponse, ModelResponseStream, StreamingChoices, TextCompletionResponse, TokenCountResponse, + echoed_cost_map_pricing_fields, + is_server_derived_pricing_key, + pricing_override_fields, ) from litellm.utils import load_credentials_from_list @@ -4822,6 +4826,16 @@ def _bind_general_settings_store(settings: SettingsStore) -> None: general_settings = settings # pyright: ignore[reportAssignmentType] # legacy global accepts mappings +@lru_cache(maxsize=4096) +def _log_ignored_cost_map_copy(model_id: str, fields: tuple[str, ...]) -> None: + verbose_proxy_logger.warning( + "Deployment %s stores a copy of the cost map in model_info (%s); ignoring it so the deployment follows the " + "current cost map. Set the price on litellm_params to override the cost map on purpose.", + model_id, + ", ".join(fields), + ) + + class ProxyConfig: """ Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic. @@ -6643,7 +6657,12 @@ class ProxyConfig: model.model_info["id"] = model.model_id if "db_model" in model.model_info and model.model_info["db_model"] is False: model.model_info["db_model"] = db_model - _model_info = RouterModelInfo(**model.model_info) + echoed_pricing: Final = echoed_cost_map_pricing_fields(model.model_info) + if echoed_pricing: + _log_ignored_cost_map_copy(str(model.model_info["id"]), echoed_pricing) + _model_info = RouterModelInfo( + **MappingProxyType({k: v for k, v in model.model_info.items() if k not in echoed_pricing}) + ) else: _model_info = RouterModelInfo(id=model.model_id, db_model=db_model) @@ -9302,6 +9321,15 @@ def select_data_generator( ) +def _pricing_override_stamps( + model_info: Mapping[str, object], litellm_params: Mapping[str, object] +) -> Mapping[str, object]: + own_pricing: Final = MappingProxyType( + {k: v for k, v in litellm_params.items() if v is not None and is_server_derived_pricing_key(k)} + ) + return MappingProxyType({**own_pricing, PRICING_OVERRIDES_KEY: pricing_override_fields(model_info, own_pricing)}) + + def get_litellm_model_info(model: dict = {}): model_info: Final = model.get("model_info", {}) model_to_lookup = model.get("litellm_params", {}).get("model", None) @@ -13602,6 +13630,8 @@ def _enrich_model_info_with_litellm_data( discovered_model_info: Final = ( llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({}) ) + for k, v in _pricing_override_stamps(model_info, model.get("litellm_params") or MappingProxyType({})).items(): + model_info[k] = v for k, v in MappingProxyType({**litellm_model_info, **discovered_model_info}).items(): if k not in model_info or (model_info[k] is None and k in discovered_model_info): model_info[k] = v @@ -15071,6 +15101,8 @@ def _get_proxy_model_info(model: dict) -> dict: discovered_model_info: Final = ( llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({}) ) + for k, v in _pricing_override_stamps(model_info, model.get("litellm_params") or MappingProxyType({})).items(): + model_info[k] = v for k, v in MappingProxyType({**litellm_model_info, **discovered_model_info}).items(): if k not in model_info or (model_info[k] is None and k in discovered_model_info): model_info[k] = v diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 748c91a4792..831041e90d3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3727,6 +3727,10 @@ def is_server_derived_pricing_key(key: str) -> bool: return key in SERVER_DERIVED_PRICING_FIELDS or ABOVE_THRESHOLD_COST_KEY_PATTERN.search(key) is not None +PRICING_OVERRIDES_KEY: Final = "pricing_overrides" +COST_MAP_LOOKUP_KEY: Final = "key" + + def without_server_derived_pricing(model_info: Mapping[str, Any]) -> Mapping[str, Any]: """Drop the pricing ``/model/info`` derives for display, keeping everything else. @@ -3736,7 +3740,32 @@ def without_server_derived_pricing(model_info: Mapping[str, Any]) -> Mapping[str deployment at that day's price where no cost map refresh can reach it. A deployment's own pricing belongs on ``litellm_params``, which is unaffected. """ - return MappingProxyType({k: v for k, v in model_info.items() if not is_server_derived_pricing_key(k)}) + return MappingProxyType( + {k: v for k, v in model_info.items() if k != PRICING_OVERRIDES_KEY and not is_server_derived_pricing_key(k)} + ) + + +def echoed_cost_map_pricing_fields(model_info: Mapping[str, Any]) -> tuple[str, ...]: + """Pricing fields a stored ``model_info`` blob copied from a ``/model/info`` response. + + Only ``litellm.get_model_info`` emits ``key`` (the resolved cost-map entry), so a stored + blob carrying it alongside pricing fields holds the cost map as it stood on the day the + row was saved, not a price anyone typed. Rows saved before 1.102 through the Admin UI + edit form look exactly like this, and a price typed into ``litellm_params`` never does. + """ + if COST_MAP_LOOKUP_KEY not in model_info: + return () + return tuple(sorted(k for k in model_info if is_server_derived_pricing_key(k))) + + +def pricing_override_fields(*sources: Mapping[str, Any]) -> tuple[str, ...]: + return tuple( + sorted( + frozenset( + k for source in sources for k, v in source.items() if v is not None and is_server_derived_pricing_key(k) + ) + ) + ) # Server-controlled fields that bound or drive an interceptor's agentic loop diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index d1fe88df26c..078aea04e03 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -3709,6 +3709,31 @@ class TestModelInfoServerDerivedPricingFilter: assert field not in info, f"{field} was persisted as a per-deployment override" assert field not in params + def test_echoed_pricing_overrides_report_is_not_persisted(self): + """LIT-8064. `/model/info` reports which pricing fields a deployment overrides; a + client echoing that response back must not store the report as a field.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-report-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + model_info=ModelInfo(id="dep-report-0", access_groups=["prod"], pricing_overrides=[]), + ), + ) + + info = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert "pricing_overrides" not in info + def test_tiered_above_threshold_pricing_is_dropped(self): """Tiered rates ride `get_model_info` on a pattern match and are declared on no model, so a filter built only from the declared pricing fields would miss them.""" diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 42cd6e4ed78..fef9d1bd534 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -16,7 +16,7 @@ import re from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime -from types import SimpleNamespace +from types import MappingProxyType, SimpleNamespace from typing import Any, Dict, Final from unittest.mock import AsyncMock, MagicMock @@ -2633,6 +2633,82 @@ def test_ProxyConfig_get_model_info_with_id_returns_router_model_info(): assert snapshot == {"id": "m-1", "db_model": True, "blocked": False} +PINNED_MODEL_INFO: Final = MappingProxyType( + { + "id": "pinned-row", + "key": "gpt-5.6", + "mode": "chat", + "access_groups": ["prod"], + "input_cost_per_token": 4e-06, + "output_cost_per_token": 2e-05, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + } +) + + +def test_ProxyConfig_get_model_info_with_id_ignores_cost_map_pricing_echoed_into_model_info(): + """LIT-8064. A pre-1.102 Admin UI save wrote the whole ``/model/info`` response back into + the row's ``model_info``, cost-map pricing included. Only that response carries ``key``, so + a stored blob with it holds a copy of the map, not a price anyone typed, and the deployment + must keep following the live cost map.""" + pc = ProxyConfig() + model = SimpleNamespace(model_id="pinned-row", model_info=dict(PINNED_MODEL_INFO), blocked=False) + out = pc.get_model_info_with_id(model=model, db_model=True).model_dump(exclude_none=True) + assert out["access_groups"] == ["prod"] + assert out["mode"] == "chat" + for field in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost_above_272k_tokens"): + assert field not in out, f"{field} still pins the deployment to the cost map of the day it was saved" + + +def test_ProxyConfig_get_model_info_with_id_keeps_pricing_typed_into_model_info(): + """A custom-priced deployment the cost map does not know never got ``key``, so its + ``model_info`` pricing is the operator's own and stays.""" + pc = ProxyConfig() + model = SimpleNamespace( + model_id="custom-row", + model_info={"id": "custom-row", "input_cost_per_token": 7e-06, "output_cost_per_token": 9e-06}, + blocked=False, + ) + out = pc.get_model_info_with_id(model=model, db_model=True).model_dump(exclude_none=True) + assert (out["input_cost_per_token"], out["output_cost_per_token"]) == (7e-06, 9e-06) + + +def test_ProxyConfig__add_deployment_pinned_row_follows_the_cost_map_across_reloads(monkeypatch, local_model_cost_map): + """The customer's symptom end to end: a row pinned before 1.102 must bill at the live cost + map price on boot and again after Reload Price Data, while a price typed on + ``litellm_params`` keeps overriding it.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.decrypt_value_helper", + lambda value, key, return_original_value: value, + ) + router = litellm.Router(model_list=[]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + pinned = SimpleNamespace( + model_id="pinned-row", + model_name="gpt-5.6", + model_info=dict(PINNED_MODEL_INFO), + litellm_params={"model": "openai/gpt-5.6", "api_key": "sk-test"}, + blocked=False, + ) + typed = SimpleNamespace( + model_id="typed-row", + model_name="gpt-5.6-typed", + model_info={"id": "typed-row", "key": "gpt-5.6", "input_cost_per_token": 4e-06}, + litellm_params={"model": "openai/gpt-5.6", "api_key": "sk-test", "input_cost_per_token": 3e-06}, + blocked=False, + ) + + assert ProxyConfig()._add_deployment(db_models=[pinned, typed]) == 2 + + monkeypatch.setitem(litellm.model_cost["gpt-5.6"], "input_cost_per_token", 1e-06) + router._replay_model_cost_registrations() + + assert litellm.model_cost.get("pinned-row", {}).get("input_cost_per_token") is None + assert router.get_deployment(model_id="pinned-row").model_info.input_cost_per_token is None + assert litellm.get_model_info("openai/gpt-5.6")["input_cost_per_token"] == 1e-06 + assert litellm.model_cost["typed-row"]["input_cost_per_token"] == 3e-06 + + def test_ProxyConfig_get_model_info_with_id_missing_model_id_raises(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) pc = ProxyConfig() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py index a1cf838ab6b..1ef35811372 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py @@ -286,6 +286,48 @@ def test_get_proxy_model_info_surfaces_supports_parallel_function_calling(local_ assert enriched["model_info"]["supports_parallel_function_calling"] is True +def _enriched_model_info(monkeypatch, litellm_params: dict, model_info: dict) -> dict: + monkeypatch.setattr(proxy_server, "llm_router", None) + enriched: Final = proxy_server._get_proxy_model_info( + model={"model_name": "gpt-5.6", "litellm_params": litellm_params, "model_info": model_info} + ) + return enriched["model_info"] + + +def test_get_proxy_model_info_reports_no_pricing_overrides_for_a_cost_map_priced_deployment( + monkeypatch, local_model_cost_map +): + """LIT-8064. A deployment with no price of its own follows the cost map, and ``/model/info`` + says so with an empty ``pricing_overrides``.""" + info = _enriched_model_info(monkeypatch, {"model": "openai/gpt-5.6"}, {"id": "dep-synced", "db_model": True}) + assert info["pricing_overrides"] == () + assert info["input_cost_per_token"] == litellm.model_cost["gpt-5.6"]["input_cost_per_token"] + + +def test_get_proxy_model_info_shows_litellm_params_pricing_and_names_it_as_an_override( + monkeypatch, local_model_cost_map +): + """A price on ``litellm_params`` is what the deployment bills at, so the model page shows that + value rather than the cost map's and lists the field under ``pricing_overrides``.""" + info = _enriched_model_info( + monkeypatch, + {"model": "openai/gpt-5.6", "input_cost_per_token_batches": 1e-09}, + {"id": "dep-batches", "db_model": True}, + ) + assert info["pricing_overrides"] == ("input_cost_per_token_batches",) + assert info["input_cost_per_token_batches"] == 1e-09 + assert info["input_cost_per_token"] == litellm.model_cost["gpt-5.6"]["input_cost_per_token"] + + +def test_get_proxy_model_info_names_config_model_info_pricing_as_an_override(monkeypatch, local_model_cost_map): + """Pricing declared under ``model_info`` in config.yaml overrides the cost map too.""" + info = _enriched_model_info( + monkeypatch, {"model": "openai/gpt-5.6"}, {"id": "dep-config", "db_model": False, "output_cost_per_token": 7e-06} + ) + assert info["pricing_overrides"] == ("output_cost_per_token",) + assert info["output_cost_per_token"] == 7e-06 + + def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monkeypatch): from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.proxy.auth import model_checks diff --git a/ui/litellm-dashboard/src/components/model_dashboard/types.ts b/ui/litellm-dashboard/src/components/model_dashboard/types.ts index f580e31a933..47c7eab8ba6 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/types.ts +++ b/ui/litellm-dashboard/src/components/model_dashboard/types.ts @@ -14,6 +14,7 @@ export interface ModelInfo { blocked?: boolean; team_public_model_name?: string; key?: string; + pricing_overrides?: string[]; } export interface LiteLLMParams { diff --git a/ui/litellm-dashboard/src/components/molecules/models/ModelPricingSummary.test.tsx b/ui/litellm-dashboard/src/components/molecules/models/ModelPricingSummary.test.tsx index 921a824e671..9a9bdfc6490 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/ModelPricingSummary.test.tsx +++ b/ui/litellm-dashboard/src/components/molecules/models/ModelPricingSummary.test.tsx @@ -52,4 +52,31 @@ describe("ModelPricingSummary", () => { expect(screen.getByText("-")).toBeInTheDocument(); expect(screen.queryByText(/\$/)).not.toBeInTheDocument(); }); + + it("names the fields a deployment prices itself", () => { + render( + , + ); + expect(screen.getByText("Custom pricing")).toBeInTheDocument(); + expect( + screen.getByText("Overrides the model cost map for input_cost_per_token, output_cost_per_token"), + ).toBeInTheDocument(); + }); + + it("says the price follows the cost map when nothing is overridden", () => { + render(); + expect(screen.getByText("Follows the model cost map")).toBeInTheDocument(); + expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument(); + }); + + it("says nothing about the source when the proxy did not report it", () => { + render(); + expect(screen.queryByText(/cost map/)).not.toBeInTheDocument(); + expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/molecules/models/ModelPricingSummary.tsx b/ui/litellm-dashboard/src/components/molecules/models/ModelPricingSummary.tsx index 10b37c6100c..facbe73eed0 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/ModelPricingSummary.tsx +++ b/ui/litellm-dashboard/src/components/molecules/models/ModelPricingSummary.tsx @@ -1,10 +1,26 @@ -import { ModelData } from "@/components/model_dashboard/types"; +import { ModelData, ModelInfo } from "@/components/model_dashboard/types"; +import { Badge } from "@/components/ui/badge"; import { formatPerSecondCost } from "@/utils/dataUtils"; type PricingFields = Pick< ModelData, "input_cost" | "output_cost" | "output_cost_per_second" | "output_cost_per_second_tiers" ->; +> & { model_info?: Pick }; + +function PricingSource({ overrides }: { overrides: string[] | undefined }) { + if (overrides === undefined) return null; + if (overrides.length === 0) { + return

Follows the model cost map

; + } + return ( +

+ + Custom pricing + + Overrides the model cost map for {overrides.join(", ")} +

+ ); +} export function ModelPricingSummary({ model }: { model: PricingFields }) { const perSecond = model.output_cost_per_second; @@ -26,6 +42,7 @@ export function ModelPricingSummary({ model }: { model: PricingFields }) { Output ({resolution}): {formatPerSecondCost(cost)}

))} + ); }