Merge pull request #40878 from BerriAI/litellm_null_cost_unpriced_deployments

fix(router): report null cost for unpriced deployments instead of 0

litellm/proxy/proxy_server.py

@@ -154,7 +154,7 @@ from litellm.types.utils import (
     TextCompletionResponse,
     TokenCountResponse,
 )
-from litellm.utils import load_credentials_from_list
+from litellm.utils import cost_map_omits_token_price, load_credentials_from_list
 
 if TYPE_CHECKING:
     from aiohttp import ClientSession
@@ -13632,9 +13632,10 @@ def _enrich_model_info_with_litellm_data(
     discovered_model_info: Final = (
         llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({})
     )
+    unpriced: Final = cost_map_omits_token_price(model_info.get("id"), litellm_model_info.get("key"))
     for k, v in MappingProxyType({**litellm_model_info, **discovered_model_info}).items():
         if k not in model_info or (model_info[k] is None and k in discovered_model_info):
-            model_info[k] = v
+            model_info[k] = None if unpriced and k in ("input_cost_per_token", "output_cost_per_token") else v
     model["model_info"] = model_info
     # don't return the api key / vertex credentials
     # don't return the llm credentials
@@ -15070,45 +15071,7 @@ def _translate_model_name_for_response(model: dict) -> dict:
 
 
 def _get_proxy_model_info(model: dict) -> dict:
-    # provided model_info in config.yaml
-    model_info: Final = model.get("model_info", {})
-
-    # read litellm model_prices_and_context_window.json to get the following:
-    # input_cost_per_token, output_cost_per_token, max_tokens
-    litellm_model_info = get_litellm_model_info(model=model)
-
-    # 2nd pass on the model, try seeing if we can find model in litellm model_cost map
-    if litellm_model_info == {}:
-        # use litellm_param model_name to get model_info
-        litellm_params = model.get("litellm_params", {})
-        litellm_model = litellm_params.get("model", None)
-        try:
-            litellm_model_info = litellm.get_model_info(model=litellm_model)
-        except Exception:
-            litellm_model_info = {}
-    # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map
-    if litellm_model_info == {}:
-        # use litellm_param model_name to get model_info
-        litellm_params = model.get("litellm_params", {})
-        litellm_model = litellm_params.get("model", None)
-        split_model: Final = litellm_model.split("/")
-        if len(split_model) > 0:
-            litellm_model = split_model[-1]
-        try:
-            litellm_model_info = litellm.get_model_info(model=litellm_model, custom_llm_provider=split_model[0])
-        except Exception:
-            litellm_model_info = {}
-    discovered_model_info: Final = (
-        llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({})
-    )
-    for k, v in MappingProxyType({**litellm_model_info, **discovered_model_info}).items():
-        if k not in model_info or (model_info[k] is None and k in discovered_model_info):
-            model_info[k] = v
-    model["model_info"] = model_info
-    # don't return the llm credentials
-    model = remove_sensitive_info_from_deployment(deployment_dict=model, excluded_keys={"litellm_credential_name"})
-
-    return _translate_model_name_for_response(model)
+    return _translate_model_name_for_response(_enrich_model_info_with_litellm_data(model=model, llm_router=llm_router))
 
 
 def _model_info_json_response(data: Sequence[Mapping[str, object]] | Mapping[str, object]) -> Response:

litellm/router.py

@@ -10587,11 +10587,12 @@ class Router:
         2. If not, check if litellm model name is in model info
         3. If not, return None
         """
-        from litellm.utils import _update_dictionary
+        from litellm.utils import _update_dictionary, cost_map_omits_token_price
 
         model_info: ModelInfo | None = None
         custom_model_info: dict | None = None
         litellm_model_name_model_info: ModelInfo | None = None
+        base_model_key: str | None = None
 
         try:
             custom_model_info = (
@@ -10618,6 +10619,7 @@ class Router:
                     ## update litellm model info with base model info
                     base_model_info: Final = copy.deepcopy(litellm.get_model_info(model=base_model))
                     if base_model_info is not None:
+                        base_model_key = base_model_info.get("key")
                         # Base model provides defaults, custom model info overrides
                         custom_model_info = _update_dictionary(
                             cast(dict, base_model_info),
@@ -10645,6 +10647,15 @@ class Router:
             # custom_model_info already includes base_model defaults at this point, if applicable
             model_info = cast(ModelInfo, custom_model_info)
 
+        if model_info is None:
+            return None
+        builtin_key: Final = (
+            litellm_model_name_model_info.get("key") if litellm_model_name_model_info is not None else None
+        )
+        if cost_map_omits_token_price(model_id, builtin_key, base_model_key):
+            return cast(  # cast-ok: TypedDict spread with overridden keys loses its type
+                ModelInfo, {**model_info, "input_cost_per_token": None, "output_cost_per_token": None}
+            )
         return model_info
 
     def _set_model_group_info(self, model_group: str, user_facing_model_group_name: str) -> ModelGroupInfo | None:

litellm/utils.py

@@ -3156,6 +3156,22 @@ def reapply_runtime_model_cost_registrations() -> None:
         register_model(model_cost=dict(_runtime_registered_model_cost))  # mutable-ok: snapshot, replay rewrites it
 
 
+def cost_map_omits_token_price(*keys: object) -> bool:
+    """Whether the raw ``litellm.model_cost`` entries under ``keys`` exist but none carries a per-token price.
+
+    ``get_model_info`` substitutes 0 for a missing price, which reads exactly like a declared
+    zero. Surfaces that report pricing use this to keep an unpriced deployment at ``None``.
+    """
+    entries: Final = tuple(
+        entry
+        for entry in (litellm.model_cost.get(key) for key in keys if isinstance(key, str))
+        if isinstance(entry, dict)
+    )
+    return len(entries) > 0 and not any(
+        "input_cost_per_token" in entry or "output_cost_per_token" in entry for entry in entries
+    )
+
+
 def register_model(
     model_cost: str | dict,
     *,

tests/test_litellm/proxy/proxy_server/test_routes_model_info.py

@@ -286,6 +286,93 @@ def test_get_proxy_model_info_surfaces_supports_parallel_function_calling(local_
     assert enriched["model_info"]["supports_parallel_function_calling"] is True
 
 
+def test_model_info_reports_null_cost_for_unpriced_deployment_and_zero_for_declared_zero():
+    """A deployment configured with no cost fields must not surface the 0 that ``get_model_info``
+    defaults to, since the zero-cost budget bypass only honours a declared zero. The declared zero
+    and a catalog price still come through."""
+    import litellm
+
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "vllm-unpriced",
+                "litellm_params": {"model": "openai/vllm-unpriced", "api_key": "x", "api_base": "http://vllm"},
+            },
+            {
+                "model_name": "vllm-free",
+                "litellm_params": {
+                    "model": "openai/vllm-free",
+                    "api_key": "x",
+                    "api_base": "http://vllm",
+                    "input_cost_per_token": 0,
+                    "output_cost_per_token": 0,
+                },
+            },
+            {"model_name": "gpt-priced", "litellm_params": {"model": "gpt-4o", "api_key": "x"}},
+        ]
+    )
+
+    def enriched_cost(model_name: str) -> tuple:
+        deployment = router.get_model_list(model_name=model_name)[0]
+        info = proxy_server._enrich_model_info_with_litellm_data({**deployment, "model_info": dict(deployment["model_info"])})["model_info"]
+        return info.get("input_cost_per_token"), info.get("output_cost_per_token")
+
+    assert enriched_cost("vllm-unpriced") == (None, None)
+    assert enriched_cost("vllm-free") == (0, 0)
+    input_cost, output_cost = enriched_cost("gpt-priced")
+    assert input_cost > 0 and output_cost > 0
+
+
+def test_model_info_id_lookup_reports_the_same_cost_as_the_list(
+    client: TestClient,
+    auth_as: Callable[[], AbstractContextManager[object]],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """``GET /model/info?litellm_model_id=`` must agree with the ``GET /model/info`` list, so an
+    unpriced deployment cannot read as null in the list and as free on the id lookup."""
+    monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
+    declared: Final = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002}
+    free: Final = {"input_cost_per_token": 0, "output_cost_per_token": 0}
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": name,
+                "litellm_params": {"model": f"openai/{name}", "api_key": "x", "api_base": "http://vllm", **costs},
+                "model_info": {"id": f"{name}-id"},
+            }
+            for name, costs in (("vllm-unpriced", {}), ("vllm-free", free), ("vllm-priced", declared))
+        ]
+    )
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
+    monkeypatch.setattr(proxy_server, "user_model", None)
+
+    def costs_of(response: httpx.Response) -> dict[str, tuple[object, object]]:
+        assert response.status_code == 200, response.text
+        return {
+            row["model_info"]["id"]: (
+                row["model_info"].get("input_cost_per_token"),
+                row["model_info"].get("output_cost_per_token"),
+            )
+            for row in response.json()["data"]
+        }
+
+    with auth_as():
+        listed: Final = costs_of(client.get("/model/info"))
+        by_id: Final = {
+            model_id: costs_of(client.get("/model/info", params={"litellm_model_id": model_id}))[model_id]
+            for model_id in listed
+        }
+
+    assert listed == {
+        "vllm-unpriced-id": (None, None),
+        "vllm-free-id": (0, 0),
+        "vllm-priced-id": (declared["input_cost_per_token"], declared["output_cost_per_token"]),
+    }
+    assert by_id == listed
+    _invalidate_model_cost_lowercase_map()
+
+
 def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monkeypatch):
     from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
     from litellm.proxy.auth import model_checks

tests/test_litellm/test_router.py

@@ -1888,6 +1888,53 @@ def test_model_group_info_cost_none_when_db_model_info_has_no_cost():
         assert result.output_cost_per_token is None
 
 
+def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declared():
+    """A deployment with no cost fields anywhere must report None, not the 0 that
+    get_model_info defaults to, so the reported price matches what the zero-cost
+    budget bypass accepts. A deployment declaring 0 keeps reporting 0."""
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "vllm-unpriced",
+                "litellm_params": {
+                    "model": "openai/my-vllm-unpriced",
+                    "api_key": "fake",
+                    "api_base": "http://localhost:8000/v1",
+                },
+            },
+            {
+                "model_name": "vllm-free",
+                "litellm_params": {
+                    "model": "openai/my-vllm-free",
+                    "api_key": "fake",
+                    "api_base": "http://localhost:8000/v1",
+                    "input_cost_per_token": 0,
+                    "output_cost_per_token": 0,
+                },
+            },
+            {
+                "model_name": "gpt-priced",
+                "litellm_params": {"model": "gpt-4o", "api_key": "fake"},
+            },
+        ]
+    )
+
+    unpriced = router.get_model_group_info(model_group="vllm-unpriced")
+    assert unpriced is not None
+    assert unpriced.input_cost_per_token is None
+    assert unpriced.output_cost_per_token is None
+
+    free = router.get_model_group_info(model_group="vllm-free")
+    assert free is not None
+    assert free.input_cost_per_token == 0
+    assert free.output_cost_per_token == 0
+
+    priced = router.get_model_group_info(model_group="gpt-priced")
+    assert priced is not None
+    assert priced.input_cost_per_token is not None and priced.input_cost_per_token > 0
+    assert priced.output_cost_per_token is not None and priced.output_cost_per_token > 0
+
+
 @pytest.mark.parametrize(
     "value,expected",
     [