fix(router): list tier and batch prices at the most any deployment bills, and scope team deployment prices

A deployment with no flex, priority or batch price still bills those calls at its base rate (half of it for batch input and output), so the group counts that rate through billing's own lookup. A team-owned deployment's prices beyond input and output are listed only for that team's keys.

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -910,7 +910,9 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value:
     return 0.0
 
 
-def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None:
+def get_cost_per_unit(
+    model_info: Mapping[str, object], cost_key: str, default_value: float | None = 0.0
+) -> float | None:
     # Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
     cost_per_unit: Final = model_info.get(cost_key)
     if isinstance(cost_per_unit, float):
@@ -954,6 +956,19 @@ def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float
 _get_cost_per_unit = get_cost_per_unit
 
 
+_HALF_PRICED_BATCH_KEYS: Final = frozenset(
+    {f"input_cost_per_token{_BATCH_KEY_SUFFIX}", f"output_cost_per_token{_BATCH_KEY_SUFFIX}"}
+)
+
+
+def billed_rate(model_info: Mapping[str, object], cost_key: str) -> float | None:
+    rate: Final = get_cost_per_unit(model_info, cost_key, default_value=None)
+    if rate is not None or cost_key not in _HALF_PRICED_BATCH_KEYS:
+        return rate
+    standard_rate: Final = get_cost_per_unit(model_info, cost_key.removesuffix(_BATCH_KEY_SUFFIX), default_value=None)
+    return None if standard_rate is None else standard_rate / 2
+
+
 def deployment_pricing(model_info: ModelInfo | None) -> ModelInfo | None:
     """The prices a deployment sets itself, as floats; None when it sets none that parse."""
     if model_info is None:

litellm/router.py

@@ -144,7 +144,7 @@ from litellm.litellm_core_utils.hidden_params import (
 )
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
-from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES
+from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES, billed_rate
 from litellm.litellm_core_utils.ptu_pricing import (
     PTU_COST_ATTRIBUTION_ENV_VAR,
     declares_ptu,
@@ -732,9 +732,38 @@ def _billed_pricing_entry(deployment: Mapping[str, object], model_info: Mapping[
     return model_info if catalog_entry is None else {**catalog_entry, **model_info}
 
 
-def _deployment_prices(deployment: Mapping[str, object], model_info: Mapping[str, object]) -> Mapping[str, float]:
+def _deployment_price_entry(
+    deployment: Mapping[str, object], model_info: Mapping[str, object] | None
+) -> Mapping[str, object]:
+    if model_info is None:
+        return _configured_model_info(deployment)
     token_prices: Final = {key: model_info.get(key) for key in ("input_cost_per_token", "output_cost_per_token")}
-    return model_info_prices({**_billed_pricing_entry(deployment, model_info), **token_prices})
+    return {**_billed_pricing_entry(deployment, model_info), **token_prices}
+
+
+@dataclass(frozen=True, slots=True)
+class _GroupDeployment:
+    model_info: ModelMapInfo | None
+    price_entry: Mapping[str, object]
+    team_may_use: bool
+
+    def listed_prices(self) -> Mapping[str, float]:
+        prices: Final = model_info_prices(self.price_entry)
+        if self.team_may_use:
+            return prices
+        return MappingProxyType({key: price for key, price in prices.items() if key in ModelGroupInfo.model_fields})
+
+
+def _highest_billed_rate(cost_key: str, price_entries: Sequence[Mapping[str, object]]) -> float | None:
+    rates: Final = (price_as_float(billed_rate(entry, cost_key)) for entry in price_entries)
+    return max((rate for rate in rates if rate is not None), default=None)
+
+
+def _highest_billed_prices(
+    listed_keys: Iterable[str], price_entries: Sequence[Mapping[str, object]]
+) -> Mapping[str, float]:
+    rates: Final = ((key, _highest_billed_rate(key, price_entries)) for key in listed_keys)
+    return MappingProxyType({key: rate for key, rate in rates if rate is not None})
 
 
 def _every_deployment_refused(
@@ -12652,17 +12681,12 @@ class Router:
             (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)),
             None,
         )
-        for model in model_list:
-            is_match = False
-            if (
-                "model_name" in model
-                and model["model_name"] == model_group
-                or "model_name" in model
-                and self.pattern_router.route(model_group) is not None
-            ):  # exact match
-                is_match = True
-
-            if not is_match:
+        group_deployments: Final = tuple(
+            self._group_deployment(model, team_id) if self._is_model_group_member(model, model_group) else None
+            for model in model_list
+        )
+        for model, group_deployment in zip(model_list, group_deployments, strict=True):
+            if group_deployment is None:
                 continue
             # model in model group found #
             litellm_params = LiteLLM_Params(**model["litellm_params"])
@@ -12703,22 +12727,9 @@ class Router:
             if _deployment_otpm is None:
                 _deployment_otpm = model_info_dict.get("otpm", None)
 
-            # get model info
-            try:
-                model_id = model_info_dict.get("id", None)
-                if model_id is not None:
-                    model_info = self.get_deployment_model_info(model_id=model_id, model_name=litellm_params.model)
-                else:
-                    model_info = None
-            except Exception:
-                model_info = None
-
+            model_info = group_deployment.model_info
             deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict)
-            deployment_prices = (
-                model_info_prices(_configured_model_info(model))
-                if model_info is None
-                else _deployment_prices(model, model_info)
-            )
+            deployment_prices = group_deployment.listed_prices()
 
             # get llm provider
             litellm_model, llm_provider = self._model_group_llm_provider(litellm_params)
@@ -12835,7 +12846,7 @@ class Router:
                                 deployment_supported_endpoints(
                                     model_info.get("supported_endpoints"), model, llm_provider
                                 )
-                                if team_may_use_deployment(_configured_model_info(model).get("team_id"), team_id)
+                                if group_deployment.team_may_use
                                 else ()
                             ),
                         )
@@ -12899,7 +12910,39 @@ class Router:
             if configurable_clientside_auth_params is not None:
                 model_group_info.configurable_clientside_auth_params = configurable_clientside_auth_params
 
-        return model_group_info
+            usable_price_entries: Final = tuple(
+                deployment.price_entry
+                for deployment in group_deployments
+                if deployment is not None and deployment.team_may_use
+            )
+            return model_group_info.with_higher_prices(
+                _highest_billed_prices(model_info_prices(model_group_info.model_dump()), usable_price_entries)
+            )
+
+        return None
+
+    def _is_model_group_member(self, deployment: DeploymentTypedDict, model_group: str) -> bool:
+        return "model_name" in deployment and (
+            deployment["model_name"] == model_group or self.pattern_router.route(model_group) is not None
+        )
+
+    def _deployment_model_info(self, deployment: DeploymentTypedDict) -> ModelMapInfo | None:
+        model_id: Final = _configured_model_info(deployment).get("id")
+        model_name: Final = deployment["litellm_params"].get("model")
+        if not isinstance(model_id, str) or model_name is None:
+            return None
+        try:
+            return self.get_deployment_model_info(model_id=model_id, model_name=model_name)
+        except Exception:
+            return None
+
+    def _group_deployment(self, deployment: DeploymentTypedDict, team_id: str | None) -> _GroupDeployment:
+        model_info: Final = self._deployment_model_info(deployment)
+        return _GroupDeployment(
+            model_info=model_info,
+            price_entry=_deployment_price_entry(deployment, model_info),
+            team_may_use=team_may_use_deployment(_configured_model_info(deployment).get("team_id"), team_id),
+        )
 
     def get_model_group_info(
         self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None

tests/code_coverage_tests/router_code_coverage.py

@@ -175,8 +175,14 @@ ignored_function_names = [
     "_responses_continuation",  # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
     "_responses_continuation_ceilings",  # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
     "_team_deployments_named",  # Tested through the team lookups in the concurrent delete tests in test_router.py
-    "_deployment_prices",  # Read by _set_model_group_info, tested through the cost map model group info test in test_router.py
-    "_billed_pricing_entry",  # Read by _deployment_prices, tested through the custom-priced model group info test in test_router.py
+    "_deployment_price_entry",  # Read by _group_deployment, tested through the cost map model group info test in test_router.py
+    "_billed_pricing_entry",  # Read by _deployment_price_entry, tested through the custom-priced model group info test in test_router.py
+    "listed_prices",  # Read by _set_model_group_info, tested through the team-scoped model group info test in test_router.py
+    "_highest_billed_rate",  # Read by _highest_billed_prices, tested through the tier and batch model group info test in test_router.py
+    "_highest_billed_prices",  # Read by _set_model_group_info, tested through the tier and batch model group info test in test_router.py
+    "_is_model_group_member",  # Read by _set_model_group_info, tested through every model group info test in test_router.py
+    "_deployment_model_info",  # Read by _group_deployment, tested through the cost map model group info test in test_router.py
+    "_group_deployment",  # Read by _set_model_group_info, tested through the team-scoped model group info test in test_router.py
     "priority_header_update",  # Tested through the priority header tests in test_common_request_processing.py
     "_drops_params",  # Read by split_request_priority and priority_header_update, tested through both
     "_responses_event_announces_background_run",  # Tested through the background Responses stream tests in test_router.py

tests/unit/test_router/test_router.py

@@ -2725,6 +2725,128 @@ def test_model_group_info_lists_a_custom_priced_deployment_only_at_the_rates_it_
     assert "input_cost_per_token_above_200k_tokens" not in listed
 
 
+def test_model_group_info_lists_tier_and_batch_prices_at_the_most_any_deployment_bills(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    from litellm.cost_calculator import batch_cost_calculator
+    from litellm.utils import _invalidate_model_cost_lowercase_map
+
+    backends: Final = ("made-up-tiered-backend", "made-up-untiered-backend")
+    monkeypatch.setattr(
+        litellm,
+        "model_cost",
+        {
+            **litellm.model_cost,
+            backends[0]: {
+                "litellm_provider": "openai",
+                "mode": "chat",
+                "input_cost_per_token": 1e-06,
+                "output_cost_per_token": 4e-06,
+                "input_cost_per_token_flex": 5e-07,
+                "output_cost_per_token_flex": 2e-06,
+                "input_cost_per_token_batches": 4e-07,
+                "output_cost_per_token_batches": 2e-06,
+            },
+            backends[1]: {
+                "litellm_provider": "openai",
+                "mode": "chat",
+                "input_cost_per_token": 3e-06,
+                "output_cost_per_token": 1.2e-05,
+            },
+        },
+    )
+    _invalidate_model_cost_lowercase_map()
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": "mixed-backend-group",
+                "litellm_params": {"model": f"openai/{backend}", "api_key": "fake"},
+                "model_info": {"id": backend},
+            }
+            for backend in backends
+        ]
+    )
+    tokens: Final = 1_000_000
+    usage: Final = Usage(prompt_tokens=tokens, completion_tokens=tokens, total_tokens=2 * tokens)
+    flex_bills: Final = tuple(
+        litellm.cost_per_token(
+            model=backend,
+            custom_llm_provider="openai",
+            prompt_tokens=tokens,
+            completion_tokens=tokens,
+            service_tier="flex",
+        )
+        for backend in backends
+    )
+    batch_bills: Final = tuple(
+        batch_cost_calculator(usage, model=backend, custom_llm_provider="openai") for backend in backends
+    )
+    highest_bills: Final = (
+        max(prompt_cost for prompt_cost, _ in flex_bills),
+        max(completion_cost for _, completion_cost in flex_bills),
+        max(prompt_cost for prompt_cost, _ in batch_bills),
+        max(completion_cost for _, completion_cost in batch_bills),
+    )
+
+    info: Final = router.get_model_group_info(model_group="mixed-backend-group")
+
+    assert info is not None
+    listed: Final = info.model_dump()
+    listed_keys: Final = (
+        "input_cost_per_token_flex",
+        "output_cost_per_token_flex",
+        "input_cost_per_token_batches",
+        "output_cost_per_token_batches",
+    )
+    assert tuple(listed[key] * tokens for key in listed_keys) == pytest.approx(highest_bills)
+
+
+@pytest.mark.parametrize(("team_id", "listed_cache_read_cost"), [(None, 1e-07), ("team-b", 1e-07), ("team-a", 9e-06)])
+def test_model_group_info_lists_a_team_deployment_price_only_to_that_team(
+    monkeypatch: pytest.MonkeyPatch, team_id: str | None, listed_cache_read_cost: float
+):
+    from litellm.utils import _invalidate_model_cost_lowercase_map
+
+    cache_read_costs: Final = {"made-up-shared-backend": 1e-07, "made-up-team-backend": 9e-06}
+    monkeypatch.setattr(
+        litellm,
+        "model_cost",
+        {
+            **litellm.model_cost,
+            **{
+                backend: {
+                    "litellm_provider": "openai",
+                    "mode": "chat",
+                    "input_cost_per_token": 1e-06,
+                    "output_cost_per_token": 4e-06,
+                    "cache_read_input_token_cost": cache_read_cost,
+                }
+                for backend, cache_read_cost in cache_read_costs.items()
+            },
+        },
+    )
+    _invalidate_model_cost_lowercase_map()
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": "team-scoped-group",
+                "litellm_params": {"model": "openai/made-up-shared-backend", "api_key": "fake"},
+                "model_info": {"id": "shared-deployment"},
+            },
+            {
+                "model_name": "team-scoped-group",
+                "litellm_params": {"model": "openai/made-up-team-backend", "api_key": "fake"},
+                "model_info": {"id": "team-a-deployment", "team_id": "team-a"},
+            },
+        ]
+    )
+
+    info: Final = router.get_model_group_info(model_group="team-scoped-group", team_id=team_id)
+
+    assert info is not None
+    assert info.model_dump().get("cache_read_input_token_cost") == listed_cache_read_cost
+
+
 def test_model_group_info_still_validates_from_attributes():
     source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)