From 51462ad63d8ece498fbeaef5d2042095c35ea665 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Sat, 13 Jun 2026 16:48:07 +0000 Subject: [PATCH] feat(router): backfill missing cost fields from canonical entry for known models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a deployment is added via the dashboard /model/new form (or loaded from DB), the form exposes only input_cost_per_token and output_cost_per_token — none of the cache rate fields are surfaced. Router previously registered the resulting partial dict directly into litellm.model_cost under the deployment UUID, leaving cache_read_input_token_cost and cache_creation_input_token_cost absent. Downstream effect: _select_model_name_for_cost_calc prefers the deployment-UUID entry when custom_pricing=True. With cache rate fields missing, the cost calculator silently dropped cache token charges — under-billing cache-heavy requests by ~93%. Operators were treating "Reload Price Data" as a fix. It worked by accident: replacing litellm.model_cost wholesale evicted the deployment-UUID entries, causing fall-through to the canonical bare model name entry. Multi-worker proxies experienced partial recovery because the reload broadcast (force_reload flag in LiteLLM_Config) is consumed by whichever worker polls first. Fix: Router._backfill_cost_fields_from_canonical — at registration, copy missing CustomPricingLiteLLMParams fields from litellm.model_cost[bare_model_name] into the dict that will be keyed by the deployment UUID. User-supplied values always win; this only fills slots the user didn't touch. Tests: 4 unit tests cover (1) known model backfills missing cache fields, (2) user-supplied overrides preserved, (3) unknown models silently skip backfill, (4) canonical entry missing fields don't raise. --- litellm/router.py | 54 ++++++ .../test_router_backfill_cost_fields.py | 167 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 tests/test_litellm/test_router_backfill_cost_fields.py diff --git a/litellm/router.py b/litellm/router.py index e54eadfb872..f40390098a6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7921,6 +7921,43 @@ class Router: if backend_value is not None: model_info[field] = backend_value + @staticmethod + def _backfill_cost_fields_from_canonical( + model_info_dict: dict, + litellm_params: dict, + ) -> None: + """Backfill missing CustomPricingLiteLLMParams fields in a + deployment-UUID model_cost entry from the canonical static + litellm.model_cost entry for the bare model name. + + The dashboard /model/new form only exposes input_cost_per_token + and output_cost_per_token. Without this backfill, deployments + added via the dashboard end up registered under their UUID with + cache_*_input_token_cost = None, and the cost calculator's + custom-pricing path silently drops cache token charges. + + Complements ``_inherit_builtin_cache_pricing`` above, which only + runs once a user has set ``input_cost_per_token`` and only fills + the 5 cache fields. This helper always runs and fills every + CustomPricingLiteLLMParams field, covering the dashboard + zero-cost-fields path. + + User-supplied values always win. Only known models are eligible + (those that have a static model_cost entry). Mutates + ``model_info_dict`` in place. + """ + bare_model_name = litellm_params.get("model") + if not bare_model_name: + return + canonical = litellm.model_cost.get(bare_model_name) + if not canonical: + return + for field in CustomPricingLiteLLMParams.model_fields.keys(): + if model_info_dict.get(field) is None: + value = canonical.get(field) + if value is not None: + model_info_dict[field] = value + def _create_deployment( self, deployment_info: dict, @@ -7955,6 +7992,15 @@ class Router: backend_model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, ) + # Backfill any remaining missing cost fields from the canonical + # static entry for the bare model name. User-supplied values + # and the cache backfill above always win; this only fills + # slots that are still None (e.g. dashboard-added deployments + # where the user set no cost fields). + self._backfill_cost_fields_from_canonical( + model_info_dict=_model_info, + litellm_params=_litellm_params, + ) ## REGISTER MODEL INFO IN LITELLM MODEL COST MAP model_id = deployment.model_info.id @@ -8699,6 +8745,14 @@ class Router: field_value = deployment.litellm_params.get(field) if field_value is not None: _model_info_dict[field] = field_value + # Backfill any missing cost fields from the canonical static entry + # for the bare model name. User-supplied values above always win; + # this only fills slots the user didn't touch (e.g. cache rate + # fields the /model/new dashboard form doesn't expose). + self._backfill_cost_fields_from_canonical( + model_info_dict=_model_info_dict, + litellm_params=deployment.litellm_params.model_dump(), + ) if _model_info_dict.get("input_cost_per_token") is not None: Router._inherit_builtin_cache_pricing( diff --git a/tests/test_litellm/test_router_backfill_cost_fields.py b/tests/test_litellm/test_router_backfill_cost_fields.py new file mode 100644 index 00000000000..f62f6f0ee88 --- /dev/null +++ b/tests/test_litellm/test_router_backfill_cost_fields.py @@ -0,0 +1,167 @@ +""" +Tests for Router._backfill_cost_fields_from_canonical. + +Background: the dashboard /model/new form exposes only input/output cost +fields. Without backfill, deployments added via the dashboard end up +registered under their UUID with cache_*_input_token_cost = None, and +the cost calculator's custom-pricing path silently drops cache token +charges. The backfill copies missing fields from +litellm.model_cost[bare_model_name] for known upstream models, while +preserving any value the user supplied explicitly. + +This file pins three behaviors: + 1. Known model + no user cache rates → backfilled from canonical + 2. Known model + explicit user cache rates → user values win + 3. Unknown model (no static entry) → no backfill, fields stay None +""" + +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +import litellm +from litellm import Router + +KNOWN_MODEL = "claude-haiku-4-5-20251001" +UNKNOWN_MODEL = "company-private/in-house-llm-not-in-static-map" + + +@pytest.fixture(autouse=True) +def _ensure_local_model_cost_map(monkeypatch): + """Use bundled JSON deterministically — never depend on a live fetch.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + +def _build_router(deployment_uuid: str, model: str, extra_params: dict | None = None): + """Build a single-deployment Router whose litellm_params mimics what + dashboard /model/new produces — only input/output rates, plus any + overrides callers want to add via ``extra_params``.""" + litellm_params: dict = { + "model": model, + "custom_llm_provider": "anthropic", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 5e-6, + } + if extra_params: + litellm_params.update(extra_params) + return Router( + model_list=[ + { + "model_name": "alias-for-test", + "litellm_params": litellm_params, + "model_info": {"id": deployment_uuid}, + } + ] + ) + + +def test_known_model_backfills_missing_cache_fields(): + """Dashboard added a known-upstream model with only input/output rates. + The UUID entry should be backfilled from the static map so cache + pricing applies on the cost-calc UUID path.""" + deployment_uuid = "backfill-known-model-uuid" + litellm.model_cost.pop(deployment_uuid, None) + + _build_router(deployment_uuid=deployment_uuid, model=KNOWN_MODEL) + + entry = litellm.model_cost.get(deployment_uuid, {}) + assert ( + entry.get("input_cost_per_token") == 1e-6 + ), "user-supplied input_cost_per_token should be preserved" + assert ( + entry.get("output_cost_per_token") == 5e-6 + ), "user-supplied output_cost_per_token should be preserved" + + canonical = litellm.model_cost[KNOWN_MODEL] + assert ( + entry.get("cache_read_input_token_cost") + == canonical["cache_read_input_token_cost"] + ), "cache_read_input_token_cost should be backfilled from canonical entry" + assert ( + entry.get("cache_creation_input_token_cost") + == canonical["cache_creation_input_token_cost"] + ), "cache_creation_input_token_cost should be backfilled from canonical entry" + + +def test_user_supplied_cache_rates_override_backfill(): + """If the user explicitly sets cache rates in litellm_params (e.g. + they negotiated a discounted gateway rate), those values must win + over the canonical static values.""" + deployment_uuid = "backfill-user-override-uuid" + litellm.model_cost.pop(deployment_uuid, None) + + user_cache_read_rate = 2e-7 # 2x the canonical Anthropic rate + user_cache_create_rate = 9.99e-7 + + _build_router( + deployment_uuid=deployment_uuid, + model=KNOWN_MODEL, + extra_params={ + "cache_read_input_token_cost": user_cache_read_rate, + "cache_creation_input_token_cost": user_cache_create_rate, + }, + ) + + entry = litellm.model_cost.get(deployment_uuid, {}) + assert ( + entry.get("cache_read_input_token_cost") == user_cache_read_rate + ), "user-supplied cache_read_input_token_cost must win over canonical" + assert ( + entry.get("cache_creation_input_token_cost") == user_cache_create_rate + ), "user-supplied cache_creation_input_token_cost must win over canonical" + + canonical = litellm.model_cost[KNOWN_MODEL] + assert ( + entry.get("cache_read_input_token_cost") + != canonical["cache_read_input_token_cost"] + ), "test geometry weak — user value happens to equal canonical" + + +def test_unknown_model_no_backfill(): + """Custom in-house model with no static map entry: backfill must + not invent values — cache fields stay None / absent.""" + deployment_uuid = "backfill-unknown-model-uuid" + litellm.model_cost.pop(deployment_uuid, None) + litellm.model_cost.pop(UNKNOWN_MODEL, None) + assert ( + UNKNOWN_MODEL not in litellm.model_cost + ), "test setup expects UNKNOWN_MODEL to be absent from static map" + + _build_router(deployment_uuid=deployment_uuid, model=UNKNOWN_MODEL) + + entry = litellm.model_cost.get(deployment_uuid, {}) + assert ( + entry.get("input_cost_per_token") == 1e-6 + ), "user-supplied fields still register normally for unknown models" + assert ( + entry.get("cache_read_input_token_cost") is None + ), "cache_read must not be invented when no canonical entry exists" + assert ( + entry.get("cache_creation_input_token_cost") is None + ), "cache_creation must not be invented when no canonical entry exists" + + +def test_backfill_skips_when_canonical_lacks_field(): + """If the canonical static entry itself lacks a given field, the + backfill leaves the UUID entry unchanged for that field (i.e. no + None-for-None copy that would mask absence).""" + deployment_uuid = "backfill-canonical-partial-uuid" + litellm.model_cost.pop(deployment_uuid, None) + + _build_router(deployment_uuid=deployment_uuid, model=KNOWN_MODEL) + + entry = litellm.model_cost.get(deployment_uuid, {}) + # Pick a field the canonical entry definitely doesn't have. The + # bundled JSON's Anthropic claude-haiku-4-5-20251001 entry doesn't + # carry input_cost_per_audio_token, for instance. + canonical = litellm.model_cost[KNOWN_MODEL] + if canonical.get("input_cost_per_audio_token") is None: + assert ( + "input_cost_per_audio_token" not in entry + or entry["input_cost_per_audio_token"] is None + ), "backfill must not copy None-valued fields from canonical"