fix(proxy): preserve declared free response costs

litellm/cost_calculator.py

@@ -2352,17 +2352,16 @@ def completion_cost(
 
 
 def get_response_cost_from_hidden_params(
-    hidden_params: dict | BaseModel,
+    hidden_params: Mapping[str, object] | BaseModel,
 ) -> float | None:
-    if isinstance(hidden_params, BaseModel):
-        _hidden_params_dict = hidden_params.model_dump()
-    else:
-        _hidden_params_dict = hidden_params
-
-    additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
-    if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
-        return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
-    return None
+    params: Final[Mapping[str, object]] = (
+        hidden_params.model_dump() if isinstance(hidden_params, BaseModel) else hidden_params
+    )
+    additional_headers: Final = params.get("additional_headers")
+    if not isinstance(additional_headers, Mapping):
+        return None
+    headers: Final = TypeAdapter(Mapping[str, object]).validate_python(additional_headers)
+    return _billable_header_cost(headers.get("llm_provider-x-litellm-response-cost"))
 
 
 def _billable_header_cost(value: object) -> float | None:

litellm/litellm_core_utils/litellm_logging.py

@@ -54,6 +54,7 @@ from litellm.constants import (
 from litellm.cost_calculator import (
     RealtimeAPITokenUsageProcessor,
     ResponsesWebSocketTokenUsageProcessor,
+    get_response_cost_from_hidden_params,
     get_usage_object,
     pricing_entry_for_cost_calc,
     raw_cost_map_entry,
@@ -73,6 +74,7 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger
 from litellm.integrations.langtrace import langtrace_trace_endpoint
 from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
+from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
 from litellm.litellm_core_utils.classifier_logging import (
     classifier_audit_fields,
     classifier_input_snapshot,
@@ -213,7 +215,12 @@ from litellm.types.utils import (
     is_server_derived_pricing_key,
 )
 from litellm.types.videos.main import VideoObject
-from litellm.utils import get_base_model_from_metadata, print_verbose
+from litellm.utils import (
+    get_base_model_from_metadata,
+    mapped_model_fill_missing,
+    print_verbose,
+    unmapped_model_generalization,
+)
 
 from ..integrations.argilla import ArgillaLogger
 from ..integrations.arize.arize_phoenix import ArizePhoenixLogger
@@ -2569,7 +2576,11 @@ class Logging(LiteLLMLoggingBaseClass):
         )
 
     def _response_price_is_known(self, result: object, cost: float, payload: StandardLoggingPayload) -> bool:
-        if cost != 0 or self._provider_reported_cost == 0:
+        if (
+            cost != 0
+            or self._provider_reported_cost == 0
+            or get_response_cost_from_hidden_params(get_hidden_params_dict(result)) == 0
+        ):
             return True
         if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result):
             return True
@@ -2580,13 +2591,26 @@ class Logging(LiteLLMLoggingBaseClass):
             return True
         if self.custom_cost_per_token is not None or self.custom_cost_per_second is not None:
             return True
+        passthrough: Final = _LITELLM_PARAMS.validate_python(
+            self._model_call_details_view.get("passthrough_logging_payload") or {}
+        )
+        if passthrough.get("cost_per_request") is not None:
+            return True
         model_info: Final = payload["model_map_information"]["model_map_value"]
-        raw_entry: Final = raw_cost_map_entry(model_info["key"]) if model_info is not None else None
-        return raw_entry is not None and any(
+        if model_info is None:
+            return False
+        raw_entry: Final = raw_cost_map_entry(model_info["key"])
+        rule_entry: Final = (
+            unmapped_model_generalization(model_info["key"], payload["custom_llm_provider"])
+            if raw_entry is None
+            else mapped_model_fill_missing(model_info["key"], raw_entry)
+        )
+        prices: Final = MappingProxyType({**(rule_entry or {}), **(raw_entry or {})})
+        return any(
             value is not None
             and ("cost" in key or "pricing" in key)
             and (is_server_derived_pricing_key(key) or key in DEPLOYMENT_SCOPED_PRICING_FIELDS)
-            for key, value in raw_entry.items()
+            for key, value in prices.items()
         )
 
     def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:

tests/integration/pricing/test_price_data_reload_prices.py

@@ -26,6 +26,10 @@ _FINETUNE: Final = "ft:gpt-audit-reload-2025-01-01"
 _CLAUDE: Final = "claude-audit-reload-sonnet"
 _MINI: Final = "gpt-audit-reload-mini"
 _FREE: Final = "gpt-audit-reload-free"
+_RULE_FREE: Final = "gpt-audit-rule-free"
+_FILL_FREE: Final = "gpt-audit-fill-free"
+_RULE_UNPRICED: Final = "gpt-audit-rule-unpriced"
+_REPORTED_FREE: Final = "gpt-audit-reported-free"
 _USAGE: Final = re.compile(r"bill (\d+) (\d+) (\d+)")
 _CUSTOM: Final = MappingProxyType({"input_cost_per_token": 1.1e-06, "output_cost_per_token": 2.3e-06})
 _COST_MAP_LIMITS: Final = MappingProxyType(
@@ -63,6 +67,7 @@ _CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
             "output_cost_per_token": 1.5e-05,
             "cache_read_input_token_cost": 3e-07,
         },
+        _FILL_FREE: {"litellm_provider": "openai", "mode": "chat"},
         _FREE: {
             "litellm_provider": "openai",
             "mode": "chat",
@@ -108,6 +113,8 @@ class _Deployment:
     prices: Mapping[str, float]
     usage: _Usage
     rates: Callable[[float], _Rates | None]
+    reported_cost: float | None = None
+    listed_rates: Callable[[float], _Rates | None] | None = None
 
 
 @dataclass(frozen=True, slots=True)
@@ -116,10 +123,23 @@ class _Charge:
     expected: float
 
 
-def _catalog(scale: float) -> dict[str, dict[str, JsonValue]]:
+def _catalog(scale: float) -> dict[str, JsonValue]:
     return {
-        model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()}
-        for model, row in _CATALOG.items()
+        **{
+            model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()}
+            for model, row in _CATALOG.items()
+        },
+        "fallback_generalizations": {
+            "rules": [
+                {
+                    "name": "free-pricing",
+                    "pattern": f"({_RULE_FREE}|{_FILL_FREE})",
+                    "fill_missing_for_providers": ["openai"],
+                    "model_info": {"mode": "chat", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
+                },
+                {"name": "unpriced-capabilities", "pattern": _RULE_UNPRICED, "model_info": {"mode": "chat"}},
+            ]
+        },
     }
 
 
@@ -249,6 +269,7 @@ def _reply(request: Request) -> Reply:
                     "completion_tokens": usage.completion,
                     "total_tokens": usage.prompt + usage.completion,
                     "prompt_tokens_details": {"cached_tokens": usage.cached},
+                    **({"cost": 0.0} if object_value(json.loads(request.body))["model"] == _REPORTED_FREE else {}),
                 },
             }
         ).encode()
@@ -290,10 +311,16 @@ def _billed(client: httpx.Client, key: str, deployment: _Deployment) -> tuple[st
 
 def _charged_as_listed(proxy: Gateway, deployment: _Deployment, scale: float) -> _Charge:
     rates: Final = deployment.rates(scale)
-    assert _listed_rates(proxy, deployment.name) == _expected_listing(rates), deployment.name
+    assert _listed_rates(proxy, deployment.name) == _expected_listing(
+        deployment.listed_rates(scale) if deployment.listed_rates is not None else rates
+    ), deployment.name
     request_id, charged = _billed(proxy.client, proxy.key, deployment)
-    expected: Final = _expected_cost(rates, deployment.usage)
-    assert charged == (None if rates is None else pytest.approx(expected, rel=1e-6)), deployment.name
+    expected: Final = (
+        deployment.reported_cost if deployment.reported_cost is not None else _expected_cost(rates, deployment.usage)
+    )
+    assert charged == (
+        None if rates is None and deployment.reported_cost is None else pytest.approx(expected, rel=1e-6)
+    ), deployment.name
     return _Charge(request_id=request_id, expected=expected)
 
 
@@ -366,6 +393,19 @@ def test_price_data_reload_reprices_catalog_deployments_and_keeps_custom_prices(
         ),
         _Deployment("cross-provider-unpriced", f"azure/{_CLAUDE}", {}, _Usage(2000, 1000, 1000), _unpriced),
         _Deployment("catalog-free", f"openai/{_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)),
+        _Deployment("rule-free", f"openai/{_RULE_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)),
+        _Deployment(
+            "fill-rule-free",
+            f"openai/{_FILL_FREE}",
+            {},
+            _Usage(2000, 1000, 1000),
+            _catalog_rates(_FREE),
+            listed_rates=_unpriced,
+        ),
+        _Deployment("rule-unpriced", f"openai/{_RULE_UNPRICED}", {}, _Usage(2000, 1000, 1000), _unpriced),
+        _Deployment(
+            "provider-reported-free", f"openrouter/{_REPORTED_FREE}", {}, _Usage(2000, 1000, 1000), _unpriced, 0.0
+        ),
         _Deployment(
             "cross-provider-free",
             f"azure/{_CLAUDE}",

tests/unit/proxy/spend_tracking/test_cost_finalization.py

@@ -3,6 +3,7 @@ from datetime import datetime, timezone
 from typing import Final, Literal
 
 import pytest
+from pydantic import TypeAdapter
 
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging
@@ -450,18 +451,17 @@ async def test_completed_call_keeps_its_charge_when_post_call_processing_fails(
     disable_spend_updates: bool,
 ) -> None:
     from unittest.mock import create_autospec
+
     import anyio
-
     from fastapi import Request, Response
 
-    from litellm.proxy._types import ProxyException
-
     from litellm.caching.caching import DualCache
     from litellm.integrations.custom_logger import CustomLogger
+    from litellm.proxy import proxy_server
+    from litellm.proxy._types import ProxyException
     from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
     from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
     from litellm.proxy.proxy_server import ProxyConfig
-    from litellm.proxy import proxy_server
     from litellm.proxy.utils import ProxyLogging
 
     scope: Final = anyio.CancelScope()
@@ -598,6 +598,7 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage(
     logging_obj: Logging, monkeypatch: pytest.MonkeyPatch
 ) -> None:
     import httpx
+
     from litellm.caching.dual_cache import DualCache
     from litellm.integrations.custom_guardrail import CustomGuardrail
     from litellm.proxy.utils import ProxyLogging
@@ -678,10 +679,12 @@ async def test_finalized_header_distinguishes_missing_prices_from_free_models(
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("source", ["unpriced", "cache", "non-inference", "provider", "per-token", "per-second"])
+@pytest.mark.parametrize(
+    "source", ["unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second"]
+)
 @pytest.mark.parametrize("guardrail_cost", [0.0, 0.1])
 async def test_zero_cost_headers_preserve_known_free_calls_and_guardrail_charges(
-    source: Literal["unpriced", "cache", "non-inference", "provider", "per-token", "per-second"],
+    source: Literal["unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second"],
     guardrail_cost: float,
 ) -> None:
     model: Final = "unmapped-header-cost"
@@ -702,6 +705,8 @@ async def test_zero_cost_headers_preserve_known_free_calls_and_guardrail_charges
     match source:
         case "provider":
             logging._provider_reported_cost = 0.0
+        case "provider-header":
+            result._hidden_params["additional_headers"] = {"llm_provider-x-litellm-response-cost": 0.0}
         case "per-token":
             logging.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}
         case "per-second":
@@ -755,3 +760,62 @@ async def test_free_off_peak_pricing_keeps_the_finalized_zero_cost_header(monkey
         litellm_logging_obj=logging,
     )
     assert headers["x-litellm-response-cost"] == "0.0"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("mapped", [False, True], ids=["unmapped-rule", "fill-missing-rule"])
+@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"])
+async def test_rule_pricing_preserves_free_headers_without_treating_capabilities_as_prices(
+    monkeypatch: pytest.MonkeyPatch, mapped: bool, price: float | None
+) -> None:
+    from litellm.litellm_core_utils.fallback_generalizations import (
+        get_fallback_generalization_rules,
+        set_fallback_generalizations,
+    )
+
+    model: Final = "gpt-rule-header-model"
+    previous_rules: Final = TypeAdapter(list[object]).validate_python(get_fallback_generalization_rules())
+    if mapped:
+        monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat"})
+    try:
+        set_fallback_generalizations(
+            [
+                {
+                    "name": "header-pricing",
+                    "pattern": model,
+                    "fill_missing_for_providers": ["openai"],
+                    "model_info": {
+                        "mode": "chat",
+                        **(
+                            {"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {}
+                        ),
+                    },
+                }
+            ]
+        )
+        logging: Final = Logging(
+            model=model,
+            messages=[],
+            stream=False,
+            call_type="acompletion",
+            start_time=NOW,
+            litellm_call_id="rule-header",
+            function_id="rule-header",
+        )
+        logging.update_environment_variables(
+            model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
+        )
+        result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
+        finalized: Final = await finalize_response_cost(logging, result, None, NOW)
+        assert finalized is not None
+        assert finalized.amount == pytest.approx((price or 0.0) * 15)
+        headers: Final = TypeAdapter(dict[str, str]).validate_python(
+            ProxyBaseLLMRequestProcessing.get_custom_headers(
+                user_api_key_dict=UserAPIKeyAuth(),
+                response_cost=finalized.amount,
+                litellm_logging_obj=logging,
+            )
+        )
+        assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount))
+    finally:
+        set_fallback_generalizations(previous_rules)