diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e66be233be4..e56a540ea04 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -916,7 +916,7 @@ def _bedrock_pricing_info( def cost_map_key_prices_anything(key: str) -> bool: - entry: Final = _raw_cost_map_entry(key) + entry: Final = raw_cost_map_entry(key) return entry is not None and _cost_map_entry_prices_anything(entry) @@ -1048,7 +1048,7 @@ def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool: key: Final = cached_cost_map_key(model, custom_llm_provider) if key is None: return False - entry: Final = _raw_cost_map_entry(key) + entry: Final = raw_cost_map_entry(key) if entry is None: return _rule_declares_rates(model, custom_llm_provider) fill_missing: Final = mapped_model_fill_missing(key, entry) @@ -2352,17 +2352,16 @@ def completion_cost( def get_response_cost_from_hidden_params( - hidden_params: dict | BaseModel, + hidden_params: Mapping[str, object] | BaseModel, ) -> float | None: - if isinstance(hidden_params, BaseModel): - _hidden_params_dict = hidden_params.model_dump() - else: - _hidden_params_dict = hidden_params - - additional_headers: Final = _hidden_params_dict.get("additional_headers", {}) - if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers: - return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"]) - return None + params: Final[Mapping[str, object]] = ( + hidden_params.model_dump() if isinstance(hidden_params, BaseModel) else hidden_params + ) + additional_headers: Final = params.get("additional_headers") + if not isinstance(additional_headers, Mapping): + return None + headers: Final = TypeAdapter(Mapping[str, object]).validate_python(additional_headers) + return _billable_header_cost(headers.get("llm_provider-x-litellm-response-cost")) def _billable_header_cost(value: object) -> float | None: @@ -2555,7 +2554,7 @@ def _deployment_model_info( ) -> ModelInfo | None: if not custom_pricing: return None - registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None + registered_deployment_info: Final = raw_cost_map_entry(router_model_id) if router_model_id is not None else None if registered_deployment_info is not None: return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id if litellm_logging_obj is None: @@ -2628,7 +2627,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, . ) -def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: +def raw_cost_map_entry(key: str) -> Mapping[str, object] | None: raw_entry: Final = litellm.model_cost.get(key) return raw_entry if isinstance(raw_entry, Mapping) else None @@ -2672,7 +2671,7 @@ def pricing_entry_for_cost_calc( ) if resolved is None: return None - return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved + return resolved["key"], raw_cost_map_entry(resolved["key"]) or resolved def ocr_cost( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 639003c9a6a..7ffcc8e5d26 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -54,8 +54,10 @@ from litellm.constants import ( from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, + get_response_cost_from_hidden_params, get_usage_object, pricing_entry_for_cost_calc, + raw_cost_map_entry, select_model_name_for_cost_calc, ) from litellm.exceptions import ( @@ -72,6 +74,7 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.langtrace import langtrace_trace_endpoint from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger +from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.litellm_core_utils.classifier_logging import ( classifier_audit_fields, classifier_input_snapshot, @@ -209,9 +212,15 @@ from litellm.types.utils import ( TranscriptionResponse, Usage, batch_chat_usage, + is_server_derived_pricing_key, ) from litellm.types.videos.main import VideoObject -from litellm.utils import get_base_model_from_metadata, print_verbose +from litellm.utils import ( + get_base_model_from_metadata, + mapped_model_fill_missing, + print_verbose, + unmapped_model_generalization, +) from ..integrations.argilla import ArgillaLogger from ..integrations.arize.arize_phoenix import ArizePhoenixLogger @@ -2566,6 +2575,44 @@ class Logging(LiteLLMLoggingBaseClass): **kwargs, ) + def _response_price_is_known(self, result: object, cost: float, payload: StandardLoggingPayload) -> bool: + if ( + cost != 0 + or self._provider_reported_cost == 0 + or get_response_cost_from_hidden_params(get_hidden_params_dict(result)) == 0 + ): + return True + if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result): + return True + params: Final = _LITELLM_PARAMS.validate_python(self._model_call_details_view.get("litellm_params") or {}) + if is_unbilled_non_inference_call( + payload["call_type"], StandardLoggingPayloadSetup.merge_litellm_metadata(params), result + ): + return True + if self.custom_cost_per_token is not None or self.custom_cost_per_second is not None: + return True + passthrough: Final = _LITELLM_PARAMS.validate_python( + self._model_call_details_view.get("passthrough_logging_payload") or {} + ) + if passthrough.get("cost_per_request") is not None: + return True + model_info: Final = payload["model_map_information"]["model_map_value"] + if model_info is None: + return False + raw_entry: Final = raw_cost_map_entry(model_info["key"]) + rule_entry: Final = ( + unmapped_model_generalization(model_info["key"], payload["custom_llm_provider"]) + if raw_entry is None + else mapped_model_fill_missing(model_info["key"], raw_entry) + ) + prices: Final = MappingProxyType({**(rule_entry or {}), **(raw_entry or {})}) + return any( + value is not None + and ("cost" in key or "pricing" in key) + and (is_server_derived_pricing_key(key) or key in DEPLOYMENT_SCOPED_PRICING_FIELDS) + for key, value in prices.items() + ) + def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None: if self.prepared_cost is not None: return self.prepared_cost @@ -2618,6 +2665,7 @@ class Logging(LiteLLMLoggingBaseClass): "cost_breakdown": copy.deepcopy(payload["cost_breakdown"]), }, usage, + price_known=self._response_price_is_known(prepared[2], raw_cost, payload), ) self.record_partial_usage_for_failure( usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"] diff --git a/litellm/litellm_core_utils/llm_cost_calc/finalization.py b/litellm/litellm_core_utils/llm_cost_calc/finalization.py index a13134ada4a..756c53ea882 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/finalization.py +++ b/litellm/litellm_core_utils/llm_cost_calc/finalization.py @@ -17,6 +17,7 @@ class PreparedCost: amount: float payload: StandardLoggingPayload usage: Usage + price_known: bool = True @dataclass(frozen=True, slots=True) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9282ee7f703..74c4b86f37f 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1884,6 +1884,13 @@ def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float | def cost_headers( response_cost: float | str | None, litellm_logging_obj: LiteLLMLoggingObj | None ) -> Mapping[str, str | None]: + if ( + response_cost == 0 + and litellm_logging_obj is not None + and litellm_logging_obj.prepared_cost is not None + and not litellm_logging_obj.prepared_cost.price_known + ): + return {} cost_breakdown: Final = _get_cost_breakdown_from_logging_obj( litellm_logging_obj=litellm_logging_obj, response_cost=response_cost ) diff --git a/tests/integration/pricing/test_price_data_reload_prices.py b/tests/integration/pricing/test_price_data_reload_prices.py index 0aa3a3bcba2..aa0addb7673 100644 --- a/tests/integration/pricing/test_price_data_reload_prices.py +++ b/tests/integration/pricing/test_price_data_reload_prices.py @@ -25,9 +25,16 @@ _GEO: Final = f"au.{_ROW}" _FINETUNE: Final = "ft:gpt-audit-reload-2025-01-01" _CLAUDE: Final = "claude-audit-reload-sonnet" _MINI: Final = "gpt-audit-reload-mini" +_FREE: Final = "gpt-audit-reload-free" +_RULE_FREE: Final = "gpt-audit-rule-free" +_FILL_FREE: Final = "gpt-audit-fill-free" +_RULE_UNPRICED: Final = "gpt-audit-rule-unpriced" +_REPORTED_FREE: Final = "gpt-audit-reported-free" _USAGE: Final = re.compile(r"bill (\d+) (\d+) (\d+)") _CUSTOM: Final = MappingProxyType({"input_cost_per_token": 1.1e-06, "output_cost_per_token": 2.3e-06}) -_COST_MAP_LIMITS: Final = MappingProxyType({"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"}) +_COST_MAP_LIMITS: Final = MappingProxyType( + {"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"} +) _PUBLISHED_SCALE: Final = 2.0 _CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType( { @@ -60,6 +67,13 @@ _CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType( "output_cost_per_token": 1.5e-05, "cache_read_input_token_cost": 3e-07, }, + _FILL_FREE: {"litellm_provider": "openai", "mode": "chat"}, + _FREE: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, _MINI: { "litellm_provider": "openai", "mode": "chat", @@ -99,6 +113,8 @@ class _Deployment: prices: Mapping[str, float] usage: _Usage rates: Callable[[float], _Rates | None] + reported_cost: float | None = None + listed_rates: Callable[[float], _Rates | None] | None = None @dataclass(frozen=True, slots=True) @@ -107,10 +123,23 @@ class _Charge: expected: float -def _catalog(scale: float) -> dict[str, dict[str, JsonValue]]: +def _catalog(scale: float) -> dict[str, JsonValue]: return { - model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()} - for model, row in _CATALOG.items() + **{ + model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()} + for model, row in _CATALOG.items() + }, + "fallback_generalizations": { + "rules": [ + { + "name": "free-pricing", + "pattern": f"({_RULE_FREE}|{_FILL_FREE})", + "fill_missing_for_providers": ["openai"], + "model_info": {"mode": "chat", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, + }, + {"name": "unpriced-capabilities", "pattern": _RULE_UNPRICED, "model_info": {"mode": "chat"}}, + ] + }, } @@ -240,6 +269,7 @@ def _reply(request: Request) -> Reply: "completion_tokens": usage.completion, "total_tokens": usage.prompt + usage.completion, "prompt_tokens_details": {"cached_tokens": usage.cached}, + **({"cost": 0.0} if object_value(json.loads(request.body))["model"] == _REPORTED_FREE else {}), }, } ).encode() @@ -281,10 +311,16 @@ def _billed(client: httpx.Client, key: str, deployment: _Deployment) -> tuple[st def _charged_as_listed(proxy: Gateway, deployment: _Deployment, scale: float) -> _Charge: rates: Final = deployment.rates(scale) - assert _listed_rates(proxy, deployment.name) == _expected_listing(rates), deployment.name + assert _listed_rates(proxy, deployment.name) == _expected_listing( + deployment.listed_rates(scale) if deployment.listed_rates is not None else rates + ), deployment.name request_id, charged = _billed(proxy.client, proxy.key, deployment) - expected: Final = _expected_cost(rates, deployment.usage) - assert charged == (None if rates is None else pytest.approx(expected, rel=1e-6)), deployment.name + expected: Final = ( + deployment.reported_cost if deployment.reported_cost is not None else _expected_cost(rates, deployment.usage) + ) + assert charged == ( + None if rates is None and deployment.reported_cost is None else pytest.approx(expected, rel=1e-6) + ), deployment.name return _Charge(request_id=request_id, expected=expected) @@ -352,8 +388,31 @@ def test_price_data_reload_reprices_catalog_deployments_and_keeps_custom_prices( _Usage(2000, 1000, 1000), _catalog_rates(_FINETUNE), ), - _Deployment("finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates), + _Deployment( + "finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates + ), _Deployment("cross-provider-unpriced", f"azure/{_CLAUDE}", {}, _Usage(2000, 1000, 1000), _unpriced), + _Deployment("catalog-free", f"openai/{_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)), + _Deployment("rule-free", f"openai/{_RULE_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)), + _Deployment( + "fill-rule-free", + f"openai/{_FILL_FREE}", + {}, + _Usage(2000, 1000, 1000), + _catalog_rates(_FREE), + listed_rates=_unpriced, + ), + _Deployment("rule-unpriced", f"openai/{_RULE_UNPRICED}", {}, _Usage(2000, 1000, 1000), _unpriced), + _Deployment( + "provider-reported-free", f"openrouter/{_REPORTED_FREE}", {}, _Usage(2000, 1000, 1000), _unpriced, 0.0 + ), + _Deployment( + "cross-provider-free", + f"azure/{_CLAUDE}", + {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, + _Usage(2000, 1000, 1000), + _catalog_rates(_FREE), + ), _Deployment( "region-custom-above-threshold", f"bedrock/ap-northeast-1.{_ROW}", diff --git a/tests/unit/proxy/spend_tracking/test_cost_finalization.py b/tests/unit/proxy/spend_tracking/test_cost_finalization.py index c9515c1232e..eea18e3fe87 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_finalization.py +++ b/tests/unit/proxy/spend_tracking/test_cost_finalization.py @@ -1,8 +1,9 @@ import asyncio from datetime import datetime, timezone -from typing import Final +from typing import Final, Literal import pytest +from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging @@ -450,18 +451,17 @@ async def test_completed_call_keeps_its_charge_when_post_call_processing_fails( disable_spend_updates: bool, ) -> None: from unittest.mock import create_autospec + import anyio - from fastapi import Request, Response - from litellm.proxy._types import ProxyException - from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger + from litellm.proxy import proxy_server + from litellm.proxy._types import ProxyException from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.proxy_server import ProxyConfig - from litellm.proxy import proxy_server from litellm.proxy.utils import ProxyLogging scope: Final = anyio.CancelScope() @@ -598,6 +598,7 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage( logging_obj: Logging, monkeypatch: pytest.MonkeyPatch ) -> None: import httpx + from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy.utils import ProxyLogging @@ -633,3 +634,188 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage( assert (payload["prompt_tokens"], payload["completion_tokens"]) == (10, 5) assert payload["response_cost"] == 0.5 ProxyLogging._callback_capabilities_cache.clear() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"]) +@pytest.mark.parametrize("discount", [0.0, 1.0], ids=["no-discount", "fully-discounted"]) +async def test_finalized_header_distinguishes_missing_prices_from_free_models( + monkeypatch: pytest.MonkeyPatch, price: float | None, discount: float +) -> None: + model: Final = "claude-header-pricing-test" + monkeypatch.setattr(litellm, "cost_discount_config", {"azure": discount}) + monkeypatch.setitem( + litellm.model_cost, + f"azure/{model}", + { + "litellm_provider": "azure", + "mode": "chat", + "regional_endpoint_uplift_multiplier": 1.0, + **({"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {}), + }, + ) + logging: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="acompletion", + start_time=NOW, + litellm_call_id="pricing-header", + function_id="pricing-header", + ) + logging.update_environment_variables( + model=model, custom_llm_provider="azure", litellm_params={}, optional_params={} + ) + result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + finalized: Final = await finalize_response_cost(logging, result, None, NOW) + assert finalized is not None + assert finalized.amount == pytest.approx((price or 0.0) * 15 * (1 - discount)) + headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), + response_cost=finalized.amount, + litellm_logging_obj=logging, + ) + assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "source", ["unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second"] +) +@pytest.mark.parametrize("guardrail_cost", [0.0, 0.1]) +async def test_zero_cost_headers_preserve_known_free_calls_and_guardrail_charges( + source: Literal["unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second"], + guardrail_cost: float, +) -> None: + model: Final = "unmapped-header-cost" + logging: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="adelete_responses" if source == "non-inference" else "acompletion", + start_time=NOW, + litellm_call_id="zero-cost-header", + function_id="zero-cost-header", + ) + logging.update_environment_variables( + model=model, custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + result._hidden_params = {"response_cost": 0.0, "cache_hit": source == "cache"} + match source: + case "provider": + logging._provider_reported_cost = 0.0 + case "provider-header": + result._hidden_params["additional_headers"] = {"llm_provider-x-litellm-response-cost": 0.0} + case "per-token": + logging.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + case "per-second": + logging.custom_cost_per_second = 0.0 + case _: + pass + finalized: Final = await finalize_response_cost(logging, result, None, NOW, guardrail_cost=guardrail_cost) + assert finalized is not None + assert finalized.amount == guardrail_cost + headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), + response_cost=finalized.amount, + litellm_logging_obj=logging, + ) + assert headers.get("x-litellm-response-cost") == ( + None if source == "unpriced" and guardrail_cost == 0 else str(guardrail_cost) + ) + + +@pytest.mark.asyncio +async def test_free_off_peak_pricing_keeps_the_finalized_zero_cost_header(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "gpt-free-off-peak-header" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "chat", + "off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, + }, + ) + logging: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="acompletion", + start_time=NOW, + litellm_call_id="off-peak-header", + function_id="off-peak-header", + ) + logging.update_environment_variables( + model=model, custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + result._hidden_params["response_cost"] = 0.0 + finalized: Final = await finalize_response_cost(logging, result, None, NOW) + assert finalized is not None and finalized.amount == 0.0 + headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), + response_cost=finalized.amount, + litellm_logging_obj=logging, + ) + assert headers["x-litellm-response-cost"] == "0.0" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mapped", [False, True], ids=["unmapped-rule", "fill-missing-rule"]) +@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"]) +async def test_rule_pricing_preserves_free_headers_without_treating_capabilities_as_prices( + monkeypatch: pytest.MonkeyPatch, mapped: bool, price: float | None +) -> None: + from litellm.litellm_core_utils.fallback_generalizations import ( + get_fallback_generalization_rules, + set_fallback_generalizations, + ) + + model: Final = "gpt-rule-header-model" + previous_rules: Final = TypeAdapter(list[object]).validate_python(get_fallback_generalization_rules()) + if mapped: + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat"}) + try: + set_fallback_generalizations( + [ + { + "name": "header-pricing", + "pattern": model, + "fill_missing_for_providers": ["openai"], + "model_info": { + "mode": "chat", + **( + {"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {} + ), + }, + } + ] + ) + logging: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="acompletion", + start_time=NOW, + litellm_call_id="rule-header", + function_id="rule-header", + ) + logging.update_environment_variables( + model=model, custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + finalized: Final = await finalize_response_cost(logging, result, None, NOW) + assert finalized is not None + assert finalized.amount == pytest.approx((price or 0.0) * 15) + headers: Final = TypeAdapter(dict[str, str]).validate_python( + ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), + response_cost=finalized.amount, + litellm_logging_obj=logging, + ) + ) + assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount)) + finally: + set_fallback_generalizations(previous_rules)