scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-01 07:03:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 129

fix(proxy): omit cost headers for unpriced responses

Browse source
This commit is contained in:
Joshua Valluru 2026-10-31 11:28:42 -07:00
parent aea0e4f2d4
commit 01fdfe78f0
6 changed files with 181 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/cost_calculator.py
View file

@ -916,7 +916,7 @@ def _bedrock_pricing_info(
def cost_map_key_prices_anything(key: str) -> bool:
entry: Final = _raw_cost_map_entry(key)
entry: Final = raw_cost_map_entry(key)
return entry is not None and _cost_map_entry_prices_anything(entry)
@ -1048,7 +1048,7 @@ def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
key: Final = cached_cost_map_key(model, custom_llm_provider)
if key is None:
return False
entry: Final = _raw_cost_map_entry(key)
entry: Final = raw_cost_map_entry(key)
if entry is None:
return _rule_declares_rates(model, custom_llm_provider)
fill_missing: Final = mapped_model_fill_missing(key, entry)
@ -2555,7 +2555,7 @@ def _deployment_model_info(
) -> ModelInfo | None:
if not custom_pricing:
return None
registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None
registered_deployment_info: Final = raw_cost_map_entry(router_model_id) if router_model_id is not None else None
if registered_deployment_info is not None:
return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id
if litellm_logging_obj is None:
@ -2628,7 +2628,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, .
)
def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
def raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
raw_entry: Final = litellm.model_cost.get(key)
return raw_entry if isinstance(raw_entry, Mapping) else None
@ -2672,7 +2672,7 @@ def pricing_entry_for_cost_calc(
)
if resolved is None:
return None
return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved
return resolved["key"], raw_cost_map_entry(resolved["key"]) or resolved
def ocr_cost(

24
litellm/litellm_core_utils/litellm_logging.py
View file

@ -56,6 +56,7 @@ from litellm.cost_calculator import (
ResponsesWebSocketTokenUsageProcessor,
get_usage_object,
pricing_entry_for_cost_calc,
raw_cost_map_entry,
select_model_name_for_cost_calc,
)
from litellm.exceptions import (
@ -209,6 +210,7 @@ from litellm.types.utils import (
TranscriptionResponse,
Usage,
batch_chat_usage,
is_server_derived_pricing_key,
)
from litellm.types.videos.main import VideoObject
from litellm.utils import get_base_model_from_metadata, print_verbose
@ -2566,6 +2568,27 @@ class Logging(LiteLLMLoggingBaseClass):
**kwargs,
)
def _response_price_is_known(self, result: object, cost: float, payload: StandardLoggingPayload) -> bool:
if cost != 0 or self._provider_reported_cost == 0:
return True
if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result):
return True
params: Final = _LITELLM_PARAMS.validate_python(self._model_call_details_view.get("litellm_params") or {})
if is_unbilled_non_inference_call(
payload["call_type"], StandardLoggingPayloadSetup.merge_litellm_metadata(params), result
):
return True
if self.custom_cost_per_token is not None or self.custom_cost_per_second is not None:
return True
model_info: Final = payload["model_map_information"]["model_map_value"]
raw_entry: Final = raw_cost_map_entry(model_info["key"]) if model_info is not None else None
return raw_entry is not None and any(
value is not None
and ("cost" in key or "pricing" in key)
and (is_server_derived_pricing_key(key) or key in DEPLOYMENT_SCOPED_PRICING_FIELDS)
for key, value in raw_entry.items()
)
def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:
if self.prepared_cost is not None:
return self.prepared_cost
@ -2618,6 +2641,7 @@ class Logging(LiteLLMLoggingBaseClass):
"cost_breakdown": copy.deepcopy(payload["cost_breakdown"]),
},
usage,
price_known=self._response_price_is_known(prepared[2], raw_cost, payload),
)
self.record_partial_usage_for_failure(
usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"]

1
litellm/litellm_core_utils/llm_cost_calc/finalization.py
View file

@ -17,6 +17,7 @@ class PreparedCost:
amount: float
payload: StandardLoggingPayload
usage: Usage
price_known: bool = True
@dataclass(frozen=True, slots=True)

7
litellm/proxy/common_request_processing.py
View file

@ -1884,6 +1884,13 @@ def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float |
def cost_headers(
response_cost: float | str | None, litellm_logging_obj: LiteLLMLoggingObj | None
) -> Mapping[str, str | None]:
if (
response_cost == 0
and litellm_logging_obj is not None
and litellm_logging_obj.prepared_cost is not None
and not litellm_logging_obj.prepared_cost.price_known
):
return {}
cost_breakdown: Final = _get_cost_breakdown_from_logging_obj(
litellm_logging_obj=litellm_logging_obj, response_cost=response_cost
)

23
tests/integration/pricing/test_price_data_reload_prices.py
View file

@ -25,9 +25,12 @@ _GEO: Final = f"au.{_ROW}"
_FINETUNE: Final = "ft:gpt-audit-reload-2025-01-01"
_CLAUDE: Final = "claude-audit-reload-sonnet"
_MINI: Final = "gpt-audit-reload-mini"
_FREE: Final = "gpt-audit-reload-free"
_USAGE: Final = re.compile(r"bill (\d+) (\d+) (\d+)")
_CUSTOM: Final = MappingProxyType({"input_cost_per_token": 1.1e-06, "output_cost_per_token": 2.3e-06})
_COST_MAP_LIMITS: Final = MappingProxyType({"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"})
_COST_MAP_LIMITS: Final = MappingProxyType(
{"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"}
)
_PUBLISHED_SCALE: Final = 2.0
_CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
{
@ -60,6 +63,12 @@ _CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
"output_cost_per_token": 1.5e-05,
"cache_read_input_token_cost": 3e-07,
},
_FREE: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
},
_MINI: {
"litellm_provider": "openai",
"mode": "chat",
@ -352,8 +361,18 @@ def test_price_data_reload_reprices_catalog_deployments_and_keeps_custom_prices(
_Usage(2000, 1000, 1000),
_catalog_rates(_FINETUNE),
),
_Deployment("finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates),
_Deployment(
"finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates
),
_Deployment("cross-provider-unpriced", f"azure/{_CLAUDE}", {}, _Usage(2000, 1000, 1000), _unpriced),
_Deployment("catalog-free", f"openai/{_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)),
_Deployment(
"cross-provider-free",
f"azure/{_CLAUDE}",
{"input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
_Usage(2000, 1000, 1000),
_catalog_rates(_FREE),
),
_Deployment(
"region-custom-above-threshold",
f"bedrock/ap-northeast-1.{_ROW}",

124
tests/unit/proxy/spend_tracking/test_cost_finalization.py
View file

@ -1,6 +1,6 @@
import asyncio
from datetime import datetime, timezone
from typing import Final
from typing import Final, Literal
import pytest
@ -633,3 +633,125 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage(
assert (payload["prompt_tokens"], payload["completion_tokens"]) == (10, 5)
assert payload["response_cost"] == 0.5
ProxyLogging._callback_capabilities_cache.clear()
@pytest.mark.asyncio
@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"])
@pytest.mark.parametrize("discount", [0.0, 1.0], ids=["no-discount", "fully-discounted"])
async def test_finalized_header_distinguishes_missing_prices_from_free_models(
monkeypatch: pytest.MonkeyPatch, price: float | None, discount: float
) -> None:
model: Final = "claude-header-pricing-test"
monkeypatch.setattr(litellm, "cost_discount_config", {"azure": discount})
monkeypatch.setitem(
litellm.model_cost,
f"azure/{model}",
{
"litellm_provider": "azure",
"mode": "chat",
"regional_endpoint_uplift_multiplier": 1.0,
**({"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {}),
},
)
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="acompletion",
start_time=NOW,
litellm_call_id="pricing-header",
function_id="pricing-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="azure", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
assert finalized is not None
assert finalized.amount == pytest.approx((price or 0.0) * 15 * (1 - discount))
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount))
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["unpriced", "cache", "non-inference", "provider", "per-token", "per-second"])
@pytest.mark.parametrize("guardrail_cost", [0.0, 0.1])
async def test_zero_cost_headers_preserve_known_free_calls_and_guardrail_charges(
source: Literal["unpriced", "cache", "non-inference", "provider", "per-token", "per-second"],
guardrail_cost: float,
) -> None:
model: Final = "unmapped-header-cost"
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="adelete_responses" if source == "non-inference" else "acompletion",
start_time=NOW,
litellm_call_id="zero-cost-header",
function_id="zero-cost-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
result._hidden_params = {"response_cost": 0.0, "cache_hit": source == "cache"}
match source:
case "provider":
logging._provider_reported_cost = 0.0
case "per-token":
logging.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}
case "per-second":
logging.custom_cost_per_second = 0.0
case _:
pass
finalized: Final = await finalize_response_cost(logging, result, None, NOW, guardrail_cost=guardrail_cost)
assert finalized is not None
assert finalized.amount == guardrail_cost
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers.get("x-litellm-response-cost") == (
None if source == "unpriced" and guardrail_cost == 0 else str(guardrail_cost)
)
@pytest.mark.asyncio
async def test_free_off_peak_pricing_keeps_the_finalized_zero_cost_header(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "gpt-free-off-peak-header"
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": "openai",
"mode": "chat",
"off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
},
)
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="acompletion",
start_time=NOW,
litellm_call_id="off-peak-header",
function_id="off-peak-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
result._hidden_params["response_cost"] = 0.0
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
assert finalized is not None and finalized.amount == 0.0
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers["x-litellm-response-cost"] == "0.0"