mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge f7bbdea4d6 into 60fc121ce8
This commit is contained in:
commit
bb2b284316
6 changed files with 329 additions and 29 deletions
|
|
@ -916,7 +916,7 @@ def _bedrock_pricing_info(
|
|||
|
||||
|
||||
def cost_map_key_prices_anything(key: str) -> bool:
|
||||
entry: Final = _raw_cost_map_entry(key)
|
||||
entry: Final = raw_cost_map_entry(key)
|
||||
return entry is not None and _cost_map_entry_prices_anything(entry)
|
||||
|
||||
|
||||
|
|
@ -1048,7 +1048,7 @@ def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
|
|||
key: Final = cached_cost_map_key(model, custom_llm_provider)
|
||||
if key is None:
|
||||
return False
|
||||
entry: Final = _raw_cost_map_entry(key)
|
||||
entry: Final = raw_cost_map_entry(key)
|
||||
if entry is None:
|
||||
return _rule_declares_rates(model, custom_llm_provider)
|
||||
fill_missing: Final = mapped_model_fill_missing(key, entry)
|
||||
|
|
@ -2352,17 +2352,16 @@ def completion_cost(
|
|||
|
||||
|
||||
def get_response_cost_from_hidden_params(
|
||||
hidden_params: dict | BaseModel,
|
||||
hidden_params: Mapping[str, object] | BaseModel,
|
||||
) -> float | None:
|
||||
if isinstance(hidden_params, BaseModel):
|
||||
_hidden_params_dict = hidden_params.model_dump()
|
||||
else:
|
||||
_hidden_params_dict = hidden_params
|
||||
|
||||
additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
|
||||
if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
|
||||
return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
|
||||
return None
|
||||
params: Final[Mapping[str, object]] = (
|
||||
hidden_params.model_dump() if isinstance(hidden_params, BaseModel) else hidden_params
|
||||
)
|
||||
additional_headers: Final = params.get("additional_headers")
|
||||
if not isinstance(additional_headers, Mapping):
|
||||
return None
|
||||
headers: Final = TypeAdapter(Mapping[str, object]).validate_python(additional_headers)
|
||||
return _billable_header_cost(headers.get("llm_provider-x-litellm-response-cost"))
|
||||
|
||||
|
||||
def _billable_header_cost(value: object) -> float | None:
|
||||
|
|
@ -2555,7 +2554,7 @@ def _deployment_model_info(
|
|||
) -> ModelInfo | None:
|
||||
if not custom_pricing:
|
||||
return None
|
||||
registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None
|
||||
registered_deployment_info: Final = raw_cost_map_entry(router_model_id) if router_model_id is not None else None
|
||||
if registered_deployment_info is not None:
|
||||
return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id
|
||||
if litellm_logging_obj is None:
|
||||
|
|
@ -2628,7 +2627,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, .
|
|||
)
|
||||
|
||||
|
||||
def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
|
||||
def raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
|
||||
raw_entry: Final = litellm.model_cost.get(key)
|
||||
return raw_entry if isinstance(raw_entry, Mapping) else None
|
||||
|
||||
|
|
@ -2672,7 +2671,7 @@ def pricing_entry_for_cost_calc(
|
|||
)
|
||||
if resolved is None:
|
||||
return None
|
||||
return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved
|
||||
return resolved["key"], raw_cost_map_entry(resolved["key"]) or resolved
|
||||
|
||||
|
||||
def ocr_cost(
|
||||
|
|
|
|||
|
|
@ -54,8 +54,10 @@ from litellm.constants import (
|
|||
from litellm.cost_calculator import (
|
||||
RealtimeAPITokenUsageProcessor,
|
||||
ResponsesWebSocketTokenUsageProcessor,
|
||||
get_response_cost_from_hidden_params,
|
||||
get_usage_object,
|
||||
pricing_entry_for_cost_calc,
|
||||
raw_cost_map_entry,
|
||||
select_model_name_for_cost_calc,
|
||||
)
|
||||
from litellm.exceptions import (
|
||||
|
|
@ -72,6 +74,7 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger
|
|||
from litellm.integrations.langtrace import langtrace_trace_endpoint
|
||||
from litellm.integrations.mlflow import MlflowLogger
|
||||
from litellm.integrations.sqs import SQSLogger
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.classifier_logging import (
|
||||
classifier_audit_fields,
|
||||
classifier_input_snapshot,
|
||||
|
|
@ -209,9 +212,15 @@ from litellm.types.utils import (
|
|||
TranscriptionResponse,
|
||||
Usage,
|
||||
batch_chat_usage,
|
||||
is_server_derived_pricing_key,
|
||||
)
|
||||
from litellm.types.videos.main import VideoObject
|
||||
from litellm.utils import get_base_model_from_metadata, print_verbose
|
||||
from litellm.utils import (
|
||||
get_base_model_from_metadata,
|
||||
mapped_model_fill_missing,
|
||||
print_verbose,
|
||||
unmapped_model_generalization,
|
||||
)
|
||||
|
||||
from ..integrations.argilla import ArgillaLogger
|
||||
from ..integrations.arize.arize_phoenix import ArizePhoenixLogger
|
||||
|
|
@ -2566,6 +2575,44 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
**kwargs,
|
||||
)
|
||||
|
||||
def _response_price_is_known(self, result: object, cost: float, payload: StandardLoggingPayload) -> bool:
|
||||
if (
|
||||
cost != 0
|
||||
or self._provider_reported_cost == 0
|
||||
or get_response_cost_from_hidden_params(get_hidden_params_dict(result)) == 0
|
||||
):
|
||||
return True
|
||||
if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result):
|
||||
return True
|
||||
params: Final = _LITELLM_PARAMS.validate_python(self._model_call_details_view.get("litellm_params") or {})
|
||||
if is_unbilled_non_inference_call(
|
||||
payload["call_type"], StandardLoggingPayloadSetup.merge_litellm_metadata(params), result
|
||||
):
|
||||
return True
|
||||
if self.custom_cost_per_token is not None or self.custom_cost_per_second is not None:
|
||||
return True
|
||||
passthrough: Final = _LITELLM_PARAMS.validate_python(
|
||||
self._model_call_details_view.get("passthrough_logging_payload") or {}
|
||||
)
|
||||
if passthrough.get("cost_per_request") is not None:
|
||||
return True
|
||||
model_info: Final = payload["model_map_information"]["model_map_value"]
|
||||
if model_info is None:
|
||||
return False
|
||||
raw_entry: Final = raw_cost_map_entry(model_info["key"])
|
||||
rule_entry: Final = (
|
||||
unmapped_model_generalization(model_info["key"], payload["custom_llm_provider"])
|
||||
if raw_entry is None
|
||||
else mapped_model_fill_missing(model_info["key"], raw_entry)
|
||||
)
|
||||
prices: Final = MappingProxyType({**(rule_entry or {}), **(raw_entry or {})})
|
||||
return any(
|
||||
value is not None
|
||||
and ("cost" in key or "pricing" in key)
|
||||
and (is_server_derived_pricing_key(key) or key in DEPLOYMENT_SCOPED_PRICING_FIELDS)
|
||||
for key, value in prices.items()
|
||||
)
|
||||
|
||||
def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:
|
||||
if self.prepared_cost is not None:
|
||||
return self.prepared_cost
|
||||
|
|
@ -2618,6 +2665,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"cost_breakdown": copy.deepcopy(payload["cost_breakdown"]),
|
||||
},
|
||||
usage,
|
||||
price_known=self._response_price_is_known(prepared[2], raw_cost, payload),
|
||||
)
|
||||
self.record_partial_usage_for_failure(
|
||||
usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"]
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ class PreparedCost:
|
|||
amount: float
|
||||
payload: StandardLoggingPayload
|
||||
usage: Usage
|
||||
price_known: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
|
|||
|
|
@ -1884,6 +1884,13 @@ def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float |
|
|||
def cost_headers(
|
||||
response_cost: float | str | None, litellm_logging_obj: LiteLLMLoggingObj | None
|
||||
) -> Mapping[str, str | None]:
|
||||
if (
|
||||
response_cost == 0
|
||||
and litellm_logging_obj is not None
|
||||
and litellm_logging_obj.prepared_cost is not None
|
||||
and not litellm_logging_obj.prepared_cost.price_known
|
||||
):
|
||||
return {}
|
||||
cost_breakdown: Final = _get_cost_breakdown_from_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj, response_cost=response_cost
|
||||
)
|
||||
|
|
|
|||
|
|
@ -25,9 +25,16 @@ _GEO: Final = f"au.{_ROW}"
|
|||
_FINETUNE: Final = "ft:gpt-audit-reload-2025-01-01"
|
||||
_CLAUDE: Final = "claude-audit-reload-sonnet"
|
||||
_MINI: Final = "gpt-audit-reload-mini"
|
||||
_FREE: Final = "gpt-audit-reload-free"
|
||||
_RULE_FREE: Final = "gpt-audit-rule-free"
|
||||
_FILL_FREE: Final = "gpt-audit-fill-free"
|
||||
_RULE_UNPRICED: Final = "gpt-audit-rule-unpriced"
|
||||
_REPORTED_FREE: Final = "gpt-audit-reported-free"
|
||||
_USAGE: Final = re.compile(r"bill (\d+) (\d+) (\d+)")
|
||||
_CUSTOM: Final = MappingProxyType({"input_cost_per_token": 1.1e-06, "output_cost_per_token": 2.3e-06})
|
||||
_COST_MAP_LIMITS: Final = MappingProxyType({"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"})
|
||||
_COST_MAP_LIMITS: Final = MappingProxyType(
|
||||
{"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"}
|
||||
)
|
||||
_PUBLISHED_SCALE: Final = 2.0
|
||||
_CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
|
||||
{
|
||||
|
|
@ -60,6 +67,13 @@ _CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
|
|||
"output_cost_per_token": 1.5e-05,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
},
|
||||
_FILL_FREE: {"litellm_provider": "openai", "mode": "chat"},
|
||||
_FREE: {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
_MINI: {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
|
|
@ -99,6 +113,8 @@ class _Deployment:
|
|||
prices: Mapping[str, float]
|
||||
usage: _Usage
|
||||
rates: Callable[[float], _Rates | None]
|
||||
reported_cost: float | None = None
|
||||
listed_rates: Callable[[float], _Rates | None] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -107,10 +123,23 @@ class _Charge:
|
|||
expected: float
|
||||
|
||||
|
||||
def _catalog(scale: float) -> dict[str, dict[str, JsonValue]]:
|
||||
def _catalog(scale: float) -> dict[str, JsonValue]:
|
||||
return {
|
||||
model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()}
|
||||
for model, row in _CATALOG.items()
|
||||
**{
|
||||
model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()}
|
||||
for model, row in _CATALOG.items()
|
||||
},
|
||||
"fallback_generalizations": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "free-pricing",
|
||||
"pattern": f"({_RULE_FREE}|{_FILL_FREE})",
|
||||
"fill_missing_for_providers": ["openai"],
|
||||
"model_info": {"mode": "chat", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
|
||||
},
|
||||
{"name": "unpriced-capabilities", "pattern": _RULE_UNPRICED, "model_info": {"mode": "chat"}},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -240,6 +269,7 @@ def _reply(request: Request) -> Reply:
|
|||
"completion_tokens": usage.completion,
|
||||
"total_tokens": usage.prompt + usage.completion,
|
||||
"prompt_tokens_details": {"cached_tokens": usage.cached},
|
||||
**({"cost": 0.0} if object_value(json.loads(request.body))["model"] == _REPORTED_FREE else {}),
|
||||
},
|
||||
}
|
||||
).encode()
|
||||
|
|
@ -281,10 +311,16 @@ def _billed(client: httpx.Client, key: str, deployment: _Deployment) -> tuple[st
|
|||
|
||||
def _charged_as_listed(proxy: Gateway, deployment: _Deployment, scale: float) -> _Charge:
|
||||
rates: Final = deployment.rates(scale)
|
||||
assert _listed_rates(proxy, deployment.name) == _expected_listing(rates), deployment.name
|
||||
assert _listed_rates(proxy, deployment.name) == _expected_listing(
|
||||
deployment.listed_rates(scale) if deployment.listed_rates is not None else rates
|
||||
), deployment.name
|
||||
request_id, charged = _billed(proxy.client, proxy.key, deployment)
|
||||
expected: Final = _expected_cost(rates, deployment.usage)
|
||||
assert charged == (None if rates is None else pytest.approx(expected, rel=1e-6)), deployment.name
|
||||
expected: Final = (
|
||||
deployment.reported_cost if deployment.reported_cost is not None else _expected_cost(rates, deployment.usage)
|
||||
)
|
||||
assert charged == (
|
||||
None if rates is None and deployment.reported_cost is None else pytest.approx(expected, rel=1e-6)
|
||||
), deployment.name
|
||||
return _Charge(request_id=request_id, expected=expected)
|
||||
|
||||
|
||||
|
|
@ -352,8 +388,31 @@ def test_price_data_reload_reprices_catalog_deployments_and_keeps_custom_prices(
|
|||
_Usage(2000, 1000, 1000),
|
||||
_catalog_rates(_FINETUNE),
|
||||
),
|
||||
_Deployment("finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates),
|
||||
_Deployment(
|
||||
"finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates
|
||||
),
|
||||
_Deployment("cross-provider-unpriced", f"azure/{_CLAUDE}", {}, _Usage(2000, 1000, 1000), _unpriced),
|
||||
_Deployment("catalog-free", f"openai/{_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)),
|
||||
_Deployment("rule-free", f"openai/{_RULE_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)),
|
||||
_Deployment(
|
||||
"fill-rule-free",
|
||||
f"openai/{_FILL_FREE}",
|
||||
{},
|
||||
_Usage(2000, 1000, 1000),
|
||||
_catalog_rates(_FREE),
|
||||
listed_rates=_unpriced,
|
||||
),
|
||||
_Deployment("rule-unpriced", f"openai/{_RULE_UNPRICED}", {}, _Usage(2000, 1000, 1000), _unpriced),
|
||||
_Deployment(
|
||||
"provider-reported-free", f"openrouter/{_REPORTED_FREE}", {}, _Usage(2000, 1000, 1000), _unpriced, 0.0
|
||||
),
|
||||
_Deployment(
|
||||
"cross-provider-free",
|
||||
f"azure/{_CLAUDE}",
|
||||
{"input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
|
||||
_Usage(2000, 1000, 1000),
|
||||
_catalog_rates(_FREE),
|
||||
),
|
||||
_Deployment(
|
||||
"region-custom-above-threshold",
|
||||
f"bedrock/ap-northeast-1.{_ROW}",
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
|
@ -450,18 +451,17 @@ async def test_completed_call_keeps_its_charge_when_post_call_processing_fails(
|
|||
disable_spend_updates: bool,
|
||||
) -> None:
|
||||
from unittest.mock import create_autospec
|
||||
|
||||
import anyio
|
||||
|
||||
from fastapi import Request, Response
|
||||
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
scope: Final = anyio.CancelScope()
|
||||
|
|
@ -598,6 +598,7 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage(
|
|||
logging_obj: Logging, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
import httpx
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
|
@ -633,3 +634,188 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage(
|
|||
assert (payload["prompt_tokens"], payload["completion_tokens"]) == (10, 5)
|
||||
assert payload["response_cost"] == 0.5
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"])
|
||||
@pytest.mark.parametrize("discount", [0.0, 1.0], ids=["no-discount", "fully-discounted"])
|
||||
async def test_finalized_header_distinguishes_missing_prices_from_free_models(
|
||||
monkeypatch: pytest.MonkeyPatch, price: float | None, discount: float
|
||||
) -> None:
|
||||
model: Final = "claude-header-pricing-test"
|
||||
monkeypatch.setattr(litellm, "cost_discount_config", {"azure": discount})
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
f"azure/{model}",
|
||||
{
|
||||
"litellm_provider": "azure",
|
||||
"mode": "chat",
|
||||
"regional_endpoint_uplift_multiplier": 1.0,
|
||||
**({"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {}),
|
||||
},
|
||||
)
|
||||
logging: Final = Logging(
|
||||
model=model,
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
start_time=NOW,
|
||||
litellm_call_id="pricing-header",
|
||||
function_id="pricing-header",
|
||||
)
|
||||
logging.update_environment_variables(
|
||||
model=model, custom_llm_provider="azure", litellm_params={}, optional_params={}
|
||||
)
|
||||
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
|
||||
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
|
||||
assert finalized is not None
|
||||
assert finalized.amount == pytest.approx((price or 0.0) * 15 * (1 - discount))
|
||||
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
response_cost=finalized.amount,
|
||||
litellm_logging_obj=logging,
|
||||
)
|
||||
assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"source", ["unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second"]
|
||||
)
|
||||
@pytest.mark.parametrize("guardrail_cost", [0.0, 0.1])
|
||||
async def test_zero_cost_headers_preserve_known_free_calls_and_guardrail_charges(
|
||||
source: Literal["unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second"],
|
||||
guardrail_cost: float,
|
||||
) -> None:
|
||||
model: Final = "unmapped-header-cost"
|
||||
logging: Final = Logging(
|
||||
model=model,
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="adelete_responses" if source == "non-inference" else "acompletion",
|
||||
start_time=NOW,
|
||||
litellm_call_id="zero-cost-header",
|
||||
function_id="zero-cost-header",
|
||||
)
|
||||
logging.update_environment_variables(
|
||||
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
|
||||
)
|
||||
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
|
||||
result._hidden_params = {"response_cost": 0.0, "cache_hit": source == "cache"}
|
||||
match source:
|
||||
case "provider":
|
||||
logging._provider_reported_cost = 0.0
|
||||
case "provider-header":
|
||||
result._hidden_params["additional_headers"] = {"llm_provider-x-litellm-response-cost": 0.0}
|
||||
case "per-token":
|
||||
logging.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}
|
||||
case "per-second":
|
||||
logging.custom_cost_per_second = 0.0
|
||||
case _:
|
||||
pass
|
||||
finalized: Final = await finalize_response_cost(logging, result, None, NOW, guardrail_cost=guardrail_cost)
|
||||
assert finalized is not None
|
||||
assert finalized.amount == guardrail_cost
|
||||
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
response_cost=finalized.amount,
|
||||
litellm_logging_obj=logging,
|
||||
)
|
||||
assert headers.get("x-litellm-response-cost") == (
|
||||
None if source == "unpriced" and guardrail_cost == 0 else str(guardrail_cost)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_free_off_peak_pricing_keeps_the_finalized_zero_cost_header(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
model: Final = "gpt-free-off-peak-header"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
|
||||
},
|
||||
)
|
||||
logging: Final = Logging(
|
||||
model=model,
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
start_time=NOW,
|
||||
litellm_call_id="off-peak-header",
|
||||
function_id="off-peak-header",
|
||||
)
|
||||
logging.update_environment_variables(
|
||||
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
|
||||
)
|
||||
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
|
||||
result._hidden_params["response_cost"] = 0.0
|
||||
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
|
||||
assert finalized is not None and finalized.amount == 0.0
|
||||
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
response_cost=finalized.amount,
|
||||
litellm_logging_obj=logging,
|
||||
)
|
||||
assert headers["x-litellm-response-cost"] == "0.0"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("mapped", [False, True], ids=["unmapped-rule", "fill-missing-rule"])
|
||||
@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"])
|
||||
async def test_rule_pricing_preserves_free_headers_without_treating_capabilities_as_prices(
|
||||
monkeypatch: pytest.MonkeyPatch, mapped: bool, price: float | None
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
get_fallback_generalization_rules,
|
||||
set_fallback_generalizations,
|
||||
)
|
||||
|
||||
model: Final = "gpt-rule-header-model"
|
||||
previous_rules: Final = TypeAdapter(list[object]).validate_python(get_fallback_generalization_rules())
|
||||
if mapped:
|
||||
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat"})
|
||||
try:
|
||||
set_fallback_generalizations(
|
||||
[
|
||||
{
|
||||
"name": "header-pricing",
|
||||
"pattern": model,
|
||||
"fill_missing_for_providers": ["openai"],
|
||||
"model_info": {
|
||||
"mode": "chat",
|
||||
**(
|
||||
{"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {}
|
||||
),
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
logging: Final = Logging(
|
||||
model=model,
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
start_time=NOW,
|
||||
litellm_call_id="rule-header",
|
||||
function_id="rule-header",
|
||||
)
|
||||
logging.update_environment_variables(
|
||||
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
|
||||
)
|
||||
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
|
||||
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
|
||||
assert finalized is not None
|
||||
assert finalized.amount == pytest.approx((price or 0.0) * 15)
|
||||
headers: Final = TypeAdapter(dict[str, str]).validate_python(
|
||||
ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
response_cost=finalized.amount,
|
||||
litellm_logging_obj=logging,
|
||||
)
|
||||
)
|
||||
assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount))
|
||||
finally:
|
||||
set_fallback_generalizations(previous_rules)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue