mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
fix(cost): bill a provider reported usage.cost on every path unless the deployment is priced (#48953)
* fix(cost): bill a provider reported usage.cost on every path unless the deployment is priced * test(integration): a reported cost is attributed to output in the gateway parity cases * fix(cost): skip a chat usage.cost that already holds litellm's stamped bill and Responses API objects * fix(cost): record litellm's bill on the Responses terminal event so a later pricing pass does not read it as the upstream's charge * fix(cost): mark usage.cost litellm wrote as its own bill and keep a breakdown for a reported cost * fix(cost): keep the token breakdown when a header reported cost repeats an already priced bill * fix(cost): read the stored breakdown total through the logging object's typed field * refactor(cost): name the providers billed from their own usage fields for what they are * fix(cost): keep the upstream's usage.cost beside litellm's bill and stamp a zero operator price on the stream frame --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu
GitHub
parent
369f3cb5f0
commit
733cfeba93
10 changed files with 387 additions and 60 deletions
|
|
@ -1544,6 +1544,12 @@ def _response_time_ms_for_cost(
|
|||
return litellm_logging_obj.get_response_ms()
|
||||
|
||||
|
||||
def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None:
|
||||
breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown
|
||||
total: Final[object] = None if breakdown is None else breakdown.get("total_cost")
|
||||
return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
|
||||
|
||||
|
||||
def _adjusted_provider_billed_cost(
|
||||
reported_cost: float,
|
||||
custom_llm_provider: str | None,
|
||||
|
|
@ -1561,9 +1567,9 @@ def _adjusted_provider_billed_cost(
|
|||
)
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=None,
|
||||
completion_tokens_cost_usd_dollar=None,
|
||||
cost_for_built_in_tools_cost_usd_dollar=None,
|
||||
prompt_tokens_cost_usd_dollar=0.0,
|
||||
completion_tokens_cost_usd_dollar=adjusted_cost,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
total_cost_usd_dollar=adjusted_cost,
|
||||
original_cost=reported_cost,
|
||||
discount_percent=discount_percent,
|
||||
|
|
@ -2376,6 +2382,15 @@ def response_cost_calculator(
|
|||
)
|
||||
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
|
||||
if provider_response_cost is not None and not custom_pricing:
|
||||
if _breakdown_total(litellm_logging_obj) != provider_response_cost:
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=0.0,
|
||||
completion_tokens_cost_usd_dollar=provider_response_cost,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
total_cost_usd_dollar=provider_response_cost,
|
||||
original_cost=provider_response_cost,
|
||||
)
|
||||
return provider_response_cost
|
||||
|
||||
response_cost = completion_cost(
|
||||
|
|
|
|||
|
|
@ -1017,13 +1017,44 @@ def chunk_usage(chunk: object) -> object:
|
|||
return getattr(chunk, "usage", None)
|
||||
|
||||
|
||||
USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
|
||||
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: Final = frozenset({"xai", "perplexity", "opencode_go"})
|
||||
|
||||
|
||||
UPSTREAM_USAGE_COST: Final = "upstream_usage_cost"
|
||||
|
||||
|
||||
def _numeric(value: object) -> float | None:
|
||||
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
|
||||
|
||||
|
||||
def stamp_litellm_bill_on_usage(response: object, usage: object, bill: float) -> None:
|
||||
"""Writes litellm's bill into ``usage.cost`` and the hidden ``response_cost`` cache, keeping the upstream's own
|
||||
``usage.cost`` under ``upstream_usage_cost`` so a later pricing pass still bills the upstream's charge instead of
|
||||
reading litellm's bill back as if the upstream had reported it."""
|
||||
hidden: Final[object] = getattr(response, "_hidden_params", None)
|
||||
if isinstance(hidden, dict):
|
||||
hidden_params: Final = cast(dict[str, object], hidden) # cast-ok: hidden params ride on the response as a dict
|
||||
upstream: Final = (
|
||||
hidden_params[UPSTREAM_USAGE_COST]
|
||||
if UPSTREAM_USAGE_COST in hidden_params
|
||||
else _numeric(getattr(usage, "cost", None))
|
||||
)
|
||||
hidden_params.update({"response_cost": bill, UPSTREAM_USAGE_COST: upstream})
|
||||
setattr(usage, "cost", bill)
|
||||
|
||||
|
||||
def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
|
||||
if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
|
||||
"""The charge an OpenAI-compatible upstream reports in ``usage.cost``, skipped for providers billed from their own
|
||||
usage fields. Once litellm wrote its own bill into ``usage.cost`` (the stream builder and the Responses terminal
|
||||
event do), the upstream's figure is read from ``upstream_usage_cost`` instead."""
|
||||
if custom_llm_provider in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS:
|
||||
return None
|
||||
cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
|
||||
hidden: Final = as_str_mapping(getattr(response, "_hidden_params", None))
|
||||
cost: Final = (
|
||||
_numeric(hidden[UPSTREAM_USAGE_COST])
|
||||
if hidden is not None and UPSTREAM_USAGE_COST in hidden
|
||||
else provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
|
||||
)
|
||||
return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,9 +13,10 @@ from typing_extensions import ReadOnly, Required
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
USAGE_COST_BILLED_PROVIDERS,
|
||||
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
|
||||
chunk_usage,
|
||||
provider_reported_usage_cost,
|
||||
stamp_litellm_bill_on_usage,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
|
||||
|
|
@ -1380,9 +1381,6 @@ def stream_chunk_builder_text_completion(
|
|||
return TextCompletionResponse(**response)
|
||||
|
||||
|
||||
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
|
||||
|
||||
|
||||
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
|
||||
if logging_obj is None:
|
||||
return None
|
||||
|
|
@ -1390,10 +1388,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object:
|
|||
return provider
|
||||
|
||||
|
||||
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
|
||||
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
|
||||
|
||||
|
||||
def _stream_builder_response_cost(
|
||||
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
|
||||
) -> float | None:
|
||||
|
|
@ -1404,9 +1398,6 @@ def _stream_builder_response_cost(
|
|||
for chunk in reversed(chunks)
|
||||
)
|
||||
return next((cost for cost in reported if cost is not None), None)
|
||||
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
|
||||
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
|
||||
return float(usage_cost)
|
||||
if logging_obj is not None:
|
||||
return None
|
||||
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
|
||||
|
|
@ -1453,13 +1444,15 @@ def _set_stream_builder_response_cost(
|
|||
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
|
||||
if logging_obj is None:
|
||||
return
|
||||
if isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
|
||||
getattr(usage, "cost", None), (int, float)
|
||||
):
|
||||
return
|
||||
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
|
||||
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:
|
||||
if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0:
|
||||
return
|
||||
setattr(usage, "cost", computed_cost)
|
||||
_set_hidden_response_cost(response, float(computed_cost))
|
||||
if computed_cost > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
stamp_litellm_bill_on_usage(response, usage, float(computed_cost))
|
||||
|
||||
|
||||
def _field_of(value: object, name: str) -> object:
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
process_response_headers,
|
||||
provider_reported_usage_cost,
|
||||
set_response_cost_in_hidden_params,
|
||||
stamp_litellm_bill_on_usage,
|
||||
)
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
|
|
@ -1869,7 +1870,7 @@ def stamp_responses_usage_cost(
|
|||
if isinstance(bill, bool) or not isinstance(bill, (int, float)):
|
||||
return
|
||||
if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)):
|
||||
setattr(usage_obj, "cost", float(bill))
|
||||
stamp_litellm_bill_on_usage(response_obj, usage_obj, float(bill))
|
||||
|
||||
|
||||
def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, cast
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
|
|
@ -11878,7 +11879,215 @@ GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JS
|
|||
}
|
||||
)
|
||||
|
||||
|
||||
# An OpenAI-compatible gateway that reports what it charged under usage.cost (#47533). The reported total is the bill,
|
||||
# split across input and output by the ratio of the cost map rates (100 * 1e-6 : 20 * 3e-6), on plain and streamed
|
||||
# replies alike. A deployment the operator priced keeps the operator's rates on both.
|
||||
_GATEWAY_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037}
|
||||
_GATEWAY_REPORTED_EXPECTED: Final = ExactExpected(
|
||||
spend=0.037, input_cost=0.0, output_cost=0.037, prompt_tokens=100, completion_tokens=20
|
||||
)
|
||||
_GATEWAY_OPERATOR_PRICED: Final = Deployment(input_cost_per_token=2e-06, output_cost_per_token=6e-06)
|
||||
_GATEWAY_OPERATOR_PRICED_EXPECTED: Final = ExactExpected(
|
||||
spend=0.00032, input_cost=0.0002, output_cost=0.00012, prompt_tokens=100, completion_tokens=20
|
||||
)
|
||||
_GATEWAY_CHUNK: Final = {
|
||||
"id": "chatcmpl-$REQUEST_ID",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gateway-reported-cost-fixture",
|
||||
}
|
||||
|
||||
GATEWAY_REPORTED_COST: Final = CostTrackingTestCase(
|
||||
name="gateway-reported-cost",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="gateway-reported-cost-fixture",
|
||||
endpoint="/v1/chat/completions",
|
||||
request={"model": "$MODEL", "messages": [{"role": "user", "content": "reported cost"}], "stream": False},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "chatcmpl-$REQUEST_ID",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gateway-reported-cost-fixture",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": _GATEWAY_REPORTED_USAGE,
|
||||
},
|
||||
),
|
||||
expected=_GATEWAY_REPORTED_EXPECTED,
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_STREAM: Final = CostTrackingTestCase(
|
||||
name="gateway-reported-cost-stream",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="gateway-reported-cost-fixture",
|
||||
endpoint="/v1/chat/completions",
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"messages": [{"role": "user", "content": "reported cost"}],
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
response=SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]},
|
||||
{**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_REPORTED_USAGE},
|
||||
done=True,
|
||||
),
|
||||
),
|
||||
expected=_GATEWAY_REPORTED_EXPECTED,
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-cost-operator_priced",
|
||||
"deployment": _GATEWAY_OPERATOR_PRICED,
|
||||
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
|
||||
}
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STREAM.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-cost-operator_priced-stream",
|
||||
"deployment": _GATEWAY_OPERATOR_PRICED,
|
||||
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
|
||||
}
|
||||
)
|
||||
|
||||
_GATEWAY_RESPONSES_USAGE: Final[Mapping[str, JsonValue]] = {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.037}
|
||||
_GATEWAY_RESPONSES_ITEM: Final[Mapping[str, JsonValue]] = {
|
||||
"id": "msg_gateway",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
|
||||
}
|
||||
_GATEWAY_RESPONSES_BODY: Final[Mapping[str, JsonValue]] = {
|
||||
"id": "resp_$REQUEST_ID",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gateway-reported-cost-fixture",
|
||||
"output": [_GATEWAY_RESPONSES_ITEM],
|
||||
"usage": _GATEWAY_RESPONSES_USAGE,
|
||||
}
|
||||
|
||||
GATEWAY_REPORTED_COST_RESPONSES: Final = CostTrackingTestCase(
|
||||
name="gateway-reported-cost-responses",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="gateway-reported-cost-fixture",
|
||||
endpoint="/v1/responses",
|
||||
request={"model": "$MODEL", "input": "reported cost", "cache": {"no-cache": True}},
|
||||
response=JsonResponse(content_type="application/json", body=_GATEWAY_RESPONSES_BODY),
|
||||
expected=_GATEWAY_REPORTED_EXPECTED,
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_RESPONSES_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-cost-responses-stream",
|
||||
"request": {**GATEWAY_REPORTED_COST_RESPONSES.request, "stream": True},
|
||||
"response": SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"type": "response.created",
|
||||
"sequence_number": 0,
|
||||
"response": {**_GATEWAY_RESPONSES_BODY, "status": "in_progress", "output": [], "usage": None},
|
||||
},
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"sequence_number": 1,
|
||||
"item_id": "msg_gateway",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "ok",
|
||||
},
|
||||
{"type": "response.completed", "sequence_number": 2, "response": _GATEWAY_RESPONSES_BODY},
|
||||
),
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-cost-responses-operator_priced",
|
||||
"deployment": _GATEWAY_OPERATOR_PRICED,
|
||||
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
|
||||
}
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES_STREAM.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-cost-responses-operator_priced-stream",
|
||||
"deployment": _GATEWAY_OPERATOR_PRICED,
|
||||
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_GATEWAY_ZERO_USAGE: Final = {**_GATEWAY_REPORTED_USAGE, "cost": 0}
|
||||
|
||||
GATEWAY_REPORTED_ZERO_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-zero-operator_priced",
|
||||
"response": JsonResponse(
|
||||
content_type="application/json",
|
||||
body={**GATEWAY_REPORTED_COST.response.body, "usage": _GATEWAY_ZERO_USAGE},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-zero-operator_priced-stream",
|
||||
"response": SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]},
|
||||
{**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_ZERO_USAGE},
|
||||
done=True,
|
||||
),
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
_GATEWAY_OPENROUTER_OPERATOR_PRICED: Final = _GATEWAY_OPERATOR_PRICED.model_copy(
|
||||
update={"model": "openrouter/vendor/gateway-reported-cost-fixture"}
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-cost-openrouter-operator_priced",
|
||||
"deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED,
|
||||
}
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM: Final = (
|
||||
GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-cost-openrouter-operator_priced-stream",
|
||||
"deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
GATEWAY_REPORTED_COST,
|
||||
GATEWAY_REPORTED_COST_STREAM,
|
||||
GATEWAY_REPORTED_COST_OPERATOR_PRICED,
|
||||
GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM,
|
||||
GATEWAY_REPORTED_COST_RESPONSES,
|
||||
GATEWAY_REPORTED_COST_RESPONSES_STREAM,
|
||||
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED,
|
||||
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM,
|
||||
GATEWAY_REPORTED_ZERO_OPERATOR_PRICED,
|
||||
GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM,
|
||||
GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED,
|
||||
GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM,
|
||||
GPT_5_6_SLASH_ALIAS,
|
||||
GPT_5_6_SLASH_ALIAS_STREAM,
|
||||
GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON,
|
||||
|
|
@ -12120,6 +12329,22 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
StreamParityTestCase(plain=GATEWAY_REPORTED_COST, streamed=GATEWAY_REPORTED_COST_STREAM),
|
||||
StreamParityTestCase(
|
||||
plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED,
|
||||
streamed=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM,
|
||||
),
|
||||
StreamParityTestCase(plain=GATEWAY_REPORTED_COST_RESPONSES, streamed=GATEWAY_REPORTED_COST_RESPONSES_STREAM),
|
||||
StreamParityTestCase(
|
||||
plain=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED,
|
||||
streamed=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM,
|
||||
),
|
||||
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM),
|
||||
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE),
|
||||
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED),
|
||||
|
|
|
|||
|
|
@ -26,6 +26,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"input_cost_per_token_above_128k_tokens": 2.2e-07,
|
||||
"output_cost_per_token_above_128k_tokens": 8.8e-07,
|
||||
},
|
||||
"gateway-reported-cost-fixture": {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 3e-06,
|
||||
},
|
||||
"openrouter/anthropic/claude-sonnet-5": {
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
|
|
|
|||
|
|
@ -69,9 +69,7 @@ def test_preserve_upstream_non_openai_attributes_keeps_scalar_and_structured_val
|
|||
scalar_attribute="scalar",
|
||||
structured_attribute={"items": ["value"]},
|
||||
)
|
||||
model_response: Final = ModelResponseStream(
|
||||
choices=[StreamingChoices(index=0, delta=Delta(content="hello"))]
|
||||
)
|
||||
model_response: Final = ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="hello"))])
|
||||
|
||||
preserve_upstream_non_openai_attributes(model_response, original_chunk)
|
||||
|
||||
|
|
@ -262,9 +260,7 @@ class TestMapFinishReasonAnthropic:
|
|||
("content_filtered", "content_filter"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_finish_reasons(
|
||||
self, provider_reason: str, expected: str
|
||||
) -> None:
|
||||
def test_anthropic_finish_reasons(self, provider_reason: str, expected: str) -> None:
|
||||
assert map_finish_reason(provider_reason) == expected
|
||||
|
||||
def test_refusal(self):
|
||||
|
|
@ -336,9 +332,7 @@ class TestMapFinishReasonZhipu:
|
|||
|
||||
|
||||
class TestMapFinishReasonOpenAIPassthrough:
|
||||
@pytest.mark.parametrize(
|
||||
"reason", ["stop", "length", "tool_calls", "function_call", "content_filter"]
|
||||
)
|
||||
@pytest.mark.parametrize("reason", ["stop", "length", "tool_calls", "function_call", "content_filter"])
|
||||
def test_openai_values_pass_through(self, reason):
|
||||
assert map_finish_reason(reason) == reason
|
||||
|
||||
|
|
@ -349,9 +343,7 @@ class TestMapFinishReasonGenericError:
|
|||
assert map_finish_reason("error") == "stop"
|
||||
|
||||
def test_lowercase_error_does_not_warn(self, mocker):
|
||||
warn = mocker.patch(
|
||||
"litellm.litellm_core_utils.core_helpers.verbose_logger.warning"
|
||||
)
|
||||
warn = mocker.patch("litellm.litellm_core_utils.core_helpers.verbose_logger.warning")
|
||||
assert map_finish_reason("error") == "stop"
|
||||
warn.assert_not_called()
|
||||
|
||||
|
|
@ -369,8 +361,7 @@ class TestFinishReasonMapOutputsAreValid:
|
|||
"""Every value in _FINISH_REASON_MAP must be a valid OpenAI finish reason."""
|
||||
for provider_reason, openai_reason in _FINISH_REASON_MAP.items():
|
||||
assert openai_reason in VALID_OPENAI_FINISH_REASONS, (
|
||||
f"Mapped value '{openai_reason}' (from '{provider_reason}') "
|
||||
f"is not a valid OpenAI finish reason"
|
||||
f"Mapped value '{openai_reason}' (from '{provider_reason}') is not a valid OpenAI finish reason"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -380,28 +371,18 @@ class TestRedactNestedMatchAndRegexKeys:
|
|||
"assessments": [
|
||||
{
|
||||
"sensitiveInformationPolicy": {
|
||||
"piiEntities": [
|
||||
{"type": "NAME", "match": "secret-name", "action": "BLOCKED"}
|
||||
]
|
||||
},
|
||||
"wordPolicy": {
|
||||
"customWords": [{"match": "badword", "action": "BLOCKED"}]
|
||||
"piiEntities": [{"type": "NAME", "match": "secret-name", "action": "BLOCKED"}]
|
||||
},
|
||||
"wordPolicy": {"customWords": [{"match": "badword", "action": "BLOCKED"}]},
|
||||
}
|
||||
],
|
||||
"regex": "should-redact-key-named-regex",
|
||||
}
|
||||
out = redact_nested_match_and_regex_keys(payload)
|
||||
assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][
|
||||
"match"
|
||||
] == "[REDACTED]"
|
||||
assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == (
|
||||
"[REDACTED]"
|
||||
)
|
||||
assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0]["match"] == "[REDACTED]"
|
||||
assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == ("[REDACTED]")
|
||||
assert out["regex"] == "[REDACTED]"
|
||||
assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][
|
||||
0
|
||||
]["match"] == "secret-name"
|
||||
assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0]["match"] == "secret-name"
|
||||
|
||||
def test_passes_through_none_and_str(self):
|
||||
assert redact_nested_match_and_regex_keys(None) is None
|
||||
|
|
@ -564,13 +545,17 @@ class TestIsExpectedClientError:
|
|||
assert is_expected_client_error(over_budget) is True
|
||||
|
||||
litellm_limit = RateLimitError(
|
||||
message="key over rpm", llm_provider="anthropic", model="claude-haiku-4-5",
|
||||
message="key over rpm",
|
||||
llm_provider="anthropic",
|
||||
model="claude-haiku-4-5",
|
||||
category=RateLimitErrorCategory.LITELLM_RATE_LIMIT,
|
||||
)
|
||||
assert is_expected_client_error(litellm_limit) is True
|
||||
|
||||
vendor_limit = RateLimitError(
|
||||
message="rate limited upstream", llm_provider="anthropic", model="claude-haiku-4-5",
|
||||
message="rate limited upstream",
|
||||
llm_provider="anthropic",
|
||||
model="claude-haiku-4-5",
|
||||
category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
|
||||
)
|
||||
assert is_expected_client_error(vendor_limit) is False
|
||||
|
|
@ -803,3 +788,34 @@ class TestReceiveStampSurvivesASecondMetadataBucket:
|
|||
}
|
||||
|
||||
assert get_litellm_metadata_from_kwargs(kwargs)["litellm_received_at"] is own
|
||||
|
||||
|
||||
def test_billed_usage_cost_bills_any_chat_upstream_without_its_own_reader() -> None:
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import billed_usage_cost
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037)
|
||||
chat = ModelResponse(usage=usage)
|
||||
assert billed_usage_cost(chat, "openai") == 0.037
|
||||
assert billed_usage_cost(chat, "openrouter") == 0.037
|
||||
assert billed_usage_cost(chat, "xai") is None
|
||||
assert billed_usage_cost(SimpleNamespace(usage=usage), "openai") == 0.037
|
||||
|
||||
|
||||
def test_billed_usage_cost_keeps_the_upstream_charge_after_litellm_wrote_its_bill_into_usage_cost() -> None:
|
||||
from litellm.litellm_core_utils.core_helpers import billed_usage_cost, stamp_litellm_bill_on_usage
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
reported = ModelResponse(usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037))
|
||||
stamp_litellm_bill_on_usage(reported, reported.usage, 0.0407)
|
||||
stamp_litellm_bill_on_usage(reported, reported.usage, 0.0407)
|
||||
assert reported.usage.cost == 0.0407
|
||||
assert reported._hidden_params["response_cost"] == 0.0407
|
||||
assert billed_usage_cost(reported, "openai") == 0.037
|
||||
|
||||
unreported = ModelResponse(usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120))
|
||||
stamp_litellm_bill_on_usage(unreported, unreported.usage, 0.0015)
|
||||
assert unreported.usage.cost == 0.0015
|
||||
assert billed_usage_cost(unreported, "openai") is None
|
||||
|
|
|
|||
|
|
@ -2659,7 +2659,7 @@ def test_perplexity_streaming_dict_cost_bills_through_its_own_calculator():
|
|||
) == pytest.approx(0.00503)
|
||||
|
||||
|
||||
def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map():
|
||||
def test_openai_compatible_streaming_reported_cost_is_billed_instead_of_the_cost_map():
|
||||
import litellm
|
||||
from litellm.cost_calculator import (
|
||||
get_response_cost_from_hidden_params,
|
||||
|
|
@ -2681,7 +2681,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map():
|
|||
id="chatcmpl-openai-compatible",
|
||||
model=model,
|
||||
choices=[],
|
||||
usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=3_144_000),
|
||||
usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.037),
|
||||
)
|
||||
|
||||
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "openai")
|
||||
|
|
@ -2693,7 +2693,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map():
|
|||
custom_llm_provider="openai",
|
||||
call_type="completion",
|
||||
optional_params={},
|
||||
) == pytest.approx(2e-5)
|
||||
) == pytest.approx(0.037)
|
||||
|
||||
|
||||
def test_xai_streaming_reported_cost_still_takes_the_margin(monkeypatch):
|
||||
|
|
|
|||
|
|
@ -254,7 +254,7 @@ def test_interaction_on_kyma_is_billed_at_the_cost_kyma_reports():
|
|||
|
||||
|
||||
@pytest.mark.parametrize("model", ("openrouter/openai/gpt-4o-mini", "deepseek/deepseek-chat"))
|
||||
def test_interaction_on_another_provider_keeps_its_upstream_cost_out_of_the_body(model: str):
|
||||
def test_interaction_on_another_openai_compatible_provider_is_billed_at_its_reported_cost(model: str):
|
||||
chat_usage: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721}
|
||||
responses_body: Final = {
|
||||
"id": "resp_upstream",
|
||||
|
|
@ -278,7 +278,7 @@ def test_interaction_on_another_provider_keeps_its_upstream_cost_out_of_the_body
|
|||
upstream.route(path__regex=r".*/responses$").respond(200, json=responses_body)
|
||||
interaction: Final = litellm.interactions.create(model=model, input="ping", api_key="k")
|
||||
|
||||
assert "cost" not in interaction.model_dump(exclude_none=True)["usage"]
|
||||
assert interaction.model_dump(exclude_none=True)["usage"]["cost"] == pytest.approx(0.00721)
|
||||
|
||||
|
||||
def test_sdk_completion_cost_prices_kyma_at_the_cost_kyma_reports():
|
||||
|
|
@ -410,8 +410,8 @@ def test_cost_headers_show_the_discount_and_margin_taken_from_the_cost_kyma_repo
|
|||
assert float(headers["x-litellm-response-cost-discount-amount"] or "nan") == pytest.approx(0.004)
|
||||
assert float(headers["x-litellm-response-cost-margin-amount"] or "nan") == pytest.approx(0.0004)
|
||||
assert float(headers["x-litellm-response-cost"] or "nan") == pytest.approx(0.0044)
|
||||
assert headers["x-litellm-response-cost-input"] is None
|
||||
assert headers["x-litellm-response-cost-output"] is None
|
||||
assert float(headers["x-litellm-response-cost-input"] or "nan") == 0.0
|
||||
assert float(headers["x-litellm-response-cost-output"] or "nan") == pytest.approx(0.0044)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", ({"type": "enabled", "budget_tokens": 1024}, {"type": "adaptive"}))
|
||||
|
|
|
|||
|
|
@ -5443,6 +5443,46 @@ def test_stream_chunk_builder_keeps_provider_reported_usage_cost():
|
|||
assert response._hidden_params["response_cost"] == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_completion_cost_on_a_stamped_stream_still_bills_the_reported_charge():
|
||||
usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "")
|
||||
usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5)
|
||||
chunks: Final = [
|
||||
_stream_builder_text_chunk("gpt-4o", "Hello ", finish_reason="stop"),
|
||||
usage_chunk,
|
||||
]
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=_stream_builder_logging_obj()
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_stream_chunk_builder_stamps_a_zero_operator_price_over_a_reported_cost():
|
||||
logging_obj: Final = _stream_builder_logging_obj()
|
||||
logging_obj.litellm_params = {
|
||||
**logging_obj.litellm_params,
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
}
|
||||
logging_obj.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}
|
||||
usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "")
|
||||
usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5)
|
||||
chunks: Final = [
|
||||
_stream_builder_text_chunk("gpt-4o", "Hello ", finish_reason="stop"),
|
||||
usage_chunk,
|
||||
]
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert getattr(response.usage, "cost", None) == 0.0
|
||||
assert response._hidden_params["response_cost"] == 0.0
|
||||
|
||||
|
||||
def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk():
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue