diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a61033b7e30..d0066e16a14 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2210,6 +2210,16 @@ def _billable_header_cost(value: object) -> float | None: return cost if math.isfinite(cost) and cost >= 0 else None +def _with_cost_adjustments( + cost: float, custom_llm_provider: str | None, model: str | None, region_name: str | None +) -> float: + discounted_cost, _, _ = _apply_cost_discount( + base_cost=cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name + ) + adjusted_cost, _, _, _ = _apply_cost_margin(base_cost=discounted_cost, custom_llm_provider=custom_llm_provider) + return adjusted_cost + + def response_cost_calculator( response_object: ModelResponse | EmbeddingResponse @@ -2293,8 +2303,8 @@ def response_cost_calculator( if provider_response_cost is not None and not custom_pricing: return provider_response_cost usage_cost: Final = billed_usage_cost(response_object, custom_llm_provider) - if usage_cost is not None and not custom_pricing: - return usage_cost + if usage_cost is not None and not custom_pricing and base_model is None: + return _with_cost_adjustments(usage_cost, custom_llm_provider, model, region_name) response_cost = completion_cost( completion_response=response_object, diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index ade33831cdc..48c8b9a0ca0 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -266,9 +266,11 @@ class LiteLLMResponsesInteractionsConfig: # (total_input_tokens, total_output_tokens) usage: Final[object] = getattr(responses_response, "usage", None) if usage: + reported_cost: Final[object] = getattr(usage, "cost", None) interactions_response_dict["usage"] = { "total_input_tokens": getattr(usage, "input_tokens", 0), "total_output_tokens": getattr(usage, "output_tokens", 0), + **({} if reported_cost is None else {"cost": reported_cost}), } # Add updated (same as created for now) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 9f451d89396..1cbb9ab9973 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -2,6 +2,7 @@ ## Helper utilities import copy import logging +import math import re from collections.abc import Collection, Iterable, Iterator, Mapping from types import MappingProxyType @@ -972,13 +973,14 @@ def chunk_usage(chunk: object) -> object: return getattr(chunk, "usage", None) -_USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"}) +USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"}) def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None: - if custom_llm_provider not in _USAGE_COST_BILLED_PROVIDERS: + if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS: return None - return provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"}) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 8d243e572fa..b2feeebe9a1 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -12,7 +12,11 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost +from litellm.litellm_core_utils.core_helpers import ( + USAGE_COST_BILLED_PROVIDERS, + chunk_usage, + provider_reported_usage_cost, +) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -1351,7 +1355,7 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) +_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS def _logged_provider(logging_obj: Optional["Logging"]) -> object: diff --git a/tests/unit/llms/openai_like/test_kyma_provider.py b/tests/unit/llms/openai_like/test_kyma_provider.py index 4e63adc8f58..7fe4457ab88 100644 --- a/tests/unit/llms/openai_like/test_kyma_provider.py +++ b/tests/unit/llms/openai_like/test_kyma_provider.py @@ -2,6 +2,7 @@ import contextlib import json import re from collections.abc import Callable +from datetime import datetime from itertools import chain from pathlib import Path from typing import Final @@ -11,7 +12,9 @@ import respx import litellm from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.proxy.auth.model_checks import get_known_models_from_wildcard +from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage _REPO_ROOT: Final = Path(__file__).resolve().parents[4] _PRIMARY_PRICES: Final = json.loads((_REPO_ROOT / "model_prices_and_context_window.json").read_text()) @@ -202,6 +205,110 @@ def test_deployment_pricing_overrides_the_cost_kyma_reports(): assert response._hidden_params["response_cost"] == pytest.approx(4 * 0.001 + 3 * 0.002) + +def _catalog_price(model: str, prompt_tokens: int, completion_tokens: int) -> float: + row: Final = _PRIMARY_PRICES[model] + return prompt_tokens * row["input_cost_per_token"] + completion_tokens * row["output_cost_per_token"] + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("reported_cost", ("-0.5", "1e999")) +def test_unusable_cost_from_kyma_falls_back_to_the_catalog_price(reported_cost: str): + usage: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": "REPORTED_COST"} + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, + content=json.dumps(_kyma_chat_body(usage)).replace('"REPORTED_COST"', reported_cost), + headers={"content-type": "application/json"}, + ) + response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k") + + assert response._hidden_params["response_cost"] == pytest.approx(_catalog_price("kyma/sonar", 4, 3)) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_deployment_base_model_overrides_the_cost_kyma_reports(): + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721}) + ) + response: Final = litellm.completion( + model="kyma/sonar", + messages=[{"role": "user", "content": "ping"}], + api_key="k", + base_model="kyma/qwen3.7-flash", + ) + + assert response._hidden_params["response_cost"] == pytest.approx(_catalog_price("kyma/qwen3.7-flash", 4, 3)) + + +def test_interaction_on_kyma_is_billed_at_the_cost_kyma_reports(): + reported_cost: Final = 0.00721 + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": reported_cost}) + ) + interaction: Final = litellm.interactions.create(model="kyma/qwen3.7-flash", input="ping", api_key="k") + + assert interaction._hidden_params["response_cost"] == pytest.approx(reported_cost) + + +def _with_kyma_discount_and_margin(monkeypatch: pytest.MonkeyPatch) -> float: + monkeypatch.setattr(litellm, "cost_discount_config", {"kyma": 0.5}) + monkeypatch.setattr(litellm, "cost_margin_config", {"kyma": 0.1}) + return 0.5 * 1.1 + + +def test_cost_kyma_reports_takes_the_configured_discount_and_margin(monkeypatch: pytest.MonkeyPatch): + adjustment: Final = _with_kyma_discount_and_margin(monkeypatch) + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.008}) + ) + response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k") + + assert response._hidden_params["response_cost"] == pytest.approx(0.008 * adjustment) + + +def _kyma_stream_chunk(content: str, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl_kyma", + created=1_792_000_000, + model="sonar", + object="chat.completion.chunk", + choices=[StreamingChoices(finish_reason=finish_reason, index=0, delta=Delta(content=content, role="assistant"))], + ) + + +def test_streamed_cost_kyma_reports_takes_the_configured_discount_and_margin(monkeypatch: pytest.MonkeyPatch): + adjustment: Final = _with_kyma_discount_and_margin(monkeypatch) + usage_chunk: Final = _kyma_stream_chunk("") + usage_chunk.usage = Usage(prompt_tokens=4, completion_tokens=3, total_tokens=7, cost=0.008) + logging_obj: Final = LiteLLMLogging( + model="sonar", + messages=[{"role": "user", "content": "ping"}], + stream=True, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="kyma-call", + function_id="kyma-function", + ) + logging_obj.update_environment_variables( + model="sonar", + user=None, + optional_params={}, + litellm_params={"custom_llm_provider": "kyma"}, + custom_llm_provider="kyma", + ) + + response: Final = litellm.stream_chunk_builder( + chunks=[_kyma_stream_chunk("pong", finish_reason="stop"), usage_chunk], + messages=[{"role": "user", "content": "ping"}], + logging_obj=logging_obj, + ) + + assert logging_obj.response_cost_calculator(result=response) == pytest.approx(0.008 * adjustment) + def test_kyma_model_missing_from_the_catalog_keeps_its_tools(): tools: Final = [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}] with respx.mock() as upstream: