mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(providers): apply cost discount and margin to Kyma's reported cost and guard bad values
Kyma's usage.cost now takes cost_discount_config and cost_margin_config, falls back to the catalog price when negative or non-finite, yields to a deployment base_model, and reaches /v1beta/interactions through the Responses bridge usage
This commit is contained in:
parent
af6a35f3f9
commit
b55f14e121
5 changed files with 132 additions and 7 deletions
|
|
@ -2210,6 +2210,16 @@ def _billable_header_cost(value: object) -> float | None:
|
|||
return cost if math.isfinite(cost) and cost >= 0 else None
|
||||
|
||||
|
||||
def _with_cost_adjustments(
|
||||
cost: float, custom_llm_provider: str | None, model: str | None, region_name: str | None
|
||||
) -> float:
|
||||
discounted_cost, _, _ = _apply_cost_discount(
|
||||
base_cost=cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name
|
||||
)
|
||||
adjusted_cost, _, _, _ = _apply_cost_margin(base_cost=discounted_cost, custom_llm_provider=custom_llm_provider)
|
||||
return adjusted_cost
|
||||
|
||||
|
||||
def response_cost_calculator(
|
||||
response_object: ModelResponse
|
||||
| EmbeddingResponse
|
||||
|
|
@ -2293,8 +2303,8 @@ def response_cost_calculator(
|
|||
if provider_response_cost is not None and not custom_pricing:
|
||||
return provider_response_cost
|
||||
usage_cost: Final = billed_usage_cost(response_object, custom_llm_provider)
|
||||
if usage_cost is not None and not custom_pricing:
|
||||
return usage_cost
|
||||
if usage_cost is not None and not custom_pricing and base_model is None:
|
||||
return _with_cost_adjustments(usage_cost, custom_llm_provider, model, region_name)
|
||||
|
||||
response_cost = completion_cost(
|
||||
completion_response=response_object,
|
||||
|
|
|
|||
|
|
@ -266,9 +266,11 @@ class LiteLLMResponsesInteractionsConfig:
|
|||
# (total_input_tokens, total_output_tokens)
|
||||
usage: Final[object] = getattr(responses_response, "usage", None)
|
||||
if usage:
|
||||
reported_cost: Final[object] = getattr(usage, "cost", None)
|
||||
interactions_response_dict["usage"] = {
|
||||
"total_input_tokens": getattr(usage, "input_tokens", 0),
|
||||
"total_output_tokens": getattr(usage, "output_tokens", 0),
|
||||
**({} if reported_cost is None else {"cost": reported_cost}),
|
||||
}
|
||||
|
||||
# Add updated (same as created for now)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
## Helper utilities
|
||||
import copy
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Collection, Iterable, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
|
|
@ -972,13 +973,14 @@ def chunk_usage(chunk: object) -> object:
|
|||
return getattr(chunk, "usage", None)
|
||||
|
||||
|
||||
_USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
|
||||
USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
|
||||
|
||||
|
||||
def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
|
||||
if custom_llm_provider not in _USAGE_COST_BILLED_PROVIDERS:
|
||||
if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
|
||||
return None
|
||||
return provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
|
||||
cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
|
||||
return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None
|
||||
|
||||
|
||||
_MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"})
|
||||
|
|
|
|||
|
|
@ -12,7 +12,11 @@ from typing_extensions import ReadOnly, Required
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
USAGE_COST_BILLED_PROVIDERS,
|
||||
chunk_usage,
|
||||
provider_reported_usage_cost,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -1351,7 +1355,7 @@ def stream_chunk_builder_text_completion(
|
|||
return TextCompletionResponse(**response)
|
||||
|
||||
|
||||
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value})
|
||||
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
|
||||
|
||||
|
||||
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import contextlib
|
|||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
|
@ -11,7 +12,9 @@ import respx
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage
|
||||
|
||||
_REPO_ROOT: Final = Path(__file__).resolve().parents[4]
|
||||
_PRIMARY_PRICES: Final = json.loads((_REPO_ROOT / "model_prices_and_context_window.json").read_text())
|
||||
|
|
@ -202,6 +205,110 @@ def test_deployment_pricing_overrides_the_cost_kyma_reports():
|
|||
assert response._hidden_params["response_cost"] == pytest.approx(4 * 0.001 + 3 * 0.002)
|
||||
|
||||
|
||||
|
||||
def _catalog_price(model: str, prompt_tokens: int, completion_tokens: int) -> float:
|
||||
row: Final = _PRIMARY_PRICES[model]
|
||||
return prompt_tokens * row["input_cost_per_token"] + completion_tokens * row["output_cost_per_token"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("reported_cost", ("-0.5", "1e999"))
|
||||
def test_unusable_cost_from_kyma_falls_back_to_the_catalog_price(reported_cost: str):
|
||||
usage: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": "REPORTED_COST"}
|
||||
with respx.mock() as upstream:
|
||||
upstream.post(_CHAT_URL).respond(
|
||||
200,
|
||||
content=json.dumps(_kyma_chat_body(usage)).replace('"REPORTED_COST"', reported_cost),
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k")
|
||||
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(_catalog_price("kyma/sonar", 4, 3))
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_deployment_base_model_overrides_the_cost_kyma_reports():
|
||||
with respx.mock() as upstream:
|
||||
upstream.post(_CHAT_URL).respond(
|
||||
200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721})
|
||||
)
|
||||
response: Final = litellm.completion(
|
||||
model="kyma/sonar",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
api_key="k",
|
||||
base_model="kyma/qwen3.7-flash",
|
||||
)
|
||||
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(_catalog_price("kyma/qwen3.7-flash", 4, 3))
|
||||
|
||||
|
||||
def test_interaction_on_kyma_is_billed_at_the_cost_kyma_reports():
|
||||
reported_cost: Final = 0.00721
|
||||
with respx.mock() as upstream:
|
||||
upstream.post(_CHAT_URL).respond(
|
||||
200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": reported_cost})
|
||||
)
|
||||
interaction: Final = litellm.interactions.create(model="kyma/qwen3.7-flash", input="ping", api_key="k")
|
||||
|
||||
assert interaction._hidden_params["response_cost"] == pytest.approx(reported_cost)
|
||||
|
||||
|
||||
def _with_kyma_discount_and_margin(monkeypatch: pytest.MonkeyPatch) -> float:
|
||||
monkeypatch.setattr(litellm, "cost_discount_config", {"kyma": 0.5})
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"kyma": 0.1})
|
||||
return 0.5 * 1.1
|
||||
|
||||
|
||||
def test_cost_kyma_reports_takes_the_configured_discount_and_margin(monkeypatch: pytest.MonkeyPatch):
|
||||
adjustment: Final = _with_kyma_discount_and_margin(monkeypatch)
|
||||
with respx.mock() as upstream:
|
||||
upstream.post(_CHAT_URL).respond(
|
||||
200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.008})
|
||||
)
|
||||
response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k")
|
||||
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(0.008 * adjustment)
|
||||
|
||||
|
||||
def _kyma_stream_chunk(content: str, finish_reason: str | None = None) -> ModelResponseStream:
|
||||
return ModelResponseStream(
|
||||
id="chatcmpl_kyma",
|
||||
created=1_792_000_000,
|
||||
model="sonar",
|
||||
object="chat.completion.chunk",
|
||||
choices=[StreamingChoices(finish_reason=finish_reason, index=0, delta=Delta(content=content, role="assistant"))],
|
||||
)
|
||||
|
||||
|
||||
def test_streamed_cost_kyma_reports_takes_the_configured_discount_and_margin(monkeypatch: pytest.MonkeyPatch):
|
||||
adjustment: Final = _with_kyma_discount_and_margin(monkeypatch)
|
||||
usage_chunk: Final = _kyma_stream_chunk("")
|
||||
usage_chunk.usage = Usage(prompt_tokens=4, completion_tokens=3, total_tokens=7, cost=0.008)
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
model="sonar",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="kyma-call",
|
||||
function_id="kyma-function",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model="sonar",
|
||||
user=None,
|
||||
optional_params={},
|
||||
litellm_params={"custom_llm_provider": "kyma"},
|
||||
custom_llm_provider="kyma",
|
||||
)
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[_kyma_stream_chunk("pong", finish_reason="stop"), usage_chunk],
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
assert logging_obj.response_cost_calculator(result=response) == pytest.approx(0.008 * adjustment)
|
||||
|
||||
def test_kyma_model_missing_from_the_catalog_keeps_its_tools():
|
||||
tools: Final = [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
|
||||
with respx.mock() as upstream:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue