fix(kyma): bill completion_cost from usage.cost, keep other providers' upstream cost out of interactions, map thinking, refuse images

This commit is contained in:
mateo-berri 2026-10-28 18:18:43 -07:00
parent 69a21bf218
commit d968ab3b91
7 changed files with 168 additions and 29 deletions

View file

@ -1396,9 +1396,9 @@ def _apply_cost_margin(
def _store_cost_breakdown_in_logging_obj(
litellm_logging_obj: LitellmLoggingObject | None,
prompt_tokens_cost_usd_dollar: float,
completion_tokens_cost_usd_dollar: float,
cost_for_built_in_tools_cost_usd_dollar: float,
prompt_tokens_cost_usd_dollar: float | None,
completion_tokens_cost_usd_dollar: float | None,
cost_for_built_in_tools_cost_usd_dollar: float | None,
total_cost_usd_dollar: float,
additional_costs: dict | None = None,
original_cost: float | None = None,
@ -1505,6 +1505,39 @@ def _response_time_ms_for_cost(
return 0.0
def _adjusted_provider_billed_cost(
reported_cost: float,
custom_llm_provider: str | None,
model: str | None,
region_name: str | None,
litellm_logging_obj: LitellmLoggingObject | None,
service_tier: str | None,
data_residency: str | None,
) -> float:
discounted_cost, discount_percent, discount_amount = _apply_cost_discount(
base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name
)
adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin(
base_cost=discounted_cost, custom_llm_provider=custom_llm_provider
)
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=None,
completion_tokens_cost_usd_dollar=None,
cost_for_built_in_tools_cost_usd_dollar=None,
total_cost_usd_dollar=adjusted_cost,
original_cost=reported_cost,
discount_percent=discount_percent,
discount_amount=discount_amount,
margin_percent=margin_percent,
margin_fixed_amount=margin_fixed_amount,
margin_total_amount=margin_total_amount,
service_tier=service_tier,
data_residency=data_residency,
)
return adjusted_cost
def completion_cost(
completion_response: object | None = None,
model: str | None = None,
@ -1674,6 +1707,7 @@ def completion_cost(
raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
)
priced_response: Final[object] = completion_response
for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names):
try:
if verbose_logger.isEnabledFor(logging.DEBUG):
@ -1991,6 +2025,19 @@ def completion_cost(
total_time=total_time,
)
model_for_discount = custom_priced_request_model or model
if (
not explicit_pricing
and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None
):
return _adjusted_provider_billed_cost(
reported_cost=reported_cost,
custom_llm_provider=custom_llm_provider,
model=model_for_discount,
region_name=region_name if isinstance(region_name, str) else None,
litellm_logging_obj=litellm_logging_obj,
service_tier=service_tier,
data_residency=data_residency,
)
# Calculate cost based on prompt_tokens, completion_tokens
if (
"togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
@ -2210,16 +2257,6 @@ def _billable_header_cost(value: object) -> float | None:
return cost if math.isfinite(cost) and cost >= 0 else None
def _with_cost_adjustments(
cost: float, custom_llm_provider: str | None, model: str | None, region_name: str | None
) -> float:
discounted_cost, _, _ = _apply_cost_discount(
base_cost=cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name
)
adjusted_cost, _, _, _ = _apply_cost_margin(base_cost=discounted_cost, custom_llm_provider=custom_llm_provider)
return adjusted_cost
def response_cost_calculator(
response_object: ModelResponse
| EmbeddingResponse
@ -2302,9 +2339,6 @@ def response_cost_calculator(
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
if provider_response_cost is not None and not custom_pricing:
return provider_response_cost
usage_cost: Final = billed_usage_cost(response_object, custom_llm_provider)
if usage_cost is not None and not custom_pricing and base_model is None:
return _with_cost_adjustments(usage_cost, custom_llm_provider, model, region_name)
response_cost = completion_cost(
completion_response=response_object,

View file

@ -644,6 +644,8 @@ def image_generation(
timeout=timeout,
client=custom_client,
)
elif custom_llm_provider == LlmProviders.KYMA.value:
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
return model_response
except Exception as e:

View file

@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler:
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
responses_response=responses_api_response,
model=model,
custom_llm_provider=custom_llm_provider,
)
async def async_interactions_api_handler(
@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler:
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
responses_response=responses_api_response,
model=model,
custom_llm_provider=responses_request.get("custom_llm_provider"),
)

View file

@ -12,6 +12,7 @@ from typing import Final, cast
from pydantic import BaseModel
from litellm.litellm_core_utils.core_helpers import billed_usage_cost
from litellm.types.interactions import (
InteractionInput,
InteractionsAPIOptionalRequestParams,
@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig:
def transform_responses_response_to_interactions_response(
responses_response: ResponsesAPIResponse,
model: str | None = None,
custom_llm_provider: str | None = None,
) -> InteractionsAPIResponse:
"""
Transform a Responses API response to an Interactions API response.
@ -266,7 +268,7 @@ class LiteLLMResponsesInteractionsConfig:
# (total_input_tokens, total_output_tokens)
usage: Final[object] = getattr(responses_response, "usage", None)
if usage:
reported_cost: Final[object] = getattr(usage, "cost", None)
reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider)
interactions_response_dict["usage"] = {
"total_input_tokens": getattr(usage, "input_tokens", 0),
"total_output_tokens": getattr(usage, "output_tokens", 0),

View file

@ -1939,10 +1939,10 @@ class Logging(LiteLLMLoggingBaseClass):
def set_cost_breakdown(
self,
input_cost: float,
output_cost: float,
input_cost: float | None,
output_cost: float | None,
total_cost: float,
cost_for_built_in_tools_cost_usd_dollar: float,
cost_for_built_in_tools_cost_usd_dollar: float | None,
additional_costs: dict | None = None,
original_cost: float | None = None,
discount_percent: float | None = None,
@ -1981,14 +1981,17 @@ class Logging(LiteLLMLoggingBaseClass):
self.billed_token_rates = billed_token_rates
self.cost_breakdown = CostBreakdown(
input_cost=input_cost,
output_cost=output_cost,
total_cost=total_cost,
tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
)
if input_cost is not None:
self.cost_breakdown["input_cost"] = input_cost
if output_cost is not None:
self.cost_breakdown["output_cost"] = output_cost
if cost_for_built_in_tools_cost_usd_dollar is not None:
self.cost_breakdown["tool_usage_cost"] = cost_for_built_in_tools_cost_usd_dollar
if cache_read_cost is not None and cache_read_cost > 0:
self.cost_breakdown["cache_read_cost"] = cache_read_cost
if cache_creation_cost is not None and cache_creation_cost > 0:

View file

@ -203,7 +203,7 @@ from litellm.types.llms.openai import (
ChatCompletionUserMessage,
ToolMessageContentPart,
)
from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage
from litellm.utils import supports_mid_conversation_system
from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper
@ -362,6 +362,9 @@ class AnthropicAdapter:
_BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object])
_CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.value})
class LiteLLMAnthropicMessagesAdapter:
def __init__(self):
pass
@ -1127,7 +1130,8 @@ class LiteLLMAnthropicMessagesAdapter:
"""Translate Anthropic thinking to either thinking or reasoning_effort.
A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one
speaks that param. Carrying its adaptive effort tier alongside takes two different params,
speaks that param, except Kyma, which serves Claude over the OpenAI dialect and takes only
``reasoning_effort``, so its Claude models get the non-Claude mapping. Carrying its adaptive effort tier alongside takes two different params,
because the two are not interchangeable at the provider mapping below.
Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking``
@ -1158,7 +1162,9 @@ class LiteLLMAnthropicMessagesAdapter:
is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model(
model
)
is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)
is_claude_target: Final = (
self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)
) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS
output_config: Final = anthropic_message_request.get("output_config")
if is_claude_target:

View file

@ -1,4 +1,3 @@
import contextlib
import json
import re
from collections.abc import Callable
@ -14,6 +13,7 @@ import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.proxy.common_request_processing import cost_headers
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage
_REPO_ROOT: Final = Path(__file__).resolve().parents[4]
@ -253,6 +253,45 @@ def test_interaction_on_kyma_is_billed_at_the_cost_kyma_reports():
assert interaction._hidden_params["response_cost"] == pytest.approx(reported_cost)
@pytest.mark.parametrize("model", ("openrouter/openai/gpt-4o-mini", "deepseek/deepseek-chat"))
def test_interaction_on_another_provider_keeps_its_upstream_cost_out_of_the_body(model: str):
chat_usage: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721}
responses_body: Final = {
"id": "resp_upstream",
"object": "response",
"created_at": 1_792_000_000,
"status": "completed",
"model": "x",
"output": [
{
"type": "message",
"id": "msg_upstream",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "pong", "annotations": []}],
}
],
"usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7, "cost": 0.00721},
}
with respx.mock(assert_all_called=False) as upstream:
upstream.route(path__regex=r".*/chat/completions$").respond(200, json=_kyma_chat_body(chat_usage))
upstream.route(path__regex=r".*/responses$").respond(200, json=responses_body)
interaction: Final = litellm.interactions.create(model=model, input="ping", api_key="k")
assert "cost" not in interaction.model_dump(exclude_none=True)["usage"]
def test_sdk_completion_cost_prices_kyma_at_the_cost_kyma_reports():
reported_cost: Final = 0.00721
with respx.mock() as upstream:
upstream.post(_CHAT_URL).respond(
200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": reported_cost})
)
response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k")
assert litellm.completion_cost(completion_response=response) == pytest.approx(reported_cost)
def _with_kyma_discount_and_margin(monkeypatch: pytest.MonkeyPatch) -> float:
monkeypatch.setattr(litellm, "cost_discount_config", {"kyma": 0.5})
monkeypatch.setattr(litellm, "cost_margin_config", {"kyma": 0.1})
@ -309,6 +348,57 @@ def test_streamed_cost_kyma_reports_takes_the_configured_discount_and_margin(mon
assert logging_obj.response_cost_calculator(result=response) == pytest.approx(0.008 * adjustment)
def test_cost_headers_show_the_discount_and_margin_taken_from_the_cost_kyma_reports(monkeypatch: pytest.MonkeyPatch):
_with_kyma_discount_and_margin(monkeypatch)
logging_obj: Final = LiteLLMLogging(
model="sonar",
messages=[{"role": "user", "content": "ping"}],
stream=False,
call_type="completion",
start_time=datetime.now(),
litellm_call_id="kyma-call",
function_id="kyma-function",
)
logging_obj.update_environment_variables(
model="sonar",
user=None,
optional_params={},
litellm_params={"custom_llm_provider": "kyma"},
custom_llm_provider="kyma",
)
response: Final = litellm.ModelResponse(
model="sonar", usage=Usage(prompt_tokens=4, completion_tokens=3, total_tokens=7, cost=0.008)
)
headers: Final = cost_headers(logging_obj.response_cost_calculator(result=response), logging_obj)
assert float(headers["x-litellm-response-cost-original"] or "nan") == pytest.approx(0.008)
assert float(headers["x-litellm-response-cost-discount-amount"] or "nan") == pytest.approx(0.004)
assert float(headers["x-litellm-response-cost-margin-amount"] or "nan") == pytest.approx(0.0004)
assert float(headers["x-litellm-response-cost"] or "nan") == pytest.approx(0.0044)
assert headers["x-litellm-response-cost-input"] is None
assert headers["x-litellm-response-cost-output"] is None
@pytest.mark.parametrize("thinking", ({"type": "enabled", "budget_tokens": 1024}, {"type": "adaptive"}))
def test_messages_thinking_on_a_kyma_claude_model_is_sent_as_reasoning_effort(thinking: dict[str, object]):
with respx.mock() as upstream:
route: Final = upstream.post(_CHAT_URL).respond(
200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7})
)
litellm.anthropic.messages.create(
model="kyma/claude-haiku-4-5",
max_tokens=2048,
messages=[{"role": "user", "content": "ping"}],
thinking=thinking,
api_key="k",
)
sent: Final = json.loads(route.calls.last.request.content)
assert "thinking" not in sent
assert sent["reasoning_effort"] in ("low", "medium", "high")
def test_kyma_model_missing_from_the_catalog_keeps_its_tools():
tools: Final = [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
with respx.mock() as upstream:
@ -333,10 +423,10 @@ def _transcribe() -> object:
@pytest.mark.parametrize("call", (_generate_image, _transcribe))
def test_unpriced_kyma_media_calls_are_never_sent_to_kyma(call: Callable[[], object]):
def test_unpriced_kyma_media_calls_are_refused_without_calling_kyma(call: Callable[[], object]):
with respx.mock(assert_all_called=False) as upstream:
kyma: Final = upstream.route(host="kymaapi.com").respond(202, json={"id": "job_kyma"})
with contextlib.suppress(Exception):
with pytest.raises(litellm.BadRequestError):
call()
assert kyma.call_count == 0