From d968ab3b91aa9c15a48700252f67af36c1bb3aa9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:18:43 -0700 Subject: [PATCH] fix(kyma): bill completion_cost from usage.cost, keep other providers' upstream cost out of interactions, map thinking, refuse images --- litellm/cost_calculator.py | 66 +++++++++---- litellm/images/main.py | 2 + .../handler.py | 2 + .../transformation.py | 4 +- litellm/litellm_core_utils/litellm_logging.py | 15 +-- .../messages/adapters/transformation.py | 12 ++- .../llms/openai_like/test_kyma_provider.py | 96 ++++++++++++++++++- 7 files changed, 168 insertions(+), 29 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index d0066e16a14..61d1ac3acbd 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1396,9 +1396,9 @@ def _apply_cost_margin( def _store_cost_breakdown_in_logging_obj( litellm_logging_obj: LitellmLoggingObject | None, - prompt_tokens_cost_usd_dollar: float, - completion_tokens_cost_usd_dollar: float, - cost_for_built_in_tools_cost_usd_dollar: float, + prompt_tokens_cost_usd_dollar: float | None, + completion_tokens_cost_usd_dollar: float | None, + cost_for_built_in_tools_cost_usd_dollar: float | None, total_cost_usd_dollar: float, additional_costs: dict | None = None, original_cost: float | None = None, @@ -1505,6 +1505,39 @@ def _response_time_ms_for_cost( return 0.0 +def _adjusted_provider_billed_cost( + reported_cost: float, + custom_llm_provider: str | None, + model: str | None, + region_name: str | None, + litellm_logging_obj: LitellmLoggingObject | None, + service_tier: str | None, + data_residency: str | None, +) -> float: + discounted_cost, discount_percent, discount_amount = _apply_cost_discount( + base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name + ) + adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin( + base_cost=discounted_cost, custom_llm_provider=custom_llm_provider + ) + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=None, + completion_tokens_cost_usd_dollar=None, + cost_for_built_in_tools_cost_usd_dollar=None, + total_cost_usd_dollar=adjusted_cost, + original_cost=reported_cost, + discount_percent=discount_percent, + discount_amount=discount_amount, + margin_percent=margin_percent, + margin_fixed_amount=margin_fixed_amount, + margin_total_amount=margin_total_amount, + service_tier=service_tier, + data_residency=data_residency, + ) + return adjusted_cost + + def completion_cost( completion_response: object | None = None, model: str | None = None, @@ -1674,6 +1707,7 @@ def completion_cost( raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates) ) + priced_response: Final[object] = completion_response for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names): try: if verbose_logger.isEnabledFor(logging.DEBUG): @@ -1991,6 +2025,19 @@ def completion_cost( total_time=total_time, ) model_for_discount = custom_priced_request_model or model + if ( + not explicit_pricing + and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None + ): + return _adjusted_provider_billed_cost( + reported_cost=reported_cost, + custom_llm_provider=custom_llm_provider, + model=model_for_discount, + region_name=region_name if isinstance(region_name, str) else None, + litellm_logging_obj=litellm_logging_obj, + service_tier=service_tier, + data_residency=data_residency, + ) # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" @@ -2210,16 +2257,6 @@ def _billable_header_cost(value: object) -> float | None: return cost if math.isfinite(cost) and cost >= 0 else None -def _with_cost_adjustments( - cost: float, custom_llm_provider: str | None, model: str | None, region_name: str | None -) -> float: - discounted_cost, _, _ = _apply_cost_discount( - base_cost=cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name - ) - adjusted_cost, _, _, _ = _apply_cost_margin(base_cost=discounted_cost, custom_llm_provider=custom_llm_provider) - return adjusted_cost - - def response_cost_calculator( response_object: ModelResponse | EmbeddingResponse @@ -2302,9 +2339,6 @@ def response_cost_calculator( provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params) if provider_response_cost is not None and not custom_pricing: return provider_response_cost - usage_cost: Final = billed_usage_cost(response_object, custom_llm_provider) - if usage_cost is not None and not custom_pricing and base_model is None: - return _with_cost_adjustments(usage_cost, custom_llm_provider, model, region_name) response_cost = completion_cost( completion_response=response_object, diff --git a/litellm/images/main.py b/litellm/images/main.py index 84156919774..793d0bf92a8 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -644,6 +644,8 @@ def image_generation( timeout=timeout, client=custom_client, ) + elif custom_llm_provider == LlmProviders.KYMA.value: + raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) return model_response except Exception as e: diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py index d84f0b5ffbd..f95b5611831 100644 --- a/litellm/interactions/litellm_responses_transformation/handler.py +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=custom_llm_provider, ) async def async_interactions_api_handler( @@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=responses_request.get("custom_llm_provider"), ) diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index 48c8b9a0ca0..2e9b62dcfe2 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -12,6 +12,7 @@ from typing import Final, cast from pydantic import BaseModel +from litellm.litellm_core_utils.core_helpers import billed_usage_cost from litellm.types.interactions import ( InteractionInput, InteractionsAPIOptionalRequestParams, @@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig: def transform_responses_response_to_interactions_response( responses_response: ResponsesAPIResponse, model: str | None = None, + custom_llm_provider: str | None = None, ) -> InteractionsAPIResponse: """ Transform a Responses API response to an Interactions API response. @@ -266,7 +268,7 @@ class LiteLLMResponsesInteractionsConfig: # (total_input_tokens, total_output_tokens) usage: Final[object] = getattr(responses_response, "usage", None) if usage: - reported_cost: Final[object] = getattr(usage, "cost", None) + reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider) interactions_response_dict["usage"] = { "total_input_tokens": getattr(usage, "input_tokens", 0), "total_output_tokens": getattr(usage, "output_tokens", 0), diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 48e8a44b9a4..053c53aef93 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1939,10 +1939,10 @@ class Logging(LiteLLMLoggingBaseClass): def set_cost_breakdown( self, - input_cost: float, - output_cost: float, + input_cost: float | None, + output_cost: float | None, total_cost: float, - cost_for_built_in_tools_cost_usd_dollar: float, + cost_for_built_in_tools_cost_usd_dollar: float | None, additional_costs: dict | None = None, original_cost: float | None = None, discount_percent: float | None = None, @@ -1981,14 +1981,17 @@ class Logging(LiteLLMLoggingBaseClass): self.billed_token_rates = billed_token_rates self.cost_breakdown = CostBreakdown( - input_cost=input_cost, - output_cost=output_cost, total_cost=total_cost, - tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, ) + if input_cost is not None: + self.cost_breakdown["input_cost"] = input_cost + if output_cost is not None: + self.cost_breakdown["output_cost"] = output_cost + if cost_for_built_in_tools_cost_usd_dollar is not None: + self.cost_breakdown["tool_usage_cost"] = cost_for_built_in_tools_cost_usd_dollar if cache_read_cost is not None and cache_read_cost > 0: self.cost_breakdown["cache_read_cost"] = cache_read_cost if cache_creation_cost is not None and cache_creation_cost > 0: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 1ed552d46e3..d7faac631d4 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -203,7 +203,7 @@ from litellm.types.llms.openai import ( ChatCompletionUserMessage, ToolMessageContentPart, ) -from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage +from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage from litellm.utils import supports_mid_conversation_system from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper @@ -362,6 +362,9 @@ class AnthropicAdapter: _BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object]) +_CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.value}) + + class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass @@ -1127,7 +1130,8 @@ class LiteLLMAnthropicMessagesAdapter: """Translate Anthropic thinking to either thinking or reasoning_effort. A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one - speaks that param. Carrying its adaptive effort tier alongside takes two different params, + speaks that param, except Kyma, which serves Claude over the OpenAI dialect and takes only + ``reasoning_effort``, so its Claude models get the non-Claude mapping. Carrying its adaptive effort tier alongside takes two different params, because the two are not interchangeable at the provider mapping below. Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking`` @@ -1158,7 +1162,9 @@ class LiteLLMAnthropicMessagesAdapter: is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model( model ) - is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + is_claude_target: Final = ( + self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS output_config: Final = anthropic_message_request.get("output_config") if is_claude_target: diff --git a/tests/unit/llms/openai_like/test_kyma_provider.py b/tests/unit/llms/openai_like/test_kyma_provider.py index 7fe4457ab88..a05ca10713f 100644 --- a/tests/unit/llms/openai_like/test_kyma_provider.py +++ b/tests/unit/llms/openai_like/test_kyma_provider.py @@ -1,4 +1,3 @@ -import contextlib import json import re from collections.abc import Callable @@ -14,6 +13,7 @@ import litellm from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.proxy.auth.model_checks import get_known_models_from_wildcard +from litellm.proxy.common_request_processing import cost_headers from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage _REPO_ROOT: Final = Path(__file__).resolve().parents[4] @@ -253,6 +253,45 @@ def test_interaction_on_kyma_is_billed_at_the_cost_kyma_reports(): assert interaction._hidden_params["response_cost"] == pytest.approx(reported_cost) +@pytest.mark.parametrize("model", ("openrouter/openai/gpt-4o-mini", "deepseek/deepseek-chat")) +def test_interaction_on_another_provider_keeps_its_upstream_cost_out_of_the_body(model: str): + chat_usage: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721} + responses_body: Final = { + "id": "resp_upstream", + "object": "response", + "created_at": 1_792_000_000, + "status": "completed", + "model": "x", + "output": [ + { + "type": "message", + "id": "msg_upstream", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], + } + ], + "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7, "cost": 0.00721}, + } + with respx.mock(assert_all_called=False) as upstream: + upstream.route(path__regex=r".*/chat/completions$").respond(200, json=_kyma_chat_body(chat_usage)) + upstream.route(path__regex=r".*/responses$").respond(200, json=responses_body) + interaction: Final = litellm.interactions.create(model=model, input="ping", api_key="k") + + assert "cost" not in interaction.model_dump(exclude_none=True)["usage"] + + +def test_sdk_completion_cost_prices_kyma_at_the_cost_kyma_reports(): + reported_cost: Final = 0.00721 + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": reported_cost}) + ) + response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k") + + assert litellm.completion_cost(completion_response=response) == pytest.approx(reported_cost) + + def _with_kyma_discount_and_margin(monkeypatch: pytest.MonkeyPatch) -> float: monkeypatch.setattr(litellm, "cost_discount_config", {"kyma": 0.5}) monkeypatch.setattr(litellm, "cost_margin_config", {"kyma": 0.1}) @@ -309,6 +348,57 @@ def test_streamed_cost_kyma_reports_takes_the_configured_discount_and_margin(mon assert logging_obj.response_cost_calculator(result=response) == pytest.approx(0.008 * adjustment) + +def test_cost_headers_show_the_discount_and_margin_taken_from_the_cost_kyma_reports(monkeypatch: pytest.MonkeyPatch): + _with_kyma_discount_and_margin(monkeypatch) + logging_obj: Final = LiteLLMLogging( + model="sonar", + messages=[{"role": "user", "content": "ping"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="kyma-call", + function_id="kyma-function", + ) + logging_obj.update_environment_variables( + model="sonar", + user=None, + optional_params={}, + litellm_params={"custom_llm_provider": "kyma"}, + custom_llm_provider="kyma", + ) + response: Final = litellm.ModelResponse( + model="sonar", usage=Usage(prompt_tokens=4, completion_tokens=3, total_tokens=7, cost=0.008) + ) + + headers: Final = cost_headers(logging_obj.response_cost_calculator(result=response), logging_obj) + + assert float(headers["x-litellm-response-cost-original"] or "nan") == pytest.approx(0.008) + assert float(headers["x-litellm-response-cost-discount-amount"] or "nan") == pytest.approx(0.004) + assert float(headers["x-litellm-response-cost-margin-amount"] or "nan") == pytest.approx(0.0004) + assert float(headers["x-litellm-response-cost"] or "nan") == pytest.approx(0.0044) + assert headers["x-litellm-response-cost-input"] is None + assert headers["x-litellm-response-cost-output"] is None + + +@pytest.mark.parametrize("thinking", ({"type": "enabled", "budget_tokens": 1024}, {"type": "adaptive"})) +def test_messages_thinking_on_a_kyma_claude_model_is_sent_as_reasoning_effort(thinking: dict[str, object]): + with respx.mock() as upstream: + route: Final = upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}) + ) + litellm.anthropic.messages.create( + model="kyma/claude-haiku-4-5", + max_tokens=2048, + messages=[{"role": "user", "content": "ping"}], + thinking=thinking, + api_key="k", + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert "thinking" not in sent + assert sent["reasoning_effort"] in ("low", "medium", "high") + def test_kyma_model_missing_from_the_catalog_keeps_its_tools(): tools: Final = [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}] with respx.mock() as upstream: @@ -333,10 +423,10 @@ def _transcribe() -> object: @pytest.mark.parametrize("call", (_generate_image, _transcribe)) -def test_unpriced_kyma_media_calls_are_never_sent_to_kyma(call: Callable[[], object]): +def test_unpriced_kyma_media_calls_are_refused_without_calling_kyma(call: Callable[[], object]): with respx.mock(assert_all_called=False) as upstream: kyma: Final = upstream.route(host="kymaapi.com").respond(202, json={"id": "job_kyma"}) - with contextlib.suppress(Exception): + with pytest.raises(litellm.BadRequestError): call() assert kyma.call_count == 0