mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(kyma): bill completion_cost from usage.cost, keep other providers' upstream cost out of interactions, map thinking, refuse images
This commit is contained in:
parent
69a21bf218
commit
d968ab3b91
7 changed files with 168 additions and 29 deletions
|
|
@ -1396,9 +1396,9 @@ def _apply_cost_margin(
|
|||
|
||||
def _store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj: LitellmLoggingObject | None,
|
||||
prompt_tokens_cost_usd_dollar: float,
|
||||
completion_tokens_cost_usd_dollar: float,
|
||||
cost_for_built_in_tools_cost_usd_dollar: float,
|
||||
prompt_tokens_cost_usd_dollar: float | None,
|
||||
completion_tokens_cost_usd_dollar: float | None,
|
||||
cost_for_built_in_tools_cost_usd_dollar: float | None,
|
||||
total_cost_usd_dollar: float,
|
||||
additional_costs: dict | None = None,
|
||||
original_cost: float | None = None,
|
||||
|
|
@ -1505,6 +1505,39 @@ def _response_time_ms_for_cost(
|
|||
return 0.0
|
||||
|
||||
|
||||
def _adjusted_provider_billed_cost(
|
||||
reported_cost: float,
|
||||
custom_llm_provider: str | None,
|
||||
model: str | None,
|
||||
region_name: str | None,
|
||||
litellm_logging_obj: LitellmLoggingObject | None,
|
||||
service_tier: str | None,
|
||||
data_residency: str | None,
|
||||
) -> float:
|
||||
discounted_cost, discount_percent, discount_amount = _apply_cost_discount(
|
||||
base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name
|
||||
)
|
||||
adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin(
|
||||
base_cost=discounted_cost, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=None,
|
||||
completion_tokens_cost_usd_dollar=None,
|
||||
cost_for_built_in_tools_cost_usd_dollar=None,
|
||||
total_cost_usd_dollar=adjusted_cost,
|
||||
original_cost=reported_cost,
|
||||
discount_percent=discount_percent,
|
||||
discount_amount=discount_amount,
|
||||
margin_percent=margin_percent,
|
||||
margin_fixed_amount=margin_fixed_amount,
|
||||
margin_total_amount=margin_total_amount,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
return adjusted_cost
|
||||
|
||||
|
||||
def completion_cost(
|
||||
completion_response: object | None = None,
|
||||
model: str | None = None,
|
||||
|
|
@ -1674,6 +1707,7 @@ def completion_cost(
|
|||
raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
|
||||
)
|
||||
|
||||
priced_response: Final[object] = completion_response
|
||||
for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names):
|
||||
try:
|
||||
if verbose_logger.isEnabledFor(logging.DEBUG):
|
||||
|
|
@ -1991,6 +2025,19 @@ def completion_cost(
|
|||
total_time=total_time,
|
||||
)
|
||||
model_for_discount = custom_priced_request_model or model
|
||||
if (
|
||||
not explicit_pricing
|
||||
and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None
|
||||
):
|
||||
return _adjusted_provider_billed_cost(
|
||||
reported_cost=reported_cost,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model=model_for_discount,
|
||||
region_name=region_name if isinstance(region_name, str) else None,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
# Calculate cost based on prompt_tokens, completion_tokens
|
||||
if (
|
||||
"togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
|
||||
|
|
@ -2210,16 +2257,6 @@ def _billable_header_cost(value: object) -> float | None:
|
|||
return cost if math.isfinite(cost) and cost >= 0 else None
|
||||
|
||||
|
||||
def _with_cost_adjustments(
|
||||
cost: float, custom_llm_provider: str | None, model: str | None, region_name: str | None
|
||||
) -> float:
|
||||
discounted_cost, _, _ = _apply_cost_discount(
|
||||
base_cost=cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name
|
||||
)
|
||||
adjusted_cost, _, _, _ = _apply_cost_margin(base_cost=discounted_cost, custom_llm_provider=custom_llm_provider)
|
||||
return adjusted_cost
|
||||
|
||||
|
||||
def response_cost_calculator(
|
||||
response_object: ModelResponse
|
||||
| EmbeddingResponse
|
||||
|
|
@ -2302,9 +2339,6 @@ def response_cost_calculator(
|
|||
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
|
||||
if provider_response_cost is not None and not custom_pricing:
|
||||
return provider_response_cost
|
||||
usage_cost: Final = billed_usage_cost(response_object, custom_llm_provider)
|
||||
if usage_cost is not None and not custom_pricing and base_model is None:
|
||||
return _with_cost_adjustments(usage_cost, custom_llm_provider, model, region_name)
|
||||
|
||||
response_cost = completion_cost(
|
||||
completion_response=response_object,
|
||||
|
|
|
|||
|
|
@ -644,6 +644,8 @@ def image_generation(
|
|||
timeout=timeout,
|
||||
client=custom_client,
|
||||
)
|
||||
elif custom_llm_provider == LlmProviders.KYMA.value:
|
||||
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
return model_response
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler:
|
|||
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
|
||||
responses_response=responses_api_response,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
async def async_interactions_api_handler(
|
||||
|
|
@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler:
|
|||
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
|
||||
responses_response=responses_api_response,
|
||||
model=model,
|
||||
custom_llm_provider=responses_request.get("custom_llm_provider"),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import Final, cast
|
|||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import billed_usage_cost
|
||||
from litellm.types.interactions import (
|
||||
InteractionInput,
|
||||
InteractionsAPIOptionalRequestParams,
|
||||
|
|
@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig:
|
|||
def transform_responses_response_to_interactions_response(
|
||||
responses_response: ResponsesAPIResponse,
|
||||
model: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> InteractionsAPIResponse:
|
||||
"""
|
||||
Transform a Responses API response to an Interactions API response.
|
||||
|
|
@ -266,7 +268,7 @@ class LiteLLMResponsesInteractionsConfig:
|
|||
# (total_input_tokens, total_output_tokens)
|
||||
usage: Final[object] = getattr(responses_response, "usage", None)
|
||||
if usage:
|
||||
reported_cost: Final[object] = getattr(usage, "cost", None)
|
||||
reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider)
|
||||
interactions_response_dict["usage"] = {
|
||||
"total_input_tokens": getattr(usage, "input_tokens", 0),
|
||||
"total_output_tokens": getattr(usage, "output_tokens", 0),
|
||||
|
|
|
|||
|
|
@ -1939,10 +1939,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
def set_cost_breakdown(
|
||||
self,
|
||||
input_cost: float,
|
||||
output_cost: float,
|
||||
input_cost: float | None,
|
||||
output_cost: float | None,
|
||||
total_cost: float,
|
||||
cost_for_built_in_tools_cost_usd_dollar: float,
|
||||
cost_for_built_in_tools_cost_usd_dollar: float | None,
|
||||
additional_costs: dict | None = None,
|
||||
original_cost: float | None = None,
|
||||
discount_percent: float | None = None,
|
||||
|
|
@ -1981,14 +1981,17 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
self.billed_token_rates = billed_token_rates
|
||||
self.cost_breakdown = CostBreakdown(
|
||||
input_cost=input_cost,
|
||||
output_cost=output_cost,
|
||||
total_cost=total_cost,
|
||||
tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
)
|
||||
if input_cost is not None:
|
||||
self.cost_breakdown["input_cost"] = input_cost
|
||||
if output_cost is not None:
|
||||
self.cost_breakdown["output_cost"] = output_cost
|
||||
if cost_for_built_in_tools_cost_usd_dollar is not None:
|
||||
self.cost_breakdown["tool_usage_cost"] = cost_for_built_in_tools_cost_usd_dollar
|
||||
if cache_read_cost is not None and cache_read_cost > 0:
|
||||
self.cost_breakdown["cache_read_cost"] = cache_read_cost
|
||||
if cache_creation_cost is not None and cache_creation_cost > 0:
|
||||
|
|
|
|||
|
|
@ -203,7 +203,7 @@ from litellm.types.llms.openai import (
|
|||
ChatCompletionUserMessage,
|
||||
ToolMessageContentPart,
|
||||
)
|
||||
from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
|
||||
from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage
|
||||
from litellm.utils import supports_mid_conversation_system
|
||||
|
||||
from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper
|
||||
|
|
@ -362,6 +362,9 @@ class AnthropicAdapter:
|
|||
_BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object])
|
||||
|
||||
|
||||
_CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.value})
|
||||
|
||||
|
||||
class LiteLLMAnthropicMessagesAdapter:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
|
@ -1127,7 +1130,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"""Translate Anthropic thinking to either thinking or reasoning_effort.
|
||||
|
||||
A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one
|
||||
speaks that param. Carrying its adaptive effort tier alongside takes two different params,
|
||||
speaks that param, except Kyma, which serves Claude over the OpenAI dialect and takes only
|
||||
``reasoning_effort``, so its Claude models get the non-Claude mapping. Carrying its adaptive effort tier alongside takes two different params,
|
||||
because the two are not interchangeable at the provider mapping below.
|
||||
|
||||
Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking``
|
||||
|
|
@ -1158,7 +1162,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model(
|
||||
model
|
||||
)
|
||||
is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)
|
||||
is_claude_target: Final = (
|
||||
self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)
|
||||
) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS
|
||||
output_config: Final = anthropic_message_request.get("output_config")
|
||||
|
||||
if is_claude_target:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import contextlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
|
|
@ -14,6 +13,7 @@ import litellm
|
|||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.proxy.common_request_processing import cost_headers
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage
|
||||
|
||||
_REPO_ROOT: Final = Path(__file__).resolve().parents[4]
|
||||
|
|
@ -253,6 +253,45 @@ def test_interaction_on_kyma_is_billed_at_the_cost_kyma_reports():
|
|||
assert interaction._hidden_params["response_cost"] == pytest.approx(reported_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("openrouter/openai/gpt-4o-mini", "deepseek/deepseek-chat"))
|
||||
def test_interaction_on_another_provider_keeps_its_upstream_cost_out_of_the_body(model: str):
|
||||
chat_usage: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721}
|
||||
responses_body: Final = {
|
||||
"id": "resp_upstream",
|
||||
"object": "response",
|
||||
"created_at": 1_792_000_000,
|
||||
"status": "completed",
|
||||
"model": "x",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_upstream",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "pong", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7, "cost": 0.00721},
|
||||
}
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream.route(path__regex=r".*/chat/completions$").respond(200, json=_kyma_chat_body(chat_usage))
|
||||
upstream.route(path__regex=r".*/responses$").respond(200, json=responses_body)
|
||||
interaction: Final = litellm.interactions.create(model=model, input="ping", api_key="k")
|
||||
|
||||
assert "cost" not in interaction.model_dump(exclude_none=True)["usage"]
|
||||
|
||||
|
||||
def test_sdk_completion_cost_prices_kyma_at_the_cost_kyma_reports():
|
||||
reported_cost: Final = 0.00721
|
||||
with respx.mock() as upstream:
|
||||
upstream.post(_CHAT_URL).respond(
|
||||
200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": reported_cost})
|
||||
)
|
||||
response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k")
|
||||
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(reported_cost)
|
||||
|
||||
|
||||
def _with_kyma_discount_and_margin(monkeypatch: pytest.MonkeyPatch) -> float:
|
||||
monkeypatch.setattr(litellm, "cost_discount_config", {"kyma": 0.5})
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"kyma": 0.1})
|
||||
|
|
@ -309,6 +348,57 @@ def test_streamed_cost_kyma_reports_takes_the_configured_discount_and_margin(mon
|
|||
|
||||
assert logging_obj.response_cost_calculator(result=response) == pytest.approx(0.008 * adjustment)
|
||||
|
||||
|
||||
def test_cost_headers_show_the_discount_and_margin_taken_from_the_cost_kyma_reports(monkeypatch: pytest.MonkeyPatch):
|
||||
_with_kyma_discount_and_margin(monkeypatch)
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
model="sonar",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="kyma-call",
|
||||
function_id="kyma-function",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model="sonar",
|
||||
user=None,
|
||||
optional_params={},
|
||||
litellm_params={"custom_llm_provider": "kyma"},
|
||||
custom_llm_provider="kyma",
|
||||
)
|
||||
response: Final = litellm.ModelResponse(
|
||||
model="sonar", usage=Usage(prompt_tokens=4, completion_tokens=3, total_tokens=7, cost=0.008)
|
||||
)
|
||||
|
||||
headers: Final = cost_headers(logging_obj.response_cost_calculator(result=response), logging_obj)
|
||||
|
||||
assert float(headers["x-litellm-response-cost-original"] or "nan") == pytest.approx(0.008)
|
||||
assert float(headers["x-litellm-response-cost-discount-amount"] or "nan") == pytest.approx(0.004)
|
||||
assert float(headers["x-litellm-response-cost-margin-amount"] or "nan") == pytest.approx(0.0004)
|
||||
assert float(headers["x-litellm-response-cost"] or "nan") == pytest.approx(0.0044)
|
||||
assert headers["x-litellm-response-cost-input"] is None
|
||||
assert headers["x-litellm-response-cost-output"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", ({"type": "enabled", "budget_tokens": 1024}, {"type": "adaptive"}))
|
||||
def test_messages_thinking_on_a_kyma_claude_model_is_sent_as_reasoning_effort(thinking: dict[str, object]):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post(_CHAT_URL).respond(
|
||||
200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7})
|
||||
)
|
||||
litellm.anthropic.messages.create(
|
||||
model="kyma/claude-haiku-4-5",
|
||||
max_tokens=2048,
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
thinking=thinking,
|
||||
api_key="k",
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert "thinking" not in sent
|
||||
assert sent["reasoning_effort"] in ("low", "medium", "high")
|
||||
|
||||
def test_kyma_model_missing_from_the_catalog_keeps_its_tools():
|
||||
tools: Final = [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
|
||||
with respx.mock() as upstream:
|
||||
|
|
@ -333,10 +423,10 @@ def _transcribe() -> object:
|
|||
|
||||
|
||||
@pytest.mark.parametrize("call", (_generate_image, _transcribe))
|
||||
def test_unpriced_kyma_media_calls_are_never_sent_to_kyma(call: Callable[[], object]):
|
||||
def test_unpriced_kyma_media_calls_are_refused_without_calling_kyma(call: Callable[[], object]):
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
kyma: Final = upstream.route(host="kymaapi.com").respond(202, json={"id": "job_kyma"})
|
||||
with contextlib.suppress(Exception):
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
call()
|
||||
|
||||
assert kyma.call_count == 0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue