mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(decisions): report provider charges on the cost header and read Vercel's cache split from the serving provider
Decisions goes back to reporting a provider's own charge through the response cost header, the way main does for OpenRouter. Moving it onto upstream_usage_cost dropped every cost header on free calls, priced base_model deployments from the cost map, and put the margin in the response header but not in the SystemOne body cost. Vercel's cache split now comes from the provider_metadata entry of the provider that served the call, and only when that entry reports input_tokens_details. Any credentialType value is accepted. The Jev 1.13.0 and 1.14.0 rows record their image probes.
This commit is contained in:
parent
c8122bc6a5
commit
ec26925103
6 changed files with 152 additions and 69 deletions
|
|
@ -12,7 +12,7 @@ from typing_extensions import assert_never
|
|||
|
||||
import litellm
|
||||
from litellm.cost_calculator import cost_map_key_prices_anything
|
||||
from litellm.litellm_core_utils.core_helpers import UPSTREAM_USAGE_COST, normalize_drop_params
|
||||
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
|
||||
from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_PROVIDERS
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -768,7 +768,7 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
|
|||
"model": f"{call.custom_llm_provider}/{call.model}",
|
||||
"custom_llm_provider": call.custom_llm_provider,
|
||||
"provider_response_model": _pricing_model(response, call),
|
||||
**({UPSTREAM_USAGE_COST: provider_cost} if provider_cost is not None else {}),
|
||||
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
|
||||
}
|
||||
)
|
||||
return formatted
|
||||
|
|
|
|||
|
|
@ -1,13 +1,18 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from itertools import chain
|
||||
from typing import Final, Literal, TypeVar
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import usage_reported_cost
|
||||
from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig
|
||||
from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse, DecisionsIRUsage, OpenAIDecisionUsage
|
||||
from litellm.types.decisions import (
|
||||
DecisionsIRRequest,
|
||||
DecisionsIRResponse,
|
||||
DecisionsIRUsage,
|
||||
OpenAIDecisionInputTokensDetails,
|
||||
)
|
||||
|
||||
_Model: Final = TypeVar("_Model", bound=BaseModel)
|
||||
|
||||
|
|
@ -15,7 +20,8 @@ _Model: Final = TypeVar("_Model", bound=BaseModel)
|
|||
class _ProviderAttempt(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
credential_type: Literal["system", "byok"] | None = Field(default=None, alias="credentialType")
|
||||
provider: str | None = None
|
||||
credential_type: str | None = Field(default=None, alias="credentialType")
|
||||
success: bool = False
|
||||
|
||||
|
||||
|
|
@ -30,28 +36,40 @@ class _Routing(BaseModel):
|
|||
|
||||
model_attempts: tuple[_ModelAttempt, ...] = Field(default=(), alias="modelAttempts")
|
||||
|
||||
def served_by(self) -> _ProviderAttempt | None:
|
||||
attempts: Final = chain.from_iterable(model.provider_attempts for model in self.model_attempts)
|
||||
return next((attempt for attempt in attempts if attempt.success), None)
|
||||
|
||||
|
||||
class _GatewayRouting(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
routing: _Routing = _Routing()
|
||||
|
||||
|
||||
class _GatewayCharge(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
cost: float
|
||||
market_cost: float | None = Field(default=None, alias="marketCost")
|
||||
routing: _Routing = _Routing()
|
||||
|
||||
def served_with_own_key(self) -> bool:
|
||||
attempts: Final = chain.from_iterable(model.provider_attempts for model in self.routing.model_attempts)
|
||||
return any(attempt.success and attempt.credential_type == "byok" for attempt in attempts)
|
||||
|
||||
def billed_cost(self) -> float:
|
||||
if self.market_cost is None or not self.served_with_own_key():
|
||||
def billed_cost(self, served_by: _ProviderAttempt | None) -> float:
|
||||
if self.market_cost is None or served_by is None or served_by.credential_type != "byok":
|
||||
return self.cost
|
||||
return self.cost + self.market_cost
|
||||
|
||||
|
||||
class _UpstreamUsage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
input_tokens: int
|
||||
input_tokens_details: OpenAIDecisionInputTokensDetails
|
||||
|
||||
|
||||
class _UpstreamReport(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
usage: OpenAIDecisionUsage
|
||||
usage: _UpstreamUsage
|
||||
|
||||
|
||||
class _GatewayDecisionResponse(BaseModel):
|
||||
|
|
@ -67,15 +85,27 @@ def _validated(model: type[_Model], value: object) -> _Model | None:
|
|||
return None
|
||||
|
||||
|
||||
def _served_by(gateway: object) -> _ProviderAttempt | None:
|
||||
gateway_routing: Final = _validated(_GatewayRouting, gateway)
|
||||
return None if gateway_routing is None else gateway_routing.routing.served_by()
|
||||
|
||||
|
||||
def _upstream_usage(metadata: Mapping[str, object], served_by: _ProviderAttempt | None) -> _UpstreamUsage | None:
|
||||
if served_by is None or served_by.provider is None:
|
||||
return None
|
||||
report: Final = _validated(_UpstreamReport, metadata.get(served_by.provider))
|
||||
return None if report is None else report.usage
|
||||
|
||||
|
||||
def _gateway_usage(usage: DecisionsIRUsage, metadata: Mapping[str, object]) -> DecisionsIRUsage:
|
||||
charge: Final = _validated(_GatewayCharge, metadata.get("gateway"))
|
||||
reports: Final = (_validated(_UpstreamReport, entry) for entry in metadata.values())
|
||||
upstream: Final = next(
|
||||
(report.usage for report in reports if report is not None and report.usage.input_tokens == usage.input_tokens),
|
||||
None,
|
||||
gateway: Final = metadata.get("gateway")
|
||||
served_by: Final = _served_by(gateway)
|
||||
charge: Final = _validated(_GatewayCharge, gateway)
|
||||
priced: Final = (
|
||||
usage if charge is None else replace(usage, extra={**usage.extra, "cost": charge.billed_cost(served_by)})
|
||||
)
|
||||
priced: Final = usage if charge is None else replace(usage, extra={**usage.extra, "cost": charge.billed_cost()})
|
||||
if upstream is None:
|
||||
upstream: Final = _upstream_usage(metadata, served_by)
|
||||
if upstream is None or upstream.input_tokens != usage.input_tokens:
|
||||
return priced
|
||||
return replace(
|
||||
priced,
|
||||
|
|
|
|||
|
|
@ -52565,7 +52565,7 @@
|
|||
"litellm_provider": "vercel_ai_gateway",
|
||||
"max_input_tokens": 32000,
|
||||
"metadata": {
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.13.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000."
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.13.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.99 for a gradient and 0.12 for solid gray."
|
||||
},
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
|
|
@ -52581,7 +52581,7 @@
|
|||
"litellm_provider": "vercel_ai_gateway",
|
||||
"max_input_tokens": 32000,
|
||||
"metadata": {
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.14.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000."
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.14.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.84 for a gradient and 0.13 for solid gray."
|
||||
},
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
|
|
|
|||
|
|
@ -52565,7 +52565,7 @@
|
|||
"litellm_provider": "vercel_ai_gateway",
|
||||
"max_input_tokens": 32000,
|
||||
"metadata": {
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.13.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000."
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.13.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.99 for a gradient and 0.12 for solid gray."
|
||||
},
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
|
|
@ -52581,7 +52581,7 @@
|
|||
"litellm_provider": "vercel_ai_gateway",
|
||||
"max_input_tokens": 32000,
|
||||
"metadata": {
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.14.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000."
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.14.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.84 for a gradient and 0.13 for solid gray."
|
||||
},
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from respx.models import Call
|
|||
|
||||
import litellm
|
||||
from litellm.constants import AWS_SIGNING_MAX_THREADS
|
||||
from litellm.cost_calculator import get_response_cost_from_hidden_params
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded, refetch_model_cost_map
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
|
@ -1812,7 +1813,7 @@ async def test_openrouter_decisions_uses_provider_reported_cost_without_cost_map
|
|||
)
|
||||
|
||||
assert route.called
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(cost)
|
||||
assert get_response_cost_from_hidden_params(response.hidden_params) == cost
|
||||
|
||||
|
||||
_SAFETY_IDENTIFIER: Final = "end-user-7"
|
||||
|
|
@ -5025,7 +5026,7 @@ async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports(
|
|||
)
|
||||
|
||||
assert response.answers[0].probability == 0.9
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(billed)
|
||||
assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(billed)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -5049,26 +5050,6 @@ async def test_vercel_ai_gateway_decisions_price_an_unreadable_charge_from_the_c
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("vercel_gateway_key")
|
||||
async def test_vercel_ai_gateway_decisions_apply_the_operator_discount_and_margin_to_the_gateway_charge(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "cost_discount_config", {"vercel_ai_gateway": 0.5})
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"vercel_ai_gateway": 0.1})
|
||||
respx_mock.post(_VERCEL_DECISIONS_URL).respond(
|
||||
json=_vercel_response("openai/gpt-6-luna-decisions", {"gateway": {"cost": "0.00005722"}})
|
||||
)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model="vercel_ai_gateway/openai/gpt-6-luna-decisions",
|
||||
input="The package arrived broken.",
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
)
|
||||
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(5.722e-05 * 0.5 * 1.1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("vercel_gateway_key")
|
||||
@pytest.mark.parametrize("model", ("typesafe-ai/jev", "typesafe-ai/jev-1.13.0", "typesafe-ai/jev-1.14.0"))
|
||||
|
|
@ -5153,36 +5134,65 @@ async def test_vercel_ai_gateway_jev_sends_box_questions(
|
|||
assert response.answers[0].confidence == 0.8
|
||||
|
||||
|
||||
_UPSTREAM_SPLIT: Final[Mapping[str, int]] = {"cached_tokens": _INPUT_TOKENS - 20, "cache_write_tokens": 20}
|
||||
_TOP_LEVEL_SPLIT: Final[Mapping[str, int]] = {"cached_tokens": 300, "cache_write_tokens": 7}
|
||||
|
||||
|
||||
def _upstream_usage(input_tokens: int, details: Mapping[str, int] | None) -> Mapping[str, object]:
|
||||
return {
|
||||
"usage": {
|
||||
"input_tokens": input_tokens,
|
||||
**({} if details is None else {"input_tokens_details": details}),
|
||||
"output_tokens": 0,
|
||||
"total_tokens": input_tokens,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("vercel_gateway_key")
|
||||
@pytest.mark.parametrize(
|
||||
("upstream_input_tokens", "cached_tokens", "cache_write_tokens"),
|
||||
((_INPUT_TOKENS, _INPUT_TOKENS - 20, 20), (_INPUT_TOKENS + 1, 0, 0)),
|
||||
ids=("upstream_reports_this_call", "upstream_count_disagrees"),
|
||||
("top_level_split", "served_by", "upstream", "expected_split"),
|
||||
(
|
||||
(None, "openai", _upstream_usage(_INPUT_TOKENS, _UPSTREAM_SPLIT), _UPSTREAM_SPLIT),
|
||||
(None, "openai", _upstream_usage(_INPUT_TOKENS + 1, _UPSTREAM_SPLIT), {"cached_tokens": 0, "cache_write_tokens": 0}),
|
||||
(_TOP_LEVEL_SPLIT, "openai", _upstream_usage(_INPUT_TOKENS, None), _TOP_LEVEL_SPLIT),
|
||||
(None, "anthropic", _upstream_usage(_INPUT_TOKENS, _UPSTREAM_SPLIT), {"cached_tokens": 0, "cache_write_tokens": 0}),
|
||||
),
|
||||
ids=(
|
||||
"serving_provider_reports_this_call",
|
||||
"upstream_count_disagrees",
|
||||
"upstream_omits_the_split",
|
||||
"report_from_a_provider_that_did_not_serve",
|
||||
),
|
||||
)
|
||||
async def test_vercel_ai_gateway_decisions_report_the_upstream_cache_split(
|
||||
upstream_input_tokens: int,
|
||||
cached_tokens: int,
|
||||
cache_write_tokens: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
top_level_split: Mapping[str, int] | None,
|
||||
served_by: str,
|
||||
upstream: Mapping[str, object],
|
||||
expected_split: Mapping[str, int],
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False)
|
||||
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key")
|
||||
respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond(
|
||||
respx_mock.post(_VERCEL_DECISIONS_URL).respond(
|
||||
json={
|
||||
"model": "openai/gpt-6-luna-decisions",
|
||||
"answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}],
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS},
|
||||
"usage": {
|
||||
"input_tokens": _INPUT_TOKENS,
|
||||
**({} if top_level_split is None else {"input_tokens_details": top_level_split}),
|
||||
"output_tokens": 0,
|
||||
"total_tokens": _INPUT_TOKENS,
|
||||
},
|
||||
"provider_metadata": {
|
||||
"openai": {
|
||||
"usage": {
|
||||
"input_tokens": upstream_input_tokens,
|
||||
"input_tokens_details": {"cached_tokens": _INPUT_TOKENS - 20, "cache_write_tokens": 20},
|
||||
"output_tokens": 0,
|
||||
"total_tokens": upstream_input_tokens,
|
||||
}
|
||||
"openai": upstream,
|
||||
"gateway": {
|
||||
"cost": "0.00005722",
|
||||
"routing": {
|
||||
"modelAttempts": [
|
||||
{"providerAttempts": [{"provider": served_by, "credentialType": "system", "success": True}]}
|
||||
]
|
||||
},
|
||||
},
|
||||
"gateway": {"cost": "0.00005722"},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
|
@ -5190,9 +5200,9 @@ async def test_vercel_ai_gateway_decisions_report_the_upstream_cache_split(
|
|||
response: Final = await litellm.adecisions(
|
||||
model="vercel_ai_gateway/openai/gpt-6-luna-decisions",
|
||||
input="The package arrived broken.",
|
||||
questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
)
|
||||
|
||||
assert response.usage.input_tokens_details.cached_tokens == cached_tokens
|
||||
assert response.usage.input_tokens_details.cache_write_tokens == cache_write_tokens
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(5.722e-05)
|
||||
assert response.usage.input_tokens_details.cached_tokens == expected_split["cached_tokens"]
|
||||
assert response.usage.input_tokens_details.cache_write_tokens == expected_split["cache_write_tokens"]
|
||||
assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(5.722e-05)
|
||||
|
|
|
|||
|
|
@ -1943,3 +1943,46 @@ def test_proxy_decisions_dispatches_vercel_ai_gateway_deployment(
|
|||
)
|
||||
assert expected_cost > 0
|
||||
assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gateway_cost", ("0.00005722", "0"), ids=("paid_call", "free_call"))
|
||||
def test_proxy_decisions_report_the_billed_gateway_charge_in_the_header_and_body(
|
||||
gateway_cost: str,
|
||||
client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False)
|
||||
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key")
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"vercel_ai_gateway": 0.5})
|
||||
deployment_router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gateway-decider",
|
||||
"litellm_params": {"model": "vercel_ai_gateway/example/unpriced-decider"},
|
||||
"model_info": {"mode": "evaluation"},
|
||||
}
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", deployment_router)
|
||||
respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond(
|
||||
json={
|
||||
"model": "example/unpriced-decider",
|
||||
"answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}],
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS},
|
||||
"provider_metadata": {"gateway": {"cost": gateway_cost}},
|
||||
}
|
||||
)
|
||||
|
||||
response: Final = client.post(
|
||||
"/v1/systemone",
|
||||
json={
|
||||
"model": "gateway-decider",
|
||||
"state": {"source": "proxy-test"},
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(response.json()["usage"]["cost"])
|
||||
assert response.json()["usage"]["cost"] == pytest.approx(float(gateway_cost))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue