fix(decisions): bill Vercel gateway charges with margin and BYOK, forward providerOptions

Backport of #50642 to lts/1.108.x.
Cherry-picked from 1f50a7c91f (mishdotdev:vercel-ai-gateway-decisions).
Adapted to the line: the line has no upstream_usage_cost billing path (main got it in #48953), so the reported charge still goes out as the line's response-cost header and the operator discount and margin do not apply to it there, the same as OpenRouter decisions on the line. The image check is resolved once per request beside the line's box-question check, and the tests read the billed cost the way the line bills it.
This commit is contained in:
mateo-berri 2026-11-09 16:40:59 -08:00
parent 3606c8ec07
commit 0222d0bdf4
6 changed files with 312 additions and 48 deletions

View file

@ -294,12 +294,15 @@ def _unsupported_params(
custom_llm_provider: str,
headers: Mapping[str, str],
supports_box_questions: bool,
supports_images: bool,
) -> tuple[str, ...]:
refuses_safety_identifier: Final = (
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
)
refused_image_param: Final = (
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
None
if supports_images and provider_config.supports_images_in(ir_request.input, headers)
else image_param(ir_request.input)
)
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
@ -332,12 +335,13 @@ def _without_unsupported_params(
provider_config: BaseDecisionsConfig,
headers: Mapping[str, str],
supports_box_questions: bool,
supports_images: bool,
) -> DecisionsIRRequest:
return replace(
ir_request,
input=(
ir_request.input
if provider_config.supports_images_in(ir_request.input, headers)
if supports_images and provider_config.supports_images_in(ir_request.input, headers)
else with_images_where(ir_request.input, _no_image)
),
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
@ -391,14 +395,17 @@ def _provider_ir_request(
supports_box_questions: Final = provider_config.supports_box_questions(
model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs)
)
supports_images: Final = provider_config.supports_image_input(model, custom_llm_provider)
unsupported: Final = _unsupported_params(
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions, supports_images
)
if not unsupported:
return ir_request
if not _drops_params(kwargs):
return _UnsupportedParams(names=unsupported)
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
supported: Final = _without_unsupported_params(
ir_request, provider_config, headers, supports_box_questions, supports_images
)
if _leaves_no_input(supported.input):
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
if not supported.questions:
@ -683,7 +690,7 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]:
model=call.requested_model,
llm_provider=call.custom_llm_provider,
)
return body
return {**body, **call.provider_config.provider_request_fields(call.litellm_params)}
def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:

View file

@ -63,6 +63,7 @@ _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter
_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"})
_REMOTE_URL_PREFIXES: Final = ("http://", "https://")
_DATA_URL_PREFIX: Final = "data:"
_NO_PROVIDER_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
def decisions_text(value: DecisionsJSON) -> str:
@ -541,6 +542,12 @@ class BaseDecisionsConfig(ABC):
) -> bool:
return self.supports_images
def supports_image_input(self, model: str, custom_llm_provider: str) -> bool:
return True
def provider_request_fields(self, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return _NO_PROVIDER_FIELDS
def supports_assets(self, headers: Mapping[str, str]) -> bool:
return False

View file

@ -1,50 +1,114 @@
from collections.abc import Mapping
from dataclasses import replace
from typing import Final
from itertools import chain
from types import MappingProxyType
from typing import Final, Literal, TypeVar
from pydantic import BaseModel, ConfigDict, ValidationError
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from litellm.llms.base_llm.decisions.transformation import usage_reported_cost
from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig
from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse
from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse, DecisionsIRUsage, OpenAIDecisionUsage
from litellm.utils import is_vision_explicitly_disabled
_Model: Final = TypeVar("_Model", bound=BaseModel)
_PROVIDER_OPTIONS: Final = "providerOptions"
class _ProviderAttempt(BaseModel):
model_config = ConfigDict(frozen=True)
credential_type: Literal["system", "byok"] | None = Field(default=None, alias="credentialType")
success: bool = False
class _ModelAttempt(BaseModel):
model_config = ConfigDict(frozen=True)
provider_attempts: tuple[_ProviderAttempt, ...] = Field(default=(), alias="providerAttempts")
class _Routing(BaseModel):
model_config = ConfigDict(frozen=True)
model_attempts: tuple[_ModelAttempt, ...] = Field(default=(), alias="modelAttempts")
class _GatewayCharge(BaseModel):
model_config = ConfigDict(frozen=True)
cost: float | None = None
cost: float
market_cost: float | None = Field(default=None, alias="marketCost")
routing: _Routing = _Routing()
def served_with_own_key(self) -> bool:
attempts: Final = chain.from_iterable(model.provider_attempts for model in self.routing.model_attempts)
return any(attempt.success and attempt.credential_type == "byok" for attempt in attempts)
def billed_cost(self) -> float:
if self.market_cost is None or not self.served_with_own_key():
return self.cost
return self.cost + self.market_cost
class _GatewayProviderMetadata(BaseModel):
class _UpstreamReport(BaseModel):
model_config = ConfigDict(frozen=True)
gateway: _GatewayCharge = _GatewayCharge()
usage: OpenAIDecisionUsage
class _GatewayDecisionResponse(BaseModel):
model_config = ConfigDict(frozen=True)
provider_metadata: _GatewayProviderMetadata = _GatewayProviderMetadata()
provider_metadata: Mapping[str, object]
def _gateway_charge(payload: object) -> float | None:
def _validated(model: type[_Model], value: object) -> _Model | None:
try:
return _GatewayDecisionResponse.model_validate(payload).provider_metadata.gateway.cost
return model.model_validate(value)
except ValidationError:
return None
def _gateway_usage(usage: DecisionsIRUsage, metadata: Mapping[str, object]) -> DecisionsIRUsage:
charge: Final = _validated(_GatewayCharge, metadata.get("gateway"))
reports: Final = (_validated(_UpstreamReport, entry) for entry in metadata.values())
upstream: Final = next(
(report.usage for report in reports if report is not None and report.usage.input_tokens == usage.input_tokens),
None,
)
priced: Final = usage if charge is None else replace(usage, extra={**usage.extra, "cost": charge.billed_cost()})
if upstream is None:
return priced
return replace(
priced,
cached_tokens=upstream.input_tokens_details.cached_tokens,
cache_write_tokens=upstream.input_tokens_details.cache_write_tokens,
)
class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig):
"""get_llm_provider already resolves the gateway's base and key, so never fall back to OpenAI credentials."""
inlines_remote_images = True
def resolve_api_key(self, api_key: str | None) -> str | None:
return api_key
def supports_image_input(self, model: str, custom_llm_provider: str) -> bool:
return not is_vision_explicitly_disabled(model, custom_llm_provider)
def provider_request_fields(self, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
if _PROVIDER_OPTIONS not in litellm_params:
return MappingProxyType({})
return MappingProxyType({_PROVIDER_OPTIONS: litellm_params[_PROVIDER_OPTIONS]})
def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
response: Final = super().parse_response(payload, request)
charge: Final = _gateway_charge(payload)
if charge is None:
metadata: Final = _validated(_GatewayDecisionResponse, payload)
if metadata is None:
return response
return replace(response, usage=replace(response.usage, extra={**response.usage.extra, "cost": charge}))
return replace(response, usage=_gateway_usage(response.usage, metadata.provider_metadata))
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
return usage_reported_cost(response)

View file

@ -51269,12 +51269,13 @@
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 32000,
"metadata": {
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000."
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with an input_image answered 400 invalid request body on 2026-11-09."
},
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://ai-gateway.vercel.sh/v1/models",
"supports_box_questions": true
"supports_box_questions": true,
"supports_vision": false
},
"vercel_ai_gateway/typesafe-ai/jev-1.13.0": {
"input_cost_per_token": 4.2e-08,

View file

@ -51269,12 +51269,13 @@
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 32000,
"metadata": {
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000."
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with an input_image answered 400 invalid request body on 2026-11-09."
},
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://ai-gateway.vercel.sh/v1/models",
"supports_box_questions": true
"supports_box_questions": true,
"supports_vision": false
},
"vercel_ai_gateway/typesafe-ai/jev-1.13.0": {
"input_cost_per_token": 4.2e-08,

View file

@ -3399,42 +3399,181 @@ async def test_vercel_ai_gateway_decisions_never_fall_back_to_the_openai_key(
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model", "gateway_cost", "reported_cost"),
(
("example/unpriced-decider", "0.000001386", 1.386e-06),
("openai/gpt-6-luna-decisions", "0.00005722", 5.722e-05),
("openai/gpt-6-luna-decisions", "not-a-number", None),
),
ids=("unpriced_model", "cache_hit_on_priced_model", "unreadable_charge"),
)
async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports(
model: str,
gateway_cost: str,
reported_cost: float | None,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
_VERCEL_DECISIONS_URL: Final = "https://ai-gateway.vercel.sh/v1/decisions"
def _gateway_routing(*attempts: tuple[str, bool]) -> Mapping[str, object]:
return {
"modelAttempts": [
{
"providerAttempts": [
{"provider": "openai", "credentialType": credential_type, "success": success}
for credential_type, success in attempts
]
}
]
}
def _vercel_response(model: str, provider_metadata: Mapping[str, object]) -> Mapping[str, object]:
return {
"model": model,
"answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}],
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS},
"provider_metadata": provider_metadata,
}
@pytest.fixture
def vercel_gateway_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False)
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key")
respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond(
json={
"model": model,
"answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}],
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS},
"provider_metadata": {"gateway": {"cost": gateway_cost}},
}
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("vercel_gateway_key")
@pytest.mark.parametrize(
("model", "gateway", "billed"),
(
("example/unpriced-decider", {"cost": "0.000001386"}, 1.386e-06),
("openai/gpt-6-luna-decisions", {"cost": "0.00005722"}, 5.722e-05),
(
"openai/gpt-6-luna-decisions",
{"cost": "0", "marketCost": "0.0000164", "routing": _gateway_routing(("byok", True))},
1.64e-05,
),
(
"openai/gpt-6-luna-decisions",
{
"cost": "0.0000164",
"marketCost": "0.0000164",
"routing": _gateway_routing(("byok", False), ("system", True)),
},
1.64e-05,
),
),
ids=("unpriced_model", "cache_hit_on_priced_model", "own_provider_key", "own_key_failed_over_to_gateway_credits"),
)
async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports(
model: str, gateway: Mapping[str, object], billed: float, respx_mock: respx.MockRouter
) -> None:
respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {"gateway": gateway}))
response: Final = await litellm.adecisions(
model=f"vercel_ai_gateway/{model}",
input="The package arrived broken.",
questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
questions=_PREDICATE_QUESTIONS,
)
assert response.answers[0].probability == 0.9
assert get_response_cost_from_hidden_params(response.hidden_params) == reported_cost
assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(billed)
@pytest.mark.asyncio
@pytest.mark.usefixtures("vercel_gateway_key")
async def test_vercel_ai_gateway_decisions_price_an_unreadable_charge_from_the_cost_map(
respx_mock: respx.MockRouter,
) -> None:
respx_mock.post(_VERCEL_DECISIONS_URL).respond(
json=_vercel_response("openai/gpt-6-luna-decisions", {"gateway": {"cost": "not-a-number"}})
)
response: Final = await litellm.adecisions(
model="vercel_ai_gateway/openai/gpt-6-luna-decisions",
input="The package arrived broken.",
questions=_PREDICATE_QUESTIONS,
)
luna: Final = litellm.model_cost["vercel_ai_gateway/openai/gpt-6-luna-decisions"]
assert get_response_cost_from_hidden_params(response.hidden_params) is None
assert litellm.completion_cost(completion_response=response) == pytest.approx(
_INPUT_TOKENS * float(luna["input_cost_per_token"])
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("vercel_gateway_key")
@pytest.mark.parametrize(
("model", "drop_params", "sent_content"),
(
(
"typesafe-ai/jev-1.14.0",
False,
[
{"type": "input_text", "text": _SCREEN_TEXT},
{"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "high"},
],
),
("typesafe-ai/jev", True, [{"type": "input_text", "text": _SCREEN_TEXT}]),
),
ids=("model_reads_images", "text_only_model_drops_them"),
)
async def test_vercel_ai_gateway_decisions_send_images_only_to_models_that_read_them(
model: str, drop_params: bool, sent_content: list[Mapping[str, object]], respx_mock: respx.MockRouter
) -> None:
route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {}))
await litellm.adecisions(
model=f"vercel_ai_gateway/{model}",
input=_screen_messages(_PNG_DATA_URL),
questions=_PREDICATE_QUESTIONS,
drop_params=drop_params,
)
assert json.loads(route.calls.last.request.content)["input"][0]["content"] == sent_content
@pytest.mark.asyncio
@pytest.mark.usefixtures("vercel_gateway_key")
async def test_vercel_ai_gateway_text_only_decisions_model_refuses_images_before_http(
respx_mock: respx.MockRouter,
) -> None:
with pytest.raises(litellm.UnsupportedParamsError, match="input_image"):
await litellm.adecisions(
model="vercel_ai_gateway/typesafe-ai/jev",
input=_screen_messages(_PNG_DATA_URL),
questions=_PREDICATE_QUESTIONS,
)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
@pytest.mark.usefixtures("vercel_gateway_key")
async def test_vercel_ai_gateway_decisions_inline_remote_images(
respx_mock: respx.MockRouter, async_only_image_fetch
) -> None:
route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond(
json=_vercel_response("openai/gpt-6-luna-decisions", {})
)
await litellm.adecisions(
model="vercel_ai_gateway/openai/gpt-6-luna-decisions",
input=_screen_messages(_REMOTE_IMAGE_URL),
questions=_PREDICATE_QUESTIONS,
)
assert async_only_image_fetch.fetched == [_REMOTE_IMAGE_URL]
sent_image: Final = json.loads(route.calls.last.request.content)["input"][0]["content"][1]
assert sent_image["image_url"] == async_only_image_fetch.data_url
@pytest.mark.asyncio
@pytest.mark.usefixtures("vercel_gateway_key")
async def test_vercel_ai_gateway_decisions_forward_provider_options(respx_mock: respx.MockRouter) -> None:
provider_options: Final = {"gateway": {"order": ["openai"], "byok": {"openai": [{"apiKey": "caller-openai-key"}]}}}
route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond(
json=_vercel_response("openai/gpt-6-luna-decisions", {})
)
await litellm.adecisions(
model="vercel_ai_gateway/openai/gpt-6-luna-decisions",
input="The package arrived broken.",
questions=_PREDICATE_QUESTIONS,
providerOptions=provider_options,
)
assert json.loads(route.calls.last.request.content)["providerOptions"] == provider_options
@pytest.mark.asyncio
@ -3466,3 +3605,48 @@ async def test_vercel_ai_gateway_jev_sends_box_questions(
assert json.loads(route.calls.last.request.content)["questions"] == [box_question]
assert response.answers[0].confidence == 0.8
@pytest.mark.asyncio
@pytest.mark.parametrize(
("upstream_input_tokens", "cached_tokens", "cache_write_tokens"),
((_INPUT_TOKENS, _INPUT_TOKENS - 20, 20), (_INPUT_TOKENS + 1, 0, 0)),
ids=("upstream_reports_this_call", "upstream_count_disagrees"),
)
async def test_vercel_ai_gateway_decisions_report_the_upstream_cache_split(
upstream_input_tokens: int,
cached_tokens: int,
cache_write_tokens: int,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False)
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key")
respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond(
json={
"model": "openai/gpt-6-luna-decisions",
"answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}],
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS},
"provider_metadata": {
"openai": {
"usage": {
"input_tokens": upstream_input_tokens,
"input_tokens_details": {"cached_tokens": _INPUT_TOKENS - 20, "cache_write_tokens": 20},
"output_tokens": 0,
"total_tokens": upstream_input_tokens,
}
},
"gateway": {"cost": "0.00005722"},
},
}
)
response: Final = await litellm.adecisions(
model="vercel_ai_gateway/openai/gpt-6-luna-decisions",
input="The package arrived broken.",
questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
)
assert response.usage.input_tokens_details.cached_tokens == cached_tokens
assert response.usage.input_tokens_details.cache_write_tokens == cache_write_tokens
assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(5.722e-05)