fix(decisions): bill Vercel AI Gateway decisions at the charge Vercel reports

Backport of #50642 to lts/1.108.x.
Cherry-picked from 7c40b75f56 (mishdotdev:vercel-ai-gateway-decisions).
Adapted to the line: its decisions usage is never None, so the None guards are dropped, and the resolve_credential override stays out as in the first pick.
This commit is contained in:
mateo-berri 2026-11-09 16:35:45 -08:00
parent 5703b6295e
commit 3606c8ec07
6 changed files with 126 additions and 9 deletions

View file

@ -461,6 +461,13 @@ def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
return is_decisions_model_mode(model_mode)
def usage_reported_cost(response: DecisionsIRResponse) -> float | None:
cost: Final = response.usage.extra.get("cost")
if not isinstance(cost, (int, float)):
return None
return float(cost)
class BaseDecisionsConfig(ABC):
path: str = "/v1/systemone"
api_key_env: tuple[str, ...] = ()

View file

@ -1,6 +1,4 @@
from typing import Final
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, usage_reported_cost
from litellm.types.decisions import DecisionsIRResponse
@ -13,7 +11,4 @@ class OpenRouterDecisionsConfig(BaseDecisionsConfig):
return "https://openrouter.ai/api"
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
cost: Final = response.usage.extra.get("cost")
if not isinstance(cost, (int, float)):
return None
return float(cost)
return usage_reported_cost(response)

View file

@ -1,4 +1,36 @@
from dataclasses import replace
from typing import Final
from pydantic import BaseModel, ConfigDict, ValidationError
from litellm.llms.base_llm.decisions.transformation import usage_reported_cost
from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig
from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse
class _GatewayCharge(BaseModel):
model_config = ConfigDict(frozen=True)
cost: float | None = None
class _GatewayProviderMetadata(BaseModel):
model_config = ConfigDict(frozen=True)
gateway: _GatewayCharge = _GatewayCharge()
class _GatewayDecisionResponse(BaseModel):
model_config = ConfigDict(frozen=True)
provider_metadata: _GatewayProviderMetadata = _GatewayProviderMetadata()
def _gateway_charge(payload: object) -> float | None:
try:
return _GatewayDecisionResponse.model_validate(payload).provider_metadata.gateway.cost
except ValidationError:
return None
class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig):
@ -6,3 +38,13 @@ class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig):
def resolve_api_key(self, api_key: str | None) -> str | None:
return api_key
def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
response: Final = super().parse_response(payload, request)
charge: Final = _gateway_charge(payload)
if charge is None:
return response
return replace(response, usage=replace(response.usage, extra={**response.usage.extra, "cost": charge}))
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
return usage_reported_cost(response)

View file

@ -51273,7 +51273,8 @@
},
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://ai-gateway.vercel.sh/v1/models"
"source": "https://ai-gateway.vercel.sh/v1/models",
"supports_box_questions": true
},
"vercel_ai_gateway/typesafe-ai/jev-1.13.0": {
"input_cost_per_token": 4.2e-08,
@ -51305,6 +51306,7 @@
"text",
"image"
],
"supports_box_questions": true,
"supports_vision": true
},
"vercel_ai_gateway/vercel/v0-1.0-md": {

View file

@ -51273,7 +51273,8 @@
},
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://ai-gateway.vercel.sh/v1/models"
"source": "https://ai-gateway.vercel.sh/v1/models",
"supports_box_questions": true
},
"vercel_ai_gateway/typesafe-ai/jev-1.13.0": {
"input_cost_per_token": 4.2e-08,
@ -51305,6 +51306,7 @@
"text",
"image"
],
"supports_box_questions": true,
"supports_vision": true
},
"vercel_ai_gateway/vercel/v0-1.0-md": {

View file

@ -3397,3 +3397,72 @@ async def test_vercel_ai_gateway_decisions_never_fall_back_to_the_openai_key(
)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model", "gateway_cost", "reported_cost"),
(
("example/unpriced-decider", "0.000001386", 1.386e-06),
("openai/gpt-6-luna-decisions", "0.00005722", 5.722e-05),
("openai/gpt-6-luna-decisions", "not-a-number", None),
),
ids=("unpriced_model", "cache_hit_on_priced_model", "unreadable_charge"),
)
async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports(
model: str,
gateway_cost: str,
reported_cost: float | None,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False)
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key")
respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond(
json={
"model": model,
"answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}],
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS},
"provider_metadata": {"gateway": {"cost": gateway_cost}},
}
)
response: Final = await litellm.adecisions(
model=f"vercel_ai_gateway/{model}",
input="The package arrived broken.",
questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
)
assert response.answers[0].probability == 0.9
assert get_response_cost_from_hidden_params(response.hidden_params) == reported_cost
@pytest.mark.asyncio
@pytest.mark.parametrize("model", ("typesafe-ai/jev", "typesafe-ai/jev-1.14.0"))
async def test_vercel_ai_gateway_jev_sends_box_questions(
model: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False)
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key")
box_question: Final = {"type": "box", "name": "crack", "instructions": "the crack on the screen"}
route: Final = respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond(
json={
"model": model,
"answers": [
{
"type": "box",
"name": "crack",
"box": {"top": 0.1, "bottom": 0.9, "left": 0.1, "right": 0.9},
"confidence": 0.8,
}
],
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS},
}
)
response: Final = await litellm.adecisions(
model=f"vercel_ai_gateway/{model}", input="The screen arrived cracked.", questions=[box_question]
)
assert json.loads(route.calls.last.request.content)["questions"] == [box_question]
assert response.answers[0].confidence == 0.8