From 3606c8ec0751b34e3de17fc9d85114e68cdbeaaf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:35:45 -0800 Subject: [PATCH] fix(decisions): bill Vercel AI Gateway decisions at the charge Vercel reports Backport of #50642 to lts/1.108.x. Cherry-picked from 7c40b75f5657ec067f9a6fe2c7c91f8662c81a78 (mishdotdev:vercel-ai-gateway-decisions). Adapted to the line: its decisions usage is never None, so the None guards are dropped, and the resolve_credential override stays out as in the first pick. --- .../llms/base_llm/decisions/transformation.py | 7 ++ .../openrouter/decisions/transformation.py | 9 +-- .../decisions/transformation.py | 42 +++++++++++ ...odel_prices_and_context_window_backup.json | 4 +- model_prices_and_context_window.json | 4 +- tests/unit/decisions/test_main.py | 69 +++++++++++++++++++ 6 files changed, 126 insertions(+), 9 deletions(-) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 041473cb634..0f33a42ab61 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -461,6 +461,13 @@ def _is_decisions_model(model: str, custom_llm_provider: str) -> bool: return is_decisions_model_mode(model_mode) +def usage_reported_cost(response: DecisionsIRResponse) -> float | None: + cost: Final = response.usage.extra.get("cost") + if not isinstance(cost, (int, float)): + return None + return float(cost) + + class BaseDecisionsConfig(ABC): path: str = "/v1/systemone" api_key_env: tuple[str, ...] = () diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 0fa5c7ca7cb..e84a6162713 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,6 +1,4 @@ -from typing import Final - -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, usage_reported_cost from litellm.types.decisions import DecisionsIRResponse @@ -13,7 +11,4 @@ class OpenRouterDecisionsConfig(BaseDecisionsConfig): return "https://openrouter.ai/api" def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: - cost: Final = response.usage.extra.get("cost") - if not isinstance(cost, (int, float)): - return None - return float(cost) + return usage_reported_cost(response) diff --git a/litellm/llms/vercel_ai_gateway/decisions/transformation.py b/litellm/llms/vercel_ai_gateway/decisions/transformation.py index 36b2275435b..1152529d6ca 100644 --- a/litellm/llms/vercel_ai_gateway/decisions/transformation.py +++ b/litellm/llms/vercel_ai_gateway/decisions/transformation.py @@ -1,4 +1,36 @@ +from dataclasses import replace +from typing import Final + +from pydantic import BaseModel, ConfigDict, ValidationError + +from litellm.llms.base_llm.decisions.transformation import usage_reported_cost from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig +from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse + + +class _GatewayCharge(BaseModel): + model_config = ConfigDict(frozen=True) + + cost: float | None = None + + +class _GatewayProviderMetadata(BaseModel): + model_config = ConfigDict(frozen=True) + + gateway: _GatewayCharge = _GatewayCharge() + + +class _GatewayDecisionResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + provider_metadata: _GatewayProviderMetadata = _GatewayProviderMetadata() + + +def _gateway_charge(payload: object) -> float | None: + try: + return _GatewayDecisionResponse.model_validate(payload).provider_metadata.gateway.cost + except ValidationError: + return None class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig): @@ -6,3 +38,13 @@ class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig): def resolve_api_key(self, api_key: str | None) -> str | None: return api_key + + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + response: Final = super().parse_response(payload, request) + charge: Final = _gateway_charge(payload) + if charge is None: + return response + return replace(response, usage=replace(response.usage, extra={**response.usage.extra, "cost": charge})) + + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: + return usage_reported_cost(response) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6f43833c225..59f5d5a213a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -51273,7 +51273,8 @@ }, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://ai-gateway.vercel.sh/v1/models" + "source": "https://ai-gateway.vercel.sh/v1/models", + "supports_box_questions": true }, "vercel_ai_gateway/typesafe-ai/jev-1.13.0": { "input_cost_per_token": 4.2e-08, @@ -51305,6 +51306,7 @@ "text", "image" ], + "supports_box_questions": true, "supports_vision": true }, "vercel_ai_gateway/vercel/v0-1.0-md": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6f43833c225..59f5d5a213a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -51273,7 +51273,8 @@ }, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://ai-gateway.vercel.sh/v1/models" + "source": "https://ai-gateway.vercel.sh/v1/models", + "supports_box_questions": true }, "vercel_ai_gateway/typesafe-ai/jev-1.13.0": { "input_cost_per_token": 4.2e-08, @@ -51305,6 +51306,7 @@ "text", "image" ], + "supports_box_questions": true, "supports_vision": true }, "vercel_ai_gateway/vercel/v0-1.0-md": { diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index bc13f4dc005..f8662a8bc59 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3397,3 +3397,72 @@ async def test_vercel_ai_gateway_decisions_never_fall_back_to_the_openai_key( ) assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "gateway_cost", "reported_cost"), + ( + ("example/unpriced-decider", "0.000001386", 1.386e-06), + ("openai/gpt-6-luna-decisions", "0.00005722", 5.722e-05), + ("openai/gpt-6-luna-decisions", "not-a-number", None), + ), + ids=("unpriced_model", "cache_hit_on_priced_model", "unreadable_charge"), +) +async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports( + model: str, + gateway_cost: str, + reported_cost: float | None, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": model, + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + "provider_metadata": {"gateway": {"cost": gateway_cost}}, + } + ) + + response: Final = await litellm.adecisions( + model=f"vercel_ai_gateway/{model}", + input="The package arrived broken.", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + ) + + assert response.answers[0].probability == 0.9 + assert get_response_cost_from_hidden_params(response.hidden_params) == reported_cost + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("typesafe-ai/jev", "typesafe-ai/jev-1.14.0")) +async def test_vercel_ai_gateway_jev_sends_box_questions( + model: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + box_question: Final = {"type": "box", "name": "crack", "instructions": "the crack on the screen"} + route: Final = respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": model, + "answers": [ + { + "type": "box", + "name": "crack", + "box": {"top": 0.1, "bottom": 0.9, "left": 0.1, "right": 0.9}, + "confidence": 0.8, + } + ], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + } + ) + + response: Final = await litellm.adecisions( + model=f"vercel_ai_gateway/{model}", input="The screen arrived cracked.", questions=[box_question] + ) + + assert json.loads(route.calls.last.request.content)["questions"] == [box_question] + assert response.answers[0].confidence == 0.8