From 0222d0bdf48ba604410e30aa692a451ac0f358d0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:40:59 -0800 Subject: [PATCH] fix(decisions): bill Vercel gateway charges with margin and BYOK, forward providerOptions Backport of #50642 to lts/1.108.x. Cherry-picked from 1f50a7c91f5032cbd350ab71af204245f2c640a7 (mishdotdev:vercel-ai-gateway-decisions). Adapted to the line: the line has no upstream_usage_cost billing path (main got it in #48953), so the reported charge still goes out as the line's response-cost header and the operator discount and margin do not apply to it there, the same as OpenRouter decisions on the line. The image check is resolved once per request beside the line's box-question check, and the tests read the billed cost the way the line bills it. --- litellm/decisions/call.py | 17 +- .../llms/base_llm/decisions/transformation.py | 7 + .../decisions/transformation.py | 88 ++++++- ...odel_prices_and_context_window_backup.json | 5 +- model_prices_and_context_window.json | 5 +- tests/unit/decisions/test_main.py | 238 ++++++++++++++++-- 6 files changed, 312 insertions(+), 48 deletions(-) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 307634b55d8..3e3b965173a 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -294,12 +294,15 @@ def _unsupported_params( custom_llm_provider: str, headers: Mapping[str, str], supports_box_questions: bool, + supports_images: bool, ) -> tuple[str, ...]: refuses_safety_identifier: Final = ( ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier ) refused_image_param: Final = ( - None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input) + None + if supports_images and provider_config.supports_images_in(ir_request.input, headers) + else image_param(ir_request.input) ) refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers) refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions @@ -332,12 +335,13 @@ def _without_unsupported_params( provider_config: BaseDecisionsConfig, headers: Mapping[str, str], supports_box_questions: bool, + supports_images: bool, ) -> DecisionsIRRequest: return replace( ir_request, input=( ir_request.input - if provider_config.supports_images_in(ir_request.input, headers) + if supports_images and provider_config.supports_images_in(ir_request.input, headers) else with_images_where(ir_request.input, _no_image) ), safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None, @@ -391,14 +395,17 @@ def _provider_ir_request( supports_box_questions: Final = provider_config.supports_box_questions( model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs) ) + supports_images: Final = provider_config.supports_image_input(model, custom_llm_provider) unsupported: Final = _unsupported_params( - ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions + ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions, supports_images ) if not unsupported: return ir_request if not _drops_params(kwargs): return _UnsupportedParams(names=unsupported) - supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions) + supported: Final = _without_unsupported_params( + ir_request, provider_config, headers, supports_box_questions, supports_images + ) if _leaves_no_input(supported.input): return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on") if not supported.questions: @@ -683,7 +690,7 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]: model=call.requested_model, llm_provider=call.custom_llm_provider, ) - return body + return {**body, **call.provider_config.provider_request_fields(call.litellm_params)} def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 0f33a42ab61..c025e66ccaf 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -63,6 +63,7 @@ _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) _REMOTE_URL_PREFIXES: Final = ("http://", "https://") _DATA_URL_PREFIX: Final = "data:" +_NO_PROVIDER_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) def decisions_text(value: DecisionsJSON) -> str: @@ -541,6 +542,12 @@ class BaseDecisionsConfig(ABC): ) -> bool: return self.supports_images + def supports_image_input(self, model: str, custom_llm_provider: str) -> bool: + return True + + def provider_request_fields(self, litellm_params: Mapping[str, object]) -> Mapping[str, object]: + return _NO_PROVIDER_FIELDS + def supports_assets(self, headers: Mapping[str, str]) -> bool: return False diff --git a/litellm/llms/vercel_ai_gateway/decisions/transformation.py b/litellm/llms/vercel_ai_gateway/decisions/transformation.py index 1152529d6ca..01d02294abf 100644 --- a/litellm/llms/vercel_ai_gateway/decisions/transformation.py +++ b/litellm/llms/vercel_ai_gateway/decisions/transformation.py @@ -1,50 +1,114 @@ +from collections.abc import Mapping from dataclasses import replace -from typing import Final +from itertools import chain +from types import MappingProxyType +from typing import Final, Literal, TypeVar -from pydantic import BaseModel, ConfigDict, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError from litellm.llms.base_llm.decisions.transformation import usage_reported_cost from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig -from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse +from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse, DecisionsIRUsage, OpenAIDecisionUsage +from litellm.utils import is_vision_explicitly_disabled + +_Model: Final = TypeVar("_Model", bound=BaseModel) +_PROVIDER_OPTIONS: Final = "providerOptions" + + +class _ProviderAttempt(BaseModel): + model_config = ConfigDict(frozen=True) + + credential_type: Literal["system", "byok"] | None = Field(default=None, alias="credentialType") + success: bool = False + + +class _ModelAttempt(BaseModel): + model_config = ConfigDict(frozen=True) + + provider_attempts: tuple[_ProviderAttempt, ...] = Field(default=(), alias="providerAttempts") + + +class _Routing(BaseModel): + model_config = ConfigDict(frozen=True) + + model_attempts: tuple[_ModelAttempt, ...] = Field(default=(), alias="modelAttempts") class _GatewayCharge(BaseModel): model_config = ConfigDict(frozen=True) - cost: float | None = None + cost: float + market_cost: float | None = Field(default=None, alias="marketCost") + routing: _Routing = _Routing() + + def served_with_own_key(self) -> bool: + attempts: Final = chain.from_iterable(model.provider_attempts for model in self.routing.model_attempts) + return any(attempt.success and attempt.credential_type == "byok" for attempt in attempts) + + def billed_cost(self) -> float: + if self.market_cost is None or not self.served_with_own_key(): + return self.cost + return self.cost + self.market_cost -class _GatewayProviderMetadata(BaseModel): +class _UpstreamReport(BaseModel): model_config = ConfigDict(frozen=True) - gateway: _GatewayCharge = _GatewayCharge() + usage: OpenAIDecisionUsage class _GatewayDecisionResponse(BaseModel): model_config = ConfigDict(frozen=True) - provider_metadata: _GatewayProviderMetadata = _GatewayProviderMetadata() + provider_metadata: Mapping[str, object] -def _gateway_charge(payload: object) -> float | None: +def _validated(model: type[_Model], value: object) -> _Model | None: try: - return _GatewayDecisionResponse.model_validate(payload).provider_metadata.gateway.cost + return model.model_validate(value) except ValidationError: return None +def _gateway_usage(usage: DecisionsIRUsage, metadata: Mapping[str, object]) -> DecisionsIRUsage: + charge: Final = _validated(_GatewayCharge, metadata.get("gateway")) + reports: Final = (_validated(_UpstreamReport, entry) for entry in metadata.values()) + upstream: Final = next( + (report.usage for report in reports if report is not None and report.usage.input_tokens == usage.input_tokens), + None, + ) + priced: Final = usage if charge is None else replace(usage, extra={**usage.extra, "cost": charge.billed_cost()}) + if upstream is None: + return priced + return replace( + priced, + cached_tokens=upstream.input_tokens_details.cached_tokens, + cache_write_tokens=upstream.input_tokens_details.cache_write_tokens, + ) + + class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig): """get_llm_provider already resolves the gateway's base and key, so never fall back to OpenAI credentials.""" + inlines_remote_images = True + def resolve_api_key(self, api_key: str | None) -> str | None: return api_key + def supports_image_input(self, model: str, custom_llm_provider: str) -> bool: + return not is_vision_explicitly_disabled(model, custom_llm_provider) + + def provider_request_fields(self, litellm_params: Mapping[str, object]) -> Mapping[str, object]: + if _PROVIDER_OPTIONS not in litellm_params: + return MappingProxyType({}) + return MappingProxyType({_PROVIDER_OPTIONS: litellm_params[_PROVIDER_OPTIONS]}) + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: response: Final = super().parse_response(payload, request) - charge: Final = _gateway_charge(payload) - if charge is None: + metadata: Final = _validated(_GatewayDecisionResponse, payload) + if metadata is None: return response - return replace(response, usage=replace(response.usage, extra={**response.usage.extra, "cost": charge})) + return replace(response, usage=_gateway_usage(response.usage, metadata.provider_metadata)) def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: return usage_reported_cost(response) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 59f5d5a213a..91788da2de6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -51269,12 +51269,13 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "metadata": { - "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000." + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with an input_image answered 400 invalid request body on 2026-11-09." }, "mode": "evaluation", "output_cost_per_token": 0.0, "source": "https://ai-gateway.vercel.sh/v1/models", - "supports_box_questions": true + "supports_box_questions": true, + "supports_vision": false }, "vercel_ai_gateway/typesafe-ai/jev-1.13.0": { "input_cost_per_token": 4.2e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 59f5d5a213a..91788da2de6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -51269,12 +51269,13 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "metadata": { - "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000." + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with an input_image answered 400 invalid request body on 2026-11-09." }, "mode": "evaluation", "output_cost_per_token": 0.0, "source": "https://ai-gateway.vercel.sh/v1/models", - "supports_box_questions": true + "supports_box_questions": true, + "supports_vision": false }, "vercel_ai_gateway/typesafe-ai/jev-1.13.0": { "input_cost_per_token": 4.2e-08, diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index f8662a8bc59..463192ec2fa 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3399,42 +3399,181 @@ async def test_vercel_ai_gateway_decisions_never_fall_back_to_the_openai_key( assert len(respx_mock.calls) == 0 -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("model", "gateway_cost", "reported_cost"), - ( - ("example/unpriced-decider", "0.000001386", 1.386e-06), - ("openai/gpt-6-luna-decisions", "0.00005722", 5.722e-05), - ("openai/gpt-6-luna-decisions", "not-a-number", None), - ), - ids=("unpriced_model", "cache_hit_on_priced_model", "unreadable_charge"), -) -async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports( - model: str, - gateway_cost: str, - reported_cost: float | None, - monkeypatch: pytest.MonkeyPatch, - respx_mock: respx.MockRouter, -) -> None: +_VERCEL_DECISIONS_URL: Final = "https://ai-gateway.vercel.sh/v1/decisions" + + +def _gateway_routing(*attempts: tuple[str, bool]) -> Mapping[str, object]: + return { + "modelAttempts": [ + { + "providerAttempts": [ + {"provider": "openai", "credentialType": credential_type, "success": success} + for credential_type, success in attempts + ] + } + ] + } + + +def _vercel_response(model: str, provider_metadata: Mapping[str, object]) -> Mapping[str, object]: + return { + "model": model, + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + "provider_metadata": provider_metadata, + } + + +@pytest.fixture +def vercel_gateway_key(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") - respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( - json={ - "model": model, - "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], - "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, - "provider_metadata": {"gateway": {"cost": gateway_cost}}, - } - ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +@pytest.mark.parametrize( + ("model", "gateway", "billed"), + ( + ("example/unpriced-decider", {"cost": "0.000001386"}, 1.386e-06), + ("openai/gpt-6-luna-decisions", {"cost": "0.00005722"}, 5.722e-05), + ( + "openai/gpt-6-luna-decisions", + {"cost": "0", "marketCost": "0.0000164", "routing": _gateway_routing(("byok", True))}, + 1.64e-05, + ), + ( + "openai/gpt-6-luna-decisions", + { + "cost": "0.0000164", + "marketCost": "0.0000164", + "routing": _gateway_routing(("byok", False), ("system", True)), + }, + 1.64e-05, + ), + ), + ids=("unpriced_model", "cache_hit_on_priced_model", "own_provider_key", "own_key_failed_over_to_gateway_credits"), +) +async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports( + model: str, gateway: Mapping[str, object], billed: float, respx_mock: respx.MockRouter +) -> None: + respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {"gateway": gateway})) response: Final = await litellm.adecisions( model=f"vercel_ai_gateway/{model}", input="The package arrived broken.", - questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + questions=_PREDICATE_QUESTIONS, ) assert response.answers[0].probability == 0.9 - assert get_response_cost_from_hidden_params(response.hidden_params) == reported_cost + assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(billed) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_price_an_unreadable_charge_from_the_cost_map( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {"gateway": {"cost": "not-a-number"}}) + ) + + response: Final = await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + ) + + luna: Final = litellm.model_cost["vercel_ai_gateway/openai/gpt-6-luna-decisions"] + assert get_response_cost_from_hidden_params(response.hidden_params) is None + assert litellm.completion_cost(completion_response=response) == pytest.approx( + _INPUT_TOKENS * float(luna["input_cost_per_token"]) + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +@pytest.mark.parametrize( + ("model", "drop_params", "sent_content"), + ( + ( + "typesafe-ai/jev-1.14.0", + False, + [ + {"type": "input_text", "text": _SCREEN_TEXT}, + {"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "high"}, + ], + ), + ("typesafe-ai/jev", True, [{"type": "input_text", "text": _SCREEN_TEXT}]), + ), + ids=("model_reads_images", "text_only_model_drops_them"), +) +async def test_vercel_ai_gateway_decisions_send_images_only_to_models_that_read_them( + model: str, drop_params: bool, sent_content: list[Mapping[str, object]], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {})) + + await litellm.adecisions( + model=f"vercel_ai_gateway/{model}", + input=_screen_messages(_PNG_DATA_URL), + questions=_PREDICATE_QUESTIONS, + drop_params=drop_params, + ) + + assert json.loads(route.calls.last.request.content)["input"][0]["content"] == sent_content + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_text_only_decisions_model_refuses_images_before_http( + respx_mock: respx.MockRouter, +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="input_image"): + await litellm.adecisions( + model="vercel_ai_gateway/typesafe-ai/jev", + input=_screen_messages(_PNG_DATA_URL), + questions=_PREDICATE_QUESTIONS, + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_inline_remote_images( + respx_mock: respx.MockRouter, async_only_image_fetch +) -> None: + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {}) + ) + + await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input=_screen_messages(_REMOTE_IMAGE_URL), + questions=_PREDICATE_QUESTIONS, + ) + + assert async_only_image_fetch.fetched == [_REMOTE_IMAGE_URL] + sent_image: Final = json.loads(route.calls.last.request.content)["input"][0]["content"][1] + assert sent_image["image_url"] == async_only_image_fetch.data_url + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_forward_provider_options(respx_mock: respx.MockRouter) -> None: + provider_options: Final = {"gateway": {"order": ["openai"], "byok": {"openai": [{"apiKey": "caller-openai-key"}]}}} + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {}) + ) + + await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + providerOptions=provider_options, + ) + + assert json.loads(route.calls.last.request.content)["providerOptions"] == provider_options @pytest.mark.asyncio @@ -3466,3 +3605,48 @@ async def test_vercel_ai_gateway_jev_sends_box_questions( assert json.loads(route.calls.last.request.content)["questions"] == [box_question] assert response.answers[0].confidence == 0.8 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("upstream_input_tokens", "cached_tokens", "cache_write_tokens"), + ((_INPUT_TOKENS, _INPUT_TOKENS - 20, 20), (_INPUT_TOKENS + 1, 0, 0)), + ids=("upstream_reports_this_call", "upstream_count_disagrees"), +) +async def test_vercel_ai_gateway_decisions_report_the_upstream_cache_split( + upstream_input_tokens: int, + cached_tokens: int, + cache_write_tokens: int, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": "openai/gpt-6-luna-decisions", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + "provider_metadata": { + "openai": { + "usage": { + "input_tokens": upstream_input_tokens, + "input_tokens_details": {"cached_tokens": _INPUT_TOKENS - 20, "cache_write_tokens": 20}, + "output_tokens": 0, + "total_tokens": upstream_input_tokens, + } + }, + "gateway": {"cost": "0.00005722"}, + }, + } + ) + + response: Final = await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + ) + + assert response.usage.input_tokens_details.cached_tokens == cached_tokens + assert response.usage.input_tokens_details.cache_write_tokens == cache_write_tokens + assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(5.722e-05)