From fd6c00200c52e079ea010c3752ad241eb5ac45e5 Mon Sep 17 00:00:00 2001 From: Michael Bennett Date: Sat, 14 Nov 2026 15:43:08 +0100 Subject: [PATCH] feat(decisions): support Vercel AI Gateway decision models (#50642) * feat(decisions): support Vercel AI Gateway decision models * test(decisions): cover Vercel AI Gateway dispatch through the proxy * feat(decisions): add Vercel AI Gateway cost map rows for Jev * fix(decisions): bill Vercel AI Gateway decisions at the charge Vercel reports Vercel returns the charge for every call under provider_metadata.gateway.cost, already net of cache discounts, so unpriced evaluation models and luna cache hits now bill what Vercel charged instead of zero or the full input rate. Jev 1.14.0 and the jev alias also take box questions, as Vercel serves them * fix(decisions): bill Vercel gateway charges with margin and BYOK, forward providerOptions Decisions now records a provider's reported charge as upstream_usage_cost, so the operator's cost discount and margin apply to it like any other provider-billed cost. Vercel calls served on the caller's own provider key bill marketCost on top of the gateway fee, the cache split comes from the upstream usage Vercel nests under provider_metadata, providerOptions reach the gateway, remote images are inlined, and typesafe-ai/jev, which reads text only, refuses or drops images * fix(decisions): stop forwarding caller providerOptions to Vercel and send Jev images Decisions no longer sends a caller's providerOptions to Vercel. Gateway routing and provider keys are set on the Vercel team instead. Vercel serves typesafe-ai/jev images at 768x768 (the earlier 400 came from a 96x96 image TypeSafe rejects for its size), so the Jev rows read images and the per-model image refusal is removed. * fix(decisions): report provider charges on the cost header and read Vercel's cache split from the serving provider Decisions goes back to reporting a provider's own charge through the response cost header, the way main does for OpenRouter. Moving it onto upstream_usage_cost dropped every cost header on free calls, priced base_model deployments from the cost map, and put the margin in the response header but not in the SystemOne body cost. Vercel's cache split now comes from the provider_metadata entry of the provider that served the call, and only when that entry reports input_tokens_details. Any credentialType value is accepted. The Jev 1.13.0 and 1.14.0 rows record their image probes. * test(decisions): type the async image fetch fixture in the Vercel inline image test --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 4 + .../llms/base_llm/decisions/transformation.py | 7 + .../openrouter/decisions/transformation.py | 9 +- .../decisions/transformation.py | 138 ++++++++ ...odel_prices_and_context_window_backup.json | 63 ++++ litellm/utils.py | 2 + model_prices_and_context_window.json | 63 ++++ tests/unit/decisions/test_main.py | 301 ++++++++++++++++++ .../decisions_endpoints/test_endpoints.py | 99 ++++++ 10 files changed, 682 insertions(+), 7 deletions(-) create mode 100644 litellm/llms/vercel_ai_gateway/decisions/transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 98c36dc582a..b556eb2e307 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1797,6 +1797,9 @@ if TYPE_CHECKING: from .llms.azure_ai.decisions.transformation import ( AzureAIDecisionsConfig as AzureAIDecisionsConfig, ) + from .llms.vercel_ai_gateway.decisions.transformation import ( + VercelAIGatewayDecisionsConfig as VercelAIGatewayDecisionsConfig, + ) from .llms.nvidia_nim.rerank.transformation import ( NvidiaNimRerankConfig as NvidiaNimRerankConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 6d3c18042a6..d660f8f4ac9 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -135,6 +135,10 @@ LITELLM_ATTRIBUTES: Final[Mapping[str, tuple[str, str]]] = MappingProxyType( "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"), "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), + "VercelAIGatewayDecisionsConfig": ( + ".llms.vercel_ai_gateway.decisions.transformation", + "VercelAIGatewayDecisionsConfig", + ), "NvidiaNimRerankConfig": (".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig"), "NvidiaNimRankingConfig": (".llms.nvidia_nim.rerank.ranking_transformation", "NvidiaNimRankingConfig"), "VertexAIRerankConfig": (".llms.vertex_ai.rerank.transformation", "VertexAIRerankConfig"), diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index b84d5ce0ea1..56bc9f444db 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -520,6 +520,13 @@ def _is_decisions_model(model: str, custom_llm_provider: str) -> bool: return is_decisions_model_mode(model_mode) +def usage_reported_cost(response: DecisionsIRResponse) -> float | None: + cost: Final = response.usage.extra.get("cost") if response.usage is not None else None + if not isinstance(cost, (int, float)): + return None + return float(cost) + + @dataclass(frozen=True, slots=True) class DecisionsInputLimits: max_request_tokens: int | None = None diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 30d33f09efd..e84a6162713 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,6 +1,4 @@ -from typing import Final - -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, usage_reported_cost from litellm.types.decisions import DecisionsIRResponse @@ -13,7 +11,4 @@ class OpenRouterDecisionsConfig(BaseDecisionsConfig): return "https://openrouter.ai/api" def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: - cost: Final = response.usage.extra.get("cost") if response.usage is not None else None - if not isinstance(cost, (int, float)): - return None - return float(cost) + return usage_reported_cost(response) diff --git a/litellm/llms/vercel_ai_gateway/decisions/transformation.py b/litellm/llms/vercel_ai_gateway/decisions/transformation.py new file mode 100644 index 00000000000..af29267b17a --- /dev/null +++ b/litellm/llms/vercel_ai_gateway/decisions/transformation.py @@ -0,0 +1,138 @@ +from collections.abc import Mapping +from dataclasses import replace +from itertools import chain +from typing import Final, TypeVar + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from litellm.llms.base_llm.decisions.transformation import usage_reported_cost +from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig +from litellm.types.decisions import ( + DecisionsIRRequest, + DecisionsIRResponse, + DecisionsIRUsage, + OpenAIDecisionInputTokensDetails, +) + +_Model: Final = TypeVar("_Model", bound=BaseModel) + + +class _ProviderAttempt(BaseModel): + model_config = ConfigDict(frozen=True) + + provider: str | None = None + credential_type: str | None = Field(default=None, alias="credentialType") + success: bool = False + + +class _ModelAttempt(BaseModel): + model_config = ConfigDict(frozen=True) + + provider_attempts: tuple[_ProviderAttempt, ...] = Field(default=(), alias="providerAttempts") + + +class _Routing(BaseModel): + model_config = ConfigDict(frozen=True) + + model_attempts: tuple[_ModelAttempt, ...] = Field(default=(), alias="modelAttempts") + + def served_by(self) -> _ProviderAttempt | None: + attempts: Final = chain.from_iterable(model.provider_attempts for model in self.model_attempts) + return next((attempt for attempt in attempts if attempt.success), None) + + +class _GatewayRouting(BaseModel): + model_config = ConfigDict(frozen=True) + + routing: _Routing = _Routing() + + +class _GatewayCharge(BaseModel): + model_config = ConfigDict(frozen=True) + + cost: float + market_cost: float | None = Field(default=None, alias="marketCost") + + def billed_cost(self, served_by: _ProviderAttempt | None) -> float: + if self.market_cost is None or served_by is None or served_by.credential_type != "byok": + return self.cost + return self.cost + self.market_cost + + +class _UpstreamUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + input_tokens: int + input_tokens_details: OpenAIDecisionInputTokensDetails + + +class _UpstreamReport(BaseModel): + model_config = ConfigDict(frozen=True) + + usage: _UpstreamUsage + + +class _GatewayDecisionResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + provider_metadata: Mapping[str, object] + + +def _validated(model: type[_Model], value: object) -> _Model | None: + try: + return model.model_validate(value) + except ValidationError: + return None + + +def _served_by(gateway: object) -> _ProviderAttempt | None: + gateway_routing: Final = _validated(_GatewayRouting, gateway) + return None if gateway_routing is None else gateway_routing.routing.served_by() + + +def _upstream_usage(metadata: Mapping[str, object], served_by: _ProviderAttempt | None) -> _UpstreamUsage | None: + if served_by is None or served_by.provider is None: + return None + report: Final = _validated(_UpstreamReport, metadata.get(served_by.provider)) + return None if report is None else report.usage + + +def _gateway_usage(usage: DecisionsIRUsage, metadata: Mapping[str, object]) -> DecisionsIRUsage: + gateway: Final = metadata.get("gateway") + served_by: Final = _served_by(gateway) + charge: Final = _validated(_GatewayCharge, gateway) + priced: Final = ( + usage if charge is None else replace(usage, extra={**usage.extra, "cost": charge.billed_cost(served_by)}) + ) + upstream: Final = _upstream_usage(metadata, served_by) + if upstream is None or upstream.input_tokens != usage.input_tokens: + return priced + return replace( + priced, + cached_tokens=upstream.input_tokens_details.cached_tokens, + cache_write_tokens=upstream.input_tokens_details.cache_write_tokens, + ) + + +class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig): + """get_llm_provider already resolves the gateway's base and key, so never fall back to OpenAI credentials.""" + + inlines_remote_images = True + + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key + + def resolve_credential( + self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object] + ) -> str | None: + return api_key + + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + response: Final = super().parse_response(payload, request) + metadata: Final = _validated(_GatewayDecisionResponse, payload) + if metadata is None or response.usage is None: + return response + return replace(response, usage=_gateway_usage(response.usage, metadata.provider_metadata)) + + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: + return usage_reported_cost(response) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0e352ded72a..544815aaa0b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -52612,6 +52612,19 @@ "supports_tool_choice": true, "supports_response_schema": true }, + "vercel_ai_gateway/openai/gpt-6-luna-decisions": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 1050000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models" + }, "vercel_ai_gateway/openai/o1": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 7.5e-06, @@ -52723,6 +52736,56 @@ "mode": "chat", "output_cost_per_token": 8e-06 }, + "vercel_ai_gateway/typesafe-ai/jev": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with a 768x768 input_image answered 200 on 2026-11-10, while a 96x96 one answered 400 invalid request body." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_box_questions": true, + "supports_vision": true + }, + "vercel_ai_gateway/typesafe-ai/jev-1.13.0": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.13.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.99 for a gradient and 0.12 for solid gray." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "vercel_ai_gateway/typesafe-ai/jev-1.14.0": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.14.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.84 for a gradient and 0.13 for solid gray." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_box_questions": true, + "supports_vision": true + }, "vercel_ai_gateway/vercel/v0-1.0-md": { "input_cost_per_token": 3e-06, "litellm_provider": "vercel_ai_gateway", diff --git a/litellm/utils.py b/litellm/utils.py index e84f933abec..eae9addc8e0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10395,6 +10395,7 @@ class ProviderConfigManager: LlmProviders.AZURE_AI, LlmProviders.EU_JEV, LlmProviders.PARALLEL_AI, + LlmProviders.VERCEL_AI_GATEWAY, ): configs: Final = { LlmProviders.EMPIRIOLABS: litellm.EmpirioLabsDecisionsConfig, @@ -10403,6 +10404,7 @@ class ProviderConfigManager: LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig, LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig, LlmProviders.PARALLEL_AI: litellm.ParallelAIDecisionsConfig, + LlmProviders.VERCEL_AI_GATEWAY: litellm.VercelAIGatewayDecisionsConfig, } return configs[provider]() return None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0e352ded72a..544815aaa0b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -52612,6 +52612,19 @@ "supports_tool_choice": true, "supports_response_schema": true }, + "vercel_ai_gateway/openai/gpt-6-luna-decisions": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 1050000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models" + }, "vercel_ai_gateway/openai/o1": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 7.5e-06, @@ -52723,6 +52736,56 @@ "mode": "chat", "output_cost_per_token": 8e-06 }, + "vercel_ai_gateway/typesafe-ai/jev": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with a 768x768 input_image answered 200 on 2026-11-10, while a 96x96 one answered 400 invalid request body." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_box_questions": true, + "supports_vision": true + }, + "vercel_ai_gateway/typesafe-ai/jev-1.13.0": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.13.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.99 for a gradient and 0.12 for solid gray." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "vercel_ai_gateway/typesafe-ai/jev-1.14.0": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.14.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.84 for a gradient and 0.13 for solid gray." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_box_questions": true, + "supports_vision": true + }, "vercel_ai_gateway/vercel/v0-1.0-md": { "input_cost_per_token": 3e-06, "litellm_provider": "vercel_ai_gateway", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 8b470248be7..76add4d6fdd 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -4924,6 +4924,307 @@ def test_hosted_vllm_systemone_maps_image_rejection_to_bad_request( assert route.call_count == 1 +@pytest.mark.asyncio +async def test_vercel_ai_gateway_decisions_send_the_gateway_key_to_the_gateway_and_price_the_call( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") + monkeypatch.setenv("OPENAI_BASE_URL", "https://openai.example") + route: Final = respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": "openai/gpt-6-luna-decisions", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + } + ) + + response: Final = await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + ) + + assert route.calls.last.request.headers["authorization"] == "Bearer gateway-key" + assert json.loads(route.calls.last.request.content) == { + "model": "openai/gpt-6-luna-decisions", + "input": "The package arrived broken.", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } + assert response.answers[0].probability == 0.9 + input_rate: Final = litellm.model_cost["vercel_ai_gateway/openai/gpt-6-luna-decisions"]["input_cost_per_token"] + assert input_rate > 0 + assert litellm.completion_cost(completion_response=response) == pytest.approx(_INPUT_TOKENS * input_rate) + + +@pytest.mark.asyncio +async def test_vercel_ai_gateway_decisions_never_fall_back_to_the_openai_key( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_KEY", raising=False) + monkeypatch.delenv("VERCEL_OIDC_TOKEN", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") + + with pytest.raises(litellm.AuthenticationError): + await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="review", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + ) + + assert len(respx_mock.calls) == 0 + + +_VERCEL_DECISIONS_URL: Final = "https://ai-gateway.vercel.sh/v1/decisions" + + +def _gateway_routing(*attempts: tuple[str, bool]) -> Mapping[str, object]: + return { + "modelAttempts": [ + { + "providerAttempts": [ + {"provider": "openai", "credentialType": credential_type, "success": success} + for credential_type, success in attempts + ] + } + ] + } + + +def _vercel_response(model: str, provider_metadata: Mapping[str, object]) -> Mapping[str, object]: + return { + "model": model, + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + "provider_metadata": provider_metadata, + } + + +@pytest.fixture +def vercel_gateway_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +@pytest.mark.parametrize( + ("model", "gateway", "billed"), + ( + ("example/unpriced-decider", {"cost": "0.000001386"}, 1.386e-06), + ("openai/gpt-6-luna-decisions", {"cost": "0.00005722"}, 5.722e-05), + ( + "openai/gpt-6-luna-decisions", + {"cost": "0", "marketCost": "0.0000164", "routing": _gateway_routing(("byok", True))}, + 1.64e-05, + ), + ( + "openai/gpt-6-luna-decisions", + { + "cost": "0.0000164", + "marketCost": "0.0000164", + "routing": _gateway_routing(("byok", False), ("system", True)), + }, + 1.64e-05, + ), + ), + ids=("unpriced_model", "cache_hit_on_priced_model", "own_provider_key", "own_key_failed_over_to_gateway_credits"), +) +async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports( + model: str, gateway: Mapping[str, object], billed: float, respx_mock: respx.MockRouter +) -> None: + respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {"gateway": gateway})) + + response: Final = await litellm.adecisions( + model=f"vercel_ai_gateway/{model}", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + ) + + assert response.answers[0].probability == 0.9 + assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(billed) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_price_an_unreadable_charge_from_the_cost_map( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {"gateway": {"cost": "not-a-number"}}) + ) + + response: Final = await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + ) + + luna: Final = litellm.model_cost["vercel_ai_gateway/openai/gpt-6-luna-decisions"] + assert litellm.completion_cost(completion_response=response) == pytest.approx( + _INPUT_TOKENS * float(luna["input_cost_per_token"]) + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +@pytest.mark.parametrize("model", ("typesafe-ai/jev", "typesafe-ai/jev-1.13.0", "typesafe-ai/jev-1.14.0")) +async def test_vercel_ai_gateway_jev_decisions_send_images(model: str, respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {})) + + await litellm.adecisions( + model=f"vercel_ai_gateway/{model}", input=_screen_messages(_PNG_DATA_URL), questions=_PREDICATE_QUESTIONS + ) + + assert json.loads(route.calls.last.request.content)["input"][0]["content"] == [ + {"type": "input_text", "text": _SCREEN_TEXT}, + {"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "high"}, + ] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_inline_remote_images( + respx_mock: respx.MockRouter, async_only_image_fetch: AsyncOnlyImageFetch +) -> None: + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {}) + ) + + await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input=_screen_messages(_REMOTE_IMAGE_URL), + questions=_PREDICATE_QUESTIONS, + ) + + assert async_only_image_fetch.fetched == [_REMOTE_IMAGE_URL] + sent_image: Final = json.loads(route.calls.last.request.content)["input"][0]["content"][1] + assert sent_image["image_url"] == async_only_image_fetch.data_url + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_never_send_caller_provider_options(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {}) + ) + + await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + providerOptions={"gateway": {"models": ["openai/gpt-6"], "byok": {"openai": [{"apiKey": "caller-openai-key"}]}}}, + ) + + assert "providerOptions" not in json.loads(route.calls.last.request.content) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("typesafe-ai/jev", "typesafe-ai/jev-1.14.0")) +async def test_vercel_ai_gateway_jev_sends_box_questions( + model: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + box_question: Final = {"type": "box", "name": "crack", "instructions": "the crack on the screen"} + route: Final = respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": model, + "answers": [ + { + "type": "box", + "name": "crack", + "box": {"top": 0.1, "bottom": 0.9, "left": 0.1, "right": 0.9}, + "confidence": 0.8, + } + ], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + } + ) + + response: Final = await litellm.adecisions( + model=f"vercel_ai_gateway/{model}", input="The screen arrived cracked.", questions=[box_question] + ) + + assert json.loads(route.calls.last.request.content)["questions"] == [box_question] + assert response.answers[0].confidence == 0.8 + + +_UPSTREAM_SPLIT: Final[Mapping[str, int]] = {"cached_tokens": _INPUT_TOKENS - 20, "cache_write_tokens": 20} +_TOP_LEVEL_SPLIT: Final[Mapping[str, int]] = {"cached_tokens": 300, "cache_write_tokens": 7} + + +def _upstream_usage(input_tokens: int, details: Mapping[str, int] | None) -> Mapping[str, object]: + return { + "usage": { + "input_tokens": input_tokens, + **({} if details is None else {"input_tokens_details": details}), + "output_tokens": 0, + "total_tokens": input_tokens, + } + } + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +@pytest.mark.parametrize( + ("top_level_split", "served_by", "upstream", "expected_split"), + ( + (None, "openai", _upstream_usage(_INPUT_TOKENS, _UPSTREAM_SPLIT), _UPSTREAM_SPLIT), + (None, "openai", _upstream_usage(_INPUT_TOKENS + 1, _UPSTREAM_SPLIT), {"cached_tokens": 0, "cache_write_tokens": 0}), + (_TOP_LEVEL_SPLIT, "openai", _upstream_usage(_INPUT_TOKENS, None), _TOP_LEVEL_SPLIT), + (None, "anthropic", _upstream_usage(_INPUT_TOKENS, _UPSTREAM_SPLIT), {"cached_tokens": 0, "cache_write_tokens": 0}), + ), + ids=( + "serving_provider_reports_this_call", + "upstream_count_disagrees", + "upstream_omits_the_split", + "report_from_a_provider_that_did_not_serve", + ), +) +async def test_vercel_ai_gateway_decisions_report_the_upstream_cache_split( + top_level_split: Mapping[str, int] | None, + served_by: str, + upstream: Mapping[str, object], + expected_split: Mapping[str, int], + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json={ + "model": "openai/gpt-6-luna-decisions", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": { + "input_tokens": _INPUT_TOKENS, + **({} if top_level_split is None else {"input_tokens_details": top_level_split}), + "output_tokens": 0, + "total_tokens": _INPUT_TOKENS, + }, + "provider_metadata": { + "openai": upstream, + "gateway": { + "cost": "0.00005722", + "routing": { + "modelAttempts": [ + {"providerAttempts": [{"provider": served_by, "credentialType": "system", "success": True}]} + ] + }, + }, + }, + } + ) + + response: Final = await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + ) + + assert response.usage.input_tokens_details.cached_tokens == expected_split["cached_tokens"] + assert response.usage.input_tokens_details.cache_write_tokens == expected_split["cache_write_tokens"] + assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(5.722e-05) + + _NACE_SYSTEMONE_URL: Final = "https://console.nace.ai/v1/systemone" _DREX_MAX_RESPONSE: Final[Mapping[str, object]] = {**_RESPONSE, "model": "drex-max"} diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index c67d1012475..d668e904deb 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -1887,3 +1887,102 @@ def test_proxy_routes_ollama_decisions_with_optional_authentication( else [{"type": "predicate", "name": "is_defect", "probability": 0.9}] ) assert response.json()["answers"] == expected + + +def test_proxy_decisions_dispatches_vercel_ai_gateway_deployment( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + deployment_router: Final = litellm.Router( + model_list=[ + { + "model_name": "gateway-decider", + "litellm_params": {"model": "vercel_ai_gateway/openai/gpt-6-luna-decisions"}, + "model_info": {"mode": "evaluation"}, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", deployment_router) + upstream: Final = respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": "openai/gpt-6-luna-decisions", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": { + "input_tokens": _INPUT_TOKENS, + "output_tokens": _OUTPUT_TOKENS, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + }, + } + ) + + response: Final = client.post( + "/v1/decisions", + json={ + "model": "gateway-decider", + "input": "The package arrived broken.", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == [{"type": "predicate", "name": "is_defect", "probability": 0.9}] + assert upstream.called + assert str(upstream.calls.last.request.url) == "https://ai-gateway.vercel.sh/v1/decisions" + assert upstream.calls.last.request.headers["authorization"] == "Bearer gateway-key" + assert json.loads(upstream.calls.last.request.content) == { + "model": "openai/gpt-6-luna-decisions", + "input": "The package arrived broken.", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } + gateway_cost: Final = litellm.model_cost["vercel_ai_gateway/openai/gpt-6-luna-decisions"] + expected_cost: Final = _INPUT_TOKENS * float(gateway_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + gateway_cost["output_cost_per_token"] + ) + assert expected_cost > 0 + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) + + +@pytest.mark.parametrize("gateway_cost", ("0.00005722", "0"), ids=("paid_call", "free_call")) +def test_proxy_decisions_report_the_billed_gateway_charge_in_the_header_and_body( + gateway_cost: str, + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + monkeypatch.setattr(litellm, "cost_margin_config", {"vercel_ai_gateway": 0.5}) + deployment_router: Final = litellm.Router( + model_list=[ + { + "model_name": "gateway-decider", + "litellm_params": {"model": "vercel_ai_gateway/example/unpriced-decider"}, + "model_info": {"mode": "evaluation"}, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", deployment_router) + respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": "example/unpriced-decider", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + "provider_metadata": {"gateway": {"cost": gateway_cost}}, + } + ) + + response: Final = client.post( + "/v1/systemone", + json={ + "model": "gateway-decider", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + ) + + assert response.status_code == 200, response.text + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(response.json()["usage"]["cost"]) + assert response.json()["usage"]["cost"] == pytest.approx(float(gateway_cost))