diff --git a/litellm/__init__.py b/litellm/__init__.py index 98c36dc582a..b556eb2e307 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1797,6 +1797,9 @@ if TYPE_CHECKING: from .llms.azure_ai.decisions.transformation import ( AzureAIDecisionsConfig as AzureAIDecisionsConfig, ) + from .llms.vercel_ai_gateway.decisions.transformation import ( + VercelAIGatewayDecisionsConfig as VercelAIGatewayDecisionsConfig, + ) from .llms.nvidia_nim.rerank.transformation import ( NvidiaNimRerankConfig as NvidiaNimRerankConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 6d3c18042a6..d660f8f4ac9 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -135,6 +135,10 @@ LITELLM_ATTRIBUTES: Final[Mapping[str, tuple[str, str]]] = MappingProxyType( "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"), "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), + "VercelAIGatewayDecisionsConfig": ( + ".llms.vercel_ai_gateway.decisions.transformation", + "VercelAIGatewayDecisionsConfig", + ), "NvidiaNimRerankConfig": (".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig"), "NvidiaNimRankingConfig": (".llms.nvidia_nim.rerank.ranking_transformation", "NvidiaNimRankingConfig"), "VertexAIRerankConfig": (".llms.vertex_ai.rerank.transformation", "VertexAIRerankConfig"), diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index b84d5ce0ea1..56bc9f444db 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -520,6 +520,13 @@ def _is_decisions_model(model: str, custom_llm_provider: str) -> bool: return is_decisions_model_mode(model_mode) +def usage_reported_cost(response: DecisionsIRResponse) -> float | None: + cost: Final = response.usage.extra.get("cost") if response.usage is not None else None + if not isinstance(cost, (int, float)): + return None + return float(cost) + + @dataclass(frozen=True, slots=True) class DecisionsInputLimits: max_request_tokens: int | None = None diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 30d33f09efd..e84a6162713 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,6 +1,4 @@ -from typing import Final - -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, usage_reported_cost from litellm.types.decisions import DecisionsIRResponse @@ -13,7 +11,4 @@ class OpenRouterDecisionsConfig(BaseDecisionsConfig): return "https://openrouter.ai/api" def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: - cost: Final = response.usage.extra.get("cost") if response.usage is not None else None - if not isinstance(cost, (int, float)): - return None - return float(cost) + return usage_reported_cost(response) diff --git a/litellm/llms/vercel_ai_gateway/decisions/transformation.py b/litellm/llms/vercel_ai_gateway/decisions/transformation.py new file mode 100644 index 00000000000..af29267b17a --- /dev/null +++ b/litellm/llms/vercel_ai_gateway/decisions/transformation.py @@ -0,0 +1,138 @@ +from collections.abc import Mapping +from dataclasses import replace +from itertools import chain +from typing import Final, TypeVar + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from litellm.llms.base_llm.decisions.transformation import usage_reported_cost +from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig +from litellm.types.decisions import ( + DecisionsIRRequest, + DecisionsIRResponse, + DecisionsIRUsage, + OpenAIDecisionInputTokensDetails, +) + +_Model: Final = TypeVar("_Model", bound=BaseModel) + + +class _ProviderAttempt(BaseModel): + model_config = ConfigDict(frozen=True) + + provider: str | None = None + credential_type: str | None = Field(default=None, alias="credentialType") + success: bool = False + + +class _ModelAttempt(BaseModel): + model_config = ConfigDict(frozen=True) + + provider_attempts: tuple[_ProviderAttempt, ...] = Field(default=(), alias="providerAttempts") + + +class _Routing(BaseModel): + model_config = ConfigDict(frozen=True) + + model_attempts: tuple[_ModelAttempt, ...] = Field(default=(), alias="modelAttempts") + + def served_by(self) -> _ProviderAttempt | None: + attempts: Final = chain.from_iterable(model.provider_attempts for model in self.model_attempts) + return next((attempt for attempt in attempts if attempt.success), None) + + +class _GatewayRouting(BaseModel): + model_config = ConfigDict(frozen=True) + + routing: _Routing = _Routing() + + +class _GatewayCharge(BaseModel): + model_config = ConfigDict(frozen=True) + + cost: float + market_cost: float | None = Field(default=None, alias="marketCost") + + def billed_cost(self, served_by: _ProviderAttempt | None) -> float: + if self.market_cost is None or served_by is None or served_by.credential_type != "byok": + return self.cost + return self.cost + self.market_cost + + +class _UpstreamUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + input_tokens: int + input_tokens_details: OpenAIDecisionInputTokensDetails + + +class _UpstreamReport(BaseModel): + model_config = ConfigDict(frozen=True) + + usage: _UpstreamUsage + + +class _GatewayDecisionResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + provider_metadata: Mapping[str, object] + + +def _validated(model: type[_Model], value: object) -> _Model | None: + try: + return model.model_validate(value) + except ValidationError: + return None + + +def _served_by(gateway: object) -> _ProviderAttempt | None: + gateway_routing: Final = _validated(_GatewayRouting, gateway) + return None if gateway_routing is None else gateway_routing.routing.served_by() + + +def _upstream_usage(metadata: Mapping[str, object], served_by: _ProviderAttempt | None) -> _UpstreamUsage | None: + if served_by is None or served_by.provider is None: + return None + report: Final = _validated(_UpstreamReport, metadata.get(served_by.provider)) + return None if report is None else report.usage + + +def _gateway_usage(usage: DecisionsIRUsage, metadata: Mapping[str, object]) -> DecisionsIRUsage: + gateway: Final = metadata.get("gateway") + served_by: Final = _served_by(gateway) + charge: Final = _validated(_GatewayCharge, gateway) + priced: Final = ( + usage if charge is None else replace(usage, extra={**usage.extra, "cost": charge.billed_cost(served_by)}) + ) + upstream: Final = _upstream_usage(metadata, served_by) + if upstream is None or upstream.input_tokens != usage.input_tokens: + return priced + return replace( + priced, + cached_tokens=upstream.input_tokens_details.cached_tokens, + cache_write_tokens=upstream.input_tokens_details.cache_write_tokens, + ) + + +class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig): + """get_llm_provider already resolves the gateway's base and key, so never fall back to OpenAI credentials.""" + + inlines_remote_images = True + + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key + + def resolve_credential( + self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object] + ) -> str | None: + return api_key + + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + response: Final = super().parse_response(payload, request) + metadata: Final = _validated(_GatewayDecisionResponse, payload) + if metadata is None or response.usage is None: + return response + return replace(response, usage=_gateway_usage(response.usage, metadata.provider_metadata)) + + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: + return usage_reported_cost(response) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0e352ded72a..544815aaa0b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -52612,6 +52612,19 @@ "supports_tool_choice": true, "supports_response_schema": true }, + "vercel_ai_gateway/openai/gpt-6-luna-decisions": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 1050000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models" + }, "vercel_ai_gateway/openai/o1": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 7.5e-06, @@ -52723,6 +52736,56 @@ "mode": "chat", "output_cost_per_token": 8e-06 }, + "vercel_ai_gateway/typesafe-ai/jev": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with a 768x768 input_image answered 200 on 2026-11-10, while a 96x96 one answered 400 invalid request body." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_box_questions": true, + "supports_vision": true + }, + "vercel_ai_gateway/typesafe-ai/jev-1.13.0": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.13.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.99 for a gradient and 0.12 for solid gray." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "vercel_ai_gateway/typesafe-ai/jev-1.14.0": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.14.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.84 for a gradient and 0.13 for solid gray." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_box_questions": true, + "supports_vision": true + }, "vercel_ai_gateway/vercel/v0-1.0-md": { "input_cost_per_token": 3e-06, "litellm_provider": "vercel_ai_gateway", diff --git a/litellm/utils.py b/litellm/utils.py index e84f933abec..eae9addc8e0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10395,6 +10395,7 @@ class ProviderConfigManager: LlmProviders.AZURE_AI, LlmProviders.EU_JEV, LlmProviders.PARALLEL_AI, + LlmProviders.VERCEL_AI_GATEWAY, ): configs: Final = { LlmProviders.EMPIRIOLABS: litellm.EmpirioLabsDecisionsConfig, @@ -10403,6 +10404,7 @@ class ProviderConfigManager: LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig, LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig, LlmProviders.PARALLEL_AI: litellm.ParallelAIDecisionsConfig, + LlmProviders.VERCEL_AI_GATEWAY: litellm.VercelAIGatewayDecisionsConfig, } return configs[provider]() return None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0e352ded72a..544815aaa0b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -52612,6 +52612,19 @@ "supports_tool_choice": true, "supports_response_schema": true }, + "vercel_ai_gateway/openai/gpt-6-luna-decisions": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 1050000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models" + }, "vercel_ai_gateway/openai/o1": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 7.5e-06, @@ -52723,6 +52736,56 @@ "mode": "chat", "output_cost_per_token": 8e-06 }, + "vercel_ai_gateway/typesafe-ai/jev": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with a 768x768 input_image answered 200 on 2026-11-10, while a 96x96 one answered 400 invalid request body." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_box_questions": true, + "supports_vision": true + }, + "vercel_ai_gateway/typesafe-ai/jev-1.13.0": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.13.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.99 for a gradient and 0.12 for solid gray." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "vercel_ai_gateway/typesafe-ai/jev-1.14.0": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 32000, + "metadata": { + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev-1.14.0, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. Direct /v1/decisions probes on 2026-11-10 asking whether a 768x768 picture is a smooth gradient answered 0.84 for a gradient and 0.13 for solid gray." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ai-gateway.vercel.sh/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supports_box_questions": true, + "supports_vision": true + }, "vercel_ai_gateway/vercel/v0-1.0-md": { "input_cost_per_token": 3e-06, "litellm_provider": "vercel_ai_gateway", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 8b470248be7..76add4d6fdd 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -4924,6 +4924,307 @@ def test_hosted_vllm_systemone_maps_image_rejection_to_bad_request( assert route.call_count == 1 +@pytest.mark.asyncio +async def test_vercel_ai_gateway_decisions_send_the_gateway_key_to_the_gateway_and_price_the_call( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") + monkeypatch.setenv("OPENAI_BASE_URL", "https://openai.example") + route: Final = respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": "openai/gpt-6-luna-decisions", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + } + ) + + response: Final = await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + ) + + assert route.calls.last.request.headers["authorization"] == "Bearer gateway-key" + assert json.loads(route.calls.last.request.content) == { + "model": "openai/gpt-6-luna-decisions", + "input": "The package arrived broken.", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } + assert response.answers[0].probability == 0.9 + input_rate: Final = litellm.model_cost["vercel_ai_gateway/openai/gpt-6-luna-decisions"]["input_cost_per_token"] + assert input_rate > 0 + assert litellm.completion_cost(completion_response=response) == pytest.approx(_INPUT_TOKENS * input_rate) + + +@pytest.mark.asyncio +async def test_vercel_ai_gateway_decisions_never_fall_back_to_the_openai_key( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_KEY", raising=False) + monkeypatch.delenv("VERCEL_OIDC_TOKEN", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") + + with pytest.raises(litellm.AuthenticationError): + await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="review", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + ) + + assert len(respx_mock.calls) == 0 + + +_VERCEL_DECISIONS_URL: Final = "https://ai-gateway.vercel.sh/v1/decisions" + + +def _gateway_routing(*attempts: tuple[str, bool]) -> Mapping[str, object]: + return { + "modelAttempts": [ + { + "providerAttempts": [ + {"provider": "openai", "credentialType": credential_type, "success": success} + for credential_type, success in attempts + ] + } + ] + } + + +def _vercel_response(model: str, provider_metadata: Mapping[str, object]) -> Mapping[str, object]: + return { + "model": model, + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + "provider_metadata": provider_metadata, + } + + +@pytest.fixture +def vercel_gateway_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +@pytest.mark.parametrize( + ("model", "gateway", "billed"), + ( + ("example/unpriced-decider", {"cost": "0.000001386"}, 1.386e-06), + ("openai/gpt-6-luna-decisions", {"cost": "0.00005722"}, 5.722e-05), + ( + "openai/gpt-6-luna-decisions", + {"cost": "0", "marketCost": "0.0000164", "routing": _gateway_routing(("byok", True))}, + 1.64e-05, + ), + ( + "openai/gpt-6-luna-decisions", + { + "cost": "0.0000164", + "marketCost": "0.0000164", + "routing": _gateway_routing(("byok", False), ("system", True)), + }, + 1.64e-05, + ), + ), + ids=("unpriced_model", "cache_hit_on_priced_model", "own_provider_key", "own_key_failed_over_to_gateway_credits"), +) +async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports( + model: str, gateway: Mapping[str, object], billed: float, respx_mock: respx.MockRouter +) -> None: + respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {"gateway": gateway})) + + response: Final = await litellm.adecisions( + model=f"vercel_ai_gateway/{model}", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + ) + + assert response.answers[0].probability == 0.9 + assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(billed) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_price_an_unreadable_charge_from_the_cost_map( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {"gateway": {"cost": "not-a-number"}}) + ) + + response: Final = await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + ) + + luna: Final = litellm.model_cost["vercel_ai_gateway/openai/gpt-6-luna-decisions"] + assert litellm.completion_cost(completion_response=response) == pytest.approx( + _INPUT_TOKENS * float(luna["input_cost_per_token"]) + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +@pytest.mark.parametrize("model", ("typesafe-ai/jev", "typesafe-ai/jev-1.13.0", "typesafe-ai/jev-1.14.0")) +async def test_vercel_ai_gateway_jev_decisions_send_images(model: str, respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {})) + + await litellm.adecisions( + model=f"vercel_ai_gateway/{model}", input=_screen_messages(_PNG_DATA_URL), questions=_PREDICATE_QUESTIONS + ) + + assert json.loads(route.calls.last.request.content)["input"][0]["content"] == [ + {"type": "input_text", "text": _SCREEN_TEXT}, + {"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "high"}, + ] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_inline_remote_images( + respx_mock: respx.MockRouter, async_only_image_fetch: AsyncOnlyImageFetch +) -> None: + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {}) + ) + + await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input=_screen_messages(_REMOTE_IMAGE_URL), + questions=_PREDICATE_QUESTIONS, + ) + + assert async_only_image_fetch.fetched == [_REMOTE_IMAGE_URL] + sent_image: Final = json.loads(route.calls.last.request.content)["input"][0]["content"][1] + assert sent_image["image_url"] == async_only_image_fetch.data_url + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_never_send_caller_provider_options(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {}) + ) + + await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + providerOptions={"gateway": {"models": ["openai/gpt-6"], "byok": {"openai": [{"apiKey": "caller-openai-key"}]}}}, + ) + + assert "providerOptions" not in json.loads(route.calls.last.request.content) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("typesafe-ai/jev", "typesafe-ai/jev-1.14.0")) +async def test_vercel_ai_gateway_jev_sends_box_questions( + model: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + box_question: Final = {"type": "box", "name": "crack", "instructions": "the crack on the screen"} + route: Final = respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": model, + "answers": [ + { + "type": "box", + "name": "crack", + "box": {"top": 0.1, "bottom": 0.9, "left": 0.1, "right": 0.9}, + "confidence": 0.8, + } + ], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + } + ) + + response: Final = await litellm.adecisions( + model=f"vercel_ai_gateway/{model}", input="The screen arrived cracked.", questions=[box_question] + ) + + assert json.loads(route.calls.last.request.content)["questions"] == [box_question] + assert response.answers[0].confidence == 0.8 + + +_UPSTREAM_SPLIT: Final[Mapping[str, int]] = {"cached_tokens": _INPUT_TOKENS - 20, "cache_write_tokens": 20} +_TOP_LEVEL_SPLIT: Final[Mapping[str, int]] = {"cached_tokens": 300, "cache_write_tokens": 7} + + +def _upstream_usage(input_tokens: int, details: Mapping[str, int] | None) -> Mapping[str, object]: + return { + "usage": { + "input_tokens": input_tokens, + **({} if details is None else {"input_tokens_details": details}), + "output_tokens": 0, + "total_tokens": input_tokens, + } + } + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +@pytest.mark.parametrize( + ("top_level_split", "served_by", "upstream", "expected_split"), + ( + (None, "openai", _upstream_usage(_INPUT_TOKENS, _UPSTREAM_SPLIT), _UPSTREAM_SPLIT), + (None, "openai", _upstream_usage(_INPUT_TOKENS + 1, _UPSTREAM_SPLIT), {"cached_tokens": 0, "cache_write_tokens": 0}), + (_TOP_LEVEL_SPLIT, "openai", _upstream_usage(_INPUT_TOKENS, None), _TOP_LEVEL_SPLIT), + (None, "anthropic", _upstream_usage(_INPUT_TOKENS, _UPSTREAM_SPLIT), {"cached_tokens": 0, "cache_write_tokens": 0}), + ), + ids=( + "serving_provider_reports_this_call", + "upstream_count_disagrees", + "upstream_omits_the_split", + "report_from_a_provider_that_did_not_serve", + ), +) +async def test_vercel_ai_gateway_decisions_report_the_upstream_cache_split( + top_level_split: Mapping[str, int] | None, + served_by: str, + upstream: Mapping[str, object], + expected_split: Mapping[str, int], + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json={ + "model": "openai/gpt-6-luna-decisions", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": { + "input_tokens": _INPUT_TOKENS, + **({} if top_level_split is None else {"input_tokens_details": top_level_split}), + "output_tokens": 0, + "total_tokens": _INPUT_TOKENS, + }, + "provider_metadata": { + "openai": upstream, + "gateway": { + "cost": "0.00005722", + "routing": { + "modelAttempts": [ + {"providerAttempts": [{"provider": served_by, "credentialType": "system", "success": True}]} + ] + }, + }, + }, + } + ) + + response: Final = await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + ) + + assert response.usage.input_tokens_details.cached_tokens == expected_split["cached_tokens"] + assert response.usage.input_tokens_details.cache_write_tokens == expected_split["cache_write_tokens"] + assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(5.722e-05) + + _NACE_SYSTEMONE_URL: Final = "https://console.nace.ai/v1/systemone" _DREX_MAX_RESPONSE: Final[Mapping[str, object]] = {**_RESPONSE, "model": "drex-max"} diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index c67d1012475..d668e904deb 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -1887,3 +1887,102 @@ def test_proxy_routes_ollama_decisions_with_optional_authentication( else [{"type": "predicate", "name": "is_defect", "probability": 0.9}] ) assert response.json()["answers"] == expected + + +def test_proxy_decisions_dispatches_vercel_ai_gateway_deployment( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + deployment_router: Final = litellm.Router( + model_list=[ + { + "model_name": "gateway-decider", + "litellm_params": {"model": "vercel_ai_gateway/openai/gpt-6-luna-decisions"}, + "model_info": {"mode": "evaluation"}, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", deployment_router) + upstream: Final = respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": "openai/gpt-6-luna-decisions", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": { + "input_tokens": _INPUT_TOKENS, + "output_tokens": _OUTPUT_TOKENS, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + }, + } + ) + + response: Final = client.post( + "/v1/decisions", + json={ + "model": "gateway-decider", + "input": "The package arrived broken.", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == [{"type": "predicate", "name": "is_defect", "probability": 0.9}] + assert upstream.called + assert str(upstream.calls.last.request.url) == "https://ai-gateway.vercel.sh/v1/decisions" + assert upstream.calls.last.request.headers["authorization"] == "Bearer gateway-key" + assert json.loads(upstream.calls.last.request.content) == { + "model": "openai/gpt-6-luna-decisions", + "input": "The package arrived broken.", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } + gateway_cost: Final = litellm.model_cost["vercel_ai_gateway/openai/gpt-6-luna-decisions"] + expected_cost: Final = _INPUT_TOKENS * float(gateway_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + gateway_cost["output_cost_per_token"] + ) + assert expected_cost > 0 + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) + + +@pytest.mark.parametrize("gateway_cost", ("0.00005722", "0"), ids=("paid_call", "free_call")) +def test_proxy_decisions_report_the_billed_gateway_charge_in_the_header_and_body( + gateway_cost: str, + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + monkeypatch.setattr(litellm, "cost_margin_config", {"vercel_ai_gateway": 0.5}) + deployment_router: Final = litellm.Router( + model_list=[ + { + "model_name": "gateway-decider", + "litellm_params": {"model": "vercel_ai_gateway/example/unpriced-decider"}, + "model_info": {"mode": "evaluation"}, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", deployment_router) + respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": "example/unpriced-decider", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + "provider_metadata": {"gateway": {"cost": gateway_cost}}, + } + ) + + response: Final = client.post( + "/v1/systemone", + json={ + "model": "gateway-decider", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + ) + + assert response.status_code == 200, response.text + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(response.json()["usage"]["cost"]) + assert response.json()["usage"]["cost"] == pytest.approx(float(gateway_cost))