diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 307634b55d8..3e3b965173a 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -294,12 +294,15 @@ def _unsupported_params( custom_llm_provider: str, headers: Mapping[str, str], supports_box_questions: bool, + supports_images: bool, ) -> tuple[str, ...]: refuses_safety_identifier: Final = ( ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier ) refused_image_param: Final = ( - None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input) + None + if supports_images and provider_config.supports_images_in(ir_request.input, headers) + else image_param(ir_request.input) ) refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers) refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions @@ -332,12 +335,13 @@ def _without_unsupported_params( provider_config: BaseDecisionsConfig, headers: Mapping[str, str], supports_box_questions: bool, + supports_images: bool, ) -> DecisionsIRRequest: return replace( ir_request, input=( ir_request.input - if provider_config.supports_images_in(ir_request.input, headers) + if supports_images and provider_config.supports_images_in(ir_request.input, headers) else with_images_where(ir_request.input, _no_image) ), safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None, @@ -391,14 +395,17 @@ def _provider_ir_request( supports_box_questions: Final = provider_config.supports_box_questions( model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs) ) + supports_images: Final = provider_config.supports_image_input(model, custom_llm_provider) unsupported: Final = _unsupported_params( - ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions + ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions, supports_images ) if not unsupported: return ir_request if not _drops_params(kwargs): return _UnsupportedParams(names=unsupported) - supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions) + supported: Final = _without_unsupported_params( + ir_request, provider_config, headers, supports_box_questions, supports_images + ) if _leaves_no_input(supported.input): return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on") if not supported.questions: @@ -683,7 +690,7 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]: model=call.requested_model, llm_provider=call.custom_llm_provider, ) - return body + return {**body, **call.provider_config.provider_request_fields(call.litellm_params)} def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 0f33a42ab61..c025e66ccaf 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -63,6 +63,7 @@ _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) _REMOTE_URL_PREFIXES: Final = ("http://", "https://") _DATA_URL_PREFIX: Final = "data:" +_NO_PROVIDER_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) def decisions_text(value: DecisionsJSON) -> str: @@ -541,6 +542,12 @@ class BaseDecisionsConfig(ABC): ) -> bool: return self.supports_images + def supports_image_input(self, model: str, custom_llm_provider: str) -> bool: + return True + + def provider_request_fields(self, litellm_params: Mapping[str, object]) -> Mapping[str, object]: + return _NO_PROVIDER_FIELDS + def supports_assets(self, headers: Mapping[str, str]) -> bool: return False diff --git a/litellm/llms/vercel_ai_gateway/decisions/transformation.py b/litellm/llms/vercel_ai_gateway/decisions/transformation.py index 1152529d6ca..01d02294abf 100644 --- a/litellm/llms/vercel_ai_gateway/decisions/transformation.py +++ b/litellm/llms/vercel_ai_gateway/decisions/transformation.py @@ -1,50 +1,114 @@ +from collections.abc import Mapping from dataclasses import replace -from typing import Final +from itertools import chain +from types import MappingProxyType +from typing import Final, Literal, TypeVar -from pydantic import BaseModel, ConfigDict, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError from litellm.llms.base_llm.decisions.transformation import usage_reported_cost from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig -from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse +from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse, DecisionsIRUsage, OpenAIDecisionUsage +from litellm.utils import is_vision_explicitly_disabled + +_Model: Final = TypeVar("_Model", bound=BaseModel) +_PROVIDER_OPTIONS: Final = "providerOptions" + + +class _ProviderAttempt(BaseModel): + model_config = ConfigDict(frozen=True) + + credential_type: Literal["system", "byok"] | None = Field(default=None, alias="credentialType") + success: bool = False + + +class _ModelAttempt(BaseModel): + model_config = ConfigDict(frozen=True) + + provider_attempts: tuple[_ProviderAttempt, ...] = Field(default=(), alias="providerAttempts") + + +class _Routing(BaseModel): + model_config = ConfigDict(frozen=True) + + model_attempts: tuple[_ModelAttempt, ...] = Field(default=(), alias="modelAttempts") class _GatewayCharge(BaseModel): model_config = ConfigDict(frozen=True) - cost: float | None = None + cost: float + market_cost: float | None = Field(default=None, alias="marketCost") + routing: _Routing = _Routing() + + def served_with_own_key(self) -> bool: + attempts: Final = chain.from_iterable(model.provider_attempts for model in self.routing.model_attempts) + return any(attempt.success and attempt.credential_type == "byok" for attempt in attempts) + + def billed_cost(self) -> float: + if self.market_cost is None or not self.served_with_own_key(): + return self.cost + return self.cost + self.market_cost -class _GatewayProviderMetadata(BaseModel): +class _UpstreamReport(BaseModel): model_config = ConfigDict(frozen=True) - gateway: _GatewayCharge = _GatewayCharge() + usage: OpenAIDecisionUsage class _GatewayDecisionResponse(BaseModel): model_config = ConfigDict(frozen=True) - provider_metadata: _GatewayProviderMetadata = _GatewayProviderMetadata() + provider_metadata: Mapping[str, object] -def _gateway_charge(payload: object) -> float | None: +def _validated(model: type[_Model], value: object) -> _Model | None: try: - return _GatewayDecisionResponse.model_validate(payload).provider_metadata.gateway.cost + return model.model_validate(value) except ValidationError: return None +def _gateway_usage(usage: DecisionsIRUsage, metadata: Mapping[str, object]) -> DecisionsIRUsage: + charge: Final = _validated(_GatewayCharge, metadata.get("gateway")) + reports: Final = (_validated(_UpstreamReport, entry) for entry in metadata.values()) + upstream: Final = next( + (report.usage for report in reports if report is not None and report.usage.input_tokens == usage.input_tokens), + None, + ) + priced: Final = usage if charge is None else replace(usage, extra={**usage.extra, "cost": charge.billed_cost()}) + if upstream is None: + return priced + return replace( + priced, + cached_tokens=upstream.input_tokens_details.cached_tokens, + cache_write_tokens=upstream.input_tokens_details.cache_write_tokens, + ) + + class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig): """get_llm_provider already resolves the gateway's base and key, so never fall back to OpenAI credentials.""" + inlines_remote_images = True + def resolve_api_key(self, api_key: str | None) -> str | None: return api_key + def supports_image_input(self, model: str, custom_llm_provider: str) -> bool: + return not is_vision_explicitly_disabled(model, custom_llm_provider) + + def provider_request_fields(self, litellm_params: Mapping[str, object]) -> Mapping[str, object]: + if _PROVIDER_OPTIONS not in litellm_params: + return MappingProxyType({}) + return MappingProxyType({_PROVIDER_OPTIONS: litellm_params[_PROVIDER_OPTIONS]}) + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: response: Final = super().parse_response(payload, request) - charge: Final = _gateway_charge(payload) - if charge is None: + metadata: Final = _validated(_GatewayDecisionResponse, payload) + if metadata is None: return response - return replace(response, usage=replace(response.usage, extra={**response.usage.extra, "cost": charge})) + return replace(response, usage=_gateway_usage(response.usage, metadata.provider_metadata)) def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: return usage_reported_cost(response) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 59f5d5a213a..91788da2de6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -51269,12 +51269,13 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "metadata": { - "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000." + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with an input_image answered 400 invalid request body on 2026-11-09." }, "mode": "evaluation", "output_cost_per_token": 0.0, "source": "https://ai-gateway.vercel.sh/v1/models", - "supports_box_questions": true + "supports_box_questions": true, + "supports_vision": false }, "vercel_ai_gateway/typesafe-ai/jev-1.13.0": { "input_cost_per_token": 4.2e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 59f5d5a213a..91788da2de6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -51269,12 +51269,13 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "metadata": { - "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000." + "notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with an input_image answered 400 invalid request body on 2026-11-09." }, "mode": "evaluation", "output_cost_per_token": 0.0, "source": "https://ai-gateway.vercel.sh/v1/models", - "supports_box_questions": true + "supports_box_questions": true, + "supports_vision": false }, "vercel_ai_gateway/typesafe-ai/jev-1.13.0": { "input_cost_per_token": 4.2e-08, diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index f8662a8bc59..463192ec2fa 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3399,42 +3399,181 @@ async def test_vercel_ai_gateway_decisions_never_fall_back_to_the_openai_key( assert len(respx_mock.calls) == 0 -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("model", "gateway_cost", "reported_cost"), - ( - ("example/unpriced-decider", "0.000001386", 1.386e-06), - ("openai/gpt-6-luna-decisions", "0.00005722", 5.722e-05), - ("openai/gpt-6-luna-decisions", "not-a-number", None), - ), - ids=("unpriced_model", "cache_hit_on_priced_model", "unreadable_charge"), -) -async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports( - model: str, - gateway_cost: str, - reported_cost: float | None, - monkeypatch: pytest.MonkeyPatch, - respx_mock: respx.MockRouter, -) -> None: +_VERCEL_DECISIONS_URL: Final = "https://ai-gateway.vercel.sh/v1/decisions" + + +def _gateway_routing(*attempts: tuple[str, bool]) -> Mapping[str, object]: + return { + "modelAttempts": [ + { + "providerAttempts": [ + {"provider": "openai", "credentialType": credential_type, "success": success} + for credential_type, success in attempts + ] + } + ] + } + + +def _vercel_response(model: str, provider_metadata: Mapping[str, object]) -> Mapping[str, object]: + return { + "model": model, + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + "provider_metadata": provider_metadata, + } + + +@pytest.fixture +def vercel_gateway_key(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") - respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( - json={ - "model": model, - "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], - "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, - "provider_metadata": {"gateway": {"cost": gateway_cost}}, - } - ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +@pytest.mark.parametrize( + ("model", "gateway", "billed"), + ( + ("example/unpriced-decider", {"cost": "0.000001386"}, 1.386e-06), + ("openai/gpt-6-luna-decisions", {"cost": "0.00005722"}, 5.722e-05), + ( + "openai/gpt-6-luna-decisions", + {"cost": "0", "marketCost": "0.0000164", "routing": _gateway_routing(("byok", True))}, + 1.64e-05, + ), + ( + "openai/gpt-6-luna-decisions", + { + "cost": "0.0000164", + "marketCost": "0.0000164", + "routing": _gateway_routing(("byok", False), ("system", True)), + }, + 1.64e-05, + ), + ), + ids=("unpriced_model", "cache_hit_on_priced_model", "own_provider_key", "own_key_failed_over_to_gateway_credits"), +) +async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports( + model: str, gateway: Mapping[str, object], billed: float, respx_mock: respx.MockRouter +) -> None: + respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {"gateway": gateway})) response: Final = await litellm.adecisions( model=f"vercel_ai_gateway/{model}", input="The package arrived broken.", - questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + questions=_PREDICATE_QUESTIONS, ) assert response.answers[0].probability == 0.9 - assert get_response_cost_from_hidden_params(response.hidden_params) == reported_cost + assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(billed) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_price_an_unreadable_charge_from_the_cost_map( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {"gateway": {"cost": "not-a-number"}}) + ) + + response: Final = await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + ) + + luna: Final = litellm.model_cost["vercel_ai_gateway/openai/gpt-6-luna-decisions"] + assert get_response_cost_from_hidden_params(response.hidden_params) is None + assert litellm.completion_cost(completion_response=response) == pytest.approx( + _INPUT_TOKENS * float(luna["input_cost_per_token"]) + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +@pytest.mark.parametrize( + ("model", "drop_params", "sent_content"), + ( + ( + "typesafe-ai/jev-1.14.0", + False, + [ + {"type": "input_text", "text": _SCREEN_TEXT}, + {"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "high"}, + ], + ), + ("typesafe-ai/jev", True, [{"type": "input_text", "text": _SCREEN_TEXT}]), + ), + ids=("model_reads_images", "text_only_model_drops_them"), +) +async def test_vercel_ai_gateway_decisions_send_images_only_to_models_that_read_them( + model: str, drop_params: bool, sent_content: list[Mapping[str, object]], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {})) + + await litellm.adecisions( + model=f"vercel_ai_gateway/{model}", + input=_screen_messages(_PNG_DATA_URL), + questions=_PREDICATE_QUESTIONS, + drop_params=drop_params, + ) + + assert json.loads(route.calls.last.request.content)["input"][0]["content"] == sent_content + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_text_only_decisions_model_refuses_images_before_http( + respx_mock: respx.MockRouter, +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="input_image"): + await litellm.adecisions( + model="vercel_ai_gateway/typesafe-ai/jev", + input=_screen_messages(_PNG_DATA_URL), + questions=_PREDICATE_QUESTIONS, + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_inline_remote_images( + respx_mock: respx.MockRouter, async_only_image_fetch +) -> None: + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {}) + ) + + await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input=_screen_messages(_REMOTE_IMAGE_URL), + questions=_PREDICATE_QUESTIONS, + ) + + assert async_only_image_fetch.fetched == [_REMOTE_IMAGE_URL] + sent_image: Final = json.loads(route.calls.last.request.content)["input"][0]["content"][1] + assert sent_image["image_url"] == async_only_image_fetch.data_url + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("vercel_gateway_key") +async def test_vercel_ai_gateway_decisions_forward_provider_options(respx_mock: respx.MockRouter) -> None: + provider_options: Final = {"gateway": {"order": ["openai"], "byok": {"openai": [{"apiKey": "caller-openai-key"}]}}} + route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond( + json=_vercel_response("openai/gpt-6-luna-decisions", {}) + ) + + await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=_PREDICATE_QUESTIONS, + providerOptions=provider_options, + ) + + assert json.loads(route.calls.last.request.content)["providerOptions"] == provider_options @pytest.mark.asyncio @@ -3466,3 +3605,48 @@ async def test_vercel_ai_gateway_jev_sends_box_questions( assert json.loads(route.calls.last.request.content)["questions"] == [box_question] assert response.answers[0].confidence == 0.8 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("upstream_input_tokens", "cached_tokens", "cache_write_tokens"), + ((_INPUT_TOKENS, _INPUT_TOKENS - 20, 20), (_INPUT_TOKENS + 1, 0, 0)), + ids=("upstream_reports_this_call", "upstream_count_disagrees"), +) +async def test_vercel_ai_gateway_decisions_report_the_upstream_cache_split( + upstream_input_tokens: int, + cached_tokens: int, + cache_write_tokens: int, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False) + monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key") + respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond( + json={ + "model": "openai/gpt-6-luna-decisions", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS}, + "provider_metadata": { + "openai": { + "usage": { + "input_tokens": upstream_input_tokens, + "input_tokens_details": {"cached_tokens": _INPUT_TOKENS - 20, "cache_write_tokens": 20}, + "output_tokens": 0, + "total_tokens": upstream_input_tokens, + } + }, + "gateway": {"cost": "0.00005722"}, + }, + } + ) + + response: Final = await litellm.adecisions( + model="vercel_ai_gateway/openai/gpt-6-luna-decisions", + input="The package arrived broken.", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + ) + + assert response.usage.input_tokens_details.cached_tokens == cached_tokens + assert response.usage.input_tokens_details.cache_write_tokens == cache_write_tokens + assert get_response_cost_from_hidden_params(response.hidden_params) == pytest.approx(5.722e-05)