mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(decisions): stop forwarding caller providerOptions to Vercel and send Jev images
Backport of #50642 to lts/1.108.x.
Cherry-picked from c8122bc6a5 (mishdotdev:vercel-ai-gateway-decisions).
Adapted to the line: decisions/call.py goes back to the line's own version, since the line never had the per-model support record main edits here, and the discount and margin test this commit touches on main is not on the line.
This commit is contained in:
parent
0222d0bdf4
commit
9bed368bfd
6 changed files with 27 additions and 76 deletions
|
|
@ -294,15 +294,12 @@ def _unsupported_params(
|
|||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
supports_images: bool,
|
||||
) -> tuple[str, ...]:
|
||||
refuses_safety_identifier: Final = (
|
||||
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
|
||||
)
|
||||
refused_image_param: Final = (
|
||||
None
|
||||
if supports_images and provider_config.supports_images_in(ir_request.input, headers)
|
||||
else image_param(ir_request.input)
|
||||
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
|
||||
)
|
||||
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
|
||||
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
|
||||
|
|
@ -335,13 +332,12 @@ def _without_unsupported_params(
|
|||
provider_config: BaseDecisionsConfig,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
supports_images: bool,
|
||||
) -> DecisionsIRRequest:
|
||||
return replace(
|
||||
ir_request,
|
||||
input=(
|
||||
ir_request.input
|
||||
if supports_images and provider_config.supports_images_in(ir_request.input, headers)
|
||||
if provider_config.supports_images_in(ir_request.input, headers)
|
||||
else with_images_where(ir_request.input, _no_image)
|
||||
),
|
||||
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
|
||||
|
|
@ -395,17 +391,14 @@ def _provider_ir_request(
|
|||
supports_box_questions: Final = provider_config.supports_box_questions(
|
||||
model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs)
|
||||
)
|
||||
supports_images: Final = provider_config.supports_image_input(model, custom_llm_provider)
|
||||
unsupported: Final = _unsupported_params(
|
||||
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions, supports_images
|
||||
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions
|
||||
)
|
||||
if not unsupported:
|
||||
return ir_request
|
||||
if not _drops_params(kwargs):
|
||||
return _UnsupportedParams(names=unsupported)
|
||||
supported: Final = _without_unsupported_params(
|
||||
ir_request, provider_config, headers, supports_box_questions, supports_images
|
||||
)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
|
||||
if _leaves_no_input(supported.input):
|
||||
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
|
||||
if not supported.questions:
|
||||
|
|
@ -690,7 +683,7 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]:
|
|||
model=call.requested_model,
|
||||
llm_provider=call.custom_llm_provider,
|
||||
)
|
||||
return {**body, **call.provider_config.provider_request_fields(call.litellm_params)}
|
||||
return body
|
||||
|
||||
|
||||
def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
||||
|
|
|
|||
|
|
@ -63,7 +63,6 @@ _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter
|
|||
_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"})
|
||||
_REMOTE_URL_PREFIXES: Final = ("http://", "https://")
|
||||
_DATA_URL_PREFIX: Final = "data:"
|
||||
_NO_PROVIDER_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def decisions_text(value: DecisionsJSON) -> str:
|
||||
|
|
@ -542,12 +541,6 @@ class BaseDecisionsConfig(ABC):
|
|||
) -> bool:
|
||||
return self.supports_images
|
||||
|
||||
def supports_image_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
return True
|
||||
|
||||
def provider_request_fields(self, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return _NO_PROVIDER_FIELDS
|
||||
|
||||
def supports_assets(self, headers: Mapping[str, str]) -> bool:
|
||||
return False
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeVar
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
|
|
@ -9,10 +8,8 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
|||
from litellm.llms.base_llm.decisions.transformation import usage_reported_cost
|
||||
from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig
|
||||
from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse, DecisionsIRUsage, OpenAIDecisionUsage
|
||||
from litellm.utils import is_vision_explicitly_disabled
|
||||
|
||||
_Model: Final = TypeVar("_Model", bound=BaseModel)
|
||||
_PROVIDER_OPTIONS: Final = "providerOptions"
|
||||
|
||||
|
||||
class _ProviderAttempt(BaseModel):
|
||||
|
|
@ -95,14 +92,6 @@ class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig):
|
|||
def resolve_api_key(self, api_key: str | None) -> str | None:
|
||||
return api_key
|
||||
|
||||
def supports_image_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
return not is_vision_explicitly_disabled(model, custom_llm_provider)
|
||||
|
||||
def provider_request_fields(self, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
if _PROVIDER_OPTIONS not in litellm_params:
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType({_PROVIDER_OPTIONS: litellm_params[_PROVIDER_OPTIONS]})
|
||||
|
||||
def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
|
||||
response: Final = super().parse_response(payload, request)
|
||||
metadata: Final = _validated(_GatewayDecisionResponse, payload)
|
||||
|
|
|
|||
|
|
@ -51269,13 +51269,17 @@
|
|||
"litellm_provider": "vercel_ai_gateway",
|
||||
"max_input_tokens": 32000,
|
||||
"metadata": {
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with an input_image answered 400 invalid request body on 2026-11-09."
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with a 768x768 input_image answered 200 on 2026-11-10, while a 96x96 one answered 400 invalid request body."
|
||||
},
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://ai-gateway.vercel.sh/v1/models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supports_box_questions": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": true
|
||||
},
|
||||
"vercel_ai_gateway/typesafe-ai/jev-1.13.0": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
|
|
|
|||
|
|
@ -51269,13 +51269,17 @@
|
|||
"litellm_provider": "vercel_ai_gateway",
|
||||
"max_input_tokens": 32000,
|
||||
"metadata": {
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with an input_image answered 400 invalid request body on 2026-11-09."
|
||||
"notes": "Vercel AI Gateway model list entry typesafe-ai/jev, read 2026-11-09: input 0.000000042 per token, output 0, context window 32000. A direct /v1/decisions probe with a 768x768 input_image answered 200 on 2026-11-10, while a 96x96 one answered 400 invalid request body."
|
||||
},
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://ai-gateway.vercel.sh/v1/models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supports_box_questions": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": true
|
||||
},
|
||||
"vercel_ai_gateway/typesafe-ai/jev-1.13.0": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
|
|
|
|||
|
|
@ -3493,49 +3493,18 @@ async def test_vercel_ai_gateway_decisions_price_an_unreadable_charge_from_the_c
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("vercel_gateway_key")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "drop_params", "sent_content"),
|
||||
(
|
||||
(
|
||||
"typesafe-ai/jev-1.14.0",
|
||||
False,
|
||||
[
|
||||
{"type": "input_text", "text": _SCREEN_TEXT},
|
||||
{"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "high"},
|
||||
],
|
||||
),
|
||||
("typesafe-ai/jev", True, [{"type": "input_text", "text": _SCREEN_TEXT}]),
|
||||
),
|
||||
ids=("model_reads_images", "text_only_model_drops_them"),
|
||||
)
|
||||
async def test_vercel_ai_gateway_decisions_send_images_only_to_models_that_read_them(
|
||||
model: str, drop_params: bool, sent_content: list[Mapping[str, object]], respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
@pytest.mark.parametrize("model", ("typesafe-ai/jev", "typesafe-ai/jev-1.13.0", "typesafe-ai/jev-1.14.0"))
|
||||
async def test_vercel_ai_gateway_jev_decisions_send_images(model: str, respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond(json=_vercel_response(model, {}))
|
||||
|
||||
await litellm.adecisions(
|
||||
model=f"vercel_ai_gateway/{model}",
|
||||
input=_screen_messages(_PNG_DATA_URL),
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
drop_params=drop_params,
|
||||
model=f"vercel_ai_gateway/{model}", input=_screen_messages(_PNG_DATA_URL), questions=_PREDICATE_QUESTIONS
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["input"][0]["content"] == sent_content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("vercel_gateway_key")
|
||||
async def test_vercel_ai_gateway_text_only_decisions_model_refuses_images_before_http(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="input_image"):
|
||||
await litellm.adecisions(
|
||||
model="vercel_ai_gateway/typesafe-ai/jev",
|
||||
input=_screen_messages(_PNG_DATA_URL),
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
assert json.loads(route.calls.last.request.content)["input"][0]["content"] == [
|
||||
{"type": "input_text", "text": _SCREEN_TEXT},
|
||||
{"type": "input_image", "image_url": _PNG_DATA_URL, "detail": "high"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -3560,8 +3529,7 @@ async def test_vercel_ai_gateway_decisions_inline_remote_images(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("vercel_gateway_key")
|
||||
async def test_vercel_ai_gateway_decisions_forward_provider_options(respx_mock: respx.MockRouter) -> None:
|
||||
provider_options: Final = {"gateway": {"order": ["openai"], "byok": {"openai": [{"apiKey": "caller-openai-key"}]}}}
|
||||
async def test_vercel_ai_gateway_decisions_never_send_caller_provider_options(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_VERCEL_DECISIONS_URL).respond(
|
||||
json=_vercel_response("openai/gpt-6-luna-decisions", {})
|
||||
)
|
||||
|
|
@ -3570,10 +3538,10 @@ async def test_vercel_ai_gateway_decisions_forward_provider_options(respx_mock:
|
|||
model="vercel_ai_gateway/openai/gpt-6-luna-decisions",
|
||||
input="The package arrived broken.",
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
providerOptions=provider_options,
|
||||
providerOptions={"gateway": {"models": ["openai/gpt-6"], "byok": {"openai": [{"apiKey": "caller-openai-key"}]}}},
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["providerOptions"] == provider_options
|
||||
assert "providerOptions" not in json.loads(route.calls.last.request.content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue