From 2137d34545d2723fdea9a92f6fed9ab7d94bf55f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:10:36 -0700 Subject: [PATCH] fix(decisions): send images only to OpenRouter upstreams that take them, answer Jev 1.14 box questions OpenRouter decision models other than the TypeSafe, Cloudflare and Perplexity upstreams are text-only, so images to them go back to main's behavior: a 400 before the call without drop_params, dropped under drop_params. Add an openrouter/typesafe/jev-1.14 capability row so box questions reach Jev 1.14 through OpenRouter, and log the body that was actually sent when an image drops after remote images were inlined. --- litellm/decisions/call.py | 21 +++-- .../llms/base_llm/decisions/transformation.py | 2 +- .../openrouter/decisions/transformation.py | 15 +++- .../llms/typesafe/decisions/transformation.py | 2 +- ...odel_prices_and_context_window_backup.json | 8 ++ model_prices_and_context_window.json | 8 ++ tests/unit/decisions/test_main.py | 89 +++++++++++++++++-- 7 files changed, 129 insertions(+), 16 deletions(-) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index feebd664246..bcadff6bd8b 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -302,7 +302,7 @@ def _unsupported_params( ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier ) refused_image_param: Final = ( - None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input) + None if provider_config.supports_images_in(model, ir_request.input, headers) else image_param(ir_request.input) ) refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers) refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions @@ -333,6 +333,7 @@ def _is_not_box_question(question: DecisionsIRQuestion) -> bool: def _without_unsupported_params( ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, + model: str, headers: Mapping[str, str], supports_box_questions: bool, ) -> DecisionsIRRequest: @@ -340,7 +341,7 @@ def _without_unsupported_params( ir_request, input=( ir_request.input - if provider_config.supports_images_in(ir_request.input, headers) + if provider_config.supports_images_in(model, ir_request.input, headers) else with_images_where(ir_request.input, _no_image) ), safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None, @@ -401,7 +402,7 @@ def _provider_ir_request( return ir_request if not _drops_params(kwargs): return _UnsupportedParams(names=unsupported) - supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions) + supported: Final = _without_unsupported_params(ir_request, provider_config, model, headers, supports_box_questions) if _leaves_no_input(supported.input): return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on") if not supported.questions: @@ -716,6 +717,16 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]: return body +def _image_count(call: DecisionsCall) -> int: + return len(input_images(call.ir_request.input)) + + +def _logged_body(supported: DecisionsCall, inlined: DecisionsCall, body: Mapping[str, object]) -> Mapping[str, object]: + if inlined is supported or _image_count(inlined) < _image_count(supported): + return body + return _request_body(supported) + + def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str: returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}") if returned is not None and cost_map_key_prices_anything(returned): @@ -789,7 +800,7 @@ async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionRespon provider_config=inlined.provider_config, request=inlined.ir_request, body=body, - logged_body=body if inlined is supported else _request_body(supported), + logged_body=_logged_body(supported, inlined, body), api_base=inlined.api_base, api_key=inlined.api_key, headers=inlined.headers, @@ -813,7 +824,7 @@ def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: provider_config=inlined.provider_config, request=inlined.ir_request, body=body, - logged_body=body if inlined is supported else _request_body(supported), + logged_body=_logged_body(supported, inlined, body), api_base=inlined.api_base, api_key=inlined.api_key, headers=inlined.headers, diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 7edc96bf525..e5f63ac4f5a 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -666,7 +666,7 @@ class BaseDecisionsConfig(ABC): return headers, None def supports_images_in( - self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] ) -> bool: return self.supports_images diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 49d42036cd7..86573580181 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -6,7 +6,7 @@ from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.cloudflare.decisions.transformation import CloudflareDecisionsConfig from litellm.llms.perplexity.decisions.transformation import PerplexityDecisionsConfig from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig -from litellm.types.decisions import DecisionsIRResponse +from litellm.types.decisions import DecisionsIRMessages, DecisionsIRResponse, DecisionsIRState _ORG_IMAGE_RULES: Final[Mapping[str, BaseDecisionsConfig]] = MappingProxyType( { @@ -17,20 +17,27 @@ _ORG_IMAGE_RULES: Final[Mapping[str, BaseDecisionsConfig]] = MappingProxyType( ) +def _upstream_org(model: str) -> str: + return model.removeprefix("~").partition("/")[0] + + class OpenRouterDecisionsConfig(BaseDecisionsConfig): path = "/alpha/decisions" api_key_env = ("OPENROUTER_API_KEY",) api_base_env = ("OPENROUTER_API_BASE",) - supports_images = True - image_media_types = frozenset({"image/png", "image/jpeg", "image/jpg", "image/webp"}) inlines_remote_images = True images_in_state = True def get_default_api_base(self) -> str | None: return "https://openrouter.ai/api" + def supports_images_in( + self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + ) -> bool: + return _upstream_org(model) in _ORG_IMAGE_RULES + def image_rules_for(self, model: str) -> BaseDecisionsConfig: - return _ORG_IMAGE_RULES.get(model.removeprefix("~").partition("/")[0], self) + return _ORG_IMAGE_RULES.get(_upstream_org(model), self) def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: cost: Final = response.usage.extra.get("cost") if response.usage is not None else None diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py index f77474d2872..3ba6504c576 100644 --- a/litellm/llms/typesafe/decisions/transformation.py +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -84,7 +84,7 @@ class TypeSafeDecisionsConfig(BaseDecisionsConfig): return "https://api.typesafe.ai" def supports_images_in( - self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] + self, model: str, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] ) -> bool: return isinstance(decision_input, DecisionsIRMessages) and self.supports_assets(headers) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9c3bd9dacbb..da437981d72 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -70175,6 +70175,14 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "openrouter/typesafe/jev-1.14": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "openrouter", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://openrouter.ai/typesafe/jev-1.14", + "supports_box_questions": true + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9c3bd9dacbb..da437981d72 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -70175,6 +70175,14 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "openrouter/typesafe/jev-1.14": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "openrouter", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://openrouter.ai/typesafe/jev-1.14", + "supports_box_questions": true + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index a9f27fe176c..cc27d78e9cc 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -2467,14 +2467,19 @@ def test_images_over_cloudflares_size_limits_are_refused_before_http( assert not route.called +_SMALL_PNG: Final = _png_data_url(1) +_DROPPED_OVERSIZED_IMAGES: Final[tuple[tuple[Sequence[object], Sequence[object]], ...]] = ( + *((images, images[:kept]) for (images, _), kept in zip(_OVERSIZED_IMAGES, (0, 0, 2), strict=True)), + ((_png_data_url(4 * _MIB + 1), _SMALL_PNG), (_SMALL_PNG,)), +) + + @pytest.mark.usefixtures("cloudflare_account") @pytest.mark.parametrize( - ("images", "kept_count"), - tuple(zip((images for images, _ in _OVERSIZED_IMAGES), (0, 0, 2), strict=True)), - ids=_OVERSIZED_IMAGE_IDS, + ("images", "kept"), _DROPPED_OVERSIZED_IMAGES, ids=(*_OVERSIZED_IMAGE_IDS, "oversized_then_small") ) def test_images_over_cloudflares_size_limits_are_dropped_under_drop_params( - images: Sequence[object], kept_count: int, respx_mock: respx.MockRouter + images: Sequence[object], kept: Sequence[object], respx_mock: respx.MockRouter ) -> None: route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) @@ -2486,7 +2491,28 @@ def test_images_over_cloudflares_size_limits_are_dropped_under_drop_params( drop_params=True, ) - assert json.loads(route.calls.last.request.content).get("images", []) == list(images[:kept_count]) + assert json.loads(route.calls.last.request.content).get("images", []) == list(kept) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_account") +async def test_logging_callbacks_get_only_the_images_left_after_an_oversized_one_drops( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE) + recording_logger: Final = _InputRecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + await litellm.asystemone( + model="cloudflare/clef", + state=_SCREEN_TEXT, + questions=_SYSTEMONE_PREDICATE, + images=[_png_data_url(4 * _MIB + 1), _SMALL_PNG], + drop_params=True, + ) + await _drain_logging_worker() + + assert recording_logger.inputs == [json.loads(route.calls.last.request.content)] @pytest.mark.usefixtures("cloudflare_account") @@ -3615,6 +3641,31 @@ async def test_a_deployment_that_advertises_box_questions_gets_box_answers_from_ assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)] +@pytest.mark.asyncio +async def test_openrouter_jev_1_14_gets_box_answers_while_jev_1_13_is_still_refused( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + box_answer: Final = {key: value for key, value in _CRACK_BOX_ANSWER.items() if key != "name"} + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond( + json={"model": "typesafe/jev-1.14", "answers": {"crack": box_answer}} + ) + + response: Final = await litellm.adecisions( + model="openrouter/typesafe/jev-1.14", input="review", questions=[_CRACK_BOX_QUESTION], api_key="caller-key" + ) + + assert json.loads(route.calls.last.request.content)["questions"] == { + "crack": {"type": "box", "instructions": "the crack on the screen"} + } + assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)] + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['box questions'\]"): + await litellm.adecisions( + model="openrouter/typesafe/jev-1.13", input="review", questions=[_CRACK_BOX_QUESTION], api_key="caller-key" + ) + assert route.call_count == 1 + + @pytest.mark.asyncio async def test_a_deployment_that_turns_box_questions_off_is_refused_even_when_the_cost_map_allows_them( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch @@ -4309,6 +4360,34 @@ def test_openrouter_jev_rejects_webp_before_http( assert not route.called +@pytest.mark.asyncio +async def test_images_to_an_openrouter_upstream_without_image_rules_are_refused_or_dropped_as_before( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + request: Final[Mapping[str, object]] = MappingProxyType( + { + "model": "openrouter/upstage/solar-decide-flash", + "state": "Product photo", + "questions": _SYSTEMONE_PREDICATE, + "images": [_SMALL_PNG], + "api_key": "caller-key", + } + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['images'\]"): + await litellm.asystemone(**request) + await litellm.asystemone(**request, drop_params=True) + + assert route.call_count == 1 + assert json.loads(route.calls.last.request.content) == { + "model": "upstage/solar-decide-flash", + "state": "Product photo", + "questions": _SYSTEMONE_PREDICATE, + } + + def test_openrouter_jev_drops_webp_and_keeps_png_under_drop_params(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)