diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index da437981d72..979bae225fd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -44596,6 +44596,18 @@ "supports_system_messages": true, "supports_vision": false }, + "openrouter/typesafe/jev-1.14": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, + "openrouter/~typesafe/jev-latest": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", @@ -70175,14 +70187,6 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, - "openrouter/typesafe/jev-1.14": { - "input_cost_per_token": 4.2e-08, - "litellm_provider": "openrouter", - "mode": "evaluation", - "output_cost_per_token": 0.0, - "source": "https://openrouter.ai/typesafe/jev-1.14", - "supports_box_questions": true - }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index da437981d72..979bae225fd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -44596,6 +44596,18 @@ "supports_system_messages": true, "supports_vision": false }, + "openrouter/typesafe/jev-1.14": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, + "openrouter/~typesafe/jev-latest": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "source": "https://openrouter.ai/docs/guides/community/jev", + "supports_box_questions": true + }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", @@ -70175,14 +70187,6 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, - "openrouter/typesafe/jev-1.14": { - "input_cost_per_token": 4.2e-08, - "litellm_provider": "openrouter", - "mode": "evaluation", - "output_cost_per_token": 0.0, - "source": "https://openrouter.ai/typesafe/jev-1.14", - "supports_box_questions": true - }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index cc27d78e9cc..f2c9f1ce314 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3642,28 +3642,44 @@ async def test_a_deployment_that_advertises_box_questions_gets_box_answers_from_ @pytest.mark.asyncio -async def test_openrouter_jev_1_14_gets_box_answers_while_jev_1_13_is_still_refused( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("model", ("openrouter/typesafe/jev-1.14", "openrouter/~typesafe/jev-latest")) +async def test_openrouter_jev_sends_box_questions_and_returns_box_answers( + model: str, respx_mock: respx.MockRouter ) -> None: - monkeypatch.setattr(litellm, "drop_params", False) - box_answer: Final = {key: value for key, value in _CRACK_BOX_ANSWER.items() if key != "name"} route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond( - json={"model": "typesafe/jev-1.14", "answers": {"crack": box_answer}} + json={**_RESPONSE, "answers": {"crack": {k: v for k, v in _CRACK_BOX_ANSWER.items() if k != "name"}}} ) response: Final = await litellm.adecisions( - model="openrouter/typesafe/jev-1.14", input="review", questions=[_CRACK_BOX_QUESTION], api_key="caller-key" + model=model, + input=[ + { + "role": "user", + "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}], + } + ], + questions=[_CRACK_BOX_QUESTION], + api_key="caller-key", ) assert json.loads(route.calls.last.request.content)["questions"] == { "crack": {"type": "box", "instructions": "the crack on the screen"} } assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)] + + +def test_openrouter_jev_1_13_still_refuses_box_questions( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['box questions'\]"): - await litellm.adecisions( - model="openrouter/typesafe/jev-1.13", input="review", questions=[_CRACK_BOX_QUESTION], api_key="caller-key" + litellm.decisions( + model="openrouter/typesafe/jev-1.13", input="review", questions=[_CRACK_BOX_QUESTION], api_key="k" ) - assert route.call_count == 1 + + assert not route.called @pytest.mark.asyncio