scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-13 08:52:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 353

feat(openrouter): allow box questions for Jev 1.14 and jev-latest

Browse source
This commit is contained in:
mtrefilek 2026-10-29 12:14:40 -05:00
parent f716db54a3
commit f25509e8e4
3 changed files with 65 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
litellm/model_prices_and_context_window_backup.json
View file

@ -44596,6 +44596,18 @@
"supports_system_messages": true,
"supports_vision": false
},
"openrouter/typesafe/jev-1.14": {
"litellm_provider": "openrouter",
"mode": "evaluation",
"source": "https://openrouter.ai/docs/guides/community/jev",
"supports_box_questions": true
},
"openrouter/~typesafe/jev-latest": {
"litellm_provider": "openrouter",
"mode": "evaluation",
"source": "https://openrouter.ai/docs/guides/community/jev",
"supports_box_questions": true
},
"ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
"input_cost_per_token": 6.7e-07,
"litellm_provider": "ovhcloud",

12
model_prices_and_context_window.json
View file

@ -44596,6 +44596,18 @@
"supports_system_messages": true,
"supports_vision": false
},
"openrouter/typesafe/jev-1.14": {
"litellm_provider": "openrouter",
"mode": "evaluation",
"source": "https://openrouter.ai/docs/guides/community/jev",
"supports_box_questions": true
},
"openrouter/~typesafe/jev-latest": {
"litellm_provider": "openrouter",
"mode": "evaluation",
"source": "https://openrouter.ai/docs/guides/community/jev",
"supports_box_questions": true
},
"ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
"input_cost_per_token": 6.7e-07,
"litellm_provider": "ovhcloud",

41
tests/unit/decisions/test_main.py
View file

@ -3615,6 +3615,47 @@ async def test_a_deployment_that_advertises_box_questions_gets_box_answers_from_
assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)]
@pytest.mark.asyncio
@pytest.mark.parametrize("model", ("openrouter/typesafe/jev-1.14", "openrouter/~typesafe/jev-latest"))
async def test_openrouter_jev_sends_box_questions_and_returns_box_answers(
model: str, respx_mock: respx.MockRouter
) -> None:
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(
json={**_RESPONSE, "answers": {"crack": {k: v for k, v in _CRACK_BOX_ANSWER.items() if k != "name"}}}
)
response: Final = await litellm.adecisions(
model=model,
input=[
{
"role": "user",
"content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}],
}
],
questions=[_CRACK_BOX_QUESTION],
api_key="caller-key",
)
assert json.loads(route.calls.last.request.content)["questions"] == {
"crack": {"type": "box", "instructions": "the crack on the screen"}
}
assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)]
def test_openrouter_jev_1_13_still_refuses_box_questions(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match=r"\['box questions'\]"):
litellm.decisions(
model="openrouter/typesafe/jev-1.13", input="review", questions=[_CRACK_BOX_QUESTION], api_key="k"
)
assert not route.called
@pytest.mark.asyncio
async def test_a_deployment_that_turns_box_questions_off_is_refused_even_when_the_cost_map_allows_them(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch