scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-10 08:28:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

Merge branch 'openrouter-decisions-images' of https://github.com/mtrefilek/litellm into openrouter-decisions-images

Browse source 
# Conflicts:
#	tests/unit/decisions/test_main.py
This commit is contained in:
mateo-berri 2026-10-29 13:12:52 -07:00
commit 859e15fe13
3 changed files with 49 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
litellm/model_prices_and_context_window_backup.json
View file

@ -44596,6 +44596,18 @@
"supports_system_messages": true,
"supports_vision": false
},
"openrouter/typesafe/jev-1.14": {
"litellm_provider": "openrouter",
"mode": "evaluation",
"source": "https://openrouter.ai/docs/guides/community/jev",
"supports_box_questions": true
},
"openrouter/~typesafe/jev-latest": {
"litellm_provider": "openrouter",
"mode": "evaluation",
"source": "https://openrouter.ai/docs/guides/community/jev",
"supports_box_questions": true
},
"ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
"input_cost_per_token": 6.7e-07,
"litellm_provider": "ovhcloud",
@ -70175,14 +70187,6 @@
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
},
"openrouter/typesafe/jev-1.14": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "openrouter",
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://openrouter.ai/typesafe/jev-1.14",
"supports_box_questions": true
},
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 2e-07,

20
model_prices_and_context_window.json
View file

@ -44596,6 +44596,18 @@
"supports_system_messages": true,
"supports_vision": false
},
"openrouter/typesafe/jev-1.14": {
"litellm_provider": "openrouter",
"mode": "evaluation",
"source": "https://openrouter.ai/docs/guides/community/jev",
"supports_box_questions": true
},
"openrouter/~typesafe/jev-latest": {
"litellm_provider": "openrouter",
"mode": "evaluation",
"source": "https://openrouter.ai/docs/guides/community/jev",
"supports_box_questions": true
},
"ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
"input_cost_per_token": 6.7e-07,
"litellm_provider": "ovhcloud",
@ -70175,14 +70187,6 @@
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
},
"openrouter/typesafe/jev-1.14": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "openrouter",
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://openrouter.ai/typesafe/jev-1.14",
"supports_box_questions": true
},
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 2e-07,

34
tests/unit/decisions/test_main.py
View file

@ -3642,28 +3642,44 @@ async def test_a_deployment_that_advertises_box_questions_gets_box_answers_from_
@pytest.mark.asyncio
async def test_openrouter_jev_1_14_gets_box_answers_while_jev_1_13_is_still_refused(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
@pytest.mark.parametrize("model", ("openrouter/typesafe/jev-1.14", "openrouter/~typesafe/jev-latest"))
async def test_openrouter_jev_sends_box_questions_and_returns_box_answers(
model: str, respx_mock: respx.MockRouter
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
box_answer: Final = {key: value for key, value in _CRACK_BOX_ANSWER.items() if key != "name"}
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(
json={"model": "typesafe/jev-1.14", "answers": {"crack": box_answer}}
json={**_RESPONSE, "answers": {"crack": {k: v for k, v in _CRACK_BOX_ANSWER.items() if k != "name"}}}
)
response: Final = await litellm.adecisions(
model="openrouter/typesafe/jev-1.14", input="review", questions=[_CRACK_BOX_QUESTION], api_key="caller-key"
model=model,
input=[
{
"role": "user",
"content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}],
}
],
questions=[_CRACK_BOX_QUESTION],
api_key="caller-key",
)
assert json.loads(route.calls.last.request.content)["questions"] == {
"crack": {"type": "box", "instructions": "the crack on the screen"}
}
assert [answer.model_dump(mode="json") for answer in response.answers] == [dict(_CRACK_BOX_ANSWER)]
def test_openrouter_jev_1_13_still_refuses_box_questions(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match=r"\['box questions'\]"):
await litellm.adecisions(
model="openrouter/typesafe/jev-1.13", input="review", questions=[_CRACK_BOX_QUESTION], api_key="caller-key"
litellm.decisions(
model="openrouter/typesafe/jev-1.13", input="review", questions=[_CRACK_BOX_QUESTION], api_key="k"
)
assert route.call_count == 1
assert not route.called
@pytest.mark.asyncio