diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 4572e59944b..e5e0027a808 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -11407,7 +11407,7 @@ }, "/systemone": { "post": { - "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_systemone_post", "requestBody": { "content": { @@ -11813,7 +11813,6 @@ ] }, "description": "Named noul, choice and score questions. Each key becomes a key in the answers", - "maxProperties": 128, "minProperties": 1, "propertyNames": { "minLength": 1 @@ -12814,7 +12813,7 @@ }, "/v1/systemone": { "post": { - "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_v1_systemone_post", "requestBody": { "content": { @@ -13220,7 +13219,6 @@ ] }, "description": "Named noul, choice and score questions. Each key becomes a key in the answers", - "maxProperties": 128, "minProperties": 1, "propertyNames": { "minLength": 1 diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 366842d23a6..a8684dd9757 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -228,7 +228,7 @@ async def systemone( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): """ - Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + Judge a `state` against one or more named `questions` and get one calibrated answer per question name. Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`, diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index a1d48f730c0..5ad6e45e6d2 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -73,7 +73,7 @@ DecisionQuestion: TypeAlias = Annotated[ DecisionQuestionMap: TypeAlias = Annotated[ Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion], - Field(min_length=1, max_length=MAX_DECISION_QUESTIONS), + Field(min_length=1), ] diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 021eb289ad7..4384507e841 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -126,6 +126,51 @@ def test_proxy_decisions_route_returns_answers_and_cost( assert upstream.calls[0].request.headers["authorization"] == "Bearer test-key" +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone")) +@pytest.mark.parametrize("question_count", (128, 129, 154)) +def test_proxy_systemone_preserves_large_question_batches( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, + question_count: int, +) -> None: + questions: Final = { + f"q{index}": {"type": "noul", "instructions": f"Question {index}?"} for index in range(question_count) + } + answers: Final = {f"q{index}": {"type": "noul", "noul": index / question_count} for index in range(question_count)} + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={**_RESPONSE, "answers": answers} + ) + + response: Final = client.post(endpoint, json={**_REQUEST, "questions": questions}) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + **_REQUEST, + "model": "pplx-decider-v1-27b", + "questions": questions, + } + assert response.json()["answers"] == answers + + +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone")) +@pytest.mark.parametrize( + "questions", + ({}, {"": {"type": "noul", "instructions": "Is this a defect?"}}, {"is_defect": {"type": "noul"}}), + ids=("empty", "empty_name", "missing_instructions_or_criteria"), +) +def test_proxy_systemone_rejects_invalid_questions_before_http( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, + questions: Mapping[str, object], +) -> None: + response: Final = client.post(endpoint, json={**_REQUEST, "questions": questions}) + + assert response.status_code == 400, response.text + assert len(respx_mock.calls) == 0 + + def test_proxy_decisions_dispatches_typesafe_deployment( client: TestClient, monkeypatch: pytest.MonkeyPatch, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2e2ff77963d..4dd9e5d6073 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16025,7 +16025,7 @@ export interface paths { put?: never; /** * Ask named questions about a state (System One format) - * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + * @description Judge a `state` against one or more named `questions` and get one calibrated answer per question name. * * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with * `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`, @@ -22346,7 +22346,7 @@ export interface paths { put?: never; /** * Ask named questions about a state (System One format) - * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + * @description Judge a `state` against one or more named `questions` and get one calibrated answer per question name. * * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with * `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,