fix(decisions): remove the System One question count cap (#47985)

litellm/proxy/_lazy_openapi_snapshot.json

@@ -11407,7 +11407,7 @@
       },
       "/systemone": {
         "post": {
-          "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
+          "description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
           "operationId": "systemone_systemone_post",
           "requestBody": {
             "content": {
@@ -11813,7 +11813,6 @@
                         ]
                       },
                       "description": "Named noul, choice and score questions. Each key becomes a key in the answers",
-                      "maxProperties": 128,
                       "minProperties": 1,
                       "propertyNames": {
                         "minLength": 1
@@ -12814,7 +12813,7 @@
       },
       "/v1/systemone": {
         "post": {
-          "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
+          "description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
           "operationId": "systemone_v1_systemone_post",
           "requestBody": {
             "content": {
@@ -13220,7 +13219,6 @@
                         ]
                       },
                       "description": "Named noul, choice and score questions. Each key becomes a key in the answers",
-                      "maxProperties": 128,
                       "minProperties": 1,
                       "propertyNames": {
                         "minLength": 1

litellm/proxy/decisions_endpoints/endpoints.py

@@ -228,7 +228,7 @@ async def systemone(
     user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
 ):
     """
-    Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
+    Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
 
     Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
     `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,

litellm/types/decisions.py

@@ -73,7 +73,7 @@ DecisionQuestion: TypeAlias = Annotated[
 
 DecisionQuestionMap: TypeAlias = Annotated[
     Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion],
-    Field(min_length=1, max_length=MAX_DECISION_QUESTIONS),
+    Field(min_length=1),
 ]
 
 

tests/unit/proxy/decisions_endpoints/test_endpoints.py

@@ -126,6 +126,51 @@ def test_proxy_decisions_route_returns_answers_and_cost(
     assert upstream.calls[0].request.headers["authorization"] == "Bearer test-key"
 
 
+@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone"))
+@pytest.mark.parametrize("question_count", (128, 129, 154))
+def test_proxy_systemone_preserves_large_question_batches(
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+    endpoint: str,
+    question_count: int,
+) -> None:
+    questions: Final = {
+        f"q{index}": {"type": "noul", "instructions": f"Question {index}?"} for index in range(question_count)
+    }
+    answers: Final = {f"q{index}": {"type": "noul", "noul": index / question_count} for index in range(question_count)}
+    upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(
+        json={**_RESPONSE, "answers": answers}
+    )
+
+    response: Final = client.post(endpoint, json={**_REQUEST, "questions": questions})
+
+    assert response.status_code == 200, response.text
+    assert json.loads(upstream.calls[0].request.content) == {
+        **_REQUEST,
+        "model": "pplx-decider-v1-27b",
+        "questions": questions,
+    }
+    assert response.json()["answers"] == answers
+
+
+@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone"))
+@pytest.mark.parametrize(
+    "questions",
+    ({}, {"": {"type": "noul", "instructions": "Is this a defect?"}}, {"is_defect": {"type": "noul"}}),
+    ids=("empty", "empty_name", "missing_instructions_or_criteria"),
+)
+def test_proxy_systemone_rejects_invalid_questions_before_http(
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+    endpoint: str,
+    questions: Mapping[str, object],
+) -> None:
+    response: Final = client.post(endpoint, json={**_REQUEST, "questions": questions})
+
+    assert response.status_code == 400, response.text
+    assert len(respx_mock.calls) == 0
+
+
 def test_proxy_decisions_dispatches_typesafe_deployment(
     client: TestClient,
     monkeypatch: pytest.MonkeyPatch,

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -16025,7 +16025,7 @@ export interface paths {
         put?: never;
         /**
          * Ask named questions about a state (System One format)
-         * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
+         * @description Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
          *
          *     Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
          *     `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,
@@ -22346,7 +22346,7 @@ export interface paths {
         put?: never;
         /**
          * Ask named questions about a state (System One format)
-         * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
+         * @description Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
          *
          *     Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
          *     `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,