fix(decisions): keep box answers from OpenAI-format providers instead of turning them into refusals (#46633)

* fix(decisions): keep box answers from OpenAI-format providers instead of turning them into refusals

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(integration): prove an OpenAI-format box answer reaches the caller as a box

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(integration): give the box wire test a deployment that advertises supports_box_questions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/llms/openai/decisions/transformation.py

@@ -239,6 +239,8 @@ def _ir_answer(question: DecisionsIRQuestion, answer: OpenAIDecisionAnswer | Non
                     for item in answer.probabilities
                 ),
             )
+        case DecisionsIRBoxQuestion(), OpenAIBoxAnswer():
+            return DecisionsIRBoxAnswer(box=answer.box, confidence=answer.confidence)
         case _:
             return DecisionsIRRefusal()
 

tests/integration/providers/test_decisions_openai_format_wire.py

@@ -286,14 +286,10 @@ def _expected_spend(provider: _Provider) -> float:
     cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes())
     prices: Final = object_value(cost_map[provider.cost_map_key])
     input_key: Final = (
-        "input_cost_per_token_decisions"
-        if "input_cost_per_token_decisions" in prices
-        else "input_cost_per_token"
+        "input_cost_per_token_decisions" if "input_cost_per_token_decisions" in prices else "input_cost_per_token"
     )
     output_key: Final = (
-        "output_cost_per_token_decisions"
-        if "output_cost_per_token_decisions" in prices
-        else "output_cost_per_token"
+        "output_cost_per_token_decisions" if "output_cost_per_token_decisions" in prices else "output_cost_per_token"
     )
     return _INPUT_TOKENS * _number(prices[input_key]) + _OUTPUT_TOKENS * _number(prices[output_key])
 
@@ -524,6 +520,24 @@ def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway)
         assert call["body"] == {"model": _OPENAI.body_model, "input": _OPENAI_IMAGE_MESSAGES, "questions": _QUESTIONS}
 
 
+def test_openai_box_answers_reach_the_caller_as_boxes_not_refusals(gateway: Gateway) -> None:
+    with gateway.scenario() as scenario:
+        box: Final[dict[str, JsonValue]] = {"top": 0.1, "bottom": 0.4, "left": 0.2, "right": 0.6}
+        box_answer: Final[dict[str, JsonValue]] = {"type": "box", "name": "crack", "box": box, "confidence": 0.7}
+        handle: Final = _register(scenario, {**_OPENAI.upstream_reply(), "answers": [box_answer]})
+        model: Final = scenario.model(
+            model=_OPENAI.model,
+            api_base=handle.api_base(),
+            api_key=_OPENAI.api_key,
+            model_info={"supports_box_questions": True},
+        )
+        response: Final = _decide(
+            gateway, model, questions=[{"type": "box", "name": "crack", "instructions": "Where is the crack?"}]
+        )
+        assert response.status_code == 200, response.text
+        assert response.json()["answers"] == [box_answer], response.text
+
+
 def test_a_message_list_input_reaches_a_system_one_provider_as_its_flattened_text(gateway: Gateway) -> None:
     with gateway.scenario() as scenario:
         handle: Final = _register(scenario, _PERPLEXITY.upstream_reply())

tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py

@@ -116,6 +116,25 @@ def test_answers_openai_did_not_return_are_refusals() -> None:
     assert [answer.name for answer in response.answers] == ["damaged", None, "severity", "fraud"]
 
 
+def test_a_box_answer_from_openai_keeps_its_box_instead_of_becoming_a_refusal() -> None:
+    request: Final = {
+        "input": _OPENAI_REQUEST["input"],
+        "questions": [{"type": "box", "name": "crack", "instructions": "Where is the crack?"}],
+    }
+    box: Final = {"top": 0.1, "bottom": 0.4, "left": 0.2, "right": 0.6}
+    payload: Final = {**_OPENAI_RESPONSE, "answers": [{"type": "box", "name": "crack", "box": box, "confidence": 0.7}]}
+    ir: Final = _openai_ir(request)
+
+    response: Final = ir_to_openai_response(OpenAIDecisionsConfig().parse_response(payload, ir), ir, "requested")
+
+    assert response.answers[0].model_dump(mode="json") == {
+        "type": "box",
+        "name": "crack",
+        "box": box,
+        "confidence": 0.7,
+    }
+
+
 def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_their_keys() -> None:
     request: Final = _SYSTEMONE_BODY.validate_python(
         {