From 606ecf56debff12b2c64c802ea8a2d2dddd11beb Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:01:59 -0700 Subject: [PATCH] fix(decisions): keep box answers from OpenAI-format providers instead of turning them into refusals (#46633) * fix(decisions): keep box answers from OpenAI-format providers instead of turning them into refusals Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): prove an OpenAI-format box answer reaches the caller as a box Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): give the box wire test a deployment that advertises supports_box_questions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/openai/decisions/transformation.py | 2 ++ .../test_decisions_openai_format_wire.py | 26 ++++++++++++++----- .../test_openai_decisions_transformation.py | 19 ++++++++++++++ 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 95511f5221b..7ce5b8676ee 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -239,6 +239,8 @@ def _ir_answer(question: DecisionsIRQuestion, answer: OpenAIDecisionAnswer | Non for item in answer.probabilities ), ) + case DecisionsIRBoxQuestion(), OpenAIBoxAnswer(): + return DecisionsIRBoxAnswer(box=answer.box, confidence=answer.confidence) case _: return DecisionsIRRefusal() diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index cb021e02e88..2f82561d1b1 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -286,14 +286,10 @@ def _expected_spend(provider: _Provider) -> float: cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes()) prices: Final = object_value(cost_map[provider.cost_map_key]) input_key: Final = ( - "input_cost_per_token_decisions" - if "input_cost_per_token_decisions" in prices - else "input_cost_per_token" + "input_cost_per_token_decisions" if "input_cost_per_token_decisions" in prices else "input_cost_per_token" ) output_key: Final = ( - "output_cost_per_token_decisions" - if "output_cost_per_token_decisions" in prices - else "output_cost_per_token" + "output_cost_per_token_decisions" if "output_cost_per_token_decisions" in prices else "output_cost_per_token" ) return _INPUT_TOKENS * _number(prices[input_key]) + _OUTPUT_TOKENS * _number(prices[output_key]) @@ -524,6 +520,24 @@ def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) assert call["body"] == {"model": _OPENAI.body_model, "input": _OPENAI_IMAGE_MESSAGES, "questions": _QUESTIONS} +def test_openai_box_answers_reach_the_caller_as_boxes_not_refusals(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + box: Final[dict[str, JsonValue]] = {"top": 0.1, "bottom": 0.4, "left": 0.2, "right": 0.6} + box_answer: Final[dict[str, JsonValue]] = {"type": "box", "name": "crack", "box": box, "confidence": 0.7} + handle: Final = _register(scenario, {**_OPENAI.upstream_reply(), "answers": [box_answer]}) + model: Final = scenario.model( + model=_OPENAI.model, + api_base=handle.api_base(), + api_key=_OPENAI.api_key, + model_info={"supports_box_questions": True}, + ) + response: Final = _decide( + gateway, model, questions=[{"type": "box", "name": "crack", "instructions": "Where is the crack?"}] + ) + assert response.status_code == 200, response.text + assert response.json()["answers"] == [box_answer], response.text + + def test_a_message_list_input_reaches_a_system_one_provider_as_its_flattened_text(gateway: Gateway) -> None: with gateway.scenario() as scenario: handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index db781a1caf7..2a07462fe81 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -116,6 +116,25 @@ def test_answers_openai_did_not_return_are_refusals() -> None: assert [answer.name for answer in response.answers] == ["damaged", None, "severity", "fraud"] +def test_a_box_answer_from_openai_keeps_its_box_instead_of_becoming_a_refusal() -> None: + request: Final = { + "input": _OPENAI_REQUEST["input"], + "questions": [{"type": "box", "name": "crack", "instructions": "Where is the crack?"}], + } + box: Final = {"top": 0.1, "bottom": 0.4, "left": 0.2, "right": 0.6} + payload: Final = {**_OPENAI_RESPONSE, "answers": [{"type": "box", "name": "crack", "box": box, "confidence": 0.7}]} + ir: Final = _openai_ir(request) + + response: Final = ir_to_openai_response(OpenAIDecisionsConfig().parse_response(payload, ir), ir, "requested") + + assert response.answers[0].model_dump(mode="json") == { + "type": "box", + "name": "crack", + "box": box, + "confidence": 0.7, + } + + def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_their_keys() -> None: request: Final = _SYSTEMONE_BODY.validate_python( {