diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index fcae4bcc49c..003fccb7a8d 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -45,10 +45,14 @@ from litellm.types.decisions import ( OpenAIScoreAnswer, OpenAIScoreProbability, OpenAIScoreQuestion, + UnsupportedDecisionsRequest, systemone_choice_key, ) _OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) +_SINGLE_OPTION: Final = UnsupportedDecisionsRequest( + reason="OpenAI needs at least 2 choices or levels on every choice or score question" +) def _ir_input(decision_input: OpenAIDecisionInput) -> DecisionsIRState | DecisionsIRMessages: @@ -94,6 +98,10 @@ def _text_field(key: str, value: DecisionsJSON | None) -> Mapping[str, str]: return {} if value is None else {key: decisions_text(value)} +def _instructions(value: DecisionsJSON | None, default: str) -> str: + return default if value is None else decisions_text(value) + + def _predicate_instructions(question: DecisionsIRPredicateQuestion) -> str: instructions: Final = () if question.instructions is None else (decisions_text(question.instructions),) criteria: Final = tuple( @@ -110,13 +118,13 @@ def _openai_question(question: DecisionsIRQuestion) -> Mapping[str, object]: return { "type": "predicate", **_text_field("name", question.name), - "instructions": _predicate_instructions(question), + "instructions": _predicate_instructions(question) or "Is this true of the input?", } case DecisionsIRChoiceQuestion(): return { "type": "choice", **_text_field("name", question.name), - **_text_field("instructions", question.instructions), + "instructions": _instructions(question.instructions, "Which choice best fits the input?"), "choices": [ {"value": option.value, **_text_field("description", option.description)} for option in question.choices @@ -126,7 +134,7 @@ def _openai_question(question: DecisionsIRQuestion) -> Mapping[str, object]: return { "type": "score", **_text_field("name", question.name), - **_text_field("instructions", question.instructions), + "instructions": _instructions(question.instructions, "Which level best fits the input?"), "levels": [ {"label": decisions_text(level.label), **_text_field("description", level.description)} for level in question.levels @@ -146,7 +154,19 @@ def _openai_input(decision_input: DecisionsIRState | DecisionsIRMessages) -> str assert_never(decision_input) -def ir_to_openai_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]: +def _has_one_option(question: DecisionsIRQuestion) -> bool: + match question: + case DecisionsIRChoiceQuestion(): + return len(question.choices) < 2 + case DecisionsIRScoreQuestion(): + return len(question.levels) < 2 + case _: + return False + + +def ir_to_openai_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if any(_has_one_option(question) for question in request.questions): + return _SINGLE_OPTION return { "model": model, "input": _openai_input(request.input), @@ -298,7 +318,9 @@ class OpenAIDecisionsEndpoint: def endpoint_url(self, api_base: str, model: str) -> str: return f"{api_base.rstrip('/').removesuffix('/v1')}/v1/decisions" - def request_body(self, model: str, request: DecisionsIRRequest) -> Mapping[str, object]: + def request_body( + self, model: str, request: DecisionsIRRequest + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: return ir_to_openai_request(model, request) def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index 942fad1db6d..249a1a7640a 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -1,6 +1,7 @@ from collections.abc import Mapping from typing import Final +import pytest from pydantic import TypeAdapter from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_response, systemone_request_to_ir @@ -10,7 +11,12 @@ from litellm.llms.openai.decisions.transformation import ( ir_to_openai_response, openai_request_to_ir, ) -from litellm.types.decisions import DecisionsIRRequest, DecisionsRequestBody, OpenAIDecisionRequestBody +from litellm.types.decisions import ( + DecisionsIRRequest, + DecisionsRequestBody, + OpenAIDecisionRequestBody, + UnsupportedDecisionsRequest, +) _SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) _OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) @@ -148,11 +154,49 @@ def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_t "instructions": '{"policy": "refund-v2"}', "choices": [{"value": "refund", "description": "Within 30 days"}, {"value": "escalate"}], }, - {"type": "score", "name": "severity", "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}]}, + { + "type": "score", + "name": "severity", + "instructions": "Which level best fits the input?", + "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}], + }, ], } +def test_systemone_questions_without_instructions_become_valid_openai_questions() -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": "Screen cracked", + "questions": { + "damaged": {"type": "noul", "criteria": {"false": None}}, + "action": {"type": "choice", "criteria": {"refund": None, "escalate": None}}, + "severity": {"type": "score", "criteria": ["minor", "major"]}, + }, + } + ) + + body: Final = ir_to_openai_request("gpt-6-luna", systemone_request_to_ir(request)) + + assert all(question.instructions for question in _OPENAI_BODY.validate_python(body).questions) + + +@pytest.mark.parametrize( + "question", + ({"type": "choice", "criteria": {"refund": None}}, {"type": "score", "criteria": ["minor"]}), + ids=("choice", "score"), +) +def test_a_systemone_question_with_one_option_is_unsupported_by_openai(question: Mapping[str, object]) -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": "Screen cracked", + "questions": {"damaged": {"type": "noul", "instructions": "Damaged?"}, "q": question}, + } + ) + + assert isinstance(ir_to_openai_request("gpt-6-luna", systemone_request_to_ir(request)), UnsupportedDecisionsRequest) + + def test_an_openai_response_becomes_systemone_answers_with_the_callers_score_labels() -> None: ir: Final = systemone_request_to_ir( _SYSTEMONE_BODY.validate_python(