From d209a85076d61511a011494916eee4b212b4d1ff Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:47:54 -0700 Subject: [PATCH] refactor(decisions): move the OpenAI wire translation into llms/openai --- litellm/decisions/openai_transformation.py | 98 -------------- .../llms/openai/decisions/transformation.py | 120 +++++++++++++++++- .../decisions/test_openai_transformation.py | 103 +-------------- tests/unit/llms/openai/decisions/__init__.py | 0 .../test_openai_decisions_transformation.py | 109 ++++++++++++++++ 5 files changed, 227 insertions(+), 203 deletions(-) create mode 100644 tests/unit/llms/openai/decisions/__init__.py create mode 100644 tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py diff --git a/litellm/decisions/openai_transformation.py b/litellm/decisions/openai_transformation.py index bdbfd7df0a1..af2c838b9ab 100644 --- a/litellm/decisions/openai_transformation.py +++ b/litellm/decisions/openai_transformation.py @@ -1,21 +1,14 @@ -import json from collections.abc import Mapping, Sequence from typing import Final -from pydantic import TypeAdapter from typing_extensions import assert_never from litellm.types.decisions import ( ChoiceAnswer, - ChoiceQuestion, DecisionAnswer, - DecisionQuestion, - DecisionsJSON, - DecisionsRequestBody, DecisionsResponse, DecisionsUsage, NoulAnswer, - NoulQuestion, OpenAIChoiceAnswer, OpenAIChoiceProbability, OpenAIChoiceQuestion, @@ -33,12 +26,10 @@ from litellm.types.decisions import ( OpenAIScoreProbability, OpenAIScoreQuestion, ScoreAnswer, - ScoreQuestion, systemone_choice_key, ) _OPENAI_ONLY_FIELDS: Final = frozenset({"input", "questions", "safety_identifier"}) -_OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) def _message_text(message: OpenAIDecisionInputMessage) -> str: @@ -134,92 +125,3 @@ def to_openai_response( total_tokens=usage.input_tokens + usage.output_tokens, ), ) - - -def _text(value: DecisionsJSON) -> str: - return value if isinstance(value, str) else json.dumps(value) - - -def _instructions(instructions: DecisionsJSON | None) -> Mapping[str, str]: - return {} if instructions is None else {"instructions": _text(instructions)} - - -def _predicate_instructions(question: NoulQuestion) -> str: - instructions: Final = () if question.instructions is None else (_text(question.instructions),) - criteria: Final = tuple( - f"Answer {answer} when: {_text(rule)}" for answer, rule in (question.criteria or {}).items() if rule is not None - ) - return "\n\n".join((*instructions, *criteria)) - - -def _choice(value: str, description: DecisionsJSON | None) -> Mapping[str, str]: - return {"value": value} if description is None else {"value": value, "description": _text(description)} - - -def _openai_question(name: str, question: DecisionQuestion) -> Mapping[str, object]: - match question: - case NoulQuestion(): - return {"type": "predicate", "name": name, "instructions": _predicate_instructions(question)} - case ChoiceQuestion(): - return { - "type": "choice", - "name": name, - **_instructions(question.instructions), - "choices": [_choice(value, description) for value, description in question.criteria.items()], - } - case ScoreQuestion(): - return { - "type": "score", - "name": name, - **_instructions(question.instructions), - "levels": [{"label": _text(criterion)} for criterion in question.criteria], - } - case _: - assert_never(question) - - -def to_openai_request(model: str, request: DecisionsRequestBody) -> Mapping[str, object]: - return { - "model": model, - "input": _text(request.state), - "questions": [_openai_question(name, question) for name, question in request.questions.items()], - } - - -def _systemone_answer(answer: OpenAIDecisionAnswer) -> DecisionAnswer | None: - match answer: - case OpenAIPredicateAnswer(): - return NoulAnswer(type="noul", noul=answer.probability) - case OpenAIChoiceAnswer(): - return ChoiceAnswer( - type="choice", - choice=systemone_choice_key(answer.choice), - confidence=answer.confidence, - probabilities={systemone_choice_key(item.value): item.probability for item in answer.probabilities}, - ) - case OpenAIScoreAnswer(): - return ScoreAnswer( - type="score", - score=answer.score, - confidence=answer.confidence, - legend={str(item.value): item.label for item in answer.probabilities}, - probabilities={str(item.value): item.probability for item in answer.probabilities}, - ) - case OpenAIRefusalAnswer(): - return None - case _: - assert_never(answer) - - -def _named_answers(answers: Sequence[OpenAIDecisionAnswer]) -> Mapping[str, DecisionAnswer]: - translated: Final = ((answer.name, _systemone_answer(answer)) for answer in answers) - return {name: answer for name, answer in translated if name is not None and answer is not None} - - -def to_systemone_response(payload: object) -> DecisionsResponse: - response: Final = _OPENAI_RESPONSE_ADAPTER.validate_python(payload) - return DecisionsResponse( - model=response.model, - answers=_named_answers(response.answers), - usage=DecisionsUsage(input_tokens=response.usage.input_tokens, output_tokens=response.usage.output_tokens), - ) diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 7c7bee2b099..ea07a8a5b89 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -1,9 +1,123 @@ -from collections.abc import Mapping +import json +from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Final -from litellm.decisions.openai_transformation import to_openai_request, to_systemone_response -from litellm.types.decisions import DecisionsRequestBody, DecisionsResponse +from pydantic import TypeAdapter +from typing_extensions import assert_never + +from litellm.types.decisions import ( + ChoiceAnswer, + ChoiceQuestion, + DecisionAnswer, + DecisionQuestion, + DecisionsJSON, + DecisionsRequestBody, + DecisionsResponse, + DecisionsUsage, + NoulAnswer, + NoulQuestion, + OpenAIChoiceAnswer, + OpenAIDecisionAnswer, + OpenAIDecisionResponse, + OpenAIPredicateAnswer, + OpenAIRefusalAnswer, + OpenAIScoreAnswer, + ScoreAnswer, + ScoreQuestion, + systemone_choice_key, +) + +_OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) + + +def _text(value: DecisionsJSON) -> str: + return value if isinstance(value, str) else json.dumps(value) + + +def _instructions(instructions: DecisionsJSON | None) -> Mapping[str, str]: + return {} if instructions is None else {"instructions": _text(instructions)} + + +def _predicate_instructions(question: NoulQuestion) -> str: + instructions: Final = () if question.instructions is None else (_text(question.instructions),) + criteria: Final = tuple( + f"Answer {answer} when: {_text(rule)}" for answer, rule in (question.criteria or {}).items() if rule is not None + ) + return "\n\n".join((*instructions, *criteria)) + + +def _choice(value: str, description: DecisionsJSON | None) -> Mapping[str, str]: + return {"value": value} if description is None else {"value": value, "description": _text(description)} + + +def _openai_question(name: str, question: DecisionQuestion) -> Mapping[str, object]: + match question: + case NoulQuestion(): + return {"type": "predicate", "name": name, "instructions": _predicate_instructions(question)} + case ChoiceQuestion(): + return { + "type": "choice", + "name": name, + **_instructions(question.instructions), + "choices": [_choice(value, description) for value, description in question.criteria.items()], + } + case ScoreQuestion(): + return { + "type": "score", + "name": name, + **_instructions(question.instructions), + "levels": [{"label": _text(criterion)} for criterion in question.criteria], + } + case _: + assert_never(question) + + +def to_openai_request(model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return { + "model": model, + "input": _text(request.state), + "questions": [_openai_question(name, question) for name, question in request.questions.items()], + } + + +def _systemone_answer(answer: OpenAIDecisionAnswer) -> DecisionAnswer | None: + match answer: + case OpenAIPredicateAnswer(): + return NoulAnswer(type="noul", noul=answer.probability) + case OpenAIChoiceAnswer(): + return ChoiceAnswer( + type="choice", + choice=systemone_choice_key(answer.choice), + confidence=answer.confidence, + probabilities={systemone_choice_key(item.value): item.probability for item in answer.probabilities}, + ) + case OpenAIScoreAnswer(): + return ScoreAnswer( + type="score", + score=answer.score, + confidence=answer.confidence, + legend={str(item.value): item.label for item in answer.probabilities}, + probabilities={str(item.value): item.probability for item in answer.probabilities}, + ) + case OpenAIRefusalAnswer(): + return None + case _: + assert_never(answer) + + +def _named_answers(answers: Sequence[OpenAIDecisionAnswer]) -> Mapping[str, DecisionAnswer]: + translated: Final = ((answer.name, _systemone_answer(answer)) for answer in answers) + return {name: answer for name, answer in translated if name is not None and answer is not None} + + +def to_systemone_response(payload: object) -> DecisionsResponse: + response: Final = _OPENAI_RESPONSE_ADAPTER.validate_python(payload) + return DecisionsResponse( + model=response.model, + answers=_named_answers(response.answers), + usage=DecisionsUsage(input_tokens=response.usage.input_tokens, output_tokens=response.usage.output_tokens), + ) @dataclass(frozen=True, slots=True) diff --git a/tests/unit/decisions/test_openai_transformation.py b/tests/unit/decisions/test_openai_transformation.py index 381c2aac643..a3cc1583d40 100644 --- a/tests/unit/decisions/test_openai_transformation.py +++ b/tests/unit/decisions/test_openai_transformation.py @@ -4,7 +4,7 @@ from typing import Final import pytest from pydantic import TypeAdapter, ValidationError -from litellm.decisions.openai_transformation import to_openai_request, to_systemone_request, to_systemone_response +from litellm.decisions.openai_transformation import to_systemone_request from litellm.types.decisions import MAX_DECISION_QUESTIONS, DecisionsRequestBody, OpenAIDecisionRequestBody _OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) @@ -30,104 +30,3 @@ def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_req def test_an_openai_request_with_more_questions_than_systemone_takes_is_rejected_before_translation() -> None: with pytest.raises(ValidationError, match="questions"): _OPENAI_BODY.validate_python(_openai_request(MAX_DECISION_QUESTIONS + 1)) - - -def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_their_keys() -> None: - request: Final = _SYSTEMONE_BODY.validate_python( - { - "state": {"ticket": 1234, "text": "Screen cracked"}, - "questions": { - "damaged": { - "type": "noul", - "instructions": "Is the item damaged?", - "criteria": {"true": "Visible damage", "false": None}, - "provider_field": "dropped", - }, - "rubric_only": {"type": "noul", "criteria": {"true": {"signal": "refund"}}}, - "action": { - "type": "choice", - "instructions": {"policy": "refund-v2"}, - "criteria": {"refund": "Within 30 days", "escalate": None}, - }, - "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, - }, - } - ) - - assert to_openai_request("gpt-6-luna", request) == { - "model": "gpt-6-luna", - "input": '{"ticket": 1234, "text": "Screen cracked"}', - "questions": [ - { - "type": "predicate", - "name": "damaged", - "instructions": "Is the item damaged?\n\nAnswer true when: Visible damage", - }, - {"type": "predicate", "name": "rubric_only", "instructions": 'Answer true when: {"signal": "refund"}'}, - { - "type": "choice", - "name": "action", - "instructions": '{"policy": "refund-v2"}', - "choices": [{"value": "refund", "description": "Within 30 days"}, {"value": "escalate"}], - }, - {"type": "score", "name": "severity", "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}]}, - ], - } - - -def test_an_openai_response_becomes_systemone_answers_keyed_by_question_name_without_refusals() -> None: - response: Final = to_systemone_response( - { - "model": "gpt-6-luna", - "answers": [ - {"type": "predicate", "name": "damaged", "probability": 0.95}, - { - "type": "choice", - "name": "action", - "choice": True, - "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], - "confidence": 0.8, - }, - { - "type": "score", - "name": "severity", - "score": 1.72, - "probabilities": [ - {"value": 0, "label": "minor", "probability": 0.0}, - {"value": 1, "label": "major", "probability": 0.28}, - {"value": 2, "label": "critical", "probability": 0.72}, - ], - "confidence": 0.58, - }, - {"type": "refusal", "name": "fraud"}, - ], - "usage": { - "input_tokens": 383, - "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, - "output_tokens": 2, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 385, - }, - } - ) - - assert response.model_dump(mode="json") == { - "model": "gpt-6-luna", - "answers": { - "damaged": {"type": "noul", "noul": 0.95}, - "action": { - "type": "choice", - "choice": "true", - "confidence": 0.8, - "probabilities": {"true": 0.9, "escalate": 0.1}, - }, - "severity": { - "type": "score", - "score": 1.72, - "confidence": 0.58, - "legend": {"0": "minor", "1": "major", "2": "critical"}, - "probabilities": {"0": 0.0, "1": 0.28, "2": 0.72}, - }, - }, - "usage": {"input_tokens": 383, "output_tokens": 2}, - } diff --git a/tests/unit/llms/openai/decisions/__init__.py b/tests/unit/llms/openai/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py new file mode 100644 index 00000000000..56c950cc8d0 --- /dev/null +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -0,0 +1,109 @@ +from typing import Final + +from pydantic import TypeAdapter + +from litellm.llms.openai.decisions.transformation import to_openai_request, to_systemone_response +from litellm.types.decisions import DecisionsRequestBody + +_SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) + + +def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_their_keys() -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": {"ticket": 1234, "text": "Screen cracked"}, + "questions": { + "damaged": { + "type": "noul", + "instructions": "Is the item damaged?", + "criteria": {"true": "Visible damage", "false": None}, + "provider_field": "dropped", + }, + "rubric_only": {"type": "noul", "criteria": {"true": {"signal": "refund"}}}, + "action": { + "type": "choice", + "instructions": {"policy": "refund-v2"}, + "criteria": {"refund": "Within 30 days", "escalate": None}, + }, + "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, + }, + } + ) + + assert to_openai_request("gpt-6-luna", request) == { + "model": "gpt-6-luna", + "input": '{"ticket": 1234, "text": "Screen cracked"}', + "questions": [ + { + "type": "predicate", + "name": "damaged", + "instructions": "Is the item damaged?\n\nAnswer true when: Visible damage", + }, + {"type": "predicate", "name": "rubric_only", "instructions": 'Answer true when: {"signal": "refund"}'}, + { + "type": "choice", + "name": "action", + "instructions": '{"policy": "refund-v2"}', + "choices": [{"value": "refund", "description": "Within 30 days"}, {"value": "escalate"}], + }, + {"type": "score", "name": "severity", "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}]}, + ], + } + + +def test_an_openai_response_becomes_systemone_answers_keyed_by_question_name_without_refusals() -> None: + response: Final = to_systemone_response( + { + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "damaged", "probability": 0.95}, + { + "type": "choice", + "name": "action", + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 1.72, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.0}, + {"value": 1, "label": "major", "probability": 0.28}, + {"value": 2, "label": "critical", "probability": 0.72}, + ], + "confidence": 0.58, + }, + {"type": "refusal", "name": "fraud"}, + ], + "usage": { + "input_tokens": 383, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 385, + }, + } + ) + + assert response.model_dump(mode="json") == { + "model": "gpt-6-luna", + "answers": { + "damaged": {"type": "noul", "noul": 0.95}, + "action": { + "type": "choice", + "choice": "true", + "confidence": 0.8, + "probabilities": {"true": 0.9, "escalate": 0.1}, + }, + "severity": { + "type": "score", + "score": 1.72, + "confidence": 0.58, + "legend": {"0": "minor", "1": "major", "2": "critical"}, + "probabilities": {"0": 0.0, "1": 0.28, "2": 0.72}, + }, + }, + "usage": {"input_tokens": 383, "output_tokens": 2}, + }