From b94bbbc3187ae2afa3141cc818a0de8b6a52c119 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:34:58 -0700 Subject: [PATCH] feat(decisions): add OpenAI as a Decisions provider System One requests to openai/ models are translated to OpenAI's Decisions shape on the way out and OpenAI's answers are translated back, so both /v1/systemone and /v1/decisions can route to gpt-6-luna. --- litellm/decisions/main.py | 13 +- litellm/decisions/openai_transformation.py | 98 ++++++++++++++ .../llms/base_llm/decisions/transformation.py | 18 +++ .../cloudflare/decisions/transformation.py | 5 + .../llms/openai/decisions/transformation.py | 37 ++++++ ...odel_prices_and_context_window_backup.json | 3 +- litellm/types/decisions.py | 5 +- model_prices_and_context_window.json | 3 +- tests/unit/decisions/test_main.py | 73 +++++++++++ .../decisions/test_openai_transformation.py | 103 ++++++++++++++- .../decisions_endpoints/test_endpoints.py | 121 ++++++++++++++---- 11 files changed, 443 insertions(+), 36 deletions(-) create mode 100644 litellm/llms/openai/decisions/transformation.py diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 8304c52a886..46b00f098ce 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT from litellm.llms.custom_httpx.http_handler import get_async_httpx_client, get_httpx_client +from litellm.llms.openai.decisions.transformation import OPENAI_DECISIONS_ENDPOINT from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT @@ -31,6 +32,7 @@ DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxy "openrouter": OPENROUTER_DECISIONS_ENDPOINT, "cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT, "strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT, + "openai": OPENAI_DECISIONS_ENDPOINT, } ) @@ -147,16 +149,7 @@ def _prepare_request( "Content-Type": "application/json", } ) - body: Final = MappingProxyType( - { - "model": endpoint.request_model(canonical_model), - "state": validated_request.state, - "questions": { - name: question.model_dump(mode="json", exclude_none=True) - for name, question in validated_request.questions.items() - }, - } - ) + body: Final = MappingProxyType(endpoint.request_body(endpoint.request_model(canonical_model), validated_request)) return _PreparedDecisionsRequest( config=endpoint, provider=provider, diff --git a/litellm/decisions/openai_transformation.py b/litellm/decisions/openai_transformation.py index af2c838b9ab..bdbfd7df0a1 100644 --- a/litellm/decisions/openai_transformation.py +++ b/litellm/decisions/openai_transformation.py @@ -1,14 +1,21 @@ +import json from collections.abc import Mapping, Sequence from typing import Final +from pydantic import TypeAdapter from typing_extensions import assert_never from litellm.types.decisions import ( ChoiceAnswer, + ChoiceQuestion, DecisionAnswer, + DecisionQuestion, + DecisionsJSON, + DecisionsRequestBody, DecisionsResponse, DecisionsUsage, NoulAnswer, + NoulQuestion, OpenAIChoiceAnswer, OpenAIChoiceProbability, OpenAIChoiceQuestion, @@ -26,10 +33,12 @@ from litellm.types.decisions import ( OpenAIScoreProbability, OpenAIScoreQuestion, ScoreAnswer, + ScoreQuestion, systemone_choice_key, ) _OPENAI_ONLY_FIELDS: Final = frozenset({"input", "questions", "safety_identifier"}) +_OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) def _message_text(message: OpenAIDecisionInputMessage) -> str: @@ -125,3 +134,92 @@ def to_openai_response( total_tokens=usage.input_tokens + usage.output_tokens, ), ) + + +def _text(value: DecisionsJSON) -> str: + return value if isinstance(value, str) else json.dumps(value) + + +def _instructions(instructions: DecisionsJSON | None) -> Mapping[str, str]: + return {} if instructions is None else {"instructions": _text(instructions)} + + +def _predicate_instructions(question: NoulQuestion) -> str: + instructions: Final = () if question.instructions is None else (_text(question.instructions),) + criteria: Final = tuple( + f"Answer {answer} when: {_text(rule)}" for answer, rule in (question.criteria or {}).items() if rule is not None + ) + return "\n\n".join((*instructions, *criteria)) + + +def _choice(value: str, description: DecisionsJSON | None) -> Mapping[str, str]: + return {"value": value} if description is None else {"value": value, "description": _text(description)} + + +def _openai_question(name: str, question: DecisionQuestion) -> Mapping[str, object]: + match question: + case NoulQuestion(): + return {"type": "predicate", "name": name, "instructions": _predicate_instructions(question)} + case ChoiceQuestion(): + return { + "type": "choice", + "name": name, + **_instructions(question.instructions), + "choices": [_choice(value, description) for value, description in question.criteria.items()], + } + case ScoreQuestion(): + return { + "type": "score", + "name": name, + **_instructions(question.instructions), + "levels": [{"label": _text(criterion)} for criterion in question.criteria], + } + case _: + assert_never(question) + + +def to_openai_request(model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return { + "model": model, + "input": _text(request.state), + "questions": [_openai_question(name, question) for name, question in request.questions.items()], + } + + +def _systemone_answer(answer: OpenAIDecisionAnswer) -> DecisionAnswer | None: + match answer: + case OpenAIPredicateAnswer(): + return NoulAnswer(type="noul", noul=answer.probability) + case OpenAIChoiceAnswer(): + return ChoiceAnswer( + type="choice", + choice=systemone_choice_key(answer.choice), + confidence=answer.confidence, + probabilities={systemone_choice_key(item.value): item.probability for item in answer.probabilities}, + ) + case OpenAIScoreAnswer(): + return ScoreAnswer( + type="score", + score=answer.score, + confidence=answer.confidence, + legend={str(item.value): item.label for item in answer.probabilities}, + probabilities={str(item.value): item.probability for item in answer.probabilities}, + ) + case OpenAIRefusalAnswer(): + return None + case _: + assert_never(answer) + + +def _named_answers(answers: Sequence[OpenAIDecisionAnswer]) -> Mapping[str, DecisionAnswer]: + translated: Final = ((answer.name, _systemone_answer(answer)) for answer in answers) + return {name: answer for name, answer in translated if name is not None and answer is not None} + + +def to_systemone_response(payload: object) -> DecisionsResponse: + response: Final = _OPENAI_RESPONSE_ADAPTER.validate_python(payload) + return DecisionsResponse( + model=response.model, + answers=_named_answers(response.answers), + usage=DecisionsUsage(input_tokens=response.usage.input_tokens, output_tokens=response.usage.output_tokens), + ) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index d4fcea24793..b001942a8ba 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -1,6 +1,19 @@ +from collections.abc import Mapping from dataclasses import dataclass from typing import Protocol +from litellm.types.decisions import DecisionsRequestBody + + +def systemone_request_body(model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return { + "model": model, + "state": request.state, + "questions": { + name: question.model_dump(mode="json", exclude_none=True) for name, question in request.questions.items() + }, + } + @dataclass(frozen=True, slots=True) class JevCompatibleDecisionsEndpoint: @@ -25,6 +38,9 @@ class JevCompatibleDecisionsEndpoint: def endpoint_url(self, api_base: str, model: str) -> str: return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" + def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return systemone_request_body(model, request) + def unwrap_response(self, payload: object) -> object: return payload @@ -49,4 +65,6 @@ class DecisionsProviderConfig(Protocol): def endpoint_url(self, api_base: str, model: str) -> str: ... + def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: ... + def unwrap_response(self, payload: object) -> object: ... diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 6e8b2999778..82c457630e6 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -4,10 +4,12 @@ from typing import Final from pydantic import TypeAdapter +from litellm.llms.base_llm.decisions.transformation import systemone_request_body from litellm.secret_managers.main import ( get_secret_str, normalize_nonempty_secret_str, ) +from litellm.types.decisions import DecisionsRequestBody _RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) @@ -43,6 +45,9 @@ class CloudflareDecisionsEndpoint: return f"{normalized_api_base}/{model}" return f"{normalized_api_base}/ai/run/{model}" + def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return systemone_request_body(model, request) + def unwrap_response(self, payload: object) -> object: if not isinstance(payload, Mapping): return payload diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py new file mode 100644 index 00000000000..7c7bee2b099 --- /dev/null +++ b/litellm/llms/openai/decisions/transformation.py @@ -0,0 +1,37 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from litellm.decisions.openai_transformation import to_openai_request, to_systemone_response +from litellm.types.decisions import DecisionsRequestBody, DecisionsResponse + + +@dataclass(frozen=True, slots=True) +class OpenAIDecisionsEndpoint: + api_key_env: tuple[str, ...] = ("OPENAI_API_KEY",) + api_base_env: str = "OPENAI_BASE_URL" + api_key_required: bool = True + + def default_api_base(self) -> str | None: + return "https://api.openai.com" + + def missing_api_base_message(self, provider: str) -> str: + return f"api_base is required for Decisions provider '{provider}'" + + def canonical_model(self, model: str) -> str: + return model + + def request_model(self, model: str) -> str: + return model + + def endpoint_url(self, api_base: str, model: str) -> str: + return f"{api_base.rstrip('/').removesuffix('/v1')}/v1/decisions" + + def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return to_openai_request(model, request) + + def unwrap_response(self, payload: object) -> DecisionsResponse: + return to_systemone_response(payload) + + +OPENAI_DECISIONS_ENDPOINT: Final[OpenAIDecisionsEndpoint] = OpenAIDecisionsEndpoint() diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c2aa83a9216..92a0c0a22c6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -34527,7 +34527,8 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", - "/v1/responses" + "/v1/responses", + "/v1/decisions" ], "supported_modalities": [ "text", diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 563329bb6e3..ff7d4dc8b12 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -260,7 +260,10 @@ class OpenAIRefusalAnswer(LiteLLMPydanticObjectBase): model_config = ConfigDict(frozen=True) -OpenAIDecisionAnswer: TypeAlias = OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIRefusalAnswer +OpenAIDecisionAnswer: TypeAlias = Annotated[ + OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIRefusalAnswer, + Field(discriminator="type"), +] class OpenAIDecisionInputTokensDetails(LiteLLMPydanticObjectBase): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c2aa83a9216..92a0c0a22c6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -34527,7 +34527,8 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", - "/v1/responses" + "/v1/responses", + "/v1/decisions" ], "supported_modalities": [ "text", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 684248b3561..beb3cba4c5a 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -79,6 +79,21 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( ), ) +_OPENAI_RESPONSE: Final[Mapping[str, object]] = { + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + {"type": "refusal", "name": "sentiment"}, + ], + "usage": { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + }, +} + class _RecordingLogger(CustomLogger): def __init__(self) -> None: @@ -599,3 +614,61 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( assert provider_resolution[:2] == ("strands-decider-2B-hobson-v19", "strands_decider") assert route.called assert response.model == _STRANDS_RESPONSE["model"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api_base", "url"), + ( + (None, "https://api.openai.com/v1/decisions"), + ("https://gateway.example/v1", "https://gateway.example/v1/decisions"), + ), +) +async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_and_back( + api_base: str | None, + url: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE) + + response: Final = await litellm.adecisions( + model="openai/gpt-6-luna", + state="The package arrived broken.", + questions={ + "is_defect": {"type": "noul", "instructions": "Is this a defect?"}, + "sentiment": { + "type": "choice", + "instructions": "How does the customer feel?", + "criteria": {"positive": None, "negative": "unhappy"}, + }, + }, + api_key="caller-key", + api_base=api_base, + ) + + assert route.called + request: Final = respx_mock.calls[0].request + assert request.headers["authorization"] == "Bearer caller-key" + assert json.loads(request.content) == { + "model": "gpt-6-luna", + "input": "The package arrived broken.", + "questions": [ + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "sentiment", + "instructions": "How does the customer feel?", + "choices": [{"value": "positive"}, {"value": "negative", "description": "unhappy"}], + }, + ], + } + assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} + assert response.hidden_params["custom_llm_provider"] == "openai" + luna_cost: Final = litellm.model_cost["gpt-6-luna"] + expected_cost: Final = _INPUT_TOKENS * float(luna_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + luna_cost["output_cost_per_token"] + ) + assert expected_cost > 0 + assert litellm.completion_cost(completion_response=response) == pytest.approx(expected_cost) diff --git a/tests/unit/decisions/test_openai_transformation.py b/tests/unit/decisions/test_openai_transformation.py index a3cc1583d40..381c2aac643 100644 --- a/tests/unit/decisions/test_openai_transformation.py +++ b/tests/unit/decisions/test_openai_transformation.py @@ -4,7 +4,7 @@ from typing import Final import pytest from pydantic import TypeAdapter, ValidationError -from litellm.decisions.openai_transformation import to_systemone_request +from litellm.decisions.openai_transformation import to_openai_request, to_systemone_request, to_systemone_response from litellm.types.decisions import MAX_DECISION_QUESTIONS, DecisionsRequestBody, OpenAIDecisionRequestBody _OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) @@ -30,3 +30,104 @@ def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_req def test_an_openai_request_with_more_questions_than_systemone_takes_is_rejected_before_translation() -> None: with pytest.raises(ValidationError, match="questions"): _OPENAI_BODY.validate_python(_openai_request(MAX_DECISION_QUESTIONS + 1)) + + +def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_their_keys() -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": {"ticket": 1234, "text": "Screen cracked"}, + "questions": { + "damaged": { + "type": "noul", + "instructions": "Is the item damaged?", + "criteria": {"true": "Visible damage", "false": None}, + "provider_field": "dropped", + }, + "rubric_only": {"type": "noul", "criteria": {"true": {"signal": "refund"}}}, + "action": { + "type": "choice", + "instructions": {"policy": "refund-v2"}, + "criteria": {"refund": "Within 30 days", "escalate": None}, + }, + "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, + }, + } + ) + + assert to_openai_request("gpt-6-luna", request) == { + "model": "gpt-6-luna", + "input": '{"ticket": 1234, "text": "Screen cracked"}', + "questions": [ + { + "type": "predicate", + "name": "damaged", + "instructions": "Is the item damaged?\n\nAnswer true when: Visible damage", + }, + {"type": "predicate", "name": "rubric_only", "instructions": 'Answer true when: {"signal": "refund"}'}, + { + "type": "choice", + "name": "action", + "instructions": '{"policy": "refund-v2"}', + "choices": [{"value": "refund", "description": "Within 30 days"}, {"value": "escalate"}], + }, + {"type": "score", "name": "severity", "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}]}, + ], + } + + +def test_an_openai_response_becomes_systemone_answers_keyed_by_question_name_without_refusals() -> None: + response: Final = to_systemone_response( + { + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "damaged", "probability": 0.95}, + { + "type": "choice", + "name": "action", + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 1.72, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.0}, + {"value": 1, "label": "major", "probability": 0.28}, + {"value": 2, "label": "critical", "probability": 0.72}, + ], + "confidence": 0.58, + }, + {"type": "refusal", "name": "fraud"}, + ], + "usage": { + "input_tokens": 383, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 385, + }, + } + ) + + assert response.model_dump(mode="json") == { + "model": "gpt-6-luna", + "answers": { + "damaged": {"type": "noul", "noul": 0.95}, + "action": { + "type": "choice", + "choice": "true", + "confidence": 0.8, + "probabilities": {"true": 0.9, "escalate": 0.1}, + }, + "severity": { + "type": "score", + "score": 1.72, + "confidence": 0.58, + "legend": {"0": "minor", "1": "major", "2": "critical"}, + "probabilities": {"0": 0.0, "1": 0.28, "2": 0.72}, + }, + }, + "usage": {"input_tokens": 383, "output_tokens": 2}, + } diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index f61507e1d1d..9768d252c4a 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -320,6 +320,28 @@ _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST: Final[Mapping[str, object]] = { "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, } +_OPENAI_FORMAT_ANSWERS: Final = [ + {"type": "predicate", "name": "damaged", "probability": 0.95}, + { + "type": "choice", + "name": None, + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": "major", "probability": 0.7}, + ], + "confidence": 0.6, + }, + {"type": "refusal", "name": "fraud"}, +] + @pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions")) def test_openai_format_decisions_translate_through_systemone( @@ -354,27 +376,7 @@ def test_openai_format_decisions_translate_through_systemone( } body: Final = response.json() assert body["model"] == _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST["model"] - assert body["answers"] == [ - {"type": "predicate", "name": "damaged", "probability": 0.95}, - { - "type": "choice", - "name": None, - "choice": True, - "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], - "confidence": 0.8, - }, - { - "type": "score", - "name": "severity", - "score": 0.7, - "probabilities": [ - {"value": 0, "label": "minor", "probability": 0.3}, - {"value": 1, "label": "major", "probability": 0.7}, - ], - "confidence": 0.6, - }, - {"type": "refusal", "name": "fraud"}, - ] + assert body["answers"] == _OPENAI_FORMAT_ANSWERS assert body["usage"] == { "input_tokens": _INPUT_TOKENS, "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, @@ -402,7 +404,9 @@ def test_openai_format_decisions_translate_through_systemone( }, { "model": "decider", - "input": [{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]}], + "input": [ + {"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]} + ], "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], }, ), @@ -437,6 +441,79 @@ def test_a_body_that_is_not_json_is_a_client_error( assert not upstream.called +def test_openai_format_decisions_reach_an_openai_deployment_with_the_callers_questions( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[{"model_name": "decider", "litellm_params": {"model": "openai/gpt-6-luna", "api_key": "k"}}] + ), + ) + upstream: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond( + json={ + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "0", "probability": 0.95}, + { + "type": "choice", + "name": "1", + "choice": "true", + "probabilities": [{"value": "true", "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "2", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": "major: Product unusable", "probability": 0.7}, + ], + "confidence": 0.6, + }, + {"type": "refusal", "name": "3"}, + ], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS, "total_tokens": 370}, + } + ) + + response: Final = client.post("/v1/decisions", json=_OPENAI_FORMAT_REQUEST) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "The package arrived with a broken screen.\n\nI want a refund.\n\nOrder 1234.", + "questions": [ + {"type": "predicate", "name": "0", "instructions": "Does the customer report a damaged item?"}, + { + "type": "choice", + "name": "1", + "instructions": "Should we refund?", + "choices": [{"value": "true", "description": "Refund now"}, {"value": "escalate"}], + }, + { + "type": "score", + "name": "2", + "instructions": "How severe is the issue?", + "levels": [{"label": "minor"}, {"label": "major: Product unusable"}], + }, + {"type": "predicate", "name": "3", "instructions": "Is this fraud?"}, + ], + } + assert response.json()["answers"] == _OPENAI_FORMAT_ANSWERS + luna_cost: Final = litellm.model_cost["gpt-6-luna"] + expected_cost: Final = _INPUT_TOKENS * float(luna_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + luna_cost["output_cost_per_token"] + ) + assert expected_cost > 0 + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) + + def _decisions_feature() -> LazyFeature: return next(feature for feature in LAZY_FEATURES if feature.name == "decisions")