From 54ee06fce585f5f4e47f48ee332f918539bf2b62 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:42:28 -0700 Subject: [PATCH 01/18] feat(decisions): add the OpenAI Decisions spec types and the System One translation (#45129) * feat(decisions): add the OpenAI Decisions spec types and the System One translation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): share one DecisionsModel config and require model and usage on responses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): rename the shared pydantic parent to DecisionsObjectBase Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): match the SDK on strict choice values and drop the invented min_length bounds Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): compose DecisionsRequest from model and body and read the translator top down Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): use the plural Decisions prefix only for the request and response envelopes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): import assert_never from typing_extensions for Python 3.10 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 79f62db62023f1cd60b7241c40b08e8c7151a1e6) --- litellm/llms/base_llm/decisions/systemone.py | 247 +++++++++++++++++ litellm/types/openai_decisions.py | 162 +++++++++++ .../llms/base_llm/decisions/test_systemone.py | 262 ++++++++++++++++++ tests/unit/types/test_openai_decisions.py | 150 ++++++++++ 4 files changed, 821 insertions(+) create mode 100644 litellm/llms/base_llm/decisions/systemone.py create mode 100644 litellm/types/openai_decisions.py create mode 100644 tests/unit/llms/base_llm/decisions/test_systemone.py create mode 100644 tests/unit/types/test_openai_decisions.py diff --git a/litellm/llms/base_llm/decisions/systemone.py b/litellm/llms/base_llm/decisions/systemone.py new file mode 100644 index 00000000000..43be31b0c30 --- /dev/null +++ b/litellm/llms/base_llm/decisions/systemone.py @@ -0,0 +1,247 @@ +"""The Jev / System One wire shape and its translation to and from the OpenAI Decisions shape. + +System One (TypeSafe, Perplexity, OpenRouter, Cloudflare Clef, Strands Decider) takes +{"model", "state", "questions": {name: question}} and answers with {"model", "answers": {name: answer}, "usage"}. +Predicates are `noul` questions, choice options are a `criteria` map, score levels are a `criteria` list. +""" + +import itertools +from collections.abc import Mapping, Sequence +from typing import Final, Literal, TypeAlias + +from pydantic import ConfigDict, TypeAdapter +from typing_extensions import assert_never + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.base import LiteLLMPydanticObjectBase +from litellm.types.openai_decisions import ( + ChoiceAnswer, + ChoiceProbability, + ChoiceQuestion, + DecisionAnswer, + DecisionChoice, + DecisionInput, + DecisionInputMessage, + DecisionInputPart, + DecisionInputTokensDetails, + DecisionOutputTokensDetails, + DecisionQuestion, + DecisionsRequest, + DecisionsRequestBody, + DecisionsResponse, + DecisionUsage, + PredicateAnswer, + PredicateQuestion, + ScoreAnswer, + ScoreProbability, + ScoreQuestion, +) + + +class SystemOneObjectBase(LiteLLMPydanticObjectBase): + model_config = ConfigDict(extra="allow", frozen=True) + + +class SystemOneNoulAnswer(SystemOneObjectBase): + type: Literal["noul"] + noul: float + + +class SystemOneChoiceAnswer(SystemOneObjectBase): + type: Literal["choice"] + choice: str + confidence: float + probabilities: Mapping[str, float] + + +class SystemOneScoreAnswer(SystemOneObjectBase): + type: Literal["score"] + score: float + confidence: float + probabilities: Mapping[str, float] + + +SystemOneAnswer: TypeAlias = SystemOneNoulAnswer | SystemOneChoiceAnswer | SystemOneScoreAnswer + + +class SystemOneUsage(SystemOneObjectBase): + input_tokens: int = 0 + output_tokens: int = 0 + + +class SystemOneResponse(SystemOneObjectBase): + model: str | None = None + answers: Mapping[str, SystemOneAnswer] + usage: SystemOneUsage | None = None + + +SYSTEM_ONE_RESPONSE_ADAPTER: Final[TypeAdapter[SystemOneResponse]] = TypeAdapter(SystemOneResponse) + + +def _unsupported(what: str, custom_llm_provider: str) -> BaseLLMException: + return BaseLLMException( + status_code=400, + message=f"Decisions provider '{custom_llm_provider}' does not support {what}", + ) + + +def to_system_one_request(model: str, body: DecisionsRequestBody, custom_llm_provider: str) -> dict[str, object]: + keys: Final = question_keys(body.questions, custom_llm_provider) + return { + "model": model, + "state": _state(body.input, custom_llm_provider), + "questions": { + key: _question(question, custom_llm_provider) for key, question in zip(keys, body.questions, strict=True) + }, + } + + +def question_keys(questions: Sequence[DecisionQuestion], custom_llm_provider: str) -> tuple[str, ...]: + """System One keys questions and answers by name, so unnamed questions get a positional key.""" + names: Final = tuple(question.name for question in questions if question.name is not None) + if len(set(names)) != len(names): + raise BaseLLMException( + status_code=400, + message=f"Decisions provider '{custom_llm_provider}' requires a unique name per question", + ) + taken: Final = frozenset(names) + return tuple( + question.name if question.name is not None else _positional_key(index, taken) + for index, question in enumerate(questions) + ) + + +def _positional_key(index: int, taken: frozenset[str]) -> str: + candidates: Final = (f"{'_' * depth}q{index}" for depth in itertools.count()) + return next(key for key in candidates if key not in taken) + + +def _state(input_value: DecisionInput, custom_llm_provider: str) -> str: + if isinstance(input_value, str): + return input_value + return "\n".join(_message_text(message, custom_llm_provider) for message in input_value) + + +def _message_text(message: DecisionInputMessage, custom_llm_provider: str) -> str: + if isinstance(message.content, str): + return message.content + return "\n".join(_part_text(part, custom_llm_provider) for part in message.content) + + +def _part_text(part: DecisionInputPart, custom_llm_provider: str) -> str: + if part.type != "input_text": + raise _unsupported("input_image parts", custom_llm_provider) + return part.text + + +def _question(question: DecisionQuestion, custom_llm_provider: str) -> dict[str, object]: + match question: + case PredicateQuestion(): + return {"type": "noul", "instructions": question.instructions} + case ChoiceQuestion(): + return { + "type": "choice", + "instructions": question.instructions, + "criteria": _choice_criteria(question, custom_llm_provider), + } + case ScoreQuestion(): + return { + "type": "score", + "instructions": question.instructions, + "criteria": [_level_text(level.label, level.description) for level in question.levels], + } + case _: + assert_never(question) + + +def _choice_criteria(question: ChoiceQuestion, custom_llm_provider: str) -> dict[str, str | None]: + values: Final = tuple(_choice_key(choice, custom_llm_provider) for choice in question.choices) + if len(set(values)) != len(values): + raise _unsupported("repeated choice values", custom_llm_provider) + return {value: choice.description for value, choice in zip(values, question.choices, strict=True)} + + +def _choice_key(choice: DecisionChoice, custom_llm_provider: str) -> str: + if not isinstance(choice.value, str): + raise _unsupported("boolean choice values", custom_llm_provider) + return choice.value + + +def _level_text(label: str, description: str | None) -> str: + return description if description is not None else label + + +def to_decisions_response( + system_one: SystemOneResponse, + request: DecisionsRequest, + custom_llm_provider: str, +) -> DecisionsResponse: + keys: Final = question_keys(request.body.questions, custom_llm_provider) + return DecisionsResponse( + model=system_one.model if system_one.model is not None else request.model, + answers=[ + _answer(key, question, system_one.answers, custom_llm_provider) + for key, question in zip(keys, request.body.questions, strict=True) + ], + usage=_usage(system_one.usage), + ) + + +def _answer( + key: str, + question: DecisionQuestion, + answers: Mapping[str, SystemOneAnswer], + custom_llm_provider: str, +) -> DecisionAnswer: + match question, answers.get(key): + case PredicateQuestion(), SystemOneNoulAnswer() as answer: + return PredicateAnswer(type="predicate", name=question.name, probability=answer.noul) + case ChoiceQuestion(), SystemOneChoiceAnswer() as answer: + return ChoiceAnswer( + type="choice", + name=question.name, + choice=answer.choice, + probabilities=_choice_probabilities(question, answer), + confidence=answer.confidence, + ) + case ScoreQuestion(), SystemOneScoreAnswer() as answer: + return ScoreAnswer( + type="score", + name=question.name, + score=answer.score, + probabilities=_score_probabilities(question, answer), + confidence=answer.confidence, + ) + case _: + raise BaseLLMException( + status_code=500, + message=( + f"Decisions provider '{custom_llm_provider}' returned no {question.type} answer " + f"for question '{key}'" + ), + ) + + +def _choice_probabilities(question: ChoiceQuestion, answer: SystemOneChoiceAnswer) -> list[ChoiceProbability]: + return [ + ChoiceProbability(value=choice.value, probability=answer.probabilities.get(str(choice.value), 0.0)) + for choice in question.choices + ] + + +def _score_probabilities(question: ScoreQuestion, answer: SystemOneScoreAnswer) -> list[ScoreProbability]: + return [ + ScoreProbability(value=index, label=level.label, probability=answer.probabilities.get(str(index), 0.0)) + for index, level in enumerate(question.levels) + ] + + +def _usage(usage: SystemOneUsage | None) -> DecisionUsage: + counted: Final = usage if usage is not None else SystemOneUsage() + return DecisionUsage( + input_tokens=counted.input_tokens, + input_tokens_details=DecisionInputTokensDetails(cached_tokens=0, cache_write_tokens=0), + output_tokens=counted.output_tokens, + output_tokens_details=DecisionOutputTokensDetails(reasoning_tokens=0), + total_tokens=counted.input_tokens + counted.output_tokens, + ) diff --git a/litellm/types/openai_decisions.py b/litellm/types/openai_decisions.py new file mode 100644 index 00000000000..074c0a803f3 --- /dev/null +++ b/litellm/types/openai_decisions.py @@ -0,0 +1,162 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Annotated, Literal, TypeAlias + +from pydantic import ConfigDict, Field, PrivateAttr, StrictBool, StrictStr + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +ChoiceValue: TypeAlias = StrictStr | StrictBool + + +class DecisionsObjectBase(LiteLLMPydanticObjectBase): + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionInputText(DecisionsObjectBase): + type: Literal["input_text"] + text: str + + +class DecisionInputImage(DecisionsObjectBase): + type: Literal["input_image"] + image_url: str + detail: Literal["low", "high", "auto", "original"] | None = None + + +DecisionInputPart: TypeAlias = Annotated[DecisionInputText | DecisionInputImage, Field(discriminator="type")] + + +class DecisionInputMessage(DecisionsObjectBase): + role: Literal["user"] + content: str | Sequence[DecisionInputPart] + type: Literal["message"] | None = None + + +DecisionInput: TypeAlias = str | Sequence[DecisionInputMessage] + + +class DecisionChoice(DecisionsObjectBase): + value: ChoiceValue + description: str | None = None + + +class DecisionLevel(DecisionsObjectBase): + label: str + description: str | None = None + + +class PredicateQuestion(DecisionsObjectBase): + type: Literal["predicate"] + instructions: str + name: str | None = None + + +class ChoiceQuestion(DecisionsObjectBase): + type: Literal["choice"] + instructions: str + choices: Sequence[DecisionChoice] + name: str | None = None + + +class ScoreQuestion(DecisionsObjectBase): + type: Literal["score"] + instructions: str + levels: Sequence[DecisionLevel] + name: str | None = None + + +DecisionQuestion: TypeAlias = Annotated[ + PredicateQuestion | ChoiceQuestion | ScoreQuestion, + Field(discriminator="type"), +] + +DecisionQuestions: TypeAlias = Sequence[DecisionQuestion] + + +class DecisionsRequestBody(DecisionsObjectBase): + input: DecisionInput + questions: DecisionQuestions + safety_identifier: str | None = None + + +@dataclass(frozen=True, slots=True) +class DecisionsRequest: + model: str + body: DecisionsRequestBody + + +class PredicateAnswer(DecisionsObjectBase): + type: Literal["predicate"] + name: str | None = None + probability: float + + +class ChoiceProbability(DecisionsObjectBase): + value: ChoiceValue + probability: float + + +class ChoiceAnswer(DecisionsObjectBase): + type: Literal["choice"] + name: str | None = None + choice: ChoiceValue + probabilities: Sequence[ChoiceProbability] + confidence: float + + +class ScoreProbability(DecisionsObjectBase): + value: int + label: str + probability: float + + +class ScoreAnswer(DecisionsObjectBase): + type: Literal["score"] + name: str | None = None + score: float + probabilities: Sequence[ScoreProbability] + confidence: float + + +class RefusalAnswer(DecisionsObjectBase): + type: Literal["refusal"] + name: str | None = None + + +DecisionAnswer: TypeAlias = Annotated[ + PredicateAnswer | ChoiceAnswer | ScoreAnswer | RefusalAnswer, + Field(discriminator="type"), +] + + +class DecisionInputTokensDetails(DecisionsObjectBase): + cached_tokens: int + cache_write_tokens: int + + +class DecisionOutputTokensDetails(DecisionsObjectBase): + reasoning_tokens: int + + +class DecisionUsage(DecisionsObjectBase): + input_tokens: int + input_tokens_details: DecisionInputTokensDetails + output_tokens: int + output_tokens_details: DecisionOutputTokensDetails + total_tokens: int + + +class DecisionsResponse(DecisionsObjectBase): + model: str + answers: Sequence[DecisionAnswer] + usage: DecisionUsage + + _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) + + @property + def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation + return self._hidden_params + + def set_hidden_params(self, params: Mapping[str, object]) -> None: + self._hidden_params.update(params) diff --git a/tests/unit/llms/base_llm/decisions/test_systemone.py b/tests/unit/llms/base_llm/decisions/test_systemone.py new file mode 100644 index 00000000000..42a1c41b3d3 --- /dev/null +++ b/tests/unit/llms/base_llm/decisions/test_systemone.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.decisions.systemone import ( + SYSTEM_ONE_RESPONSE_ADAPTER, + question_keys, + to_decisions_response, + to_system_one_request, +) +from litellm.types.openai_decisions import ( + ChoiceAnswer, + DecisionsRequest, + DecisionsRequestBody, + PredicateAnswer, + ScoreAnswer, +) + +_INPUT: Final = "The export job hangs at 99% and never finishes" +_QUESTIONS: Final[Sequence[Mapping[str, object]]] = ( + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "sentiment", + "instructions": "How does the customer feel?", + "choices": [{"value": "positive"}, {"value": "negative", "description": "unhappy"}], + }, + { + "type": "score", + "name": "severity", + "instructions": "How severe is it?", + "levels": [{"label": "none"}, {"label": "low"}, {"label": "high", "description": "blocks users"}], + }, +) +_SYSTEM_ONE_QUESTIONS: Final[Mapping[str, object]] = { + "is_defect": {"type": "noul", "instructions": "Is this a defect?"}, + "sentiment": { + "type": "choice", + "instructions": "How does the customer feel?", + "criteria": {"positive": None, "negative": "unhappy"}, + }, + "severity": {"type": "score", "instructions": "How severe is it?", "criteria": ["none", "low", "blocks users"]}, +} +_SYSTEM_ONE_RESPONSE: Final[Mapping[str, object]] = { + "model": "jev-1.13", + "answers": { + "is_defect": {"type": "noul", "noul": 0.9}, + "sentiment": { + "type": "choice", + "choice": "positive", + "confidence": 0.8, + "probabilities": {"positive": 0.8, "negative": 0.2}, + }, + "severity": { + "type": "score", + "score": 1, + "confidence": 0.7, + "legend": {"0": "none", "1": "low", "2": "high"}, + "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}, + }, + }, + "usage": {"input_tokens": 367, "output_tokens": 3}, +} +_EXPECTED_ANSWERS: Final[Sequence[Mapping[str, object]]] = ( + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + { + "type": "choice", + "name": "sentiment", + "choice": "positive", + "probabilities": [{"value": "positive", "probability": 0.8}, {"value": "negative", "probability": 0.2}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 1.0, + "probabilities": [ + {"value": 0, "label": "none", "probability": 0.1}, + {"value": 1, "label": "low", "probability": 0.8}, + {"value": 2, "label": "high", "probability": 0.1}, + ], + "confidence": 0.7, + }, +) +_EXPECTED_USAGE: Final[Mapping[str, object]] = { + "input_tokens": 367, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 370, +} +_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) + + +def _body( + input_value: object = _INPUT, + questions: Sequence[Mapping[str, object]] = _QUESTIONS, +) -> DecisionsRequestBody: + return _BODY_ADAPTER.validate_python({"input": input_value, "questions": questions}) + + +def _request( + input_value: object = _INPUT, + questions: Sequence[Mapping[str, object]] = _QUESTIONS, + model: str = "jev-1.13", +) -> DecisionsRequest: + return DecisionsRequest(model=model, body=_body(input_value, questions)) + + +def _predicate(name: str | None = "is_defect") -> tuple[Mapping[str, object]]: + return ({"type": "predicate", "name": name, "instructions": "Is this a defect?"},) + + +def test_openai_request_becomes_the_system_one_body() -> None: + assert to_system_one_request("jev-1.13", _body(), "typesafe") == { + "model": "jev-1.13", + "state": _INPUT, + "questions": _SYSTEM_ONE_QUESTIONS, + } + + +def test_system_one_answers_become_openai_answers_in_question_order() -> None: + response: Final = to_decisions_response( + SYSTEM_ONE_RESPONSE_ADAPTER.validate_python(_SYSTEM_ONE_RESPONSE), _request(), "typesafe" + ) + + assert response.model_dump(mode="json") == { + "model": "jev-1.13", + "answers": list(_EXPECTED_ANSWERS), + "usage": _EXPECTED_USAGE, + } + assert isinstance(response.answers[0], PredicateAnswer) + assert isinstance(response.answers[1], ChoiceAnswer) + assert isinstance(response.answers[2], ScoreAnswer) + + +def test_user_messages_are_joined_into_one_system_one_state() -> None: + messages: Final = [ + {"role": "user", "content": "first"}, + { + "role": "user", + "content": [{"type": "input_text", "text": "second"}, {"type": "input_text", "text": "third"}], + }, + ] + + body: Final = to_system_one_request("jev-1.13", _body(messages, _predicate()), "typesafe") + + assert body["state"] == "first\nsecond\nthird" + + +@pytest.mark.parametrize( + ("label", "input_value", "questions"), + ( + ( + "input_image", + [{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]}], + _predicate(), + ), + ("boolean choice", _INPUT, [{"type": "choice", "instructions": "Refund?", "choices": [{"value": True}]}]), + ("unique name", _INPUT, [*_predicate(), *_predicate()]), + ( + "repeated choice", + _INPUT, + [{"type": "choice", "instructions": "Refund?", "choices": [{"value": "yes"}, {"value": "yes"}]}], + ), + ), +) +def test_what_system_one_cannot_express_is_a_400( + label: str, + input_value: object, + questions: Sequence[Mapping[str, object]], +) -> None: + with pytest.raises(BaseLLMException, match=label) as error: + to_system_one_request("jev-1.13", _body(input_value, questions), "perplexity") + + assert error.value.status_code == 400 + assert "perplexity" in error.value.message + + +def test_unnamed_questions_get_positional_keys_that_never_shadow_a_supplied_name() -> None: + body: Final = _body(questions=[*_predicate(None), *_predicate("q0"), *_predicate(None)]) + + assert question_keys(body.questions, "typesafe") == ("_q0", "q0", "q2") + noul: Final = {"type": "noul", "instructions": "Is this a defect?"} + assert to_system_one_request("jev-1.13", body, "typesafe")["questions"] == {"_q0": noul, "q0": noul, "q2": noul} + + +def test_positional_answers_come_back_in_question_order_without_a_name() -> None: + request: Final = _request(questions=[*_predicate(None), *_predicate("q0")]) + system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python( + {"answers": {"_q0": {"type": "noul", "noul": 0.25}, "q0": {"type": "noul", "noul": 0.75}}} + ) + + response: Final = to_decisions_response(system_one, request, "typesafe") + + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": None, "probability": 0.25}, + {"type": "predicate", "name": "q0", "probability": 0.75}, + ] + assert response.usage.model_dump(mode="json") == { + **_EXPECTED_USAGE, + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + } + + +def test_a_reply_without_a_model_reports_the_requested_model() -> None: + system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python( + {k: v for k, v in _SYSTEM_ONE_RESPONSE.items() if k != "model"} + ) + + response: Final = to_decisions_response(system_one, _request(model="typesafe/jev-1.13.0"), "typesafe") + + assert response.model == "typesafe/jev-1.13.0" + + +def test_a_choice_the_provider_left_out_of_probabilities_is_reported_at_zero() -> None: + system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python( + { + "answers": { + "sentiment": { + "type": "choice", + "choice": "positive", + "confidence": 1.0, + "probabilities": {"positive": 1.0}, + } + } + } + ) + request: Final = _request(questions=_QUESTIONS[1:2]) + + response: Final = to_decisions_response(system_one, request, "typesafe") + + assert response.answers[0].model_dump(mode="json") == { + "type": "choice", + "name": "sentiment", + "choice": "positive", + "probabilities": [{"value": "positive", "probability": 1.0}, {"value": "negative", "probability": 0.0}], + "confidence": 1.0, + } + + +@pytest.mark.parametrize( + "answers", + ( + {}, + {"is_defect": {"type": "choice", "choice": "yes", "confidence": 1.0, "probabilities": {"yes": 1.0}}}, + ), +) +def test_a_reply_without_a_matching_answer_is_a_server_error(answers: Mapping[str, object]) -> None: + system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python({"answers": answers}) + + with pytest.raises(BaseLLMException, match="no predicate answer for question 'is_defect'") as error: + to_decisions_response(system_one, _request(questions=_predicate()), "typesafe") + + assert error.value.status_code == 500 diff --git a/tests/unit/types/test_openai_decisions.py b/tests/unit/types/test_openai_decisions.py new file mode 100644 index 00000000000..d9387d644d5 --- /dev/null +++ b/tests/unit/types/test_openai_decisions.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final + +import pytest +from pydantic import TypeAdapter, ValidationError + +from litellm.types.openai_decisions import ( + ChoiceAnswer, + DecisionsRequestBody, + DecisionsResponse, + RefusalAnswer, + ScoreQuestion, +) + +_REQUEST: Final[Mapping[str, object]] = { + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Is this receipt a valid business expense?"}, + {"type": "input_image", "image_url": "https://example.com/receipt.png", "detail": "high"}, + ], + } + ], + "questions": [ + {"type": "predicate", "name": "is_expense", "instructions": "Is this a business expense?"}, + { + "type": "choice", + "name": "approve", + "instructions": "Should this be approved?", + "choices": [{"value": True, "description": "approve"}, {"value": False, "description": "reject"}], + }, + { + "type": "score", + "name": "risk", + "instructions": "How risky is this expense?", + "levels": [{"label": "low"}, {"label": "high", "description": "needs a manager"}], + }, + ], + "safety_identifier": "user-123", +} +_RESPONSE: Final[Mapping[str, object]] = { + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "is_expense", "probability": 0.92}, + { + "type": "choice", + "name": "approve", + "choice": True, + "probabilities": [{"value": True, "probability": 0.7}, {"value": False, "probability": 0.3}], + "confidence": 0.7, + }, + {"type": "refusal", "name": "risk"}, + ], + "usage": { + "input_tokens": 120, + "input_tokens_details": {"cached_tokens": 100, "cache_write_tokens": 0}, + "output_tokens": 12, + "output_tokens_details": {"reasoning_tokens": 4}, + "total_tokens": 132, + }, +} +_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) + + +def test_the_documented_request_round_trips_with_its_boolean_choices_and_image_part() -> None: + request: Final = _REQUEST_ADAPTER.validate_python(_REQUEST) + + assert request.model_dump(mode="json", exclude_none=True) == _REQUEST + assert isinstance(request.questions[2], ScoreQuestion) + + +def test_the_documented_response_keeps_answer_order_refusals_and_token_details() -> None: + response: Final = _RESPONSE_ADAPTER.validate_python(_RESPONSE) + + assert response.model_dump(mode="json") == _RESPONSE + assert isinstance(response.answers[1], ChoiceAnswer) + assert isinstance(response.answers[2], RefusalAnswer) + + +_OFF_SPEC: Final[tuple[tuple[str, object], ...]] = ( + ("questions", [{"type": "noul", "name": "q", "instructions": "x"}]), + ("questions", [{"type": "choice", "name": "q", "instructions": "x", "choices": [{"value": 1}]}]), + ("questions", [{"type": "score", "name": "q", "levels": [{"label": "low"}]}]), + ("input", {"state": "not an OpenAI input"}), +) + + +@pytest.mark.parametrize(("field", "value"), _OFF_SPEC) +def test_requests_off_the_spec_are_rejected(field: str, value: object) -> None: + with pytest.raises(ValidationError): + _REQUEST_ADAPTER.validate_python({**_REQUEST, field: value}) + + +_EMPTY_COLLECTIONS: Final[tuple[tuple[str, list[object]], ...]] = ( + ("questions", []), + ("questions", [{"type": "choice", "name": "q", "instructions": "x", "choices": []}]), + ("questions", [{"type": "score", "name": "q", "instructions": "x", "levels": []}]), +) + + +@pytest.mark.parametrize(("field", "value"), _EMPTY_COLLECTIONS) +def test_empty_collections_are_left_for_the_provider_to_judge(field: str, value: list[object]) -> None: + request: Final = _REQUEST_ADAPTER.validate_python({**_REQUEST, field: value}) + + assert request.model_dump(mode="json", exclude_none=True)[field] == value + + +def test_choice_values_keep_their_type_so_a_string_true_and_a_boolean_true_stay_distinct() -> None: + answer: Final = { + "type": "choice", + "name": "approve", + "choice": "true", + "probabilities": [{"value": "true", "probability": 0.6}, {"value": True, "probability": 0.4}], + "confidence": 0.6, + } + + response: Final = _RESPONSE_ADAPTER.validate_python({**_RESPONSE, "answers": [answer]}) + + assert response.model_dump(mode="json")["answers"] == [answer] + with pytest.raises(ValidationError): + _RESPONSE_ADAPTER.validate_python({**_RESPONSE, "answers": [{**answer, "choice": 1}]}) + + +_OFF_SPEC_RESPONSE: Final[tuple[str, ...]] = ("model", "usage") + + +@pytest.mark.parametrize("field", _OFF_SPEC_RESPONSE) +def test_responses_missing_a_required_field_are_rejected(field: str) -> None: + with pytest.raises(ValidationError): + _RESPONSE_ADAPTER.validate_python({k: v for k, v in _RESPONSE.items() if k != field}) + + +def test_usage_without_token_details_is_rejected() -> None: + usage: Final = {"input_tokens": 120, "output_tokens": 12, "total_tokens": 132} + + with pytest.raises(ValidationError): + _RESPONSE_ADAPTER.validate_python({**_RESPONSE, "usage": usage}) + + +def test_hidden_params_live_outside_the_wire_body() -> None: + response: Final = _RESPONSE_ADAPTER.validate_python(_RESPONSE) + + response.set_hidden_params({"custom_llm_provider": "openai"}) + + assert response.hidden_params == {"custom_llm_provider": "openai"} + assert "_hidden_params" not in response.model_dump(mode="json") From ee13623808eeac365f44c8a06166afedc07d6e66 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:45:45 -0700 Subject: [PATCH 02/18] refactor(decisions): dispatch /v1/decisions through provider configs and the shared HTTP handler (#45130) * refactor(decisions): dispatch /v1/decisions through provider configs and the shared HTTP handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): map OpenRouter connection failures to APIConnectionError and drop explanatory docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): keep APIConnectionError for unreachable deployments on every provider `_map_upstream_exception` now maps a status `_handle_error` synthesized for a non-HTTP failure to `APIConnectionError` before `exception_type` runs, so a Perplexity (openai-compatible) deployment that cannot be reached no longer answers `InternalServerError` where main answers `APIConnectionError`. Audit cells: the unreachable-deployment integration cell is parametrized over the five providers, OpenRouter connection failures are pinned on chat, stream, embeddings, decisions and the SDK, a deployment whose provider has no decisions config gets the 400 naming every supported provider, and a unit cell pins the mapper alone. The first docstring sentence of the five translation bases is dropped. --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> (cherry picked from commit 0e09e991468d174a528a8c3e7a90a3616d4bbb27) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 18 + litellm/_lazy_imports_registry.py | 15 + litellm/decisions/main.py | 344 ++++++++---------- .../exception_mapping_utils.py | 5 +- litellm/llms/base_llm/decisions/__init__.py | 4 +- .../llms/base_llm/decisions/transformation.py | 105 +++--- .../cloudflare/decisions/transformation.py | 57 +-- litellm/llms/custom_httpx/llm_http_handler.py | 97 +++++ .../llms/openai/decisions/transformation.py | 44 +-- .../openrouter/decisions/transformation.py | 16 +- .../perplexity/decisions/transformation.py | 16 +- .../decisions/transformation.py | 14 +- .../llms/typesafe/decisions/transformation.py | 15 +- litellm/types/decisions.py | 7 + litellm/utils.py | 17 + .../providers/test_decisions_wire.py | 86 ++++- tests/integration/proxy_config.yaml | 50 ++- .../translation/decisions/__init__.py | 0 .../translation/decisions/bases/__init__.py | 0 .../translation/decisions/bases/cloudflare.py | 84 +++++ .../translation/decisions/bases/openrouter.py | 83 +++++ .../translation/decisions/bases/perplexity.py | 79 ++++ .../decisions/bases/strands_decider.py | 35 ++ .../translation/decisions/bases/typesafe.py | 79 ++++ .../translation/decisions/basic/__init__.py | 0 .../basic/test_decisions_basic_cloudflare.py | 11 + .../basic/test_decisions_basic_openrouter.py | 11 + .../basic/test_decisions_basic_perplexity.py | 11 + .../test_decisions_basic_strands_decider.py | 11 + .../basic/test_decisions_basic_typesafe.py | 11 + tests/unit/decisions/test_main.py | 103 +++++- .../test_exception_mapping_utils.py | 28 ++ .../test_openai_decisions_transformation.py | 8 +- 33 files changed, 1086 insertions(+), 378 deletions(-) create mode 100644 tests/integration/translation/decisions/__init__.py create mode 100644 tests/integration/translation/decisions/bases/__init__.py create mode 100644 tests/integration/translation/decisions/bases/cloudflare.py create mode 100644 tests/integration/translation/decisions/bases/openrouter.py create mode 100644 tests/integration/translation/decisions/bases/perplexity.py create mode 100644 tests/integration/translation/decisions/bases/strands_decider.py create mode 100644 tests/integration/translation/decisions/bases/typesafe.py create mode 100644 tests/integration/translation/decisions/basic/__init__.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py diff --git a/litellm/__init__.py b/litellm/__init__.py index d576ea03c94..49a64d89376 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1654,6 +1654,24 @@ if TYPE_CHECKING: from .llms.hosted_vllm.rerank.transformation import ( HostedVLLMRerankConfig as HostedVLLMRerankConfig, ) + from .llms.perplexity.decisions.transformation import ( + PerplexityDecisionsConfig as PerplexityDecisionsConfig, + ) + from .llms.typesafe.decisions.transformation import ( + TypeSafeDecisionsConfig as TypeSafeDecisionsConfig, + ) + from .llms.openrouter.decisions.transformation import ( + OpenRouterDecisionsConfig as OpenRouterDecisionsConfig, + ) + from .llms.cloudflare.decisions.transformation import ( + CloudflareDecisionsConfig as CloudflareDecisionsConfig, + ) + from .llms.strands_decider.decisions.transformation import ( + StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig, + ) + from .llms.openai.decisions.transformation import ( + OpenAIDecisionsConfig as OpenAIDecisionsConfig, + ) from .llms.nvidia_nim.rerank.transformation import ( NvidiaNimRerankConfig as NvidiaNimRerankConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index ffcbffb05b6..2e68fb3459b 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -153,6 +153,12 @@ LLM_CONFIG_NAMES: Final = ( "JinaAIRerankConfig", "DeepinfraRerankConfig", "HostedVLLMRerankConfig", + "PerplexityDecisionsConfig", + "TypeSafeDecisionsConfig", + "OpenRouterDecisionsConfig", + "CloudflareDecisionsConfig", + "StrandsDeciderDecisionsConfig", + "OpenAIDecisionsConfig", "NvidiaNimRerankConfig", "NvidiaNimRankingConfig", "VertexAIRerankConfig", @@ -696,6 +702,15 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.hosted_vllm.rerank.transformation", "HostedVLLMRerankConfig", ), + "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"), + "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"), + "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), + "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"), + "StrandsDeciderDecisionsConfig": ( + ".llms.strands_decider.decisions.transformation", + "StrandsDeciderDecisionsConfig", + ), + "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "NvidiaNimRerankConfig": ( ".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig", diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 01fc149da3c..25ec501fd48 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,6 +1,5 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, field -from types import MappingProxyType from typing import Final, TypeAlias import httpx @@ -9,22 +8,14 @@ from typing_extensions import assert_never import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( - DecisionsProviderConfig, + BaseDecisionsConfig, ir_to_systemone_response, systemone_request_to_ir, ) -from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT -from litellm.llms.custom_httpx.http_handler import _get_httpx_client, get_async_httpx_client -from litellm.llms.openai.decisions.transformation import ( - OPENAI_DECISIONS_ENDPOINT, - ir_to_openai_response, - openai_request_to_ir, -) -from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT -from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT -from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT -from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_ENDPOINT +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir from litellm.types.decisions import ( DecisionQuestion, DecisionsIRRequest, @@ -38,18 +29,8 @@ from litellm.types.decisions import ( OpenAIDecisionResponse, UnsupportedDecisionsRequest, ) -from litellm.utils import client - -DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxyType( - { - "perplexity": PERPLEXITY_DECISIONS_ENDPOINT, - "typesafe": TYPESAFE_DECISIONS_ENDPOINT, - "openrouter": OPENROUTER_DECISIONS_ENDPOINT, - "cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT, - "strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT, - "openai": OPENAI_DECISIONS_ENDPOINT, - } -) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, client DecisionsQuestions: TypeAlias = ( Mapping[str, DecisionQuestion | Mapping[str, object]] | Sequence[OpenAIDecisionQuestion | Mapping[str, object]] @@ -58,62 +39,48 @@ DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequest _SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) _OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) -_DECISIONS_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) +_HANDLER: Final = BaseLLMHTTPHandler() @dataclass(frozen=True, slots=True, repr=False) -class _PreparedDecisionsRequest: - config: DecisionsProviderConfig - provider: str - upstream_model: str - url: str - api_key: str | None = field(repr=False) - headers: Mapping[str, str] = field(repr=False) +class _DecisionsCall: + model: str + requested_model: str + custom_llm_provider: str + provider_config: BaseDecisionsConfig + request: DecisionsRequestFormat + ir_request: DecisionsIRRequest body: Mapping[str, object] = field(repr=False) - request: DecisionsRequestFormat = field(repr=False) - ir_request: DecisionsIRRequest = field(repr=False) + api_base: str + api_key: str | None = field(repr=False) + logging_obj: LiteLLMLoggingObj | None + headers: Mapping[str, str] + timeout: float | httpx.Timeout | None -def _resolve_provider_model(model: str, custom_llm_provider: str | None) -> tuple[str, str]: - provider: Final = model.partition("/")[0] if custom_llm_provider is None else custom_llm_provider - if provider not in DECISIONS_ENDPOINTS: - supported: Final = ", ".join(DECISIONS_ENDPOINTS) - raise litellm.BadRequestError( - message=f"Unknown Decisions provider '{provider}'. Supported providers: {supported}", - model=model, - llm_provider=provider, - ) - upstream_model: Final = model.removeprefix(f"{provider}/") if model.startswith(f"{provider}/") else model - if not upstream_model: - raise litellm.BadRequestError( - message="A model name is required for the Decisions API", - model=model, - llm_provider=provider, - ) - return provider, upstream_model - - -def _resolve_api_key( - *, - provider: str, - model: str, - endpoint: DecisionsProviderConfig, - api_key: str | None, -) -> str | None: - if api_key is not None: - return api_key - configured_api_key: Final = endpoint.configured_api_key() - if configured_api_key: - return configured_api_key - if not endpoint.api_key_required: - return None - raise litellm.AuthenticationError( - message=f"Missing API key for Decisions provider '{provider}'", - model=model, - llm_provider=provider, +def _supported_providers() -> tuple[str, ...]: + return tuple( + provider.value + for provider in LlmProviders + if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None ) +def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig: + provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None) + provider_config: Final = ( + None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider) + ) + if provider_config is None: + supported: Final = ", ".join(_supported_providers()) + raise litellm.BadRequestError( + message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}", + model=model, + llm_provider=custom_llm_provider, + ) + return provider_config + + def _validate_request( *, state: DecisionsJSON | None, @@ -138,7 +105,7 @@ def _ir_request(request: DecisionsRequestFormat) -> DecisionsIRRequest: assert_never(request) -def _prepare_request( +def _prepare_call( *, model: str, state: DecisionsJSON | None, @@ -147,10 +114,25 @@ def _prepare_request( safety_identifier: str | None, api_key: str | None, api_base: str | None, + timeout: float | httpx.Timeout | None, custom_llm_provider: str | None, extra_headers: Mapping[str, str] | None, -) -> _PreparedDecisionsRequest: - provider, upstream_model = _resolve_provider_model(model, custom_llm_provider) + kwargs: Mapping[str, object], +) -> _DecisionsCall: + upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + provider_config: Final = _provider_config(upstream_model, provider) + canonical_model: Final = provider_config.canonical_model(upstream_model) + if not upstream_model: + raise litellm.BadRequestError( + message="A model name is required for the Decisions API", + model=model, + llm_provider=provider, + ) if state is not None and decision_input is not None: raise litellm.BadRequestError( message="Pass either state (System One format) or input (OpenAI format) to the Decisions API, not both", @@ -158,7 +140,7 @@ def _prepare_request( llm_provider=provider, ) try: - validated_request: Final = _validate_request( + request: Final = _validate_request( state=state, questions=questions, decision_input=decision_input, safety_identifier=safety_identifier ) except ValidationError as error: @@ -168,120 +150,94 @@ def _prepare_request( llm_provider=provider, ) from error - endpoint: Final = DECISIONS_ENDPOINTS[provider] - resolved_api_base: Final = api_base or endpoint.configured_api_base() + resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base) if resolved_api_base is None: raise litellm.BadRequestError( - message=endpoint.missing_api_base_message(provider), + message=provider_config.missing_api_base_message(provider), + model=model, + llm_provider=provider, + ) + resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key) + if resolved_api_key is None and provider_config.api_key_required: + raise litellm.AuthenticationError( + message=f"Missing API key for Decisions provider '{provider}'", model=model, llm_provider=provider, ) - resolved_api_key: Final = _resolve_api_key( - provider=provider, - model=model, - endpoint=endpoint, - api_key=api_key, + ir_request: Final = _ir_request(request) + body: Final = provider_config.transform_decisions_request( + model=canonical_model, request=ir_request, custom_llm_provider=provider ) - - canonical_model: Final = endpoint.canonical_model(upstream_model) - outbound_headers: Final = MappingProxyType( - { - **{ - name: value - for name, value in (extra_headers or {}).items() - if name.lower() not in {"authorization", "content-type"} - }, - **({"Authorization": f"Bearer {resolved_api_key}"} if resolved_api_key is not None else {}), - "Content-Type": "application/json", - } - ) - ir_request: Final = _ir_request(validated_request) - body: Final = endpoint.request_body(endpoint.request_model(canonical_model), ir_request) if isinstance(body, UnsupportedDecisionsRequest): raise litellm.BadRequestError( message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}", model=model, llm_provider=provider, ) - return _PreparedDecisionsRequest( - config=endpoint, - provider=provider, - upstream_model=canonical_model, - url=endpoint.endpoint_url(resolved_api_base, canonical_model), - api_key=resolved_api_key, - headers=outbound_headers, - body=MappingProxyType(body), - request=validated_request, - ir_request=ir_request, - ) - -def _log_request( - prepared: _PreparedDecisionsRequest, - kwargs: Mapping[str, object], -) -> LiteLLMLoggingObj | None: logging_obj: Final = kwargs.get("litellm_logging_obj") - if not isinstance(logging_obj, LiteLLMLoggingObj): - return None - logging_obj.update_from_kwargs( - kwargs=dict(kwargs), - model=prepared.upstream_model, - litellm_params={ - "litellm_call_id": kwargs.get("litellm_call_id"), - "api_base": prepared.url, - }, - custom_llm_provider=prepared.provider, + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.update_from_kwargs( + kwargs=dict(kwargs), + model=canonical_model, + litellm_params={ + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), + }, + custom_llm_provider=provider, + ) + return _DecisionsCall( + model=canonical_model, + requested_model=model, + custom_llm_provider=provider, + provider_config=provider_config, + request=request, + ir_request=ir_request, + body=body, + api_base=resolved_api_base, + api_key=resolved_api_key, + logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, + headers=extra_headers or {}, + timeout=timeout, ) - request_body: Final = dict(prepared.body) - request_headers: Final = dict(prepared.headers) - logging_obj.pre_call( - input=request_body, - api_key=prepared.api_key, - model=prepared.upstream_model, - additional_args={ - "api_base": prepared.url, - "complete_input_dict": request_body, - "headers": request_headers, - }, - ) - return logging_obj -def _format_response( - response: DecisionsIRResponse, prepared: _PreparedDecisionsRequest, model: str -) -> DecisionsResponse | OpenAIDecisionResponse: - match prepared.request: - case DecisionsRequestBody(): - return ir_to_systemone_response(response, prepared.ir_request) - case OpenAIDecisionRequestBody(): - return ir_to_openai_response(response, prepared.ir_request, model) - case _: - assert_never(prepared.request) - - -def _parse_response( - response: httpx.Response, - prepared: _PreparedDecisionsRequest, - model: str, -) -> DecisionsResponse | OpenAIDecisionResponse: - response.raise_for_status() - payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content) - result: Final = _format_response(prepared.config.parse_response(payload, prepared.ir_request), prepared, model) - result._hidden_params.update( +def _format_response(response: DecisionsIRResponse, call: _DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + formatted: Final = _formatted_response(response, call) + formatted.set_hidden_params( { - "model": f"{prepared.provider}/{prepared.upstream_model}", - "custom_llm_provider": prepared.provider, - "provider_response_model": f"{prepared.provider}/{prepared.upstream_model}", + "model": f"{call.custom_llm_provider}/{call.model}", + "custom_llm_provider": call.custom_llm_provider, + "provider_response_model": f"{call.custom_llm_provider}/{call.model}", } ) - return result + return formatted -def _map_upstream_exception(error: Exception, prepared: _PreparedDecisionsRequest) -> Exception: +def _formatted_response( + response: DecisionsIRResponse, call: _DecisionsCall +) -> DecisionsResponse | OpenAIDecisionResponse: + match call.request: + case DecisionsRequestBody(): + return ir_to_systemone_response(response, call.ir_request) + case OpenAIDecisionRequestBody(): + return ir_to_openai_response(response, call.ir_request, call.requested_model) + case _: + assert_never(call.request) + + +def _map_upstream_exception(error: Exception, call: _DecisionsCall) -> Exception: + if isinstance(error, BaseLLMException) and error.status_code_is_synthesized: + provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception" + return litellm.APIConnectionError( + message=f"{provider_label} - {error.message}", + llm_provider=call.custom_llm_provider, + model=f"{call.custom_llm_provider}/{call.model}", + ) return litellm.exception_type( - model=f"{prepared.provider}/{prepared.upstream_model}", - custom_llm_provider=prepared.provider, + model=f"{call.custom_llm_provider}/{call.model}", + custom_llm_provider=call.custom_llm_provider, original_exception=error, ) @@ -300,7 +256,7 @@ async def adecisions( safety_identifier: str | None = None, **kwargs: object, ) -> DecisionsResponse | OpenAIDecisionResponse: - prepared: Final = _prepare_request( + call: Final = _prepare_call( model=model, state=state, questions=questions, @@ -308,22 +264,27 @@ async def adecisions( safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, + timeout=timeout, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + kwargs=kwargs, ) - logging_obj: Final = _log_request(prepared, kwargs) try: - handler: Final = get_async_httpx_client(llm_provider=prepared.provider) - response: Final = await handler.post( - prepared.url, - json=dict(prepared.body), - headers=dict(prepared.headers), - timeout=timeout, - logging_obj=logging_obj, + response: Final = await _HANDLER.adecisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.ir_request, + body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, ) - return _parse_response(response=response, prepared=prepared, model=model) except Exception as error: - raise _map_upstream_exception(error, prepared) from error + raise _map_upstream_exception(error, call) from error + return _format_response(response, call) @client @@ -340,7 +301,7 @@ def decisions( safety_identifier: str | None = None, **kwargs: object, ) -> DecisionsResponse | OpenAIDecisionResponse: - prepared: Final = _prepare_request( + call: Final = _prepare_call( model=model, state=state, questions=questions, @@ -348,22 +309,27 @@ def decisions( safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, + timeout=timeout, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + kwargs=kwargs, ) - logging_obj: Final = _log_request(prepared, kwargs) try: - handler: Final = _get_httpx_client() - response: Final = handler.post( - prepared.url, - json=dict(prepared.body), - headers=dict(prepared.headers), - timeout=timeout, - logging_obj=logging_obj, + response: Final = _HANDLER.decisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.ir_request, + body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, ) - return _parse_response(response=response, prepared=prepared, model=model) except Exception as error: - raise _map_upstream_exception(error, prepared) from error + raise _map_upstream_exception(error, call) from error + return _format_response(response, call) -__all__ = ["DECISIONS_ENDPOINTS", "adecisions", "decisions"] +__all__ = ["adecisions", "decisions"] diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 0fdfb301291..9fb7f233242 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -2146,7 +2146,10 @@ def _map_openrouter_exception( exception_provider: str, extra_information: str, ) -> None: - if hasattr(original_exception, "status_code"): + received_status: Final = hasattr(original_exception, "status_code") and not getattr( + original_exception, "status_code_is_synthesized", False + ) + if received_status: if original_exception.status_code == 400: raise BadRequestError( message=f"{exception_provider} - {error_str}", diff --git a/litellm/llms/base_llm/decisions/__init__.py b/litellm/llms/base_llm/decisions/__init__.py index c18ac9b00f2..313e57e9762 100644 --- a/litellm/llms/base_llm/decisions/__init__.py +++ b/litellm/llms/base_llm/decisions/__init__.py @@ -1,3 +1,3 @@ -from .transformation import DecisionsProviderConfig, JevCompatibleDecisionsEndpoint +from .transformation import BaseDecisionsConfig -__all__ = ["DecisionsProviderConfig", "JevCompatibleDecisionsEndpoint"] +__all__ = ["BaseDecisionsConfig"] diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 14a955053da..a8d14ca59a3 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -1,12 +1,14 @@ import json +from abc import ABC from collections.abc import Mapping, Sequence -from dataclasses import dataclass from types import MappingProxyType -from typing import Final, Protocol +from typing import Final -from pydantic import TypeAdapter +import httpx +from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( ChoiceAnswer, @@ -46,7 +48,9 @@ from litellm.types.decisions import ( systemone_choice_key, ) +_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) +_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) _TEXT_ONLY: Final = UnsupportedDecisionsRequest( reason="input_image content parts are not supported because System One providers accept text input only" ) @@ -292,22 +296,27 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR ) -@dataclass(frozen=True, slots=True) -class JevCompatibleDecisionsEndpoint: - default_api_base_value: str | None - path: str - api_key_env: tuple[str, ...] - api_base_env: str +class BaseDecisionsConfig(ABC): + path: str = "/v1/systemone" + api_key_env: tuple[str, ...] = () + api_base_env: tuple[str, ...] = () api_key_required: bool = True - def configured_api_key(self) -> str | None: - return next((key for key in (get_secret_str(name) for name in self.api_key_env) if key), None) + def get_default_api_base(self) -> str | None: + return None - def configured_api_base(self) -> str | None: - return get_secret_str(self.api_base_env) or self.default_api_base_value + def missing_api_base_message(self, custom_llm_provider: str) -> str: + return f"api_base is required for Decisions provider '{custom_llm_provider}'" - def missing_api_base_message(self, provider: str) -> str: - return f"api_base is required for Decisions provider '{provider}'" + def resolve_api_base(self, api_base: str | None) -> str | None: + return api_base or self._first_secret(self.api_base_env) or self.get_default_api_base() + + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key or self._first_secret(self.api_key_env) + + @staticmethod + def _first_secret(names: tuple[str, ...]) -> str | None: + return next((value for value in (get_secret_str(name) for name in names) if value), None) def canonical_model(self, model: str) -> str: return model @@ -315,36 +324,50 @@ class JevCompatibleDecisionsEndpoint: def request_model(self, model: str) -> str: return model - def endpoint_url(self, api_base: str, model: str) -> str: + def validate_environment(self, headers: Mapping[str, str], model: str, api_key: str | None) -> dict[str, str]: + return { + **{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS}, + **({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}), + "Content-Type": "application/json", + } + + def get_complete_url(self, api_base: str, model: str) -> str: return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" - def request_body( - self, model: str, request: DecisionsIRRequest + def transform_decisions_request( + self, + model: str, + request: DecisionsIRRequest, + custom_llm_provider: str, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_systemone_request(model, request) + return ir_to_systemone_request(self.request_model(model), request) + + def unwrap_response(self, payload: object) -> object: + return payload def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: - return parse_systemone_response(payload, request) + return parse_systemone_response(self.unwrap_response(payload), request) + def transform_decisions_response( + self, + model: str, + custom_llm_provider: str, + raw_response: httpx.Response, + request: DecisionsIRRequest, + ) -> DecisionsIRResponse: + payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content) + try: + return self.parse_response(payload, request) + except ValidationError as error: + raise BaseLLMException( + status_code=500, + message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: {error}", + ) from error -class DecisionsProviderConfig(Protocol): - @property - def api_key_required(self) -> bool: ... - - def configured_api_key(self) -> str | None: ... - - def configured_api_base(self) -> str | None: ... - - def missing_api_base_message(self, provider: str) -> str: ... - - def canonical_model(self, model: str) -> str: ... - - def request_model(self, model: str) -> str: ... - - def endpoint_url(self, api_base: str, model: str) -> str: ... - - def request_body( - self, model: str, request: DecisionsIRRequest - ) -> Mapping[str, object] | UnsupportedDecisionsRequest: ... - - def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: ... + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, + ) -> BaseLLMException: + return BaseLLMException(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 83805e9d931..fa8c68ad3a4 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -1,36 +1,28 @@ from collections.abc import Mapping -from dataclasses import dataclass from typing import Final -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError -from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_request, parse_systemone_response +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.secret_managers.main import ( get_secret_str, normalize_nonempty_secret_str, ) -from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse, UnsupportedDecisionsRequest _RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) -@dataclass(frozen=True, slots=True) -class CloudflareDecisionsEndpoint: - api_key_required: bool = True +class CloudflareDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("CLOUDFLARE_API_KEY",) + api_base_env = ("CLOUDFLARE_API_BASE",) - def configured_api_key(self) -> str | None: - return get_secret_str("CLOUDFLARE_API_KEY") or None - - def configured_api_base(self) -> str | None: - env_api_base: Final = get_secret_str("CLOUDFLARE_API_BASE") - if env_api_base: - return env_api_base + def get_default_api_base(self) -> str | None: account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID")) if account_id is None: return None return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run" - def missing_api_base_message(self, provider: str) -> str: + def missing_api_base_message(self, custom_llm_provider: str) -> str: return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly" def canonical_model(self, model: str) -> str: @@ -41,7 +33,7 @@ class CloudflareDecisionsEndpoint: def request_model(self, model: str) -> str: return model.rsplit("/", maxsplit=1)[-1] - def endpoint_url(self, api_base: str, model: str) -> str: + def get_complete_url(self, api_base: str, model: str) -> str: normalized_api_base: Final = api_base.rstrip("/") if normalized_api_base.endswith("/ai/v1"): return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}" @@ -49,25 +41,14 @@ class CloudflareDecisionsEndpoint: return f"{normalized_api_base}/{model}" return f"{normalized_api_base}/ai/run/{model}" - def request_body( - self, model: str, request: DecisionsIRRequest - ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_systemone_request(model, request) - - def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: - return parse_systemone_response(_unwrap_result(payload), request) - - -def _unwrap_result(payload: object) -> object: - if not isinstance(payload, Mapping): - return payload - response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) - if "answers" in response_mapping: - return payload - result: Final = response_mapping.get("result") - if isinstance(result, Mapping): - return result - return payload - - -CLOUDFLARE_DECISIONS_ENDPOINT: Final[CloudflareDecisionsEndpoint] = CloudflareDecisionsEndpoint() + def unwrap_response(self, payload: object) -> object: + try: + response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) + except ValidationError: + return payload + if "answers" in response_mapping: + return response_mapping + try: + return _RESPONSE_MAPPING_ADAPTER.validate_python(response_mapping.get("result")) + except ValidationError: + return response_mapping diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index dd97db45a88..5e5c2e467f4 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -68,6 +68,7 @@ from litellm.llms.base_llm.base_model_iterator import ( from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.base_llm.containers.transformation import BaseContainerConfig +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.base_llm.files.transformation import ( @@ -120,6 +121,7 @@ from litellm.types.containers.main import ( ContainerObject, DeleteContainerResult, ) +from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig from litellm.types.integrations.custom_logger import ( NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, @@ -1413,6 +1415,100 @@ class BaseLLMHTTPHandler: request_data=request_data, ) + def _prepare_decisions_request( + self, + model: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + ) -> tuple[str, dict[str, str], dict[str, object]]: + outbound_headers: Final = provider_config.validate_environment(headers=headers, model=model, api_key=api_key) + url: Final = provider_config.get_complete_url(api_base=api_base, model=model) + data: Final = dict(body) + if logging_obj is not None: + logging_obj.pre_call( + input=data, + api_key=api_key, + model=model, + additional_args={"api_base": url, "complete_input_dict": data, "headers": outbound_headers}, + ) + return url, outbound_headers, data + + def decisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsIRRequest, + body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + client: HTTPHandler | None = None, + ) -> DecisionsIRResponse: + url, outbound_headers, data = self._prepare_decisions_request( + model=model, + logging_obj=logging_obj, + provider_config=provider_config, + body=body, + api_base=api_base, + api_key=api_key, + headers=headers, + ) + sync_httpx_client: Final = client if client is not None else _get_httpx_client() + try: + response: Final = sync_httpx_client.post( + url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj + ) + except httpx.HTTPError as e: + raise self._handle_error(e=e, provider_config=provider_config) + return provider_config.transform_decisions_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request + ) + + async def adecisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsIRRequest, + body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + client: AsyncHTTPHandler | None = None, + ) -> DecisionsIRResponse: + url, outbound_headers, data = self._prepare_decisions_request( + model=model, + logging_obj=logging_obj, + provider_config=provider_config, + body=body, + api_base=api_base, + api_key=api_key, + headers=headers, + ) + async_httpx_client: Final = ( + client + if client is not None + else get_async_httpx_client(llm_provider=litellm.LlmProviders(custom_llm_provider)) + ) + try: + response: Final = await async_httpx_client.post( + url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj + ) + except httpx.HTTPError as e: + raise self._handle_error(e=e, provider_config=provider_config) + return provider_config.transform_decisions_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request + ) + def _prepare_audio_transcription_request( self, model: str, @@ -5926,6 +6022,7 @@ class BaseLLMHTTPHandler: e: Exception, provider_config: Union[ BaseConfig, + BaseDecisionsConfig, BaseRerankConfig, BaseResponsesAPIConfig, BaseImageEditConfig, diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 003fccb7a8d..0afbe1a434e 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -1,12 +1,11 @@ from collections.abc import Mapping, Sequence -from dataclasses import dataclass from typing import Final from pydantic import TypeAdapter from typing_extensions import assert_never import litellm -from litellm.llms.base_llm.decisions.transformation import decisions_text +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( DecisionsIRAnswer, @@ -291,40 +290,31 @@ def ir_to_openai_response( ) -@dataclass(frozen=True, slots=True) -class OpenAIDecisionsEndpoint: - api_key_required: bool = True +class OpenAIDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" - def configured_api_key(self) -> str | None: - return litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + def get_default_api_base(self) -> str | None: + return "https://api.openai.com" - def configured_api_base(self) -> str | None: + def resolve_api_base(self, api_base: str | None) -> str | None: return ( - litellm.api_base + api_base + or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE") - or "https://api.openai.com" + or self.get_default_api_base() ) - def missing_api_base_message(self, provider: str) -> str: - return f"api_base is required for Decisions provider '{provider}'" + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") - def canonical_model(self, model: str) -> str: - return model - - def request_model(self, model: str) -> str: - return model - - def endpoint_url(self, api_base: str, model: str) -> str: - return f"{api_base.rstrip('/').removesuffix('/v1')}/v1/decisions" - - def request_body( - self, model: str, request: DecisionsIRRequest + def transform_decisions_request( + self, + model: str, + request: DecisionsIRRequest, + custom_llm_provider: str, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_openai_request(model, request) + return ir_to_openai_request(self.request_model(model), request) def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: return openai_response_to_ir(_OPENAI_RESPONSE_ADAPTER.validate_python(payload), request) - - -OPENAI_DECISIONS_ENDPOINT: Final[OpenAIDecisionsEndpoint] = OpenAIDecisionsEndpoint() diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 7a7466b1239..53ddce8b402 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,10 +1,10 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -OPENROUTER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://openrouter.ai/api", - path="/alpha/decisions", - api_key_env=("OPENROUTER_API_KEY",), - api_base_env="OPENROUTER_API_BASE", -) +class OpenRouterDecisionsConfig(BaseDecisionsConfig): + path = "/alpha/decisions" + api_key_env = ("OPENROUTER_API_KEY",) + api_base_env = ("OPENROUTER_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://openrouter.ai/api" diff --git a/litellm/llms/perplexity/decisions/transformation.py b/litellm/llms/perplexity/decisions/transformation.py index 69a4753f4a3..11e2a38f06e 100644 --- a/litellm/llms/perplexity/decisions/transformation.py +++ b/litellm/llms/perplexity/decisions/transformation.py @@ -1,10 +1,10 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -PERPLEXITY_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://api.perplexity.ai", - path="/v1/decisions", - api_key_env=("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"), - api_base_env="PERPLEXITY_API_BASE", -) +class PerplexityDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" + api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY") + api_base_env = ("PERPLEXITY_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://api.perplexity.ai" diff --git a/litellm/llms/strands_decider/decisions/transformation.py b/litellm/llms/strands_decider/decisions/transformation.py index 265afb2b148..9ef966e18f2 100644 --- a/litellm/llms/strands_decider/decisions/transformation.py +++ b/litellm/llms/strands_decider/decisions/transformation.py @@ -1,11 +1,7 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -STRANDS_DECIDER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value=None, - path="/v1/systemone", - api_key_env=("STRANDS_DECIDER_API_KEY",), - api_base_env="STRANDS_DECIDER_API_BASE", - api_key_required=False, -) +class StrandsDeciderDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("STRANDS_DECIDER_API_KEY",) + api_base_env = ("STRANDS_DECIDER_API_BASE",) + api_key_required = False diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py index 17fb24ce443..e71c526a840 100644 --- a/litellm/llms/typesafe/decisions/transformation.py +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -1,10 +1,9 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -TYPESAFE_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://api.typesafe.ai", - path="/v1/systemone", - api_key_env=("TYPESAFE_API_KEY",), - api_base_env="TYPESAFE_API_BASE", -) +class TypeSafeDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("TYPESAFE_API_KEY",) + api_base_env = ("TYPESAFE_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://api.typesafe.ai" diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 6488ef45308..b24690dbd9d 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -121,6 +121,13 @@ class DecisionsUsage(LiteLLMPydanticObjectBase): class _HiddenParamsResponse(LiteLLMPydanticObjectBase): _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) + @property + def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation + return self._hidden_params + + def set_hidden_params(self, params: Mapping[str, object]) -> None: + self._hidden_params.update(params) + class DecisionsResponse(_HiddenParamsResponse): model: str | None = None diff --git a/litellm/utils.py b/litellm/utils.py index f2da0aa42ab..fe2bac6ca8c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -381,6 +381,7 @@ if TYPE_CHECKING: # Type stubs for lazy-loaded config classes and types from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.containers.transformation import BaseContainerConfig + from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.files.transformation import BaseFilesConfig from litellm.llms.base_llm.google_genai.transformation import ( @@ -8897,6 +8898,22 @@ class ProviderConfigManager: return get_dashscope_family_rerank_config(provider.value) return litellm.CohereRerankConfig() + @staticmethod + def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None: + if provider == LlmProviders.PERPLEXITY: + return litellm.PerplexityDecisionsConfig() + if provider == LlmProviders.TYPESAFE: + return litellm.TypeSafeDecisionsConfig() + if provider == LlmProviders.OPENROUTER: + return litellm.OpenRouterDecisionsConfig() + if provider == LlmProviders.CLOUDFLARE: + return litellm.CloudflareDecisionsConfig() + if provider == LlmProviders.STRANDS_DECIDER: + return litellm.StrandsDeciderDecisionsConfig() + if provider == LlmProviders.OPENAI: + return litellm.OpenAIDecisionsConfig() + return None + @staticmethod def get_provider_anthropic_messages_config( model: str, diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 709ea42b76e..84b31088d60 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -25,6 +25,7 @@ _PASS_THROUGH_MODEL: Final = "gpt-6-luna" _PASS_THROUGH_AUTHORIZATION: Final = "Bearer customer-held-upstream-key" _PASS_THROUGH_NEIGHBOUR: Final = "decisions-beside-a-pass-through" _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_QUESTION_MAPPINGS: Final = TypeAdapter(dict[str, dict[str, object]]) _USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 367, "output_tokens": 3} _STATE: Final[dict[str, JsonValue]] = {"ticket": "The export job hangs at 99%", "component": "billing"} _QUESTIONS: Final[dict[str, JsonValue]] = { @@ -32,6 +33,7 @@ _QUESTIONS: Final[dict[str, JsonValue]] = { "severity": {"type": "choice", "criteria": {"low": "cosmetic", "high": "blocks users"}, "weight": 2}, "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, } +_SDK_QUESTIONS: Final = _QUESTION_MAPPINGS.validate_python(_QUESTIONS) _ANSWERS: Final[dict[str, JsonValue]] = { "defect": {"type": "noul", "noul": 0.93}, "severity": {"type": "choice", "choice": "high", "confidence": 0.8, "probabilities": {"low": 0.2, "high": 0.8}}, @@ -103,6 +105,11 @@ _PROVIDERS: Final = ( ), ) _PERPLEXITY: Final = _PROVIDERS[0] +_OPENROUTER: Final = _PROVIDERS[2] +_OPENROUTER_CHAT_MODEL: Final = "openrouter/openai/gpt-5-mini" +_UNSUPPORTED_PROVIDER_MODEL: Final = "openai/gpt-6-luna" +_CONNECTION_ERROR: Final = "litellm.APIConnectionError" +_GENERIC_API_ERROR: Final = "litellm.APIError" _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( ("missing questions", {"state": _STATE}), ("missing state", {"questions": _QUESTIONS}), @@ -177,6 +184,12 @@ def _spend_row(call_id: str) -> dict[str, JsonValue]: return rows[0] +def _assert_connection_error(response: httpx.Response) -> None: + assert 500 <= response.status_code < 600, response.text + assert _CONNECTION_ERROR in response.text, response.text + assert _GENERIC_API_ERROR not in response.text, response.text + + def _free_closed_port() -> int: with socket.socket() as probe: probe.bind(("127.0.0.1", 0)) @@ -257,10 +270,10 @@ async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway with gateway.scenario() as scenario: handle: Final = _register(scenario, _answer_body(provider)) synchronous: Final = litellm.decisions( - model=provider.model, state=_STATE, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY ) asynchronous: Final = await litellm.adecisions( - model=provider.model, state=_STATE, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY ) for response in (synchronous, asynchronous): assert response.model_dump(mode="json") == { @@ -443,16 +456,75 @@ def test_upstream_success_without_answers_is_a_gateway_side_server_error(gateway assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" -def test_unreachable_upstream_fails_only_its_own_deployment(gateway: Gateway) -> None: +@pytest.mark.parametrize("provider", _PROVIDERS, ids=lambda provider: provider.name) +def test_unreachable_upstream_fails_only_its_own_deployment(gateway: Gateway, provider: _Provider) -> None: with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) - healthy: Final = _deployment(scenario, handle, _PERPLEXITY) + handle: Final = _register(scenario, _answer_body(provider)) + healthy: Final = _deployment(scenario, handle, provider) dead: Final = scenario.model( - model=_PERPLEXITY.model, api_base=f"http://127.0.0.1:{_free_closed_port()}", api_key=_API_KEY + model=provider.model, api_base=f"http://127.0.0.1:{_free_closed_port()}", api_key=provider.api_key ) failed: Final = _decide(gateway, dead, num_retries=0) - assert 500 <= failed.status_code < 600, failed.text + _assert_connection_error(failed) assert _spend_row(failed.headers["x-litellm-call-id"])["status"] == "failure" served: Final = _decide(gateway, healthy) assert served.status_code == 200, served.text assert len(_upstream_calls(gateway, handle)) == 1 + + +def test_an_unreachable_openrouter_deployment_reports_a_connection_error_on_chat_embeddings_and_decisions( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + dead_api_base: Final = f"http://127.0.0.1:{_free_closed_port()}" + chat_model: Final = scenario.model(model=_OPENROUTER_CHAT_MODEL, api_base=dead_api_base, api_key=_API_KEY) + decisions_model: Final = scenario.model(model=_OPENROUTER.model, api_base=dead_api_base, api_key=_API_KEY) + chat: Final = _chat(gateway, chat_model, num_retries=0) + streamed: Final = _chat(gateway, chat_model, num_retries=0, stream=True) + embeddings: Final = gateway.request( + "POST", "/v1/embeddings", {"model": chat_model, "input": "hi", "num_retries": 0} + ) + decisions: Final = _decide(gateway, decisions_model, num_retries=0) + for response in (chat, streamed, embeddings, decisions): + _assert_connection_error(response) + for response in (chat, decisions): + assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" + + +async def test_sdk_openrouter_connection_failures_raise_a_connection_error(gateway: Gateway) -> None: + dead_api_base: Final = f"http://127.0.0.1:{_free_closed_port()}" + with pytest.raises(litellm.APIConnectionError): + litellm.completion( + model=_OPENROUTER_CHAT_MODEL, + messages=[{"role": "user", "content": "hi"}], + api_base=dead_api_base, + api_key=_API_KEY, + ) + with pytest.raises(litellm.APIConnectionError): + await litellm.acompletion( + model=_OPENROUTER_CHAT_MODEL, + messages=[{"role": "user", "content": "hi"}], + api_base=dead_api_base, + api_key=_API_KEY, + ) + with pytest.raises(litellm.APIConnectionError): + litellm.decisions( + model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY + ) + with pytest.raises(litellm.APIConnectionError): + await litellm.adecisions( + model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY + ) + + +def test_a_deployment_whose_provider_has_no_decisions_support_is_refused_naming_every_supported_provider( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) + model: Final = scenario.model(model=_UNSUPPORTED_PROVIDER_MODEL, api_base=handle.api_base(), api_key=_API_KEY) + response: Final = _decide(gateway, model) + assert response.status_code == 400, response.text + for provider in _PROVIDERS: + assert provider.name in response.text, response.text + assert _upstream_calls(gateway, handle) == [] diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 085adea81ac..993de299526 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -1,27 +1,25 @@ -model_list: [] -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - database_url: os.environ/DATABASE_URL - store_model_in_db: true - disable_spend_logs: false - proxy_batch_write_at: 1 - proxy_batch_polling_interval: 1 -litellm_settings: - enable_redis_auth_cache: true - cache: true - cache_params: - type: redis - host: os.environ/REDIS_HOST - port: os.environ/REDIS_PORT -router_settings: - disable_cooldowns: true -vector_store_registry: - - vector_store_name: integration-config-store +model_list: + - model_name: perplexity/pplx-decider-v1-27b litellm_params: - vector_store_id: vs_integration_config_store - custom_llm_provider: openai - api_base: os.environ/INTEGRATION_UPSTREAM_URL - api_key: integration-provider-key - vector_store_description: declared in tests/integration/proxy_config.yaml -environment_variables: - PERPLEXITYAI_API_KEY: synthetic-decisions-env-key + model: perplexity/pplx-decider-v1-27b + api_base: http://127.0.0.1:8191 + api_key: synthetic-perplexity-key + - model_name: typesafe/jev-1.13.0 + litellm_params: + model: typesafe/jev-1.13.0 + api_base: http://127.0.0.1:8191 + api_key: synthetic-typesafe-key + - model_name: openrouter/typesafe/jev-1.13 + litellm_params: + model: openrouter/typesafe/jev-1.13 + api_base: http://127.0.0.1:8191 + api_key: synthetic-openrouter-key + - model_name: cloudflare/@cf/cloudflare/clef + litellm_params: + model: cloudflare/@cf/cloudflare/clef + api_base: http://127.0.0.1:8191 + api_key: synthetic-cloudflare-key + - model_name: strands_decider/strands-decider-2B-hobson-v19 + litellm_params: + model: strands_decider/strands-decider-2B-hobson-v19 + api_base: http://127.0.0.1:8191 diff --git a/tests/integration/translation/decisions/__init__.py b/tests/integration/translation/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/decisions/bases/__init__.py b/tests/integration/translation/decisions/bases/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/decisions/bases/cloudflare.py b/tests/integration/translation/decisions/bases/cloudflare.py new file mode 100644 index 00000000000..1668dc215a8 --- /dev/null +++ b/tests/integration/translation/decisions/bases/cloudflare.py @@ -0,0 +1,84 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://developers.cloudflare.com/workers-ai (POST /ai/run/@cf/cloudflare/clef, reply wrapped in result). Mock reply captured live on 2026-10-06. +""" +CLEF_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "cloudflare/@cf/cloudflare/clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/ai/run/@cf/cloudflare/clef", + expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "result": { + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + }, + expected_litellm_response={ + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, +) diff --git a/tests/integration/translation/decisions/bases/openrouter.py b/tests/integration/translation/decisions/bases/openrouter.py new file mode 100644 index 00000000000..f6217bcf818 --- /dev/null +++ b/tests/integration/translation/decisions/bases/openrouter.py @@ -0,0 +1,83 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://openrouter.ai/docs (POST /api/alpha/decisions). Mock reply captured live on 2026-10-06. +""" +TYPESAFE_JEV_1_13_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "openrouter/typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/alpha/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-openrouter-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, + expected_litellm_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, +) diff --git a/tests/integration/translation/decisions/bases/perplexity.py b/tests/integration/translation/decisions/bases/perplexity.py new file mode 100644 index 00000000000..71ef7a85220 --- /dev/null +++ b/tests/integration/translation/decisions/bases/perplexity.py @@ -0,0 +1,79 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://docs.perplexity.ai (POST /v1/decisions). Mock reply captured live on 2026-10-06. +""" +PPLX_DECIDER_V1_27B_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "perplexity/pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-perplexity-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, + expected_litellm_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, +) diff --git a/tests/integration/translation/decisions/bases/strands_decider.py b/tests/integration/translation/decisions/bases/strands_decider.py new file mode 100644 index 00000000000..83d7ae5e33a --- /dev/null +++ b/tests/integration/translation/decisions/bases/strands_decider.py @@ -0,0 +1,35 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://github.com/strands-agents/decider (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-06. +""" +STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "strands_decider/strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"content-type": "application/json"}, + expected_provider_request={ + "model": "strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + }, + mock_provider_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, + expected_litellm_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, +) diff --git a/tests/integration/translation/decisions/bases/typesafe.py b/tests/integration/translation/decisions/bases/typesafe.py new file mode 100644 index 00000000000..2fb384a94ae --- /dev/null +++ b/tests/integration/translation/decisions/bases/typesafe.py @@ -0,0 +1,79 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://docs.typesafe.ai (POST /v1/systemone). Mock reply captured live on 2026-10-06. +""" +JEV_1_13_0_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "typesafe/jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"authorization": "Bearer synthetic-typesafe-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, + expected_litellm_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, +) diff --git a/tests/integration/translation/decisions/basic/__init__.py b/tests/integration/translation/decisions/basic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py new file mode 100644 index 00000000000..79fc0574b3b --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [CLEF_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_cloudflare(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py new file mode 100644 index 00000000000..a74cb23f6b4 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.openrouter import TYPESAFE_JEV_1_13_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [TYPESAFE_JEV_1_13_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_openrouter(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py new file mode 100644 index 00000000000..e341d24ba53 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.perplexity import PPLX_DECIDER_V1_27B_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [PPLX_DECIDER_V1_27B_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_perplexity(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py new file mode 100644 index 00000000000..cae7ca70c78 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.strands_decider import STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_strands_decider(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py new file mode 100644 index 00000000000..302636e6fb4 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_typesafe(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index cbc8f7f6d01..820451ed943 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -6,6 +6,7 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Final +import httpx import pytest import respx @@ -270,10 +271,7 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None: answers={}, usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS), ) - response._hidden_params = { - "model": "perplexity/pplx-decider-v1-27b", - "custom_llm_provider": "perplexity", - } + response.set_hidden_params({"model": "perplexity/pplx-decider-v1-27b", "custom_llm_provider": "perplexity"}) cost: Final = litellm.completion_cost(completion_response=response) perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] @@ -325,6 +323,15 @@ def test_custom_token_pricing_bills_cached_decisions_input_tokens_once( ) +def test_decisions_response_hidden_params_getter_preserves_mutable_identity() -> None: + response: Final = DecisionsResponse(model="decider", answers={}, usage=None) + + assert response.hidden_params is response._hidden_params + + response.hidden_params["mutation"] = "visible" + assert response._hidden_params["mutation"] == "visible" + + @pytest.mark.asyncio async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.MockRouter) -> None: respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) @@ -357,7 +364,7 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo @pytest.mark.asyncio async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: - with pytest.raises(litellm.BadRequestError, match="Supported providers"): + with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"): await litellm.adecisions( model="unknown/jev-1.13", state="review", @@ -369,17 +376,79 @@ async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRo @pytest.mark.asyncio -async def test_empty_custom_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: - with pytest.raises(litellm.BadRequestError, match="Supported providers"): +async def test_provider_without_decisions_support_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match=r"Unknown Decisions provider 'anthropic'\. Supported providers"): + await litellm.adecisions( + model="anthropic/claude-sonnet-4-5", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_empty_custom_provider_falls_back_to_the_model_prefix(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + custom_llm_provider="", + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content)["model"] == "pplx-decider-v1-27b" + assert response._hidden_params["custom_llm_provider"] == "perplexity" + + +@pytest.mark.asyncio +async def test_upstream_reply_without_answers_is_a_server_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={"model": "pplx-decider-v1-27b", "usage": {"input_tokens": 10, "output_tokens": 0}} + ) + + with pytest.raises(litellm.InternalServerError, match="unexpected response"): await litellm.adecisions( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, api_key="caller-key", - custom_llm_provider="", ) - assert len(respx_mock.calls) == 0 + +@pytest.mark.asyncio +async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_already_resolved( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": { + "model": "openrouter/typesafe/jev-1.13", + "api_key": "deployment-key", + "api_base": "https://egress.example/openrouter", + }, + } + ] + ) + upstream: Final = respx_mock.post("https://egress.example/openrouter/alpha/decisions").respond(json=_RESPONSE) + + await router.adecisions( + model="jev", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert upstream.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer deployment-key" + assert json.loads(respx_mock.calls[0].request.content)["model"] == "typesafe/jev-1.13" @pytest.mark.asyncio @@ -410,6 +479,20 @@ def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter ) +def test_unreachable_upstream_maps_to_a_connection_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").mock( + side_effect=httpx.ConnectError("Cannot connect to host api.perplexity.ai:443") + ) + + with pytest.raises(litellm.APIConnectionError, match="PerplexityException - Cannot connect to host"): + litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + def test_server_key_is_sent_to_an_explicit_api_base( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, @@ -701,7 +784,7 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_ ], } assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} - assert response._hidden_params["custom_llm_provider"] == "openai" + assert response.hidden_params["custom_llm_provider"] == "openai" luna_cost: Final = litellm.model_cost["gpt-6-luna"] expected_cost: Final = ( (_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * float(luna_cost["input_cost_per_token"]) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 9de768ea47b..20ccd49973e 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1195,6 +1195,34 @@ def test_branchless_provider_transport_error_maps_to_api_connection_error(): ) +def test_openrouter_transport_error_maps_to_api_connection_error(): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + original_exception = BaseLLMException(status_code=500, message="[Errno 111] Connection refused") + original_exception.status_code_is_synthesized = True + + with pytest.raises(litellm.APIConnectionError): + exception_type( + model="typesafe/jev-1.13", + original_exception=original_exception, + custom_llm_provider="openrouter", + ) + + +def test_openrouter_upstream_500_still_maps_to_api_error(): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + original_exception = BaseLLMException(status_code=500, message="upstream exploded") + + with pytest.raises(litellm.APIError) as excinfo: + exception_type( + model="typesafe/jev-1.13", + original_exception=original_exception, + custom_llm_provider="openrouter", + ) + assert excinfo.value.status_code == 500 + + def test_branchless_provider_upstream_500_still_maps_to_internal_server_error(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index 249a1a7640a..e73c5b92ab6 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -6,7 +6,7 @@ from pydantic import TypeAdapter from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_response, systemone_request_to_ir from litellm.llms.openai.decisions.transformation import ( - OPENAI_DECISIONS_ENDPOINT, + OpenAIDecisionsConfig, ir_to_openai_request, ir_to_openai_response, openai_request_to_ir, @@ -101,7 +101,7 @@ def test_an_openai_request_reaches_openai_unchanged() -> None: def test_an_openai_response_reaches_the_caller_unchanged() -> None: ir: Final = _openai_ir(_OPENAI_REQUEST) - parsed: Final = OPENAI_DECISIONS_ENDPOINT.parse_response(_OPENAI_RESPONSE, ir) + parsed: Final = OpenAIDecisionsConfig().parse_response(_OPENAI_RESPONSE, ir) assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE @@ -110,7 +110,7 @@ def test_answers_openai_did_not_return_are_refusals() -> None: ir: Final = _openai_ir(_OPENAI_REQUEST) payload: Final = {**_OPENAI_RESPONSE, "answers": [_PREDICATE_ANSWER]} - response: Final = ir_to_openai_response(OPENAI_DECISIONS_ENDPOINT.parse_response(payload, ir), ir, "requested") + response: Final = ir_to_openai_response(OpenAIDecisionsConfig().parse_response(payload, ir), ir, "requested") assert [answer.type for answer in response.answers] == ["predicate", "refusal", "refusal", "refusal"] assert [answer.name for answer in response.answers] == ["damaged", None, "severity", "fraud"] @@ -230,7 +230,7 @@ def test_an_openai_response_becomes_systemone_answers_with_the_callers_score_lab ], } - response: Final = ir_to_systemone_response(OPENAI_DECISIONS_ENDPOINT.parse_response(payload, ir), ir) + response: Final = ir_to_systemone_response(OpenAIDecisionsConfig().parse_response(payload, ir), ir) assert response.model_dump(mode="json") == { "model": "gpt-6-luna", From b6be484cc8fc4ede381daa4cc6c351074db29a39 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 8 Oct 2026 13:23:43 -0700 Subject: [PATCH 03/18] test(decisions): post System One bodies to /v1/systemone in the translation bases (#45327) (cherry picked from commit a7b04a8028cbf25ca85b740bd1c2d38dac4a3034) --- tests/integration/translation/decisions/bases/cloudflare.py | 2 +- tests/integration/translation/decisions/bases/openrouter.py | 2 +- tests/integration/translation/decisions/bases/perplexity.py | 2 +- .../integration/translation/decisions/bases/strands_decider.py | 2 +- tests/integration/translation/decisions/bases/typesafe.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/translation/decisions/bases/cloudflare.py b/tests/integration/translation/decisions/bases/cloudflare.py index 1668dc215a8..ae86a2ccd87 100644 --- a/tests/integration/translation/decisions/bases/cloudflare.py +++ b/tests/integration/translation/decisions/bases/cloudflare.py @@ -6,7 +6,7 @@ from integration.translation.case import TranslationTestCase """ CLEF_TEST_CASE: Final = TranslationTestCase( scenario="basic", - litellm_endpoint="/v1/decisions", + litellm_endpoint="/v1/systemone", litellm_request={ "model": "cloudflare/@cf/cloudflare/clef", "state": "Ticket (billing): The export job hangs at 99% and never finishes", diff --git a/tests/integration/translation/decisions/bases/openrouter.py b/tests/integration/translation/decisions/bases/openrouter.py index f6217bcf818..b65acff6b06 100644 --- a/tests/integration/translation/decisions/bases/openrouter.py +++ b/tests/integration/translation/decisions/bases/openrouter.py @@ -6,7 +6,7 @@ from integration.translation.case import TranslationTestCase """ TYPESAFE_JEV_1_13_TEST_CASE: Final = TranslationTestCase( scenario="basic", - litellm_endpoint="/v1/decisions", + litellm_endpoint="/v1/systemone", litellm_request={ "model": "openrouter/typesafe/jev-1.13", "state": "Ticket (billing): The export job hangs at 99% and never finishes", diff --git a/tests/integration/translation/decisions/bases/perplexity.py b/tests/integration/translation/decisions/bases/perplexity.py index 71ef7a85220..02b1de51f24 100644 --- a/tests/integration/translation/decisions/bases/perplexity.py +++ b/tests/integration/translation/decisions/bases/perplexity.py @@ -6,7 +6,7 @@ from integration.translation.case import TranslationTestCase """ PPLX_DECIDER_V1_27B_TEST_CASE: Final = TranslationTestCase( scenario="basic", - litellm_endpoint="/v1/decisions", + litellm_endpoint="/v1/systemone", litellm_request={ "model": "perplexity/pplx-decider-v1-27b", "state": "Ticket (billing): The export job hangs at 99% and never finishes", diff --git a/tests/integration/translation/decisions/bases/strands_decider.py b/tests/integration/translation/decisions/bases/strands_decider.py index 83d7ae5e33a..8a1a51f8c3e 100644 --- a/tests/integration/translation/decisions/bases/strands_decider.py +++ b/tests/integration/translation/decisions/bases/strands_decider.py @@ -6,7 +6,7 @@ from integration.translation.case import TranslationTestCase """ STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE: Final = TranslationTestCase( scenario="basic", - litellm_endpoint="/v1/decisions", + litellm_endpoint="/v1/systemone", litellm_request={ "model": "strands_decider/strands-decider-2B-hobson-v19", "state": "Help! My payouts have been failing for 3 days!", diff --git a/tests/integration/translation/decisions/bases/typesafe.py b/tests/integration/translation/decisions/bases/typesafe.py index 2fb384a94ae..5da5b05cb09 100644 --- a/tests/integration/translation/decisions/bases/typesafe.py +++ b/tests/integration/translation/decisions/bases/typesafe.py @@ -6,7 +6,7 @@ from integration.translation.case import TranslationTestCase """ JEV_1_13_0_TEST_CASE: Final = TranslationTestCase( scenario="basic", - litellm_endpoint="/v1/decisions", + litellm_endpoint="/v1/systemone", litellm_request={ "model": "typesafe/jev-1.13.0", "state": "Ticket (billing): The export job hangs at 99% and never finishes", From 3e70ececf08803e76910b7840c137a4013dc3506 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:44:18 -0700 Subject: [PATCH 04/18] test: realign stale decisions and lens tests with merged behaviour (#45346) Co-authored-by: yuneng (cherry picked from commit 6e54dced8c2950a6e9e99e965cb58df43b0dae38) --- tests/integration/providers/test_decisions_wire.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 84b31088d60..db6629a43d2 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -107,7 +107,7 @@ _PROVIDERS: Final = ( _PERPLEXITY: Final = _PROVIDERS[0] _OPENROUTER: Final = _PROVIDERS[2] _OPENROUTER_CHAT_MODEL: Final = "openrouter/openai/gpt-5-mini" -_UNSUPPORTED_PROVIDER_MODEL: Final = "openai/gpt-6-luna" +_UNSUPPORTED_PROVIDER_MODEL: Final = "anthropic/claude-opus-5-5" _CONNECTION_ERROR: Final = "litellm.APIConnectionError" _GENERIC_API_ERROR: Final = "litellm.APIError" _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( From fe3fdaebb3c192ea7ebc03b09572033d7f210c8c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:04:07 -0700 Subject: [PATCH 05/18] feat(ui): add TypeSafe and Strands Decider to the Add Model provider list (#45335) * feat(ui): add TypeSafe and Strands Decider to the Add Model provider list * test(public_endpoints): type the provider fields helper against the route's model --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> (cherry picked from commit 3a3b8cad809f1a2d530d7572cd770dc35b796d30) --- .../provider_create_fields.json | 56 ++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 51 +++++++++++++++- .../public/assets/logos/typesafe.png | Bin 0 -> 4311 bytes .../components/provider_info_helpers.test.tsx | 49 +++++++++++++++ .../src/components/provider_info_helpers.tsx | 10 ++++ 5 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 ui/litellm-dashboard/public/assets/logos/typesafe.png diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 11d2ff61b95..bdfeddba3b3 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3161,6 +3161,34 @@ ], "default_model_placeholder": "soniox/stt-async-v5" }, + { + "provider": "StrandsDecider", + "provider_display_name": "Strands Decider", + "litellm_provider": "strands_decider", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "http://localhost:8000", + "tooltip": "URL of your self-hosted Strands Decider server", + "required": true, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "strands_decider/strands-decider-2B-hobson-v19" + }, { "provider": "TEXT_COMPLETION_CODESTRAL", "provider_display_name": "Text-Completion-Codestral", @@ -3263,6 +3291,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "TypeSafe", + "provider_display_name": "TypeSafe", + "litellm_provider": "typesafe", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.typesafe.ai", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "typesafe/jev-latest" + }, { "provider": "V0", "provider_display_name": "V0", diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 85b667e1be3..90db461efd3 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -9,6 +9,7 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from pydantic import TypeAdapter from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.public_endpoints import router @@ -16,6 +17,7 @@ from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_pres from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, ) +from litellm.types.proxy.public_endpoints.public_endpoints import ProviderCreateInfo from litellm.types.utils import LlmProviders @@ -377,6 +379,53 @@ def test_chatgpt_provider_fields(): assert chatgpt["credential_fields"] == [] +def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: + app_instance: Final = FastAPI() + app_instance.include_router(router) + test_client: Final = TestClient(app_instance) + + response: Final = test_client.get("/public/providers/fields") + assert response.status_code == 200 + providers: Final = TypeAdapter(list[ProviderCreateInfo]).validate_python(response.json()) + entry: Final = next((p for p in providers if p.provider == provider), None) + assert entry is not None, f"{provider} provider entry not found" + return entry + + +def test_typesafe_provider_fields(): + typesafe: Final = _decisions_provider_entry("TypeSafe") + + assert typesafe.provider_display_name == "TypeSafe" + assert typesafe.litellm_provider == LlmProviders.TYPESAFE.value + assert typesafe.default_model_placeholder is not None + assert typesafe.default_model_placeholder.startswith("typesafe/") + + fields_by_key: Final = {f.key: f for f in typesafe.credential_fields} + + assert fields_by_key["api_key"].required is True + assert fields_by_key["api_key"].field_type == "password" + + assert fields_by_key["api_base"].required is False + assert fields_by_key["api_base"].field_type == "text" + + +def test_strands_decider_provider_fields(): + strands: Final = _decisions_provider_entry("StrandsDecider") + + assert strands.provider_display_name == "Strands Decider" + assert strands.litellm_provider == LlmProviders.STRANDS_DECIDER.value + assert strands.default_model_placeholder is not None + assert strands.default_model_placeholder.startswith("strands_decider/") + + fields_by_key: Final = {f.key: f for f in strands.credential_fields} + + assert fields_by_key["api_base"].required is True + assert fields_by_key["api_base"].field_type == "text" + + assert fields_by_key["api_key"].required is False + assert fields_by_key["api_key"].field_type == "password" + + ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( { "a2a", @@ -411,13 +460,11 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( "sagemaker_nova", "scaleway", "stability", - "strands_decider", "synthetic", "tencent", "tensormesh", "text-completion-inception", "transcribe", - "typesafe", "valkey", "xiaomi_mimo", "zai", diff --git a/ui/litellm-dashboard/public/assets/logos/typesafe.png b/ui/litellm-dashboard/public/assets/logos/typesafe.png new file mode 100644 index 0000000000000000000000000000000000000000..45107c7d09d7e53da800df531d92a5eaafd08f99 GIT binary patch literal 4311 zcmW+)c{J3G_nytp7$ekRe1?x@4BmJVq6R}VL$Yrnp-3f$q=}kA%uvh>X|tpV5!oWC zER!WAS>vrN*-4hsg7owK{c-Pop68rWyu_ngaibT}j>juD4IAW{@E$r%EHa{d>h z@ExV@c-!_)(R8$TwXr`HG3E4p%JKQMQ^S-~!%kLEP}l+8)Z)A5j0*;V_8OM|s_0^l zBp=yn5sq?paDjLz{>N#kFX+C$qlkr*y&@qH3EBSz6mlnf_l}5+vhj>^Id?MZ!g1OO zi0!H1v&VBQvjQOyls1KA=^8WgE$6>GUyG5p3D7{LSAW+W9zB0(R>C(H!To6o-ci~l!ht#x`b<3E07^Sc1rT+daRN0%!~Lk;#Q5{KlBZu>C& z!Z4Mgq;lq09hxsZSt@i;h!fHh0R>0GqtZxfX=&V$*B|g`-TkG$VBCOu22CmFM#`+) z?*Y!Op00V-8v@j0GphOvQmNlW*7W1B(6Fr!?y$@}t1(-q0+r5pJw3 zpIA%ppc3RaAe~YdJrHsh#wF_`vidnn=4$3Jw3#|Y%S{QO`2OwA| zB_rs=P``m^;aHa`FA%`Pu zStqaNZn-B^0U1GSj7WVW9-Z7r-V!{pgVD4o{p-Jbnr_6~5RoAJQNuMIXz9b1Oz|KU zy6gHWzU2Z}C@B3>zu;6*%b=Qc2Xdo$?o#>dK{W-k=Uzt4$7{J&Ll$AiB?#v`LwAD( zo>wx6nw*wb*R8a6mZ)|JZKg`XH(ca-EOh*foyL?mlr_>KkljSjb246A*3 zm52S{Bmo|kN%a>00pkaVTE)d5{)c04QzmQ?Z7foX1l;{lp%y=kMV^0_*8;p!JbN4{ z+F#t@C;~y$4S!4i&jGu5A<$YTbp+qZx}4`Mvl~Uw6s^EM9~PA)SE;(c!0zYWgH{^E za~7Op3CdFB`>OIaTu++OPdBs);Ae%xTO1(6L_ms6yJS%6*>mLw$J^i&LN60g(bRyc z#DnI5SRPWDz!{`BX0aJWrN7B753lphDG+A1cEE1tCLU7AfB9op||I&oab|{XVc=@YQB# z^4iT7ZQ%B?Js#JII%`D&ao~5{L@WjJ<(6?{`^7Hit~gt-b!bFmup*V2dD*se@d*Sn ze0bq3^6xHK$*(uJ+@1mu{jv;xDx*K{#M_h0NBl zgF*S>FXK28iStdT?t>_z`OJ+Hdmw72vJYfJR`pNr5e2xb=Fy*?9LqIN^YRnplv&;< zY$>*F>Jy3~yiJMg`v}HHC-=w^12Pim5Z>;Wr{@xhZzf)dQN}9ze-XPmX-QUq;sN5>b9$=0mA^Sz||?7}mIp8p2)Vk&L9{{*L^@ z7!he8-b6}AlRbXtH3g6Kspa1llsOkkcZ}y%g~qmi)IV$=CLH8}=C!VF@jM^?PWi@&-A>S2;&sF}+=;RE ztV67h|GYJ=$6j98PiHn>7xtX-`er?vl+PW&cBF!t^yQz<=I;rjkC|H;1+)r+9(!t! z_7`kN9T*W-?f#=ItOov@2hKfmn;6X)fhuMNqvn}9m5afDo}Su>kx-b;KiA6b!!tNR z$%niuBXu#;J}JQ$hgmSu=50LIo<=mi8Q_H#&xbmD9p8G~@KZ#DK{kjmH?lz#P zRSIJ=O*lr%ps0`0eKGP(b-Xs<~tV@w_)!^aUek7j>y|_8eC3Hfp#6kBXwpdLCG3MeY+b&)ZBp;5(`|)~_ z{3eKLc5SX+XT6FQ0z@|=O&J_0Wq009lSteB95E<)VzIII3LuN%OwRG}ybuwZtO(D@ z+aMZAI$2};Zh4?j`AI0?e)5<7=hG<7{r&^Q#eub!+&2K%O#Y~{BY*wlHZ?_Pk^r+ zBuWxEu9OM$zUY*L+lkVT6v6;oaq=Bd(FT9Z(Pya$Fa>1yafq~k%+TMS{j5Um);BRGR|3=zuez6gcwiEtaNajW06m6#!dF^3v8ddyCK{ z`DK{?Kg5SXy+L%7)OyTc>1EhO3hvXlvR>t7jQ9M?QA>AV`$n=qnsPI-z=eksBcRqK zSX?2VDkZ4rs}0&6c%|yTS(-GU@Ie0IowYnLo49xm+pE7|&yz14yS+m{DWYUy<={`- z!m)_NNkqUGhCTH7V{`f${i!i!P0oLtt0+>yx5NB?7wxMU{j+qZ0CywKH5cs@>?GZQ ze14f3aIwgh3XYEmTKHucV7SOt92`&e1oTT`Xg$9bUJSXX7L zcdpLpOebM}kD5IBtS6#i>A-t!DgGklb`7lL#rq7L*bx$Ojh zw25&sYhgb_0J`fYcB+kdWg<5ch^HGuXxc16_fJXD0sniwp*g;&xthzc#D-mOqti_b zy9hMV#OPJABgb88XM`LlMOpRO*Ci!X)bK>O3ab&Tj}WsAk{cF6&wLS(qwWneOa z37gU!Yl~-TukIoD#VYQTu`obzT_^j}52R$pUl(-9kesV!GXxCFzkVN(Kz4q$c$#9BRx}0wSbJ!5<0pdbA}5TT(o#| zGyd_q*2h)x>GhANiwZ)!2)kAQlXN|JRD`_;Kq zfAgQY=}IF=5k-j_{*=k}k3bxd8E*)+143*z?!kKSI0;YKKS@r1Rn_L4RG$1wa0ij= z1RfRvO|=NI>3wh8l~f=?X_`QKciGmdcs~)`6Z2fOBrK>qDi;E0nPx~E!B5L{d_Za?9zx@Np%~Lt`M91pp$PWTXKU$O^+7yjYass z`$LgjH2FJhnBkOb;ikTs+=i0I!dB~?EFY<6^V9?qR^jkJ$YGcxR@ku zi)|tGUTEUc4fRAT+jKRv*!n(6$%3fST@-gG3vY$l@b_BTl&a48=J(dw9hG1UoxBW* z2@SFyuuc+b+9VSmVx7$)tl0U&GrcC0F@U*L!-OFkx&vZ=X?ObiNH1-`ODfjC*s;gw z%>mZdXgH#$HD!$ftX>e2ye9&amQ?K$zb7J^E+X^$&_i4nd>p#6xDS@D*n=SjbxHHl z5z9?>-_&wmp;m^b#-SH|Ou6-nvj%E$*GLAv?Zq-qrOh%=pR^9hwPxjnVX_p*#dXgN z%aq%Osk|PmRoCBlqVyakbMC_naQY!_U7Jm{*to&Jb9Z7O4-9LV^NW*P^4p)*J(2b7 zd?lTIG^}C0icXx^Z2n-ay)r%cUvD<`%oP(uwYgo%?x6D59#%o5(g?=~lB@x4V@$Sz z2za}eT!r@yO||ChK7&5eBH@z_o@KBhe#2b!$_XW;Y;ag$oc_x0{7{V6_Eo)uA(9#i z!T|>&5`@#VE(fg`YAD_Rm@kaXgg!u}tW3I=?StDEYP-4$OHe%?J^6U(N3L+HhEG+K zIkEn@5d4bNJdSQlHn^G|?KY#d1CR{rt-2X_{e ABLDyZ literal 0 HcmV?d00001 diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 29ce2d4865a..3337a7f9ffd 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -209,6 +209,26 @@ describe("provider_info_helpers", () => { const { logo } = getProviderLogoAndName("openai"); expect(logo).toContain("openai_small"); }); + + it("should resolve the typesafe slug and TypeSafe enum key to the TypeSafe name and bundled logo", () => { + const fromSlug = getProviderLogoAndName("typesafe"); + expect(fromSlug.displayName).toBe(Providers.TypeSafe); + expect(fromSlug.logo).toContain("typesafe"); + + const fromEnumKey = getProviderLogoAndName("TypeSafe"); + expect(fromEnumKey.displayName).toBe(Providers.TypeSafe); + expect(fromEnumKey.logo).toBe(fromSlug.logo); + }); + + it("should resolve the strands_decider slug and StrandsDecider enum key to the Strands Decider name and bundled logo", () => { + const fromSlug = getProviderLogoAndName("strands_decider"); + expect(fromSlug.displayName).toBe("Strands Decider"); + expect(fromSlug.logo).toContain("strands"); + + const fromEnumKey = getProviderLogoAndName("StrandsDecider"); + expect(fromEnumKey.displayName).toBe(Providers.StrandsDecider); + expect(fromEnumKey.logo).toBe(fromSlug.logo); + }); }); describe("getPlaceholder", () => { @@ -309,6 +329,11 @@ describe("provider_info_helpers", () => { expect(getPlaceholder("CHATGPT")).toBe("chatgpt/gpt-5.4"); }); + it("should return decision model placeholders for the TypeSafe and StrandsDecider dropdown keys", () => { + expect(getPlaceholder("TypeSafe")).toBe("typesafe/jev-latest"); + expect(getPlaceholder("StrandsDecider")).toBe("strands_decider/strands-decider-2B-hobson-v19"); + }); + it("should return default gpt-3.5-turbo placeholder for unknown provider", () => { expect(getPlaceholder("UnknownProvider" as any)).toBe("gpt-3.5-turbo"); }); @@ -412,6 +437,30 @@ describe("provider_info_helpers", () => { expect(getProviderModels("Sail" as Providers, modelMap)).toEqual(["sail/openai/gpt-oss-120b"]); }); + it("should list only typesafe decision models for the 'TypeSafe' provider key, not the OpenRouter-hosted one", () => { + const modelMap = { + "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, + "typesafe/jev-preview": { litellm_provider: "typesafe", mode: "evaluation" }, + "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" }, + "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" }, + }; + expect(getProviderModels("TypeSafe" as Providers, modelMap)).toEqual([ + "typesafe/jev-latest", + "typesafe/jev-preview", + ]); + }); + + it("should list only strands_decider models for the 'StrandsDecider' provider key", () => { + const modelMap = { + "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" }, + "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, + "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" }, + }; + expect(getProviderModels("StrandsDecider" as Providers, modelMap)).toEqual([ + "strands_decider/strands-decider-2B-hobson-v19", + ]); + }); + it("should include bedrock converse but exclude standalone bedrock_mantle when called with 'Bedrock' provider key", () => { const modelMap = { "bedrock-base": { litellm_provider: "bedrock" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index b0e33338bab..a88615df185 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -55,8 +55,10 @@ import sapLogo from "../../public/assets/logos/sap.png"; import scxAiLogo from "../../public/assets/logos/scx_ai.svg"; import snowflakeLogo from "../../public/assets/logos/snowflake.svg"; import sonioxLogo from "../../public/assets/logos/soniox.svg"; +import strandsLogo from "../../public/assets/logos/strands.svg"; import togetheraiLogo from "../../public/assets/logos/togetherai.svg"; import topazLogo from "../../public/assets/logos/topaz.svg"; +import typesafeLogo from "../../public/assets/logos/typesafe.png"; import v0Logo from "../../public/assets/logos/v0.svg"; import vercelLogo from "../../public/assets/logos/vercel.svg"; import vllmLogo from "../../public/assets/logos/vllm.png"; @@ -166,10 +168,12 @@ export enum Providers { SCX_AI = "SCX.ai", Snowflake = "Snowflake", Soniox = "Soniox", + StrandsDecider = "Strands Decider", TEXT_COMPLETION_CODESTRAL = "Text-Completion-Codestral", TogetherAI = "TogetherAI", TOPAZ = "Topaz", Triton = "Triton", + TypeSafe = "TypeSafe", V0 = "V0", VERCEL_AI_GATEWAY = "Vercel Ai Gateway", Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)", @@ -285,10 +289,12 @@ export const provider_map: Record = { SCX_AI: "scx-ai", Snowflake: "snowflake", Soniox: "soniox", + StrandsDecider: "strands_decider", TEXT_COMPLETION_CODESTRAL: "text-completion-codestral", TogetherAI: "together_ai", TOPAZ: "topaz", Triton: "triton", + TypeSafe: "typesafe", V0: "v0", VERCEL_AI_GATEWAY: "vercel_ai_gateway", Vertex_AI: "vertex_ai", @@ -384,10 +390,12 @@ export const providerLogoMap: Partial> = { [Providers.SCX_AI]: scxAiLogo.src, [Providers.Snowflake]: snowflakeLogo.src, [Providers.Soniox]: sonioxLogo.src, + [Providers.StrandsDecider]: strandsLogo.src, [Providers.TEXT_COMPLETION_CODESTRAL]: mistralLogo.src, [Providers.TogetherAI]: togetheraiLogo.src, [Providers.TOPAZ]: topazLogo.src, [Providers.Triton]: nvidiaTritonLogo.src, + [Providers.TypeSafe]: typesafeLogo.src, [Providers.V0]: v0Logo.src, [Providers.VERCEL_AI_GATEWAY]: vercelLogo.src, [Providers.Vertex_AI]: googleLogo.src, @@ -453,6 +461,8 @@ const providerPlaceholderMap: Partial> = { [Providers.Sail]: "sail/openai/gpt-oss-120b", [Providers.SCX_AI]: "scx-ai/GLM-5.2", [Providers.Snowflake]: "snowflake/mistral-7b", + [Providers.StrandsDecider]: "strands_decider/strands-decider-2B-hobson-v19", + [Providers.TypeSafe]: "typesafe/jev-latest", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", [Providers.Voyage]: "voyage/", From 67494a043f16b5c0c86abe5633f2cca80e39fe4b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:34:06 -0700 Subject: [PATCH 06/18] chore(decisions): remove the System One converters and OpenAI spec types left dead by #45214 (#45442) #45214 routed both decision formats through the shared decisions IR in litellm/llms/base_llm/decisions/transformation.py and litellm/types/decisions.py. That left litellm/llms/base_llm/decisions/systemone.py (to_system_one_request, question_keys, to_decisions_response, the SystemOne* models and their adapter) and litellm/types/openai_decisions.py with no importer outside their own unit tests, so both modules and both test files go. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> (cherry picked from commit 0e48048bd56c913488b5c52e247cd7a0f6d9bf30) --- litellm/llms/base_llm/decisions/systemone.py | 247 ----------------- litellm/types/openai_decisions.py | 162 ----------- .../llms/base_llm/decisions/test_systemone.py | 262 ------------------ tests/unit/types/test_openai_decisions.py | 150 ---------- 4 files changed, 821 deletions(-) delete mode 100644 litellm/llms/base_llm/decisions/systemone.py delete mode 100644 litellm/types/openai_decisions.py delete mode 100644 tests/unit/llms/base_llm/decisions/test_systemone.py delete mode 100644 tests/unit/types/test_openai_decisions.py diff --git a/litellm/llms/base_llm/decisions/systemone.py b/litellm/llms/base_llm/decisions/systemone.py deleted file mode 100644 index 43be31b0c30..00000000000 --- a/litellm/llms/base_llm/decisions/systemone.py +++ /dev/null @@ -1,247 +0,0 @@ -"""The Jev / System One wire shape and its translation to and from the OpenAI Decisions shape. - -System One (TypeSafe, Perplexity, OpenRouter, Cloudflare Clef, Strands Decider) takes -{"model", "state", "questions": {name: question}} and answers with {"model", "answers": {name: answer}, "usage"}. -Predicates are `noul` questions, choice options are a `criteria` map, score levels are a `criteria` list. -""" - -import itertools -from collections.abc import Mapping, Sequence -from typing import Final, Literal, TypeAlias - -from pydantic import ConfigDict, TypeAdapter -from typing_extensions import assert_never - -from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.types.llms.base import LiteLLMPydanticObjectBase -from litellm.types.openai_decisions import ( - ChoiceAnswer, - ChoiceProbability, - ChoiceQuestion, - DecisionAnswer, - DecisionChoice, - DecisionInput, - DecisionInputMessage, - DecisionInputPart, - DecisionInputTokensDetails, - DecisionOutputTokensDetails, - DecisionQuestion, - DecisionsRequest, - DecisionsRequestBody, - DecisionsResponse, - DecisionUsage, - PredicateAnswer, - PredicateQuestion, - ScoreAnswer, - ScoreProbability, - ScoreQuestion, -) - - -class SystemOneObjectBase(LiteLLMPydanticObjectBase): - model_config = ConfigDict(extra="allow", frozen=True) - - -class SystemOneNoulAnswer(SystemOneObjectBase): - type: Literal["noul"] - noul: float - - -class SystemOneChoiceAnswer(SystemOneObjectBase): - type: Literal["choice"] - choice: str - confidence: float - probabilities: Mapping[str, float] - - -class SystemOneScoreAnswer(SystemOneObjectBase): - type: Literal["score"] - score: float - confidence: float - probabilities: Mapping[str, float] - - -SystemOneAnswer: TypeAlias = SystemOneNoulAnswer | SystemOneChoiceAnswer | SystemOneScoreAnswer - - -class SystemOneUsage(SystemOneObjectBase): - input_tokens: int = 0 - output_tokens: int = 0 - - -class SystemOneResponse(SystemOneObjectBase): - model: str | None = None - answers: Mapping[str, SystemOneAnswer] - usage: SystemOneUsage | None = None - - -SYSTEM_ONE_RESPONSE_ADAPTER: Final[TypeAdapter[SystemOneResponse]] = TypeAdapter(SystemOneResponse) - - -def _unsupported(what: str, custom_llm_provider: str) -> BaseLLMException: - return BaseLLMException( - status_code=400, - message=f"Decisions provider '{custom_llm_provider}' does not support {what}", - ) - - -def to_system_one_request(model: str, body: DecisionsRequestBody, custom_llm_provider: str) -> dict[str, object]: - keys: Final = question_keys(body.questions, custom_llm_provider) - return { - "model": model, - "state": _state(body.input, custom_llm_provider), - "questions": { - key: _question(question, custom_llm_provider) for key, question in zip(keys, body.questions, strict=True) - }, - } - - -def question_keys(questions: Sequence[DecisionQuestion], custom_llm_provider: str) -> tuple[str, ...]: - """System One keys questions and answers by name, so unnamed questions get a positional key.""" - names: Final = tuple(question.name for question in questions if question.name is not None) - if len(set(names)) != len(names): - raise BaseLLMException( - status_code=400, - message=f"Decisions provider '{custom_llm_provider}' requires a unique name per question", - ) - taken: Final = frozenset(names) - return tuple( - question.name if question.name is not None else _positional_key(index, taken) - for index, question in enumerate(questions) - ) - - -def _positional_key(index: int, taken: frozenset[str]) -> str: - candidates: Final = (f"{'_' * depth}q{index}" for depth in itertools.count()) - return next(key for key in candidates if key not in taken) - - -def _state(input_value: DecisionInput, custom_llm_provider: str) -> str: - if isinstance(input_value, str): - return input_value - return "\n".join(_message_text(message, custom_llm_provider) for message in input_value) - - -def _message_text(message: DecisionInputMessage, custom_llm_provider: str) -> str: - if isinstance(message.content, str): - return message.content - return "\n".join(_part_text(part, custom_llm_provider) for part in message.content) - - -def _part_text(part: DecisionInputPart, custom_llm_provider: str) -> str: - if part.type != "input_text": - raise _unsupported("input_image parts", custom_llm_provider) - return part.text - - -def _question(question: DecisionQuestion, custom_llm_provider: str) -> dict[str, object]: - match question: - case PredicateQuestion(): - return {"type": "noul", "instructions": question.instructions} - case ChoiceQuestion(): - return { - "type": "choice", - "instructions": question.instructions, - "criteria": _choice_criteria(question, custom_llm_provider), - } - case ScoreQuestion(): - return { - "type": "score", - "instructions": question.instructions, - "criteria": [_level_text(level.label, level.description) for level in question.levels], - } - case _: - assert_never(question) - - -def _choice_criteria(question: ChoiceQuestion, custom_llm_provider: str) -> dict[str, str | None]: - values: Final = tuple(_choice_key(choice, custom_llm_provider) for choice in question.choices) - if len(set(values)) != len(values): - raise _unsupported("repeated choice values", custom_llm_provider) - return {value: choice.description for value, choice in zip(values, question.choices, strict=True)} - - -def _choice_key(choice: DecisionChoice, custom_llm_provider: str) -> str: - if not isinstance(choice.value, str): - raise _unsupported("boolean choice values", custom_llm_provider) - return choice.value - - -def _level_text(label: str, description: str | None) -> str: - return description if description is not None else label - - -def to_decisions_response( - system_one: SystemOneResponse, - request: DecisionsRequest, - custom_llm_provider: str, -) -> DecisionsResponse: - keys: Final = question_keys(request.body.questions, custom_llm_provider) - return DecisionsResponse( - model=system_one.model if system_one.model is not None else request.model, - answers=[ - _answer(key, question, system_one.answers, custom_llm_provider) - for key, question in zip(keys, request.body.questions, strict=True) - ], - usage=_usage(system_one.usage), - ) - - -def _answer( - key: str, - question: DecisionQuestion, - answers: Mapping[str, SystemOneAnswer], - custom_llm_provider: str, -) -> DecisionAnswer: - match question, answers.get(key): - case PredicateQuestion(), SystemOneNoulAnswer() as answer: - return PredicateAnswer(type="predicate", name=question.name, probability=answer.noul) - case ChoiceQuestion(), SystemOneChoiceAnswer() as answer: - return ChoiceAnswer( - type="choice", - name=question.name, - choice=answer.choice, - probabilities=_choice_probabilities(question, answer), - confidence=answer.confidence, - ) - case ScoreQuestion(), SystemOneScoreAnswer() as answer: - return ScoreAnswer( - type="score", - name=question.name, - score=answer.score, - probabilities=_score_probabilities(question, answer), - confidence=answer.confidence, - ) - case _: - raise BaseLLMException( - status_code=500, - message=( - f"Decisions provider '{custom_llm_provider}' returned no {question.type} answer " - f"for question '{key}'" - ), - ) - - -def _choice_probabilities(question: ChoiceQuestion, answer: SystemOneChoiceAnswer) -> list[ChoiceProbability]: - return [ - ChoiceProbability(value=choice.value, probability=answer.probabilities.get(str(choice.value), 0.0)) - for choice in question.choices - ] - - -def _score_probabilities(question: ScoreQuestion, answer: SystemOneScoreAnswer) -> list[ScoreProbability]: - return [ - ScoreProbability(value=index, label=level.label, probability=answer.probabilities.get(str(index), 0.0)) - for index, level in enumerate(question.levels) - ] - - -def _usage(usage: SystemOneUsage | None) -> DecisionUsage: - counted: Final = usage if usage is not None else SystemOneUsage() - return DecisionUsage( - input_tokens=counted.input_tokens, - input_tokens_details=DecisionInputTokensDetails(cached_tokens=0, cache_write_tokens=0), - output_tokens=counted.output_tokens, - output_tokens_details=DecisionOutputTokensDetails(reasoning_tokens=0), - total_tokens=counted.input_tokens + counted.output_tokens, - ) diff --git a/litellm/types/openai_decisions.py b/litellm/types/openai_decisions.py deleted file mode 100644 index 074c0a803f3..00000000000 --- a/litellm/types/openai_decisions.py +++ /dev/null @@ -1,162 +0,0 @@ -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from typing import Annotated, Literal, TypeAlias - -from pydantic import ConfigDict, Field, PrivateAttr, StrictBool, StrictStr - -from litellm.types.llms.base import LiteLLMPydanticObjectBase - -ChoiceValue: TypeAlias = StrictStr | StrictBool - - -class DecisionsObjectBase(LiteLLMPydanticObjectBase): - model_config = ConfigDict(extra="allow", frozen=True) - - -class DecisionInputText(DecisionsObjectBase): - type: Literal["input_text"] - text: str - - -class DecisionInputImage(DecisionsObjectBase): - type: Literal["input_image"] - image_url: str - detail: Literal["low", "high", "auto", "original"] | None = None - - -DecisionInputPart: TypeAlias = Annotated[DecisionInputText | DecisionInputImage, Field(discriminator="type")] - - -class DecisionInputMessage(DecisionsObjectBase): - role: Literal["user"] - content: str | Sequence[DecisionInputPart] - type: Literal["message"] | None = None - - -DecisionInput: TypeAlias = str | Sequence[DecisionInputMessage] - - -class DecisionChoice(DecisionsObjectBase): - value: ChoiceValue - description: str | None = None - - -class DecisionLevel(DecisionsObjectBase): - label: str - description: str | None = None - - -class PredicateQuestion(DecisionsObjectBase): - type: Literal["predicate"] - instructions: str - name: str | None = None - - -class ChoiceQuestion(DecisionsObjectBase): - type: Literal["choice"] - instructions: str - choices: Sequence[DecisionChoice] - name: str | None = None - - -class ScoreQuestion(DecisionsObjectBase): - type: Literal["score"] - instructions: str - levels: Sequence[DecisionLevel] - name: str | None = None - - -DecisionQuestion: TypeAlias = Annotated[ - PredicateQuestion | ChoiceQuestion | ScoreQuestion, - Field(discriminator="type"), -] - -DecisionQuestions: TypeAlias = Sequence[DecisionQuestion] - - -class DecisionsRequestBody(DecisionsObjectBase): - input: DecisionInput - questions: DecisionQuestions - safety_identifier: str | None = None - - -@dataclass(frozen=True, slots=True) -class DecisionsRequest: - model: str - body: DecisionsRequestBody - - -class PredicateAnswer(DecisionsObjectBase): - type: Literal["predicate"] - name: str | None = None - probability: float - - -class ChoiceProbability(DecisionsObjectBase): - value: ChoiceValue - probability: float - - -class ChoiceAnswer(DecisionsObjectBase): - type: Literal["choice"] - name: str | None = None - choice: ChoiceValue - probabilities: Sequence[ChoiceProbability] - confidence: float - - -class ScoreProbability(DecisionsObjectBase): - value: int - label: str - probability: float - - -class ScoreAnswer(DecisionsObjectBase): - type: Literal["score"] - name: str | None = None - score: float - probabilities: Sequence[ScoreProbability] - confidence: float - - -class RefusalAnswer(DecisionsObjectBase): - type: Literal["refusal"] - name: str | None = None - - -DecisionAnswer: TypeAlias = Annotated[ - PredicateAnswer | ChoiceAnswer | ScoreAnswer | RefusalAnswer, - Field(discriminator="type"), -] - - -class DecisionInputTokensDetails(DecisionsObjectBase): - cached_tokens: int - cache_write_tokens: int - - -class DecisionOutputTokensDetails(DecisionsObjectBase): - reasoning_tokens: int - - -class DecisionUsage(DecisionsObjectBase): - input_tokens: int - input_tokens_details: DecisionInputTokensDetails - output_tokens: int - output_tokens_details: DecisionOutputTokensDetails - total_tokens: int - - -class DecisionsResponse(DecisionsObjectBase): - model: str - answers: Sequence[DecisionAnswer] - usage: DecisionUsage - - _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) - - @property - def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation - return self._hidden_params - - def set_hidden_params(self, params: Mapping[str, object]) -> None: - self._hidden_params.update(params) diff --git a/tests/unit/llms/base_llm/decisions/test_systemone.py b/tests/unit/llms/base_llm/decisions/test_systemone.py deleted file mode 100644 index 42a1c41b3d3..00000000000 --- a/tests/unit/llms/base_llm/decisions/test_systemone.py +++ /dev/null @@ -1,262 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping, Sequence -from typing import Final - -import pytest -from pydantic import TypeAdapter - -from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.base_llm.decisions.systemone import ( - SYSTEM_ONE_RESPONSE_ADAPTER, - question_keys, - to_decisions_response, - to_system_one_request, -) -from litellm.types.openai_decisions import ( - ChoiceAnswer, - DecisionsRequest, - DecisionsRequestBody, - PredicateAnswer, - ScoreAnswer, -) - -_INPUT: Final = "The export job hangs at 99% and never finishes" -_QUESTIONS: Final[Sequence[Mapping[str, object]]] = ( - {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, - { - "type": "choice", - "name": "sentiment", - "instructions": "How does the customer feel?", - "choices": [{"value": "positive"}, {"value": "negative", "description": "unhappy"}], - }, - { - "type": "score", - "name": "severity", - "instructions": "How severe is it?", - "levels": [{"label": "none"}, {"label": "low"}, {"label": "high", "description": "blocks users"}], - }, -) -_SYSTEM_ONE_QUESTIONS: Final[Mapping[str, object]] = { - "is_defect": {"type": "noul", "instructions": "Is this a defect?"}, - "sentiment": { - "type": "choice", - "instructions": "How does the customer feel?", - "criteria": {"positive": None, "negative": "unhappy"}, - }, - "severity": {"type": "score", "instructions": "How severe is it?", "criteria": ["none", "low", "blocks users"]}, -} -_SYSTEM_ONE_RESPONSE: Final[Mapping[str, object]] = { - "model": "jev-1.13", - "answers": { - "is_defect": {"type": "noul", "noul": 0.9}, - "sentiment": { - "type": "choice", - "choice": "positive", - "confidence": 0.8, - "probabilities": {"positive": 0.8, "negative": 0.2}, - }, - "severity": { - "type": "score", - "score": 1, - "confidence": 0.7, - "legend": {"0": "none", "1": "low", "2": "high"}, - "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}, - }, - }, - "usage": {"input_tokens": 367, "output_tokens": 3}, -} -_EXPECTED_ANSWERS: Final[Sequence[Mapping[str, object]]] = ( - {"type": "predicate", "name": "is_defect", "probability": 0.9}, - { - "type": "choice", - "name": "sentiment", - "choice": "positive", - "probabilities": [{"value": "positive", "probability": 0.8}, {"value": "negative", "probability": 0.2}], - "confidence": 0.8, - }, - { - "type": "score", - "name": "severity", - "score": 1.0, - "probabilities": [ - {"value": 0, "label": "none", "probability": 0.1}, - {"value": 1, "label": "low", "probability": 0.8}, - {"value": 2, "label": "high", "probability": 0.1}, - ], - "confidence": 0.7, - }, -) -_EXPECTED_USAGE: Final[Mapping[str, object]] = { - "input_tokens": 367, - "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, - "output_tokens": 3, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 370, -} -_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) - - -def _body( - input_value: object = _INPUT, - questions: Sequence[Mapping[str, object]] = _QUESTIONS, -) -> DecisionsRequestBody: - return _BODY_ADAPTER.validate_python({"input": input_value, "questions": questions}) - - -def _request( - input_value: object = _INPUT, - questions: Sequence[Mapping[str, object]] = _QUESTIONS, - model: str = "jev-1.13", -) -> DecisionsRequest: - return DecisionsRequest(model=model, body=_body(input_value, questions)) - - -def _predicate(name: str | None = "is_defect") -> tuple[Mapping[str, object]]: - return ({"type": "predicate", "name": name, "instructions": "Is this a defect?"},) - - -def test_openai_request_becomes_the_system_one_body() -> None: - assert to_system_one_request("jev-1.13", _body(), "typesafe") == { - "model": "jev-1.13", - "state": _INPUT, - "questions": _SYSTEM_ONE_QUESTIONS, - } - - -def test_system_one_answers_become_openai_answers_in_question_order() -> None: - response: Final = to_decisions_response( - SYSTEM_ONE_RESPONSE_ADAPTER.validate_python(_SYSTEM_ONE_RESPONSE), _request(), "typesafe" - ) - - assert response.model_dump(mode="json") == { - "model": "jev-1.13", - "answers": list(_EXPECTED_ANSWERS), - "usage": _EXPECTED_USAGE, - } - assert isinstance(response.answers[0], PredicateAnswer) - assert isinstance(response.answers[1], ChoiceAnswer) - assert isinstance(response.answers[2], ScoreAnswer) - - -def test_user_messages_are_joined_into_one_system_one_state() -> None: - messages: Final = [ - {"role": "user", "content": "first"}, - { - "role": "user", - "content": [{"type": "input_text", "text": "second"}, {"type": "input_text", "text": "third"}], - }, - ] - - body: Final = to_system_one_request("jev-1.13", _body(messages, _predicate()), "typesafe") - - assert body["state"] == "first\nsecond\nthird" - - -@pytest.mark.parametrize( - ("label", "input_value", "questions"), - ( - ( - "input_image", - [{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]}], - _predicate(), - ), - ("boolean choice", _INPUT, [{"type": "choice", "instructions": "Refund?", "choices": [{"value": True}]}]), - ("unique name", _INPUT, [*_predicate(), *_predicate()]), - ( - "repeated choice", - _INPUT, - [{"type": "choice", "instructions": "Refund?", "choices": [{"value": "yes"}, {"value": "yes"}]}], - ), - ), -) -def test_what_system_one_cannot_express_is_a_400( - label: str, - input_value: object, - questions: Sequence[Mapping[str, object]], -) -> None: - with pytest.raises(BaseLLMException, match=label) as error: - to_system_one_request("jev-1.13", _body(input_value, questions), "perplexity") - - assert error.value.status_code == 400 - assert "perplexity" in error.value.message - - -def test_unnamed_questions_get_positional_keys_that_never_shadow_a_supplied_name() -> None: - body: Final = _body(questions=[*_predicate(None), *_predicate("q0"), *_predicate(None)]) - - assert question_keys(body.questions, "typesafe") == ("_q0", "q0", "q2") - noul: Final = {"type": "noul", "instructions": "Is this a defect?"} - assert to_system_one_request("jev-1.13", body, "typesafe")["questions"] == {"_q0": noul, "q0": noul, "q2": noul} - - -def test_positional_answers_come_back_in_question_order_without_a_name() -> None: - request: Final = _request(questions=[*_predicate(None), *_predicate("q0")]) - system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python( - {"answers": {"_q0": {"type": "noul", "noul": 0.25}, "q0": {"type": "noul", "noul": 0.75}}} - ) - - response: Final = to_decisions_response(system_one, request, "typesafe") - - assert [answer.model_dump(mode="json") for answer in response.answers] == [ - {"type": "predicate", "name": None, "probability": 0.25}, - {"type": "predicate", "name": "q0", "probability": 0.75}, - ] - assert response.usage.model_dump(mode="json") == { - **_EXPECTED_USAGE, - "input_tokens": 0, - "output_tokens": 0, - "total_tokens": 0, - } - - -def test_a_reply_without_a_model_reports_the_requested_model() -> None: - system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python( - {k: v for k, v in _SYSTEM_ONE_RESPONSE.items() if k != "model"} - ) - - response: Final = to_decisions_response(system_one, _request(model="typesafe/jev-1.13.0"), "typesafe") - - assert response.model == "typesafe/jev-1.13.0" - - -def test_a_choice_the_provider_left_out_of_probabilities_is_reported_at_zero() -> None: - system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python( - { - "answers": { - "sentiment": { - "type": "choice", - "choice": "positive", - "confidence": 1.0, - "probabilities": {"positive": 1.0}, - } - } - } - ) - request: Final = _request(questions=_QUESTIONS[1:2]) - - response: Final = to_decisions_response(system_one, request, "typesafe") - - assert response.answers[0].model_dump(mode="json") == { - "type": "choice", - "name": "sentiment", - "choice": "positive", - "probabilities": [{"value": "positive", "probability": 1.0}, {"value": "negative", "probability": 0.0}], - "confidence": 1.0, - } - - -@pytest.mark.parametrize( - "answers", - ( - {}, - {"is_defect": {"type": "choice", "choice": "yes", "confidence": 1.0, "probabilities": {"yes": 1.0}}}, - ), -) -def test_a_reply_without_a_matching_answer_is_a_server_error(answers: Mapping[str, object]) -> None: - system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python({"answers": answers}) - - with pytest.raises(BaseLLMException, match="no predicate answer for question 'is_defect'") as error: - to_decisions_response(system_one, _request(questions=_predicate()), "typesafe") - - assert error.value.status_code == 500 diff --git a/tests/unit/types/test_openai_decisions.py b/tests/unit/types/test_openai_decisions.py deleted file mode 100644 index d9387d644d5..00000000000 --- a/tests/unit/types/test_openai_decisions.py +++ /dev/null @@ -1,150 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping -from typing import Final - -import pytest -from pydantic import TypeAdapter, ValidationError - -from litellm.types.openai_decisions import ( - ChoiceAnswer, - DecisionsRequestBody, - DecisionsResponse, - RefusalAnswer, - ScoreQuestion, -) - -_REQUEST: Final[Mapping[str, object]] = { - "input": [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "Is this receipt a valid business expense?"}, - {"type": "input_image", "image_url": "https://example.com/receipt.png", "detail": "high"}, - ], - } - ], - "questions": [ - {"type": "predicate", "name": "is_expense", "instructions": "Is this a business expense?"}, - { - "type": "choice", - "name": "approve", - "instructions": "Should this be approved?", - "choices": [{"value": True, "description": "approve"}, {"value": False, "description": "reject"}], - }, - { - "type": "score", - "name": "risk", - "instructions": "How risky is this expense?", - "levels": [{"label": "low"}, {"label": "high", "description": "needs a manager"}], - }, - ], - "safety_identifier": "user-123", -} -_RESPONSE: Final[Mapping[str, object]] = { - "model": "gpt-6-luna", - "answers": [ - {"type": "predicate", "name": "is_expense", "probability": 0.92}, - { - "type": "choice", - "name": "approve", - "choice": True, - "probabilities": [{"value": True, "probability": 0.7}, {"value": False, "probability": 0.3}], - "confidence": 0.7, - }, - {"type": "refusal", "name": "risk"}, - ], - "usage": { - "input_tokens": 120, - "input_tokens_details": {"cached_tokens": 100, "cache_write_tokens": 0}, - "output_tokens": 12, - "output_tokens_details": {"reasoning_tokens": 4}, - "total_tokens": 132, - }, -} -_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) -_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) - - -def test_the_documented_request_round_trips_with_its_boolean_choices_and_image_part() -> None: - request: Final = _REQUEST_ADAPTER.validate_python(_REQUEST) - - assert request.model_dump(mode="json", exclude_none=True) == _REQUEST - assert isinstance(request.questions[2], ScoreQuestion) - - -def test_the_documented_response_keeps_answer_order_refusals_and_token_details() -> None: - response: Final = _RESPONSE_ADAPTER.validate_python(_RESPONSE) - - assert response.model_dump(mode="json") == _RESPONSE - assert isinstance(response.answers[1], ChoiceAnswer) - assert isinstance(response.answers[2], RefusalAnswer) - - -_OFF_SPEC: Final[tuple[tuple[str, object], ...]] = ( - ("questions", [{"type": "noul", "name": "q", "instructions": "x"}]), - ("questions", [{"type": "choice", "name": "q", "instructions": "x", "choices": [{"value": 1}]}]), - ("questions", [{"type": "score", "name": "q", "levels": [{"label": "low"}]}]), - ("input", {"state": "not an OpenAI input"}), -) - - -@pytest.mark.parametrize(("field", "value"), _OFF_SPEC) -def test_requests_off_the_spec_are_rejected(field: str, value: object) -> None: - with pytest.raises(ValidationError): - _REQUEST_ADAPTER.validate_python({**_REQUEST, field: value}) - - -_EMPTY_COLLECTIONS: Final[tuple[tuple[str, list[object]], ...]] = ( - ("questions", []), - ("questions", [{"type": "choice", "name": "q", "instructions": "x", "choices": []}]), - ("questions", [{"type": "score", "name": "q", "instructions": "x", "levels": []}]), -) - - -@pytest.mark.parametrize(("field", "value"), _EMPTY_COLLECTIONS) -def test_empty_collections_are_left_for_the_provider_to_judge(field: str, value: list[object]) -> None: - request: Final = _REQUEST_ADAPTER.validate_python({**_REQUEST, field: value}) - - assert request.model_dump(mode="json", exclude_none=True)[field] == value - - -def test_choice_values_keep_their_type_so_a_string_true_and_a_boolean_true_stay_distinct() -> None: - answer: Final = { - "type": "choice", - "name": "approve", - "choice": "true", - "probabilities": [{"value": "true", "probability": 0.6}, {"value": True, "probability": 0.4}], - "confidence": 0.6, - } - - response: Final = _RESPONSE_ADAPTER.validate_python({**_RESPONSE, "answers": [answer]}) - - assert response.model_dump(mode="json")["answers"] == [answer] - with pytest.raises(ValidationError): - _RESPONSE_ADAPTER.validate_python({**_RESPONSE, "answers": [{**answer, "choice": 1}]}) - - -_OFF_SPEC_RESPONSE: Final[tuple[str, ...]] = ("model", "usage") - - -@pytest.mark.parametrize("field", _OFF_SPEC_RESPONSE) -def test_responses_missing_a_required_field_are_rejected(field: str) -> None: - with pytest.raises(ValidationError): - _RESPONSE_ADAPTER.validate_python({k: v for k, v in _RESPONSE.items() if k != field}) - - -def test_usage_without_token_details_is_rejected() -> None: - usage: Final = {"input_tokens": 120, "output_tokens": 12, "total_tokens": 132} - - with pytest.raises(ValidationError): - _RESPONSE_ADAPTER.validate_python({**_RESPONSE, "usage": usage}) - - -def test_hidden_params_live_outside_the_wire_body() -> None: - response: Final = _RESPONSE_ADAPTER.validate_python(_RESPONSE) - - response.set_hidden_params({"custom_llm_provider": "openai"}) - - assert response.hidden_params == {"custom_llm_provider": "openai"} - assert "_hidden_params" not in response.model_dump(mode="json") From e855acee1d131b03a18bfa54dc83de1219badf48 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:41:51 -0700 Subject: [PATCH 07/18] feat(decisions): add hosted_vllm provider (#45501) * feat(decisions): add hosted_vllm provider Adds HostedVLLMDecisionsConfig so hosted_vllm/ works on POST /v1/systemone and POST /v1/decisions against a vLLM server that serves the System One body (vllm-project/vllm#59299, after v0.31.0). vLLM only supports choice questions and rejects others, so the base decisions config gains a per-provider health_check_questions attribute and the evaluation-mode health probe sends a choice question for hosted_vllm while every other provider keeps the noul probe. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): move the hosted_vllm placeholder key to constants Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 61e5f2dd3c1e0671c07f25fa44f10097d161740e) --- litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 5 + litellm/constants.py | 1 + .../health_check_helpers.py | 21 ++- .../llms/base_llm/decisions/transformation.py | 3 + .../hosted_vllm/decisions/transformation.py | 26 ++++ litellm/utils.py | 2 + provider_endpoints_support.json | 3 +- .../management/test_model_health_check.py | 39 +++++ .../providers/test_decisions_wire.py | 3 + tests/integration/proxy_config.yaml | 4 + .../decisions/bases/hosted_vllm.py | 67 +++++++++ .../basic/test_decisions_basic_hosted_vllm.py | 11 ++ tests/unit/decisions/test_main.py | 133 ++++++++++++++++++ .../test_health_check_helpers.py | 75 ++++++++++ 15 files changed, 393 insertions(+), 3 deletions(-) create mode 100644 litellm/llms/hosted_vllm/decisions/transformation.py create mode 100644 tests/integration/translation/decisions/bases/hosted_vllm.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 49a64d89376..c559ff8cf11 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1669,6 +1669,9 @@ if TYPE_CHECKING: from .llms.strands_decider.decisions.transformation import ( StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig, ) + from .llms.hosted_vllm.decisions.transformation import ( + HostedVLLMDecisionsConfig as HostedVLLMDecisionsConfig, + ) from .llms.openai.decisions.transformation import ( OpenAIDecisionsConfig as OpenAIDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 2e68fb3459b..cb5ec8a016b 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -158,6 +158,7 @@ LLM_CONFIG_NAMES: Final = ( "OpenRouterDecisionsConfig", "CloudflareDecisionsConfig", "StrandsDeciderDecisionsConfig", + "HostedVLLMDecisionsConfig", "OpenAIDecisionsConfig", "NvidiaNimRerankConfig", "NvidiaNimRankingConfig", @@ -710,6 +711,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.strands_decider.decisions.transformation", "StrandsDeciderDecisionsConfig", ), + "HostedVLLMDecisionsConfig": ( + ".llms.hosted_vllm.decisions.transformation", + "HostedVLLMDecisionsConfig", + ), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "NvidiaNimRerankConfig": ( ".llms.nvidia_nim.rerank.transformation", diff --git a/litellm/constants.py b/litellm/constants.py index af4d1268c03..9ee73ec7b98 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -266,6 +266,7 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int( # Provider-specific API base URLs XAI_API_BASE: Final = "https://api.x.ai/v1" +PLACEHOLDER_API_KEY: Final = "fake-api-key" OPEN_SANDBOX_API_BASE_ENV_VAR: Final = "OPEN_SANDBOX_API_BASE" OPEN_SANDBOX_API_KEY_ENV_VAR: Final = "OPEN_SANDBOX_API_KEY" OPEN_SANDBOX_DEFAULT_TEMPLATE: Final = "opensandbox/code-interpreter:v1.1.0" diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 41965404351..6d6eb1b2b5b 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -3,7 +3,7 @@ Helper functions for health check calls. """ import base64 -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from typing import TYPE_CHECKING, Final, Literal from litellm.llms.base_llm.ocr.transformation import DocumentType @@ -30,6 +30,21 @@ def get_image_file_for_health_check() -> bytes: return base64.b64decode(TEST_IMAGE_BASE64) +def _decisions_health_check_questions(model: str, custom_llm_provider: str) -> Mapping[str, Mapping[str, object]]: + import litellm + from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + from litellm.utils import ProviderConfigManager + + provider: Final = next((member for member in litellm.LlmProviders if member.value == custom_llm_provider), None) + config: Final = ( + None + if provider is None + else ProviderConfigManager.get_provider_decisions_config(model=model, provider=provider) + ) + questions: Final = BaseDecisionsConfig.health_check_questions if config is None else config.health_check_questions + return {name: dict(question) for name, question in questions.items()} + + def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType: native: Final = NATIVE_OCR_HEALTH_CHECK_DOCUMENT.load() if native is None: @@ -262,7 +277,9 @@ class HealthCheckHelpers: **DECISIONS_CALL_PARAMS.validate_python( { "state": prompt or "health check", - "questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}}, + "questions": _decisions_health_check_questions( + model=model, custom_llm_provider=custom_llm_provider + ), **_filter_model_params(model_params=model_params), } ) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index a8d14ca59a3..9d1ed188d7b 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -301,6 +301,9 @@ class BaseDecisionsConfig(ABC): api_key_env: tuple[str, ...] = () api_base_env: tuple[str, ...] = () api_key_required: bool = True + health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType( + {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})} + ) def get_default_api_base(self) -> str | None: return None diff --git a/litellm/llms/hosted_vllm/decisions/transformation.py b/litellm/llms/hosted_vllm/decisions/transformation.py new file mode 100644 index 00000000000..ef99f2de173 --- /dev/null +++ b/litellm/llms/hosted_vllm/decisions/transformation.py @@ -0,0 +1,26 @@ +from types import MappingProxyType +from typing import Final + +from litellm.constants import PLACEHOLDER_API_KEY +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class HostedVLLMDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("HOSTED_VLLM_API_KEY",) + api_base_env = ("HOSTED_VLLM_API_BASE",) + api_key_required = False + health_check_questions = MappingProxyType( + { + "reachable": MappingProxyType( + { + "type": "choice", + "instructions": "Is the service reachable?", + "criteria": {"yes": None, "no": None}, + } + ) + } + ) + + def resolve_api_key(self, api_key: str | None) -> str | None: + resolved: Final = super().resolve_api_key(api_key) + return None if resolved == PLACEHOLDER_API_KEY else resolved diff --git a/litellm/utils.py b/litellm/utils.py index fe2bac6ca8c..5a61cd087de 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8910,6 +8910,8 @@ class ProviderConfigManager: return litellm.CloudflareDecisionsConfig() if provider == LlmProviders.STRANDS_DECIDER: return litellm.StrandsDeciderDecisionsConfig() + if provider == LlmProviders.HOSTED_VLLM: + return litellm.HostedVLLMDecisionsConfig() if provider == LlmProviders.OPENAI: return litellm.OpenAIDecisionsConfig() return None diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 7eefae0f748..4ce6924edb0 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1369,7 +1369,8 @@ "rerank": true, "a2a": true, "interactions": true, - "video_generations": true + "video_generations": true, + "systemone": true } }, "huggingface": { diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 17bd1c2ebfe..11b902b540a 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -24,6 +24,16 @@ _CONFIGURED_PROBE_REPLY: Final = JsonResponse( "usage": {"input_tokens": 10, "output_tokens": 1}, }, ) +_VLLM_PROBE_REPLY: Final = JsonResponse( + content_type="application/json", + body={ + "model": "Qwen/Qwen3-0.6B", + "answers": { + "reachable": {"type": "choice", "choice": "yes", "confidence": 1.0, "probabilities": {"yes": 1.0, "no": 0.0}} + }, + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, +) _STRANDS_PROBE_REPLY: Final = JsonResponse( content_type="application/json", body={ @@ -147,3 +157,32 @@ def test_evaluation_mode_health_check_of_the_self_hosted_strands_model_resolves_ }, ) ] + + +def test_evaluation_mode_health_check_of_hosted_vllm_sends_a_choice_probe(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _VLLM_PROBE_REPLY) + scenario.cleanups.callback(delete_scenario, handle) + model: Final = scenario.model( + model="hosted_vllm/Qwen/Qwen3-0.6B", + api_base=handle.api_base(), + model_info={"mode": "evaluation"}, + ) + report: Final = _health_report(gateway, model) + assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report + assert _probes_sent_to(gateway, handle) == [ + ( + f"/{handle.scenario_id}/v1/systemone", + { + "model": "Qwen/Qwen3-0.6B", + "state": os.environ.get("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"), + "questions": { + "reachable": { + "type": "choice", + "instructions": "Is the service reachable?", + "criteria": {"yes": None, "no": None}, + } + }, + }, + ) + ] diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index db6629a43d2..2f75d05bf46 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -94,6 +94,9 @@ _PROVIDERS: Final = ( _Provider( "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None ), + _Provider( + "hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None + ), _Provider( "cloudflare", "cloudflare/clef", diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 993de299526..4cbd270a53c 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -23,3 +23,7 @@ model_list: litellm_params: model: strands_decider/strands-decider-2B-hobson-v19 api_base: http://127.0.0.1:8191 + - model_name: hosted_vllm/Qwen/Qwen3-0.6B + litellm_params: + model: hosted_vllm/Qwen/Qwen3-0.6B + api_base: http://127.0.0.1:8191 diff --git a/tests/integration/translation/decisions/bases/hosted_vllm.py b/tests/integration/translation/decisions/bases/hosted_vllm.py new file mode 100644 index 00000000000..2da37f83a86 --- /dev/null +++ b/tests/integration/translation/decisions/bases/hosted_vllm.py @@ -0,0 +1,67 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from a real vLLM serve (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-09 from a vLLM CPU build of vllm main 702ce313f serving Qwen/Qwen3-0.6B. +""" +QWEN3_0_6B_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "hosted_vllm/Qwen/Qwen3-0.6B", + "state": "Help! My payouts have been failing for 3 days!", + "questions": { + "is_urgent": { + "type": "choice", + "instructions": "Does this convey urgency?", + "criteria": {"yes": "the message conveys urgency", "no": "the message does not convey urgency"}, + } + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"content-type": "application/json"}, + expected_provider_request={ + "model": "Qwen/Qwen3-0.6B", + "state": "Help! My payouts have been failing for 3 days!", + "questions": { + "is_urgent": { + "type": "choice", + "instructions": "Does this convey urgency?", + "criteria": {"yes": "the message conveys urgency", "no": "the message does not convey urgency"}, + } + }, + }, + mock_provider_response={ + "id": "decision-9c26c028d7f4457e", + "object": "structured_decision", + "created": 1791517765, + "model": "Qwen/Qwen3-0.6B", + "answers": { + "is_urgent": { + "type": "choice", + "choice": "yes", + "probabilities": {"yes": 0.9669140238756562, "no": 0.03308597612434374}, + "confidence": 0.9649145337527952, + } + }, + "usage": {"input_tokens": 61, "output_tokens": 1}, + "diagnostics": {"is_urgent": {"label_mass": 0.9979320910923947, "argmax_is_label": True}}, + }, + expected_litellm_response={ + "id": "decision-9c26c028d7f4457e", + "object": "structured_decision", + "created": 1791517765, + "model": "Qwen/Qwen3-0.6B", + "answers": { + "is_urgent": { + "type": "choice", + "choice": "yes", + "probabilities": {"yes": 0.9669140238756562, "no": 0.03308597612434374}, + "confidence": 0.9649145337527952, + } + }, + "usage": {"input_tokens": 61, "output_tokens": 1}, + "diagnostics": {"is_urgent": {"label_mass": 0.9979320910923947, "argmax_is_label": True}}, + }, +) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py b/tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py new file mode 100644 index 00000000000..f52d9147273 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.hosted_vllm import QWEN3_0_6B_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [QWEN3_0_6B_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_hosted_vllm(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 820451ed943..c7a29ed8e0e 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -733,6 +733,139 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( assert response.model == _STRANDS_RESPONSE["model"] +_VLLM_RESPONSE: Final[Mapping[str, object]] = { + "id": "systemone-4af3d2c1", + "object": "structured_decision", + "created": 1760012345, + "model": "Qwen/Qwen3-0.6B", + "answers": { + "is_defect": { + "type": "choice", + "choice": "yes", + "confidence": 0.91, + "probabilities": {"yes": 0.91, "no": 0.09}, + } + }, + "usage": {"input_tokens": 154, "output_tokens": 2}, + "diagnostics": {"engine": "vllm", "logprobs_mode": "raw_logprobs"}, +} + + +@pytest.mark.asyncio +async def test_hosted_vllm_requires_api_base_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + + with pytest.raises(litellm.BadRequestError, match="api_base is required"): + await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="review", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_hosted_vllm_without_key_sends_no_authorization_and_preserves_response_extras( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + response: Final = await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + api_base="http://vllm.local:8000", + ) + + assert route.called + assert "authorization" not in respx_mock.calls[0].request.headers + assert response.model_extra["id"] == _VLLM_RESPONSE["id"] + assert response.model_extra["object"] == _VLLM_RESPONSE["object"] + assert response.model_extra["diagnostics"] == _VLLM_RESPONSE["diagnostics"] + + +@pytest.mark.asyncio +async def test_hosted_vllm_uses_key_and_base_from_environment( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://vllm.local:8000/v1") + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "vllm-key") + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + ) + + assert route.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer vllm-key" + + +@pytest.mark.asyncio +async def test_hosted_vllm_api_base_with_v1_suffix_posts_to_v1_systemone( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + api_base="http://vllm.local:8000/v1", + ) + + assert route.called + + +@pytest.mark.asyncio +async def test_hosted_vllm_choice_question_goes_out_as_a_jev_choice_with_the_served_model_name( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={ + "is_defect": { + "type": "choice", + "instructions": "Is this a defect?", + "criteria": {"yes": "defective", "no": "intact"}, + } + }, + api_base="http://vllm.local:8000", + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "Qwen/Qwen3-0.6B", + "state": "The package arrived broken.", + "questions": { + "is_defect": { + "type": "choice", + "instructions": "Is this a defect?", + "criteria": {"yes": "defective", "no": "intact"}, + } + }, + } + + @pytest.mark.asyncio @pytest.mark.parametrize( ("api_base", "url"), diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 941e44feb26..72d4ed0ffde 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -647,3 +647,78 @@ async def test_ahealth_check_probes_strands_through_decisions_without_mode( assert "error" not in result, result assert upstream.called assert "authorization" not in upstream.calls[0].request.headers + + +@pytest.mark.asyncio +async def test_ahealth_check_probes_hosted_vllm_with_a_choice_question( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond( + json={ + "model": "Qwen/Qwen3-0.6B", + "answers": { + "reachable": { + "type": "choice", + "choice": "yes", + "confidence": 1.0, + "probabilities": {"yes": 1.0, "no": 0.0}, + } + }, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check( + { + "model": "hosted_vllm/Qwen/Qwen3-0.6B", + "api_base": "http://vllm.local:8000", + }, + mode="evaluation", + ) + + assert "error" not in result, result + assert upstream.called + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["questions"] == { + "reachable": { + "type": "choice", + "instructions": "Is the service reachable?", + "criteria": {"yes": None, "no": None}, + } + } + + +@pytest.mark.asyncio +async def test_ahealth_check_keeps_the_noul_probe_for_other_decisions_providers( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond( + json={ + "model": "jev-1.13", + "answers": {"reachable": {"type": "noul", "noul": 1.0}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check( + { + "model": "typesafe/jev-1.13", + "api_key": "sk-test", + }, + mode="evaluation", + ) + + assert "error" not in result, result + assert upstream.called + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["questions"] == { + "reachable": {"type": "noul", "instructions": "Is the service reachable?"} + } From a4266cbc4ed79672171b600aa890bede9d0f7801 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:01:13 -0700 Subject: [PATCH 08/18] feat(azure): add azure_ai/Microsoft-Decision-1 (#45639) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> (cherry picked from commit de6a0717b7e2fa1ba5b6565d8a9e3ec2223719bc) --- litellm/model_prices_and_context_window_backup.json | 7 +++++++ model_prices_and_context_window.json | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 94ebc0bd735..35deaadcd97 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -79765,5 +79765,12 @@ "metadata": { "notes": "Self-hosted decision model; infrastructure costs are paid separately" } + }, + "azure_ai/Microsoft-Decision-1": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "azure_ai", + "mode": "evaluation", + "output_cost_per_token": 4.2e-08, + "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 94ebc0bd735..35deaadcd97 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -79765,5 +79765,12 @@ "metadata": { "notes": "Self-hosted decision model; infrastructure costs are paid separately" } + }, + "azure_ai/Microsoft-Decision-1": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "azure_ai", + "mode": "evaluation", + "output_cost_per_token": 4.2e-08, + "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" } } From effe6031f1fe9d814c6efaa42258333095a69166 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:28:31 -0700 Subject: [PATCH 09/18] fix(openrouter): bill responses, decisions and pass-through from usage.cost and always allow reasoning params (#45640) [partial: decisions and pass-through billing only] Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 465f20f2747c2b6f0e1965b7d45b4813d3e1e815) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/decisions/main.py | 3 ++ .../llms/base_llm/decisions/transformation.py | 3 ++ .../llms/openrouter/chat/transformation.py | 13 ++----- .../openrouter/decisions/transformation.py | 9 +++++ .../typesafe_passthrough_logging_handler.py | 5 ++- .../providers/test_decisions_wire.py | 35 +++++++++++++------ tests/unit/decisions/test_main.py | 31 ++++++++++++++++ .../test_openrouter_chat_transformation.py | 17 ++++++--- ...st_typesafe_passthrough_logging_handler.py | 15 ++++---- 9 files changed, 98 insertions(+), 33 deletions(-) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 25ec501fd48..335980bed2b 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -7,6 +7,7 @@ from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never import litellm +from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( @@ -205,11 +206,13 @@ def _prepare_call( def _format_response(response: DecisionsIRResponse, call: _DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: formatted: Final = _formatted_response(response, call) + provider_cost: Final = call.provider_config.provider_reported_cost(response) formatted.set_hidden_params( { "model": f"{call.custom_llm_provider}/{call.model}", "custom_llm_provider": call.custom_llm_provider, "provider_response_model": f"{call.custom_llm_provider}/{call.model}", + **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}), } ) return formatted diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 9d1ed188d7b..c57c643983a 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -308,6 +308,9 @@ class BaseDecisionsConfig(ABC): def get_default_api_base(self) -> str | None: return None + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: + return None + def missing_api_base_message(self, custom_llm_provider: str) -> str: return f"api_base is required for Decisions provider '{custom_llm_provider}'" diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 08d43c169f4..71e36a3abbe 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx -import litellm from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -40,18 +39,10 @@ class CacheControlSupportedModels(str, Enum): class OpenrouterConfig(OpenAIGPTConfig): def get_supported_openai_params(self, model: str) -> list: """ - Allow reasoning parameters for models flagged as reasoning-capable. + Include reasoning parameters regardless of model-map capabilities. """ supported_params: Final = super().get_supported_openai_params(model=model) - try: - if litellm.supports_reasoning(model=model, custom_llm_provider="openrouter") or litellm.supports_reasoning( - model=model - ): - supported_params.append("reasoning_effort") - supported_params.append("thinking") - except Exception: - pass - return list(dict.fromkeys(supported_params)) + return list(dict.fromkeys([*supported_params, "reasoning_effort", "thinking"])) def map_openai_params( self, diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 53ddce8b402..0fa5c7ca7cb 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,4 +1,7 @@ +from typing import Final + from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.types.decisions import DecisionsIRResponse class OpenRouterDecisionsConfig(BaseDecisionsConfig): @@ -8,3 +11,9 @@ class OpenRouterDecisionsConfig(BaseDecisionsConfig): def get_default_api_base(self) -> str | None: return "https://openrouter.ai/api" + + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: + cost: Final = response.usage.extra.get("cost") + if not isinstance(cost, (int, float)): + return None + return float(cost) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py index 3ad92acb48a..e13c9d0c1ae 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py @@ -17,6 +17,7 @@ from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject class _TypeSafeUsage(BaseModel): input_tokens: int = 0 output_tokens: int = 0 + cost: float | None = None class _TypeSafeResponse(BaseModel): @@ -82,7 +83,9 @@ class TypeSafePassthroughLoggingHandler: ) pricing: Final = _pricing_for(candidate_model_keys) response_cost: Final = ( - input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token + usage.cost + if custom_llm_provider == "openrouter" and usage.cost is not None + else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token ) usage_object: Final = Usage( prompt_tokens=input_tokens, diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 2f75d05bf46..895375e5241 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -69,6 +69,7 @@ class _Provider: api_key: str | None wraps_result: bool cost_map_key: str | None + provider_reported_cost: float | None = None _PROVIDERS: Final = ( @@ -89,7 +90,8 @@ _PROVIDERS: Final = ( "typesafe/jev-1.13", _API_KEY, False, - "openrouter/typesafe/jev-1.13", + None, + 1.5834e-5, ), _Provider( "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None @@ -133,17 +135,32 @@ def _number(value: JsonValue) -> float: return float(value) -def _expected_spend(cost_map_key: str | None) -> float: - if cost_map_key is None: +def _expected_spend(provider: _Provider) -> float: + if provider.provider_reported_cost is not None: + return provider.provider_reported_cost + if provider.cost_map_key is None: return 0.0 - prices: Final = object_value(json.loads(Path("model_prices_and_context_window.json").read_text())[cost_map_key]) + prices: Final = object_value( + json.loads(Path("model_prices_and_context_window.json").read_text())[provider.cost_map_key] + ) return _number(_USAGE["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number( _USAGE["output_tokens"] ) * _number(prices["output_cost_per_token"]) +def _usage(provider: _Provider) -> dict[str, JsonValue]: + return { + **_USAGE, + **({"cost": provider.provider_reported_cost} if provider.provider_reported_cost is not None else {}), + } + + def _answer_body(provider: _Provider) -> dict[str, JsonValue]: - answer: Final[dict[str, JsonValue]] = {"model": provider.body_model, "answers": _ANSWERS, "usage": _USAGE} + answer: Final[dict[str, JsonValue]] = { + "model": provider.body_model, + "answers": _ANSWERS, + "usage": _usage(provider), + } return {"result": answer, "success": True} if provider.wraps_result else answer @@ -226,16 +243,14 @@ def _pass_through_config(directory: Path, pass_through_target: str, native_api_b @pytest.mark.parametrize("provider", _PROVIDERS, ids=lambda provider: provider.name) -def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cost_map( - gateway: Gateway, provider: _Provider -) -> None: - expected_spend: Final = _expected_spend(provider.cost_map_key) +def test_each_provider_gets_its_own_path_key_and_body_and_is_billed(gateway: Gateway, provider: _Provider) -> None: + expected_spend: Final = _expected_spend(provider) with gateway.scenario() as scenario: handle: Final = _register(scenario, _answer_body(provider)) model: Final = _deployment(scenario, handle, provider) response: Final = _decide(gateway, model) assert response.status_code == 200, response.text - assert response.json() == {"model": provider.body_model, "answers": _ANSWERS, "usage": _USAGE} + assert response.json() == {"model": provider.body_model, "answers": _ANSWERS, "usage": _usage(provider)} assert response.headers["x-litellm-model-group"] == model assert math.isclose(float(response.headers.get("x-litellm-response-cost", "0")), expected_spend, rel_tol=1e-9) (call,) = _upstream_calls(gateway, handle) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index c7a29ed8e0e..b7e3f8687d8 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -11,6 +11,7 @@ import pytest import respx import litellm +from litellm.cost_calculator import get_response_cost_from_hidden_params from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.decisions import ( @@ -1055,3 +1056,33 @@ async def test_requests_a_provider_cannot_serve_are_rejected_before_http( await litellm.adecisions(model=model, api_key="caller-key", **request_kwargs) assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_openrouter_decisions_uses_provider_reported_cost_without_cost_map( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delitem(litellm.model_cost, "openrouter/typesafe/jev-1.13", raising=False) + cost: Final = 1.1802e-5 + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond( + json={ + "model": "typesafe/jev-1.13", + "answers": _RESPONSE["answers"], + "usage": { + "input_tokens": _INPUT_TOKENS, + "output_tokens": _OUTPUT_TOKENS, + "cost": cost, + }, + } + ) + + response: Final = await litellm.adecisions( + model="openrouter/typesafe/jev-1.13", + state={"source": "unit-test"}, + questions=_QUESTIONS, + api_key="caller-key", + ) + + assert route.called + assert get_response_cost_from_hidden_params(response.hidden_params) == cost diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index b177b80aed1..beffcf9b69c 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -96,6 +96,15 @@ def test_openrouter_extra_body_transformation(): ] +def test_openrouter_supported_params_include_reasoning_for_unmapped_model(): + supported_params = OpenrouterConfig().get_supported_openai_params( + "openrouter/unmapped-model" + ) + + assert "reasoning_effort" in supported_params + assert "thinking" in supported_params + + def test_openrouter_cache_control_flag_removal(): transformed_request = OpenrouterConfig().transform_request( model="openrouter/deepseek/deepseek-chat", @@ -537,17 +546,15 @@ def test_openrouter_reasoning_models_allow_reasoning_effort_param(): assert supported_params.count("reasoning_effort") == 1 -def test_openrouter_non_reasoning_models_do_not_add_reasoning_effort(): - """ - Models without reasoning support should not gain reasoning-specific params. - """ +def test_openrouter_non_reasoning_models_support_reasoning_params(): config = OpenrouterConfig() supported_params = config.get_supported_openai_params( model="openrouter/anthropic/claude-3-5-haiku" ) - assert "reasoning_effort" not in supported_params + assert "reasoning_effort" in supported_params + assert "thinking" in supported_params def test_openrouter_reasoning_effort_max_maps_to_xhigh(): diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index e0a5ef063e8..b76cb17e5bd 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Final from unittest.mock import MagicMock import httpx @@ -12,7 +13,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndp @pytest.fixture(autouse=True) -def local_model_cost_map(monkeypatch: pytest.MonkeyPatch): +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) @@ -137,14 +138,17 @@ def test_success_handler_dispatches_to_typesafe_handler(): assert normalized["kwargs"]["model"] == "typesafe/jev-1.13.0" -def test_openrouter_decisions_response_is_priced_from_request_model_registry_row(): +def test_openrouter_decisions_response_uses_provider_cost_without_cost_map_row( + monkeypatch: pytest.MonkeyPatch, +): logging_obj = _logging_obj() - model_cost = litellm.model_cost["openrouter/typesafe/jev-1.13"] + cost: Final = 1.5834e-5 + monkeypatch.delitem(litellm.model_cost, "openrouter/typesafe/jev-1.13", raising=False) response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( httpx_response=_response(), response_body={ "model": "typesafe/jev-1.13-20260917", - "usage": {"input_tokens": 282, "output_tokens": 20}, + "usage": {"input_tokens": 282, "output_tokens": 20, "cost": cost}, }, logging_obj=logging_obj, url_route="https://openrouter.ai/api/alpha/decisions", @@ -156,10 +160,9 @@ def test_openrouter_decisions_response_is_priced_from_request_model_registry_row custom_llm_provider="openrouter", ) - expected_cost = 282 * model_cost["input_cost_per_token"] + 20 * model_cost["output_cost_per_token"] assert response["kwargs"]["model"] == "openrouter/typesafe/jev-1.13-20260917" assert response["kwargs"]["custom_llm_provider"] == "openrouter" - assert response["kwargs"]["response_cost"] == pytest.approx(expected_cost) + assert response["kwargs"]["response_cost"] == cost assert response["kwargs"]["combined_usage_object"].prompt_tokens == 282 assert response["kwargs"]["combined_usage_object"].completion_tokens == 20 assert response["kwargs"]["combined_usage_object"].total_tokens == 302 From d17ba6cd97e7b40fe0f2a6e09661ec89c4570718 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:40:43 -0700 Subject: [PATCH 10/18] feat(azure_ai): support Microsoft-Decision-1 on the decisions API (#45673) * feat(azure_ai): support Microsoft-Decision-1 on the decisions API Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(azure_ai): strip project path and API suffix together in decisions base Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 2fcc7804982903a1a6911b0423dd3a839a10fe7e) --- litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 2 + .../llms/azure_ai/decisions/transformation.py | 24 ++++++ litellm/utils.py | 2 + .../providers/test_decisions_wire.py | 3 + tests/integration/proxy_config.yaml | 7 ++ .../translation/decisions/bases/azure_ai.py | 79 +++++++++++++++++ .../basic/test_decisions_basic_azure_ai.py | 11 +++ tests/unit/decisions/test_main.py | 85 +++++++++++++++++++ 9 files changed, 216 insertions(+) create mode 100644 litellm/llms/azure_ai/decisions/transformation.py create mode 100644 tests/integration/translation/decisions/bases/azure_ai.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_azure_ai.py diff --git a/litellm/__init__.py b/litellm/__init__.py index c559ff8cf11..c228e63e928 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1675,6 +1675,9 @@ if TYPE_CHECKING: from .llms.openai.decisions.transformation import ( OpenAIDecisionsConfig as OpenAIDecisionsConfig, ) + from .llms.azure_ai.decisions.transformation import ( + AzureAIDecisionsConfig as AzureAIDecisionsConfig, + ) from .llms.nvidia_nim.rerank.transformation import ( NvidiaNimRerankConfig as NvidiaNimRerankConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index cb5ec8a016b..257e748ab98 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -160,6 +160,7 @@ LLM_CONFIG_NAMES: Final = ( "StrandsDeciderDecisionsConfig", "HostedVLLMDecisionsConfig", "OpenAIDecisionsConfig", + "AzureAIDecisionsConfig", "NvidiaNimRerankConfig", "NvidiaNimRankingConfig", "VertexAIRerankConfig", @@ -716,6 +717,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "HostedVLLMDecisionsConfig", ), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), + "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), "NvidiaNimRerankConfig": ( ".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig", diff --git a/litellm/llms/azure_ai/decisions/transformation.py b/litellm/llms/azure_ai/decisions/transformation.py new file mode 100644 index 00000000000..9b10f393c7c --- /dev/null +++ b/litellm/llms/azure_ai/decisions/transformation.py @@ -0,0 +1,24 @@ +import re +from typing import Final + +import httpx + +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + +_FOUNDRY_ROUTE_SUFFIX: Final = re.compile(r"(/api/projects/[^/]+)?(/openai/v1|/openai|/models|/v1)?$") + + +class AzureAIDecisionsConfig(BaseDecisionsConfig): + """Microsoft Foundry decision models (Microsoft-Decision-1), where model is the deployment name""" + + path = "/providers/microsoft/v1/systemone" + api_key_env = ("AZURE_AI_API_KEY",) + api_base_env = ("AZURE_AI_API_BASE",) + + def missing_api_base_message(self, custom_llm_provider: str) -> str: + return "Missing AZURE_AI_API_BASE - set AZURE_AI_API_BASE or pass api_base, e.g. https://.services.ai.azure.com" + + def get_complete_url(self, api_base: str, model: str) -> str: + url: Final = httpx.URL(api_base) + resource_path: Final = _FOUNDRY_ROUTE_SUFFIX.sub("", url.path.rstrip("/")) + return str(url.copy_with(path=f"{resource_path}{self.path}", query=None, fragment=None)) diff --git a/litellm/utils.py b/litellm/utils.py index 5a61cd087de..990bd689144 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8914,6 +8914,8 @@ class ProviderConfigManager: return litellm.HostedVLLMDecisionsConfig() if provider == LlmProviders.OPENAI: return litellm.OpenAIDecisionsConfig() + if provider == LlmProviders.AZURE_AI: + return litellm.AzureAIDecisionsConfig() return None @staticmethod diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 895375e5241..9a12ab2ec16 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -108,6 +108,9 @@ _PROVIDERS: Final = ( True, "cloudflare/@cf/cloudflare/clef", ), + _Provider( + "azure_ai", "azure_ai/decision-1", "/providers/microsoft/v1/systemone", "decision-1", _API_KEY, False, None + ), ) _PERPLEXITY: Final = _PROVIDERS[0] _OPENROUTER: Final = _PROVIDERS[2] diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 4cbd270a53c..8ed5237868a 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -27,3 +27,10 @@ model_list: litellm_params: model: hosted_vllm/Qwen/Qwen3-0.6B api_base: http://127.0.0.1:8191 + - model_name: azure_ai/decision-1 + litellm_params: + model: azure_ai/decision-1 + api_base: http://127.0.0.1:8191 + api_key: synthetic-azure-ai-key + model_info: + base_model: azure_ai/Microsoft-Decision-1 diff --git a/tests/integration/translation/decisions/bases/azure_ai.py b/tests/integration/translation/decisions/bases/azure_ai.py new file mode 100644 index 00000000000..27e968c55cb --- /dev/null +++ b/tests/integration/translation/decisions/bases/azure_ai.py @@ -0,0 +1,79 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from the Microsoft Foundry Decision playground (POST /providers/microsoft/v1/systemone, model is the deployment name). Mock reply captured live from a Microsoft-Decision-1 deployment on 2026-10-09. +""" +MICROSOFT_DECISION_1_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "azure_ai/decision-1", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/providers/microsoft/v1/systemone", + expected_provider_headers={"authorization": "Bearer synthetic-azure-ai-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "decision-1", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "microsoft-decision-1", + "answers": { + "confidence": { + "confidence": 0.6351489346076444, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.1824255326961778, "1": 0.8175744673038222}, + "score": 0.8175744673038222, + "type": "score", + }, + "defect": {"noul": 0.982013764002278, "type": "noul"}, + "severity": { + "choice": "high", + "confidence": 0.9866142978108987, + "probabilities": {"high": 0.9933071489054494, "low": 0.006692851094550702}, + "type": "choice", + }, + }, + "usage": {"input_tokens": 87, "output_tokens": 3}, + }, + expected_litellm_response={ + "model": "microsoft-decision-1", + "answers": { + "defect": {"type": "noul", "noul": 0.982013764002278}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9866142978108987, + "probabilities": {"high": 0.9933071489054494, "low": 0.006692851094550702}, + }, + "confidence": { + "type": "score", + "score": 0.8175744673038222, + "confidence": 0.6351489346076444, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.1824255326961778, "1": 0.8175744673038222}, + }, + }, + "usage": {"input_tokens": 87, "output_tokens": 3}, + }, +) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_azure_ai.py b/tests/integration/translation/decisions/basic/test_decisions_basic_azure_ai.py new file mode 100644 index 00000000000..1d6b5cc4af1 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_azure_ai.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.azure_ai import MICROSOFT_DECISION_1_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [MICROSOFT_DECISION_1_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_azure_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index b7e3f8687d8..ec5375bc71e 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1086,3 +1086,88 @@ async def test_openrouter_decisions_uses_provider_reported_cost_without_cost_map assert route.called assert get_response_cost_from_hidden_params(response.hidden_params) == cost + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "api_base", + ( + "https://res.services.ai.azure.com", + "https://res.services.ai.azure.com/", + "https://res.services.ai.azure.com/models", + "https://res.services.ai.azure.com/openai/v1", + "https://res.services.ai.azure.com/api/projects/proj", + "https://res.services.ai.azure.com/api/projects/proj/openai/v1", + "https://res.services.ai.azure.com/api/projects/proj/models", + "https://res.services.ai.azure.com/models?api-version=2024-05-01-preview", + ), +) +async def test_azure_ai_decision_posts_deployment_to_foundry_systemone_route( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + api_base: str, +) -> None: + monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) + monkeypatch.delenv("AZURE_AI_API_KEY", raising=False) + route: Final = respx_mock.post("https://res.services.ai.azure.com/providers/microsoft/v1/systemone").respond( + json={**_RESPONSE, "model": "microsoft-decision-1"} + ) + + response: Final = await litellm.adecisions( + model="azure_ai/decision-1", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_base=api_base, + api_key="foundry-key", + ) + + assert route.called + request: Final = respx_mock.calls[0].request + assert request.headers["authorization"] == "Bearer foundry-key" + assert "api-key" not in request.headers + assert json.loads(request.content) == { + "model": "decision-1", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} + assert response._hidden_params["model"] == "azure_ai/decision-1" + + +@pytest.mark.asyncio +async def test_azure_ai_decision_reads_foundry_env_and_keeps_gateway_path_prefix( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("AZURE_AI_API_BASE", "https://gateway.example.com/foundry/models") + monkeypatch.setenv("AZURE_AI_API_KEY", "env-foundry-key") + route: Final = respx_mock.post("https://gateway.example.com/foundry/providers/microsoft/v1/systemone").respond( + json=_RESPONSE + ) + + await litellm.adecisions( + model="azure_ai/decision-1", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert route.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer env-foundry-key" + + +@pytest.mark.asyncio +async def test_azure_ai_decision_requires_api_base_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) + monkeypatch.setenv("AZURE_AI_API_KEY", "foundry-key") + + with pytest.raises(litellm.BadRequestError, match="Missing AZURE_AI_API_BASE"): + await litellm.adecisions( + model="azure_ai/decision-1", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert len(respx_mock.calls) == 0 From 15268b74ce61576b33f6c6e7b81775be18c685d3 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:14:48 -0700 Subject: [PATCH 11/18] fix(decisions)!: refuse safety_identifier on providers that cannot take it unless drop_params drops it (#44955) * feat(decisions): add the OpenAI Decisions spec types and the System One translation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): share one DecisionsModel config and require model and usage on responses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): rename the shared pydantic parent to DecisionsObjectBase Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): dispatch /v1/decisions through provider configs and the shared HTTP handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): map OpenRouter connection failures to APIConnectionError and drop explanatory docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(decisions): switch /v1/decisions to the OpenAI Decisions schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(decisions): build the usage the OpenAI response schema now requires Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(lens): ask signal questions through the OpenAI Decisions schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(decisions): refuse safety_identifier for System One providers unless drop_params * feat(decisions): refuse safety_identifier on providers that cannot take it unless drop_params drops it * test(decisions): cover hosted_vllm in the safety_identifier and per-provider wire tests * fix(decisions): check a System One request's safety_identifier against the provider too * fix(decisions): drop a non-string safety_identifier under drop_params A malformed safety_identifier now follows the drop_params convention in both body shapes: it answers 400 without drop_params and is dropped before the provider call with it. A string identifier on OpenAI stays on the wire either way. * fix(decisions): let /v1/decisions drop a non-string safety_identifier under drop_params The route checked the whole body before routing, so a non-string safety_identifier answered 400 even when the deployment or the body set drop_params. The route now leaves that field to the Decisions call, which knows every drop_params source * refactor(decisions): return the unsupported safety_identifier as a value and raise it in _prepare_call --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> (cherry picked from commit 546402c98c7ecea5f32d263fadd3c95bd9b934ff) --- litellm/decisions/main.py | 66 +- .../llms/base_llm/decisions/transformation.py | 1 + .../llms/openai/decisions/transformation.py | 1 + .../proxy/decisions_endpoints/endpoints.py | 7 +- .../test_decisions_openai_format_wire.py | 586 ++++++++++++++++++ .../translation/decisions/bases/cloudflare.py | 97 +++ .../translation/decisions/bases/openrouter.py | 94 +++ .../translation/decisions/bases/perplexity.py | 95 +++ .../decisions/bases/strands_decider.py | 34 + .../translation/decisions/bases/typesafe.py | 92 +++ .../basic/test_decisions_basic_cloudflare.py | 4 +- .../basic/test_decisions_basic_openrouter.py | 9 +- .../basic/test_decisions_basic_perplexity.py | 9 +- .../test_decisions_basic_strands_decider.py | 11 +- .../basic/test_decisions_basic_typesafe.py | 4 +- tests/unit/decisions/test_main.py | 135 ++++ .../decisions_endpoints/test_endpoints.py | 62 ++ 17 files changed, 1289 insertions(+), 18 deletions(-) create mode 100644 tests/integration/providers/test_decisions_openai_format_wire.py diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 335980bed2b..2dd0d80aeb6 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,13 +1,13 @@ from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Final, TypeAlias import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError from typing_extensions import assert_never import litellm -from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER +from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( @@ -40,6 +40,9 @@ DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequest _SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) _OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) +_SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( + str | None, config=ConfigDict(title="safety_identifier") +) _HANDLER: Final = BaseLLMHTTPHandler() @@ -96,16 +99,46 @@ def _validate_request( ) -def _ir_request(request: DecisionsRequestFormat) -> DecisionsIRRequest: +def _ir_request(request: DecisionsRequestFormat, safety_identifier: str | None) -> DecisionsIRRequest: match request: case DecisionsRequestBody(): - return systemone_request_to_ir(request) + return replace(systemone_request_to_ir(request), safety_identifier=safety_identifier) case OpenAIDecisionRequestBody(): return openai_request_to_ir(request) case _: assert_never(request) +def _drops_params(kwargs: Mapping[str, object]) -> bool: + return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True + + +def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, object]) -> str | None: + if isinstance(safety_identifier, str) or not _drops_params(kwargs): + return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier) + return None + + +@dataclass(frozen=True, slots=True) +class _UnsupportedSafetyIdentifier: + pass + + +def _provider_ir_request( + request: DecisionsRequestFormat, + *, + safety_identifier: str | None, + provider_config: BaseDecisionsConfig, + kwargs: Mapping[str, object], +) -> DecisionsIRRequest | _UnsupportedSafetyIdentifier: + ir_request: Final = _ir_request(request, safety_identifier) + if ir_request.safety_identifier is None or provider_config.supports_safety_identifier: + return ir_request + if _drops_params(kwargs): + return replace(ir_request, safety_identifier=None) + return _UnsupportedSafetyIdentifier() + + def _prepare_call( *, model: str, @@ -141,8 +174,12 @@ def _prepare_call( llm_provider=provider, ) try: + request_safety_identifier: Final = _request_safety_identifier(safety_identifier, kwargs) request: Final = _validate_request( - state=state, questions=questions, decision_input=decision_input, safety_identifier=safety_identifier + state=state, + questions=questions, + decision_input=decision_input, + safety_identifier=request_safety_identifier, ) except ValidationError as error: raise litellm.BadRequestError( @@ -166,7 +203,22 @@ def _prepare_call( llm_provider=provider, ) - ir_request: Final = _ir_request(request) + ir_request: Final = _provider_ir_request( + request, + safety_identifier=request_safety_identifier, + provider_config=provider_config, + kwargs=kwargs, + ) + if isinstance(ir_request, _UnsupportedSafetyIdentifier): + raise litellm.UnsupportedParamsError( + message=( + f"{provider} does not support parameters: ['safety_identifier'], for model={model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ), + model=model, + llm_provider=provider, + ) body: Final = provider_config.transform_decisions_request( model=canonical_model, request=ir_request, custom_llm_provider=provider ) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index c57c643983a..baeeb3f0449 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -301,6 +301,7 @@ class BaseDecisionsConfig(ABC): api_key_env: tuple[str, ...] = () api_base_env: tuple[str, ...] = () api_key_required: bool = True + supports_safety_identifier: bool = False health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType( {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})} ) diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 0afbe1a434e..317e5abeeb5 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -292,6 +292,7 @@ def ir_to_openai_response( class OpenAIDecisionsConfig(BaseDecisionsConfig): path = "/v1/decisions" + supports_safety_identifier = True def get_default_api_base(self) -> str | None: return "https://api.openai.com" diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 86fc4f9da2c..779fb85192b 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -1,4 +1,5 @@ from collections.abc import Mapping +from types import MappingProxyType from typing import Annotated, Final from fastapi import APIRouter, Depends, Request, Response @@ -38,6 +39,10 @@ async def _invalid_request( ) +def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType({key: value for key, value in data.items() if key != "safety_identifier"}) + + async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: body: Final = await request.body() try: @@ -75,7 +80,7 @@ async def _process_decisions( data: Final = await _request_data(request, user_api_key_dict) try: - body_adapter.validate_python(data) + body_adapter.validate_python(_fields_checked_before_routing(data)) except ValidationError as error: raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict) general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings) diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py new file mode 100644 index 00000000000..f0e7163e264 --- /dev/null +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -0,0 +1,586 @@ +import math +import uuid +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario +from integration.cost_calculation.cost_tracking_case import JsonResponse +from pydantic import JsonValue, TypeAdapter + +import litellm + +_API_KEY: Final = "synthetic-decisions-key" +_SAFETY_IDENTIFIER: Final = "end-user-7" +_DROPPING_DEPLOYMENT: Final = "decisions-under-litellm-settings-drop-params" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_INPUT: Final = "Ticket (billing): The export job hangs at 99%" +_FOLLOW_UP: Final = "Customer: still stuck after retrying" +_QUESTIONS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [{"value": "low", "description": "cosmetic"}, {"value": "high", "description": "blocks users"}], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, +] +_SYSTEM_ONE_QUESTIONS: Final[dict[str, JsonValue]] = { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, +} +_SYSTEM_ONE_ANSWERS: Final[dict[str, JsonValue]] = { + "defect": {"type": "noul", "noul": 0.93}, + "severity": {"type": "choice", "choice": "high", "confidence": 0.8, "probabilities": {"low": 0.2, "high": 0.8}}, + "confidence": { + "type": "score", + "score": 1.0, + "confidence": 0.7, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.3, "1": 0.7}, + }, +} +_ANSWERS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "defect", "probability": 0.93}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.2}, {"value": "high", "probability": 0.8}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "confidence", + "score": 1.0, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.3}, + {"value": 1, "label": "sure", "probability": 0.7}, + ], + "confidence": 0.7, + }, +] +_INPUT_TOKENS: Final = 367 +_OUTPUT_TOKENS: Final = 3 +_SYSTEM_ONE_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS} +_USAGE: Final[dict[str, JsonValue]] = { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, +} +_SDK_QUESTIONS: Final = TypeAdapter(list[dict[str, object]]).validate_python(_QUESTIONS) +_SPEND_QUERY: Final = ( + "SELECT spend, status, call_type, model_group, custom_llm_provider, api_base, prompt_tokens, completion_tokens, " + 'request_tags FROM "LiteLLM_SpendLogs" WHERE request_id = %s' +) + + +@dataclass(frozen=True, slots=True) +class _Provider: + name: str + model: str + path: str + body_model: str + api_key: str | None + wraps_result: bool + cost_map_key: str | None + speaks_openai: bool = False + + def upstream_body(self) -> dict[str, JsonValue]: + if self.speaks_openai: + return {"model": self.body_model, "input": _INPUT, "questions": _QUESTIONS} + return {"model": self.body_model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS} + + def upstream_reply(self) -> dict[str, JsonValue]: + if self.speaks_openai: + return {"model": self.body_model, "answers": _ANSWERS, "usage": _USAGE} + answer: Final[dict[str, JsonValue]] = { + "model": self.body_model, + "answers": _SYSTEM_ONE_ANSWERS, + "usage": _SYSTEM_ONE_USAGE, + } + return {"result": answer, "success": True} if self.wraps_result else answer + + def litellm_response(self) -> dict[str, JsonValue]: + return {"model": self.body_model, "answers": _ANSWERS, "usage": _USAGE} + + +_PROVIDERS: Final = ( + _Provider( + "perplexity", + "perplexity/pplx-decider-v1-27b", + "/v1/decisions", + "pplx-decider-v1-27b", + _API_KEY, + False, + "perplexity/pplx-decider-v1-27b", + ), + _Provider("typesafe", "typesafe/jev-1.13.0", "/v1/systemone", "jev-1.13.0", _API_KEY, False, "typesafe/jev-1.13.0"), + _Provider( + "openrouter", + "openrouter/typesafe/jev-1.13", + "/alpha/decisions", + "typesafe/jev-1.13", + _API_KEY, + False, + "openrouter/typesafe/jev-1.13", + ), + _Provider( + "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None + ), + _Provider( + "cloudflare", + "cloudflare/clef", + "/ai/run/@cf/cloudflare/clef", + "clef", + _API_KEY, + True, + "cloudflare/@cf/cloudflare/clef", + ), + _Provider("hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None), + _Provider( + "databricks", + "databricks/databricks-openjev-qwen35-4b", + "/databricks-openjev-qwen35-4b/invocations", + "databricks-openjev-qwen35-4b", + _API_KEY, + False, + None, + ), + _Provider( + "azure_ai", "azure_ai/decision-1", "/providers/microsoft/v1/systemone", "decision-1", _API_KEY, False, None + ), + _Provider("openai", "openai/gpt-6-luna", "/v1/decisions", "gpt-6-luna", _API_KEY, False, "gpt-6-luna", True), +) +_SYSTEM_ONE_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if not provider.speaks_openai) +_PERPLEXITY: Final = _PROVIDERS[0] +_TYPESAFE: Final = _PROVIDERS[1] +_OPENAI: Final = next(provider for provider in _PROVIDERS if provider.speaks_openai) +_PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"} +_INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( + ("missing questions", {"input": _INPUT}), + ("missing input", {"questions": _QUESTIONS}), + ("numeric input", {"input": 5, "questions": _QUESTIONS}), + ("assistant message input", {"input": [{"role": "assistant", "content": "hi"}], "questions": _QUESTIONS}), + ("empty questions", {"input": _INPUT, "questions": []}), + ("questions as a map", {"input": _INPUT, "questions": {"q": _PREDICATE}}), + ("predicate without instructions", {"input": _INPUT, "questions": [{"type": "predicate", "name": "q"}]}), + ( + "choice with one choice", + {"input": _INPUT, "questions": [{**_PREDICATE, "type": "choice", "choices": [{"value": "only"}]}]}, + ), + ( + "score with one level", + {"input": _INPUT, "questions": [{**_PREDICATE, "type": "score", "levels": [{"label": "only"}]}]}, + ), + ("unknown question type", {"input": _INPUT, "questions": [{**_PREDICATE, "type": "ranking"}]}), + ("numeric safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": 7}), + ("list safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": [_SAFETY_IDENTIFIER]}), +) +_IMAGE_INPUT: Final[list[JsonValue]] = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": _INPUT}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, + ], + } +] +_OPENAI_IMAGE_MESSAGES: Final[list[JsonValue]] = [ + { + "role": "user", + "type": "message", + "content": [ + {"type": "input_text", "text": _INPUT}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, + ], + } +] +_MESSAGE_LIST_INPUT: Final[list[JsonValue]] = [ + {"role": "user", "content": _INPUT}, + {"role": "user", "content": [{"type": "input_text", "text": _FOLLOW_UP}]}, +] + + +def _response_cost(response: httpx.Response) -> float: + return float(response.headers["x-litellm-response-cost"]) if "x-litellm-response-cost" in response.headers else 0.0 + + +def _provider_id(provider: _Provider) -> str: + return provider.name + + +def _number(value: JsonValue) -> float: + assert isinstance(value, (int, float)) and not isinstance(value, bool), value + return float(value) + + +def _expected_spend(cost_map_key: str | None) -> float: + if cost_map_key is None: + return 0.0 + cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes()) + prices: Final = object_value(cost_map[cost_map_key]) + return _INPUT_TOKENS * _number(prices["input_cost_per_token"]) + _OUTPUT_TOKENS * _number( + prices["output_cost_per_token"] + ) + + +def _register(scenario: Scenario, body: dict[str, JsonValue], *, status: int = 200) -> ScenarioHandle: + handle: Final = register_scenario( + f"decisions-{uuid.uuid4().hex[:12]}", JsonResponse(content_type="application/json", body=body, status=status) + ) + scenario.cleanups.callback(delete_scenario, handle) + return handle + + +def _deployment(scenario: Scenario, handle: ScenarioHandle, provider: _Provider, *, drop_params: bool = False) -> str: + if drop_params: + return scenario.model( + model=provider.model, api_base=handle.api_base(), api_key=provider.api_key, drop_params=True + ) + return scenario.model(model=provider.model, api_base=handle.api_base(), api_key=provider.api_key) + + +def _decide(gateway: Gateway, model: str, *, route: str = "/v1/decisions", **extra: JsonValue) -> httpx.Response: + return gateway.request("POST", route, {"model": model, "input": _INPUT, "questions": _QUESTIONS, **extra}) + + +def _decide_in_system_one_format(gateway: Gateway, model: str, **extra: JsonValue) -> httpx.Response: + return gateway.request( + "POST", "/v1/systemone", {"model": model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS, **extra} + ) + + +def _observed_requests(gateway: Gateway) -> tuple[dict[str, JsonValue], ...]: + with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream: + observations: Final = _JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"] + assert isinstance(observations, list), observations + return tuple(map(object_value, observations)) + + +def _calls_to(requests: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> list[dict[str, JsonValue]]: + return [request for request in requests if string_value(request["path"]).startswith(f"/{handle.scenario_id}/")] + + +def _upstream_calls(gateway: Gateway, handle: ScenarioHandle) -> list[dict[str, JsonValue]]: + return _calls_to(_observed_requests(gateway), handle) + + +def _spend_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70) + return rows[0] + + +def _assert_refused_as_invalid(gateway: Gateway, model: str, label: str, body: dict[str, JsonValue]) -> None: + response: Final = gateway.request("POST", "/v1/decisions", {"model": model, **body}) + assert response.status_code == 400, (label, response.text) + assert "Invalid Decisions request" in response.text, (label, response.text) + + +def _assert_refused_for_safety_identifier(response: httpx.Response) -> None: + assert response.status_code == 400, response.text + assert "safety_identifier" in response.text and "drop_params" in response.text, response.text + + +def _drop_params_config(directory: Path, api_base: str) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = { + **base, + "litellm_settings": {**object_value(base["litellm_settings"]), "drop_params": True}, + "model_list": [ + { + "model_name": _DROPPING_DEPLOYMENT, + "litellm_params": {"model": _PERPLEXITY.model, "api_base": api_base, "api_key": _API_KEY}, + } + ], + } + path: Final = directory / "decisions-drop-params.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.mark.parametrize("provider", _PROVIDERS, ids=_provider_id) +def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cost_map( + gateway: Gateway, provider: _Provider +) -> None: + expected_spend: Final = _expected_spend(provider.cost_map_key) + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + model: Final = _deployment(scenario, handle, provider) + response: Final = _decide(gateway, model) + assert response.status_code == 200, response.text + assert response.json() == provider.litellm_response() + assert response.headers["x-litellm-model-group"] == model + assert math.isclose(_response_cost(response), expected_spend, rel_tol=1e-9) + (call,) = _upstream_calls(gateway, handle) + assert call["path"] == f"/{handle.scenario_id}{provider.path}" + assert call["authorization"] == (f"Bearer {provider.api_key}" if provider.api_key else "") + assert call["body"] == provider.upstream_body() + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + assert ( + row["status"], + row["call_type"], + row["custom_llm_provider"], + row["model_group"], + row["api_base"], + row["prompt_tokens"], + row["completion_tokens"], + ) == ( + "success", + "adecisions", + provider.name, + model, + f"{handle.api_base()}{provider.path}", + _INPUT_TOKENS, + _OUTPUT_TOKENS, + ) + assert math.isclose(_number(row["spend"]), expected_spend, rel_tol=1e-9), row + + +def test_the_unversioned_alias_serves_the_same_request(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide(gateway, model, route="/decisions") + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_repeated_identical_requests_each_reach_the_upstream_and_are_each_billed(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + responses: Final = tuple(_decide(gateway, model) for _ in range(2)) + assert [response.status_code for response in responses] == [200, 200], [r.text for r in responses] + call_ids: Final = tuple(response.headers["x-litellm-call-id"] for response in responses) + assert len(set(call_ids)) == 2, call_ids + assert len(_upstream_calls(gateway, handle)) == 2 + for call_id in call_ids: + assert _spend_row(call_id)["status"] == "success" + + +async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _TYPESAFE.upstream_reply()) + synchronous: Final = litellm.decisions( + model=_TYPESAFE.model, input=_INPUT, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + ) + asynchronous: Final = await litellm.adecisions( + model=_TYPESAFE.model, input=_INPUT, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + ) + for response in (synchronous, asynchronous): + assert response.model_dump(mode="json") == _TYPESAFE.litellm_response() + calls: Final = _upstream_calls(gateway, handle) + assert len(calls) == 2, calls + for call in calls: + assert call["path"] == f"/{handle.scenario_id}{_TYPESAFE.path}" + assert call["authorization"] == f"Bearer {_API_KEY}" + assert call["body"] == _TYPESAFE.upstream_body() + + +def test_gateway_only_fields_stay_at_the_gateway_and_tags_reach_the_spend_log(gateway: Gateway) -> None: + tag: Final = f"decisions-openai-format-{uuid.uuid4().hex[:8]}" + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide( + gateway, model, user="auditor", num_retries=0, temperature=0.2, metadata={"tags": [tag]} + ) + assert response.status_code == 200, response.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + tags: Final = row["request_tags"] + assert isinstance(tags, list) and tag in tags, row + + +def test_invalid_bodies_are_refused_at_the_gateway_without_an_upstream_call(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + for label, body in _INVALID_BODIES: + _assert_refused_as_invalid(gateway, model, label, body) + assert _upstream_calls(gateway, handle) == [] + + +@pytest.mark.parametrize("provider", _SYSTEM_ONE_PROVIDERS, ids=_provider_id) +def test_system_one_providers_refuse_images_at_the_gateway_without_an_upstream_call( + gateway: Gateway, provider: _Provider +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + model: Final = _deployment(scenario, handle, provider) + response: Final = _decide(gateway, model, input=_IMAGE_INPUT) + assert response.status_code == 400, response.text + assert "input_image" in response.text + assert _upstream_calls(gateway, handle) == [] + + +def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + model: Final = _deployment(scenario, handle, _OPENAI) + response: Final = _decide(gateway, model, input=_IMAGE_INPUT) + assert response.status_code == 200, response.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == {"model": _OPENAI.body_model, "input": _OPENAI_IMAGE_MESSAGES, "questions": _QUESTIONS} + + +def test_a_message_list_input_reaches_a_system_one_provider_as_its_flattened_text(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide(gateway, model, input=_MESSAGE_LIST_INPUT) + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == { + "model": _PERPLEXITY.body_model, + "state": f"{_INPUT}\n\n{_FOLLOW_UP}", + "questions": _SYSTEM_ONE_QUESTIONS, + } + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + assert (row["status"], row["call_type"], row["prompt_tokens"], row["completion_tokens"]) == ( + "success", + "adecisions", + _INPUT_TOKENS, + _OUTPUT_TOKENS, + ) + + +@pytest.mark.parametrize("provider", _SYSTEM_ONE_PROVIDERS, ids=_provider_id) +def test_safety_identifier_is_refused_by_system_one_providers_unless_the_deployment_drops_params( + gateway: Gateway, provider: _Provider +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + strict: Final = _deployment(scenario, handle, provider) + refused: Final = _decide(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER) + _assert_refused_for_safety_identifier(refused) + assert _upstream_calls(gateway, handle) == [] + + dropping: Final = _deployment(scenario, handle, provider, drop_params=True) + accepted: Final = _decide(gateway, dropping, safety_identifier=_SAFETY_IDENTIFIER) + assert accepted.status_code == 200, accepted.text + assert accepted.json() == provider.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == provider.upstream_body() + row: Final = _spend_row(accepted.headers["x-litellm-call-id"]) + assert (row["status"], row["call_type"], row["model_group"]) == ("success", "adecisions", dropping) + + +@pytest.mark.parametrize("value", ("", "x" * 5000), ids=("empty", "5kb")) +def test_every_string_safety_identifier_is_refused_or_dropped_like_the_usual_one(gateway: Gateway, value: str) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + strict: Final = _deployment(scenario, handle, _PERPLEXITY) + _assert_refused_for_safety_identifier(_decide(gateway, strict, safety_identifier=value)) + dropping: Final = _deployment(scenario, handle, _PERPLEXITY, drop_params=True) + accepted: Final = _decide(gateway, dropping, safety_identifier=value) + assert accepted.status_code == 200, accepted.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_a_request_body_drop_params_drops_the_safety_identifier_like_chat(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + strict: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER, drop_params=True) + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_openai_keeps_the_safety_identifier_on_the_wire_without_drop_params(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + model: Final = _deployment(scenario, handle, _OPENAI) + response: Final = _decide(gateway, model, safety_identifier=_SAFETY_IDENTIFIER) + assert response.status_code == 200, response.text + assert response.json() == _OPENAI.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == {**_OPENAI.upstream_body(), "safety_identifier": _SAFETY_IDENTIFIER} + + +def test_a_system_one_format_safety_identifier_is_refused_unless_the_deployment_drops_params(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + strict: Final = _deployment(scenario, handle, _PERPLEXITY) + refused: Final = _decide_in_system_one_format(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER) + _assert_refused_for_safety_identifier(refused) + assert _upstream_calls(gateway, handle) == [] + + dropping: Final = _deployment(scenario, handle, _PERPLEXITY, drop_params=True) + accepted: Final = _decide_in_system_one_format(gateway, dropping, safety_identifier=_SAFETY_IDENTIFIER) + assert accepted.status_code == 200, accepted.text + assert accepted.json()["answers"] == _SYSTEM_ONE_ANSWERS + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_openai_keeps_a_system_one_format_safety_identifier_on_the_wire(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + model: Final = _deployment(scenario, handle, _OPENAI) + response: Final = _decide_in_system_one_format(gateway, model, safety_identifier=_SAFETY_IDENTIFIER) + assert response.status_code == 200, response.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == {**_OPENAI.upstream_body(), "safety_identifier": _SAFETY_IDENTIFIER} + + +@pytest.mark.parametrize("decide", (_decide, _decide_in_system_one_format), ids=("openai_format", "system_one_format")) +@pytest.mark.parametrize("value", (7, [_SAFETY_IDENTIFIER]), ids=("numeric", "list")) +def test_a_non_string_safety_identifier_is_refused_as_invalid_unless_the_deployment_drops_params( + gateway: Gateway, value: JsonValue, decide: Callable[..., httpx.Response] +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + strict: Final = _deployment(scenario, handle, _OPENAI) + refused: Final = decide(gateway, strict, safety_identifier=value) + assert refused.status_code == 400, refused.text + assert "Invalid Decisions request" in refused.text, refused.text + assert _upstream_calls(gateway, handle) == [] + + dropping: Final = _deployment(scenario, handle, _OPENAI, drop_params=True) + accepted: Final = decide(gateway, dropping, safety_identifier=value) + assert accepted.status_code == 200, accepted.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _OPENAI.upstream_body() + + +def test_litellm_settings_drop_params_drops_the_safety_identifier_for_a_strict_deployment( + gateway: Gateway, tmp_path: Path +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + config: Final = _drop_params_config(tmp_path, handle.api_base()) + with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: + response: Final = _decide(owned.gateway, _DROPPING_DEPLOYMENT, safety_identifier=_SAFETY_IDENTIFIER) + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() diff --git a/tests/integration/translation/decisions/bases/cloudflare.py b/tests/integration/translation/decisions/bases/cloudflare.py index ae86a2ccd87..3efc96eeb87 100644 --- a/tests/integration/translation/decisions/bases/cloudflare.py +++ b/tests/integration/translation/decisions/bases/cloudflare.py @@ -6,6 +6,103 @@ from integration.translation.case import TranslationTestCase """ CLEF_TEST_CASE: Final = TranslationTestCase( scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "cloudflare/@cf/cloudflare/clef", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/ai/run/@cf/cloudflare/clef", + expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "result": { + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + }, + expected_litellm_response={ + "model": "clef", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.9345}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.0509}, {"value": "high", "probability": 0.9491}], + "confidence": 0.8067, + }, + { + "type": "score", + "name": "confidence", + "score": 0.9036, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.0964}, + {"value": 1, "label": "sure", "probability": 0.9036}, + ], + "confidence": 0.6515, + }, + ], + "usage": { + "input_tokens": 290, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 290, + }, + }, +) +CLEF_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", litellm_endpoint="/v1/systemone", litellm_request={ "model": "cloudflare/@cf/cloudflare/clef", diff --git a/tests/integration/translation/decisions/bases/openrouter.py b/tests/integration/translation/decisions/bases/openrouter.py index b65acff6b06..ed39bc1a55b 100644 --- a/tests/integration/translation/decisions/bases/openrouter.py +++ b/tests/integration/translation/decisions/bases/openrouter.py @@ -6,6 +6,100 @@ from integration.translation.case import TranslationTestCase """ TYPESAFE_JEV_1_13_TEST_CASE: Final = TranslationTestCase( scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "openrouter/typesafe/jev-1.13", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/alpha/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-openrouter-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, + expected_litellm_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.81}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.01}, {"value": "high", "probability": 0.99}], + "confidence": 0.99, + }, + { + "type": "score", + "name": "confidence", + "score": 0.5, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.5}, + {"value": 1, "label": "sure", "probability": 0.5}, + ], + "confidence": 0, + }, + ], + "usage": { + "input_tokens": 377, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 62, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 439, + }, + }, +) +TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", litellm_endpoint="/v1/systemone", litellm_request={ "model": "openrouter/typesafe/jev-1.13", diff --git a/tests/integration/translation/decisions/bases/perplexity.py b/tests/integration/translation/decisions/bases/perplexity.py index 02b1de51f24..9e29ceab8cc 100644 --- a/tests/integration/translation/decisions/bases/perplexity.py +++ b/tests/integration/translation/decisions/bases/perplexity.py @@ -6,6 +6,101 @@ from integration.translation.case import TranslationTestCase """ PPLX_DECIDER_V1_27B_TEST_CASE: Final = TranslationTestCase( scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "perplexity/pplx-decider-v1-27b", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-perplexity-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, + expected_litellm_response={ + "model": "pplx-decider-v1-27b", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.9989100737587077}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [ + {"value": "low", "probability": 0.0017684392321610232}, + {"value": "high", "probability": 0.9982315607678389}, + ], + "confidence": 0.9964631215356778, + }, + { + "type": "score", + "name": "confidence", + "score": 0.07367392327139817, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.9263260767286018}, + {"value": 1, "label": "sure", "probability": 0.07367392327139817}, + ], + "confidence": 0.8526521534572037, + }, + ], + "usage": { + "input_tokens": 318, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 321, + }, + }, +) +PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", litellm_endpoint="/v1/systemone", litellm_request={ "model": "perplexity/pplx-decider-v1-27b", diff --git a/tests/integration/translation/decisions/bases/strands_decider.py b/tests/integration/translation/decisions/bases/strands_decider.py index 8a1a51f8c3e..2950b090737 100644 --- a/tests/integration/translation/decisions/bases/strands_decider.py +++ b/tests/integration/translation/decisions/bases/strands_decider.py @@ -6,6 +6,40 @@ from integration.translation.case import TranslationTestCase """ STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE: Final = TranslationTestCase( scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "strands_decider/strands-decider-2B-hobson-v19", + "input": "Help! My payouts have been failing for 3 days!", + "questions": [{"type": "predicate", "name": "is_urgent", "instructions": "Does this convey urgency?"}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"content-type": "application/json"}, + expected_provider_request={ + "model": "strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + }, + mock_provider_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, + expected_litellm_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": [{"type": "predicate", "name": "is_urgent", "probability": 0.8277}], + "usage": { + "input_tokens": 86, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 87, + }, + }, +) +STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", litellm_endpoint="/v1/systemone", litellm_request={ "model": "strands_decider/strands-decider-2B-hobson-v19", diff --git a/tests/integration/translation/decisions/bases/typesafe.py b/tests/integration/translation/decisions/bases/typesafe.py index 5da5b05cb09..7d47ce6b082 100644 --- a/tests/integration/translation/decisions/bases/typesafe.py +++ b/tests/integration/translation/decisions/bases/typesafe.py @@ -6,6 +6,98 @@ from integration.translation.case import TranslationTestCase """ JEV_1_13_0_TEST_CASE: Final = TranslationTestCase( scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "typesafe/jev-1.13.0", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"authorization": "Bearer synthetic-typesafe-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, + expected_litellm_response={ + "model": "jev-1.13.0", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.78}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.01}, {"value": "high", "probability": 0.99}], + "confidence": 0.99, + }, + { + "type": "score", + "name": "confidence", + "score": 0.51, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.49}, + {"value": 1, "label": "sure", "probability": 0.51}, + ], + "confidence": 0.03, + }, + ], + "usage": { + "input_tokens": 377, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 62, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 439, + }, + }, +) +JEV_1_13_0_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", litellm_endpoint="/v1/systemone", litellm_request={ "model": "typesafe/jev-1.13.0", diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py index 79fc0574b3b..21e0538897d 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py @@ -2,10 +2,10 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE +from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE, CLEF_SYSTEMONE_TEST_CASE from integration.translation.runner import assert_translation -@pytest.mark.parametrize("case", [CLEF_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [CLEF_TEST_CASE, CLEF_SYSTEMONE_TEST_CASE], ids=lambda case: case.id) def test_decisions_basic_cloudflare(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py index a74cb23f6b4..973526d7272 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py @@ -2,10 +2,15 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.openrouter import TYPESAFE_JEV_1_13_TEST_CASE +from integration.translation.decisions.bases.openrouter import ( + TYPESAFE_JEV_1_13_TEST_CASE, + TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE, +) from integration.translation.runner import assert_translation -@pytest.mark.parametrize("case", [TYPESAFE_JEV_1_13_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize( + "case", [TYPESAFE_JEV_1_13_TEST_CASE, TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE], ids=lambda case: case.id +) def test_decisions_basic_openrouter(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py index e341d24ba53..d9662ec4b9f 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py @@ -2,10 +2,15 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.perplexity import PPLX_DECIDER_V1_27B_TEST_CASE +from integration.translation.decisions.bases.perplexity import ( + PPLX_DECIDER_V1_27B_TEST_CASE, + PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE, +) from integration.translation.runner import assert_translation -@pytest.mark.parametrize("case", [PPLX_DECIDER_V1_27B_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize( + "case", [PPLX_DECIDER_V1_27B_TEST_CASE, PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE], ids=lambda case: case.id +) def test_decisions_basic_perplexity(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py index cae7ca70c78..2d2cd9caaf5 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py @@ -2,10 +2,17 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.strands_decider import STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE +from integration.translation.decisions.bases.strands_decider import ( + STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE, + STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE, +) from integration.translation.runner import assert_translation -@pytest.mark.parametrize("case", [STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize( + "case", + [STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE, STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE], + ids=lambda case: case.id, +) def test_decisions_basic_strands_decider(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py index 302636e6fb4..c1111cc6ba9 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py @@ -2,10 +2,10 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE +from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE, JEV_1_13_0_SYSTEMONE_TEST_CASE from integration.translation.runner import assert_translation -@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE, JEV_1_13_0_SYSTEMONE_TEST_CASE], ids=lambda case: case.id) def test_decisions_basic_typesafe(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index ec5375bc71e..9afea66835b 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1088,6 +1088,141 @@ async def test_openrouter_decisions_uses_provider_reported_cost_without_cost_map assert get_response_cost_from_hidden_params(response.hidden_params) == cost +_SAFETY_IDENTIFIER: Final = "end-user-7" +_PREDICATE_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, +) +_PREDICATE_REQUESTS: Final[tuple[Mapping[str, object], ...]] = ( + MappingProxyType({"input": "review", "questions": _PREDICATE_QUESTIONS}), + MappingProxyType( + {"state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}} + ), +) +_PREDICATE_REQUEST_IDS: Final = ("input_format", "state_format") + + +@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +def test_safety_identifier_is_refused_before_http_when_the_provider_cannot_take_it( + request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"safety_identifier.*drop_params") as caught: + litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + safety_identifier=_SAFETY_IDENTIFIER, + api_key="caller-key", + **request_kwargs, + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope", ("call", "global")) +async def test_safety_identifier_is_dropped_from_the_wire_under_drop_params( + scope: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", scope == "global") + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + input="review", + questions=_PREDICATE_QUESTIONS, + safety_identifier=_SAFETY_IDENTIFIER, + api_key="caller-key", + **({"drop_params": True} if scope == "call" else {}), + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert isinstance(response, OpenAIDecisionResponse) + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": "is_defect", "probability": 0.9} + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params")) +async def test_safety_identifier_reaches_openai_whether_or_not_params_are_dropped( + drop_params: bool, + request_kwargs: Mapping[str, object], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + + await litellm.adecisions( + model="openai/gpt-6-luna", + safety_identifier=_SAFETY_IDENTIFIER, + api_key="caller-key", + **request_kwargs, + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + "safety_identifier": _SAFETY_IDENTIFIER, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +async def test_non_string_safety_identifier_is_rejected_before_http( + request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + invalid_safety_identifier: Final[Mapping[str, object]] = MappingProxyType({"safety_identifier": 7}) + + with pytest.raises(litellm.BadRequestError, match=r"(?s)Invalid Decisions request.*safety_identifier"): + await litellm.adecisions( + model="openai/gpt-6-luna", api_key="caller-key", **request_kwargs, **invalid_safety_identifier + ) + + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_params( + request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + dropped_safety_identifier: Final[Mapping[str, object]] = MappingProxyType( + {"safety_identifier": 7, "drop_params": True} + ) + + await litellm.adecisions( + model="openai/gpt-6-luna", api_key="caller-key", **request_kwargs, **dropped_safety_identifier + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } + + @pytest.mark.asyncio @pytest.mark.parametrize( "api_base", diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index ccfa12e10d6..e2f7b6ab884 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -530,6 +530,68 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_ assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) +@pytest.mark.parametrize("safety_identifier", (7, ["end-user-1"]), ids=("numeric", "list")) +@pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params")) +def test_a_non_string_safety_identifier_is_refused_unless_the_deployment_drops_params( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + safety_identifier: object, + deployment_drops_params: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "decider", + "litellm_params": { + "model": "openai/gpt-6-luna", + "api_key": "k", + "drop_params": deployment_drops_params, + }, + } + ] + ), + ) + upstream: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond( + json={ + "model": "gpt-6-luna", + "answers": _OPENAI_FORMAT_ANSWERS[:1], + "usage": { + "input_tokens": _INPUT_TOKENS, + "output_tokens": _OUTPUT_TOKENS, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + }, + } + ) + request_body: Final = { + "model": "decider", + "input": "The package arrived with a broken screen.", + "questions": _OPENAI_FORMAT_REQUEST["questions"][:1], + "safety_identifier": safety_identifier, + } + + response: Final = client.post("/v1/decisions", json=request_body) + + if not deployment_drops_params: + assert response.status_code == 400, response.text + assert "safety_identifier" in response.json()["error"]["message"] + assert not upstream.called + return + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "The package arrived with a broken screen.", + "questions": _OPENAI_FORMAT_REQUEST["questions"][:1], + } + + def _decisions_feature() -> LazyFeature: return next(feature for feature in LAZY_FEATURES if feature.name == "decisions") From a25782c96942dfbfc1e7cfc71147e8a18b6008f0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 02:40:28 +0000 Subject: [PATCH 12/18] fix(model_prices): drop unverified output price for azure_ai/Microsoft-Decision-1 (#45717) Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 674208bb7dac89b177df00293bc8cd03725352e8) --- ci_cd/cost_map_guard.py | 56 +++++++++++++++-- ...odel_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- tests/unit/test_cost_map_guard.py | 62 ++++++++++++++++++- 4 files changed, 114 insertions(+), 8 deletions(-) diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 5842cf6f1ac..14eecf01461 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -14,8 +14,9 @@ import argparse import json import subprocess import sys -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Final from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors @@ -25,6 +26,7 @@ BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json" GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" +EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({}) CostMap = dict[str, object] @@ -51,7 +53,31 @@ def _rendered_schema(cost_map: CostMap) -> str: return str(error) -def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]: +def _evaluation_output_price_failure( + key: str, entry: object, evaluation_output_price_sources: Mapping[str, str] +) -> str | None: + if not isinstance(entry, dict) or entry.get("mode") != "evaluation": + return None + output_cost: Final = entry.get("output_cost_per_token") + if ( + not isinstance(output_cost, (int, float)) + or isinstance(output_cost, bool) + or output_cost <= 0 + or key in evaluation_output_price_sources + ): + return None + return ( + f"{key}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor " + "prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)" + ) + + +def _file_failures( + head: Snapshot, + head_map: CostMap, + *, + evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES, +) -> tuple[str, ...]: schema_text: Final = _rendered_schema(head_map) if not schema_text.startswith("{"): return (schema_text,) @@ -75,6 +101,16 @@ def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]: f"{COST_MAP_PATH} does not validate against its schema: {error}" for error in validation_errors(head_map, json.loads(schema_text))[:20] ), + *( + failure + for key, entry in head_map.items() + if ( + failure := _evaluation_output_price_failure( + key, entry, evaluation_output_price_sources + ) + ) + is not None + ), ) @@ -121,13 +157,25 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str: return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched" -def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]: +def guard_failures( + base: Snapshot, + head: Snapshot, + changed_files: Sequence[str], + bot: bool, + *, + evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES, +) -> tuple[str, ...]: if not bot and not touches_cost_map(changed_files): return () head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH) if isinstance(head_map, str): return (head_map,) - return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ())) + return ( + *_file_failures( + head, head_map, evaluation_output_price_sources=evaluation_output_price_sources + ), + *(_bot_failures(base, head_map, changed_files) if bot else ()), + ) def _git(*args: str) -> str | None: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 35deaadcd97..d42e66605d7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -79770,7 +79770,7 @@ "input_cost_per_token": 4.2e-08, "litellm_provider": "azure_ai", "mode": "evaluation", - "output_cost_per_token": 4.2e-08, + "output_cost_per_token": 0.0, "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 35deaadcd97..d42e66605d7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -79770,7 +79770,7 @@ "input_cost_per_token": 4.2e-08, "litellm_provider": "azure_ai", "mode": "evaluation", - "output_cost_per_token": 4.2e-08, + "output_cost_per_token": 0.0, "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" } } diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index 1595bd9864f..e127aa08d29 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping import importlib.util import json import subprocess @@ -62,8 +63,19 @@ def _snapshot(cost_map: dict[str, object], backup: str | None = None, schema: st BASE: Final = _snapshot(BASE_MAP) -def _failures(head: object, changed_files: tuple[str, ...] = MAP_FILES, bot: bool = True) -> tuple[str, ...]: - return guard.guard_failures(BASE, head, changed_files, bot) +def _failures( + head: object, + changed_files: tuple[str, ...] = MAP_FILES, + bot: bool = True, + evaluation_output_price_sources: Mapping[str, str] = guard.EVALUATION_OUTPUT_PRICE_SOURCES, +) -> tuple[str, ...]: + return guard.guard_failures( + BASE, + head, + changed_files, + bot, + evaluation_output_price_sources=evaluation_output_price_sources, + ) def test_in_sync_files_pass_for_humans_and_bots() -> None: @@ -71,6 +83,52 @@ def test_in_sync_files_pass_for_humans_and_bots() -> None: assert _failures(BASE, bot=True) == () +EVALUATION_MODEL: Final = "azure_ai/Microsoft-Decision-1" +EVALUATION_OUTPUT_PRICE_FAILURE: Final = ( + f"{EVALUATION_MODEL}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor " + "prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)" +) + + +@pytest.mark.parametrize("bot", (True, False)) +def test_evaluation_rows_with_output_price_fail_for_humans_and_bots(bot: bool) -> None: + head: Final = _snapshot( + {**BASE_MAP, EVALUATION_MODEL: _entry(mode="evaluation", output_cost_per_token=4.2e-08)} + ) + assert _failures(head, bot=bot) == (EVALUATION_OUTPUT_PRICE_FAILURE,) + + +def test_evaluation_rows_with_zero_or_missing_output_price_pass() -> None: + zero_output: Final = _snapshot( + {**BASE_MAP, EVALUATION_MODEL: _entry(mode="evaluation", output_cost_per_token=0.0)} + ) + missing_output_entry: Final = { + key: value + for key, value in _entry(mode="evaluation").items() + if key != "output_cost_per_token" + } + missing_output: Final = _snapshot({**BASE_MAP, EVALUATION_MODEL: missing_output_entry}) + + assert _failures(zero_output) == () + assert _failures(missing_output) == () + + +def test_chat_rows_with_output_price_pass() -> None: + head: Final = _snapshot( + {**BASE_MAP, "openrouter/chat-with-output": _entry(output_cost_per_token=4.2e-08)} + ) + assert _failures(head) == () + + +def test_evaluation_output_price_source_allowlist_passes() -> None: + head: Final = _snapshot( + {**BASE_MAP, EVALUATION_MODEL: _entry(mode="evaluation", output_cost_per_token=4.2e-08)} + ) + source: Final = 'https://vendor.example/pricing "Output tokens are charged per token."' + + assert _failures(head, evaluation_output_price_sources={EVALUATION_MODEL: source}) == () + + def test_bot_may_add_and_reprice_models() -> None: head = _snapshot({**BASE_MAP, "openrouter/a": _entry(9e-06, supports_vision=True), "openrouter/c": _entry()}) assert _failures(head) == () From c2f7d15d978125166a79b601e848bc5c10383ab5 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:51:19 -0700 Subject: [PATCH 13/18] feat(strands_decider): call a Strands Decider on an AgentCore runtime ARN (#45678) * feat(strands_decider): call a Strands Decider on an AgentCore runtime ARN A strands_decider deployment whose api_base is a bedrock-agentcore runtime ARN now calls InvokeAgentRuntime on that runtime, signed with SigV4 from the deployment's aws_* params or the default chain, reuses one session per runtime, and maps the runtime's 200 error envelopes to 400, 503, or 500 * ci: run the strands_decider unit tests in the llms job * fix(strands_decider): take the AgentCore session id from the deployment, never the caller A caller could pick the runtime session through extra_headers, and every new id starts another microVM. The session id now comes from the deployment's agentcore_runtime_session_id (banned in request bodies) or the ARN default, and caller-supplied host, x-amz-*, and x-amzn-bedrock-agentcore-runtime-* headers are not forwarded * fix(strands_decider): map runtime error replies only on AgentCore ARN calls * fix(strands_decider): sign plain decisions calls inline, off the AWS signing executor * test(strands_decider): cover AgentCore runtime ARN calls in integration wire tests --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> (cherry picked from commit ed9c31cfa8574b9d7516443f43d1466de4053cf3) --- litellm/decisions/main.py | 4 + .../llms/base_llm/decisions/transformation.py | 14 + litellm/llms/custom_httpx/llm_http_handler.py | 38 +- .../decisions/transformation.py | 154 ++++ litellm/proxy/auth/auth_utils.py | 2 + tests/integration/_support/forward_proxy.py | 117 +++ .../test_strands_decider_agentcore_wire.py | 827 ++++++++++++++++++ tests/unit/decisions/test_main.py | 200 +++++ tests/unit/llms/strands_decider/__init__.py | 0 .../strands_decider/decisions/__init__.py | 0 ...trands_decider_decisions_transformation.py | 256 ++++++ tests/unit/proxy/auth/test_auth_utils.py | 16 + 12 files changed, 1625 insertions(+), 3 deletions(-) create mode 100644 tests/integration/_support/forward_proxy.py create mode 100644 tests/integration/providers/test_strands_decider_agentcore_wire.py create mode 100644 tests/unit/llms/strands_decider/__init__.py create mode 100644 tests/unit/llms/strands_decider/decisions/__init__.py create mode 100644 tests/unit/llms/strands_decider/decisions/test_strands_decider_decisions_transformation.py diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 2dd0d80aeb6..1c4108654fa 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -60,6 +60,7 @@ class _DecisionsCall: logging_obj: LiteLLMLoggingObj | None headers: Mapping[str, str] timeout: float | httpx.Timeout | None + litellm_params: Mapping[str, object] = field(repr=False) def _supported_providers() -> tuple[str, ...]: @@ -253,6 +254,7 @@ def _prepare_call( logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, headers=extra_headers or {}, timeout=timeout, + litellm_params=kwargs, ) @@ -336,6 +338,7 @@ async def adecisions( api_key=call.api_key, headers=call.headers, timeout=call.timeout, + litellm_params=call.litellm_params, ) except Exception as error: raise _map_upstream_exception(error, call) from error @@ -381,6 +384,7 @@ def decisions( api_key=call.api_key, headers=call.headers, timeout=call.timeout, + litellm_params=call.litellm_params, ) except Exception as error: raise _map_upstream_exception(error, call) from error diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index baeeb3f0449..724ca23c2ab 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -341,6 +341,20 @@ class BaseDecisionsConfig(ABC): def get_complete_url(self, api_base: str, model: str) -> str: return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" + def signs_with_aws(self, api_base: str) -> bool: + return False + + def sign_request( + self, + headers: Mapping[str, str], + url: str, + api_base: str, + body: Mapping[str, object], + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> tuple[Mapping[str, str], bytes | None]: + return headers, None + def transform_decisions_request( self, model: str, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 5e5c2e467f4..8383078b7ba 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -3,7 +3,7 @@ import json import ssl from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager -from functools import lru_cache +from functools import lru_cache, partial from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -1449,6 +1449,7 @@ class BaseLLMHTTPHandler: api_key: str | None, headers: Mapping[str, str], timeout: float | httpx.Timeout | None, + litellm_params: Mapping[str, object], client: HTTPHandler | None = None, ) -> DecisionsIRResponse: url, outbound_headers, data = self._prepare_decisions_request( @@ -1460,10 +1461,23 @@ class BaseLLMHTTPHandler: api_key=api_key, headers=headers, ) + signed_headers, signed_body = provider_config.sign_request( + headers=outbound_headers, + url=url, + api_base=api_base, + body=data, + api_key=api_key, + litellm_params=litellm_params, + ) sync_httpx_client: Final = client if client is not None else _get_httpx_client() try: response: Final = sync_httpx_client.post( - url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj + url, + content=signed_body, + json=data if signed_body is None else None, + headers=dict(signed_headers), + timeout=timeout, + logging_obj=logging_obj, ) except httpx.HTTPError as e: raise self._handle_error(e=e, provider_config=provider_config) @@ -1483,6 +1497,7 @@ class BaseLLMHTTPHandler: api_key: str | None, headers: Mapping[str, str], timeout: float | httpx.Timeout | None, + litellm_params: Mapping[str, object], client: AsyncHTTPHandler | None = None, ) -> DecisionsIRResponse: url, outbound_headers, data = self._prepare_decisions_request( @@ -1494,6 +1509,18 @@ class BaseLLMHTTPHandler: api_key=api_key, headers=headers, ) + sign: Final = partial( + provider_config.sign_request, + headers=outbound_headers, + url=url, + api_base=api_base, + body=data, + api_key=api_key, + litellm_params=litellm_params, + ) + signed_headers, signed_body = ( + await run_aws_signing(sign) if provider_config.signs_with_aws(api_base) else sign() + ) async_httpx_client: Final = ( client if client is not None @@ -1501,7 +1528,12 @@ class BaseLLMHTTPHandler: ) try: response: Final = await async_httpx_client.post( - url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj + url, + content=signed_body, + json=data if signed_body is None else None, + headers=dict(signed_headers), + timeout=timeout, + logging_obj=logging_obj, ) except httpx.HTTPError as e: raise self._handle_error(e=e, provider_config=provider_config) diff --git a/litellm/llms/strands_decider/decisions/transformation.py b/litellm/llms/strands_decider/decisions/transformation.py index 9ef966e18f2..fd8a643be2a 100644 --- a/litellm/llms/strands_decider/decisions/transformation.py +++ b/litellm/llms/strands_decider/decisions/transformation.py @@ -1,7 +1,161 @@ +import hashlib +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from functools import partial +from types import MappingProxyType +from typing import Final +from urllib.parse import quote, unquote + +import httpx +from pydantic import BaseModel, ConfigDict, ValidationError + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, sign_aws_json_post +from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse +from litellm.types.llms.bedrock import AwsAuthParams + +AGENTCORE_SESSION_HEADER: Final = "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id" +_RESERVED_HEADER_PREFIXES: Final = ("x-amzn-bedrock-agentcore-runtime-", "x-amz-") +_RUNTIME_ARN: Final = re.compile( + r"\Aarn:aws(?:-[a-z]+)*:bedrock-agentcore:(?P[a-z0-9-]+):[0-9]{12}:runtime/[A-Za-z0-9_-]+\Z" +) +_INVOCATIONS_PATH: Final = re.compile(r"\A/runtimes/(?P[^/]+)/invocations\Z") +_RUNTIME_ERROR_STATUS: Final = MappingProxyType({"bad_request": 400, "invalid_request": 400, "loading": 503}) + + +@dataclass(frozen=True, slots=True) +class AgentCoreRuntime: + arn: str + region: str + + @property + def invocations_url(self) -> str: + host: Final = f"bedrock-agentcore.{self.region}.{get_aws_dns_suffix(self.region)}" + return f"https://{host}/runtimes/{quote(self.arn, safe='')}/invocations" + + @property + def default_session_id(self) -> str: + return f"litellm-decider-{hashlib.sha256(self.arn.encode()).hexdigest()}" + + +def agentcore_runtime(api_base: str) -> AgentCoreRuntime | None: + match: Final = _RUNTIME_ARN.match(api_base) + return None if match is None else AgentCoreRuntime(arn=api_base, region=match["region"]) + + +def invoked_runtime(url: httpx.URL) -> AgentCoreRuntime | None: + match: Final = _INVOCATIONS_PATH.match(url.raw_path.decode("ascii")) + runtime: Final = None if match is None else agentcore_runtime(unquote(match["arn"])) + return runtime if runtime is not None and runtime.invocations_url == str(url) else None + + +class _RuntimeError(BaseModel): + model_config = ConfigDict(frozen=True) + + code: str + message: str = "" + + +class _RuntimeReply(BaseModel): + model_config = ConfigDict(frozen=True) + + error: _RuntimeError | None = None + answers: object = None + + +def _runtime_error(content: bytes) -> _RuntimeError | None: + try: + reply: Final = _RuntimeReply.model_validate_json(content) + except ValidationError: + return None + return reply.error if reply.answers is None else None + + +class _SessionParams(BaseModel): + model_config = ConfigDict(frozen=True) + + agentcore_runtime_session_id: str | None = None + + +def _is_reserved_header(name: str) -> bool: + lowered: Final = name.lower() + return lowered == "host" or lowered.startswith(_RESERVED_HEADER_PREFIXES) + + +def _runtime_headers( + headers: Mapping[str, str], runtime: AgentCoreRuntime, litellm_params: Mapping[str, object] +) -> Mapping[str, str]: + dropped: Final = sorted(name for name in headers if _is_reserved_header(name)) + if dropped: + verbose_logger.warning("Strands Decider: not forwarding reserved AgentCore header(s) %s", dropped) + configured: Final = _SessionParams.model_validate(litellm_params).agentcore_runtime_session_id + return { + **{name: value for name, value in headers.items() if not _is_reserved_header(name)}, + AGENTCORE_SESSION_HEADER: runtime.default_session_id if configured is None else configured, + } class StrandsDeciderDecisionsConfig(BaseDecisionsConfig): api_key_env = ("STRANDS_DECIDER_API_KEY",) api_base_env = ("STRANDS_DECIDER_API_BASE",) api_key_required = False + + def __init__(self, aws: BaseAWSLLM | None = None) -> None: + self._aws: Final = aws if aws is not None else BaseAWSLLM() + + def get_complete_url(self, api_base: str, model: str) -> str: + runtime: Final = agentcore_runtime(api_base) + return super().get_complete_url(api_base, model) if runtime is None else runtime.invocations_url + + def signs_with_aws(self, api_base: str) -> bool: + return agentcore_runtime(api_base) is not None + + def sign_request( + self, + headers: Mapping[str, str], + url: str, + api_base: str, + body: Mapping[str, object], + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> tuple[Mapping[str, str], bytes | None]: + runtime: Final = agentcore_runtime(api_base) + if runtime is None: + return headers, None + session_headers: Final = _runtime_headers(headers, runtime, litellm_params) + if api_key is not None: + return session_headers, None + payload: Final = json.dumps(body) + signed: Final = sign_aws_json_post( + get_credentials=partial( + self._aws.resolve_credentials, AwsAuthParams.model_validate(litellm_params), runtime.region + ), + service_name="bedrock-agentcore", + aws_region_name=runtime.region, + url=url, + body=payload, + headers=session_headers, + ) + return dict(signed.headers.items()), payload.encode() + + def transform_decisions_response( + self, + model: str, + custom_llm_provider: str, + raw_response: httpx.Response, + request: DecisionsIRRequest, + ) -> DecisionsIRResponse: + runtime_error: Final = ( + None if invoked_runtime(raw_response.request.url) is None else _runtime_error(raw_response.content) + ) + if runtime_error is not None: + raise BaseLLMException( + status_code=_RUNTIME_ERROR_STATUS.get(runtime_error.code, 500), + message=f"Strands Decider runtime error '{runtime_error.code}': {runtime_error.message}", + ) + return super().transform_decisions_response(model, custom_llm_provider, raw_response, request) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 813d72ed7ae..f20bdada863 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -343,6 +343,8 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( "anthropic_workspace_id", "anthropic-workspace-id", "bedrock_tags", + # Deployment-pinned AgentCore session: every new caller-chosen value provisions another microVM. + "agentcore_runtime_session_id", # Provider-specific endpoint overrides that flow into the outbound # request via ``optional_params``. Same threat as ``api_base``: # ``s3_endpoint_url`` redirects Bedrock file uploads to attacker diff --git a/tests/integration/_support/forward_proxy.py b/tests/integration/_support/forward_proxy.py new file mode 100644 index 00000000000..0d92c3d0719 --- /dev/null +++ b/tests/integration/_support/forward_proxy.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import contextlib +import socket +import threading +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from queue import SimpleQueue +from typing import Final + + +@dataclass(frozen=True, slots=True) +class Tunnel: + method: str + target: str + headers: Mapping[str, str] + + +@dataclass(frozen=True, slots=True) +class ForwardProxy: + url: str + port: int + received: SimpleQueue[Tunnel] + + def drain(self) -> tuple[Tunnel, ...]: + return tuple(self.received.get_nowait() for _ in range(self.received.qsize())) + + def targets(self) -> tuple[str, ...]: + return tuple(tunnel.target for tunnel in self.drain()) + + +@contextmanager +def refusing_forward_proxy(port: int = 0) -> Generator[ForwardProxy, None, None]: + """Owned HTTP forward proxy: records the host each CONNECT asks for and refuses the tunnel with 403. + + A litellm proxy booted with ``HTTPS_PROXY`` pointed here makes the host it dials for a provider + observable without a byte leaving the box; the caller sees litellm's own connection error. + """ + received: Final[SimpleQueue[Tunnel]] = SimpleQueue() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + timeout = 5 + + def refuse(self) -> None: + received.put(Tunnel(self.command, self.path, {name.lower(): value for name, value in self.headers.items()})) + self.send_response(403) + self.send_header("content-length", "0") + self.send_header("connection", "close") + self.end_headers() + self.close_connection = True + + do_CONNECT = refuse + do_GET = refuse + do_POST = refuse + do_PUT = refuse + do_DELETE = refuse + + def log_message(self, format: str, *args: object) -> None: + pass + + with ThreadingHTTPServer(("127.0.0.1", port), Handler) as server: + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) + thread.start() + try: + yield ForwardProxy(f"http://127.0.0.1:{server.server_port}", server.server_port, received) + finally: + server.shutdown() + thread.join(timeout=6) + assert not thread.is_alive(), "Owned forward proxy survived cleanup" + server.server_close() + + +def _pipe(source: socket.socket, sink: socket.socket) -> None: + with contextlib.suppress(OSError): + for chunk in iter(lambda: source.recv(65536), b""): + sink.sendall(chunk) + with contextlib.suppress(OSError): + sink.shutdown(socket.SHUT_WR) + + +@contextmanager +def tunnelling_forward_proxy(destination_port: int, port: int = 0) -> Generator[ForwardProxy, None, None]: + received: Final[SimpleQueue[Tunnel]] = SimpleQueue() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + rbufsize = 0 + timeout = 30 + request: socket.socket + + def do_CONNECT(self) -> None: + received.put(Tunnel(self.command, self.path, {name.lower(): value for name, value in self.headers.items()})) + self.close_connection = True + with socket.create_connection(("127.0.0.1", destination_port), timeout=30) as destination: + self.send_response(200, "Connection established") + self.end_headers() + outbound: Final = threading.Thread(target=_pipe, args=(self.request, destination)) + outbound.start() + _pipe(destination, self.request) + outbound.join(timeout=35) + + def log_message(self, format: str, *args: object) -> None: + pass + + with ThreadingHTTPServer(("127.0.0.1", port), Handler) as server: + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) + thread.start() + try: + yield ForwardProxy(f"http://127.0.0.1:{server.server_port}", server.server_port, received) + finally: + server.shutdown() + thread.join(timeout=6) + assert not thread.is_alive(), "Owned forward proxy survived cleanup" + server.server_close() diff --git a/tests/integration/providers/test_strands_decider_agentcore_wire.py b/tests/integration/providers/test_strands_decider_agentcore_wire.py new file mode 100644 index 00000000000..bf5da453137 --- /dev/null +++ b/tests/integration/providers/test_strands_decider_agentcore_wire.py @@ -0,0 +1,827 @@ +import hashlib +import json +import os +import re +import threading +import time +import uuid +from collections.abc import Callable, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, replace +from functools import partial +from pathlib import Path +from typing import Final +from urllib.parse import quote, unquote + +import httpx +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, gateway_from_environment +from integration._support.database import read_rows +from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy +from integration._support.process import owned_proxy +from integration._support.sigv4 import encoded_path, signature +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm.llms.strands_decider.decisions.transformation import StrandsDeciderDecisionsConfig + +_ACCOUNT: Final = "123456789012" +_ACCESS_KEY: Final = "AKIAAGENTCOREAUDIT01" +_SECRET_KEY: Final = "synthetic-agentcore-audit-secret" +_SIGNING: Final[Mapping[str, JsonValue]] = {"aws_access_key_id": _ACCESS_KEY, "aws_secret_access_key": _SECRET_KEY} +_JWT: Final = "synthetic-agentcore-jwt" +_ENV_JWT: Final = "synthetic-agentcore-env-jwt" +_US_EAST: Final = "us-east-1" +_EU_WEST: Final = "eu-west-1" +_MODEL: Final = "strands_decider/systemone-decider" +_BODY_MODEL: Final = "systemone-decider" +_SESSION_HEADER: Final = "x-amzn-bedrock-agentcore-runtime-session-id" +_CONFIGURED_SESSION: Final = "agentcore-audit-configured-session-0000000001" +_STATE: Final[dict[str, JsonValue]] = {"ticket": "Refund requested twice for one order", "queue": "payments"} +_QUESTIONS: Final[dict[str, JsonValue]] = { + "duplicate": {"type": "noul", "instructions": "Is this a duplicate refund request?"}, + "urgency": {"type": "choice", "criteria": {"low": "can wait", "high": "money at risk"}}, +} +_SDK_QUESTIONS: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python(_QUESTIONS) +_ANSWERS: Final[dict[str, JsonValue]] = { + "duplicate": {"type": "noul", "noul": 0.88}, + "urgency": {"type": "choice", "choice": "high", "confidence": 0.75, "probabilities": {"low": 0.25, "high": 0.75}}, +} +_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 211, "output_tokens": 2} +_UPSTREAM_BODY: Final[dict[str, JsonValue]] = {"model": _BODY_MODEL, "state": _STATE, "questions": _QUESTIONS} +_INPUT: Final = "Refund requested twice for one order" +_OPENAI_QUESTIONS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "duplicate", "instructions": "Is this a duplicate refund request?"}, + { + "type": "choice", + "name": "urgency", + "instructions": "How urgent is it?", + "choices": [{"value": "low", "description": "can wait"}, {"value": "high", "description": "money at risk"}], + }, +] +_OPENAI_UPSTREAM_BODY: Final[dict[str, JsonValue]] = { + "model": _BODY_MODEL, + "state": _INPUT, + "questions": { + "duplicate": {"type": "noul", "instructions": "Is this a duplicate refund request?"}, + "urgency": { + "type": "choice", + "instructions": "How urgent is it?", + "criteria": {"low": "can wait", "high": "money at risk"}, + }, + }, +} +_OPENAI_ANSWER: Final[dict[str, JsonValue]] = { + "model": _BODY_MODEL, + "answers": [ + {"type": "predicate", "name": "duplicate", "probability": 0.88}, + { + "type": "choice", + "name": "urgency", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.25}, {"value": "high", "probability": 0.75}], + "confidence": 0.75, + }, + ], + "usage": { + "input_tokens": 211, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 213, + }, +} + + +@dataclass(frozen=True, slots=True) +class _Route: + path: str + body: Mapping[str, JsonValue] + upstream: Mapping[str, JsonValue] + answer: Mapping[str, JsonValue] + + +_SYSTEMONE: Final = _Route( + "/v1/systemone", + {"state": _STATE, "questions": _QUESTIONS}, + _UPSTREAM_BODY, + {"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE}, +) +_OPENAI: Final = _Route( + "/v1/decisions", {"input": _INPUT, "questions": _OPENAI_QUESTIONS}, _OPENAI_UPSTREAM_BODY, _OPENAI_ANSWER +) +_ROUTES: Final = (_SYSTEMONE, _OPENAI, replace(_SYSTEMONE, path="/systemone"), replace(_OPENAI, path="/decisions")) +_SPEND_QUERY: Final = ( + "SELECT status, call_type, custom_llm_provider, model_group, api_base, prompt_tokens, completion_tokens " + 'FROM "LiteLLM_SpendLogs" WHERE request_id = %s' +) +_RUNTIME_TARGET: Final = re.compile( + r"\A/runtimes/arn:aws:bedrock-agentcore:[a-z0-9-]+:[0-9]{12}:runtime/(?P[A-Za-z0-9_-]+)/invocations\Z" +) +_ERROR_STATUS: Final = {"bad_request": 400, "invalid_request": 400, "loading": 503, "inference_error": 500} +_HTTP_ERRORS: Final = { + 403: "AccessDeniedException", + 404: "ResourceNotFoundException", + 429: "ThrottlingException", +} +_MALFORMED_SESSIONS: Final[Mapping[str, JsonValue]] = {"int": 1234567, "list": ["a" * 40], "empty": ""} +_LONG_SESSION: Final = "s" * 5120 +_NEAR_MISS_ARNS: Final = { + "endpoint": f"arn:aws:bedrock-agentcore:us-east-1:{_ACCOUNT}:runtime/audit_qualified/runtime-endpoint/DEFAULT", + "account": "arn:aws:bedrock-agentcore:us-east-1:12345:runtime/audit_short_account", + "service": f"arn:aws:bedrock:us-east-1:{_ACCOUNT}:runtime/audit_other_service", + "host": f"arn:aws:bedrock-agentcore:us-east-1.example.com:{_ACCOUNT}:runtime/audit_crafted_host", +} +_DEPLOYMENT_HEADERS: Final[dict[str, JsonValue]] = { + "x-amz-target": "AmazonBedrockAgentCore.Forged", + "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id": "deployment-forged-session-00000000000001", + "Host": "bedrock-agentcore.us-west-2.amazonaws.com", + "x-audit-trace": "deployment-trace", +} +_SIGNED: Final = "agentcore-audit-signed" +_REGION: Final = "agentcore-audit-region" +_SESSION: Final = "agentcore-audit-session" +_LONG: Final = "agentcore-audit-session-long" +_BEARER: Final = "agentcore-audit-bearer" +_HEADERS: Final = "agentcore-audit-headers" +_OPT_IN: Final = "agentcore-audit-opt-in" +_HEALTH: Final = "agentcore-audit-health" +_DROPPED: Final = "agentcore-audit-dropped" +_OUTAGE: Final = "agentcore-audit-outage" +_ENV: Final = "agentcore-audit-env" +_UNSIGNED: Final = "agentcore-audit-unsigned" +_SCRUBBED_PREFIXES: Final = ("AWS_", "STRANDS_DECIDER_") +_SCRUBBED_NAMES: Final = frozenset( + {"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY", "SSL_VERIFY", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE"} +) +_load_yaml: Final[Callable[[str], object]] = yaml.safe_load +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_OWNED_PROXY_CELL_SECONDS: Final = 2 * max(30.0, float(os.environ.get("INTEGRATION_PROXY_READY_SECONDS") or 70)) + 120 +pytestmark: Final = pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) + + +def _host(region: str) -> str: + return f"bedrock-agentcore.{region}.amazonaws.com" + + +def _arn(runtime: str, region: str = _US_EAST) -> str: + return f"arn:aws:bedrock-agentcore:{region}:{_ACCOUNT}:runtime/{runtime}" + + +def _invocations_url(runtime: str, region: str = _US_EAST) -> str: + return f"https://{_host(region)}/runtimes/{quote(_arn(runtime, region), safe='')}/invocations" + + +def _default_session(runtime: str, region: str = _US_EAST) -> str: + return f"litellm-decider-{hashlib.sha256(_arn(runtime, region).encode()).hexdigest()}" + + +def _runtime_of(request: Request) -> str | None: + match: Final = _RUNTIME_TARGET.match(unquote(request.target)) + return None if match is None else match["runtime"] + + +def _json_reply(body: Mapping[str, JsonValue], status: int = 200, **headers: str) -> Reply: + return Reply(status=status, body=json.dumps(body).encode(), headers=headers) + + +def _respond(outage: threading.Event, request: Request) -> Reply: + runtime: Final = _runtime_of(request) or "" + if runtime == "audit_dropped" or (runtime == "audit_outage" and outage.is_set()): + return Reply(drop_connection=True) + if runtime.startswith("audit_error_"): + code: Final = runtime.removeprefix("audit_error_") + return _json_reply({"error": {"code": code, "message": f"runtime refused with {code}"}}) + if runtime.startswith("audit_status_"): + status: Final = int(runtime.removeprefix("audit_status_")) + return _json_reply( + {"message": f"scripted {_HTTP_ERRORS[status]}"}, status, **{"x-amzn-errortype": _HTTP_ERRORS[status]} + ) + if not runtime: + return _json_reply({"message": f"unknown target {request.target}"}, 404) + return _json_reply({"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE}) + + +@dataclass(frozen=True, slots=True) +class _Runtime: + wire: Wire + tunnel: ForwardProxy + cert: Path + outage: threading.Event + + def reset(self) -> None: + self.outage.clear() + self.wire.drain() + self.tunnel.drain() + + def calls(self) -> tuple[Request, ...]: + return self.wire.drain() + + +def _calls_to(calls: Sequence[Request], runtime: str) -> tuple[Request, ...]: + return tuple(call for call in calls if _runtime_of(call) == runtime) + + +def _deployment(name: str, **litellm_params: JsonValue) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": {"model": _MODEL, **litellm_params}, + "model_info": {"id": name, "mode": "evaluation"}, + } + + +def _signed_deployment(name: str, runtime: str, region: str = _US_EAST, **extra: JsonValue) -> dict[str, JsonValue]: + return _deployment(name, api_base=_arn(runtime, region), **_SIGNING, **extra) + + +def _deployments() -> list[dict[str, JsonValue]]: + return [ + _signed_deployment(_SIGNED, "audit_signed", aws_region_name=_US_EAST), + _signed_deployment(_REGION, "audit_region", _EU_WEST, aws_region_name=_US_EAST), + _signed_deployment(_SESSION, "audit_session", agentcore_runtime_session_id=_CONFIGURED_SESSION), + _signed_deployment(_LONG, "audit_session_long", agentcore_runtime_session_id=_LONG_SESSION), + _deployment(_BEARER, api_base=_arn("audit_bearer"), api_key=_JWT), + _signed_deployment(_HEADERS, "audit_headers", extra_headers=_DEPLOYMENT_HEADERS), + _signed_deployment( + _OPT_IN, "audit_opt_in", configurable_clientside_auth_params=["agentcore_runtime_session_id"] + ), + _signed_deployment(_HEALTH, "audit_health"), + _signed_deployment(_DROPPED, "audit_dropped"), + _signed_deployment(_OUTAGE, "audit_outage"), + _deployment(_ENV, **_SIGNING), + _deployment(_UNSIGNED, api_base=_arn("audit_unsigned")), + *(_signed_deployment(f"agentcore-audit-error-{code}", f"audit_error_{code}") for code in _ERROR_STATUS), + *(_signed_deployment(f"agentcore-audit-status-{status}", f"audit_status_{status}") for status in _HTTP_ERRORS), + *( + _signed_deployment( + f"agentcore-audit-session-{label}", f"audit_session_{label}", agentcore_runtime_session_id=value + ) + for label, value in _MALFORMED_SESSIONS.items() + ), + *( + _deployment(f"agentcore-audit-near-miss-{label}", api_base=value, **_SIGNING) + for label, value in _NEAR_MISS_ARNS.items() + ), + ] + + +def _write_config(directory: Path) -> Path: + base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text())) + configuration: Final = { + **base, + "model_list": _deployments(), + "router_settings": {"disable_cooldowns": True, "num_retries": 0}, + } + path: Final = directory / "agentcore.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +def _scrubbed() -> tuple[str, ...]: + return tuple( + name for name in os.environ if name.upper().startswith(_SCRUBBED_PREFIXES) or name.upper() in _SCRUBBED_NAMES + ) + + +def _isolated_aws(directory: Path) -> dict[str, str]: + return { + "AWS_EC2_METADATA_DISABLED": "true", + "AWS_SHARED_CREDENTIALS_FILE": str(directory / "absent-credentials"), + "AWS_CONFIG_FILE": str(directory / "absent-config"), + } + + +def _proxy_environment(runtime: _Runtime, directory: Path) -> dict[str, str]: + return { + "HTTPS_PROXY": runtime.tunnel.url, + "NO_PROXY": "127.0.0.1,localhost", + "SSL_CERT_FILE": str(runtime.cert), + "STRANDS_DECIDER_API_BASE": _arn("audit_env"), + **_isolated_aws(directory), + } + + +@pytest.fixture(scope="module") +def module_gateway() -> Iterator[Gateway]: + with gateway_from_environment() as value: + yield value + + +@pytest.fixture(scope="module") +def runtime(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Runtime]: + cert, key = write_self_signed_cert(tmp_path_factory.mktemp("agentcore-cert"), (_host(_US_EAST), _host(_EU_WEST))) + outage: Final = threading.Event() + with wire_server(partial(_respond, outage), tls=server_context(cert, key)) as wire: + with tunnelling_forward_proxy(int(wire.url.rsplit(":", 1)[1])) as tunnel: + yield _Runtime(wire, tunnel, cert, outage) + + +@pytest.fixture(scope="module") +def agentcore( + module_gateway: Gateway, runtime: _Runtime, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("agentcore-proxy") + with owned_proxy( + module_gateway, + directory, + _proxy_environment(runtime, directory), + config=_write_config(directory), + remove_environment=_scrubbed(), + workers=2, + ) as candidate: + yield candidate + + +def _decide(gateway: Gateway, model: str, route: _Route = _SYSTEMONE, **extra: JsonValue) -> httpx.Response: + return gateway.request("POST", route.path, {"model": model, **route.body, **extra}) + + +def _authorization_fields(request: Request) -> dict[str, str]: + authorization: Final = request.headers.get("authorization", "") + assert authorization.startswith("AWS4-HMAC-SHA256 "), dict(request.headers) + return dict(part.split("=", 1) for part in authorization.removeprefix("AWS4-HMAC-SHA256 ").split(", ")) + + +def _assert_signed(request: Request, region: str = _US_EAST) -> None: + fields: Final = _authorization_fields(request) + access_key, scope = fields["Credential"].split("/", 1) + assert access_key == _ACCESS_KEY, fields + assert scope == f"{request.headers['x-amz-date'][:8]}/{region}/bedrock-agentcore/aws4_request", fields + signed: Final = fields["SignedHeaders"] + assert {"host", "x-amz-date", _SESSION_HEADER}.issubset(signed.split(";")), signed + expected: Final = signature( + "POST", encoded_path(request.target), request.headers, signed, request.body, _SECRET_KEY, scope + ) + assert fields["Signature"] == expected[1], (request.target, fields) + + +def _assert_answered(response: httpx.Response, model: str, route: _Route = _SYSTEMONE) -> None: + assert response.status_code == 200, response.text + assert response.json() == route.answer, response.text + assert response.headers["x-litellm-model-group"] == model, dict(response.headers) + + +def _assert_runtime_target(request: Request, runtime: str, region: str = _US_EAST) -> None: + assert request.method == "POST", request.method + assert request.target == f"/runtimes/{quote(_arn(runtime, region), safe='')}/invocations", request.target + assert request.headers["host"] == _host(region), dict(request.headers) + + +def _assert_runtime_call(request: Request, runtime: str, region: str = _US_EAST, route: _Route = _SYSTEMONE) -> None: + _assert_runtime_target(request, runtime, region) + assert json.loads(request.body) == route.upstream, request.body + + +def _canonical(body: JsonValue | Mapping[str, JsonValue]) -> str: + return json.dumps(body, sort_keys=True) + + +def _spend_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70) + return rows[0] + + +def _spend_rows(call_ids: Sequence[str]) -> list[dict[str, JsonValue]]: + query: Final = ( + "SELECT request_id, status FROM \"LiteLLM_SpendLogs\" WHERE request_id = ANY(string_to_array(%s, ','))" + ) + return eventually( + lambda: read_rows(query, (",".join(call_ids),)), lambda found: len(found) >= len(call_ids), seconds=70 + ) + + +def _connect_targets(runtime: _Runtime) -> frozenset[str]: + return frozenset(runtime.tunnel.targets()) + + +def test_an_arn_deployment_answers_on_every_decisions_route_signed_for_bedrock_agentcore( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + responses: Final = tuple(_decide(agentcore, _SIGNED, route) for route in _ROUTES) + for response, route in zip(responses, _ROUTES, strict=True): + _assert_answered(response, _SIGNED, route) + assert _connect_targets(runtime) == {f"{_host(_US_EAST)}:443"} + calls: Final = runtime.calls() + assert len(calls) == len(_ROUTES), calls + for call, route in zip(calls, _ROUTES, strict=True): + _assert_runtime_call(call, "audit_signed", route=route) + assert call.headers[_SESSION_HEADER] == _default_session("audit_signed"), dict(call.headers) + _assert_signed(call) + for response in responses: + row = _spend_row(response.headers["x-litellm-call-id"]) + assert row == { + "status": "success", + "call_type": "adecisions", + "custom_llm_provider": "strands_decider", + "model_group": _SIGNED, + "api_base": _invocations_url("audit_signed"), + "prompt_tokens": 211, + "completion_tokens": 2, + }, row + + +def test_the_arn_region_picks_the_host_and_signing_scope_over_aws_region_name( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + _assert_answered(_decide(agentcore, _REGION), _REGION) + assert _connect_targets(runtime) == {f"{_host(_EU_WEST)}:443"} + (call,) = runtime.calls() + _assert_runtime_call(call, "audit_region", _EU_WEST) + _assert_signed(call, _EU_WEST) + + +def test_a_configured_session_id_is_sent_as_written_and_signed(agentcore: Gateway, runtime: _Runtime) -> None: + runtime.reset() + _assert_answered(_decide(agentcore, _SESSION), _SESSION) + _assert_answered(_decide(agentcore, _LONG), _LONG) + calls: Final = runtime.calls() + (configured,) = _calls_to(calls, "audit_session") + (long,) = _calls_to(calls, "audit_session_long") + assert configured.headers[_SESSION_HEADER] == _CONFIGURED_SESSION, dict(configured.headers) + assert long.headers[_SESSION_HEADER] == _LONG_SESSION + _assert_signed(configured) + _assert_signed(long) + + +def test_a_deployment_api_key_sends_bearer_and_the_session_without_sigv4(agentcore: Gateway, runtime: _Runtime) -> None: + runtime.reset() + _assert_answered(_decide(agentcore, _BEARER), _BEARER) + (call,) = runtime.calls() + _assert_runtime_call(call, "audit_bearer") + assert call.headers["authorization"] == f"Bearer {_JWT}", dict(call.headers) + assert call.headers[_SESSION_HEADER] == _default_session("audit_bearer"), dict(call.headers) + assert "x-amz-date" not in call.headers, dict(call.headers) + + +def test_reserved_extra_headers_are_dropped_and_the_rest_are_kept(agentcore: Gateway, runtime: _Runtime) -> None: + runtime.reset() + _assert_answered(_decide(agentcore, _HEADERS), _HEADERS) + caller_headers: Final[dict[str, JsonValue]] = { + "x-amz-security-token": "caller-forged-token", + "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id": "caller-forged-session-0000000000000000001", + "x-audit-caller": "caller-trace", + } + _assert_answered(_decide(agentcore, _HEADERS, extra_headers=caller_headers), _HEADERS) + deployment, caller = runtime.calls() + for call in (deployment, caller): + _assert_runtime_call(call, "audit_headers") + assert call.headers[_SESSION_HEADER] == _default_session("audit_headers"), dict(call.headers) + assert "x-amz-target" not in call.headers, dict(call.headers) + assert "x-amz-security-token" not in call.headers, dict(call.headers) + _assert_signed(call) + assert deployment.headers["x-audit-trace"] == "deployment-trace", dict(deployment.headers) + assert caller.headers["x-audit-caller"] == "caller-trace", dict(caller.headers) + + +def test_an_opted_in_caller_session_id_becomes_the_header_and_stays_out_of_the_body( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + chosen: Final = f"agentcore-audit-caller-session-{uuid.uuid4().hex}" + _assert_answered(_decide(agentcore, _OPT_IN, agentcore_runtime_session_id=chosen), _OPT_IN) + _assert_answered(_decide(agentcore, _OPT_IN), _OPT_IN) + first, second = runtime.calls() + for call in (first, second): + _assert_runtime_call(call, "audit_opt_in") + _assert_signed(call) + assert first.headers[_SESSION_HEADER] == chosen, dict(first.headers) + assert second.headers[_SESSION_HEADER] == _default_session("audit_opt_in"), dict(second.headers) + + +def test_an_arn_deployment_without_api_base_takes_the_runtime_from_the_environment( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + response: Final = _decide(agentcore, _ENV) + _assert_answered(response, _ENV) + (call,) = runtime.calls() + _assert_runtime_call(call, "audit_env") + _assert_signed(call) + assert _spend_row(response.headers["x-litellm-call-id"])["api_base"] == _invocations_url("audit_env") + + +def test_an_arn_deployment_added_through_the_management_api_answers_signed( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + with agentcore.scenario() as scenario: + model: Final = scenario.model( + model_info=None, + model=_MODEL, + api_base=_arn("audit_stored"), + api_key=None, + aws_access_key_id=_ACCESS_KEY, + aws_secret_access_key=_SECRET_KEY, + ) + _assert_answered(_decide(agentcore, model), model) + (call,) = runtime.calls() + _assert_runtime_call(call, "audit_stored") + _assert_signed(call) + + +def test_health_and_test_connection_probe_the_runtime_signed(agentcore: Gateway, runtime: _Runtime) -> None: + runtime.reset() + health: Final = agentcore.request("GET", "/health", params={"model": _HEALTH}) + assert health.status_code == 200, health.text + assert (health.json()["healthy_count"], health.json()["unhealthy_count"]) == (1, 0), health.text + (probe,) = runtime.calls() + assert probe.target == f"/runtimes/{quote(_arn('audit_health'), safe='')}/invocations", probe.target + _assert_signed(probe) + connection: Final = agentcore.request( + "POST", + "/health/test_connection", + { + "litellm_params": {"model": _MODEL, "api_base": _arn("audit_test_connection"), **_SIGNING}, + "mode": "evaluation", + }, + ) + assert connection.status_code == 200, connection.text + assert connection.json()["status"] == "success", connection.text + (tested,) = runtime.calls() + assert _runtime_of(tested) == "audit_test_connection", tested.target + _assert_signed(tested) + + +@pytest.mark.parametrize(("code", "status"), tuple(_ERROR_STATUS.items()), ids=tuple(_ERROR_STATUS)) +def test_a_runtime_error_reply_maps_to_its_status_with_the_runtime_message( + agentcore: Gateway, runtime: _Runtime, code: str, status: int +) -> None: + runtime.reset() + model: Final = f"agentcore-audit-error-{code}" + response: Final = _decide(agentcore, model) + assert response.status_code == status, response.text + assert f"Strands Decider runtime error '{code}': runtime refused with {code}" in response.text, response.text + assert len(_calls_to(runtime.calls(), f"audit_error_{code}")) == 1 + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + assert (row["status"], row["model_group"]) == ("failure", model), row + + +@pytest.mark.parametrize("status", tuple(_HTTP_ERRORS), ids=tuple(str(status) for status in _HTTP_ERRORS)) +def test_a_runtime_http_error_keeps_its_status_and_message(agentcore: Gateway, runtime: _Runtime, status: int) -> None: + runtime.reset() + model: Final = f"agentcore-audit-status-{status}" + response: Final = _decide(agentcore, model) + assert response.status_code == status, response.text + assert f"scripted {_HTTP_ERRORS[status]}" in response.text, response.text + assert len(_calls_to(runtime.calls(), f"audit_status_{status}")) == 1 + assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" + + +@pytest.mark.parametrize("label", tuple(_MALFORMED_SESSIONS)) +def test_a_malformed_configured_session_id_fails_only_its_own_deployment( + agentcore: Gateway, runtime: _Runtime, label: str +) -> None: + runtime.reset() + model: Final = f"agentcore-audit-session-{label}" + response: Final = _decide(agentcore, model) + calls: Final = _calls_to(runtime.calls(), f"audit_session_{label}") + if label == "empty": + _assert_answered(response, model) + (call,) = calls + assert call.headers[_SESSION_HEADER] == "", dict(call.headers) + else: + assert response.status_code == 500, response.text + assert "agentcore_runtime_session_id" in response.text, response.text + assert calls == () + _assert_answered(_decide(agentcore, _SIGNED), _SIGNED) + + +def test_a_deployment_without_aws_credentials_fails_before_dialing_the_runtime( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + response: Final = _decide(agentcore, _UNSIGNED) + assert response.status_code == 500, response.text + assert "credentials" in response.text.lower(), response.text + assert runtime.calls() == () + assert _connect_targets(runtime) == frozenset() + _assert_answered(_decide(agentcore, _SIGNED), _SIGNED) + + +@pytest.mark.parametrize("label", tuple(_NEAR_MISS_ARNS)) +def test_a_near_miss_arn_keeps_the_plain_api_base_path(agentcore: Gateway, runtime: _Runtime, label: str) -> None: + runtime.reset() + response: Final = _decide(agentcore, f"agentcore-audit-near-miss-{label}") + assert 500 <= response.status_code < 600, response.text + assert "litellm.APIConnectionError" in response.text, response.text + assert _connect_targets(runtime) == frozenset() + assert runtime.calls() == () + + +def test_a_dropped_connection_is_a_connection_error_after_one_attempt(agentcore: Gateway, runtime: _Runtime) -> None: + runtime.reset() + response: Final = _decide(agentcore, _DROPPED) + assert response.status_code == 500, response.text + assert "litellm.APIConnectionError" in response.text, response.text + assert len(_calls_to(runtime.calls(), "audit_dropped")) == 1 + assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" + + +def _burst_routes(size: int) -> tuple[_Route, ...]: + return tuple(_ROUTES[index % len(_ROUTES)] for index in range(size)) + + +def _burst(gateway: Gateway, model: str, size: int) -> tuple[httpx.Response, ...]: + with ThreadPoolExecutor(max_workers=size) as pool: + return tuple(pool.map(partial(_decide, gateway, model), _burst_routes(size))) + + +def _health_counts(gateway: Gateway, model: str) -> tuple[int, int, int]: + response: Final = gateway.request("GET", "/health", params={"model": model}) + body: Final = JSON_OBJECT.validate_json(response.content) + healthy: Final = body["healthy_count"] + unhealthy: Final = body["unhealthy_count"] + assert isinstance(healthy, int) and isinstance(unhealthy, int), body + return response.status_code, healthy, unhealthy + + +def test_a_runtime_outage_mid_burst_fails_each_call_once_and_recovers_signed( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + runtime.outage.set() + failed: Final = _burst(agentcore, _OUTAGE, 10) + assert [response.status_code for response in failed] == [500] * 10, [response.text for response in failed] + assert _health_counts(agentcore, _OUTAGE) == (503, 0, 1) + runtime.outage.clear() + runtime.calls() + served: Final = _burst(agentcore, _OUTAGE, 20) + for response, route in zip(served, _burst_routes(20), strict=True): + _assert_answered(response, _OUTAGE, route) + calls: Final = runtime.calls() + assert len(calls) == 20, calls + for call in calls: + _assert_runtime_target(call, "audit_outage") + _assert_signed(call) + assert sorted(_canonical(_JSON.validate_json(call.body)) for call in calls) == sorted( + _canonical(route.upstream) for route in _burst_routes(20) + ) + assert _health_counts(agentcore, _OUTAGE) == (200, 1, 0) + expected: Final = { + **{response.headers["x-litellm-call-id"]: "failure" for response in failed}, + **{response.headers["x-litellm-call-id"]: "success" for response in served}, + } + assert len(expected) == 30, expected + rows: Final = _spend_rows(tuple(expected)) + assert sorted((str(row["request_id"]), row["status"]) for row in rows) == sorted(expected.items()), rows + + +async def test_sdk_sync_and_async_calls_reach_the_runtime_signed( + runtime: _Runtime, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + runtime.reset() + for name in _scrubbed(): + monkeypatch.delenv(name) + for name, value in { + "HTTPS_PROXY": runtime.tunnel.url, + "NO_PROXY": "127.0.0.1,localhost", + "SSL_CERT_FILE": str(runtime.cert), + **_isolated_aws(tmp_path), + }.items(): + monkeypatch.setenv(name, value) + litellm.in_memory_llm_clients_cache.flush_cache() + try: + synchronous: Final = litellm.decisions( + model=_MODEL, + state=_STATE, + questions=_SDK_QUESTIONS, + api_base=_arn("audit_sdk"), + aws_access_key_id=_ACCESS_KEY, + aws_secret_access_key=_SECRET_KEY, + ) + asynchronous: Final = await litellm.adecisions( + model=_MODEL, + state=_STATE, + questions=_SDK_QUESTIONS, + api_base=_arn("audit_sdk"), + aws_access_key_id=_ACCESS_KEY, + aws_secret_access_key=_SECRET_KEY, + ) + monkeypatch.setenv("STRANDS_DECIDER_API_KEY", _ENV_JWT) + bearer: Final = await litellm.adecisions( + model=_MODEL, state=_STATE, questions=_SDK_QUESTIONS, api_base=_arn("audit_sdk_env_key") + ) + finally: + litellm.in_memory_llm_clients_cache.flush_cache() + for response in (synchronous, asynchronous, bearer): + assert response.model_dump(mode="json") == {"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE} + calls: Final = runtime.calls() + signed: Final = _calls_to(calls, "audit_sdk") + assert len(signed) == 2, calls + for call in signed: + _assert_runtime_call(call, "audit_sdk") + assert call.headers[_SESSION_HEADER] == _default_session("audit_sdk"), dict(call.headers) + _assert_signed(call) + (env_key,) = _calls_to(calls, "audit_sdk_env_key") + assert env_key.headers["authorization"] == f"Bearer {_ENV_JWT}", dict(env_key.headers) + assert "x-amz-date" not in env_key.headers, dict(env_key.headers) + + +def _answering(request: Request) -> Reply: + return _json_reply({"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE}) + + +def _envelope(request: Request) -> Reply: + return _json_reply({"error": {"code": "bad_request", "message": "plain server refused"}}) + + +def _plain_model(scenario: Scenario, wire: Wire, **extra: JsonValue) -> str: + return scenario.model(model_info=None, model=_MODEL, api_base=wire.url, api_key=None, **extra) + + +def test_a_request_body_session_id_is_refused_on_every_route_without_an_upstream_call(gateway: Gateway) -> None: + with wire_server(_answering) as wire, gateway.scenario() as scenario: + model: Final = _plain_model(scenario, wire) + responses: Final = tuple( + _decide(gateway, model, route, agentcore_runtime_session_id=f"caller-{uuid.uuid4().hex}") + for route in _ROUTES + ) + for response in responses: + assert response.status_code == 401, response.text + assert "agentcore_runtime_session_id is not allowed in request body" in response.text, response.text + assert wire.drain() == () + + +def test_a_plain_api_base_opted_into_the_session_param_sends_no_session_header_or_body_field( + gateway: Gateway, +) -> None: + with wire_server(_answering) as wire, gateway.scenario() as scenario: + model: Final = _plain_model( + scenario, wire, configurable_clientside_auth_params=["agentcore_runtime_session_id"] + ) + _assert_answered(_decide(gateway, model, agentcore_runtime_session_id=f"caller-{uuid.uuid4().hex}"), model) + (call,) = wire.drain() + assert call.target == "/v1/systemone", call.target + assert _SESSION_HEADER not in call.headers, dict(call.headers) + assert json.loads(call.body) == _UPSTREAM_BODY, call.body + + +def test_a_plain_api_base_error_envelope_keeps_the_unexpected_response_server_error(gateway: Gateway) -> None: + with wire_server(_envelope) as wire, gateway.scenario() as scenario: + model: Final = _plain_model(scenario, wire) + response: Final = _decide(gateway, model) + assert response.status_code == 500, response.text + assert "Strands Decider runtime error" not in response.text, response.text + assert len(wire.drain()) == 1 + + +def test_a_plain_api_base_forwards_amazon_shaped_extra_headers_unchanged(gateway: Gateway) -> None: + headers: Final[dict[str, JsonValue]] = { + "x-amz-target": "plain-target", + "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id": "plain-session-kept", + "x-audit-trace": "plain-trace", + } + with wire_server(_answering) as wire, gateway.scenario() as scenario: + model: Final = _plain_model(scenario, wire, extra_headers=headers) + _assert_answered(_decide(gateway, model), model) + (call,) = wire.drain() + assert (call.headers["x-amz-target"], call.headers[_SESSION_HEADER], call.headers["x-audit-trace"]) == ( + "plain-target", + "plain-session-kept", + "plain-trace", + ), dict(call.headers) + assert "authorization" not in call.headers, dict(call.headers) + assert json.loads(call.body) == _UPSTREAM_BODY, call.body + + +def _worst_case_api_base() -> str: + return "arn:aws" + "-a" * 131072 + "!" + + +def test_a_worst_case_runtime_arn_pattern_input_is_classified_in_linear_time( + gateway: Gateway, record_property: Callable[[str, object], None] +) -> None: + worst: Final = _worst_case_api_base() + started: Final = time.perf_counter() + url: Final = StrandsDeciderDecisionsConfig().get_complete_url(worst, _BODY_MODEL) + classified: Final = time.perf_counter() - started + record_property("classifier_seconds", f"{classified:.6f}") + assert url == f"{worst}/v1/systemone" + assert classified < 0.5, classified + with wire_server(_answering) as wire, gateway.scenario() as scenario: + model: Final = _plain_model(scenario, wire, configurable_clientside_auth_params=["api_base"]) + with ThreadPoolExecutor(max_workers=2) as pool: + timed: Final = pool.submit(_timed_decide, gateway, model, worst) + liveliness_started: Final = time.perf_counter() + liveliness: Final = gateway.request("GET", "/health/liveliness") + liveliness_seconds: Final = time.perf_counter() - liveliness_started + response, decide_seconds = timed.result() + record_property("decide_seconds", f"{decide_seconds:.6f}") + record_property("liveliness_seconds", f"{liveliness_seconds:.6f}") + assert liveliness.status_code == 200, liveliness.text + assert 500 <= response.status_code < 600, response.text + assert "litellm.APIConnectionError" in response.text, response.text[:2000] + assert decide_seconds < 5, decide_seconds + assert liveliness_seconds < 2, liveliness_seconds + assert wire.drain() == () + + +def _timed_decide(gateway: Gateway, model: str, api_base: str) -> tuple[httpx.Response, float]: + started: Final = time.perf_counter() + response: Final = _decide(gateway, model, api_base=api_base) + return response, time.perf_counter() - started diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 9afea66835b..207228c4b53 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -2,6 +2,8 @@ from __future__ import annotations import asyncio import json +import threading +from concurrent.futures import wait from collections.abc import Mapping from types import MappingProxyType from typing import Final @@ -9,11 +11,16 @@ from typing import Final import httpx import pytest import respx +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials import litellm from litellm.cost_calculator import get_response_cost_from_hidden_params +from litellm.constants import AWS_SIGNING_MAX_THREADS from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR from litellm.types.decisions import ( ChoiceAnswer, DecisionsResponse, @@ -682,6 +689,33 @@ async def test_strands_decider_without_key_preserves_response_extras( assert severity.legend == {"0": "none", "1": "low", "2": "high"} +@pytest.mark.asyncio +async def test_strands_decider_plain_api_base_answers_while_every_aws_signing_thread_is_busy( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + release: Final = threading.Event() + busy: Final = tuple(AWS_SIGNING_EXECUTOR.submit(release.wait) for _ in range(AWS_SIGNING_MAX_THREADS)) + + try: + response: Final = await asyncio.wait_for( + litellm.adecisions( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base="https://strands.example", + ), + timeout=5, + ) + finally: + release.set() + wait(busy) + + assert isinstance(response.answers["severity"], ScoreAnswer) + + @pytest.mark.asyncio async def test_strands_decider_uses_key_from_matching_environment_base( monkeypatch: pytest.MonkeyPatch, @@ -734,6 +768,172 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( assert response.model == _STRANDS_RESPONSE["model"] +_RUNTIME_ARN: Final = "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/qa_decider-AbC123xyz0" +_RUNTIME_URL: Final = ( + "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/" + "arn%3Aaws%3Abedrock-agentcore%3Aus-east-1%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0/invocations" +) +_RUNTIME_CREDENTIALS: Final = Credentials(access_key="AKIDEXAMPLE", secret_key="example-secret") +_SESSION_HEADER: Final = "x-amzn-bedrock-agentcore-runtime-session-id" + + +def _sigv4_verifies(request: httpx.Request) -> bool: + authorization: Final = request.headers["authorization"] + signed_names: Final = authorization.split("SignedHeaders=")[1].split(",")[0].split(";") + replayed: Final = AWSRequest( + method=request.method, + url=str(request.url), + data=request.content, + headers={name: request.headers[name] for name in signed_names}, + ) + replayed.context["timestamp"] = request.headers["x-amz-date"] + auth: Final = SigV4Auth(_RUNTIME_CREDENTIALS, "bedrock-agentcore", "us-east-1") + string_to_sign: Final = auth.string_to_sign(replayed, auth.canonical_request(replayed)) + return authorization.endswith(f"Signature={auth.signature(string_to_sign, replayed)}") + + +@pytest.mark.parametrize("use_async", [True, False]) +@pytest.mark.asyncio +async def test_strands_decider_runtime_arn_sends_a_sigv4_signed_invoke_agent_runtime_call( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + use_async: bool, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + route: Final = respx_mock.post(_RUNTIME_URL).respond(json=_STRANDS_RESPONSE) + call_args: Final[Mapping[str, object]] = { + "model": "strands_decider/strands-decider-2B-hobson-v19", + "state": "review", + "questions": {"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + "api_base": _RUNTIME_ARN, + "aws_access_key_id": _RUNTIME_CREDENTIALS.access_key, + "aws_secret_access_key": _RUNTIME_CREDENTIALS.secret_key, + } + + response: Final = await litellm.adecisions(**call_args) if use_async else litellm.decisions(**call_args) + + assert route.call_count == 1 + sent: Final = route.calls[0].request + assert _sigv4_verifies(sent) + assert json.loads(sent.content)["model"] == "strands-decider-2B-hobson-v19" + assert sent.headers[_SESSION_HEADER].startswith("litellm-decider-") + assert response.model == _STRANDS_RESPONSE["model"] + + +@pytest.mark.asyncio +async def test_strands_decider_runtime_arn_router_deployment_answers_openai_format_calls( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "strands", + "litellm_params": { + "model": "strands_decider/strands-decider-2B-hobson-v19", + "api_base": _RUNTIME_ARN, + "aws_access_key_id": _RUNTIME_CREDENTIALS.access_key, + "aws_secret_access_key": _RUNTIME_CREDENTIALS.secret_key, + "aws_region_name": "eu-west-1", + "agentcore_runtime_session_id": "pooled-session-0123456789abcdef0123456", + }, + } + ] + ) + route: Final = respx_mock.post(_RUNTIME_URL).respond(json=_STRANDS_RESPONSE) + + response: Final = await router.adecisions( + model="strands", + input="review", + extra_headers={_SESSION_HEADER: "caller-session-0123456789abcdef01234567"}, + questions=[ + { + "type": "score", + "name": "severity", + "instructions": "How bad?", + "levels": [{"label": "none"}, {"label": "low"}, {"label": "high"}], + } + ], + ) + + sent: Final = route.calls[0].request + assert _sigv4_verifies(sent) + assert sent.headers[_SESSION_HEADER] == "pooled-session-0123456789abcdef0123456" + assert isinstance(response, OpenAIDecisionResponse) + + +@pytest.mark.asyncio +async def test_strands_decider_runtime_arn_with_api_key_sends_the_bearer_token_unsigned( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + route: Final = respx_mock.post(_RUNTIME_URL).respond(json=_STRANDS_RESPONSE) + + await litellm.adecisions( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base=_RUNTIME_ARN, + api_key="runtime-jwt", + ) + + sent: Final = route.calls[0].request + assert sent.headers["authorization"] == "Bearer runtime-jwt" + assert "x-amz-date" not in sent.headers + assert _SESSION_HEADER in sent.headers + + +@pytest.mark.parametrize( + ("code", "error_class"), + [ + ("bad_request", litellm.BadRequestError), + ("loading", litellm.ServiceUnavailableError), + ("inference_error", litellm.InternalServerError), + ], +) +@pytest.mark.asyncio +async def test_strands_decider_runtime_error_envelope_raises_instead_of_answering( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + code: str, + error_class: type[Exception], +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + respx_mock.post(_RUNTIME_URL).respond(json={"error": {"code": code, "message": "at most 16 questions"}}) + + with pytest.raises(error_class, match="at most 16 questions"): + await litellm.adecisions( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base=_RUNTIME_ARN, + aws_access_key_id=_RUNTIME_CREDENTIALS.access_key, + aws_secret_access_key=_RUNTIME_CREDENTIALS.secret_key, + ) + + +@pytest.mark.asyncio +async def test_strands_decider_plain_api_base_error_envelope_stays_an_unexpected_response( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + respx_mock.post("https://strands.example/v1/systemone").respond( + json={"error": {"code": "bad_request", "message": "at most 16 questions"}} + ) + + with pytest.raises(litellm.InternalServerError, match="returned an unexpected response"): + await litellm.adecisions( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base="https://strands.example", + ) + + _VLLM_RESPONSE: Final[Mapping[str, object]] = { "id": "systemone-4af3d2c1", "object": "structured_decision", diff --git a/tests/unit/llms/strands_decider/__init__.py b/tests/unit/llms/strands_decider/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/strands_decider/decisions/__init__.py b/tests/unit/llms/strands_decider/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/strands_decider/decisions/test_strands_decider_decisions_transformation.py b/tests/unit/llms/strands_decider/decisions/test_strands_decider_decisions_transformation.py new file mode 100644 index 00000000000..4d249e59233 --- /dev/null +++ b/tests/unit/llms/strands_decider/decisions/test_strands_decider_decisions_transformation.py @@ -0,0 +1,256 @@ +import json +from collections.abc import Mapping +from typing import Final + +import httpx +import pytest +from botocore.credentials import Credentials +from pydantic import TypeAdapter + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.decisions.transformation import systemone_request_to_ir +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.strands_decider.decisions.transformation import ( + AGENTCORE_SESSION_HEADER, + StrandsDeciderDecisionsConfig, +) +from litellm.types.decisions import DecisionsIRResponse, DecisionsRequestBody +from litellm.types.llms.bedrock import AwsAuthParams + +_ARN: Final = "arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/qa_decider-AbC123xyz0" +_BODY: Final[Mapping[str, object]] = { + "model": "strands-decider-2B-hobson-v21", + "state": "My order arrived broken.", + "questions": {"is_complaint": {"type": "noul", "instructions": "Is this a complaint?"}}, +} +_IR_REQUEST: Final = systemone_request_to_ir( + TypeAdapter(DecisionsRequestBody).validate_python({"state": _BODY["state"], "questions": _BODY["questions"]}) +) + + +class _RecordingAWS(BaseAWSLLM): + def __init__(self) -> None: + super().__init__() + self.resolved: tuple[tuple[AwsAuthParams, str | None], ...] = () + + def resolve_credentials(self, auth_params: AwsAuthParams, aws_region_name: str | None) -> Credentials: + self.resolved = (*self.resolved, (auth_params, aws_region_name)) + return Credentials(access_key="AKIDEXAMPLE", secret_key="example-secret") + + +def _sign( + config: StrandsDeciderDecisionsConfig, + api_base: str, + headers: Mapping[str, str], + api_key: str | None = None, + litellm_params: Mapping[str, object] | None = None, +) -> tuple[Mapping[str, str], bytes | None]: + return config.sign_request( + headers=headers, + url=config.get_complete_url(api_base=api_base, model="strands-decider-2B-hobson-v21"), + api_base=api_base, + body=_BODY, + api_key=api_key, + litellm_params=litellm_params or {}, + ) + + +@pytest.mark.parametrize( + ("arn", "expected_url"), + [ + ( + _ARN, + "https://bedrock-agentcore.us-west-2.amazonaws.com/runtimes/" + "arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0/invocations", + ), + ( + "arn:aws-cn:bedrock-agentcore:cn-north-1:123456789012:runtime/qa_decider-AbC123xyz0", + "https://bedrock-agentcore.cn-north-1.amazonaws.com.cn/runtimes/" + "arn%3Aaws-cn%3Abedrock-agentcore%3Acn-north-1%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0" + "/invocations", + ), + ( + "arn:aws-us-gov:bedrock-agentcore:us-gov-west-1:123456789012:runtime/qa_decider-AbC123xyz0", + "https://bedrock-agentcore.us-gov-west-1.amazonaws.com/runtimes/" + "arn%3Aaws-us-gov%3Abedrock-agentcore%3Aus-gov-west-1%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0" + "/invocations", + ), + ], +) +def test_runtime_arn_targets_the_regional_invoke_agent_runtime_url(arn: str, expected_url: str) -> None: + assert StrandsDeciderDecisionsConfig().get_complete_url(api_base=arn, model="any") == expected_url + + +@pytest.mark.parametrize( + "api_base", + [ + "https://strands.example", + "arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/qa_decider-AbC123xyz0/runtime-endpoint/DEFAULT", + "arn:aws:bedrock-agentcore:evil.example/x:123456789012:runtime/qa_decider-AbC123xyz0", + "arn:aws:bedrock:us-west-2:123456789012:runtime/qa_decider-AbC123xyz0", + ], +) +def test_other_api_bases_keep_the_plain_systemone_request(api_base: str) -> None: + aws: Final = _RecordingAWS() + config: Final = StrandsDeciderDecisionsConfig(aws=aws) + headers: Final = {"Content-Type": "application/json"} + + assert config.get_complete_url(api_base=api_base, model="any") == f"{api_base}/v1/systemone" + assert _sign(config, api_base, headers) == (headers, None) + assert aws.resolved == () + + +def test_sign_request_signs_the_body_for_the_arn_region_with_deployment_credentials() -> None: + aws: Final = _RecordingAWS() + config: Final = StrandsDeciderDecisionsConfig(aws=aws) + + signed_headers, signed_body = _sign( + config, + _ARN, + {"Content-Type": "application/json"}, + litellm_params={"aws_profile_name": "qa", "aws_role_name": "decider-role", "aws_region_name": "us-east-1"}, + ) + + assert signed_body == json.dumps(_BODY).encode() + assert len(aws.resolved) == 1 + auth_params, region = aws.resolved[0] + assert (auth_params.aws_profile_name, auth_params.aws_role_name, region) == ("qa", "decider-role", "us-west-2") + authorization: Final = signed_headers["Authorization"] + assert authorization.startswith("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/") + assert "/us-west-2/bedrock-agentcore/aws4_request" in authorization + assert AGENTCORE_SESSION_HEADER.lower() in authorization + + +def test_default_session_id_is_stable_per_runtime_and_within_the_agentcore_length_bounds() -> None: + other_arn: Final = _ARN.replace("AbC123xyz0", "ZyX987cba1") + + first, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), _ARN, {}) + again, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), _ARN, {}) + other, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), other_arn, {}) + + session_id: Final = first[AGENTCORE_SESSION_HEADER] + assert session_id == again[AGENTCORE_SESSION_HEADER] + assert session_id != other[AGENTCORE_SESSION_HEADER] + assert 33 <= len(session_id) <= 256 + + +_CALLER_RESERVED_HEADERS: Final = { + AGENTCORE_SESSION_HEADER.lower(): "caller-session-0123456789abcdef0123456789", + "X-Amzn-Bedrock-AgentCore-Runtime-User-Id": "someone-else", + "X-Amz-Security-Token": "caller-token", + "Host": "evil.example", +} + + +@pytest.mark.parametrize("api_key", [None, "runtime-jwt"]) +def test_caller_reserved_agentcore_headers_never_reach_the_runtime(api_key: str | None) -> None: + signed_headers, _ = _sign( + StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), + _ARN, + {**_CALLER_RESERVED_HEADERS, "X-Request-Source": "batch"}, + api_key=api_key, + ) + default_headers, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), _ARN, {}, api_key=api_key) + + forwarded: Final = {name.lower(): value for name, value in signed_headers.items()} + assert forwarded[AGENTCORE_SESSION_HEADER.lower()] == default_headers[AGENTCORE_SESSION_HEADER] + assert not any(value in forwarded.values() for value in _CALLER_RESERVED_HEADERS.values()) + assert "x-amzn-bedrock-agentcore-runtime-user-id" not in forwarded + assert "host" not in forwarded + assert forwarded["x-request-source"] == "batch" + + +def test_deployment_session_param_picks_the_session_over_the_default_and_any_caller_header() -> None: + replica_session: Final = "strands-decider-replica-a-0000000000" + + signed_headers, _ = _sign( + StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), + _ARN, + _CALLER_RESERVED_HEADERS, + litellm_params={"agentcore_runtime_session_id": replica_session}, + ) + + session_headers: Final = [ + value for name, value in signed_headers.items() if name.lower() == "x-amzn-bedrock-agentcore-runtime-session-id" + ] + assert session_headers == [replica_session] + assert AGENTCORE_SESSION_HEADER.lower() in signed_headers["Authorization"] + + +def test_api_key_on_a_runtime_arn_sends_the_bearer_token_without_sigv4() -> None: + aws: Final = _RecordingAWS() + + signed_headers, signed_body = _sign( + StrandsDeciderDecisionsConfig(aws=aws), _ARN, {"Authorization": "Bearer runtime-jwt"}, api_key="runtime-jwt" + ) + + assert signed_body is None + assert signed_headers["Authorization"] == "Bearer runtime-jwt" + assert AGENTCORE_SESSION_HEADER in signed_headers + assert aws.resolved == () + + +def _response(url: str, payload: object) -> httpx.Response: + return httpx.Response(200, json=payload, request=httpx.Request("POST", url)) + + +def _transform(api_base: str, payload: object) -> DecisionsIRResponse: + config: Final = StrandsDeciderDecisionsConfig() + url: Final = config.get_complete_url(api_base=api_base, model="strands-decider-2B-hobson-v21") + return config.transform_decisions_response( + model="strands-decider-2B-hobson-v21", + custom_llm_provider="strands_decider", + raw_response=_response(url, payload), + request=_IR_REQUEST, + ) + + +@pytest.mark.parametrize( + ("code", "status_code"), + [("bad_request", 400), ("invalid_request", 400), ("loading", 503), ("inference_error", 500), ("new_code", 500)], +) +def test_runtime_error_envelope_raises_with_the_mapped_status(code: str, status_code: int) -> None: + with pytest.raises(BaseLLMException) as raised: + _transform(_ARN, {"error": {"code": code, "message": "at most 16 questions"}}) + + assert raised.value.status_code == status_code + assert code in raised.value.message + assert "at most 16 questions" in raised.value.message + + +@pytest.mark.parametrize("code", ["bad_request", "loading", "inference_error"]) +def test_plain_api_base_error_envelope_keeps_the_unexpected_response_error(code: str) -> None: + with pytest.raises(BaseLLMException) as raised: + _transform("https://strands.example", {"error": {"code": code, "message": "at most 16 questions"}}) + + assert raised.value.status_code == 500 + assert raised.value.message.startswith("Decisions provider 'strands_decider' returned an unexpected response") + + +@pytest.mark.parametrize( + "url", + [ + "https://strands.example/runtimes/arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A123456789012%3Aruntime" + "%2Fqa_decider-AbC123xyz0/invocations", + "https://bedrock-agentcore.us-west-2.amazonaws.com/runtimes/not-an-arn/invocations", + ], +) +def test_error_envelope_from_a_url_that_is_not_the_runtime_keeps_the_unexpected_response_error(url: str) -> None: + with pytest.raises(BaseLLMException) as raised: + StrandsDeciderDecisionsConfig().transform_decisions_response( + model="strands-decider-2B-hobson-v21", + custom_llm_provider="strands_decider", + raw_response=_response(url, {"error": {"code": "bad_request", "message": "at most 16 questions"}}), + request=_IR_REQUEST, + ) + + assert raised.value.status_code == 500 + + +@pytest.mark.parametrize("payload", [["not", "an", "object"], {"error": "unstructured"}]) +def test_runtime_reply_that_is_not_an_error_envelope_keeps_the_unexpected_response_error(payload: object) -> None: + with pytest.raises(BaseLLMException) as raised: + _transform(_ARN, payload) + + assert raised.value.status_code == 500 + assert raised.value.message.startswith("Decisions provider 'strands_decider' returned an unexpected response") diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 83ac56c4c85..0b65007c153 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -2727,6 +2727,22 @@ class TestIsRequestBodySafeBlocksBedrockProjectOverride: ) +def test_agentcore_runtime_session_id_in_request_body_is_rejected_without_admin_opt_in(): + body: Final = {"model": "strands-decider", "agentcore_runtime_session_id": "caller-session-0123456789abcdef0123"} + + with pytest.raises(ValueError, match="agentcore_runtime_session_id"): + is_request_body_safe(request_body=body, general_settings={}, llm_router=None, model="strands-decider") + assert ( + is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="strands-decider", + ) + is True + ) + + class TestIsRequestBodySafeBlocksClaudePlatformWorkspaceOverride: @pytest.mark.parametrize( "alias", ["workspace_id", "aws_workspace_id", "anthropic_workspace_id", "anthropic-workspace-id"] From 8f262b2089c5bdd4f4e59af4b921c36af10b9135 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 21:04:59 +0000 Subject: [PATCH 14/18] docs(proxy): describe /v1/decisions and /v1/systemone in Swagger UI (#45838) * docs(proxy): describe /v1/decisions and /v1/systemone in Swagger UI Add summaries, descriptions, inlined request body schemas with examples and typed 200 responses to the decision routes and their aliases, so the OpenAPI document and the lazy snapshot show both request formats Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): cap _inlined schema walk depth for the recursion check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit bc2cc42340a2f363c68011250dd545268d90dfdd) --- litellm/proxy/_lazy_openapi_snapshot.json | 1805 ++++++++++++++++- .../proxy/common_utils/custom_openapi_spec.py | 38 + .../proxy/decisions_endpoints/endpoints.py | 89 +- litellm/types/decisions.py | 86 +- .../code_coverage_tests/recursive_detector.py | 1 + .../common_utils/test_custom_openapi_spec.py | 92 +- .../decisions_endpoints/test_endpoints.py | 64 +- 7 files changed, 2107 insertions(+), 68 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index ca960a317ae..2c555d4bb01 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -9494,17 +9494,878 @@ }, "decisions": { "components": { - "schemas": {} + "schemas": { + "ChoiceAnswer": { + "additionalProperties": true, + "properties": { + "choice": { + "title": "Choice", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "confidence", + "probabilities" + ], + "title": "ChoiceAnswer", + "type": "object" + }, + "DecisionsResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "additionalProperties": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/ChoiceAnswer", + "noul": "#/components/schemas/NoulAnswer", + "score": "#/components/schemas/ScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/NoulAnswer" + }, + { + "$ref": "#/components/schemas/ChoiceAnswer" + }, + { + "$ref": "#/components/schemas/ScoreAnswer" + } + ] + }, + "title": "Answers", + "type": "object" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/components/schemas/DecisionsUsage" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "answers" + ], + "title": "DecisionsResponse", + "type": "object" + }, + "DecisionsUsage": { + "additionalProperties": true, + "properties": { + "input_tokens": { + "default": 0, + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "default": 0, + "title": "Output Tokens", + "type": "integer" + } + }, + "title": "DecisionsUsage", + "type": "object" + }, + "NoulAnswer": { + "additionalProperties": true, + "properties": { + "noul": { + "title": "Noul", + "type": "number" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "title": "NoulAnswer", + "type": "object" + }, + "OpenAIChoiceAnswer": { + "properties": { + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Choice" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probabilities": { + "items": { + "$ref": "#/components/schemas/OpenAIChoiceProbability" + }, + "title": "Probabilities", + "type": "array" + }, + "type": { + "const": "choice", + "default": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "choice", + "probabilities", + "confidence" + ], + "title": "OpenAIChoiceAnswer", + "type": "object" + }, + "OpenAIChoiceProbability": { + "properties": { + "probability": { + "title": "Probability", + "type": "number" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value", + "probability" + ], + "title": "OpenAIChoiceProbability", + "type": "object" + }, + "OpenAIDecisionInputTokensDetails": { + "properties": { + "cache_write_tokens": { + "default": 0, + "title": "Cache Write Tokens", + "type": "integer" + }, + "cached_tokens": { + "default": 0, + "title": "Cached Tokens", + "type": "integer" + } + }, + "title": "OpenAIDecisionInputTokensDetails", + "type": "object" + }, + "OpenAIDecisionOutputTokensDetails": { + "properties": { + "reasoning_tokens": { + "default": 0, + "title": "Reasoning Tokens", + "type": "integer" + } + }, + "title": "OpenAIDecisionOutputTokensDetails", + "type": "object" + }, + "OpenAIDecisionResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "items": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/OpenAIChoiceAnswer", + "predicate": "#/components/schemas/OpenAIPredicateAnswer", + "refusal": "#/components/schemas/OpenAIRefusalAnswer", + "score": "#/components/schemas/OpenAIScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/OpenAIPredicateAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIChoiceAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIScoreAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIRefusalAnswer" + } + ] + }, + "title": "Answers", + "type": "array" + }, + "model": { + "title": "Model", + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/OpenAIDecisionUsage" + } + }, + "required": [ + "model", + "answers", + "usage" + ], + "title": "OpenAIDecisionResponse", + "type": "object" + }, + "OpenAIDecisionUsage": { + "properties": { + "input_tokens": { + "title": "Input Tokens", + "type": "integer" + }, + "input_tokens_details": { + "$ref": "#/components/schemas/OpenAIDecisionInputTokensDetails", + "default": { + "cache_write_tokens": 0, + "cached_tokens": 0 + } + }, + "output_tokens": { + "title": "Output Tokens", + "type": "integer" + }, + "output_tokens_details": { + "$ref": "#/components/schemas/OpenAIDecisionOutputTokensDetails", + "default": { + "reasoning_tokens": 0 + } + }, + "total_tokens": { + "title": "Total Tokens", + "type": "integer" + } + }, + "required": [ + "input_tokens", + "output_tokens", + "total_tokens" + ], + "title": "OpenAIDecisionUsage", + "type": "object" + }, + "OpenAIPredicateAnswer": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probability": { + "title": "Probability", + "type": "number" + }, + "type": { + "const": "predicate", + "default": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "probability" + ], + "title": "OpenAIPredicateAnswer", + "type": "object" + }, + "OpenAIRefusalAnswer": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "type": { + "const": "refusal", + "default": "refusal", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name" + ], + "title": "OpenAIRefusalAnswer", + "type": "object" + }, + "OpenAIScoreAnswer": { + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probabilities": { + "items": { + "$ref": "#/components/schemas/OpenAIScoreProbability" + }, + "title": "Probabilities", + "type": "array" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "default": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "score", + "probabilities", + "confidence" + ], + "title": "OpenAIScoreAnswer", + "type": "object" + }, + "OpenAIScoreProbability": { + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "probability": { + "title": "Probability", + "type": "number" + }, + "value": { + "title": "Value", + "type": "integer" + } + }, + "required": [ + "value", + "label", + "probability" + ], + "title": "OpenAIScoreProbability", + "type": "object" + }, + "ScoreAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "legend": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "title": "Legend", + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "score", + "confidence", + "legend", + "probabilities" + ], + "title": "ScoreAnswer", + "type": "object" + } + } }, "paths": { "/decisions": { "post": { + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_decisions_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "model": "luna", + "questions": [ + { + "instructions": "Is the customer asking for a refund?", + "name": "is_refund_request", + "type": "predicate" + }, + { + "choices": [ + { + "description": "can wait a week", + "value": "low" + }, + { + "description": "needs action today", + "value": "high" + } + ], + "instructions": "How urgent is this?", + "name": "urgency", + "type": "choice" + }, + { + "instructions": "How frustrated is the customer?", + "levels": [ + { + "label": "calm" + }, + { + "label": "mildly annoyed" + }, + { + "label": "angry" + } + ], + "name": "frustration", + "type": "score" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "input_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "OpenAIDecisionInputText", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "image_url": { + "title": "Image Url", + "type": "string" + }, + "type": { + "const": "input_image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "OpenAIDecisionInputImage", + "type": "object" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "default": "user", + "title": "Role", + "type": "string" + }, + "type": { + "const": "message", + "default": "message", + "title": "Type", + "type": "string" + } + }, + "required": [ + "content" + ], + "title": "OpenAIDecisionInputMessage", + "type": "object" + }, + "type": "array" + } + ], + "description": "The text being judged, or user messages whose content mixes input_text and input_image parts", + "title": "Input" + }, + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "description": "Predicate, choice and score questions. Answers come back in the same order", + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The yes/no question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIPredicateQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "choices": { + "description": "The options the model picks from. Values must be unique", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "OpenAIChoiceOption", + "type": "object" + }, + "maxItems": 255, + "minItems": 2, + "title": "Choices", + "type": "array" + }, + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "choices" + ], + "title": "OpenAIChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "levels": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label" + ], + "title": "OpenAIScoreLevel", + "type": "object" + }, + "maxItems": 10, + "minItems": 2, + "title": "Levels", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "levels" + ], + "title": "OpenAIScoreQuestion", + "type": "object" + } + ] + }, + "maxItems": 128, + "minItems": 1, + "title": "Questions", + "type": "array" + }, + "safety_identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", + "title": "Safety Identifier" + } + }, + "required": [ + "input", + "questions", + "model" + ], + "title": "OpenAIDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/OpenAIDecisionResponse" + } } }, "description": "Successful Response" @@ -9515,7 +10376,7 @@ "APIKeyHeader": [] } ], - "summary": "Decisions", + "summary": "Ask questions about an input (OpenAI Decisions format)", "tags": [ "decisions" ] @@ -9523,12 +10384,290 @@ }, "/systemone": { "post": { + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "jev", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "urgency": { + "criteria": { + "high": "needs action today", + "low": "can wait a week" + }, + "instructions": "How urgent is this?", + "type": "choice" + } + }, + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + }, + "schema": { + "additionalProperties": true, + "properties": { + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": true, + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what makes the answer true and what makes it false", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The yes/no question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "NoulQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "description": "Candidate labels mapped to an optional description of each", + "maxProperties": 255, + "minProperties": 1, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "maxItems": 10, + "minItems": 1, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ScoreQuestion", + "type": "object" + } + ] + }, + "description": "Named noul, choice and score questions. Each key becomes a key in the answers", + "maxProperties": 128, + "minProperties": 1, + "propertyNames": { + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The thing being judged, such as a support ticket, a document or a chat transcript", + "title": "State" + } + }, + "required": [ + "state", + "questions", + "model" + ], + "title": "DecisionsRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/DecisionsResponse" + } } }, "description": "Successful Response" @@ -9539,7 +10678,7 @@ "APIKeyHeader": [] } ], - "summary": "Systemone", + "summary": "Ask named questions about a state (System One format)", "tags": [ "decisions" ] @@ -9547,12 +10686,382 @@ }, "/v1/decisions": { "post": { + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_v1_decisions_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "model": "luna", + "questions": [ + { + "instructions": "Is the customer asking for a refund?", + "name": "is_refund_request", + "type": "predicate" + }, + { + "choices": [ + { + "description": "can wait a week", + "value": "low" + }, + { + "description": "needs action today", + "value": "high" + } + ], + "instructions": "How urgent is this?", + "name": "urgency", + "type": "choice" + }, + { + "instructions": "How frustrated is the customer?", + "levels": [ + { + "label": "calm" + }, + { + "label": "mildly annoyed" + }, + { + "label": "angry" + } + ], + "name": "frustration", + "type": "score" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "input_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "OpenAIDecisionInputText", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "image_url": { + "title": "Image Url", + "type": "string" + }, + "type": { + "const": "input_image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "OpenAIDecisionInputImage", + "type": "object" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "default": "user", + "title": "Role", + "type": "string" + }, + "type": { + "const": "message", + "default": "message", + "title": "Type", + "type": "string" + } + }, + "required": [ + "content" + ], + "title": "OpenAIDecisionInputMessage", + "type": "object" + }, + "type": "array" + } + ], + "description": "The text being judged, or user messages whose content mixes input_text and input_image parts", + "title": "Input" + }, + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "description": "Predicate, choice and score questions. Answers come back in the same order", + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The yes/no question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIPredicateQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "choices": { + "description": "The options the model picks from. Values must be unique", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "OpenAIChoiceOption", + "type": "object" + }, + "maxItems": 255, + "minItems": 2, + "title": "Choices", + "type": "array" + }, + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "choices" + ], + "title": "OpenAIChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "levels": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label" + ], + "title": "OpenAIScoreLevel", + "type": "object" + }, + "maxItems": 10, + "minItems": 2, + "title": "Levels", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "levels" + ], + "title": "OpenAIScoreQuestion", + "type": "object" + } + ] + }, + "maxItems": 128, + "minItems": 1, + "title": "Questions", + "type": "array" + }, + "safety_identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", + "title": "Safety Identifier" + } + }, + "required": [ + "input", + "questions", + "model" + ], + "title": "OpenAIDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/OpenAIDecisionResponse" + } } }, "description": "Successful Response" @@ -9563,7 +11072,7 @@ "APIKeyHeader": [] } ], - "summary": "Decisions", + "summary": "Ask questions about an input (OpenAI Decisions format)", "tags": [ "decisions" ] @@ -9571,12 +11080,290 @@ }, "/v1/systemone": { "post": { + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_v1_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "jev", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "urgency": { + "criteria": { + "high": "needs action today", + "low": "can wait a week" + }, + "instructions": "How urgent is this?", + "type": "choice" + } + }, + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + }, + "schema": { + "additionalProperties": true, + "properties": { + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": true, + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what makes the answer true and what makes it false", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The yes/no question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "NoulQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "description": "Candidate labels mapped to an optional description of each", + "maxProperties": 255, + "minProperties": 1, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "maxItems": 10, + "minItems": 1, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ScoreQuestion", + "type": "object" + } + ] + }, + "description": "Named noul, choice and score questions. Each key becomes a key in the answers", + "maxProperties": 128, + "minProperties": 1, + "propertyNames": { + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The thing being judged, such as a support ticket, a document or a chat transcript", + "title": "State" + } + }, + "required": [ + "state", + "questions", + "model" + ], + "title": "DecisionsRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/DecisionsResponse" + } } }, "description": "Successful Response" @@ -9587,7 +11374,7 @@ "APIKeyHeader": [] } ], - "summary": "Systemone", + "summary": "Ask named questions about a state (System One format)", "tags": [ "decisions" ] diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index c1d4442893a..45ca509b7b7 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -437,3 +437,41 @@ class CustomOpenAPISpec: # Add responses API request schema return CustomOpenAPISpec.add_responses_api_request_schema(with_embeddings) + + +_MAX_INLINE_DEPTH: Final = 32 + + +def _inlined(node: JsonValue, defs: Mapping[str, JsonValue], depth: int = 0) -> JsonValue: + if depth >= _MAX_INLINE_DEPTH or not isinstance(node, (list, dict)): + return node + if isinstance(node, list): + return [_inlined(item, defs, depth + 1) for item in node] + ref: Final = node.get("$ref") + if isinstance(ref, str) and ref.startswith("#/$defs/"): + target: Final = _inlined(defs[ref.removeprefix("#/$defs/")], defs, depth + 1) + siblings: Final = {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$ref"} + return {**target, **siblings} if isinstance(target, dict) else siblings + if "propertyName" in node and "mapping" in node: + return {"propertyName": node["propertyName"]} + return {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$defs"} + + +def inline_request_body(model_class: type, example: Mapping[str, object]) -> JsonObject: + """ + Build an ``openapi_extra["requestBody"]`` for a route that reads its body with ``request.body()``. + + Swagger UI resolves ``$ref`` against the whole document, so a schema placed inline in an operation + cannot keep Pydantic's ``#/$defs/...`` references. Every reference is expanded in place instead, + which also keeps the operation self-contained inside the lazy OpenAPI snapshot. + """ + schema: Final = CustomOpenAPISpec.get_pydantic_schema(model_class) + media: Final[JsonObject] = {"example": cast(JsonValue, dict(example))} # cast-ok: JSON example literal + if schema is None: + return {"required": True, "content": {"application/json": media}} + raw_defs: Final = schema.get("$defs") + defs: Final[Mapping[str, JsonValue]] = raw_defs if isinstance(raw_defs, dict) else MappingProxyType({}) + return { + "required": True, + "content": {"application/json": {"schema": _inlined(schema, defs), **media}}, + } diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 779fb85192b..fba4d79fe48 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -9,7 +9,15 @@ from pydantic import TypeAdapter, ValidationError from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body +from litellm.types.decisions import ( + DecisionsRequest, + DecisionsRequestBody, + DecisionsResponse, + OpenAIDecisionRequest, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, +) router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) @@ -20,6 +28,51 @@ _OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBo _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) _OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) +_SYSTEMONE_REQUEST_BODY: Final = inline_request_body( + DecisionsRequest, + { + "model": "jev", + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "questions": { + "is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "urgency": { + "type": "choice", + "instructions": "How urgent is this?", + "criteria": {"low": "can wait a week", "high": "needs action today"}, + }, + "frustration": { + "type": "score", + "instructions": "How frustrated is the customer?", + "criteria": ["calm", "mildly annoyed", "angry"], + }, + }, + }, +) +_DECISIONS_REQUEST_BODY: Final = inline_request_body( + OpenAIDecisionRequest, + { + "model": "luna", + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "questions": [ + {"type": "predicate", "name": "is_refund_request", "instructions": "Is the customer asking for a refund?"}, + { + "type": "choice", + "name": "urgency", + "instructions": "How urgent is this?", + "choices": [ + {"value": "low", "description": "can wait a week"}, + {"value": "high", "description": "needs action today"}, + ], + }, + { + "type": "score", + "name": "frustration", + "instructions": "How frustrated is the customer?", + "levels": [{"label": "calm"}, {"label": "mildly annoyed"}, {"label": "angry"}], + }, + ], + }, +) async def _invalid_request( @@ -121,18 +174,35 @@ async def _process_decisions( dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask named questions about a state (System One format)", + responses={200: {"model": DecisionsResponse}}, + openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY}, ) @router.post( "/systemone", dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask named questions about a state (System One format)", + responses={200: {"model": DecisionsResponse}}, + openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY}, ) async def systemone( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): + """ + Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + + Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + + [Docs](https://docs.litellm.ai/docs/decisions) + """ return await _process_decisions( request=request, fastapi_response=fastapi_response, @@ -146,18 +216,35 @@ async def systemone( dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask questions about an input (OpenAI Decisions format)", + responses={200: {"model": OpenAIDecisionResponse}}, + openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY}, ) @router.post( "/decisions", dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask questions about an input (OpenAI Decisions format)", + responses={200: {"model": OpenAIDecisionResponse}}, + openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY}, ) async def decisions( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): + """ + Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + + Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + model list. System One decision models accept this format too, LiteLLM translates the request and the + answers. Streaming is not supported. + + [Docs](https://docs.litellm.ai/docs/decisions) + """ return await _process_decisions( request=request, fastapi_response=fastapi_response, diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index b24690dbd9d..3bc90a5109a 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -15,8 +15,13 @@ MAX_DECISION_QUESTIONS: Final = 128 class NoulQuestion(LiteLLMPydanticObjectBase): type: Literal["noul"] - instructions: DecisionsJSON | None = None - criteria: NoulCriteria | None = None + instructions: Annotated[DecisionsJSON | None, Field(description="The yes/no question to ask about the state")] = ( + None + ) + criteria: Annotated[ + NoulCriteria | None, + Field(description="Optional descriptions of what makes the answer true and what makes it false"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) @@ -29,16 +34,24 @@ class NoulQuestion(LiteLLMPydanticObjectBase): class ChoiceQuestion(LiteLLMPydanticObjectBase): type: Literal["choice"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)] + instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None + criteria: Annotated[ + Mapping[str, DecisionsJSON | None], + Field(min_length=1, max_length=255, description="Candidate labels mapped to an optional description of each"), + ] model_config = ConfigDict(extra="allow", frozen=True) class ScoreQuestion(LiteLLMPydanticObjectBase): type: Literal["score"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)] + instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None + criteria: Annotated[ + Sequence[DecisionsJSON], + Field( + min_length=1, max_length=10, description="Scale levels from lowest to highest. The score is a level index" + ), + ] model_config = ConfigDict(extra="allow", frozen=True) @@ -55,14 +68,20 @@ DecisionQuestionMap: TypeAlias = Annotated[ class DecisionsRequestBody(LiteLLMPydanticObjectBase): - state: DecisionsJSON - questions: DecisionQuestionMap + state: Annotated[ + DecisionsJSON, + Field(description="The thing being judged, such as a support ticket, a document or a chat transcript"), + ] + questions: Annotated[ + DecisionQuestionMap, + Field(description="Named noul, choice and score questions. Each key becomes a key in the answers"), + ] model_config = ConfigDict(extra="allow", frozen=True) class DecisionsRequest(DecisionsRequestBody): - model: str + model: Annotated[str, Field(description="A decision model from the proxy model_list")] @with_config(ConfigDict(extra="allow")) @@ -171,8 +190,8 @@ OpenAIDecisionInput: TypeAlias = str | Sequence[OpenAIDecisionInputMessage] class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase): type: Literal["predicate"] - name: str | None = None - instructions: str + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The yes/no question to ask about the input")] model_config = ConfigDict(extra="forbid", frozen=True) @@ -192,9 +211,12 @@ def systemone_choice_key(value: str | bool) -> str: class OpenAIChoiceQuestion(LiteLLMPydanticObjectBase): type: Literal["choice"] - name: str | None = None - instructions: str - choices: Annotated[Sequence[OpenAIChoiceOption], Field(min_length=2, max_length=255)] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The question to ask about the input")] + choices: Annotated[ + Sequence[OpenAIChoiceOption], + Field(min_length=2, max_length=255, description="The options the model picks from. Values must be unique"), + ] model_config = ConfigDict(extra="forbid", frozen=True) @@ -215,9 +237,14 @@ class OpenAIScoreLevel(LiteLLMPydanticObjectBase): class OpenAIScoreQuestion(LiteLLMPydanticObjectBase): type: Literal["score"] - name: str | None = None - instructions: str - levels: Annotated[Sequence[OpenAIScoreLevel], Field(min_length=2, max_length=10)] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The question to ask about the input")] + levels: Annotated[ + Sequence[OpenAIScoreLevel], + Field( + min_length=2, max_length=10, description="Scale levels from lowest to highest. The score is a level index" + ), + ] model_config = ConfigDict(extra="forbid", frozen=True) @@ -229,13 +256,32 @@ OpenAIDecisionQuestion: TypeAlias = Annotated[ class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): - input: OpenAIDecisionInput - questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)] - safety_identifier: str | None = None + input: Annotated[ + OpenAIDecisionInput, + Field( + description="The text being judged, or user messages whose content mixes input_text and input_image parts" + ), + ] + questions: Annotated[ + Sequence[OpenAIDecisionQuestion], + Field( + min_length=1, + max_length=MAX_DECISION_QUESTIONS, + description="Predicate, choice and score questions. Answers come back in the same order", + ), + ] + safety_identifier: Annotated[ + str | None, + Field(description="A stable id for your end user. Sent to OpenAI and dropped for other providers"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) +class OpenAIDecisionRequest(OpenAIDecisionRequestBody): + model: Annotated[str, Field(description="A decision model from the proxy model_list")] + + class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase): type: Literal["predicate"] = "predicate" name: str | None diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 863934d76f7..d664c7771cc 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -53,6 +53,7 @@ IGNORE_FUNCTIONS = [ "sanitize_oci_schema", # OCI: bounded by JSON-schema tree depth. "_freeze_for_dedupe", # OTEL: max depth set (default 16, _FREEZE_MAX_DEPTH); fails closed by returning repr(value) at the cap. "apply_json_merge_patch", # max depth set (_MAX_MERGE_DEPTH=64); fails closed by raising ValueError at the cap. + "_inlined", # max depth set (_MAX_INLINE_DEPTH=32); passes the schema node through untouched at the cap. Walks a Pydantic JSON schema once at import time. "_filter_argument_value", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by blocking the tool call at the cap. "_redact_scanned_content", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by returning "[REDACTED]" at the cap. "replace_ciphertexts", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); walks stored JSON, which has no cycles, and leaves values below the cap untouched. diff --git a/tests/unit/proxy/common_utils/test_custom_openapi_spec.py b/tests/unit/proxy/common_utils/test_custom_openapi_spec.py index 485dbcef2f6..41ab613e7ea 100644 --- a/tests/unit/proxy/common_utils/test_custom_openapi_spec.py +++ b/tests/unit/proxy/common_utils/test_custom_openapi_spec.py @@ -4,11 +4,14 @@ Simple unit tests for CustomOpenAPISpec class. Tests basic functionality of OpenAPI schema generation. """ -from unittest.mock import Mock, patch +import json +from typing import Annotated, Literal +from unittest.mock import patch import pytest +from pydantic import BaseModel, ConfigDict, Field -from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec +from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec, inline_request_body class TestCustomOpenAPISpec: @@ -27,19 +30,13 @@ class TestCustomOpenAPISpec: }, } - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) - def test_add_chat_completion_request_schema( - self, mock_add_schema, base_openapi_schema - ): + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") + def test_add_chat_completion_request_schema(self, mock_add_schema, base_openapi_schema): """Test that chat completion schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema with patch("litellm.proxy._types.ProxyChatCompletionRequest") as mock_model: - result = CustomOpenAPISpec.add_chat_completion_request_schema( - base_openapi_schema - ) + result = CustomOpenAPISpec.add_chat_completion_request_schema(base_openapi_schema) mock_add_schema.assert_called_once_with( openapi_schema=base_openapi_schema, @@ -50,9 +47,7 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") def test_add_embedding_request_schema(self, mock_add_schema, base_openapi_schema): """Test that embedding schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema @@ -69,19 +64,13 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) - def test_add_responses_api_request_schema( - self, mock_add_schema, base_openapi_schema - ): + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") + def test_add_responses_api_request_schema(self, mock_add_schema, base_openapi_schema): """Test that responses API schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema with patch("litellm.types.llms.openai.ResponsesAPIRequestParams") as mock_model: - result = CustomOpenAPISpec.add_responses_api_request_schema( - base_openapi_schema - ) + result = CustomOpenAPISpec.add_responses_api_request_schema(base_openapi_schema) mock_add_schema.assert_called_once_with( openapi_schema=base_openapi_schema, @@ -123,15 +112,11 @@ def test_defs_rewritten_in_add_schema_to_components(): ) assert "$defs" not in openapi_schema assert ( - openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][ - "items" - ]["anyOf"][0]["$ref"] + openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" ) assert ( - openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][ - "items" - ]["anyOf"][1]["$ref"] + openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" ) @@ -188,14 +173,8 @@ def test_rewrite_defs_refs(): rewritten = CustomOpenAPISpec._rewrite_defs_refs(schema=schema, renames={}) assert "$defs" not in rewritten - assert ( - rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] - == "#/components/schemas/UserMessage" - ) - assert ( - rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] - == "#/components/schemas/AssistantMessage" - ) + assert rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" + assert rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" def test_get_pydantic_schema_generates_schema_for_responses_request_typed_dict(): @@ -390,3 +369,42 @@ def test_add_schema_to_components_renames_def_with_different_required_set(): assert schemas["Block"]["required"] == ["type", "x", "y"] assert schemas["Req_Block"]["required"] == ["keys", "type", "x", "y"] assert schemas["Req"]["properties"]["b"]["$ref"] == "#/components/schemas/Req_Block" + + +class _Cat(BaseModel): + kind: Literal["cat"] + lives: int = 9 + + model_config = ConfigDict(frozen=True) + + +class _Dog(BaseModel): + kind: Literal["dog"] + good: bool = True + + model_config = ConfigDict(frozen=True) + + +class _Shelter(BaseModel): + pets: dict[str, Annotated[_Cat | _Dog, Field(discriminator="kind")]] + favorite: _Cat | None = None + + model_config = ConfigDict(frozen=True) + + +def test_inline_request_body_expands_every_ref_so_the_operation_stands_alone(): + body = inline_request_body(_Shelter, {"pets": {"tom": {"kind": "cat"}}}) + + media = body["content"]["application/json"] + assert body["required"] is True + assert media["example"] == {"pets": {"tom": {"kind": "cat"}}} + assert "$ref" not in json.dumps(media["schema"]) + assert "$defs" not in media["schema"] + pet_schema = media["schema"]["properties"]["pets"]["additionalProperties"] + variants = {variant["properties"]["kind"]["const"]: variant for variant in pet_schema["oneOf"]} + assert variants["cat"]["properties"]["lives"]["default"] == 9 + assert variants["dog"]["properties"]["good"]["type"] == "boolean" + assert pet_schema["discriminator"] == {"propertyName": "kind"} + favorite = media["schema"]["properties"]["favorite"] + assert favorite["default"] is None + assert any(option.get("properties", {}).get("lives") for option in favorite["anyOf"]) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index e2f7b6ab884..e92ebc0eec9 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -12,17 +12,19 @@ import respx from fastapi import FastAPI from fastapi.routing import APIRoute from fastapi.testclient import TestClient +from pydantic import TypeAdapter from starlette.routing import Match import litellm from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features -from litellm.proxy.decisions_endpoints.endpoints import decisions, systemone +from litellm.proxy.decisions_endpoints.endpoints import decisions, router, systemone from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder from litellm.proxy.proxy_server import ( app, cleanup_router_config_variables, initialize, ) +from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody _INPUT_TOKENS: Final[int] = 367 _OUTPUT_TOKENS: Final[int] = 3 @@ -643,3 +645,63 @@ def test_with_lazy_routes_disabled_a_config_pass_through_at_v1_decisions_still_w assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} assert _serving_endpoint(bare, "/v1/decisions") is pass_through assert _serving_endpoint(bare, "/decisions") is decisions + + +@pytest.fixture(scope="module") +def decisions_openapi() -> dict[str, object]: + app = FastAPI() + app.include_router(router) + return app.openapi() + + +@pytest.mark.parametrize( + ("path", "body_adapter", "response_schema", "required"), + ( + ("/v1/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}), + ("/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}), + ( + "/v1/decisions", + TypeAdapter(OpenAIDecisionRequestBody), + "OpenAIDecisionResponse", + {"model", "input", "questions"}, + ), + ( + "/decisions", + TypeAdapter(OpenAIDecisionRequestBody), + "OpenAIDecisionResponse", + {"model", "input", "questions"}, + ), + ), +) +def test_swagger_documents_the_request_format_and_response_of_each_decisions_route( + decisions_openapi: dict[str, object], + path: str, + body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody], + response_schema: str, + required: set[str], +) -> None: + operation = decisions_openapi["paths"][path]["post"] + media = operation["requestBody"]["content"]["application/json"] + + assert "Decisions" in operation["summary"] or "System One" in operation["summary"] + assert "docs.litellm.ai/docs/decisions" in operation["description"] + assert required <= set(media["schema"]["required"]) + assert required <= set(media["schema"]["properties"]) + assert "$ref" not in json.dumps(media["schema"]) + assert media["schema"]["properties"]["model"]["description"] + + example = {key: value for key, value in media["example"].items() if key != "model"} + assert len(body_adapter.validate_python(example).questions) == 3 + + response_ref = operation["responses"]["200"]["content"]["application/json"]["schema"]["$ref"] + assert response_ref == f"#/components/schemas/{response_schema}" + assert "answers" in decisions_openapi["components"]["schemas"][response_schema]["properties"] + + +def test_the_two_decisions_routes_document_different_request_formats(decisions_openapi: dict[str, object]) -> None: + def properties(path: str) -> set[str]: + operation = decisions_openapi["paths"][path]["post"] + return set(operation["requestBody"]["content"]["application/json"]["schema"]["properties"]) + + assert "state" in properties("/v1/systemone") and "state" not in properties("/v1/decisions") + assert "input" in properties("/v1/decisions") and "input" not in properties("/v1/systemone") From 40323053177a3615d1c13071e4faa42224f48c5a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:47:32 -0700 Subject: [PATCH 15/18] test(decisions): bill the OpenAI-format openrouter case from provider-reported cost (#45853) PR #45667 removed openrouter/typesafe/jev-1.13 from the cost map after #45640 moved OpenRouter decisions billing to usage.cost. The OpenAI-format wire test still read the removed row and raised KeyError on CircleCI integration-providers Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 5894968c5dc31dad5a20ff27750b27f399b0576f) --- .../test_decisions_openai_format_wire.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index f0e7163e264..1104a6151db 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -105,19 +105,25 @@ class _Provider: wraps_result: bool cost_map_key: str | None speaks_openai: bool = False + provider_reported_cost: float | None = None def upstream_body(self) -> dict[str, JsonValue]: if self.speaks_openai: return {"model": self.body_model, "input": _INPUT, "questions": _QUESTIONS} return {"model": self.body_model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS} + def _with_reported_cost(self, usage: dict[str, JsonValue]) -> dict[str, JsonValue]: + if self.provider_reported_cost is None: + return usage + return {**usage, "cost": self.provider_reported_cost} + def upstream_reply(self) -> dict[str, JsonValue]: if self.speaks_openai: - return {"model": self.body_model, "answers": _ANSWERS, "usage": _USAGE} + return {"model": self.body_model, "answers": _ANSWERS, "usage": self._with_reported_cost(_USAGE)} answer: Final[dict[str, JsonValue]] = { "model": self.body_model, "answers": _SYSTEM_ONE_ANSWERS, - "usage": _SYSTEM_ONE_USAGE, + "usage": self._with_reported_cost(_SYSTEM_ONE_USAGE), } return {"result": answer, "success": True} if self.wraps_result else answer @@ -143,7 +149,8 @@ _PROVIDERS: Final = ( "typesafe/jev-1.13", _API_KEY, False, - "openrouter/typesafe/jev-1.13", + None, + provider_reported_cost=1.5834e-5, ), _Provider( "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None @@ -235,11 +242,13 @@ def _number(value: JsonValue) -> float: return float(value) -def _expected_spend(cost_map_key: str | None) -> float: - if cost_map_key is None: +def _expected_spend(provider: _Provider) -> float: + if provider.provider_reported_cost is not None: + return provider.provider_reported_cost + if provider.cost_map_key is None: return 0.0 cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes()) - prices: Final = object_value(cost_map[cost_map_key]) + prices: Final = object_value(cost_map[provider.cost_map_key]) return _INPUT_TOKENS * _number(prices["input_cost_per_token"]) + _OUTPUT_TOKENS * _number( prices["output_cost_per_token"] ) @@ -320,10 +329,8 @@ def _drop_params_config(directory: Path, api_base: str) -> Path: @pytest.mark.parametrize("provider", _PROVIDERS, ids=_provider_id) -def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cost_map( - gateway: Gateway, provider: _Provider -) -> None: - expected_spend: Final = _expected_spend(provider.cost_map_key) +def test_each_provider_gets_its_own_path_key_and_body_and_is_billed(gateway: Gateway, provider: _Provider) -> None: + expected_spend: Final = _expected_spend(provider) with gateway.scenario() as scenario: handle: Final = _register(scenario, provider.upstream_reply()) model: Final = _deployment(scenario, handle, provider) From f26cff4ce353e2a874976d85893e30aaac45ded8 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:16:51 +0000 Subject: [PATCH 16/18] feat(sdk): add litellm.systemone and trim litellm.decisions to the OpenAI shape (#45846) * feat(sdk): add litellm.systemone and trim litellm.decisions to the OpenAI shape Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: run tests/unit/systemone in the core-utils unit shard Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(sdk): keep litellm.decisions(state=...) working with a DeprecationWarning Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: make tests/unit/systemone a package Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(sdk): surface the litellm.decisions(state=...) deprecation as a UserWarning Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 25d9b05386347d7b78bdd18aef7fa2f124b1b560) --- litellm/__init__.py | 1 + litellm/cost_calculator.py | 2 + litellm/decisions/call.py | 347 ++++++++++++++++ litellm/decisions/main.py | 385 +++--------------- .../health_check_helpers.py | 2 +- litellm/proxy/common_request_processing.py | 2 + .../proxy/decisions_endpoints/endpoints.py | 7 +- litellm/proxy/route_llm_request.py | 2 + litellm/router.py | 9 + litellm/systemone/__init__.py | 3 + litellm/systemone/main.py | 107 +++++ litellm/types/utils.py | 8 +- litellm/utils.py | 8 +- .../providers/test_decisions_wire.py | 8 +- .../test_strands_decider_agentcore_wire.py | 6 +- tests/unit/decisions/test_main.py | 151 ++++--- .../llms/azure/test_azure_common_utils.py | 1 + tests/unit/systemone/__init__.py | 0 tests/unit/systemone/test_main.py | 138 +++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 20 files changed, 786 insertions(+), 403 deletions(-) create mode 100644 litellm/decisions/call.py create mode 100644 litellm/systemone/__init__.py create mode 100644 litellm/systemone/main.py create mode 100644 tests/unit/systemone/__init__.py create mode 100644 tests/unit/systemone/test_main.py diff --git a/litellm/__init__.py b/litellm/__init__.py index c228e63e928..81fd6ca6ca5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1474,6 +1474,7 @@ from .rust_bridge import rust from .rag.main import * from .sandbox.main import * from .decisions.main import * +from .systemone.main import * from .search.main import * from .realtime_api.main import ( _arealtime, diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9dd6cc0f1f9..2a254cfec1f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2014,6 +2014,8 @@ def response_cost_calculator( "asearch", "decisions", "adecisions", + "systemone", + "asystemone", ], optional_params: dict, cache_hit: bool | None = None, diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py new file mode 100644 index 00000000000..757bc5a9b0a --- /dev/null +++ b/litellm/decisions/call.py @@ -0,0 +1,347 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field, replace +from typing import Final, TypeAlias + +import httpx +from pydantic import ConfigDict, TypeAdapter, ValidationError +from typing_extensions import assert_never + +import litellm +from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.decisions.transformation import ( + BaseDecisionsConfig, + ir_to_systemone_response, + systemone_request_to_ir, +) +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir +from litellm.types.decisions import ( + DecisionQuestion, + DecisionsIRRequest, + DecisionsIRResponse, + DecisionsJSON, + DecisionsRequestBody, + DecisionsResponse, + OpenAIDecisionInput, + OpenAIDecisionQuestion, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, + UnsupportedDecisionsRequest, +) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]] +OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]] +DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody + +_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) +_SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( + str | None, config=ConfigDict(title="safety_identifier") +) +_HANDLER: Final = BaseLLMHTTPHandler() + + +@dataclass(frozen=True, slots=True) +class SystemOneRequestFields: + state: DecisionsJSON | None + questions: SystemOneQuestions | None + + def validate(self, safety_identifier: str | None) -> DecisionsRequestBody: + return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": self.state, "questions": self.questions}) + + +@dataclass(frozen=True, slots=True) +class OpenAIDecisionRequestFields: + input: OpenAIDecisionInput | None + questions: OpenAIDecisionQuestions | None + + def validate(self, safety_identifier: str | None) -> OpenAIDecisionRequestBody: + return _OPENAI_REQUEST_ADAPTER.validate_python( + {"input": self.input, "questions": self.questions, "safety_identifier": safety_identifier} + ) + + +RequestFields: TypeAlias = SystemOneRequestFields | OpenAIDecisionRequestFields + + +@dataclass(frozen=True, slots=True, repr=False) +class DecisionsCall: + model: str + requested_model: str + custom_llm_provider: str + provider_config: BaseDecisionsConfig + request: DecisionsRequestFormat + ir_request: DecisionsIRRequest + body: Mapping[str, object] = field(repr=False) + api_base: str + api_key: str | None = field(repr=False) + logging_obj: LiteLLMLoggingObj | None + headers: Mapping[str, str] + timeout: float | httpx.Timeout | None + litellm_params: Mapping[str, object] = field(repr=False) + + +def _supported_providers() -> tuple[str, ...]: + return tuple( + provider.value + for provider in LlmProviders + if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None + ) + + +def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig: + provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None) + provider_config: Final = ( + None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider) + ) + if provider_config is None: + supported: Final = ", ".join(_supported_providers()) + raise litellm.BadRequestError( + message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}", + model=model, + llm_provider=custom_llm_provider, + ) + return provider_config + + +def _ir_request(request: DecisionsRequestFormat, safety_identifier: str | None) -> DecisionsIRRequest: + match request: + case DecisionsRequestBody(): + return replace(systemone_request_to_ir(request), safety_identifier=safety_identifier) + case OpenAIDecisionRequestBody(): + return openai_request_to_ir(request) + case _: + assert_never(request) + + +def _drops_params(kwargs: Mapping[str, object]) -> bool: + return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True + + +def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, object]) -> str | None: + if isinstance(safety_identifier, str) or not _drops_params(kwargs): + return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier) + return None + + +@dataclass(frozen=True, slots=True) +class _UnsupportedSafetyIdentifier: + pass + + +def _provider_ir_request( + request: DecisionsRequestFormat, + *, + safety_identifier: str | None, + provider_config: BaseDecisionsConfig, + kwargs: Mapping[str, object], +) -> DecisionsIRRequest | _UnsupportedSafetyIdentifier: + ir_request: Final = _ir_request(request, safety_identifier) + if ir_request.safety_identifier is None or provider_config.supports_safety_identifier: + return ir_request + if _drops_params(kwargs): + return replace(ir_request, safety_identifier=None) + return _UnsupportedSafetyIdentifier() + + +def prepare_call( + *, + model: str, + fields: RequestFields, + safety_identifier: str | None, + api_key: str | None, + api_base: str | None, + timeout: float | httpx.Timeout | None, + custom_llm_provider: str | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> DecisionsCall: + upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + provider_config: Final = _provider_config(upstream_model, provider) + if not upstream_model: + raise litellm.BadRequestError( + message="A model name is required for the Decisions API", + model=model, + llm_provider=provider, + ) + try: + canonical_model: Final = provider_config.canonical_model(upstream_model) + except ValueError as error: + raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error + try: + request_safety_identifier: Final = _request_safety_identifier(safety_identifier, kwargs) + request: Final = fields.validate(request_safety_identifier) + except ValidationError as error: + raise litellm.BadRequestError( + message=f"Invalid Decisions request: {error}", + model=model, + llm_provider=provider, + ) from error + + resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base) + if resolved_api_base is None: + raise litellm.BadRequestError( + message=provider_config.missing_api_base_message(provider), + model=model, + llm_provider=provider, + ) + resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key) + if resolved_api_key is None and provider_config.api_key_required: + raise litellm.AuthenticationError( + message=f"Missing API key for Decisions provider '{provider}'", + model=model, + llm_provider=provider, + ) + + ir_request: Final = _provider_ir_request( + request, + safety_identifier=request_safety_identifier, + provider_config=provider_config, + kwargs=kwargs, + ) + if isinstance(ir_request, _UnsupportedSafetyIdentifier): + raise litellm.UnsupportedParamsError( + message=( + f"{provider} does not support parameters: ['safety_identifier'], for model={model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ), + model=model, + llm_provider=provider, + ) + body: Final = provider_config.transform_decisions_request( + model=canonical_model, request=ir_request, custom_llm_provider=provider + ) + if isinstance(body, UnsupportedDecisionsRequest): + raise litellm.BadRequestError( + message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}", + model=model, + llm_provider=provider, + ) + + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.update_from_kwargs( + kwargs=dict(kwargs), + model=canonical_model, + litellm_params={ + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), + }, + custom_llm_provider=provider, + ) + return DecisionsCall( + model=canonical_model, + requested_model=model, + custom_llm_provider=provider, + provider_config=provider_config, + request=request, + ir_request=ir_request, + body=body, + api_base=resolved_api_base, + api_key=resolved_api_key, + logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, + headers=extra_headers or {}, + timeout=timeout, + litellm_params=kwargs, + ) + + +def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + formatted: Final = _formatted_response(response, call) + provider_cost: Final = call.provider_config.provider_reported_cost(response) + formatted.set_hidden_params( + { + "model": f"{call.custom_llm_provider}/{call.model}", + "custom_llm_provider": call.custom_llm_provider, + "provider_response_model": f"{call.custom_llm_provider}/{call.model}", + **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}), + } + ) + return formatted + + +def _formatted_response( + response: DecisionsIRResponse, call: DecisionsCall +) -> DecisionsResponse | OpenAIDecisionResponse: + match call.request: + case DecisionsRequestBody(): + return ir_to_systemone_response(response, call.ir_request) + case OpenAIDecisionRequestBody(): + return ir_to_openai_response(response, call.ir_request, call.requested_model) + case _: + assert_never(call.request) + + +def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception: + if isinstance(error, BaseLLMException) and error.status_code_is_synthesized: + provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception" + return litellm.APIConnectionError( + message=f"{provider_label} - {error.message}", + llm_provider=call.custom_llm_provider, + model=f"{call.custom_llm_provider}/{call.model}", + ) + return litellm.exception_type( + model=f"{call.custom_llm_provider}/{call.model}", + custom_llm_provider=call.custom_llm_provider, + original_exception=error, + ) + + +async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + try: + response: Final = await _HANDLER.adecisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.ir_request, + body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + ) + except Exception as error: + raise _map_upstream_exception(error, call) from error + return _format_response(response, call) + + +def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + try: + response: Final = _HANDLER.decisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.ir_request, + body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + ) + except Exception as error: + raise _map_upstream_exception(error, call) from error + return _format_response(response, call) + + +def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name: str, other_function: str) -> None: + if field_name not in kwargs: + return + raise litellm.BadRequestError( + message=f"Requests with '{field_name}' go through litellm.{other_function}", + model=model, + llm_provider="", + ) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 1c4108654fa..8b13d04770a 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,151 +1,56 @@ -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field, replace -from typing import Final, TypeAlias +import warnings +from collections.abc import Mapping +from typing import Final import httpx -from pydantic import ConfigDict, TypeAdapter, ValidationError -from typing_extensions import assert_never +from pydantic import TypeAdapter -import litellm -from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.base_llm.decisions.transformation import ( - BaseDecisionsConfig, - ir_to_systemone_response, - systemone_request_to_ir, +from litellm.decisions.call import ( + DecisionsCall, + OpenAIDecisionQuestions, + OpenAIDecisionRequestFields, + RequestFields, + SystemOneQuestions, + SystemOneRequestFields, + asend, + prepare_call, + send, ) -from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir -from litellm.types.decisions import ( - DecisionQuestion, - DecisionsIRRequest, - DecisionsIRResponse, - DecisionsJSON, - DecisionsRequestBody, - DecisionsResponse, - OpenAIDecisionInput, - OpenAIDecisionQuestion, - OpenAIDecisionRequestBody, - OpenAIDecisionResponse, - UnsupportedDecisionsRequest, +from litellm.types.decisions import DecisionsJSON, DecisionsResponse, OpenAIDecisionInput, OpenAIDecisionResponse +from litellm.utils import client + +_STATE_ADAPTER: Final[TypeAdapter[DecisionsJSON | None]] = TypeAdapter(DecisionsJSON | None) +_SYSTEMONE_QUESTIONS_ADAPTER: Final[TypeAdapter[SystemOneQuestions | None]] = TypeAdapter(SystemOneQuestions | None) +_OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter( + OpenAIDecisionQuestions | None ) -from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager, client - -DecisionsQuestions: TypeAlias = ( - Mapping[str, DecisionQuestion | Mapping[str, object]] | Sequence[OpenAIDecisionQuestion | Mapping[str, object]] -) -DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody - -_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) -_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) -_SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( - str | None, config=ConfigDict(title="safety_identifier") -) -_HANDLER: Final = BaseLLMHTTPHandler() -@dataclass(frozen=True, slots=True, repr=False) -class _DecisionsCall: - model: str - requested_model: str - custom_llm_provider: str - provider_config: BaseDecisionsConfig - request: DecisionsRequestFormat - ir_request: DecisionsIRRequest - body: Mapping[str, object] = field(repr=False) - api_base: str - api_key: str | None = field(repr=False) - logging_obj: LiteLLMLoggingObj | None - headers: Mapping[str, str] - timeout: float | httpx.Timeout | None - litellm_params: Mapping[str, object] = field(repr=False) - - -def _supported_providers() -> tuple[str, ...]: - return tuple( - provider.value - for provider in LlmProviders - if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None - ) - - -def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig: - provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None) - provider_config: Final = ( - None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider) - ) - if provider_config is None: - supported: Final = ", ".join(_supported_providers()) - raise litellm.BadRequestError( - message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}", - model=model, - llm_provider=custom_llm_provider, - ) - return provider_config - - -def _validate_request( - *, - state: DecisionsJSON | None, - questions: DecisionsQuestions | None, +def _request_fields( decision_input: OpenAIDecisionInput | None, - safety_identifier: str | None, -) -> DecisionsRequestFormat: - if decision_input is None: - return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": state, "questions": questions}) - return _OPENAI_REQUEST_ADAPTER.validate_python( - {"input": decision_input, "questions": questions, "safety_identifier": safety_identifier} + questions: OpenAIDecisionQuestions | SystemOneQuestions | None, + kwargs: Mapping[str, object], +) -> RequestFields: + if "state" not in kwargs: + return OpenAIDecisionRequestFields( + input=decision_input, questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions) + ) + warnings.warn( + "litellm.decisions(state=...) is deprecated, call litellm.systemone for the System One request format", + UserWarning, + stacklevel=5, + ) + return SystemOneRequestFields( + state=_STATE_ADAPTER.validate_python(kwargs["state"]), + questions=_SYSTEMONE_QUESTIONS_ADAPTER.validate_python(questions), ) -def _ir_request(request: DecisionsRequestFormat, safety_identifier: str | None) -> DecisionsIRRequest: - match request: - case DecisionsRequestBody(): - return replace(systemone_request_to_ir(request), safety_identifier=safety_identifier) - case OpenAIDecisionRequestBody(): - return openai_request_to_ir(request) - case _: - assert_never(request) - - -def _drops_params(kwargs: Mapping[str, object]) -> bool: - return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True - - -def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, object]) -> str | None: - if isinstance(safety_identifier, str) or not _drops_params(kwargs): - return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier) - return None - - -@dataclass(frozen=True, slots=True) -class _UnsupportedSafetyIdentifier: - pass - - -def _provider_ir_request( - request: DecisionsRequestFormat, - *, - safety_identifier: str | None, - provider_config: BaseDecisionsConfig, - kwargs: Mapping[str, object], -) -> DecisionsIRRequest | _UnsupportedSafetyIdentifier: - ir_request: Final = _ir_request(request, safety_identifier) - if ir_request.safety_identifier is None or provider_config.supports_safety_identifier: - return ir_request - if _drops_params(kwargs): - return replace(ir_request, safety_identifier=None) - return _UnsupportedSafetyIdentifier() - - -def _prepare_call( +def _prepare_decisions_call( *, model: str, - state: DecisionsJSON | None, - questions: DecisionsQuestions | None, decision_input: OpenAIDecisionInput | None, + questions: OpenAIDecisionQuestions | SystemOneQuestions | None, safety_identifier: str | None, api_key: str | None, api_base: str | None, @@ -153,171 +58,37 @@ def _prepare_call( custom_llm_provider: str | None, extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], -) -> _DecisionsCall: - upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider( +) -> DecisionsCall: + return prepare_call( model=model, - custom_llm_provider=custom_llm_provider, - api_base=api_base, + fields=_request_fields(decision_input, questions, kwargs), + safety_identifier=safety_identifier, api_key=api_key, - ) - provider_config: Final = _provider_config(upstream_model, provider) - canonical_model: Final = provider_config.canonical_model(upstream_model) - if not upstream_model: - raise litellm.BadRequestError( - message="A model name is required for the Decisions API", - model=model, - llm_provider=provider, - ) - if state is not None and decision_input is not None: - raise litellm.BadRequestError( - message="Pass either state (System One format) or input (OpenAI format) to the Decisions API, not both", - model=model, - llm_provider=provider, - ) - try: - request_safety_identifier: Final = _request_safety_identifier(safety_identifier, kwargs) - request: Final = _validate_request( - state=state, - questions=questions, - decision_input=decision_input, - safety_identifier=request_safety_identifier, - ) - except ValidationError as error: - raise litellm.BadRequestError( - message=f"Invalid Decisions request: {error}", - model=model, - llm_provider=provider, - ) from error - - resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base) - if resolved_api_base is None: - raise litellm.BadRequestError( - message=provider_config.missing_api_base_message(provider), - model=model, - llm_provider=provider, - ) - resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key) - if resolved_api_key is None and provider_config.api_key_required: - raise litellm.AuthenticationError( - message=f"Missing API key for Decisions provider '{provider}'", - model=model, - llm_provider=provider, - ) - - ir_request: Final = _provider_ir_request( - request, - safety_identifier=request_safety_identifier, - provider_config=provider_config, - kwargs=kwargs, - ) - if isinstance(ir_request, _UnsupportedSafetyIdentifier): - raise litellm.UnsupportedParamsError( - message=( - f"{provider} does not support parameters: ['safety_identifier'], for model={model}. " - "To drop these, set `litellm.drop_params=True` or for proxy:\n\n" - "`litellm_settings:\n drop_params: true`\n" - ), - model=model, - llm_provider=provider, - ) - body: Final = provider_config.transform_decisions_request( - model=canonical_model, request=ir_request, custom_llm_provider=provider - ) - if isinstance(body, UnsupportedDecisionsRequest): - raise litellm.BadRequestError( - message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}", - model=model, - llm_provider=provider, - ) - - logging_obj: Final = kwargs.get("litellm_logging_obj") - if isinstance(logging_obj, LiteLLMLoggingObj): - logging_obj.update_from_kwargs( - kwargs=dict(kwargs), - model=canonical_model, - litellm_params={ - "litellm_call_id": kwargs.get("litellm_call_id"), - "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), - }, - custom_llm_provider=provider, - ) - return _DecisionsCall( - model=canonical_model, - requested_model=model, - custom_llm_provider=provider, - provider_config=provider_config, - request=request, - ir_request=ir_request, - body=body, - api_base=resolved_api_base, - api_key=resolved_api_key, - logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, - headers=extra_headers or {}, + api_base=api_base, timeout=timeout, - litellm_params=kwargs, - ) - - -def _format_response(response: DecisionsIRResponse, call: _DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: - formatted: Final = _formatted_response(response, call) - provider_cost: Final = call.provider_config.provider_reported_cost(response) - formatted.set_hidden_params( - { - "model": f"{call.custom_llm_provider}/{call.model}", - "custom_llm_provider": call.custom_llm_provider, - "provider_response_model": f"{call.custom_llm_provider}/{call.model}", - **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}), - } - ) - return formatted - - -def _formatted_response( - response: DecisionsIRResponse, call: _DecisionsCall -) -> DecisionsResponse | OpenAIDecisionResponse: - match call.request: - case DecisionsRequestBody(): - return ir_to_systemone_response(response, call.ir_request) - case OpenAIDecisionRequestBody(): - return ir_to_openai_response(response, call.ir_request, call.requested_model) - case _: - assert_never(call.request) - - -def _map_upstream_exception(error: Exception, call: _DecisionsCall) -> Exception: - if isinstance(error, BaseLLMException) and error.status_code_is_synthesized: - provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception" - return litellm.APIConnectionError( - message=f"{provider_label} - {error.message}", - llm_provider=call.custom_llm_provider, - model=f"{call.custom_llm_provider}/{call.model}", - ) - return litellm.exception_type( - model=f"{call.custom_llm_provider}/{call.model}", - custom_llm_provider=call.custom_llm_provider, - original_exception=error, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs={key: value for key, value in kwargs.items() if key != "state"}, ) @client async def adecisions( model: str, - state: DecisionsJSON | None = None, - questions: DecisionsQuestions | None = None, + input: OpenAIDecisionInput | None = None, + questions: OpenAIDecisionQuestions | SystemOneQuestions | None = None, + safety_identifier: str | None = None, api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, extra_headers: Mapping[str, str] | None = None, - input: OpenAIDecisionInput | None = None, - safety_identifier: str | None = None, **kwargs: object, -) -> DecisionsResponse | OpenAIDecisionResponse: - call: Final = _prepare_call( +) -> OpenAIDecisionResponse | DecisionsResponse: + call: Final = _prepare_decisions_call( model=model, - state=state, - questions=questions, decision_input=input, + questions=questions, safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, @@ -326,44 +97,26 @@ async def adecisions( extra_headers=extra_headers, kwargs=kwargs, ) - try: - response: Final = await _HANDLER.adecisions( - model=call.model, - custom_llm_provider=call.custom_llm_provider, - logging_obj=call.logging_obj, - provider_config=call.provider_config, - request=call.ir_request, - body=call.body, - api_base=call.api_base, - api_key=call.api_key, - headers=call.headers, - timeout=call.timeout, - litellm_params=call.litellm_params, - ) - except Exception as error: - raise _map_upstream_exception(error, call) from error - return _format_response(response, call) + return await asend(call) @client def decisions( model: str, - state: DecisionsJSON | None = None, - questions: DecisionsQuestions | None = None, + input: OpenAIDecisionInput | None = None, + questions: OpenAIDecisionQuestions | SystemOneQuestions | None = None, + safety_identifier: str | None = None, api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, extra_headers: Mapping[str, str] | None = None, - input: OpenAIDecisionInput | None = None, - safety_identifier: str | None = None, **kwargs: object, -) -> DecisionsResponse | OpenAIDecisionResponse: - call: Final = _prepare_call( +) -> OpenAIDecisionResponse | DecisionsResponse: + call: Final = _prepare_decisions_call( model=model, - state=state, - questions=questions, decision_input=input, + questions=questions, safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, @@ -372,23 +125,7 @@ def decisions( extra_headers=extra_headers, kwargs=kwargs, ) - try: - response: Final = _HANDLER.decisions( - model=call.model, - custom_llm_provider=call.custom_llm_provider, - logging_obj=call.logging_obj, - provider_config=call.provider_config, - request=call.ir_request, - body=call.body, - api_base=call.api_base, - api_key=call.api_key, - headers=call.headers, - timeout=call.timeout, - litellm_params=call.litellm_params, - ) - except Exception as error: - raise _map_upstream_exception(error, call) from error - return _format_response(response, call) + return send(call) __all__ = ["adecisions", "decisions"] diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 6d6eb1b2b5b..9080796fbf6 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -273,7 +273,7 @@ class HealthCheckHelpers: **_filter_model_params(model_params=model_params), document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider), ), - "evaluation": lambda: litellm.adecisions( + "evaluation": lambda: litellm.asystemone( **DECISIONS_CALL_PARAMS.validate_python( { "state": prompt or "health check", diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index ae989bb6a00..31e84fd4d9e 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -174,6 +174,7 @@ ProxyRouteType: TypeAlias = Literal[ "aocr", "asearch", "adecisions", + "asystemone", "avideo_generation", "avideo_list", "avideo_status", @@ -1960,6 +1961,7 @@ class ProxyBaseLLMRequestProcessing: "aocr", "asearch", "adecisions", + "asystemone", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index fba4d79fe48..ac93d2178b9 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -1,6 +1,6 @@ from collections.abc import Mapping from types import MappingProxyType -from typing import Annotated, Final +from typing import Annotated, Final, Literal from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract @@ -109,6 +109,7 @@ async def _process_decisions( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody], + route_type: Literal["adecisions", "asystemone"], ) -> object: from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, @@ -146,7 +147,7 @@ async def _process_decisions( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, - route_type="adecisions", + route_type=route_type, proxy_logging_obj=proxy_logging_obj, llm_router=llm_router, general_settings=general_settings, @@ -208,6 +209,7 @@ async def systemone( fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, body_adapter=_DECISIONS_REQUEST_BODY_ADAPTER, + route_type="asystemone", ) @@ -250,4 +252,5 @@ async def decisions( fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, body_adapter=_OPENAI_DECISION_REQUEST_BODY_ADAPTER, + route_type="adecisions", ) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index a2741c9bb8f..743f3003c63 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -91,6 +91,7 @@ ROUTE_ENDPOINT_MAPPING: Final = { "aocr": "/ocr", "asearch": "/search", "adecisions": "/decisions", + "asystemone": "/systemone", "avideo_generation": "/videos", "avideo_list": "/videos", "avideo_status": "/videos/{video_id}", @@ -375,6 +376,7 @@ RouteType = Literal[ "aocr", "asearch", "adecisions", + "asystemone", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/router.py b/litellm/router.py index 76abe7e2bab..1ba2544cdb8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2091,6 +2091,11 @@ class Router: self.adecisions = self.factory_function(adecisions, call_type="adecisions") self.decisions = self.factory_function(decisions, call_type="decisions") + from litellm.systemone import asystemone, systemone + + self.asystemone = self.factory_function(asystemone, call_type="asystemone") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable + self.systemone = self.factory_function(systemone, call_type="systemone") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable + def _initialize_video_endpoints(self): """Initialize video endpoints.""" from litellm.videos import ( @@ -6646,6 +6651,8 @@ class Router: "search", "adecisions", "decisions", + "asystemone", + "systemone", "aadapter_generate_content", "avideo_generation", "video_generation", @@ -6720,6 +6727,7 @@ class Router: "ocr", "search", "decisions", + "systemone", "video_generation", "video_list", "video_status", @@ -6888,6 +6896,7 @@ class Router: "aocr", "ocr", "adecisions", + "asystemone", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/systemone/__init__.py b/litellm/systemone/__init__.py new file mode 100644 index 00000000000..eb4971a16e4 --- /dev/null +++ b/litellm/systemone/__init__.py @@ -0,0 +1,3 @@ +from litellm.systemone.main import asystemone, systemone + +__all__ = ["asystemone", "systemone"] diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py new file mode 100644 index 00000000000..754b69142f9 --- /dev/null +++ b/litellm/systemone/main.py @@ -0,0 +1,107 @@ +from collections.abc import Mapping +from typing import Final + +import httpx + +from litellm.decisions.call import ( + DecisionsCall, + SystemOneQuestions, + SystemOneRequestFields, + asend, + prepare_call, + reject_other_format, + send, +) +from litellm.types.decisions import DecisionsJSON, DecisionsResponse +from litellm.utils import client # pyright: ignore[reportUnknownVariableType] # client is an untyped decorator + + +def _prepare_systemone_call( + *, + model: str, + state: DecisionsJSON | None, + questions: SystemOneQuestions | None, + safety_identifier: str | None, + api_key: str | None, + api_base: str | None, + timeout: float | httpx.Timeout | None, + custom_llm_provider: str | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> DecisionsCall: + reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions") + return prepare_call( + model=model, + fields=SystemOneRequestFields(state=state, questions=questions), + safety_identifier=safety_identifier, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) + + +def _systemone_response(response: object) -> DecisionsResponse: + assert isinstance(response, DecisionsResponse) + return response + + +@client +async def asystemone( + model: str, + state: DecisionsJSON | None = None, + questions: SystemOneQuestions | None = None, + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: Mapping[str, str] | None = None, + safety_identifier: str | None = None, + **kwargs: object, # kwargs-ok: litellm_params forwarded by @client +) -> DecisionsResponse: + call: Final = _prepare_systemone_call( + model=model, + state=state, + questions=questions, + safety_identifier=safety_identifier, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) + return _systemone_response(await asend(call)) + + +@client +def systemone( + model: str, + state: DecisionsJSON | None = None, + questions: SystemOneQuestions | None = None, + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: Mapping[str, str] | None = None, + safety_identifier: str | None = None, + **kwargs: object, # kwargs-ok: litellm_params forwarded by @client +) -> DecisionsResponse: + call: Final = _prepare_systemone_call( + model=model, + state=state, + questions=questions, + safety_identifier=safety_identifier, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) + return _systemone_response(send(call)) + + +__all__ = ["asystemone", "systemone"] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 561965633b5..a673db3db86 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -469,6 +469,8 @@ class CallTypes(str, Enum): asearch = "asearch" decisions = "decisions" adecisions = "adecisions" + systemone = "systemone" + asystemone = "asystemone" arealtime = "_arealtime" aresponses_websocket = "_aresponses_websocket" create_batch = "create_batch" @@ -657,6 +659,8 @@ CallTypesLiteral = Literal[ "asearch", "decisions", "adecisions", + "systemone", + "asystemone", "_arealtime", "_aresponses_websocket", "create_batch", @@ -768,8 +772,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/search": [CallTypes.asearch, CallTypes.search], "/decisions": [CallTypes.adecisions, CallTypes.decisions], "/v1/decisions": [CallTypes.adecisions, CallTypes.decisions], - "/systemone": [CallTypes.adecisions, CallTypes.decisions], - "/v1/systemone": [CallTypes.adecisions, CallTypes.decisions], + "/systemone": [CallTypes.asystemone, CallTypes.systemone], + "/v1/systemone": [CallTypes.asystemone, CallTypes.systemone], # Batches "/batches": [CallTypes.acreate_batch, CallTypes.create_batch], "/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch], diff --git a/litellm/utils.py b/litellm/utils.py index 990bd689144..0c829520c72 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -972,6 +972,12 @@ def _ocr_document_summary(document: object) -> str: # Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc. + +_DECISION_CALL_TYPES: Final[frozenset[str]] = frozenset( + (CallTypes.decisions.value, CallTypes.adecisions.value, CallTypes.systemone.value, CallTypes.asystemone.value) +) + + def function_setup( original_function: str, rules_obj: Rules, @@ -1218,7 +1224,7 @@ def function_setup( if isinstance(search_query, list) else search_query ) - elif call_type in (CallTypes.decisions.value, CallTypes.adecisions.value): + elif call_type in _DECISION_CALL_TYPES: decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state") or kwargs.get("input") or "" messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state, default=str) elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 9a12ab2ec16..55fc142163c 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -269,7 +269,7 @@ def test_each_provider_gets_its_own_path_key_and_body_and_is_billed(gateway: Gat row["api_base"], row["prompt_tokens"], row["completion_tokens"], - ) == ("success", "adecisions", provider.name, model, f"{handle.api_base()}{provider.path}", 367, 3) + ) == ("success", "asystemone", provider.name, model, f"{handle.api_base()}{provider.path}", 367, 3) assert math.isclose(_number(row["spend"]), expected_spend, rel_tol=1e-9), row @@ -293,7 +293,7 @@ async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway synchronous: Final = litellm.decisions( model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY ) - asynchronous: Final = await litellm.adecisions( + asynchronous: Final = await litellm.asystemone( model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY ) for response in (synchronous, asynchronous): @@ -461,7 +461,7 @@ def test_upstream_errors_keep_their_status_and_log_an_unbilled_failure(gateway: row: Final = _spend_row(response.headers["x-litellm-call-id"]) assert (row["status"], row["call_type"], row["model_group"], _number(row["spend"])) == ( "failure", - "adecisions", + "asystemone", model, 0.0, ) @@ -533,7 +533,7 @@ async def test_sdk_openrouter_connection_failures_raise_a_connection_error(gatew model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY ) with pytest.raises(litellm.APIConnectionError): - await litellm.adecisions( + await litellm.asystemone( model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY ) diff --git a/tests/integration/providers/test_strands_decider_agentcore_wire.py b/tests/integration/providers/test_strands_decider_agentcore_wire.py index bf5da453137..493d8c238fa 100644 --- a/tests/integration/providers/test_strands_decider_agentcore_wire.py +++ b/tests/integration/providers/test_strands_decider_agentcore_wire.py @@ -415,7 +415,7 @@ def test_an_arn_deployment_answers_on_every_decisions_route_signed_for_bedrock_a row = _spend_row(response.headers["x-litellm-call-id"]) assert row == { "status": "success", - "call_type": "adecisions", + "call_type": "asystemone", "custom_llm_provider": "strands_decider", "model_group": _SIGNED, "api_base": _invocations_url("audit_signed"), @@ -694,7 +694,7 @@ async def test_sdk_sync_and_async_calls_reach_the_runtime_signed( aws_access_key_id=_ACCESS_KEY, aws_secret_access_key=_SECRET_KEY, ) - asynchronous: Final = await litellm.adecisions( + asynchronous: Final = await litellm.asystemone( model=_MODEL, state=_STATE, questions=_SDK_QUESTIONS, @@ -703,7 +703,7 @@ async def test_sdk_sync_and_async_calls_reach_the_runtime_signed( aws_secret_access_key=_SECRET_KEY, ) monkeypatch.setenv("STRANDS_DECIDER_API_KEY", _ENV_JWT) - bearer: Final = await litellm.adecisions( + bearer: Final = await litellm.asystemone( model=_MODEL, state=_STATE, questions=_SDK_QUESTIONS, api_base=_arn("audit_sdk_env_key") ) finally: diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 207228c4b53..35340646feb 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -29,6 +29,8 @@ from litellm.types.decisions import ( OpenAIDecisionResponse, ScoreAnswer, ) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager _QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( { @@ -147,7 +149,7 @@ async def test_adecisions_sends_the_provider_wire_contract( ) -> None: route: Final = respx_mock.post(url).respond(json=_RESPONSE) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model=model, state={"source": "unit-test"}, questions=_QUESTIONS, @@ -209,7 +211,7 @@ async def test_router_dispatches_typesafe_decisions_without_api_base( ) upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) - response: Final = await router.adecisions( + response: Final = await router.asystemone( model="jev", state="router-test", questions={ @@ -240,7 +242,7 @@ async def test_router_dispatches_typesafe_decisions_without_api_base( def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) - response: Final = litellm.decisions( + response: Final = litellm.systemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -260,7 +262,7 @@ def test_openrouter_response_keeps_provider_fields(respx_mock: respx.MockRouter) } respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=payload) - response: Final = litellm.decisions( + response: Final = litellm.systemone( model="openrouter/typesafe/jev-1.13", state="review", questions=_QUESTIONS, @@ -348,7 +350,7 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo litellm.callbacks = [recording_logger] try: - await litellm.adecisions( + await litellm.asystemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -373,7 +375,7 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo @pytest.mark.asyncio async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"): - await litellm.adecisions( + await litellm.asystemone( model="unknown/jev-1.13", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -386,7 +388,7 @@ async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRo @pytest.mark.asyncio async def test_provider_without_decisions_support_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: with pytest.raises(litellm.BadRequestError, match=r"Unknown Decisions provider 'anthropic'\. Supported providers"): - await litellm.adecisions( + await litellm.asystemone( model="anthropic/claude-sonnet-4-5", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -400,7 +402,7 @@ async def test_provider_without_decisions_support_is_rejected_before_http(respx_ async def test_empty_custom_provider_falls_back_to_the_model_prefix(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -420,7 +422,7 @@ async def test_upstream_reply_without_answers_is_a_server_error(respx_mock: resp ) with pytest.raises(litellm.InternalServerError, match="unexpected response"): - await litellm.adecisions( + await litellm.asystemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -448,7 +450,7 @@ async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_a ) upstream: Final = respx_mock.post("https://egress.example/openrouter/alpha/decisions").respond(json=_RESPONSE) - await router.adecisions( + await router.asystemone( model="jev", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -462,7 +464,7 @@ async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_a @pytest.mark.asyncio async def test_invalid_question_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): - await litellm.adecisions( + await litellm.asystemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"sentiment": {"type": "choice"}}, @@ -479,7 +481,7 @@ def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter ) with pytest.raises(litellm.BadRequestError): - litellm.decisions( + litellm.systemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -493,7 +495,7 @@ def test_unreachable_upstream_maps_to_a_connection_error(respx_mock: respx.MockR ) with pytest.raises(litellm.APIConnectionError, match="PerplexityException - Cannot connect to host"): - litellm.decisions( + litellm.systemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -509,7 +511,7 @@ def test_server_key_is_sent_to_an_explicit_api_base( monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False) route: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE) - litellm.decisions( + litellm.systemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -539,7 +541,7 @@ async def test_cloudflare_clef_resolves_model_and_response_envelope( "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" ).respond(json=response_body) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model=model, state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -569,7 +571,7 @@ async def test_cloudflare_clef_flash_uses_flash_endpoint_and_request_model( "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-flash" ).respond(json=_RESPONSE) - await litellm.adecisions( + await litellm.asystemone( model="cloudflare/clef-flash", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -591,7 +593,7 @@ async def test_cloudflare_api_base_from_env_uses_workers_ai_run_path( "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" ).respond(json=_RESPONSE) - await litellm.adecisions( + await litellm.asystemone( model="cloudflare/clef", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -610,7 +612,7 @@ async def test_cloudflare_requires_account_id_or_api_base_before_http( monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") with pytest.raises(litellm.BadRequestError, match="Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID"): - await litellm.adecisions( + await litellm.asystemone( model="cloudflare/clef", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -631,7 +633,7 @@ async def test_cloudflare_clef_cost_uses_the_model_cost_map( json=_RESPONSE ) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model="cloudflare/clef", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -656,7 +658,7 @@ async def test_strands_decider_requires_api_base_before_http( monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) with pytest.raises(litellm.BadRequestError, match="api_base is required"): - await litellm.adecisions( + await litellm.asystemone( model="strands_decider/strands-decider-2B-hobson-v19", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -674,7 +676,7 @@ async def test_strands_decider_without_key_preserves_response_extras( monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model="strands_decider/strands-decider-2B-hobson-v19", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -701,7 +703,7 @@ async def test_strands_decider_plain_api_base_answers_while_every_aws_signing_th try: response: Final = await asyncio.wait_for( - litellm.adecisions( + litellm.asystemone( model="strands_decider/strands-decider-2B-hobson-v19", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -725,7 +727,7 @@ async def test_strands_decider_uses_key_from_matching_environment_base( monkeypatch.setenv("STRANDS_DECIDER_API_KEY", "strands-key") route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) - await litellm.adecisions( + await litellm.asystemone( model="strands_decider/strands-decider-2B-hobson-v19", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -757,7 +759,7 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( ) route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) - response: Final = await router.adecisions( + response: Final = await router.asystemone( model="strands", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -811,7 +813,7 @@ async def test_strands_decider_runtime_arn_sends_a_sigv4_signed_invoke_agent_run "aws_secret_access_key": _RUNTIME_CREDENTIALS.secret_key, } - response: Final = await litellm.adecisions(**call_args) if use_async else litellm.decisions(**call_args) + response: Final = await litellm.asystemone(**call_args) if use_async else litellm.systemone(**call_args) assert route.call_count == 1 sent: Final = route.calls[0].request @@ -872,7 +874,7 @@ async def test_strands_decider_runtime_arn_with_api_key_sends_the_bearer_token_u monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) route: Final = respx_mock.post(_RUNTIME_URL).respond(json=_STRANDS_RESPONSE) - await litellm.adecisions( + await litellm.asystemone( model="strands_decider/strands-decider-2B-hobson-v19", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -905,7 +907,7 @@ async def test_strands_decider_runtime_error_envelope_raises_instead_of_answerin respx_mock.post(_RUNTIME_URL).respond(json={"error": {"code": code, "message": "at most 16 questions"}}) with pytest.raises(error_class, match="at most 16 questions"): - await litellm.adecisions( + await litellm.asystemone( model="strands_decider/strands-decider-2B-hobson-v19", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -926,7 +928,7 @@ async def test_strands_decider_plain_api_base_error_envelope_stays_an_unexpected ) with pytest.raises(litellm.InternalServerError, match="returned an unexpected response"): - await litellm.adecisions( + await litellm.asystemone( model="strands_decider/strands-decider-2B-hobson-v19", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -961,7 +963,7 @@ async def test_hosted_vllm_requires_api_base_before_http( monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) with pytest.raises(litellm.BadRequestError, match="api_base is required"): - await litellm.adecisions( + await litellm.asystemone( model="hosted_vllm/Qwen/Qwen3-0.6B", state="review", questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, @@ -979,7 +981,7 @@ async def test_hosted_vllm_without_key_sends_no_authorization_and_preserves_resp monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model="hosted_vllm/Qwen/Qwen3-0.6B", state="The package arrived broken.", questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, @@ -1002,7 +1004,7 @@ async def test_hosted_vllm_uses_key_and_base_from_environment( monkeypatch.setenv("HOSTED_VLLM_API_KEY", "vllm-key") route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) - await litellm.adecisions( + await litellm.asystemone( model="hosted_vllm/Qwen/Qwen3-0.6B", state="The package arrived broken.", questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, @@ -1021,7 +1023,7 @@ async def test_hosted_vllm_api_base_with_v1_suffix_posts_to_v1_systemone( monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) - await litellm.adecisions( + await litellm.asystemone( model="hosted_vllm/Qwen/Qwen3-0.6B", state="The package arrived broken.", questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, @@ -1040,7 +1042,7 @@ async def test_hosted_vllm_choice_question_goes_out_as_a_jev_choice_with_the_ser monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) - await litellm.adecisions( + await litellm.asystemone( model="hosted_vllm/Qwen/Qwen3-0.6B", state="The package arrived broken.", questions={ @@ -1067,6 +1069,30 @@ async def test_hosted_vllm_choice_question_goes_out_as_a_jev_choice_with_the_ser } +_PROVIDERS_WHOSE_CHAT_MODELS_ALSO_DECIDE: Final = frozenset( + {LlmProviders.OPENAI.value, LlmProviders.OPENROUTER.value, LlmProviders.HOSTED_VLLM.value} +) +_DEDICATED_DECIDER_PROVIDERS: Final[tuple[str, ...]] = tuple( + sorted( + provider.value + for provider in LlmProviders + if provider.value not in _PROVIDERS_WHOSE_CHAT_MODELS_ALSO_DECIDE + and ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None + ) +) + + +@pytest.mark.parametrize("provider", _DEDICATED_DECIDER_PROVIDERS) +def test_every_dedicated_decider_provider_ships_an_evaluation_mode_cost_map_entry(provider: str) -> None: + evaluation_entries: Final = tuple( + name + for name, info in litellm.model_cost.items() + if isinstance(info, Mapping) and info.get("litellm_provider") == provider and info.get("mode") == "evaluation" + ) + + assert evaluation_entries, f"a {provider} decider would be health-checked as chat without a mode: evaluation entry" + + @pytest.mark.asyncio @pytest.mark.parametrize( ("api_base", "url"), @@ -1086,7 +1112,7 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_ monkeypatch.delenv("OPENAI_API_BASE", raising=False) route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model="openai/gpt-6-luna", state="The package arrived broken.", questions={ @@ -1171,7 +1197,7 @@ def test_openai_decisions_use_the_same_settings_as_other_openai_calls( monkeypatch.setenv(name, value) route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE) - litellm.decisions( + litellm.systemone( model="openai/gpt-6-luna", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -1234,17 +1260,8 @@ async def test_openai_format_calls_to_a_systemone_provider_get_openai_format_ans }, "cannot serve this request", ), - ( - "openai/gpt-6-luna", - { - "state": "review", - "input": "review", - "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], - }, - "not both", - ), ), - ids=("image_to_systemone_provider", "state_and_input"), + ids=("image_to_systemone_provider",), ) async def test_requests_a_provider_cannot_serve_are_rejected_before_http( respx_mock: respx.MockRouter, @@ -1277,7 +1294,7 @@ async def test_openrouter_decisions_uses_provider_reported_cost_without_cost_map } ) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model="openrouter/typesafe/jev-1.13", state={"source": "unit-test"}, questions=_QUESTIONS, @@ -1292,24 +1309,27 @@ _SAFETY_IDENTIFIER: Final = "end-user-7" _PREDICATE_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = ( {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, ) -_PREDICATE_REQUESTS: Final[tuple[Mapping[str, object], ...]] = ( - MappingProxyType({"input": "review", "questions": _PREDICATE_QUESTIONS}), - MappingProxyType( - {"state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}} +_PREDICATE_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( + ("decisions", MappingProxyType({"input": "review", "questions": _PREDICATE_QUESTIONS})), + ( + "systemone", + MappingProxyType( + {"state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}} + ), ), ) -_PREDICATE_REQUEST_IDS: Final = ("input_format", "state_format") +_PREDICATE_REQUEST_IDS: Final = ("decisions", "systemone") -@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) def test_safety_identifier_is_refused_before_http_when_the_provider_cannot_take_it( - request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) with pytest.raises(litellm.UnsupportedParamsError, match=r"safety_identifier.*drop_params") as caught: - litellm.decisions( + getattr(litellm, api)( model="perplexity/pplx-decider-v1-27b", safety_identifier=_SAFETY_IDENTIFIER, api_key="caller-key", @@ -1350,10 +1370,11 @@ async def test_safety_identifier_is_dropped_from_the_wire_under_drop_params( @pytest.mark.asyncio -@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) @pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params")) async def test_safety_identifier_reaches_openai_whether_or_not_params_are_dropped( drop_params: bool, + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, @@ -1364,7 +1385,7 @@ async def test_safety_identifier_reaches_openai_whether_or_not_params_are_droppe monkeypatch.delenv("OPENAI_API_BASE", raising=False) route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) - await litellm.adecisions( + await getattr(litellm, f"a{api}")( model="openai/gpt-6-luna", safety_identifier=_SAFETY_IDENTIFIER, api_key="caller-key", @@ -1381,16 +1402,16 @@ async def test_safety_identifier_reaches_openai_whether_or_not_params_are_droppe @pytest.mark.asyncio -@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) async def test_non_string_safety_identifier_is_rejected_before_http( - request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) invalid_safety_identifier: Final[Mapping[str, object]] = MappingProxyType({"safety_identifier": 7}) with pytest.raises(litellm.BadRequestError, match=r"(?s)Invalid Decisions request.*safety_identifier"): - await litellm.adecisions( + await getattr(litellm, f"a{api}")( model="openai/gpt-6-luna", api_key="caller-key", **request_kwargs, **invalid_safety_identifier ) @@ -1398,9 +1419,9 @@ async def test_non_string_safety_identifier_is_rejected_before_http( @pytest.mark.asyncio -@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_params( - request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) monkeypatch.setattr(litellm, "api_base", None) @@ -1411,7 +1432,7 @@ async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_ {"safety_identifier": 7, "drop_params": True} ) - await litellm.adecisions( + await getattr(litellm, f"a{api}")( model="openai/gpt-6-luna", api_key="caller-key", **request_kwargs, **dropped_safety_identifier ) @@ -1448,7 +1469,7 @@ async def test_azure_ai_decision_posts_deployment_to_foundry_systemone_route( json={**_RESPONSE, "model": "microsoft-decision-1"} ) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model="azure_ai/decision-1", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -1480,7 +1501,7 @@ async def test_azure_ai_decision_reads_foundry_env_and_keeps_gateway_path_prefix json=_RESPONSE ) - await litellm.adecisions( + await litellm.asystemone( model="azure_ai/decision-1", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -1499,7 +1520,7 @@ async def test_azure_ai_decision_requires_api_base_before_http( monkeypatch.setenv("AZURE_AI_API_KEY", "foundry-key") with pytest.raises(litellm.BadRequestError, match="Missing AZURE_AI_API_BASE"): - await litellm.adecisions( + await litellm.asystemone( model="azure_ai/decision-1", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 5f254a3bc6c..7268748b7e9 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -471,6 +471,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "llm_passthrough_route", "asearch", "adecisions", + "asystemone", "avector_store_create", "avector_store_search", "acreate_skill", diff --git a/tests/unit/systemone/__init__.py b/tests/unit/systemone/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/systemone/test_main.py b/tests/unit/systemone/test_main.py new file mode 100644 index 00000000000..342b7f1ef97 --- /dev/null +++ b/tests/unit/systemone/test_main.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import pytest +import respx +from respx.models import Call + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse + +_STATE: Final = "Customer asked for a refund" +_SYSTEMONE_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( + {"is_refund": {"type": "noul", "instructions": "Is this a refund request?"}} +) +_OPENAI_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_refund", "instructions": "Is this a refund request?"}, +) +_SYSTEMONE_RESPONSE: Final[Mapping[str, object]] = { + "model": "jev-1.13", + "answers": {"is_refund": {"type": "noul", "noul": 0.93}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, +} +_TYPESAFE_URL: Final = "https://api.typesafe.ai/v1/systemone" + + +@pytest.fixture(autouse=True) +def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +class _CallTypeRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.call_types: list[str] = [] # mutable-ok: test recorder collects logger callbacks + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.call_types.append(kwargs["call_type"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", (True, False), ids=("asystemone", "systemone")) +async def test_systemone_sends_the_systemone_wire_format_and_returns_a_systemone_response( + use_async: bool, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + kwargs: Final[Mapping[str, object]] = MappingProxyType( + {"model": "typesafe/jev-1.13", "state": _STATE, "questions": _SYSTEMONE_QUESTIONS, "api_key": "caller-key"} + ) + + response: Final = await litellm.asystemone(**kwargs) if use_async else litellm.systemone(**kwargs) + + assert route.called + call: Final[Call] = respx_mock.calls.last + assert json.loads(call.request.content) == { + "model": "jev-1.13", + "state": _STATE, + "questions": {"is_refund": {"type": "noul", "instructions": "Is this a refund request?"}}, + } + assert isinstance(response, DecisionsResponse) + assert response.answers["is_refund"].model_dump(mode="json") == {"type": "noul", "noul": 0.93} + + +@pytest.mark.asyncio +async def test_systemone_rejects_openai_input_and_points_at_decisions(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=r"litellm\.decisions") as caught: + await litellm.asystemone( + model="typesafe/jev-1.13", input=_STATE, questions=_SYSTEMONE_QUESTIONS, api_key="caller-key" + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", (True, False), ids=("adecisions", "decisions")) +async def test_decisions_with_state_still_sends_the_systemone_format_and_warns( + use_async: bool, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + kwargs: Final[Mapping[str, object]] = MappingProxyType( + {"model": "typesafe/jev-1.13", "state": _STATE, "questions": _SYSTEMONE_QUESTIONS, "api_key": "caller-key"} + ) + + with pytest.warns(UserWarning, match=r"litellm\.systemone"): + response: Final = await litellm.adecisions(**kwargs) if use_async else litellm.decisions(**kwargs) + + assert route.called + call: Final[Call] = respx_mock.calls.last + assert json.loads(call.request.content) == { + "model": "jev-1.13", + "state": _STATE, + "questions": {"is_refund": {"type": "noul", "instructions": "Is this a refund request?"}}, + } + assert isinstance(response, DecisionsResponse) + assert response.answers["is_refund"].model_dump(mode="json") == {"type": "noul", "noul": 0.93} + + +@pytest.mark.asyncio +async def test_systemone_rejects_openai_question_lists_before_http(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): + await litellm.asystemone( + model="typesafe/jev-1.13", state=_STATE, questions=_OPENAI_QUESTIONS, api_key="caller-key" + ) + + assert not route.called + + +@pytest.mark.asyncio +async def test_router_asystemone_logs_the_systemone_call_type( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + recorder: Final = _CallTypeRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + router: Final = litellm.Router( + model_list=[{"model_name": "jev", "litellm_params": {"model": "typesafe/jev-1.13", "api_key": "caller-key"}}] + ) + + response: Final = await router.asystemone(model="jev", state=_STATE, questions=_SYSTEMONE_QUESTIONS) + await asyncio.sleep(0) + GLOBAL_LOGGING_WORKER.start() + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + assert isinstance(response, DecisionsResponse) + assert recorder.call_types == ["asystemone"] + assert not isinstance(response, OpenAIDecisionResponse) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 57cd9c62b68..4f971504e16 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -28090,7 +28090,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ From 212b42e932cd3a025119b552ef33aad56b191ec9 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 11 Oct 2026 00:53:53 +0000 Subject: [PATCH 17/18] chore(backport): regenerate dashboard API types and adapt the picks to rc/1.105.0 schema.d.ts is regenerated with npm run gen:api instead of hand-merged, the Strands logo the Add Model list imports is copied from main, and the picked tests get the import order and Final import this line's ruff asks for Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../basic/test_decisions_basic_cloudflare.py | 2 +- .../basic/test_decisions_basic_openrouter.py | 2 +- .../basic/test_decisions_basic_perplexity.py | 2 +- .../test_decisions_basic_strands_decider.py | 2 +- .../basic/test_decisions_basic_typesafe.py | 2 +- tests/unit/decisions/test_main.py | 4 +- tests/unit/proxy/auth/test_auth_utils.py | 6 +- tests/unit/test_cost_map_guard.py | 2 +- .../public/assets/logos/strands.svg | 4 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 1963 +++++++---------- 10 files changed, 869 insertions(+), 1120 deletions(-) create mode 100644 ui/litellm-dashboard/public/assets/logos/strands.svg diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py index 21e0538897d..7209c255efb 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py @@ -2,7 +2,7 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE, CLEF_SYSTEMONE_TEST_CASE +from integration.translation.decisions.bases.cloudflare import CLEF_SYSTEMONE_TEST_CASE, CLEF_TEST_CASE from integration.translation.runner import assert_translation diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py index 973526d7272..0e658c3ce32 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py @@ -3,8 +3,8 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase from integration.translation.decisions.bases.openrouter import ( - TYPESAFE_JEV_1_13_TEST_CASE, TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE, + TYPESAFE_JEV_1_13_TEST_CASE, ) from integration.translation.runner import assert_translation diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py index d9662ec4b9f..1b3cec8495d 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py @@ -3,8 +3,8 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase from integration.translation.decisions.bases.perplexity import ( - PPLX_DECIDER_V1_27B_TEST_CASE, PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE, + PPLX_DECIDER_V1_27B_TEST_CASE, ) from integration.translation.runner import assert_translation diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py index 2d2cd9caaf5..d13fd932961 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py @@ -3,8 +3,8 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase from integration.translation.decisions.bases.strands_decider import ( - STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE, STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE, + STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE, ) from integration.translation.runner import assert_translation diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py index c1111cc6ba9..79a017d6be5 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py @@ -2,7 +2,7 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE, JEV_1_13_0_SYSTEMONE_TEST_CASE +from integration.translation.decisions.bases.typesafe import JEV_1_13_0_SYSTEMONE_TEST_CASE, JEV_1_13_0_TEST_CASE from integration.translation.runner import assert_translation diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 35340646feb..8bb201a2750 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3,8 +3,8 @@ from __future__ import annotations import asyncio import json import threading -from concurrent.futures import wait from collections.abc import Mapping +from concurrent.futures import wait from types import MappingProxyType from typing import Final @@ -16,8 +16,8 @@ from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials import litellm -from litellm.cost_calculator import get_response_cost_from_hidden_params from litellm.constants import AWS_SIGNING_MAX_THREADS +from litellm.cost_calculator import get_response_cost_from_hidden_params from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 0b65007c153..aadb5881ce0 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -4,7 +4,7 @@ Unit tests for auth_utils functions related to rate limiting and customer ID ext import base64 import logging -from typing import Optional +from typing import Final, Optional from unittest.mock import MagicMock, patch import pytest @@ -16,8 +16,6 @@ from litellm.proxy.auth.auth_utils import ( abbreviate_api_key, check_complete_credentials, custom_auth_common_checks_warning, - log_once_if_budget_reservation_disabled, - warn_once_if_custom_auth_skips_common_checks, get_end_user_id_from_request_body, get_key_mcp_rpm_limit, get_key_model_rpm_limit, @@ -29,6 +27,8 @@ from litellm.proxy.auth.auth_utils import ( get_project_model_tpm_limit, get_request_route_template, is_request_body_safe, + log_once_if_budget_reservation_disabled, + warn_once_if_custom_auth_skips_common_checks, ) diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index e127aa08d29..e2c54d381d8 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -1,8 +1,8 @@ -from collections.abc import Mapping import importlib.util import json import subprocess import sys +from collections.abc import Mapping from pathlib import Path from types import ModuleType from typing import Final diff --git a/ui/litellm-dashboard/public/assets/logos/strands.svg b/ui/litellm-dashboard/public/assets/logos/strands.svg new file mode 100644 index 00000000000..466fb64465e --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/strands.svg @@ -0,0 +1,4 @@ + + + + diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 4f971504e16..f469fbac3fd 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1189,60 +1189,6 @@ export interface paths { patch?: never; trace?: never; }; - "/audit": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get Audit Logs - * @description Get all audit logs with filtering and pagination. - * - * Returns a paginated response of audit logs matching the specified filters. - * - * Note: object_team_id and object_key_hash use Prisma JSON path filtering, - * which requires PostgreSQL. - */ - get: operations["get_audit_logs_audit_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/audit/{id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get Audit Log By Id - * @description Get detailed information about a specific audit log entry by its ID. - * - * Args: - * id (str): The unique identifier of the audit log entry - * - * Returns: - * AuditLogResponse: Detailed information about the audit log entry - * - * Raises: - * HTTPException: If the audit log is not found or if there's a database connection error - */ - get: operations["get_audit_log_by_id_audit__id__get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/authorize": { parameters: { query?: never; @@ -4577,7 +4523,18 @@ export interface paths { }; get?: never; put?: never; - /** Decisions */ + /** + * Ask questions about an input (OpenAI Decisions format) + * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * + * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + * `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + * rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + * model list. System One decision models accept this format too, LiteLLM translates the request and the + * answers. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["decisions_decisions_post"]; delete?: never; options?: never; @@ -4602,50 +4559,6 @@ export interface paths { patch?: never; trace?: never; }; - "/email/event_settings": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get Email Event Settings - * @description Get all email event settings - */ - get: operations["get_email_event_settings_email_event_settings_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - /** - * Update Event Settings - * @description Update the settings for email events - */ - patch: operations["update_event_settings_email_event_settings_patch"]; - trace?: never; - }; - "/email/event_settings/reset": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** - * Reset Event Settings - * @description Reset all email event settings to default (new user invitations on, virtual key creation off) - */ - post: operations["reset_event_settings_email_event_settings_reset_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/embeddings": { parameters: { query?: never; @@ -12729,240 +12642,6 @@ export interface paths { patch?: never; trace?: never; }; - "/project/delete": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - post?: never; - /** - * Delete Project - * @description Delete projects - * - * Parameters: - * - project_ids: *List[str]* - List of project ids to delete - * - * Example: - * ```bash - * curl --location --request DELETE 'http://0.0.0.0:4000/project/delete' \ - * --header 'Authorization: Bearer sk-1234' \ - * --header 'Content-Type: application/json' \ - * --data '{ - * "project_ids": ["project-123", "project-456"] - * }' - * ``` - */ - delete: operations["delete_project_project_delete_delete"]; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/project/info": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Project Info - * @description Get information about a specific project - * - * Parameters: - * - project_id: *str* - The project id to fetch info for - * - * Example: - * ```bash - * curl --location 'http://0.0.0.0:4000/project/info?project_id=project-123' \ - * --header 'Authorization: Bearer sk-1234' - * ``` - */ - get: operations["project_info_project_info_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/project/list": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * List Projects - * @description List all projects that the user has access to - * - * Example: - * ```bash - * curl --location 'http://0.0.0.0:4000/project/list' \ - * --header 'Authorization: Bearer sk-1234' - * ``` - */ - get: operations["list_projects_project_list_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/project/new": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** - * New Project - * @description Create a new project. Projects sit between teams and keys in the hierarchy. - * - * Only admins or team admins can create projects. - * - * # Parameters - * - * - project_alias: *Optional[str]* - The name of the project. - * - description: *Optional[str]* - Description of the project's purpose and use case. - * - team_id: *str* - The team id that this project belongs to. Required. - * - models: *List* - The models the project has access to. - * - budget_id: *Optional[str]* - The id for a budget (tpm/rpm/max budget) for the project. - * ### IF NO BUDGET ID - CREATE ONE WITH THESE PARAMS ### - * - max_budget: *Optional[float]* - Max budget for project - * - tpm_limit: *Optional[int]* - Max tpm limit for project - * - rpm_limit: *Optional[int]* - Max rpm limit for project - * - max_parallel_requests: *Optional[int]* - Max parallel requests for project - * - soft_budget: *Optional[float]* - Get a slack alert when this soft budget is reached. Don't block requests. - * - model_max_budget: *Optional[dict]* - Max budget for a specific model. Example: {"gpt-4": 100.0, "gpt-3.5-turbo": 50.0} - * - model_rpm_limit: *Optional[dict]* - RPM limits per model. Example: {"gpt-4": 1000, "gpt-3.5-turbo": 5000} - * - model_tpm_limit: *Optional[dict]* - TPM limits per model. Example: {"gpt-4": 50000, "gpt-3.5-turbo": 100000} - * - budget_duration: *Optional[str]* - Frequency of reseting project budget - * - metadata: *Optional[dict]* - Metadata for project, store information for project. Example metadata - {"use_case_id": "SNOW-12345", "responsible_ai_id": "RAI-67890"} - * - tags: *Optional[list]* - Tags for the project. Example: ["production", "api"] - * - blocked: *bool* - Flag indicating if the project is blocked or not - will stop all calls from keys with this project_id. - * - object_permission: Optional[LiteLLM_ObjectPermissionBase] - project-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"]}. IF null or {} then no object permission. - * - * Example 1: Create new project **without** a budget_id, with model-specific limits - * - * ```bash - * curl --location 'http://0.0.0.0:4000/project/new' \ - * --header 'Authorization: Bearer sk-1234' \ - * --header 'Content-Type: application/json' \ - * --data '{ - * "project_alias": "flight-search-assistant", - * "description": "AI-powered flight search and booking assistant", - * "team_id": "team-123", - * "models": ["gpt-4", "gpt-3.5-turbo"], - * "max_budget": 100, - * "model_rpm_limit": { - * "gpt-4": 1000, - * "gpt-3.5-turbo": 5000 - * }, - * "model_tpm_limit": { - * "gpt-4": 50000, - * "gpt-3.5-turbo": 100000 - * }, - * "metadata": { - * "use_case_id": "SNOW-12345", - * "responsible_ai_id": "RAI-67890" - * } - * }' - * ``` - * - * Example 2: Create new project **with** a budget_id - * - * ```bash - * curl --location 'http://0.0.0.0:4000/project/new' \ - * --header 'Authorization: Bearer sk-1234' \ - * --header 'Content-Type: application/json' \ - * --data '{ - * "project_alias": "hotel-recommendations", - * "description": "Personalized hotel recommendation engine", - * "team_id": "team-123", - * "models": ["claude-3-sonnet"], - * "budget_id": "428eeaa8-f3ac-4e85-a8fb-7dc8d7aa8689", - * "metadata": { - * "use_case_id": "SNOW-54321" - * } - * }' - * ``` - */ - post: operations["new_project_project_new_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/project/update": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** - * Update Project - * @description Update a project - * - * Parameters: - * - project_id: *str* - The project id to update. Required. - * - project_alias: *Optional[str]* - Updated name for the project - * - description: *Optional[str]* - Updated description for the project - * - team_id: *Optional[str]* - Updated team_id for the project - * - metadata: *Optional[dict]* - Updated metadata for project - * - models: *Optional[list]* - Updated list of models for the project - * - blocked: *Optional[bool]* - Updated blocked status - * - max_budget: *Optional[float]* - Updated max budget - * - tpm_limit: *Optional[int]* - Updated tpm limit - * - rpm_limit: *Optional[int]* - Updated rpm limit - * - model_rpm_limit: *Optional[dict]* - Updated RPM limits per model - * - model_tpm_limit: *Optional[dict]* - Updated TPM limits per model - * - budget_duration: *Optional[str]* - Updated budget duration - * - tags: *Optional[list]* - Updated list of tags for the project - * - object_permission: Optional[LiteLLM_ObjectPermissionBase] - Updated object permission - * - * Example: - * ```bash - * curl --location 'http://0.0.0.0:4000/project/update' \ - * --header 'Authorization: Bearer sk-1234' \ - * --header 'Content-Type: application/json' \ - * --data '{ - * "project_id": "project-123", - * "description": "Updated flight search system with enhanced capabilities", - * "max_budget": 200, - * "model_rpm_limit": { - * "gpt-4": 2000, - * "gpt-3.5-turbo": 10000 - * }, - * "metadata": { - * "use_case_id": "SNOW-12345", - * "status": "active" - * } - * }' - * ``` - */ - post: operations["update_project_project_update_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/prompts": { parameters: { query?: never; @@ -14219,27 +13898,6 @@ export interface paths { patch?: never; trace?: never; }; - "/robots.txt": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get Robots - * @description Block all web crawlers from indexing the proxy server endpoints - * This is useful for ensuring that the API endpoints aren't indexed by search engines - */ - get: operations["get_robots_robots_txt_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/roi-calculator/connections/test": { parameters: { query?: never; @@ -16015,7 +15673,18 @@ export interface paths { }; get?: never; put?: never; - /** Systemone */ + /** + * Ask named questions about a state (System One format) + * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + * + * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + * `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + * `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + * is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + * too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["systemone_systemone_post"]; delete?: never; options?: never; @@ -18233,26 +17902,6 @@ export interface paths { patch?: never; trace?: never; }; - "/user/available_users": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Available Enterprise Users - * @description For keys with `max_users` set, return the list of users that are allowed to use the key. - */ - get: operations["available_enterprise_users_user_available_users_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/user/bulk_update": { parameters: { query?: never; @@ -19798,7 +19447,18 @@ export interface paths { }; get?: never; put?: never; - /** Decisions */ + /** + * Ask questions about an input (OpenAI Decisions format) + * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * + * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + * `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + * rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + * model list. System One decision models accept this format too, LiteLLM translates the request and the + * answers. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["decisions_v1_decisions_post"]; delete?: never; options?: never; @@ -22121,7 +21781,18 @@ export interface paths { }; get?: never; put?: never; - /** Systemone */ + /** + * Ask named questions about a state (System One format) + * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + * + * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + * `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + * `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + * is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + * too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["systemone_v1_systemone_post"]; delete?: never; options?: never; @@ -26147,37 +25818,6 @@ export interface components { */ unnamed_teams_count: number; }; - /** - * AuditLogResponse - * @description Response model for a single audit log entry - */ - AuditLogResponse: { - /** Action */ - action: string; - /** Before Value */ - before_value?: { - [key: string]: unknown; - } | null; - /** Changed By */ - changed_by: string; - /** Changed By Api Key */ - changed_by_api_key: string; - /** Id */ - id: string; - /** Object Id */ - object_id: string; - /** Table Name */ - table_name: string; - /** - * Updated At - * Format: date-time - */ - updated_at: string; - /** Updated Values */ - updated_values?: { - [key: string]: unknown; - } | null; - }; /** AutoRouterAllowance */ AutoRouterAllowance: { /** @@ -28723,6 +28363,24 @@ export interface components { /** Instruction */ instruction: string; }; + /** ChoiceAnswer */ + ChoiceAnswer: { + /** Choice */ + choice: string; + /** Confidence */ + confidence: number; + /** Probabilities */ + probabilities: { + [key: string]: number; + }; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "choice"; + } & { + [key: string]: unknown; + }; /** ChoiceLogprobs */ ChoiceLogprobs: { /** Content */ @@ -31068,6 +30726,33 @@ export interface components { */ total_tokens: number; }; + /** DecisionsResponse */ + DecisionsResponse: { + /** Answers */ + answers: { + [key: string]: components["schemas"]["NoulAnswer"] | components["schemas"]["ChoiceAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + /** Model */ + model?: string | null; + usage?: components["schemas"]["DecisionsUsage"] | null; + } & { + [key: string]: unknown; + }; + /** DecisionsUsage */ + DecisionsUsage: { + /** + * Input Tokens + * @default 0 + */ + input_tokens: number; + /** + * Output Tokens + * @default 0 + */ + output_tokens: number; + } & { + [key: string]: unknown; + }; /** * DefaultInternalUserParams * @description Default parameters to apply when a new user signs in via SSO or is created on the /user/new API endpoint @@ -31215,14 +30900,6 @@ export interface components { /** Organization Ids */ organization_ids: string[]; }; - /** - * DeleteProjectRequest - * @description Request model for DELETE /project/delete - */ - DeleteProjectRequest: { - /** Project Ids */ - project_ids: string[]; - }; /** * DeleteResponseResult * @description Result of a delete response request @@ -31466,27 +31143,6 @@ export interface components { */ type?: "message"; }; - /** - * EmailEvent - * @enum {string} - */ - EmailEvent: "Virtual Key Created" | "New User Invitation" | "Virtual Key Rotated" | "Soft Budget Crossed" | "Max Budget Alert"; - /** EmailEventSettings */ - EmailEventSettings: { - /** Enabled */ - enabled: boolean; - event: components["schemas"]["EmailEvent"]; - }; - /** EmailEventSettingsResponse */ - EmailEventSettingsResponse: { - /** Settings */ - settings: components["schemas"]["EmailEventSettings"][]; - }; - /** EmailEventSettingsUpdateRequest */ - EmailEventSettingsUpdateRequest: { - /** Settings */ - settings: components["schemas"]["EmailEventSettings"][]; - }; /** EmbeddingRequest */ EmbeddingRequest: { /** @@ -35282,65 +34938,6 @@ export interface components { } & { [key: string]: unknown; }; - /** - * LiteLLM_ProjectTable - * @description Database model representation for project - */ - LiteLLM_ProjectTable: { - /** - * Blocked - * @default false - */ - blocked: boolean; - /** Budget Id */ - budget_id?: string | null; - /** Created At */ - created_at?: string | null; - /** Created By */ - created_by?: string | null; - /** Description */ - description?: string | null; - litellm_budget_table?: components["schemas"]["LiteLLM_BudgetTable"] | null; - /** Metadata */ - metadata?: { - [key: string]: unknown; - } | null; - /** Model Rpm Limit */ - model_rpm_limit?: { - [key: string]: unknown; - } | null; - /** Model Spend */ - model_spend?: { - [key: string]: unknown; - } | null; - /** Model Tpm Limit */ - model_tpm_limit?: { - [key: string]: unknown; - } | null; - /** - * Models - * @default [] - */ - models: string[]; - object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null; - /** Object Permission Id */ - object_permission_id?: string | null; - /** Project Alias */ - project_alias?: string | null; - /** Project Id */ - project_id: string; - /** - * Spend - * @default 0 - */ - spend: number; - /** Team Id */ - team_id?: string | null; - /** Updated At */ - updated_at?: string | null; - /** Updated By */ - updated_by?: string | null; - }; /** LiteLLM_ProxyModelTable */ LiteLLM_ProxyModelTable: { /** @@ -38566,148 +38163,6 @@ export interface components { /** Users */ users?: components["schemas"]["LiteLLM_UserTable"][] | null; }; - /** - * NewProjectRequest - * @description Request model for POST /project/new - */ - NewProjectRequest: { - /** Allowed Models */ - allowed_models?: string[] | null; - /** - * Blocked - * @default false - */ - blocked: boolean; - /** Budget Duration */ - budget_duration?: string | null; - /** Budget Id */ - budget_id?: string | null; - /** Description */ - description?: string | null; - /** Guardrails */ - guardrails?: string[] | null; - /** Max Budget */ - max_budget?: number | null; - /** Max Parallel Requests */ - max_parallel_requests?: number | null; - /** Metadata */ - metadata?: { - [key: string]: unknown; - } | null; - /** Model Itpm Limit */ - model_itpm_limit?: { - [key: string]: number; - } | null; - /** Model Max Budget */ - model_max_budget?: { - [key: string]: unknown; - } | null; - /** Model Otpm Limit */ - model_otpm_limit?: { - [key: string]: number; - } | null; - /** Model Rpm Limit */ - model_rpm_limit?: { - [key: string]: unknown; - } | null; - /** Model Tpm Limit */ - model_tpm_limit?: { - [key: string]: unknown; - } | null; - /** - * Models - * @default [] - */ - models: string[]; - object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null; - /** Policies */ - policies?: string[] | null; - /** Project Alias */ - project_alias?: string | null; - /** Project Id */ - project_id?: string | null; - /** Rpm Limit */ - rpm_limit?: number | null; - /** Soft Budget */ - soft_budget?: number | null; - /** Tags */ - tags?: string[] | null; - /** Team Id */ - team_id: string; - /** Temp Budget Expiry */ - temp_budget_expiry?: string | null; - /** Temp Budget Increase */ - temp_budget_increase?: number | null; - /** Tpd Limit */ - tpd_limit?: number | null; - /** Tpm Limit */ - tpm_limit?: number | null; - }; - /** - * NewProjectResponse - * @description Response model for POST /project/new - */ - NewProjectResponse: { - /** - * Blocked - * @default false - */ - blocked: boolean; - /** Budget Id */ - budget_id?: string | null; - /** - * Created At - * Format: date-time - */ - created_at: string; - /** Created By */ - created_by?: string | null; - /** Description */ - description?: string | null; - litellm_budget_table?: components["schemas"]["LiteLLM_BudgetTable"] | null; - /** Metadata */ - metadata?: { - [key: string]: unknown; - } | null; - /** Model Rpm Limit */ - model_rpm_limit?: { - [key: string]: unknown; - } | null; - /** Model Spend */ - model_spend?: { - [key: string]: unknown; - } | null; - /** Model Tpm Limit */ - model_tpm_limit?: { - [key: string]: unknown; - } | null; - /** - * Models - * @default [] - */ - models: string[]; - object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null; - /** Object Permission Id */ - object_permission_id?: string | null; - /** Project Alias */ - project_alias?: string | null; - /** Project Id */ - project_id: string; - /** - * Spend - * @default 0 - */ - spend: number; - /** Team Id */ - team_id?: string | null; - /** - * Updated At - * Format: date-time - */ - updated_at: string; - /** Updated By */ - updated_by?: string | null; - }; /** NewTeamRequest */ NewTeamRequest: { /** Access Group Ids */ @@ -39149,6 +38604,18 @@ export interface components { /** User Role */ user_role?: ("proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer") | null; }; + /** NoulAnswer */ + NoulAnswer: { + /** Noul */ + noul: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "noul"; + } & { + [key: string]: unknown; + }; /** * OAuth2SecurityScheme * @description Defines a security scheme using OAuth 2.0. @@ -39187,6 +38654,129 @@ export interface components { [key: string]: unknown; } | null; }; + /** OpenAIChoiceAnswer */ + OpenAIChoiceAnswer: { + /** Choice */ + choice: string | boolean; + /** Confidence */ + confidence: number; + /** Name */ + name: string | null; + /** Probabilities */ + probabilities: components["schemas"]["OpenAIChoiceProbability"][]; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "choice"; + }; + /** OpenAIChoiceProbability */ + OpenAIChoiceProbability: { + /** Probability */ + probability: number; + /** Value */ + value: string | boolean; + }; + /** OpenAIDecisionInputTokensDetails */ + OpenAIDecisionInputTokensDetails: { + /** + * Cache Write Tokens + * @default 0 + */ + cache_write_tokens: number; + /** + * Cached Tokens + * @default 0 + */ + cached_tokens: number; + }; + /** OpenAIDecisionOutputTokensDetails */ + OpenAIDecisionOutputTokensDetails: { + /** + * Reasoning Tokens + * @default 0 + */ + reasoning_tokens: number; + }; + /** OpenAIDecisionResponse */ + OpenAIDecisionResponse: { + /** Answers */ + answers: (components["schemas"]["OpenAIPredicateAnswer"] | components["schemas"]["OpenAIChoiceAnswer"] | components["schemas"]["OpenAIScoreAnswer"] | components["schemas"]["OpenAIRefusalAnswer"])[]; + /** Model */ + model: string; + usage: components["schemas"]["OpenAIDecisionUsage"]; + } & { + [key: string]: unknown; + }; + /** OpenAIDecisionUsage */ + OpenAIDecisionUsage: { + /** Input Tokens */ + input_tokens: number; + /** + * @default { + * "cache_write_tokens": 0, + * "cached_tokens": 0 + * } + */ + input_tokens_details: components["schemas"]["OpenAIDecisionInputTokensDetails"]; + /** Output Tokens */ + output_tokens: number; + /** + * @default { + * "reasoning_tokens": 0 + * } + */ + output_tokens_details: components["schemas"]["OpenAIDecisionOutputTokensDetails"]; + /** Total Tokens */ + total_tokens: number; + }; + /** OpenAIPredicateAnswer */ + OpenAIPredicateAnswer: { + /** Name */ + name: string | null; + /** Probability */ + probability: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "predicate"; + }; + /** OpenAIRefusalAnswer */ + OpenAIRefusalAnswer: { + /** Name */ + name: string | null; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "refusal"; + }; + /** OpenAIScoreAnswer */ + OpenAIScoreAnswer: { + /** Confidence */ + confidence: number; + /** Name */ + name: string | null; + /** Probabilities */ + probabilities: components["schemas"]["OpenAIScoreProbability"][]; + /** Score */ + score: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "score"; + }; + /** OpenAIScoreProbability */ + OpenAIScoreProbability: { + /** Label */ + label: string; + /** Probability */ + probability: number; + /** Value */ + value: number; + }; /** * OpenIdConnectSecurityScheme * @description Defines a security scheme using OpenID Connect. @@ -39583,34 +39173,6 @@ export interface components { /** Page Size */ page_size: number; }; - /** - * PaginatedAuditLogResponse - * @description Response model for paginated audit logs - */ - PaginatedAuditLogResponse: { - /** Audit Logs */ - audit_logs: components["schemas"]["AuditLogResponse"][]; - /** - * Page - * @description Current page number - */ - page: number; - /** - * Page Size - * @description Number of items per page - */ - page_size: number; - /** - * Total - * @description Total number of audit logs matching the filters - */ - total: number; - /** - * Total Pages - * @description Total number of pages - */ - total_pages: number; - }; /** PassThroughEndpointResponse */ PassThroughEndpointResponse: { /** Endpoints */ @@ -44521,6 +44083,30 @@ export interface components { */ team_id: string; }; + /** ScoreAnswer */ + ScoreAnswer: { + /** Confidence */ + confidence: number; + /** Legend */ + legend: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[]; + }; + /** Probabilities */ + probabilities: { + [key: string]: number; + }; + /** Score */ + score: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "score"; + } & { + [key: string]: unknown; + }; /** * Screenshot * @description A screenshot action. @@ -47587,77 +47173,6 @@ export interface components { */ version?: string | null; }; - /** - * UpdateProjectRequest - * @description Request model for POST /project/update - */ - UpdateProjectRequest: { - /** Allowed Models */ - allowed_models?: string[] | null; - /** Blocked */ - blocked?: boolean | null; - /** Budget Duration */ - budget_duration?: string | null; - /** Budget Id */ - budget_id?: string | null; - /** Description */ - description?: string | null; - /** Guardrails */ - guardrails?: string[] | null; - /** Max Budget */ - max_budget?: number | null; - /** Max Parallel Requests */ - max_parallel_requests?: number | null; - /** Metadata */ - metadata?: { - [key: string]: unknown; - } | null; - /** Model Itpm Limit */ - model_itpm_limit?: { - [key: string]: number; - } | null; - /** Model Max Budget */ - model_max_budget?: { - [key: string]: unknown; - } | null; - /** Model Otpm Limit */ - model_otpm_limit?: { - [key: string]: number; - } | null; - /** Model Rpm Limit */ - model_rpm_limit?: { - [key: string]: unknown; - } | null; - /** Model Tpm Limit */ - model_tpm_limit?: { - [key: string]: unknown; - } | null; - /** Models */ - models?: string[] | null; - object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null; - /** Policies */ - policies?: string[] | null; - /** Project Alias */ - project_alias?: string | null; - /** Project Id */ - project_id: string; - /** Rpm Limit */ - rpm_limit?: number | null; - /** Soft Budget */ - soft_budget?: number | null; - /** Tags */ - tags?: string[] | null; - /** Team Id */ - team_id?: string | null; - /** Temp Budget Expiry */ - temp_budget_expiry?: string | null; - /** Temp Budget Increase */ - temp_budget_increase?: number | null; - /** Tpd Limit */ - tpd_limit?: number | null; - /** Tpm Limit */ - tpm_limit?: number | null; - }; /** * UpdatePublicModelGroupsRequest * @description Request model for updating public model groups @@ -51514,107 +51029,6 @@ export interface operations { }; }; }; - get_audit_logs_audit_get: { - parameters: { - query?: { - page?: number; - page_size?: number; - /** @description Filter by user or system that performed the action */ - changed_by?: string | null; - /** @description Filter by API key hash that performed the action */ - changed_by_api_key?: string | null; - /** @description Filter by action type (create, update, delete) */ - action?: string | null; - /** @description Filter by table name that was modified */ - table_name?: string | null; - /** @description Filter by ID of the object that was modified */ - object_id?: string | null; - /** @description Filter logs after this date */ - start_date?: string | null; - /** @description Filter logs before this date */ - end_date?: string | null; - /** @description Filter by team_id present in before_value or updated_values JSON (PostgreSQL only) */ - object_team_id?: string | null; - /** @description Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only) */ - object_key_hash?: string | null; - /** @description Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value */ - search?: string | null; - /** @description Column to sort by (e.g. 'updated_at', 'action', 'table_name') */ - sort_by?: string | null; - /** @description Sort order ('asc' or 'desc') */ - sort_order?: string; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["PaginatedAuditLogResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - get_audit_log_by_id_audit__id__get: { - parameters: { - query?: never; - header?: never; - path: { - id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["AuditLogResponse"]; - }; - }; - /** @description Audit log not found */ - 404: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - /** @description Database connection error */ - 500: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - }; - }; authorize_authorize_get: { parameters: { query: { @@ -56518,7 +55932,179 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + * "model": "luna", + * "questions": [ + * { + * "instructions": "Is the customer asking for a refund?", + * "name": "is_refund_request", + * "type": "predicate" + * }, + * { + * "choices": [ + * { + * "description": "can wait a week", + * "value": "low" + * }, + * { + * "description": "needs action today", + * "value": "high" + * } + * ], + * "instructions": "How urgent is this?", + * "name": "urgency", + * "type": "choice" + * }, + * { + * "instructions": "How frustrated is the customer?", + * "levels": [ + * { + * "label": "calm" + * }, + * { + * "label": "mildly annoyed" + * }, + * { + * "label": "angry" + * } + * ], + * "name": "frustration", + * "type": "score" + * } + * ] + * } + */ + "application/json": { + /** + * Input + * @description The text being judged, or user messages whose content mixes input_text and input_image parts + */ + input: string | { + /** Content */ + content: string | ({ + /** Text */ + text: string; + /** + * Type + * @constant + */ + type: "input_text"; + } | { + /** Detail */ + detail?: string | null; + /** Image Url */ + image_url: string; + /** + * Type + * @constant + */ + type: "input_image"; + })[]; + /** + * Role + * @default user + * @constant + */ + role?: "user"; + /** + * Type + * @default message + * @constant + */ + type?: "message"; + }[]; + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Predicate, choice and score questions. Answers come back in the same order + */ + questions: ({ + /** + * Instructions + * @description The yes/no question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "predicate"; + } | { + /** + * Choices + * @description The options the model picks from. Values must be unique + */ + choices: { + /** Description */ + description?: string | null; + /** Value */ + value: string | boolean; + }[]; + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Levels + * @description Scale levels from lowest to highest. The score is a level index + */ + levels: { + /** Description */ + description?: string | null; + /** Label */ + label: string; + }[]; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "score"; + })[]; + /** + * Safety Identifier + * @description A stable id for your end user. Sent to OpenAI and dropped for other providers + */ + safety_identifier?: string | null; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -56526,7 +56112,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["OpenAIDecisionResponse"]; }; }; }; @@ -56564,79 +56150,6 @@ export interface operations { }; }; }; - get_email_event_settings_email_event_settings_get: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["EmailEventSettingsResponse"]; - }; - }; - }; - }; - update_event_settings_email_event_settings_patch: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["EmailEventSettingsUpdateRequest"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - reset_event_settings_email_event_settings_reset_post: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - }; - }; - }; - }; embeddings_embeddings_post: { parameters: { query?: never; @@ -67523,156 +67036,6 @@ export interface operations { }; }; }; - delete_project_project_delete_delete: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["DeleteProjectRequest"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["LiteLLM_ProjectTable"][]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - project_info_project_info_get: { - parameters: { - query: { - project_id: string; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["LiteLLM_ProjectTable"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - list_projects_project_list_get: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["LiteLLM_ProjectTable"][]; - }; - }; - }; - }; - new_project_project_new_post: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["NewProjectRequest"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["NewProjectResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - update_project_project_update_post: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["UpdateProjectRequest"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["LiteLLM_ProjectTable"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; create_prompt_prompts_post: { parameters: { query?: never; @@ -68865,26 +68228,6 @@ export interface operations { }; }; }; - get_robots_robots_txt_get: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - }; - }; - }; - }; test_roi_calculator_connections_roi_calculator_connections_test_post: { parameters: { query?: never; @@ -71092,7 +70435,132 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "jev", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "urgency": { + * "criteria": { + * "high": "needs action today", + * "low": "can wait a week" + * }, + * "instructions": "How urgent is this?", + * "type": "choice" + * } + * }, + * "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + * } + */ + "application/json": { + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Named noul, choice and score questions. Each key becomes a key in the answers + */ + questions: { + [key: string]: ({ + /** + * Criteria + * @description Optional descriptions of what makes the answer true and what makes it false + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description The yes/no question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Candidate labels mapped to an optional description of each + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Scale levels from lowest to highest. The score is a level index + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[])[]; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + } & { + [key: string]: unknown; + }); + }; + /** + * State + * @description The thing being judged, such as a support ticket, a document or a chat transcript + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -71100,7 +70568,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["DecisionsResponse"]; }; }; }; @@ -74010,26 +73478,6 @@ export interface operations { }; }; }; - available_enterprise_users_user_available_users_get: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - }; - }; - }; - }; bulk_user_update_user_bulk_update_post: { parameters: { query?: never; @@ -76213,7 +75661,179 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + * "model": "luna", + * "questions": [ + * { + * "instructions": "Is the customer asking for a refund?", + * "name": "is_refund_request", + * "type": "predicate" + * }, + * { + * "choices": [ + * { + * "description": "can wait a week", + * "value": "low" + * }, + * { + * "description": "needs action today", + * "value": "high" + * } + * ], + * "instructions": "How urgent is this?", + * "name": "urgency", + * "type": "choice" + * }, + * { + * "instructions": "How frustrated is the customer?", + * "levels": [ + * { + * "label": "calm" + * }, + * { + * "label": "mildly annoyed" + * }, + * { + * "label": "angry" + * } + * ], + * "name": "frustration", + * "type": "score" + * } + * ] + * } + */ + "application/json": { + /** + * Input + * @description The text being judged, or user messages whose content mixes input_text and input_image parts + */ + input: string | { + /** Content */ + content: string | ({ + /** Text */ + text: string; + /** + * Type + * @constant + */ + type: "input_text"; + } | { + /** Detail */ + detail?: string | null; + /** Image Url */ + image_url: string; + /** + * Type + * @constant + */ + type: "input_image"; + })[]; + /** + * Role + * @default user + * @constant + */ + role?: "user"; + /** + * Type + * @default message + * @constant + */ + type?: "message"; + }[]; + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Predicate, choice and score questions. Answers come back in the same order + */ + questions: ({ + /** + * Instructions + * @description The yes/no question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "predicate"; + } | { + /** + * Choices + * @description The options the model picks from. Values must be unique + */ + choices: { + /** Description */ + description?: string | null; + /** Value */ + value: string | boolean; + }[]; + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Levels + * @description Scale levels from lowest to highest. The score is a level index + */ + levels: { + /** Description */ + description?: string | null; + /** Label */ + label: string; + }[]; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "score"; + })[]; + /** + * Safety Identifier + * @description A stable id for your end user. Sent to OpenAI and dropped for other providers + */ + safety_identifier?: string | null; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -76221,7 +75841,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["OpenAIDecisionResponse"]; }; }; }; @@ -79365,7 +78985,132 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "jev", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "urgency": { + * "criteria": { + * "high": "needs action today", + * "low": "can wait a week" + * }, + * "instructions": "How urgent is this?", + * "type": "choice" + * } + * }, + * "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + * } + */ + "application/json": { + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Named noul, choice and score questions. Each key becomes a key in the answers + */ + questions: { + [key: string]: ({ + /** + * Criteria + * @description Optional descriptions of what makes the answer true and what makes it false + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description The yes/no question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Candidate labels mapped to an optional description of each + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Scale levels from lowest to highest. The score is a level index + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[])[]; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + } & { + [key: string]: unknown; + }); + }; + /** + * State + * @description The thing being judged, such as a support ticket, a document or a chat transcript + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -79373,7 +79118,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["DecisionsResponse"]; }; }; }; From 23fec69be297bd68c43ecbb66f525b8c635d220e Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 11 Oct 2026 00:59:15 +0000 Subject: [PATCH 18/18] test(unit): flush queued logging tasks after each test so systemone callback checks stay isolated Ports the pytest_runtest_call hook from tests/unit/conftest.py on main (#44146). Without it the asystemone call type test picks up adecisions callbacks queued by earlier tests in the same worker Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/conftest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 2578cb7d78a..2e1c67c3d91 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -45,6 +45,7 @@ import litellm.router as litellm_router_module # noqa: E402 # same import-time import litellm.utils as litellm_utils_module # noqa: E402 # same import-time dependency from litellm._logging import ALL_LOGGERS # noqa: E402 # same import-time dependency from litellm.anthropic_beta_headers_manager import reload_beta_headers_config # noqa: E402 # same import-time dependency +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER # noqa: E402 # same import-time dependency from litellm.litellm_core_utils.prompt_templates import factory as prompt_factory_module # noqa: E402 # same import-time dependency from litellm.litellm_core_utils.prompt_templates import ( # noqa: E402 # same import-time dependency image_handling as image_handling_module, @@ -157,6 +158,31 @@ def _run_coroutine_if_needed(result: object) -> None: loop.create_task(coroutine) +async def _complete_test_logging() -> None: + await GLOBAL_LOGGING_WORKER.flush() + await GLOBAL_LOGGING_WORKER.stop() + + +def _flush_completed_test_logging() -> None: + queue: Final = GLOBAL_LOGGING_WORKER._queue + if queue is None or queue._unfinished_tasks == 0: + return + worker_loop: Final = GLOBAL_LOGGING_WORKER._bound_loop + if worker_loop is None or worker_loop.is_closed(): + _run_coroutine_if_needed(_complete_test_logging()) + return + if worker_loop.is_running(): + asyncio.run_coroutine_threadsafe(_complete_test_logging(), worker_loop).result() + return + worker_loop.run_until_complete(_complete_test_logging()) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_call() -> Iterator[None]: + yield + _flush_completed_test_logging() + + def _close_handler_if_needed(handler: object) -> None: close: Final = getattr(handler, "close", None) if not callable(close):