mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
feat(decisions): serve System One format at /v1/systemone and OpenAI format at /v1/decisions
The System One request format moves to /v1/systemone and /systemone. /v1/decisions and /decisions now accept the OpenAI Decisions API format, translate it into a System One request, route it through the same pipeline, and translate the answers back.
This commit is contained in:
parent
df23f11e9b
commit
3339469e73
16 changed files with 696 additions and 58 deletions
|
|
@ -62,6 +62,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
||||||
"/rerank",
|
"/rerank",
|
||||||
"/v1/decisions",
|
"/v1/decisions",
|
||||||
"/decisions",
|
"/decisions",
|
||||||
|
"/v1/systemone",
|
||||||
|
"/systemone",
|
||||||
"/v1/ocr",
|
"/v1/ocr",
|
||||||
"/ocr",
|
"/ocr",
|
||||||
"/v1/rag/",
|
"/v1/rag/",
|
||||||
|
|
|
||||||
127
litellm/decisions/openai_transformation.py
Normal file
127
litellm/decisions/openai_transformation.py
Normal file
|
|
@ -0,0 +1,127 @@
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from typing import Final, assert_never
|
||||||
|
|
||||||
|
from litellm.types.decisions import (
|
||||||
|
ChoiceAnswer,
|
||||||
|
DecisionAnswer,
|
||||||
|
DecisionsResponse,
|
||||||
|
DecisionsUsage,
|
||||||
|
NoulAnswer,
|
||||||
|
OpenAIChoiceAnswer,
|
||||||
|
OpenAIChoiceProbability,
|
||||||
|
OpenAIChoiceQuestion,
|
||||||
|
OpenAIDecisionAnswer,
|
||||||
|
OpenAIDecisionInputMessage,
|
||||||
|
OpenAIDecisionQuestion,
|
||||||
|
OpenAIDecisionRequestBody,
|
||||||
|
OpenAIDecisionResponse,
|
||||||
|
OpenAIDecisionUsage,
|
||||||
|
OpenAIPredicateAnswer,
|
||||||
|
OpenAIPredicateQuestion,
|
||||||
|
OpenAIRefusalAnswer,
|
||||||
|
OpenAIScoreAnswer,
|
||||||
|
OpenAIScoreLevel,
|
||||||
|
OpenAIScoreProbability,
|
||||||
|
OpenAIScoreQuestion,
|
||||||
|
ScoreAnswer,
|
||||||
|
systemone_choice_key,
|
||||||
|
)
|
||||||
|
|
||||||
|
_OPENAI_ONLY_FIELDS: Final = frozenset({"input", "questions", "safety_identifier"})
|
||||||
|
|
||||||
|
|
||||||
|
def _message_text(message: OpenAIDecisionInputMessage) -> str:
|
||||||
|
if isinstance(message.content, str):
|
||||||
|
return message.content
|
||||||
|
return "\n\n".join(part.text for part in message.content)
|
||||||
|
|
||||||
|
|
||||||
|
def _state(decision_input: str | Sequence[OpenAIDecisionInputMessage]) -> str:
|
||||||
|
if isinstance(decision_input, str):
|
||||||
|
return decision_input
|
||||||
|
return "\n\n".join(_message_text(message) for message in decision_input)
|
||||||
|
|
||||||
|
|
||||||
|
def _level_criterion(level: OpenAIScoreLevel) -> str:
|
||||||
|
return level.label if level.description is None else f"{level.label}: {level.description}"
|
||||||
|
|
||||||
|
|
||||||
|
def _systemone_question(question: OpenAIDecisionQuestion) -> Mapping[str, object]:
|
||||||
|
match question:
|
||||||
|
case OpenAIPredicateQuestion():
|
||||||
|
return {"type": "noul", "instructions": question.instructions}
|
||||||
|
case OpenAIChoiceQuestion():
|
||||||
|
return {
|
||||||
|
"type": "choice",
|
||||||
|
"instructions": question.instructions,
|
||||||
|
"criteria": {systemone_choice_key(option.value): option.description for option in question.choices},
|
||||||
|
}
|
||||||
|
case OpenAIScoreQuestion():
|
||||||
|
return {
|
||||||
|
"type": "score",
|
||||||
|
"instructions": question.instructions,
|
||||||
|
"criteria": [_level_criterion(level) for level in question.levels],
|
||||||
|
}
|
||||||
|
case _:
|
||||||
|
assert_never(question)
|
||||||
|
|
||||||
|
|
||||||
|
def to_systemone_request(
|
||||||
|
request_data: Mapping[str, object], body: OpenAIDecisionRequestBody
|
||||||
|
) -> Mapping[str, object]:
|
||||||
|
return {
|
||||||
|
**{key: value for key, value in request_data.items() if key not in _OPENAI_ONLY_FIELDS},
|
||||||
|
"state": _state(body.input),
|
||||||
|
"questions": {str(index): _systemone_question(question) for index, question in enumerate(body.questions)},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _openai_answer(question: OpenAIDecisionQuestion, answer: DecisionAnswer | None) -> OpenAIDecisionAnswer:
|
||||||
|
match question, answer:
|
||||||
|
case OpenAIPredicateQuestion(), NoulAnswer():
|
||||||
|
return OpenAIPredicateAnswer(name=question.name, probability=answer.noul)
|
||||||
|
case OpenAIChoiceQuestion(), ChoiceAnswer():
|
||||||
|
typed_values: Final = {systemone_choice_key(option.value): option.value for option in question.choices}
|
||||||
|
return OpenAIChoiceAnswer(
|
||||||
|
name=question.name,
|
||||||
|
choice=typed_values.get(answer.choice, answer.choice),
|
||||||
|
probabilities=tuple(
|
||||||
|
OpenAIChoiceProbability(
|
||||||
|
value=option.value,
|
||||||
|
probability=answer.probabilities.get(systemone_choice_key(option.value), 0.0),
|
||||||
|
)
|
||||||
|
for option in question.choices
|
||||||
|
),
|
||||||
|
confidence=answer.confidence,
|
||||||
|
)
|
||||||
|
case OpenAIScoreQuestion(), ScoreAnswer():
|
||||||
|
return OpenAIScoreAnswer(
|
||||||
|
name=question.name,
|
||||||
|
score=answer.score,
|
||||||
|
probabilities=tuple(
|
||||||
|
OpenAIScoreProbability(
|
||||||
|
value=index, label=level.label, probability=answer.probabilities.get(str(index), 0.0)
|
||||||
|
)
|
||||||
|
for index, level in enumerate(question.levels)
|
||||||
|
),
|
||||||
|
confidence=answer.confidence,
|
||||||
|
)
|
||||||
|
case _:
|
||||||
|
return OpenAIRefusalAnswer(name=question.name)
|
||||||
|
|
||||||
|
|
||||||
|
def to_openai_response(
|
||||||
|
response: DecisionsResponse, questions: Sequence[OpenAIDecisionQuestion], requested_model: str
|
||||||
|
) -> OpenAIDecisionResponse:
|
||||||
|
usage: Final = response.usage or DecisionsUsage()
|
||||||
|
return OpenAIDecisionResponse(
|
||||||
|
model=response.model or requested_model,
|
||||||
|
answers=tuple(
|
||||||
|
_openai_answer(question, response.answers.get(str(index))) for index, question in enumerate(questions)
|
||||||
|
),
|
||||||
|
usage=OpenAIDecisionUsage(
|
||||||
|
input_tokens=usage.input_tokens,
|
||||||
|
output_tokens=usage.output_tokens,
|
||||||
|
total_tokens=usage.input_tokens + usage.output_tokens,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
@ -267,7 +267,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
||||||
LazyFeature(
|
LazyFeature(
|
||||||
name="decisions",
|
name="decisions",
|
||||||
module_path="litellm.proxy.decisions_endpoints.endpoints",
|
module_path="litellm.proxy.decisions_endpoints.endpoints",
|
||||||
path_prefixes=("/v1/decisions", "/decisions"),
|
path_prefixes=("/v1/decisions", "/decisions", "/v1/systemone", "/systemone"),
|
||||||
),
|
),
|
||||||
LazyFeature(
|
LazyFeature(
|
||||||
name="claude_code_marketplace",
|
name="claude_code_marketplace",
|
||||||
|
|
|
||||||
|
|
@ -9521,6 +9521,30 @@
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/systemone": {
|
||||||
|
"post": {
|
||||||
|
"operationId": "systemone_systemone_post",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful Response"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"APIKeyHeader": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"summary": "Systemone",
|
||||||
|
"tags": [
|
||||||
|
"decisions"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
"/v1/decisions": {
|
"/v1/decisions": {
|
||||||
"post": {
|
"post": {
|
||||||
"operationId": "decisions_v1_decisions_post",
|
"operationId": "decisions_v1_decisions_post",
|
||||||
|
|
@ -9544,6 +9568,30 @@
|
||||||
"decisions"
|
"decisions"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"/v1/systemone": {
|
||||||
|
"post": {
|
||||||
|
"operationId": "systemone_v1_systemone_post",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful Response"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"APIKeyHeader": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"summary": "Systemone",
|
||||||
|
"tags": [
|
||||||
|
"decisions"
|
||||||
|
]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -488,6 +488,8 @@ class LiteLLMRoutes(enum.Enum):
|
||||||
"/v1/search/{search_tool_name}",
|
"/v1/search/{search_tool_name}",
|
||||||
"/decisions",
|
"/decisions",
|
||||||
"/v1/decisions",
|
"/v1/decisions",
|
||||||
|
"/systemone",
|
||||||
|
"/v1/systemone",
|
||||||
# OCR
|
# OCR
|
||||||
"/ocr",
|
"/ocr",
|
||||||
"/v1/ocr",
|
"/v1/ocr",
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,7 @@ _MANAGED_MODEL_ROUTES: Final = frozenset(
|
||||||
"moderations",
|
"moderations",
|
||||||
"rerank",
|
"rerank",
|
||||||
"decisions",
|
"decisions",
|
||||||
|
"systemone",
|
||||||
"ocr",
|
"ocr",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
@ -74,6 +75,7 @@ _MODEL_ROUTE_KINDS: Final[
|
||||||
"/audio/speech": "speech",
|
"/audio/speech": "speech",
|
||||||
"/rerank": "body",
|
"/rerank": "body",
|
||||||
"/decisions": "body",
|
"/decisions": "body",
|
||||||
|
"/systemone": "body",
|
||||||
"/messages/count_tokens": "body",
|
"/messages/count_tokens": "body",
|
||||||
":countTokens": "path",
|
":countTokens": "path",
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,39 +1,52 @@
|
||||||
|
from collections.abc import Mapping
|
||||||
from typing import Annotated, Final
|
from typing import Annotated, Final
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Request, Response
|
from fastapi import APIRouter, Depends, Request, Response
|
||||||
from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract
|
from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||||
from pydantic import TypeAdapter, ValidationError
|
from pydantic import TypeAdapter, ValidationError
|
||||||
|
|
||||||
|
from litellm.decisions.openai_transformation import to_openai_response, to_systemone_request
|
||||||
from litellm.exceptions import BadRequestError
|
from litellm.exceptions import BadRequestError
|
||||||
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
|
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
|
||||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||||
from litellm.types.decisions import DecisionsRequestBody
|
from litellm.types.decisions import DecisionsRequestBody, DecisionsResponse, OpenAIDecisionRequestBody
|
||||||
|
|
||||||
router: Final = APIRouter()
|
router: Final = APIRouter()
|
||||||
_REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
_REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||||
_DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody)
|
_DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody)
|
||||||
|
_OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(
|
||||||
|
OpenAIDecisionRequestBody
|
||||||
|
)
|
||||||
|
_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse)
|
||||||
_GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
_GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||||
_OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
|
_OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
|
||||||
_OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
|
_OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
async def _invalid_request(
|
||||||
"/v1/decisions",
|
raw_data: Mapping[str, object], error: ValidationError, user_api_key_dict: UserAPIKeyAuth
|
||||||
dependencies=[Depends(user_api_key_auth)],
|
) -> Exception:
|
||||||
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
|
from litellm.proxy.proxy_server import proxy_logging_obj, version
|
||||||
tags=["decisions"],
|
|
||||||
)
|
return await ProxyBaseLLMRequestProcessing(data=dict(raw_data))._handle_llm_api_exception(
|
||||||
@router.post(
|
e=BadRequestError(
|
||||||
"/decisions",
|
message=f"Invalid Decisions request: {error}",
|
||||||
dependencies=[Depends(user_api_key_auth)],
|
model=str(raw_data.get("model", "")),
|
||||||
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
|
llm_provider="",
|
||||||
tags=["decisions"],
|
),
|
||||||
)
|
user_api_key_dict=user_api_key_dict,
|
||||||
async def decisions(
|
proxy_logging_obj=proxy_logging_obj,
|
||||||
|
version=version,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _process_systemone(
|
||||||
request: Request,
|
request: Request,
|
||||||
fastapi_response: Response,
|
fastapi_response: Response,
|
||||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
user_api_key_dict: UserAPIKeyAuth,
|
||||||
):
|
raw_data: Mapping[str, object],
|
||||||
|
openai_body: OpenAIDecisionRequestBody | None,
|
||||||
|
) -> object:
|
||||||
from litellm.proxy.proxy_server import (
|
from litellm.proxy.proxy_server import (
|
||||||
general_settings as proxy_general_settings,
|
general_settings as proxy_general_settings,
|
||||||
)
|
)
|
||||||
|
|
@ -55,7 +68,7 @@ async def decisions(
|
||||||
user_temperature as proxy_user_temperature,
|
user_temperature as proxy_user_temperature,
|
||||||
)
|
)
|
||||||
|
|
||||||
data: Final = _REQUEST_DATA_ADAPTER.validate_json(await request.body())
|
data: Final = dict(raw_data if openai_body is None else to_systemone_request(raw_data, openai_body))
|
||||||
general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings)
|
general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings)
|
||||||
user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base)
|
user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base)
|
||||||
user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model)
|
user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model)
|
||||||
|
|
@ -63,7 +76,7 @@ async def decisions(
|
||||||
processor: Final = ProxyBaseLLMRequestProcessing(data=data)
|
processor: Final = ProxyBaseLLMRequestProcessing(data=data)
|
||||||
try:
|
try:
|
||||||
_DECISIONS_REQUEST_BODY_ADAPTER.validate_python(data)
|
_DECISIONS_REQUEST_BODY_ADAPTER.validate_python(data)
|
||||||
return await processor.base_process_llm_request(
|
result: Final[object] = await processor.base_process_llm_request(
|
||||||
request=request,
|
request=request,
|
||||||
fastapi_response=fastapi_response,
|
fastapi_response=fastapi_response,
|
||||||
user_api_key_dict=user_api_key_dict,
|
user_api_key_dict=user_api_key_dict,
|
||||||
|
|
@ -81,18 +94,15 @@ async def decisions(
|
||||||
user_api_base=user_api_base,
|
user_api_base=user_api_base,
|
||||||
version=version,
|
version=version,
|
||||||
)
|
)
|
||||||
|
if openai_body is None or isinstance(result, Response):
|
||||||
|
return result
|
||||||
|
return to_openai_response(
|
||||||
|
_DECISIONS_RESPONSE_ADAPTER.validate_python(result),
|
||||||
|
openai_body.questions,
|
||||||
|
str(data.get("model", "")),
|
||||||
|
)
|
||||||
except ValidationError as error:
|
except ValidationError as error:
|
||||||
bad_request_error: Final = BadRequestError(
|
raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict)
|
||||||
message=f"Invalid Decisions request: {error}",
|
|
||||||
model=str(data.get("model", "")),
|
|
||||||
llm_provider="",
|
|
||||||
)
|
|
||||||
raise await processor._handle_llm_api_exception(
|
|
||||||
e=bad_request_error,
|
|
||||||
user_api_key_dict=user_api_key_dict,
|
|
||||||
proxy_logging_obj=proxy_logging_obj,
|
|
||||||
version=version,
|
|
||||||
)
|
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
raise await processor._handle_llm_api_exception(
|
raise await processor._handle_llm_api_exception(
|
||||||
e=error,
|
e=error,
|
||||||
|
|
@ -100,3 +110,60 @@ async def decisions(
|
||||||
proxy_logging_obj=proxy_logging_obj,
|
proxy_logging_obj=proxy_logging_obj,
|
||||||
version=version,
|
version=version,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/v1/systemone",
|
||||||
|
dependencies=[Depends(user_api_key_auth)],
|
||||||
|
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||||
|
tags=["decisions"],
|
||||||
|
)
|
||||||
|
@router.post(
|
||||||
|
"/systemone",
|
||||||
|
dependencies=[Depends(user_api_key_auth)],
|
||||||
|
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||||
|
tags=["decisions"],
|
||||||
|
)
|
||||||
|
async def systemone(
|
||||||
|
request: Request,
|
||||||
|
fastapi_response: Response,
|
||||||
|
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||||
|
):
|
||||||
|
return await _process_systemone(
|
||||||
|
request=request,
|
||||||
|
fastapi_response=fastapi_response,
|
||||||
|
user_api_key_dict=user_api_key_dict,
|
||||||
|
raw_data=_REQUEST_DATA_ADAPTER.validate_json(await request.body()),
|
||||||
|
openai_body=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/v1/decisions",
|
||||||
|
dependencies=[Depends(user_api_key_auth)],
|
||||||
|
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||||
|
tags=["decisions"],
|
||||||
|
)
|
||||||
|
@router.post(
|
||||||
|
"/decisions",
|
||||||
|
dependencies=[Depends(user_api_key_auth)],
|
||||||
|
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||||
|
tags=["decisions"],
|
||||||
|
)
|
||||||
|
async def decisions(
|
||||||
|
request: Request,
|
||||||
|
fastapi_response: Response,
|
||||||
|
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||||
|
):
|
||||||
|
raw_data: Final = _REQUEST_DATA_ADAPTER.validate_json(await request.body())
|
||||||
|
try:
|
||||||
|
openai_body: Final = _OPENAI_DECISION_REQUEST_BODY_ADAPTER.validate_python(raw_data)
|
||||||
|
except ValidationError as error:
|
||||||
|
raise await _invalid_request(raw_data=raw_data, error=error, user_api_key_dict=user_api_key_dict)
|
||||||
|
return await _process_systemone(
|
||||||
|
request=request,
|
||||||
|
fastapi_response=fastapi_response,
|
||||||
|
user_api_key_dict=user_api_key_dict,
|
||||||
|
raw_data=raw_data,
|
||||||
|
openai_body=openai_body,
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
from collections.abc import Mapping, Sequence
|
from collections.abc import Mapping, Sequence
|
||||||
from typing import Annotated, Literal, TypeAlias
|
from typing import Annotated, Final, Literal, TypeAlias
|
||||||
|
|
||||||
from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config
|
from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config
|
||||||
from typing_extensions import ReadOnly, Required, TypedDict
|
from typing_extensions import ReadOnly, Required, TypedDict
|
||||||
|
|
@ -125,3 +125,169 @@ class DecisionsResponse(LiteLLMPydanticObjectBase):
|
||||||
@property
|
@property
|
||||||
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
|
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
|
||||||
return self._hidden_params
|
return self._hidden_params
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIDecisionInputText(LiteLLMPydanticObjectBase):
|
||||||
|
type: Literal["input_text"]
|
||||||
|
text: str
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIDecisionInputMessage(LiteLLMPydanticObjectBase):
|
||||||
|
role: Literal["user"] = "user"
|
||||||
|
type: Literal["message"] = "message"
|
||||||
|
content: str | Sequence[OpenAIDecisionInputText]
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase):
|
||||||
|
type: Literal["predicate"]
|
||||||
|
name: str | None = None
|
||||||
|
instructions: str
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIChoiceOption(LiteLLMPydanticObjectBase):
|
||||||
|
value: str | bool
|
||||||
|
description: str | None = None
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
def systemone_choice_key(value: str | bool) -> str:
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return "true" if value else "false"
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIChoiceQuestion(LiteLLMPydanticObjectBase):
|
||||||
|
type: Literal["choice"]
|
||||||
|
name: str | None = None
|
||||||
|
instructions: str
|
||||||
|
choices: Annotated[Sequence[OpenAIChoiceOption], Field(min_length=2, max_length=255)]
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def require_unique_systemone_keys(self) -> "OpenAIChoiceQuestion":
|
||||||
|
keys: Final = frozenset(systemone_choice_key(option.value) for option in self.choices)
|
||||||
|
if len(keys) != len(self.choices):
|
||||||
|
raise ValueError("Choice values must be unique, and a boolean cannot share its text with a string choice")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIScoreLevel(LiteLLMPydanticObjectBase):
|
||||||
|
label: str
|
||||||
|
description: str | None = None
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIScoreQuestion(LiteLLMPydanticObjectBase):
|
||||||
|
type: Literal["score"]
|
||||||
|
name: str | None = None
|
||||||
|
instructions: str
|
||||||
|
levels: Annotated[Sequence[OpenAIScoreLevel], Field(min_length=2, max_length=10)]
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
OpenAIDecisionQuestion: TypeAlias = Annotated[
|
||||||
|
OpenAIPredicateQuestion | OpenAIChoiceQuestion | OpenAIScoreQuestion,
|
||||||
|
Field(discriminator="type"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase):
|
||||||
|
input: str | Sequence[OpenAIDecisionInputMessage]
|
||||||
|
questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=200)]
|
||||||
|
safety_identifier: str | None = None
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="allow", frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase):
|
||||||
|
type: Literal["predicate"] = "predicate"
|
||||||
|
name: str | None
|
||||||
|
probability: float
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIChoiceProbability(LiteLLMPydanticObjectBase):
|
||||||
|
value: str | bool
|
||||||
|
probability: float
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase):
|
||||||
|
type: Literal["choice"] = "choice"
|
||||||
|
name: str | None
|
||||||
|
choice: str | bool
|
||||||
|
probabilities: tuple[OpenAIChoiceProbability, ...]
|
||||||
|
confidence: float
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIScoreProbability(LiteLLMPydanticObjectBase):
|
||||||
|
value: int
|
||||||
|
label: str
|
||||||
|
probability: float
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIScoreAnswer(LiteLLMPydanticObjectBase):
|
||||||
|
type: Literal["score"] = "score"
|
||||||
|
name: str | None
|
||||||
|
score: float
|
||||||
|
probabilities: tuple[OpenAIScoreProbability, ...]
|
||||||
|
confidence: float
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIRefusalAnswer(LiteLLMPydanticObjectBase):
|
||||||
|
type: Literal["refusal"] = "refusal"
|
||||||
|
name: str | None
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
OpenAIDecisionAnswer: TypeAlias = OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIRefusalAnswer
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIDecisionInputTokensDetails(LiteLLMPydanticObjectBase):
|
||||||
|
cached_tokens: int = 0
|
||||||
|
cache_write_tokens: int = 0
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIDecisionOutputTokensDetails(LiteLLMPydanticObjectBase):
|
||||||
|
reasoning_tokens: int = 0
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIDecisionUsage(LiteLLMPydanticObjectBase):
|
||||||
|
input_tokens: int
|
||||||
|
input_tokens_details: OpenAIDecisionInputTokensDetails = OpenAIDecisionInputTokensDetails()
|
||||||
|
output_tokens: int
|
||||||
|
output_tokens_details: OpenAIDecisionOutputTokensDetails = OpenAIDecisionOutputTokensDetails()
|
||||||
|
total_tokens: int
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAIDecisionResponse(LiteLLMPydanticObjectBase):
|
||||||
|
model: str
|
||||||
|
answers: tuple[OpenAIDecisionAnswer, ...]
|
||||||
|
usage: OpenAIDecisionUsage
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
|
||||||
|
|
@ -788,6 +788,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
|
||||||
"/v1/search": [CallTypes.asearch, CallTypes.search],
|
"/v1/search": [CallTypes.asearch, CallTypes.search],
|
||||||
"/decisions": [CallTypes.adecisions, CallTypes.decisions],
|
"/decisions": [CallTypes.adecisions, CallTypes.decisions],
|
||||||
"/v1/decisions": [CallTypes.adecisions, CallTypes.decisions],
|
"/v1/decisions": [CallTypes.adecisions, CallTypes.decisions],
|
||||||
|
"/systemone": [CallTypes.adecisions, CallTypes.decisions],
|
||||||
|
"/v1/systemone": [CallTypes.adecisions, CallTypes.decisions],
|
||||||
# Batches
|
# Batches
|
||||||
"/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
|
"/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
|
||||||
"/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
|
"/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ from starlette.routing import Match
|
||||||
|
|
||||||
import litellm
|
import litellm
|
||||||
from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features
|
from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features
|
||||||
from litellm.proxy.decisions_endpoints.endpoints import decisions
|
from litellm.proxy.decisions_endpoints.endpoints import decisions, systemone
|
||||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder
|
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder
|
||||||
from litellm.proxy.proxy_server import (
|
from litellm.proxy.proxy_server import (
|
||||||
app,
|
app,
|
||||||
|
|
@ -77,7 +77,7 @@ def client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
||||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions"))
|
@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone"))
|
||||||
def test_proxy_decisions_route_returns_answers_and_cost(
|
def test_proxy_decisions_route_returns_answers_and_cost(
|
||||||
client: TestClient,
|
client: TestClient,
|
||||||
respx_mock: respx.MockRouter,
|
respx_mock: respx.MockRouter,
|
||||||
|
|
@ -126,7 +126,7 @@ def test_proxy_decisions_dispatches_typesafe_deployment(
|
||||||
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE)
|
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE)
|
||||||
|
|
||||||
response: Final = client.post(
|
response: Final = client.post(
|
||||||
"/v1/decisions",
|
"/v1/systemone",
|
||||||
json={
|
json={
|
||||||
"model": "jev",
|
"model": "jev",
|
||||||
"state": {"source": "proxy-test"},
|
"state": {"source": "proxy-test"},
|
||||||
|
|
@ -165,7 +165,7 @@ def test_proxy_decisions_sends_the_env_key_to_the_deployment_api_base(
|
||||||
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
|
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
|
||||||
upstream: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE)
|
upstream: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE)
|
||||||
|
|
||||||
response: Final = client.post("/v1/decisions", json=_REQUEST)
|
response: Final = client.post("/v1/systemone", json=_REQUEST)
|
||||||
|
|
||||||
assert response.status_code == 200, response.text
|
assert response.status_code == 200, response.text
|
||||||
assert upstream.call_count == 1
|
assert upstream.call_count == 1
|
||||||
|
|
@ -177,7 +177,7 @@ def test_proxy_decisions_unknown_model_is_a_client_error(
|
||||||
respx_mock: respx.MockRouter,
|
respx_mock: respx.MockRouter,
|
||||||
) -> None:
|
) -> None:
|
||||||
response: Final = client.post(
|
response: Final = client.post(
|
||||||
"/v1/decisions",
|
"/v1/systemone",
|
||||||
json={
|
json={
|
||||||
"model": "missing-model",
|
"model": "missing-model",
|
||||||
"state": "review",
|
"state": "review",
|
||||||
|
|
@ -210,7 +210,7 @@ def test_proxy_decisions_missing_required_field_is_a_client_error(
|
||||||
) -> None:
|
) -> None:
|
||||||
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||||
|
|
||||||
response: Final = client.post("/v1/decisions", json=request_body)
|
response: Final = client.post("/v1/systemone", json=request_body)
|
||||||
|
|
||||||
assert response.status_code == 400, response.text
|
assert response.status_code == 400, response.text
|
||||||
assert not upstream.called
|
assert not upstream.called
|
||||||
|
|
@ -238,7 +238,7 @@ def test_proxy_decisions_dispatches_strands_decider(
|
||||||
upstream: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
|
upstream: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
|
||||||
|
|
||||||
response: Final = client.post(
|
response: Final = client.post(
|
||||||
"/v1/decisions",
|
"/v1/systemone",
|
||||||
json={
|
json={
|
||||||
"model": "strands",
|
"model": "strands",
|
||||||
"state": {"source": "proxy-test"},
|
"state": {"source": "proxy-test"},
|
||||||
|
|
@ -266,7 +266,7 @@ def test_proxy_decisions_without_model_uses_the_proxy_default_model(
|
||||||
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||||
|
|
||||||
response: Final = client.post(
|
response: Final = client.post(
|
||||||
"/v1/decisions", json={key: value for key, value in _REQUEST.items() if key != "model"}
|
"/v1/systemone", json={key: value for key, value in _REQUEST.items() if key != "model"}
|
||||||
)
|
)
|
||||||
|
|
||||||
assert response.status_code == 200, response.text
|
assert response.status_code == 200, response.text
|
||||||
|
|
@ -275,6 +275,152 @@ def test_proxy_decisions_without_model_uses_the_proxy_default_model(
|
||||||
assert json.loads(upstream.calls[0].request.content)["model"] == "pplx-decider-v1-27b"
|
assert json.loads(upstream.calls[0].request.content)["model"] == "pplx-decider-v1-27b"
|
||||||
|
|
||||||
|
|
||||||
|
_OPENAI_FORMAT_REQUEST: Final[Mapping[str, object]] = {
|
||||||
|
"model": "decider",
|
||||||
|
"input": [
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{"type": "input_text", "text": "The package arrived with a broken screen."},
|
||||||
|
{"type": "input_text", "text": "I want a refund."},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{"role": "user", "content": "Order 1234."},
|
||||||
|
],
|
||||||
|
"questions": [
|
||||||
|
{"type": "predicate", "name": "damaged", "instructions": "Does the customer report a damaged item?"},
|
||||||
|
{
|
||||||
|
"type": "choice",
|
||||||
|
"instructions": "Should we refund?",
|
||||||
|
"choices": [{"value": True, "description": "Refund now"}, {"value": "escalate"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "score",
|
||||||
|
"name": "severity",
|
||||||
|
"instructions": "How severe is the issue?",
|
||||||
|
"levels": [{"label": "minor"}, {"label": "major", "description": "Product unusable"}],
|
||||||
|
},
|
||||||
|
{"type": "predicate", "name": "fraud", "instructions": "Is this fraud?"},
|
||||||
|
],
|
||||||
|
"safety_identifier": "end-user-1",
|
||||||
|
}
|
||||||
|
_SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST: Final[Mapping[str, object]] = {
|
||||||
|
"model": "pplx-decider-v1-27b",
|
||||||
|
"answers": {
|
||||||
|
"0": {"type": "noul", "noul": 0.95},
|
||||||
|
"1": {"type": "choice", "choice": "true", "confidence": 0.8, "probabilities": {"true": 0.9, "escalate": 0.1}},
|
||||||
|
"2": {
|
||||||
|
"type": "score",
|
||||||
|
"score": 0.7,
|
||||||
|
"confidence": 0.6,
|
||||||
|
"legend": {"0": "minor", "1": "major: Product unusable"},
|
||||||
|
"probabilities": {"0": 0.3, "1": 0.7},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions"))
|
||||||
|
def test_openai_format_decisions_translate_through_systemone(
|
||||||
|
client: TestClient,
|
||||||
|
respx_mock: respx.MockRouter,
|
||||||
|
endpoint: str,
|
||||||
|
) -> None:
|
||||||
|
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(
|
||||||
|
json=_SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST
|
||||||
|
)
|
||||||
|
|
||||||
|
response: Final = client.post(endpoint, json=_OPENAI_FORMAT_REQUEST)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
assert json.loads(upstream.calls[0].request.content) == {
|
||||||
|
"model": "pplx-decider-v1-27b",
|
||||||
|
"state": "The package arrived with a broken screen.\n\nI want a refund.\n\nOrder 1234.",
|
||||||
|
"questions": {
|
||||||
|
"0": {"type": "noul", "instructions": "Does the customer report a damaged item?"},
|
||||||
|
"1": {
|
||||||
|
"type": "choice",
|
||||||
|
"instructions": "Should we refund?",
|
||||||
|
"criteria": {"true": "Refund now", "escalate": None},
|
||||||
|
},
|
||||||
|
"2": {
|
||||||
|
"type": "score",
|
||||||
|
"instructions": "How severe is the issue?",
|
||||||
|
"criteria": ["minor", "major: Product unusable"],
|
||||||
|
},
|
||||||
|
"3": {"type": "noul", "instructions": "Is this fraud?"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
body: Final = response.json()
|
||||||
|
assert body["model"] == _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST["model"]
|
||||||
|
assert body["answers"] == [
|
||||||
|
{"type": "predicate", "name": "damaged", "probability": 0.95},
|
||||||
|
{
|
||||||
|
"type": "choice",
|
||||||
|
"name": None,
|
||||||
|
"choice": True,
|
||||||
|
"probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}],
|
||||||
|
"confidence": 0.8,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "score",
|
||||||
|
"name": "severity",
|
||||||
|
"score": 0.7,
|
||||||
|
"probabilities": [
|
||||||
|
{"value": 0, "label": "minor", "probability": 0.3},
|
||||||
|
{"value": 1, "label": "major", "probability": 0.7},
|
||||||
|
],
|
||||||
|
"confidence": 0.6,
|
||||||
|
},
|
||||||
|
{"type": "refusal", "name": "fraud"},
|
||||||
|
]
|
||||||
|
assert body["usage"] == {
|
||||||
|
"input_tokens": _INPUT_TOKENS,
|
||||||
|
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
|
||||||
|
"output_tokens": _OUTPUT_TOKENS,
|
||||||
|
"output_tokens_details": {"reasoning_tokens": 0},
|
||||||
|
"total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS,
|
||||||
|
}
|
||||||
|
assert float(response.headers["x-litellm-response-cost"]) > 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"request_body",
|
||||||
|
(
|
||||||
|
_REQUEST,
|
||||||
|
{
|
||||||
|
"model": "decider",
|
||||||
|
"input": "review",
|
||||||
|
"questions": [
|
||||||
|
{
|
||||||
|
"type": "choice",
|
||||||
|
"instructions": "Pick one",
|
||||||
|
"choices": [{"value": True}, {"value": "true"}],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"model": "decider",
|
||||||
|
"input": [{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]}],
|
||||||
|
"questions": [{"type": "predicate", "instructions": "Is this a defect?"}],
|
||||||
|
},
|
||||||
|
),
|
||||||
|
ids=("systemone_body", "colliding_choice_values", "image_input"),
|
||||||
|
)
|
||||||
|
def test_openai_format_decisions_rejects_bodies_it_cannot_translate(
|
||||||
|
client: TestClient,
|
||||||
|
respx_mock: respx.MockRouter,
|
||||||
|
request_body: Mapping[str, object],
|
||||||
|
) -> None:
|
||||||
|
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||||
|
|
||||||
|
response: Final = client.post("/v1/decisions", json=request_body)
|
||||||
|
|
||||||
|
assert response.status_code == 400, response.text
|
||||||
|
assert not upstream.called
|
||||||
|
|
||||||
|
|
||||||
def _decisions_feature() -> LazyFeature:
|
def _decisions_feature() -> LazyFeature:
|
||||||
return next(feature for feature in LAZY_FEATURES if feature.name == "decisions")
|
return next(feature for feature in LAZY_FEATURES if feature.name == "decisions")
|
||||||
|
|
||||||
|
|
@ -301,6 +447,8 @@ def test_a_config_pass_through_at_v1_decisions_keeps_its_route_and_the_native_ap
|
||||||
assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"}
|
assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"}
|
||||||
assert _serving_endpoint(bare, "/v1/decisions") is pass_through
|
assert _serving_endpoint(bare, "/v1/decisions") is pass_through
|
||||||
assert _serving_endpoint(bare, "/decisions") is decisions
|
assert _serving_endpoint(bare, "/decisions") is decisions
|
||||||
|
assert _serving_endpoint(bare, "/v1/systemone") is systemone
|
||||||
|
assert _serving_endpoint(bare, "/systemone") is systemone
|
||||||
|
|
||||||
|
|
||||||
def test_with_lazy_routes_disabled_a_config_pass_through_at_v1_decisions_still_wins(
|
def test_with_lazy_routes_disabled_a_config_pass_through_at_v1_decisions_still_wins(
|
||||||
|
|
|
||||||
|
|
@ -75,7 +75,7 @@ describe("SystemOneUI integration", () => {
|
||||||
render(<SystemOneUI accessToken="session-key" />);
|
render(<SystemOneUI accessToken="session-key" />);
|
||||||
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
||||||
await user.keyboard("{ArrowDown}");
|
await user.keyboard("{ArrowDown}");
|
||||||
await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" }));
|
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
|
||||||
|
|
||||||
expect(screen.getByRole("note", { name: "Decision endpoint notice" })).toHaveTextContent(
|
expect(screen.getByRole("note", { name: "Decision endpoint notice" })).toHaveTextContent(
|
||||||
"omit model to use the proxy's configured default.",
|
"omit model to use the proxy's configured default.",
|
||||||
|
|
@ -162,7 +162,7 @@ describe("SystemOneUI integration", () => {
|
||||||
render(<SystemOneUI accessToken="session-key" />);
|
render(<SystemOneUI accessToken="session-key" />);
|
||||||
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
||||||
await user.keyboard("{ArrowDown}");
|
await user.keyboard("{ArrowDown}");
|
||||||
await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" }));
|
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
|
||||||
const editor = screen.getByRole("textbox", { name: "System One JSON payload" });
|
const editor = screen.getByRole("textbox", { name: "System One JSON payload" });
|
||||||
const draft = JSON.stringify({
|
const draft = JSON.stringify({
|
||||||
model: "my-decider",
|
model: "my-decider",
|
||||||
|
|
@ -185,12 +185,12 @@ describe("SystemOneUI integration", () => {
|
||||||
|
|
||||||
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
||||||
await user.keyboard("{ArrowDown}");
|
await user.keyboard("{ArrowDown}");
|
||||||
await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" }));
|
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
|
||||||
expect(editor).toHaveValue(draft);
|
expect(editor).toHaveValue(draft);
|
||||||
expect(screen.queryByText("Selected choice")).not.toBeInTheDocument();
|
expect(screen.queryByText("Selected choice")).not.toBeInTheDocument();
|
||||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||||
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
|
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
|
||||||
expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/decisions$/);
|
expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/systemone$/);
|
||||||
expect(JSON.parse(mockFetch.mock.calls[1]?.[1]?.body as string)).toEqual(JSON.parse(draft));
|
expect(JSON.parse(mockFetch.mock.calls[1]?.[1]?.body as string)).toEqual(JSON.parse(draft));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -199,7 +199,7 @@ describe("SystemOneUI integration", () => {
|
||||||
render(<SystemOneUI accessToken="session-key" />);
|
render(<SystemOneUI accessToken="session-key" />);
|
||||||
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
||||||
await user.keyboard("{ArrowDown}");
|
await user.keyboard("{ArrowDown}");
|
||||||
await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" }));
|
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
|
||||||
const payload = { state: "An outage", questions: { urgent: { type: "noul", instructions: "Is this urgent?" } } };
|
const payload = { state: "An outage", questions: { urgent: { type: "noul", instructions: "Is this urgent?" } } };
|
||||||
fireEvent.change(screen.getByRole("textbox", { name: "System One JSON payload" }), {
|
fireEvent.change(screen.getByRole("textbox", { name: "System One JSON payload" }), {
|
||||||
target: { value: JSON.stringify(payload) },
|
target: { value: JSON.stringify(payload) },
|
||||||
|
|
@ -207,7 +207,7 @@ describe("SystemOneUI integration", () => {
|
||||||
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
|
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
|
||||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||||
expect(await screen.findByText("jev-1.13.0")).toBeInTheDocument();
|
expect(await screen.findByText("jev-1.13.0")).toBeInTheDocument();
|
||||||
expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/decisions$/);
|
expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/systemone$/);
|
||||||
const body = JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string);
|
const body = JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string);
|
||||||
expect(body).toEqual(payload);
|
expect(body).toEqual(payload);
|
||||||
expect(body).not.toHaveProperty("model");
|
expect(body).not.toHaveProperty("model");
|
||||||
|
|
@ -227,7 +227,7 @@ describe("SystemOneUI integration", () => {
|
||||||
await screen.findByRole("button", { name: "Cancel request" });
|
await screen.findByRole("button", { name: "Cancel request" });
|
||||||
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
||||||
await user.keyboard("{ArrowDown}");
|
await user.keyboard("{ArrowDown}");
|
||||||
await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" }));
|
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
|
||||||
expect(mockFetch.mock.calls[0]?.[1]?.signal?.aborted).toBe(true);
|
expect(mockFetch.mock.calls[0]?.[1]?.signal?.aborted).toBe(true);
|
||||||
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
|
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
|
||||||
|
|
||||||
|
|
@ -249,7 +249,7 @@ describe("SystemOneUI integration", () => {
|
||||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||||
expect(await screen.findByText("jev-1.13.0")).toBeInTheDocument();
|
expect(await screen.findByText("jev-1.13.0")).toBeInTheDocument();
|
||||||
expect(screen.getByText("Selected choice")).toBeInTheDocument();
|
expect(screen.getByText("Selected choice")).toBeInTheDocument();
|
||||||
expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/decisions$/);
|
expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/systemone$/);
|
||||||
expect(mockFetch).toHaveBeenCalledTimes(2);
|
expect(mockFetch).toHaveBeenCalledTimes(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -42,11 +42,11 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
|
||||||
const [customApiKey, setCustomApiKey] = useState("");
|
const [customApiKey, setCustomApiKey] = useState("");
|
||||||
const [endpoint, setEndpoint] = useState<DecisionEndpoint>("/typesafe/v1/systemone");
|
const [endpoint, setEndpoint] = useState<DecisionEndpoint>("/typesafe/v1/systemone");
|
||||||
const [payloads, setPayloads] = useState<Record<DecisionEndpoint, string>>({
|
const [payloads, setPayloads] = useState<Record<DecisionEndpoint, string>>({
|
||||||
"/v1/decisions": DECISIONS_EXAMPLE_PAYLOAD,
|
"/v1/systemone": DECISIONS_EXAMPLE_PAYLOAD,
|
||||||
"/typesafe/v1/systemone": EXAMPLE_PAYLOAD,
|
"/typesafe/v1/systemone": EXAMPLE_PAYLOAD,
|
||||||
});
|
});
|
||||||
const rawPayload = payloads[endpoint];
|
const rawPayload = payloads[endpoint];
|
||||||
const examplePayload = endpoint === "/v1/decisions" ? DECISIONS_EXAMPLE_PAYLOAD : EXAMPLE_PAYLOAD;
|
const examplePayload = endpoint === "/v1/systemone" ? DECISIONS_EXAMPLE_PAYLOAD : EXAMPLE_PAYLOAD;
|
||||||
const activeController = useRef<AbortController | null>(null);
|
const activeController = useRef<AbortController | null>(null);
|
||||||
const validation = useMemo(() => validateSystemOnePayload(rawPayload, endpoint), [rawPayload, endpoint]);
|
const validation = useMemo(() => validateSystemOnePayload(rawPayload, endpoint), [rawPayload, endpoint]);
|
||||||
const effectiveApiKey = apiKeySource === "session" ? accessToken || "" : customApiKey.trim();
|
const effectiveApiKey = apiKeySource === "session" ? accessToken || "" : customApiKey.trim();
|
||||||
|
|
@ -105,7 +105,7 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
|
||||||
<Select
|
<Select
|
||||||
value={endpoint}
|
value={endpoint}
|
||||||
onValueChange={(value) => {
|
onValueChange={(value) => {
|
||||||
if (value === "/v1/decisions" || value === "/typesafe/v1/systemone") {
|
if (value === "/v1/systemone" || value === "/typesafe/v1/systemone") {
|
||||||
clearRequestState();
|
clearRequestState();
|
||||||
setEndpoint(value);
|
setEndpoint(value);
|
||||||
}
|
}
|
||||||
|
|
@ -113,11 +113,11 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
|
||||||
>
|
>
|
||||||
<SelectTrigger className="w-80" aria-label="Decision endpoint">
|
<SelectTrigger className="w-80" aria-label="Decision endpoint">
|
||||||
<SelectValue>
|
<SelectValue>
|
||||||
{endpoint === "/v1/decisions" ? "Decisions · /v1/decisions" : "TypeSafe · /typesafe/v1/systemone"}
|
{endpoint === "/v1/systemone" ? "System One · /v1/systemone" : "TypeSafe · /typesafe/v1/systemone"}
|
||||||
</SelectValue>
|
</SelectValue>
|
||||||
</SelectTrigger>
|
</SelectTrigger>
|
||||||
<SelectContent>
|
<SelectContent>
|
||||||
<SelectItem value="/v1/decisions">Decisions · /v1/decisions</SelectItem>
|
<SelectItem value="/v1/systemone">System One · /v1/systemone</SelectItem>
|
||||||
<SelectItem value="/typesafe/v1/systemone">TypeSafe · /typesafe/v1/systemone</SelectItem>
|
<SelectItem value="/typesafe/v1/systemone">TypeSafe · /typesafe/v1/systemone</SelectItem>
|
||||||
</SelectContent>
|
</SelectContent>
|
||||||
</Select>
|
</Select>
|
||||||
|
|
@ -182,11 +182,11 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
|
||||||
<Alert role="note" aria-label="Decision endpoint notice">
|
<Alert role="note" aria-label="Decision endpoint notice">
|
||||||
<Info />
|
<Info />
|
||||||
<AlertTitle>
|
<AlertTitle>
|
||||||
{endpoint === "/v1/decisions" ? "Decision models · Jev format" : "TypeSafe Jev · System One"}
|
{endpoint === "/v1/systemone" ? "Decision models · System One" : "TypeSafe Jev · System One"}
|
||||||
</AlertTitle>
|
</AlertTitle>
|
||||||
<AlertDescription>
|
<AlertDescription>
|
||||||
{endpoint === "/v1/decisions"
|
{endpoint === "/v1/systemone"
|
||||||
? "Sends choice, noul, and score questions through /v1/decisions. Replace the example model with a decision model configured on your proxy, or omit model to use the proxy's configured default."
|
? "Sends choice, noul, and score questions through /v1/systemone. Replace the example model with a decision model configured on your proxy, or omit model to use the proxy's configured default."
|
||||||
: "Sends requests through /typesafe/v1/systemone and requires TYPESAFE_API_KEY on the proxy."}{" "}
|
: "Sends requests through /typesafe/v1/systemone and requires TYPESAFE_API_KEY on the proxy."}{" "}
|
||||||
<a href={DECISION_MODELS_DISCUSSION_URL} target="_blank" rel="noopener noreferrer" className="underline">
|
<a href={DECISION_MODELS_DISCUSSION_URL} target="_blank" rel="noopener noreferrer" className="underline">
|
||||||
Give us feedback on what you want for decision models
|
Give us feedback on what you want for decision models
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ const request = {
|
||||||
},
|
},
|
||||||
provider_option: { enabled: true },
|
provider_option: { enabled: true },
|
||||||
};
|
};
|
||||||
const validate = (value: unknown) => validateSystemOnePayload(JSON.stringify(value), "/v1/decisions");
|
const validate = (value: unknown) => validateSystemOnePayload(JSON.stringify(value), "/v1/systemone");
|
||||||
|
|
||||||
describe("native decisions validation", () => {
|
describe("native decisions validation", () => {
|
||||||
it("accepts structured Jev criteria, optional instructions, and provider extensions without dropping fields", () => {
|
it("accepts structured Jev criteria, optional instructions, and provider extensions without dropping fields", () => {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
|
||||||
export type DecisionEndpoint = "/v1/decisions" | "/typesafe/v1/systemone";
|
export type DecisionEndpoint = "/v1/systemone" | "/typesafe/v1/systemone";
|
||||||
|
|
||||||
const decisionsJson = z.union([z.string(), z.record(z.string(), z.unknown()), z.array(z.unknown())]);
|
const decisionsJson = z.union([z.string(), z.record(z.string(), z.unknown()), z.array(z.unknown())]);
|
||||||
const decisionInstructions = decisionsJson.nullish();
|
const decisionInstructions = decisionsJson.nullish();
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,7 @@ export function validateSystemOnePayload(
|
||||||
return invalid("syntax", `Invalid JSON syntax: ${json.message}`);
|
return invalid("syntax", `Invalid JSON syntax: ${json.message}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const schema = endpoint === "/v1/decisions" ? decisionsRequestSchema : systemOneRequestSchema;
|
const schema = endpoint === "/v1/systemone" ? decisionsRequestSchema : systemOneRequestSchema;
|
||||||
const result = schema.safeParse(json.value);
|
const result = schema.safeParse(json.value);
|
||||||
if (!result.success) {
|
if (!result.success) {
|
||||||
return {
|
return {
|
||||||
|
|
|
||||||
74
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
74
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -16180,6 +16180,23 @@ export interface paths {
|
||||||
patch?: never;
|
patch?: never;
|
||||||
trace?: never;
|
trace?: never;
|
||||||
};
|
};
|
||||||
|
"/systemone": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get?: never;
|
||||||
|
put?: never;
|
||||||
|
/** Systemone */
|
||||||
|
post: operations["systemone_systemone_post"];
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
"/tag/daily/activity": {
|
"/tag/daily/activity": {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: never;
|
query?: never;
|
||||||
|
|
@ -22291,6 +22308,23 @@ export interface paths {
|
||||||
patch?: never;
|
patch?: never;
|
||||||
trace?: never;
|
trace?: never;
|
||||||
};
|
};
|
||||||
|
"/v1/systemone": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get?: never;
|
||||||
|
put?: never;
|
||||||
|
/** Systemone */
|
||||||
|
post: operations["systemone_v1_systemone_post"];
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
"/v1/threads": {
|
"/v1/threads": {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: never;
|
query?: never;
|
||||||
|
|
@ -72513,6 +72547,26 @@ export interface operations {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
systemone_systemone_post: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Successful Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
get_tag_daily_activity_tag_daily_activity_get: {
|
get_tag_daily_activity_tag_daily_activity_get: {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: {
|
query?: {
|
||||||
|
|
@ -80787,6 +80841,26 @@ export interface operations {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
systemone_v1_systemone_post: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Successful Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
create_threads_v1_threads_post: {
|
create_threads_v1_threads_post: {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: never;
|
query?: never;
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue