From bc4b7fd42c9d2b29403d71a654176b30347a7b4d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:00:08 -0700 Subject: [PATCH 01/17] feat(decisions): add unified /v1/decisions endpoint for Jev-compatible providers (#44236) Backport of #44236 to stable/1.104.x. Cherry-picked from 8b1990b4bc (main). Adapted to this line: the lazy feature helpers are ported by hand, managed agent authorization and the integration suites are not on this line, and the lazy-disabled precedence test is dropped since attach_lazy_features takes no feature list here. --- .github/workflows/test-unit.yml | 3 +- README.md | 2 + gateway/routes/allowlist.py | 4 +- litellm/__init__.py | 1 + litellm/cost_calculator.py | 14 +- litellm/decisions/__init__.py | 3 + litellm/decisions/main.py | 299 +++++++++ .../health_check_helpers.py | 12 +- .../litellm_core_utils/health_check_utils.py | 6 + litellm/litellm_core_utils/litellm_logging.py | 3 + litellm/llms/base_llm/decisions/__init__.py | 3 + .../llms/base_llm/decisions/transformation.py | 52 ++ .../cloudflare/decisions/transformation.py | 58 ++ .../openrouter/decisions/transformation.py | 10 + .../perplexity/decisions/transformation.py | 10 + .../decisions/transformation.py | 11 + .../llms/typesafe/decisions/transformation.py | 10 + ...odel_prices_and_context_window_backup.json | 50 ++ litellm/proxy/_lazy_features.py | 17 + litellm/proxy/_lazy_openapi_snapshot.json | 55 ++ litellm/proxy/_types.py | 2 + litellm/proxy/common_request_processing.py | 2 + litellm/proxy/decisions_endpoints/__init__.py | 1 + .../proxy/decisions_endpoints/endpoints.py | 102 +++ .../pass_through_endpoints.py | 50 +- litellm/proxy/route_llm_request.py | 2 + litellm/router.py | 9 + litellm/types/decisions.py | 123 ++++ litellm/types/utils.py | 8 + litellm/utils.py | 3 + model_prices_and_context_window.json | 50 ++ provider_endpoints_support.json | 14 + .../public_endpoints/test_public_endpoints.py | 2 + tests/unit/decisions/__init__.py | 0 tests/unit/decisions/test_main.py | 592 ++++++++++++++++++ .../test_health_check_helpers.py | 99 +++ .../llms/azure/test_azure_common_utils.py | 1 + .../proxy/decisions_endpoints/__init__.py | 0 .../decisions_endpoints/test_endpoints.py | 298 +++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 76 ++- 40 files changed, 2029 insertions(+), 28 deletions(-) create mode 100644 litellm/decisions/__init__.py create mode 100644 litellm/decisions/main.py create mode 100644 litellm/llms/base_llm/decisions/__init__.py create mode 100644 litellm/llms/base_llm/decisions/transformation.py create mode 100644 litellm/llms/cloudflare/decisions/transformation.py create mode 100644 litellm/llms/openrouter/decisions/transformation.py create mode 100644 litellm/llms/perplexity/decisions/transformation.py create mode 100644 litellm/llms/strands_decider/decisions/transformation.py create mode 100644 litellm/llms/typesafe/decisions/transformation.py create mode 100644 litellm/proxy/decisions_endpoints/__init__.py create mode 100644 litellm/proxy/decisions_endpoints/endpoints.py create mode 100644 litellm/types/decisions.py create mode 100644 tests/unit/decisions/__init__.py create mode 100644 tests/unit/decisions/test_main.py create mode 100644 tests/unit/proxy/decisions_endpoints/__init__.py create mode 100644 tests/unit/proxy/decisions_endpoints/test_endpoints.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f55e186e3b2..a4b80aca1dc 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -61,7 +61,7 @@ jobs: - shard: core-utils artifact-name: core-utils - test-path: "" + test-path: tests/unit/decisions unit-flag: core-utils workers: 2 reruns: 1 @@ -130,6 +130,7 @@ jobs: artifact-name: proxy-endpoints test-path: >- tests/test_litellm/proxy/analytics_endpoints + tests/unit/proxy/decisions_endpoints tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/list_api tests/test_litellm/proxy/memory diff --git a/README.md b/README.md index e927c80b8b4..80642882faf 100644 --- a/README.md +++ b/README.md @@ -364,11 +364,13 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | | | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | | | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | | +| [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | | | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | | [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | | | [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | | | | | | | | | [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ | | | | | | | | | [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ | | | | | | | | +| [Typesafe Decisions API (`typesafe`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | | | [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ | | | | | | | | | [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | | | [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index c4a3d3f7473..1999452d5db 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -1,7 +1,7 @@ """Path allowlist for the gateway component. The gateway exposes the LLM data-plane surface: chat/completions, embeddings, -audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image, +audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image, responses, vector stores, passthrough providers, realtime websockets, MCP tool-call endpoints, and operational endpoints (/health, /metrics, and the /debug/memory/summary read of the serving worker's RSS). @@ -60,6 +60,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/v1/rerank", "/v2/rerank", "/rerank", + "/v1/decisions", + "/decisions", "/v1/ocr", "/ocr", "/v1/rag/", diff --git a/litellm/__init__.py b/litellm/__init__.py index e334fbe8ca8..c484037ff7e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1467,6 +1467,7 @@ from .embeddings.dispatch import * from .rust_bridge import rust from .rag.main import * from .sandbox.main import * +from .decisions.main import * from .search.main import * from .realtime_api.main import ( _arealtime, diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a279b9f0903..fb2c23be50c 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -98,6 +98,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse +from litellm.types.decisions import DecisionsResponse, DecisionsUsage from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( HttpxBinaryResponseContent, @@ -1016,6 +1017,7 @@ def _is_known_usage_objects(usage_obj): return ( isinstance(usage_obj, litellm.Usage) or isinstance(usage_obj, ResponseAPIUsage) + or isinstance(usage_obj, DecisionsUsage) or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj) ) @@ -1433,7 +1435,12 @@ def completion_cost( "usage", litellm.Usage(**_usage_for_dump.model_dump()), ) - if usage_obj is None: + if isinstance(usage_obj, DecisionsUsage): + _usage = { + "prompt_tokens": usage_obj.input_tokens, + "completion_tokens": usage_obj.output_tokens, + } + elif usage_obj is None: _usage = {} elif isinstance(usage_obj, BaseModel): _usage = cast(BaseModel, usage_obj).model_dump() @@ -1933,7 +1940,8 @@ def response_cost_calculator( | LiteLLMRealtimeStreamLoggingObject | OpenAIModerationResponse | Response - | SearchResponse, + | SearchResponse + | DecisionsResponse, model: str, custom_llm_provider: str | None, call_type: Literal[ @@ -1955,6 +1963,8 @@ def response_cost_calculator( "arerank", "search", "asearch", + "decisions", + "adecisions", ], optional_params: dict, cache_hit: bool | None = None, diff --git a/litellm/decisions/__init__.py b/litellm/decisions/__init__.py new file mode 100644 index 00000000000..40bddd200a9 --- /dev/null +++ b/litellm/decisions/__init__.py @@ -0,0 +1,3 @@ +from litellm.decisions.main import adecisions, decisions + +__all__ = ["adecisions", "decisions"] diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py new file mode 100644 index 00000000000..da037f1d8cb --- /dev/null +++ b/litellm/decisions/main.py @@ -0,0 +1,299 @@ +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Final + +import httpx +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig +from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT +from litellm.llms.custom_httpx.http_handler import _get_httpx_client, get_async_httpx_client +from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT +from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT +from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT +from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_ENDPOINT +from litellm.secret_managers.main import get_secret_str +from litellm.types.decisions import ( + DecisionQuestion, + DecisionsJSON, + DecisionsRequest, + DecisionsResponse, +) +from litellm.utils import client + +DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxyType( + { + "perplexity": PERPLEXITY_DECISIONS_ENDPOINT, + "typesafe": TYPESAFE_DECISIONS_ENDPOINT, + "openrouter": OPENROUTER_DECISIONS_ENDPOINT, + "cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT, + "strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT, + } +) + +_DECISIONS_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequest]] = TypeAdapter(DecisionsRequest) +_DECISIONS_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) +_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) + + +@dataclass(frozen=True, slots=True, repr=False) +class _PreparedDecisionsRequest: + config: DecisionsProviderConfig + provider: str + upstream_model: str + url: str + api_key: str | None = field(repr=False) + headers: Mapping[str, str] = field(repr=False) + body: Mapping[str, object] = field(repr=False) + + +def _resolve_provider_model(model: str, custom_llm_provider: str | None) -> tuple[str, str]: + provider: Final = model.partition("/")[0] if custom_llm_provider is None else custom_llm_provider + if provider not in DECISIONS_ENDPOINTS: + supported: Final = ", ".join(DECISIONS_ENDPOINTS) + raise litellm.BadRequestError( + message=f"Unknown Decisions provider '{provider}'. Supported providers: {supported}", + model=model, + llm_provider=provider, + ) + upstream_model: Final = model.removeprefix(f"{provider}/") if model.startswith(f"{provider}/") else model + if not upstream_model: + raise litellm.BadRequestError( + message="A model name is required for the Decisions API", + model=model, + llm_provider=provider, + ) + return provider, upstream_model + + +def _resolve_api_key( + *, + provider: str, + model: str, + endpoint: DecisionsProviderConfig, + api_key: str | None, +) -> str | None: + if api_key is not None: + return api_key + + server_api_key: Final = next( + (key for key in (get_secret_str(name) for name in endpoint.api_key_env) if key), + None, + ) + if server_api_key is None: + if not endpoint.api_key_required: + return None + raise litellm.AuthenticationError( + message=f"Missing API key for Decisions provider '{provider}'", + model=model, + llm_provider=provider, + ) + + return server_api_key + + +def _prepare_request( + *, + model: str, + state: DecisionsJSON, + questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str | None, + extra_headers: Mapping[str, str] | None, +) -> _PreparedDecisionsRequest: + provider, upstream_model = _resolve_provider_model(model, custom_llm_provider) + try: + validated_request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python( + {"model": model, "state": state, "questions": questions} + ) + except ValidationError as error: + raise litellm.BadRequestError( + message=f"Invalid Decisions request: {error}", + model=model, + llm_provider=provider, + ) from error + + endpoint: Final = DECISIONS_ENDPOINTS[provider] + env_api_base: Final = get_secret_str(endpoint.api_base_env) + default_api_base: Final = endpoint.default_api_base() + resolved_api_base: Final = api_base or env_api_base or default_api_base + if resolved_api_base is None: + raise litellm.BadRequestError( + message=endpoint.missing_api_base_message(provider), + model=model, + llm_provider=provider, + ) + + resolved_api_key: Final = _resolve_api_key( + provider=provider, + model=model, + endpoint=endpoint, + api_key=api_key, + ) + + canonical_model: Final = endpoint.canonical_model(upstream_model) + outbound_headers: Final = MappingProxyType( + { + **{ + name: value + for name, value in (extra_headers or {}).items() + if name.lower() not in {"authorization", "content-type"} + }, + **({"Authorization": f"Bearer {resolved_api_key}"} if resolved_api_key is not None else {}), + "Content-Type": "application/json", + } + ) + body: Final = MappingProxyType( + { + "model": endpoint.request_model(canonical_model), + "state": validated_request.state, + "questions": { + name: question.model_dump(mode="json", exclude_none=True) + for name, question in validated_request.questions.items() + }, + } + ) + return _PreparedDecisionsRequest( + config=endpoint, + provider=provider, + upstream_model=canonical_model, + url=endpoint.endpoint_url(resolved_api_base, canonical_model), + api_key=resolved_api_key, + headers=outbound_headers, + body=body, + ) + + +def _log_request( + prepared: _PreparedDecisionsRequest, + kwargs: Mapping[str, object], +) -> LiteLLMLoggingObj | None: + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, LiteLLMLoggingObj): + return None + logging_obj.update_from_kwargs( + kwargs=dict(kwargs), + model=prepared.upstream_model, + litellm_params={ + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": prepared.url, + }, + custom_llm_provider=prepared.provider, + ) + request_body: Final = dict(prepared.body) + request_headers: Final = dict(prepared.headers) + logging_obj.pre_call( + input=request_body, + api_key=prepared.api_key, + model=prepared.upstream_model, + additional_args={ + "api_base": prepared.url, + "complete_input_dict": request_body, + "headers": request_headers, + }, + ) + return logging_obj + + +def _parse_response( + response: httpx.Response, + prepared: _PreparedDecisionsRequest, +) -> DecisionsResponse: + response.raise_for_status() + payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content) + result: Final = _DECISIONS_RESPONSE_ADAPTER.validate_python(prepared.config.unwrap_response(payload)) + result._hidden_params.update( + { + "model": f"{prepared.provider}/{prepared.upstream_model}", + "custom_llm_provider": prepared.provider, + "provider_response_model": f"{prepared.provider}/{prepared.upstream_model}", + } + ) + return result + + +def _map_upstream_exception(error: Exception, prepared: _PreparedDecisionsRequest) -> Exception: + return litellm.exception_type( + model=f"{prepared.provider}/{prepared.upstream_model}", + custom_llm_provider=prepared.provider, + original_exception=error, + ) + + +@client +async def adecisions( + model: str, + state: DecisionsJSON, + questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: Mapping[str, str] | None = None, + **kwargs: object, +) -> DecisionsResponse: + prepared: Final = _prepare_request( + model=model, + state=state, + questions=questions, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + ) + logging_obj: Final = _log_request(prepared, kwargs) + try: + handler: Final = get_async_httpx_client(llm_provider=prepared.provider) + response: Final = await handler.post( + prepared.url, + json=dict(prepared.body), + headers=dict(prepared.headers), + timeout=timeout, + logging_obj=logging_obj, + ) + return _parse_response(response=response, prepared=prepared) + except Exception as error: + raise _map_upstream_exception(error, prepared) from error + + +@client +def decisions( + model: str, + state: DecisionsJSON, + questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: Mapping[str, str] | None = None, + **kwargs: object, +) -> DecisionsResponse: + prepared: Final = _prepare_request( + model=model, + state=state, + questions=questions, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + ) + logging_obj: Final = _log_request(prepared, kwargs) + try: + handler: Final = _get_httpx_client() + response: Final = handler.post( + prepared.url, + json=dict(prepared.body), + headers=dict(prepared.headers), + timeout=timeout, + logging_obj=logging_obj, + ) + return _parse_response(response=response, prepared=prepared) + except Exception as error: + raise _map_upstream_exception(error, prepared) from error + + +__all__ = ["DECISIONS_ENDPOINTS", "adecisions", "decisions"] diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 22b8d850c83..0870a5b30ee 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -170,6 +170,7 @@ class HealthCheckHelpers: "batch", "responses", "ocr", + "evaluation", ], Callable, ]: @@ -192,7 +193,7 @@ class HealthCheckHelpers: from litellm.litellm_core_utils.audio_utils.utils import ( get_audio_file_for_health_check, ) - from litellm.litellm_core_utils.health_check_utils import _filter_model_params + from litellm.litellm_core_utils.health_check_utils import DECISIONS_CALL_PARAMS, _filter_model_params from litellm.realtime_api.main import _realtime_health_check return { @@ -259,4 +260,13 @@ class HealthCheckHelpers: **_filter_model_params(model_params=model_params), document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider), ), + "evaluation": lambda: litellm.adecisions( + **DECISIONS_CALL_PARAMS.validate_python( + { + "state": prompt or "health check", + "questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}}, + **_filter_model_params(model_params=model_params), + } + ) + ), } diff --git a/litellm/litellm_core_utils/health_check_utils.py b/litellm/litellm_core_utils/health_check_utils.py index ae56ae8f899..7fe2d830f1e 100644 --- a/litellm/litellm_core_utils/health_check_utils.py +++ b/litellm/litellm_core_utils/health_check_utils.py @@ -4,6 +4,12 @@ Utils used for litellm.ahealth_check() from typing import Final +from pydantic import TypeAdapter + +from litellm.types.decisions import DecisionsCallParams + +DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams) + def _filter_model_params(model_params: dict) -> dict: """Remove 'messages' param from model params.""" diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f3ff4dbc52a..9f210c2cc78 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -118,6 +118,7 @@ from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject +from litellm.types.decisions import DecisionsResponse from litellm.types.interactions import ( InteractionsAPIResponse, InteractionsAPIStreamingResponse, @@ -1815,6 +1816,7 @@ class Logging(LiteLLMLoggingBaseClass): LiteLLMRealtimeStreamLoggingObject, OpenAIModerationResponse, "SearchResponse", + DecisionsResponse, dict, list, ], @@ -2592,6 +2594,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, OpenAIModerationResponse) or isinstance(logging_result, OCRResponse) # OCR or isinstance(logging_result, SearchResponse) # Search API + or isinstance(logging_result, DecisionsResponse) or ( isinstance(logging_result, InteractionsAPIResponse) and logging_result.usage is not None diff --git a/litellm/llms/base_llm/decisions/__init__.py b/litellm/llms/base_llm/decisions/__init__.py new file mode 100644 index 00000000000..c18ac9b00f2 --- /dev/null +++ b/litellm/llms/base_llm/decisions/__init__.py @@ -0,0 +1,3 @@ +from .transformation import DecisionsProviderConfig, JevCompatibleDecisionsEndpoint + +__all__ = ["DecisionsProviderConfig", "JevCompatibleDecisionsEndpoint"] diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py new file mode 100644 index 00000000000..d4fcea24793 --- /dev/null +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -0,0 +1,52 @@ +from dataclasses import dataclass +from typing import Protocol + + +@dataclass(frozen=True, slots=True) +class JevCompatibleDecisionsEndpoint: + default_api_base_value: str | None + path: str + api_key_env: tuple[str, ...] + api_base_env: str + api_key_required: bool = True + + def default_api_base(self) -> str | None: + return self.default_api_base_value + + def missing_api_base_message(self, provider: str) -> str: + return f"api_base is required for Decisions provider '{provider}'" + + def canonical_model(self, model: str) -> str: + return model + + def request_model(self, model: str) -> str: + return model + + def endpoint_url(self, api_base: str, model: str) -> str: + return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" + + def unwrap_response(self, payload: object) -> object: + return payload + + +class DecisionsProviderConfig(Protocol): + @property + def api_key_env(self) -> tuple[str, ...]: ... + + @property + def api_base_env(self) -> str: ... + + @property + def api_key_required(self) -> bool: ... + + def default_api_base(self) -> str | None: ... + + def missing_api_base_message(self, provider: str) -> str: ... + + def canonical_model(self, model: str) -> str: ... + + def request_model(self, model: str) -> str: ... + + def endpoint_url(self, api_base: str, model: str) -> str: ... + + def unwrap_response(self, payload: object) -> object: ... diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py new file mode 100644 index 00000000000..6e8b2999778 --- /dev/null +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -0,0 +1,58 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from pydantic import TypeAdapter + +from litellm.secret_managers.main import ( + get_secret_str, + normalize_nonempty_secret_str, +) + +_RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +@dataclass(frozen=True, slots=True) +class CloudflareDecisionsEndpoint: + api_key_env: tuple[str, ...] = ("CLOUDFLARE_API_KEY",) + api_base_env: str = "CLOUDFLARE_API_BASE" + api_key_required: bool = True + + def default_api_base(self) -> str | None: + account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID")) + if account_id is None: + return None + return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run" + + def missing_api_base_message(self, provider: str) -> str: + return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly" + + def canonical_model(self, model: str) -> str: + if model.startswith("@cf/"): + return model + return f"@cf/cloudflare/{model}" + + def request_model(self, model: str) -> str: + return model.rsplit("/", maxsplit=1)[-1] + + def endpoint_url(self, api_base: str, model: str) -> str: + normalized_api_base: Final = api_base.rstrip("/") + if normalized_api_base.endswith("/ai/v1"): + return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}" + if normalized_api_base.endswith("/ai/run"): + return f"{normalized_api_base}/{model}" + return f"{normalized_api_base}/ai/run/{model}" + + def unwrap_response(self, payload: object) -> object: + if not isinstance(payload, Mapping): + return payload + response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) + if "answers" in response_mapping: + return payload + result: Final = response_mapping.get("result") + if isinstance(result, Mapping): + return result + return payload + + +CLOUDFLARE_DECISIONS_ENDPOINT: Final[CloudflareDecisionsEndpoint] = CloudflareDecisionsEndpoint() diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py new file mode 100644 index 00000000000..7a7466b1239 --- /dev/null +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -0,0 +1,10 @@ +from typing import Final + +from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint + +OPENROUTER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( + default_api_base_value="https://openrouter.ai/api", + path="/alpha/decisions", + api_key_env=("OPENROUTER_API_KEY",), + api_base_env="OPENROUTER_API_BASE", +) diff --git a/litellm/llms/perplexity/decisions/transformation.py b/litellm/llms/perplexity/decisions/transformation.py new file mode 100644 index 00000000000..69a4753f4a3 --- /dev/null +++ b/litellm/llms/perplexity/decisions/transformation.py @@ -0,0 +1,10 @@ +from typing import Final + +from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint + +PERPLEXITY_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( + default_api_base_value="https://api.perplexity.ai", + path="/v1/decisions", + api_key_env=("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"), + api_base_env="PERPLEXITY_API_BASE", +) diff --git a/litellm/llms/strands_decider/decisions/transformation.py b/litellm/llms/strands_decider/decisions/transformation.py new file mode 100644 index 00000000000..265afb2b148 --- /dev/null +++ b/litellm/llms/strands_decider/decisions/transformation.py @@ -0,0 +1,11 @@ +from typing import Final + +from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint + +STRANDS_DECIDER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( + default_api_base_value=None, + path="/v1/systemone", + api_key_env=("STRANDS_DECIDER_API_KEY",), + api_base_env="STRANDS_DECIDER_API_BASE", + api_key_required=False, +) diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py new file mode 100644 index 00000000000..17fb24ce443 --- /dev/null +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -0,0 +1,10 @@ +from typing import Final + +from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint + +TYPESAFE_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( + default_api_base_value="https://api.typesafe.ai", + path="/v1/systemone", + api_key_env=("TYPESAFE_API_KEY",), + api_base_env="TYPESAFE_API_BASE", +) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 46263a71971..1565f9d83c7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -77427,5 +77427,55 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_tool_choice": false + }, + "cloudflare/clef": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/clef-flash": { + "input_cost_per_token": 9e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/@cf/cloudflare/clef": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/@cf/cloudflare/clef-flash": { + "input_cost_per_token": 9e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "perplexity/pplx-decider-v1-27b": { + "input_cost_per_token": 4e-08, + "litellm_provider": "perplexity", + "max_input_tokens": 262144, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.perplexity.ai/api-reference/decisions-post" + }, + "strands_decider/strands-decider-2B-hobson-v19": { + "input_cost_per_token": 0.0, + "litellm_provider": "strands_decider", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19", + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } } } diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 5be87a8bf4d..810a0e18723 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -11,6 +11,7 @@ import importlib from collections.abc import Callable, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass, field +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Final @@ -251,6 +252,11 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( module_path="litellm.proxy.openai_evals_endpoints.endpoints", path_prefixes=("/v1/evals", "/evals"), ), + LazyFeature( + name="decisions", + module_path="litellm.proxy.decisions_endpoints.endpoints", + path_prefixes=("/v1/decisions", "/decisions"), + ), LazyFeature( name="claude_code_marketplace", module_path="litellm.proxy.anthropic_endpoints.claude_code_endpoints", @@ -361,6 +367,10 @@ def _lazy_slots(app: "FastAPI") -> Mapping[str, BaseRoute | None]: return app.state.lazy_slots if hasattr(app.state, "lazy_slots") else MappingProxyType({}) +def _lazy_routes(app: "FastAPI") -> Mapping[str, tuple[BaseRoute, ...]]: + return app.state.lazy_routes if hasattr(app.state, "lazy_routes") else MappingProxyType({}) + + def reserve_lazy_slot(app: "FastAPI", name: str, features: tuple[LazyFeature, ...] = LAZY_FEATURES) -> None: """Record the route the feature's router used to be included after, so its routes are spliced back in there once it loads and keep the same precedence. Anchoring on @@ -512,6 +522,13 @@ def _make_warmup_router(app: "FastAPI") -> "APIRouter": return router +def lazy_owned_routes(app: "FastAPI") -> frozenset[int]: + """ids of the routes lazy features have registered on this app. A route added later at + one of their paths (a config pass-through at /v1/decisions) goes ahead of them, the + precedence lazy mode gives it when the feature has not loaded by the time the config is read.""" + return frozenset(id(route) for route in chain.from_iterable(_lazy_routes(app).values())) + + def loaded_lazy_modules(app: "FastAPI") -> frozenset[str]: """The set of lazy feature modules whose routers are actually registered on this app (tracked by _force_load), empty before the middleware ever ran. diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 48ce63c4ef4..0270be5720b 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -8062,6 +8062,61 @@ } } }, + "decisions": { + "components": { + "schemas": {} + }, + "paths": { + "/decisions": { + "post": { + "operationId": "decisions_decisions_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Decisions", + "tags": [ + "decisions" + ] + } + }, + "/v1/decisions": { + "post": { + "operationId": "decisions_v1_decisions_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Decisions", + "tags": [ + "decisions" + ] + } + } + } + }, "evals": { "components": { "schemas": { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 344f83fe52a..b2716389dba 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -466,6 +466,8 @@ class LiteLLMRoutes(enum.Enum): "/v1/search", "/search/{search_tool_name}", "/v1/search/{search_tool_name}", + "/decisions", + "/v1/decisions", # OCR "/ocr", "/v1/ocr", diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7e1989b0aab..a5b107412cf 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -168,6 +168,7 @@ ProxyRouteType: TypeAlias = Literal[ "avector_store_file_delete", "aocr", "asearch", + "adecisions", "avideo_generation", "avideo_list", "avideo_status", @@ -1942,6 +1943,7 @@ class ProxyBaseLLMRequestProcessing: "avector_store_file_delete", "aocr", "asearch", + "adecisions", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/proxy/decisions_endpoints/__init__.py b/litellm/proxy/decisions_endpoints/__init__.py new file mode 100644 index 00000000000..ea9b7835485 --- /dev/null +++ b/litellm/proxy/decisions_endpoints/__init__.py @@ -0,0 +1 @@ +__all__ = () diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py new file mode 100644 index 00000000000..7dba64ee791 --- /dev/null +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -0,0 +1,102 @@ +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, Request, Response +from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract +from pydantic import TypeAdapter, ValidationError + +from litellm.exceptions import BadRequestError +from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.types.decisions import DecisionsRequestBody + +router: Final = APIRouter() +_REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) +_DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) +_OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) +_OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) + + +@router.post( + "/v1/decisions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +@router.post( + "/decisions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +async def decisions( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + from litellm.proxy.proxy_server import ( + general_settings as proxy_general_settings, + ) + from litellm.proxy.proxy_server import ( + llm_router, + proxy_config, + proxy_logging_obj, + user_max_tokens, + user_request_timeout, + version, + ) + from litellm.proxy.proxy_server import ( + user_api_base as proxy_user_api_base, + ) + from litellm.proxy.proxy_server import ( + user_model as proxy_user_model, + ) + from litellm.proxy.proxy_server import ( + user_temperature as proxy_user_temperature, + ) + + data: Final = _REQUEST_DATA_ADAPTER.validate_json(await request.body()) + general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings) + user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base) + user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model) + user_temperature: Final = _OPTIONAL_FLOAT_ADAPTER.validate_python(proxy_user_temperature) + processor: Final = ProxyBaseLLMRequestProcessing(data=data) + try: + _DECISIONS_REQUEST_BODY_ADAPTER.validate_python(data) + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="adecisions", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=None, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except ValidationError as error: + bad_request_error: Final = BadRequestError( + message=f"Invalid Decisions request: {error}", + model=str(data.get("model", "")), + llm_provider="", + ) + raise await processor._handle_llm_api_exception( + e=bad_request_error, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + except Exception as error: + raise await processor._handle_llm_api_exception( + e=error, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index e0a4184291e..35d4f18d47a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -27,6 +27,7 @@ from fastapi import ( ) from fastapi.responses import StreamingResponse from starlette.datastructures import UploadFile as StarletteUploadFile +from starlette.routing import BaseRoute, Route from starlette.websockets import WebSocketState from websockets.asyncio.client import connect from websockets.exceptions import ( @@ -65,6 +66,7 @@ from litellm.llms.base_llm.managed_resources.utils import ( ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.passthrough import BasePassthroughUtils +from litellm.proxy._lazy_features import lazy_owned_routes from litellm.proxy._types import ( ConfigFieldInfo, ConfigFieldUpdate, @@ -2868,35 +2870,34 @@ def _extract_model_from_vertex_ai_setup(setup_response: Mapping[str, object]) -> return None +def _placed_ahead(routes: Sequence[BaseRoute], moving: BaseRoute, before: BaseRoute) -> tuple[BaseRoute, ...]: + kept: Final = tuple(route for route in routes if route is not moving) + at: Final = next(index for index, route in enumerate(kept) if route is before) + return (*kept[:at], moving, *kept[at:]) + + class SafeRouteAdder: """ Wrapper class for adding routes to FastAPI app. - Only adds routes if they don't already exist on the app. + Only adds routes if they don't already exist on the app. A route a lazy feature registered + does not count: a route added at its path goes ahead of it, the precedence a config + pass-through at /v1/decisions gets in lazy mode, where the feature has not loaded yet. """ + @staticmethod + def _colliding_routes(app: FastAPI, path: str, methods: Sequence[str]) -> tuple[Route, ...]: + wanted: Final = frozenset(methods) + return tuple( + route + for route in app.routes + if isinstance(route, Route) and route.path == path and not wanted.isdisjoint(route.methods or ()) + ) + @staticmethod def _is_path_registered(app: FastAPI, path: str, methods: list[str]) -> bool: - """ - Check if a path with any of the specified methods is already registered on the app. - - Args: - app: The FastAPI application instance - path: The path to check (e.g., "/v1/chat/completions") - methods: List of HTTP methods to check (e.g., ["GET", "POST"]) - - Returns: - True if the path is already registered with any of the methods, False otherwise - """ - for route in app.routes: - # Use getattr to safely access route attributes - route_path = getattr(route, "path", None) - route_methods = getattr(route, "methods", None) - - if route_path == path and route_methods is not None: - # Check if any of the methods overlap - if any(method in route_methods for method in methods): - return True - return False + """True when a route the app itself defines already serves the path with one of the methods.""" + lazy_owned: Final = lazy_owned_routes(app) + return any(id(route) not in lazy_owned for route in SafeRouteAdder._colliding_routes(app, path, methods)) @staticmethod def add_api_route_if_not_exists( @@ -2927,12 +2928,17 @@ class SafeRouteAdder: ) return False + shadowed: Final = SafeRouteAdder._colliding_routes(app, path, methods) app.add_api_route( path=path, endpoint=endpoint, methods=methods, dependencies=dependencies, ) + if shadowed: + app.router.routes[:] = _placed_ahead( # rebind-ok: the app owns its route table + app.router.routes, app.router.routes[-1], shadowed[0] + ) verbose_proxy_logger.debug( "Successfully added route: %s with methods %s", path, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 536c58df65a..030fae5fa62 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -90,6 +90,7 @@ ROUTE_ENDPOINT_MAPPING: Final = { "acompact_responses": "/responses/compact", "aocr": "/ocr", "asearch": "/search", + "adecisions": "/decisions", "avideo_generation": "/videos", "avideo_list": "/videos", "avideo_status": "/videos/{video_id}", @@ -372,6 +373,7 @@ RouteType = Literal[ "avector_store_file_delete", "aocr", "asearch", + "adecisions", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/router.py b/litellm/router.py index 023b99cd64e..d34d0b21a5a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2047,6 +2047,11 @@ class Router: self.asearch = self.factory_function(asearch, call_type="asearch") self.search = self.factory_function(search, call_type="search") + from litellm.decisions import adecisions, decisions + + self.adecisions = self.factory_function(adecisions, call_type="adecisions") + self.decisions = self.factory_function(decisions, call_type="decisions") + def _initialize_video_endpoints(self): """Initialize video endpoints.""" from litellm.videos import ( @@ -6556,6 +6561,8 @@ class Router: "ocr", "asearch", "search", + "adecisions", + "decisions", "aadapter_generate_content", "avideo_generation", "video_generation", @@ -6629,6 +6636,7 @@ class Router: "generate_content_stream", "ocr", "search", + "decisions", "video_generation", "video_list", "video_status", @@ -6796,6 +6804,7 @@ class Router: "agenerate_content_stream", "aocr", "ocr", + "adecisions", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py new file mode 100644 index 00000000000..80db26f201d --- /dev/null +++ b/litellm/types/decisions.py @@ -0,0 +1,123 @@ +from collections.abc import Mapping, Sequence +from typing import Annotated, Literal, TypeAlias + +from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config +from typing_extensions import ReadOnly, Required, TypedDict + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +DecisionsJSON: TypeAlias = str | Mapping[str, object] | Sequence[object] +NoulCriteria: TypeAlias = Mapping[Literal["true", "false"], DecisionsJSON | None] + + +class NoulQuestion(LiteLLMPydanticObjectBase): + type: Literal["noul"] + instructions: DecisionsJSON | None = None + criteria: NoulCriteria | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + @model_validator(mode="after") + def require_instructions_or_criteria(self) -> "NoulQuestion": + if self.instructions is None and self.criteria is None: + raise ValueError("A noul question requires instructions or criteria") + return self + + +class ChoiceQuestion(LiteLLMPydanticObjectBase): + type: Literal["choice"] + instructions: DecisionsJSON | None = None + criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)] + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ScoreQuestion(LiteLLMPydanticObjectBase): + type: Literal["score"] + instructions: DecisionsJSON | None = None + criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)] + + model_config = ConfigDict(extra="allow", frozen=True) + + +DecisionQuestion: TypeAlias = Annotated[ + NoulQuestion | ChoiceQuestion | ScoreQuestion, + Field(discriminator="type"), +] + +DecisionQuestionMap: TypeAlias = Annotated[ + Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion], + Field(min_length=1, max_length=128), +] + + +class DecisionsRequestBody(LiteLLMPydanticObjectBase): + state: DecisionsJSON + questions: DecisionQuestionMap + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionsRequest(DecisionsRequestBody): + model: str + + +@with_config(ConfigDict(extra="allow")) +class DecisionsCallParams(TypedDict, total=False): + model: Required[ReadOnly[str]] + state: Required[ReadOnly[DecisionsJSON]] + questions: Required[ReadOnly[DecisionQuestionMap]] + api_key: ReadOnly[str | None] + api_base: ReadOnly[str | None] + timeout: ReadOnly[float | None] + custom_llm_provider: ReadOnly[str | None] + extra_headers: ReadOnly[Mapping[str, str] | None] + + +class NoulAnswer(LiteLLMPydanticObjectBase): + type: Literal["noul"] + noul: float + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ChoiceAnswer(LiteLLMPydanticObjectBase): + type: Literal["choice"] + choice: str + confidence: float + probabilities: Mapping[str, float] + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ScoreAnswer(LiteLLMPydanticObjectBase): + type: Literal["score"] + score: float + confidence: float + legend: Mapping[str, DecisionsJSON] + probabilities: Mapping[str, float] + + model_config = ConfigDict(extra="allow", frozen=True) + + +DecisionAnswer: TypeAlias = Annotated[ + NoulAnswer | ChoiceAnswer | ScoreAnswer, + Field(discriminator="type"), +] + + +class DecisionsUsage(LiteLLMPydanticObjectBase): + input_tokens: int = 0 + output_tokens: int = 0 + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionsResponse(LiteLLMPydanticObjectBase): + model: str | None = None + answers: Mapping[str, DecisionAnswer] + usage: DecisionsUsage | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 15b34285c5e..791541f9804 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -455,6 +455,8 @@ class CallTypes(str, Enum): arerank = "arerank" search = "search" asearch = "asearch" + decisions = "decisions" + adecisions = "adecisions" arealtime = "_arealtime" aresponses_websocket = "_aresponses_websocket" create_batch = "create_batch" @@ -641,6 +643,8 @@ CallTypesLiteral = Literal[ "arerank", "search", "asearch", + "decisions", + "adecisions", "_arealtime", "_aresponses_websocket", "create_batch", @@ -746,6 +750,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { # Search "/search": [CallTypes.asearch, CallTypes.search], "/v1/search": [CallTypes.asearch, CallTypes.search], + "/decisions": [CallTypes.adecisions, CallTypes.decisions], + "/v1/decisions": [CallTypes.adecisions, CallTypes.decisions], # Batches "/batches": [CallTypes.acreate_batch, CallTypes.create_batch], "/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch], @@ -4009,6 +4015,8 @@ class LlmProviders(str, Enum): OLLAMA_CHAT = "ollama_chat" DEEPINFRA = "deepinfra" PERPLEXITY = "perplexity" + TYPESAFE = "typesafe" + STRANDS_DECIDER = "strands_decider" MISTRAL = "mistral" MILVUS = "milvus" GROQ = "groq" diff --git a/litellm/utils.py b/litellm/utils.py index 40636b15f8b..731f1e9b3fd 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1200,6 +1200,9 @@ def function_setup( if isinstance(search_query, list) else search_query ) + elif call_type in (CallTypes.decisions.value, CallTypes.adecisions.value): + decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state", "") + messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state) elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): messages = args[1] if len(args) > 1 else kwargs.get("prompt") elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 46263a71971..1565f9d83c7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -77427,5 +77427,55 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_tool_choice": false + }, + "cloudflare/clef": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/clef-flash": { + "input_cost_per_token": 9e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/@cf/cloudflare/clef": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/@cf/cloudflare/clef-flash": { + "input_cost_per_token": 9e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "perplexity/pplx-decider-v1-27b": { + "input_cost_per_token": 4e-08, + "litellm_provider": "perplexity", + "max_input_tokens": 262144, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.perplexity.ai/api-reference/decisions-post" + }, + "strands_decider/strands-decider-2B-hobson-v19": { + "input_cost_per_token": 0.0, + "litellm_provider": "strands_decider", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19", + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e6cb0592a15..54e08240c90 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2499,6 +2499,20 @@ "image_variations": true } }, + "typesafe": { + "display_name": "TypeSafe (`typesafe`)", + "url": "https://docs.typesafe.ai/models", + "endpoints": { + "systemone": true + } + }, + "strands_decider": { + "display_name": "Strands Decider (`strands_decider`)", + "url": "https://docs.litellm.ai/docs/providers", + "endpoints": { + "systemone": true + } + }, "tavily": { "display_name": "Tavily (`tavily`)", "url": "https://docs.litellm.ai/docs/search/tavily", diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 0dec44af402..a747271019f 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -411,11 +411,13 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( "sagemaker_nova", "scaleway", "stability", + "strands_decider", "synthetic", "tencent", "tensormesh", "text-completion-inception", "transcribe", + "typesafe", "valkey", "xiaomi_mimo", "zai", diff --git a/tests/unit/decisions/__init__.py b/tests/unit/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py new file mode 100644 index 00000000000..106710d328f --- /dev/null +++ b/tests/unit/decisions/test_main.py @@ -0,0 +1,592 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import pytest +import respx + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.decisions import ( + ChoiceAnswer, + DecisionsResponse, + DecisionsUsage, + NoulAnswer, + ScoreAnswer, +) + +_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( + { + "is_defect": {"type": "noul", "instructions": "Is this a defect?", "provider_field": "kept"}, + "sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}}, + "severity": {"type": "score", "criteria": ["none", "low", "high"]}, + } +) +_INPUT_TOKENS: Final[int] = 367 +_OUTPUT_TOKENS: Final[int] = 3 +_RESPONSE: Final[Mapping[str, object]] = { + "model": "jev-1.13", + "answers": { + "is_defect": {"type": "noul", "noul": 0.9}, + "sentiment": { + "type": "choice", + "choice": "positive", + "confidence": 0.8, + "probabilities": {"positive": 0.8, "negative": 0.2}, + }, + "severity": { + "type": "score", + "score": 1, + "confidence": 0.7, + "legend": {"0": "none", "1": "low", "2": "high"}, + "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}, + }, + }, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, +} +_STRANDS_RESPONSE: Final[Mapping[str, object]] = { + "model": "strands-decider-2B-hobson-v19", + "answers": { + "severity": { + "type": "score", + "score": 1, + "confidence": 0.7, + "legend": {"0": "none", "1": "low", "2": "high"}, + "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}, + } + }, + "usage": {"input_tokens": 216, "output_tokens": 3}, + "latency_ms": 3722.17, +} +_PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( + ( + "perplexity", + "perplexity/pplx-decider-v1-27b", + "https://api.perplexity.ai/v1/decisions", + "pplx-decider-v1-27b", + ), + ("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"), + ( + "openrouter", + "openrouter/typesafe/jev-1.13", + "https://openrouter.ai/api/alpha/decisions", + "typesafe/jev-1.13", + ), +) + + +class _RecordingLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.standard_logging_object: Mapping[str, object] | None = None + + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, + ) -> None: + standard_logging_object: Final = kwargs.get("standard_logging_object") + if isinstance(standard_logging_object, dict): + self.standard_logging_object = standard_logging_object + + +async def _drain_logging_worker() -> None: + await asyncio.sleep(0) + GLOBAL_LOGGING_WORKER.start() + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + +@pytest.fixture(autouse=True) +def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("provider", "model", "url", "upstream_model"), _PROVIDERS) +async def test_adecisions_sends_the_provider_wire_contract( + provider: str, + model: str, + url: str, + upstream_model: str, + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model=model, + state={"source": "unit-test"}, + questions=_QUESTIONS, + api_key="caller-key", + extra_headers={ + "x-request-tag": "decisions-test", + "AUTHORIZATION": "attacker-key", + "Content-Type": "text/plain", + }, + internal_kwarg="must-not-leak", + ) + + assert route.called + assert len(respx_mock.calls) == 1 + request: Final = respx_mock.calls[0].request + assert request.headers["authorization"] == "Bearer caller-key" + assert request.headers["content-type"] == "application/json" + assert request.headers["x-request-tag"] == "decisions-test" + assert json.loads(request.content) == { + "model": upstream_model, + "state": {"source": "unit-test"}, + "questions": { + "is_defect": { + "type": "noul", + "instructions": "Is this a defect?", + "provider_field": "kept", + }, + "sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}}, + "severity": {"type": "score", "criteria": ["none", "low", "high"]}, + }, + } + assert isinstance(response.answers["is_defect"], NoulAnswer) + assert isinstance(response.answers["sentiment"], ChoiceAnswer) + assert isinstance(response.answers["severity"], ScoreAnswer) + assert response._hidden_params["custom_llm_provider"] == provider + + +@pytest.mark.asyncio +async def test_router_dispatches_typesafe_decisions_without_api_base( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) + monkeypatch.delenv("TYPESAFE_API_BASE", raising=False) + provider_resolution: Final = litellm.get_llm_provider("typesafe/jev-latest") + + assert provider_resolution[:2] == ("jev-latest", "typesafe") + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": { + "model": "typesafe/jev-latest", + "api_key": "k", + }, + } + ] + ) + upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) + + response: Final = await router.adecisions( + model="jev", + state="router-test", + questions={ + "sentiment": { + "type": "choice", + "criteria": {"positive": None, "negative": "unhappy"}, + } + }, + ) + + assert upstream.called + assert len(respx_mock.calls) == 1 + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "jev-latest", + "state": "router-test", + "questions": { + "sentiment": { + "type": "choice", + "criteria": {"positive": None, "negative": "unhappy"}, + } + }, + } + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer k" + assert isinstance(response.answers["sentiment"], ChoiceAnswer) + assert response.answers["sentiment"].choice == "positive" + + +def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + assert route.called + assert response.model == "jev-1.13" + + +def test_openrouter_response_keeps_provider_fields(respx_mock: respx.MockRouter) -> None: + payload: Final = { + **_RESPONSE, + "id": "decision-1", + "provider": "typesafe", + "usage": {**_RESPONSE["usage"], "cost": 0.25}, + } + respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=payload) + + response: Final = litellm.decisions( + model="openrouter/typesafe/jev-1.13", + state="review", + questions=_QUESTIONS, + api_key="caller-key", + ) + + assert response.model_extra["id"] == "decision-1" + assert response.model_extra["provider"] == "typesafe" + assert response.usage is not None + assert response.usage.model_extra["cost"] == 0.25 + + +def test_decisions_cost_uses_litellm_token_pricing() -> None: + response: Final = DecisionsResponse( + model="pplx-decider-v1-27b", + answers={}, + usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS), + ) + response._hidden_params = { + "model": "perplexity/pplx-decider-v1-27b", + "custom_llm_provider": "perplexity", + } + + cost: Final = litellm.completion_cost(completion_response=response) + perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] + expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + perplexity_cost["output_cost_per_token"] + ) + + assert expected_cost > 0 + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.asyncio +async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + recording_logger: Final = _RecordingLogger() + original_callbacks: Final = litellm.callbacks + litellm.callbacks = [recording_logger] + + try: + await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + await _drain_logging_worker() + finally: + litellm.callbacks = original_callbacks + + assert recording_logger.standard_logging_object is not None + perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] + expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + perplexity_cost["output_cost_per_token"] + ) + + assert expected_cost > 0 + assert recording_logger.standard_logging_object["response_cost"] == pytest.approx(expected_cost) + assert recording_logger.standard_logging_object["prompt_tokens"] == _INPUT_TOKENS + assert recording_logger.standard_logging_object["completion_tokens"] == _OUTPUT_TOKENS + + +@pytest.mark.asyncio +async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="Supported providers"): + await litellm.adecisions( + model="unknown/jev-1.13", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_empty_custom_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="Supported providers"): + await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + custom_llm_provider="", + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_invalid_question_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): + await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"sentiment": {"type": "choice"}}, + api_key="caller-key", + ) + + assert len(respx_mock.calls) == 0 + + +def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + status_code=400, + json={"error": {"message": "invalid decision"}}, + ) + + with pytest.raises(litellm.BadRequestError): + litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + +def test_server_key_is_sent_to_an_explicit_api_base( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("PERPLEXITYAI_API_KEY", "server-key") + monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False) + route: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE) + + litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_base="https://egress.example/perplexity", + ) + + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == "Bearer server-key" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("cloudflare/clef", "cloudflare/@cf/cloudflare/clef")) +@pytest.mark.parametrize("wrapped", (False, True)) +async def test_cloudflare_clef_resolves_model_and_response_envelope( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + model: str, + wrapped: bool, +) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + response_body: Final[Mapping[str, object]] = ( + {"result": _RESPONSE, "success": True, "errors": [], "messages": []} if wrapped else _RESPONSE + ) + route: Final = respx_mock.post( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" + ).respond(json=response_body) + + response: Final = await litellm.adecisions( + model=model, + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert route.called + request: Final = respx_mock.calls[0].request + assert request.headers["authorization"] == "Bearer cloudflare-key" + assert json.loads(request.content) == { + "model": "clef", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers == DecisionsResponse.model_validate(_RESPONSE).answers + assert response._hidden_params["model"] == "cloudflare/@cf/cloudflare/clef" + + +@pytest.mark.asyncio +async def test_cloudflare_clef_flash_uses_flash_endpoint_and_request_model( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + route: Final = respx_mock.post( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-flash" + ).respond(json=_RESPONSE) + + await litellm.adecisions( + model="cloudflare/clef-flash", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content)["model"] == "clef-flash" + + +@pytest.mark.asyncio +async def test_cloudflare_api_base_from_env_uses_workers_ai_run_path( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("CLOUDFLARE_API_BASE", "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1") + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False) + route: Final = respx_mock.post( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" + ).respond(json=_RESPONSE) + + await litellm.adecisions( + model="cloudflare/clef", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert route.called + + +@pytest.mark.asyncio +async def test_cloudflare_requires_account_id_or_api_base_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False) + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + + with pytest.raises(litellm.BadRequestError, match="Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID"): + await litellm.adecisions( + model="cloudflare/clef", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_cloudflare_clef_cost_uses_the_model_cost_map( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef").respond( + json=_RESPONSE + ) + + response: Final = await litellm.adecisions( + model="cloudflare/clef", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + cost: Final = litellm.completion_cost(completion_response=response) + clef_cost: Final = litellm.model_cost["cloudflare/@cf/cloudflare/clef"] + expected_cost: Final = _INPUT_TOKENS * float(clef_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + clef_cost["output_cost_per_token"] + ) + + assert expected_cost > 0 + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.asyncio +async def test_strands_decider_requires_api_base_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + + with pytest.raises(litellm.BadRequestError, match="api_base is required"): + await litellm.adecisions( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_strands_decider_without_key_preserves_response_extras( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + + response: Final = await litellm.adecisions( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base="https://strands.example", + ) + + assert route.called + assert "authorization" not in respx_mock.calls[0].request.headers + assert response.model_extra["latency_ms"] == _STRANDS_RESPONSE["latency_ms"] + severity: Final = response.answers["severity"] + assert isinstance(severity, ScoreAnswer) + assert severity.legend == {"0": "none", "1": "low", "2": "high"} + + +@pytest.mark.asyncio +async def test_strands_decider_uses_key_from_matching_environment_base( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("STRANDS_DECIDER_API_BASE", "https://strands.example") + monkeypatch.setenv("STRANDS_DECIDER_API_KEY", "strands-key") + route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + + await litellm.adecisions( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base="https://strands.example", + ) + + assert route.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer strands-key" + + +@pytest.mark.asyncio +async def test_strands_decider_provider_resolution_and_router_dispatch( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + provider_resolution: Final = litellm.get_llm_provider("strands_decider/strands-decider-2B-hobson-v19") + router: Final = litellm.Router( + model_list=[ + { + "model_name": "strands", + "litellm_params": { + "model": "strands_decider/strands-decider-2B-hobson-v19", + "api_base": "https://strands.example", + }, + } + ] + ) + route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + + response: Final = await router.adecisions( + model="strands", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + ) + + assert provider_resolution[:2] == ("strands-decider-2B-hobson-v19", "strands_decider") + assert route.called + assert response.model == _STRANDS_RESPONSE["model"] diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index ad0f82109f3..853fe9ddfad 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1,5 +1,6 @@ """Test health check helper functions""" +import json import socket import struct import zlib @@ -8,6 +9,7 @@ from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +import respx import litellm from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME @@ -528,3 +530,100 @@ def test_ocr_health_check_document_raises_without_the_extension(): _ocr_health_check_document(model="mistral/mistral-ocr-latest", custom_llm_provider="mistral") finally: NATIVE_OCR_HEALTH_CHECK_DOCUMENT.reset() + + +@pytest.mark.parametrize( + ("model", "upstream_url"), + ( + ("perplexity/pplx-decider-v1-27b", "https://api.perplexity.ai/v1/decisions"), + ("cloudflare/clef", "https://api.cloudflare.com/client/v4/accounts/acct-1/ai/run/@cf/cloudflare/clef"), + ), +) +async def test_ahealth_check_probes_evaluation_models_through_the_decisions_api( + model: str, + upstream_url: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct-1") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(upstream_url).respond( + json={ + "model": model, + "answers": {"reachable": {"type": "noul", "noul": 1.0}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check({"model": model, "api_key": "sk-test"}, mode=None) + + assert "error" not in result, result + assert upstream.called + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["state"] == "health check" + assert sent["questions"]["reachable"]["type"] == "noul" + + +@pytest.mark.asyncio +async def test_ahealth_check_evaluation_uses_configured_probe_state_and_questions( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={ + "model": "perplexity/pplx-decider-v1-27b", + "answers": {"ok": {"type": "noul", "noul": 1.0}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check( + { + "model": "perplexity/pplx-decider-v1-27b", + "api_key": "sk-test", + "state": "custom probe", + "questions": {"ok": {"type": "noul", "instructions": "Is it ok?"}}, + }, + mode=None, + ) + + assert "error" not in result, result + assert upstream.called + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["state"] == "custom probe" + assert set(sent["questions"]) == {"ok"} + + +@pytest.mark.asyncio +async def test_ahealth_check_probes_strands_through_decisions_without_mode( + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("http://strands.local:8080/v1/systemone").respond( + json={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"reachable": {"type": "noul", "noul": 1.0}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check( + { + "model": "strands_decider/strands-decider-2B-hobson-v19", + "api_base": "http://strands.local:8080", + }, + mode=None, + ) + + assert "error" not in result, result + assert upstream.called + assert "authorization" not in upstream.calls[0].request.headers diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index caf941ebd19..5f254a3bc6c 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -470,6 +470,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "allm_passthrough_route", "llm_passthrough_route", "asearch", + "adecisions", "avector_store_create", "avector_store_search", "acreate_skill", diff --git a/tests/unit/proxy/decisions_endpoints/__init__.py b/tests/unit/proxy/decisions_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py new file mode 100644 index 00000000000..d5f57d85ffd --- /dev/null +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import Iterator, Mapping +from pathlib import Path +from typing import Final + +import pytest +import respx +from fastapi import FastAPI +from fastapi.routing import APIRoute +from fastapi.testclient import TestClient +from starlette.routing import Match + +import litellm +from litellm.proxy._lazy_features import attach_lazy_features +from litellm.proxy.decisions_endpoints.endpoints import decisions +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder +from litellm.proxy.proxy_server import ( + app, + cleanup_router_config_variables, + initialize, +) + +_INPUT_TOKENS: Final[int] = 367 +_OUTPUT_TOKENS: Final[int] = 3 +_RESPONSE: Final[Mapping[str, object]] = { + "model": "pplx-decider-v1-27b", + "answers": { + "is_defect": {"type": "noul", "noul": 0.9}, + }, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, +} +_STRANDS_RESPONSE: Final[Mapping[str, object]] = { + "model": "strands-decider-2B-hobson-v19", + "answers": { + "is_defect": {"type": "noul", "noul": 0.9}, + }, + "usage": {"input_tokens": 216, "output_tokens": 3}, + "latency_ms": 3722.17, +} +_REQUEST: Final[Mapping[str, object]] = { + "model": "decider", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, +} + + +@pytest.fixture +def client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + monkeypatch.setenv("OPENAI_API_KEY", "fake-openai-key") + monkeypatch.setenv("OPENAI_API_BASE", "https://fake-openai.example") + monkeypatch.setenv("REDIS_HOST", "localhost") + cleanup_router_config_variables() + config_path: Final = Path(__file__).parents[1] / "test_configs" / "test_config_no_auth.yaml" + asyncio.run(initialize(config=str(config_path), debug=True)) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "decider", + "litellm_params": { + "model": "perplexity/pplx-decider-v1-27b", + "api_key": "test-key", + }, + } + ] + ), + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield TestClient(app) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions")) +def test_proxy_decisions_route_returns_answers_and_cost( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post(endpoint, json=_REQUEST) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == _RESPONSE["answers"] + assert "_hidden_params" not in response.json() + perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] + expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + perplexity_cost["output_cost_per_token"] + ) + + assert expected_cost > 0 + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) + assert upstream.called + assert json.loads(upstream.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert upstream.calls[0].request.headers["authorization"] == "Bearer test-key" + + +def test_proxy_decisions_dispatches_typesafe_deployment( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": { + "model": "typesafe/jev-latest", + "api_key": "k", + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) + + response: Final = client.post( + "/v1/decisions", + json={ + "model": "jev", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == _RESPONSE["answers"] + assert upstream.called + assert json.loads(upstream.calls[0].request.content) == { + "model": "jev-latest", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + +def test_proxy_decisions_sends_the_env_key_to_the_deployment_api_base( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("PERPLEXITYAI_API_KEY", "server-key") + monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "decider", + "litellm_params": { + "model": "perplexity/pplx-decider-v1-27b", + "api_base": "https://egress.example/perplexity", + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + upstream: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post("/v1/decisions", json=_REQUEST) + + assert response.status_code == 200, response.text + assert upstream.call_count == 1 + assert upstream.calls[0].request.headers["authorization"] == "Bearer server-key" + + +def test_proxy_decisions_unknown_model_is_a_client_error( + client: TestClient, + respx_mock: respx.MockRouter, +) -> None: + response: Final = client.post( + "/v1/decisions", + json={ + "model": "missing-model", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + ) + + assert 400 <= response.status_code < 500, response.text + assert len(respx_mock.calls) == 0 + + +@pytest.mark.parametrize( + "request_body", + ( + { + "model": "decider", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + { + "model": "decider", + "state": {"source": "proxy-test"}, + }, + ), + ids=("missing_state", "missing_questions"), +) +def test_proxy_decisions_missing_required_field_is_a_client_error( + client: TestClient, + respx_mock: respx.MockRouter, + request_body: Mapping[str, object], +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post("/v1/decisions", json=request_body) + + assert response.status_code == 400, response.text + assert not upstream.called + + +def test_proxy_decisions_dispatches_strands_decider( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "strands", + "litellm_params": { + "model": "strands_decider/strands-decider-2B-hobson-v19", + "api_base": "https://strands.example", + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + upstream: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + + response: Final = client.post( + "/v1/decisions", + json={ + "model": "strands", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == _STRANDS_RESPONSE["answers"] + assert upstream.called + assert json.loads(upstream.calls[0].request.content) == { + "model": "strands-decider-2B-hobson-v19", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert "authorization" not in upstream.calls[0].request.headers + + +def test_proxy_decisions_without_model_uses_the_proxy_default_model( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm.proxy.proxy_server, "user_model", "decider") + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post( + "/v1/decisions", json={key: value for key, value in _REQUEST.items() if key != "model"} + ) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == _RESPONSE["answers"] + assert upstream.called + assert json.loads(upstream.calls[0].request.content)["model"] == "pplx-decider-v1-27b" + + +def _serving_endpoint(bare: FastAPI, path: str) -> object: + scope: Final = {"type": "http", "method": "POST", "path": path, "root_path": "", "query_string": b"", "headers": ()} + return next( + route.endpoint for route in bare.routes if isinstance(route, APIRoute) and route.matches(scope)[0] is Match.FULL + ) + + +def test_a_config_pass_through_at_v1_decisions_keeps_its_route_and_the_native_api_serves_decisions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LITELLM_DISABLE_LAZY_ROUTES", raising=False) + + async def pass_through() -> dict[str, str]: + return {"served_by": "pass-through"} + + bare: Final = FastAPI() + attach_lazy_features(bare) + SafeRouteAdder.add_api_route_if_not_exists(bare, "/v1/decisions", pass_through, ["POST"]) + with TestClient(bare) as client: + assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} + assert _serving_endpoint(bare, "/v1/decisions") is pass_through + assert _serving_endpoint(bare, "/decisions") is decisions diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 30b322f7b73..637cfb00d84 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4398,6 +4398,23 @@ export interface paths { patch?: never; trace?: never; }; + "/decisions": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Decisions */ + post: operations["decisions_decisions_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/delete/allowed_ip": { parameters: { query?: never; @@ -18644,6 +18661,23 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/decisions": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Decisions */ + post: operations["decisions_v1_decisions_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/embeddings": { parameters: { query?: never; @@ -26667,7 +26701,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ @@ -53175,6 +53209,26 @@ export interface operations { }; }; }; + decisions_decisions_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; delete_allowed_ip_delete_allowed_ip_post: { parameters: { query?: never; @@ -70747,6 +70801,26 @@ export interface operations { }; }; }; + decisions_v1_decisions_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; embeddings_v1_embeddings_post: { parameters: { query?: never; From c24cbf6442b99726c8c16861a945cca6dc5bc7a2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:06:21 -0700 Subject: [PATCH 02/17] feat(decisions): serve System One format at /v1/systemone and OpenAI format at /v1/decisions Backport of #45184 to stable/1.104.x. Cherry-picked from 3339469e73 (litellm_systemone_endpoint, #45184 is not merged yet). The playground System One tab and managed agent authorization are not on this line, so those hunks are dropped, and the lazy-disabled precedence test is dropped as in the #44236 pick. --- gateway/routes/allowlist.py | 2 + litellm/decisions/openai_transformation.py | 127 +++++++++++++ litellm/proxy/_lazy_features.py | 2 +- litellm/proxy/_lazy_openapi_snapshot.json | 48 +++++ litellm/proxy/_types.py | 2 + .../proxy/decisions_endpoints/endpoints.py | 125 ++++++++++--- litellm/types/decisions.py | 168 +++++++++++++++++- litellm/types/utils.py | 2 + .../decisions_endpoints/test_endpoints.py | 166 ++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 74 ++++++++ 10 files changed, 677 insertions(+), 39 deletions(-) create mode 100644 litellm/decisions/openai_transformation.py diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 1999452d5db..74468fa86d9 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -62,6 +62,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/rerank", "/v1/decisions", "/decisions", + "/v1/systemone", + "/systemone", "/v1/ocr", "/ocr", "/v1/rag/", diff --git a/litellm/decisions/openai_transformation.py b/litellm/decisions/openai_transformation.py new file mode 100644 index 00000000000..b6d24b741bb --- /dev/null +++ b/litellm/decisions/openai_transformation.py @@ -0,0 +1,127 @@ +from collections.abc import Mapping, Sequence +from typing import Final, assert_never + +from litellm.types.decisions import ( + ChoiceAnswer, + DecisionAnswer, + DecisionsResponse, + DecisionsUsage, + NoulAnswer, + OpenAIChoiceAnswer, + OpenAIChoiceProbability, + OpenAIChoiceQuestion, + OpenAIDecisionAnswer, + OpenAIDecisionInputMessage, + OpenAIDecisionQuestion, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, + OpenAIDecisionUsage, + OpenAIPredicateAnswer, + OpenAIPredicateQuestion, + OpenAIRefusalAnswer, + OpenAIScoreAnswer, + OpenAIScoreLevel, + OpenAIScoreProbability, + OpenAIScoreQuestion, + ScoreAnswer, + systemone_choice_key, +) + +_OPENAI_ONLY_FIELDS: Final = frozenset({"input", "questions", "safety_identifier"}) + + +def _message_text(message: OpenAIDecisionInputMessage) -> str: + if isinstance(message.content, str): + return message.content + return "\n\n".join(part.text for part in message.content) + + +def _state(decision_input: str | Sequence[OpenAIDecisionInputMessage]) -> str: + if isinstance(decision_input, str): + return decision_input + return "\n\n".join(_message_text(message) for message in decision_input) + + +def _level_criterion(level: OpenAIScoreLevel) -> str: + return level.label if level.description is None else f"{level.label}: {level.description}" + + +def _systemone_question(question: OpenAIDecisionQuestion) -> Mapping[str, object]: + match question: + case OpenAIPredicateQuestion(): + return {"type": "noul", "instructions": question.instructions} + case OpenAIChoiceQuestion(): + return { + "type": "choice", + "instructions": question.instructions, + "criteria": {systemone_choice_key(option.value): option.description for option in question.choices}, + } + case OpenAIScoreQuestion(): + return { + "type": "score", + "instructions": question.instructions, + "criteria": [_level_criterion(level) for level in question.levels], + } + case _: + assert_never(question) + + +def to_systemone_request( + request_data: Mapping[str, object], body: OpenAIDecisionRequestBody +) -> Mapping[str, object]: + return { + **{key: value for key, value in request_data.items() if key not in _OPENAI_ONLY_FIELDS}, + "state": _state(body.input), + "questions": {str(index): _systemone_question(question) for index, question in enumerate(body.questions)}, + } + + +def _openai_answer(question: OpenAIDecisionQuestion, answer: DecisionAnswer | None) -> OpenAIDecisionAnswer: + match question, answer: + case OpenAIPredicateQuestion(), NoulAnswer(): + return OpenAIPredicateAnswer(name=question.name, probability=answer.noul) + case OpenAIChoiceQuestion(), ChoiceAnswer(): + typed_values: Final = {systemone_choice_key(option.value): option.value for option in question.choices} + return OpenAIChoiceAnswer( + name=question.name, + choice=typed_values.get(answer.choice, answer.choice), + probabilities=tuple( + OpenAIChoiceProbability( + value=option.value, + probability=answer.probabilities.get(systemone_choice_key(option.value), 0.0), + ) + for option in question.choices + ), + confidence=answer.confidence, + ) + case OpenAIScoreQuestion(), ScoreAnswer(): + return OpenAIScoreAnswer( + name=question.name, + score=answer.score, + probabilities=tuple( + OpenAIScoreProbability( + value=index, label=level.label, probability=answer.probabilities.get(str(index), 0.0) + ) + for index, level in enumerate(question.levels) + ), + confidence=answer.confidence, + ) + case _: + return OpenAIRefusalAnswer(name=question.name) + + +def to_openai_response( + response: DecisionsResponse, questions: Sequence[OpenAIDecisionQuestion], requested_model: str +) -> OpenAIDecisionResponse: + usage: Final = response.usage or DecisionsUsage() + return OpenAIDecisionResponse( + model=response.model or requested_model, + answers=tuple( + _openai_answer(question, response.answers.get(str(index))) for index, question in enumerate(questions) + ), + usage=OpenAIDecisionUsage( + input_tokens=usage.input_tokens, + output_tokens=usage.output_tokens, + total_tokens=usage.input_tokens + usage.output_tokens, + ), + ) diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 810a0e18723..29a772f33d6 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -255,7 +255,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( LazyFeature( name="decisions", module_path="litellm.proxy.decisions_endpoints.endpoints", - path_prefixes=("/v1/decisions", "/decisions"), + path_prefixes=("/v1/decisions", "/decisions", "/v1/systemone", "/systemone"), ), LazyFeature( name="claude_code_marketplace", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 0270be5720b..2759fc34bf3 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -8091,6 +8091,30 @@ ] } }, + "/systemone": { + "post": { + "operationId": "systemone_systemone_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Systemone", + "tags": [ + "decisions" + ] + } + }, "/v1/decisions": { "post": { "operationId": "decisions_v1_decisions_post", @@ -8114,6 +8138,30 @@ "decisions" ] } + }, + "/v1/systemone": { + "post": { + "operationId": "systemone_v1_systemone_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Systemone", + "tags": [ + "decisions" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b2716389dba..2b246882d5f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -468,6 +468,8 @@ class LiteLLMRoutes(enum.Enum): "/v1/search/{search_tool_name}", "/decisions", "/v1/decisions", + "/systemone", + "/v1/systemone", # OCR "/ocr", "/v1/ocr", diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 7dba64ee791..2b908a8a089 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -1,39 +1,52 @@ +from collections.abc import Mapping from typing import Annotated, Final from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract from pydantic import TypeAdapter, ValidationError +from litellm.decisions.openai_transformation import to_openai_response, to_systemone_request from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.decisions import DecisionsRequestBody +from litellm.types.decisions import DecisionsRequestBody, DecisionsResponse, OpenAIDecisionRequestBody router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter( + OpenAIDecisionRequestBody +) +_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) _OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) -@router.post( - "/v1/decisions", - dependencies=[Depends(user_api_key_auth)], - response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract - tags=["decisions"], -) -@router.post( - "/decisions", - dependencies=[Depends(user_api_key_auth)], - response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract - tags=["decisions"], -) -async def decisions( +async def _invalid_request( + raw_data: Mapping[str, object], error: ValidationError, user_api_key_dict: UserAPIKeyAuth +) -> Exception: + from litellm.proxy.proxy_server import proxy_logging_obj, version + + return await ProxyBaseLLMRequestProcessing(data=dict(raw_data))._handle_llm_api_exception( + e=BadRequestError( + message=f"Invalid Decisions request: {error}", + model=str(raw_data.get("model", "")), + llm_provider="", + ), + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +async def _process_systemone( request: Request, fastapi_response: Response, - user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], -): + user_api_key_dict: UserAPIKeyAuth, + raw_data: Mapping[str, object], + openai_body: OpenAIDecisionRequestBody | None, +) -> object: from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, ) @@ -55,7 +68,7 @@ async def decisions( user_temperature as proxy_user_temperature, ) - data: Final = _REQUEST_DATA_ADAPTER.validate_json(await request.body()) + data: Final = dict(raw_data if openai_body is None else to_systemone_request(raw_data, openai_body)) general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings) user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base) user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model) @@ -63,7 +76,7 @@ async def decisions( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: _DECISIONS_REQUEST_BODY_ADAPTER.validate_python(data) - return await processor.base_process_llm_request( + result: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -81,18 +94,15 @@ async def decisions( user_api_base=user_api_base, version=version, ) + if openai_body is None or isinstance(result, Response): + return result + return to_openai_response( + _DECISIONS_RESPONSE_ADAPTER.validate_python(result), + openai_body.questions, + str(data.get("model", "")), + ) except ValidationError as error: - bad_request_error: Final = BadRequestError( - message=f"Invalid Decisions request: {error}", - model=str(data.get("model", "")), - llm_provider="", - ) - raise await processor._handle_llm_api_exception( - e=bad_request_error, - user_api_key_dict=user_api_key_dict, - proxy_logging_obj=proxy_logging_obj, - version=version, - ) + raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict) except Exception as error: raise await processor._handle_llm_api_exception( e=error, @@ -100,3 +110,60 @@ async def decisions( proxy_logging_obj=proxy_logging_obj, version=version, ) + + +@router.post( + "/v1/systemone", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +@router.post( + "/systemone", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +async def systemone( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + return await _process_systemone( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + raw_data=_REQUEST_DATA_ADAPTER.validate_json(await request.body()), + openai_body=None, + ) + + +@router.post( + "/v1/decisions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +@router.post( + "/decisions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +async def decisions( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + raw_data: Final = _REQUEST_DATA_ADAPTER.validate_json(await request.body()) + try: + openai_body: Final = _OPENAI_DECISION_REQUEST_BODY_ADAPTER.validate_python(raw_data) + except ValidationError as error: + raise await _invalid_request(raw_data=raw_data, error=error, user_api_key_dict=user_api_key_dict) + return await _process_systemone( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + raw_data=raw_data, + openai_body=openai_body, + ) diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 80db26f201d..a49199773ec 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from typing import Annotated, Literal, TypeAlias +from typing import Annotated, Final, Literal, TypeAlias from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config from typing_extensions import ReadOnly, Required, TypedDict @@ -121,3 +121,169 @@ class DecisionsResponse(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="allow", frozen=True) _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) + + +class OpenAIDecisionInputText(LiteLLMPydanticObjectBase): + type: Literal["input_text"] + text: str + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class OpenAIDecisionInputMessage(LiteLLMPydanticObjectBase): + role: Literal["user"] = "user" + type: Literal["message"] = "message" + content: str | Sequence[OpenAIDecisionInputText] + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase): + type: Literal["predicate"] + name: str | None = None + instructions: str + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class OpenAIChoiceOption(LiteLLMPydanticObjectBase): + value: str | bool + description: str | None = None + + model_config = ConfigDict(extra="forbid", frozen=True) + + +def systemone_choice_key(value: str | bool) -> str: + if isinstance(value, bool): + return "true" if value else "false" + return value + + +class OpenAIChoiceQuestion(LiteLLMPydanticObjectBase): + type: Literal["choice"] + name: str | None = None + instructions: str + choices: Annotated[Sequence[OpenAIChoiceOption], Field(min_length=2, max_length=255)] + + model_config = ConfigDict(extra="forbid", frozen=True) + + @model_validator(mode="after") + def require_unique_systemone_keys(self) -> "OpenAIChoiceQuestion": + keys: Final = frozenset(systemone_choice_key(option.value) for option in self.choices) + if len(keys) != len(self.choices): + raise ValueError("Choice values must be unique, and a boolean cannot share its text with a string choice") + return self + + +class OpenAIScoreLevel(LiteLLMPydanticObjectBase): + label: str + description: str | None = None + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class OpenAIScoreQuestion(LiteLLMPydanticObjectBase): + type: Literal["score"] + name: str | None = None + instructions: str + levels: Annotated[Sequence[OpenAIScoreLevel], Field(min_length=2, max_length=10)] + + model_config = ConfigDict(extra="forbid", frozen=True) + + +OpenAIDecisionQuestion: TypeAlias = Annotated[ + OpenAIPredicateQuestion | OpenAIChoiceQuestion | OpenAIScoreQuestion, + Field(discriminator="type"), +] + + +class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): + input: str | Sequence[OpenAIDecisionInputMessage] + questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=200)] + safety_identifier: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase): + type: Literal["predicate"] = "predicate" + name: str | None + probability: float + + model_config = ConfigDict(frozen=True) + + +class OpenAIChoiceProbability(LiteLLMPydanticObjectBase): + value: str | bool + probability: float + + model_config = ConfigDict(frozen=True) + + +class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase): + type: Literal["choice"] = "choice" + name: str | None + choice: str | bool + probabilities: tuple[OpenAIChoiceProbability, ...] + confidence: float + + model_config = ConfigDict(frozen=True) + + +class OpenAIScoreProbability(LiteLLMPydanticObjectBase): + value: int + label: str + probability: float + + model_config = ConfigDict(frozen=True) + + +class OpenAIScoreAnswer(LiteLLMPydanticObjectBase): + type: Literal["score"] = "score" + name: str | None + score: float + probabilities: tuple[OpenAIScoreProbability, ...] + confidence: float + + model_config = ConfigDict(frozen=True) + + +class OpenAIRefusalAnswer(LiteLLMPydanticObjectBase): + type: Literal["refusal"] = "refusal" + name: str | None + + model_config = ConfigDict(frozen=True) + + +OpenAIDecisionAnswer: TypeAlias = OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIRefusalAnswer + + +class OpenAIDecisionInputTokensDetails(LiteLLMPydanticObjectBase): + cached_tokens: int = 0 + cache_write_tokens: int = 0 + + model_config = ConfigDict(frozen=True) + + +class OpenAIDecisionOutputTokensDetails(LiteLLMPydanticObjectBase): + reasoning_tokens: int = 0 + + model_config = ConfigDict(frozen=True) + + +class OpenAIDecisionUsage(LiteLLMPydanticObjectBase): + input_tokens: int + input_tokens_details: OpenAIDecisionInputTokensDetails = OpenAIDecisionInputTokensDetails() + output_tokens: int + output_tokens_details: OpenAIDecisionOutputTokensDetails = OpenAIDecisionOutputTokensDetails() + total_tokens: int + + model_config = ConfigDict(frozen=True) + + +class OpenAIDecisionResponse(LiteLLMPydanticObjectBase): + model: str + answers: tuple[OpenAIDecisionAnswer, ...] + usage: OpenAIDecisionUsage + + model_config = ConfigDict(frozen=True) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 791541f9804..c91efdb63d7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -752,6 +752,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/search": [CallTypes.asearch, CallTypes.search], "/decisions": [CallTypes.adecisions, CallTypes.decisions], "/v1/decisions": [CallTypes.adecisions, CallTypes.decisions], + "/systemone": [CallTypes.adecisions, CallTypes.decisions], + "/v1/systemone": [CallTypes.adecisions, CallTypes.decisions], # Batches "/batches": [CallTypes.acreate_batch, CallTypes.create_batch], "/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch], diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index d5f57d85ffd..2da8d472596 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -15,7 +15,7 @@ from starlette.routing import Match import litellm from litellm.proxy._lazy_features import attach_lazy_features -from litellm.proxy.decisions_endpoints.endpoints import decisions +from litellm.proxy.decisions_endpoints.endpoints import decisions, systemone from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder from litellm.proxy.proxy_server import ( app, @@ -76,7 +76,7 @@ def client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: litellm.in_memory_llm_clients_cache.flush_cache() -@pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions")) +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone")) def test_proxy_decisions_route_returns_answers_and_cost( client: TestClient, respx_mock: respx.MockRouter, @@ -125,7 +125,7 @@ def test_proxy_decisions_dispatches_typesafe_deployment( upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) response: Final = client.post( - "/v1/decisions", + "/v1/systemone", json={ "model": "jev", "state": {"source": "proxy-test"}, @@ -164,7 +164,7 @@ def test_proxy_decisions_sends_the_env_key_to_the_deployment_api_base( monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) upstream: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE) - response: Final = client.post("/v1/decisions", json=_REQUEST) + response: Final = client.post("/v1/systemone", json=_REQUEST) assert response.status_code == 200, response.text assert upstream.call_count == 1 @@ -176,7 +176,7 @@ def test_proxy_decisions_unknown_model_is_a_client_error( respx_mock: respx.MockRouter, ) -> None: response: Final = client.post( - "/v1/decisions", + "/v1/systemone", json={ "model": "missing-model", "state": "review", @@ -209,7 +209,7 @@ def test_proxy_decisions_missing_required_field_is_a_client_error( ) -> None: upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) - response: Final = client.post("/v1/decisions", json=request_body) + response: Final = client.post("/v1/systemone", json=request_body) assert response.status_code == 400, response.text assert not upstream.called @@ -237,7 +237,7 @@ def test_proxy_decisions_dispatches_strands_decider( upstream: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) response: Final = client.post( - "/v1/decisions", + "/v1/systemone", json={ "model": "strands", "state": {"source": "proxy-test"}, @@ -265,7 +265,7 @@ def test_proxy_decisions_without_model_uses_the_proxy_default_model( upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) response: Final = client.post( - "/v1/decisions", json={key: value for key, value in _REQUEST.items() if key != "model"} + "/v1/systemone", json={key: value for key, value in _REQUEST.items() if key != "model"} ) assert response.status_code == 200, response.text @@ -274,6 +274,154 @@ def test_proxy_decisions_without_model_uses_the_proxy_default_model( assert json.loads(upstream.calls[0].request.content)["model"] == "pplx-decider-v1-27b" +_OPENAI_FORMAT_REQUEST: Final[Mapping[str, object]] = { + "model": "decider", + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "The package arrived with a broken screen."}, + {"type": "input_text", "text": "I want a refund."}, + ], + }, + {"role": "user", "content": "Order 1234."}, + ], + "questions": [ + {"type": "predicate", "name": "damaged", "instructions": "Does the customer report a damaged item?"}, + { + "type": "choice", + "instructions": "Should we refund?", + "choices": [{"value": True, "description": "Refund now"}, {"value": "escalate"}], + }, + { + "type": "score", + "name": "severity", + "instructions": "How severe is the issue?", + "levels": [{"label": "minor"}, {"label": "major", "description": "Product unusable"}], + }, + {"type": "predicate", "name": "fraud", "instructions": "Is this fraud?"}, + ], + "safety_identifier": "end-user-1", +} +_SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST: Final[Mapping[str, object]] = { + "model": "pplx-decider-v1-27b", + "answers": { + "0": {"type": "noul", "noul": 0.95}, + "1": {"type": "choice", "choice": "true", "confidence": 0.8, "probabilities": {"true": 0.9, "escalate": 0.1}}, + "2": { + "type": "score", + "score": 0.7, + "confidence": 0.6, + "legend": {"0": "minor", "1": "major: Product unusable"}, + "probabilities": {"0": 0.3, "1": 0.7}, + }, + }, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, +} + + +@pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions")) +def test_openai_format_decisions_translate_through_systemone( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json=_SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST + ) + + response: Final = client.post(endpoint, json=_OPENAI_FORMAT_REQUEST) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": "The package arrived with a broken screen.\n\nI want a refund.\n\nOrder 1234.", + "questions": { + "0": {"type": "noul", "instructions": "Does the customer report a damaged item?"}, + "1": { + "type": "choice", + "instructions": "Should we refund?", + "criteria": {"true": "Refund now", "escalate": None}, + }, + "2": { + "type": "score", + "instructions": "How severe is the issue?", + "criteria": ["minor", "major: Product unusable"], + }, + "3": {"type": "noul", "instructions": "Is this fraud?"}, + }, + } + body: Final = response.json() + assert body["model"] == _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST["model"] + assert body["answers"] == [ + {"type": "predicate", "name": "damaged", "probability": 0.95}, + { + "type": "choice", + "name": None, + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": "major", "probability": 0.7}, + ], + "confidence": 0.6, + }, + {"type": "refusal", "name": "fraud"}, + ] + assert body["usage"] == { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + } + assert float(response.headers["x-litellm-response-cost"]) > 0 + + +@pytest.mark.parametrize( + "request_body", + ( + _REQUEST, + { + "model": "decider", + "input": "review", + "questions": [ + { + "type": "choice", + "instructions": "Pick one", + "choices": [{"value": True}, {"value": "true"}], + } + ], + }, + { + "model": "decider", + "input": [ + {"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]} + ], + "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], + }, + ), + ids=("systemone_body", "colliding_choice_values", "image_input"), +) +def test_openai_format_decisions_rejects_bodies_it_cannot_translate( + client: TestClient, + respx_mock: respx.MockRouter, + request_body: Mapping[str, object], +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post("/v1/decisions", json=request_body) + + assert response.status_code == 400, response.text + assert not upstream.called + + def _serving_endpoint(bare: FastAPI, path: str) -> object: scope: Final = {"type": "http", "method": "POST", "path": path, "root_path": "", "query_string": b"", "headers": ()} return next( @@ -296,3 +444,5 @@ def test_a_config_pass_through_at_v1_decisions_keeps_its_route_and_the_native_ap assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} assert _serving_endpoint(bare, "/v1/decisions") is pass_through assert _serving_endpoint(bare, "/decisions") is decisions + assert _serving_endpoint(bare, "/v1/systemone") is systemone + assert _serving_endpoint(bare, "/systemone") is systemone diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 637cfb00d84..4faccf413bb 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -15167,6 +15167,23 @@ export interface paths { patch?: never; trace?: never; }; + "/systemone": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Systemone */ + post: operations["systemone_systemone_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/tag/daily/activity": { parameters: { query?: never; @@ -20960,6 +20977,23 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/systemone": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Systemone */ + post: operations["systemone_v1_systemone_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/threads": { parameters: { query?: never; @@ -66333,6 +66367,26 @@ export interface operations { }; }; }; + systemone_systemone_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_tag_daily_activity_tag_daily_activity_get: { parameters: { query?: { @@ -73882,6 +73936,26 @@ export interface operations { }; }; }; + systemone_v1_systemone_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; create_threads_v1_threads_post: { parameters: { query?: never; From 77d57c394863ca2afd6faa0f4003dcd74d7b63b4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:12:45 -0700 Subject: [PATCH 03/17] fix(decisions): import assert_never from typing_extensions for Python 3.10 Backport of the #45184 follow-up to stable/1.104.x. Cherry-picked from 29e715473c (litellm_systemone_endpoint). --- litellm/decisions/openai_transformation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/decisions/openai_transformation.py b/litellm/decisions/openai_transformation.py index b6d24b741bb..af2c838b9ab 100644 --- a/litellm/decisions/openai_transformation.py +++ b/litellm/decisions/openai_transformation.py @@ -1,5 +1,7 @@ from collections.abc import Mapping, Sequence -from typing import Final, assert_never +from typing import Final + +from typing_extensions import assert_never from litellm.types.decisions import ( ChoiceAnswer, @@ -66,9 +68,7 @@ def _systemone_question(question: OpenAIDecisionQuestion) -> Mapping[str, object assert_never(question) -def to_systemone_request( - request_data: Mapping[str, object], body: OpenAIDecisionRequestBody -) -> Mapping[str, object]: +def to_systemone_request(request_data: Mapping[str, object], body: OpenAIDecisionRequestBody) -> Mapping[str, object]: return { **{key: value for key, value in request_data.items() if key not in _OPENAI_ONLY_FIELDS}, "state": _state(body.input), From 5f4588dec437201795c85c74bd738ae78ff2664d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:27:22 -0700 Subject: [PATCH 04/17] fix(decisions): cap OpenAI-format questions at the System One limit Backport of the #45184 follow-up to stable/1.104.x. Cherry-picked from 7ef132947a (litellm_systemone_endpoint). --- litellm/types/decisions.py | 5 +-- .../decisions/test_openai_transformation.py | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 tests/unit/decisions/test_openai_transformation.py diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index a49199773ec..22d47ccda2e 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -8,6 +8,7 @@ from litellm.types.llms.base import LiteLLMPydanticObjectBase DecisionsJSON: TypeAlias = str | Mapping[str, object] | Sequence[object] NoulCriteria: TypeAlias = Mapping[Literal["true", "false"], DecisionsJSON | None] +MAX_DECISION_QUESTIONS: Final = 128 class NoulQuestion(LiteLLMPydanticObjectBase): @@ -47,7 +48,7 @@ DecisionQuestion: TypeAlias = Annotated[ DecisionQuestionMap: TypeAlias = Annotated[ Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion], - Field(min_length=1, max_length=128), + Field(min_length=1, max_length=MAX_DECISION_QUESTIONS), ] @@ -199,7 +200,7 @@ OpenAIDecisionQuestion: TypeAlias = Annotated[ class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): input: str | Sequence[OpenAIDecisionInputMessage] - questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=200)] + questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)] safety_identifier: str | None = None model_config = ConfigDict(extra="allow", frozen=True) diff --git a/tests/unit/decisions/test_openai_transformation.py b/tests/unit/decisions/test_openai_transformation.py new file mode 100644 index 00000000000..a3cc1583d40 --- /dev/null +++ b/tests/unit/decisions/test_openai_transformation.py @@ -0,0 +1,32 @@ +from collections.abc import Mapping +from typing import Final + +import pytest +from pydantic import TypeAdapter, ValidationError + +from litellm.decisions.openai_transformation import to_systemone_request +from litellm.types.decisions import MAX_DECISION_QUESTIONS, DecisionsRequestBody, OpenAIDecisionRequestBody + +_OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) +_SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) + + +def _openai_request(question_count: int) -> Mapping[str, object]: + return { + "model": "decider", + "input": "The package arrived with a broken screen.", + "questions": [{"type": "predicate", "instructions": f"Question {index}?"} for index in range(question_count)], + } + + +def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_request() -> None: + raw: Final = _openai_request(MAX_DECISION_QUESTIONS) + + translated: Final = _SYSTEMONE_BODY.validate_python(to_systemone_request(raw, _OPENAI_BODY.validate_python(raw))) + + assert len(translated.questions) == MAX_DECISION_QUESTIONS + + +def test_an_openai_request_with_more_questions_than_systemone_takes_is_rejected_before_translation() -> None: + with pytest.raises(ValidationError, match="questions"): + _OPENAI_BODY.validate_python(_openai_request(MAX_DECISION_QUESTIONS + 1)) From d9afde2a638a26124d2a588ad231570344e20a06 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:27:38 -0700 Subject: [PATCH 05/17] fix(decisions): return 400 for bodies that are not JSON Backport of the #45184 follow-up to stable/1.104.x. Cherry-picked from b6bb885324 (litellm_systemone_endpoint). --- litellm/proxy/decisions_endpoints/endpoints.py | 12 ++++++++++-- .../proxy/decisions_endpoints/test_endpoints.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 2b908a8a089..4170e3a13c3 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -40,6 +40,14 @@ async def _invalid_request( ) +async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + body: Final = await request.body() + try: + return _REQUEST_DATA_ADAPTER.validate_json(body) + except ValidationError as error: + raise await _invalid_request(raw_data={}, error=error, user_api_key_dict=user_api_key_dict) + + async def _process_systemone( request: Request, fastapi_response: Response, @@ -133,7 +141,7 @@ async def systemone( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, - raw_data=_REQUEST_DATA_ADAPTER.validate_json(await request.body()), + raw_data=await _request_data(request, user_api_key_dict), openai_body=None, ) @@ -155,7 +163,7 @@ async def decisions( fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): - raw_data: Final = _REQUEST_DATA_ADAPTER.validate_json(await request.body()) + raw_data: Final = await _request_data(request, user_api_key_dict) try: openai_body: Final = _OPENAI_DECISION_REQUEST_BODY_ADAPTER.validate_python(raw_data) except ValidationError as error: diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 2da8d472596..0a3ee9d1a4c 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -422,6 +422,22 @@ def test_openai_format_decisions_rejects_bodies_it_cannot_translate( assert not upstream.called +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/v1/decisions")) +@pytest.mark.parametrize("raw_body", (b"", b"{not json"), ids=("empty", "malformed")) +def test_a_body_that_is_not_json_is_a_client_error( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, + raw_body: bytes, +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post(endpoint, content=raw_body, headers={"Content-Type": "application/json"}) + + assert response.status_code == 400, response.text + assert not upstream.called + + def _serving_endpoint(bare: FastAPI, path: str) -> object: scope: Final = {"type": "http", "method": "POST", "path": path, "root_path": "", "query_string": b"", "headers": ()} return next( From ef6cbbc6f59a47fa2169b376688862f21d81ae25 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:06:33 -0700 Subject: [PATCH 06/17] bump: version 1.104.2 --- pyproject.toml | 4 ++-- uv.lock | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 62bd89046d9..3ed369141e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.104.1" +version = "1.104.2" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.15" @@ -355,7 +355,7 @@ litellm-enterprise = { workspace = true } members = ["enterprise", "litellm-proxy-extras"] [tool.commitizen] -version = "1.104.1" +version = "1.104.2" version_files = [ "pyproject.toml:^version", ] diff --git a/uv.lock b/uv.lock index 9d0b0353b08..999c032012f 100644 --- a/uv.lock +++ b/uv.lock @@ -4500,7 +4500,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.104.1" +version = "1.104.2" source = { editable = "." } dependencies = [ { name = "aiohttp" }, From 138f901e090300633462be51db03615556e8f17b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:34:58 -0700 Subject: [PATCH 07/17] feat(decisions): add OpenAI as a Decisions provider System One requests to openai/ models are translated to OpenAI's Decisions shape on the way out and OpenAI's answers are translated back, so both /v1/systemone and /v1/decisions can route to gpt-6-luna. (cherry picked from commit b94bbbc3187ae2afa3141cc818a0de8b6a52c119) --- litellm/decisions/main.py | 13 +- litellm/decisions/openai_transformation.py | 98 +++++++++++++++ .../llms/base_llm/decisions/transformation.py | 18 +++ .../cloudflare/decisions/transformation.py | 5 + .../llms/openai/decisions/transformation.py | 37 ++++++ ...odel_prices_and_context_window_backup.json | 3 +- litellm/types/decisions.py | 5 +- model_prices_and_context_window.json | 3 +- tests/unit/decisions/test_main.py | 73 +++++++++++ .../decisions/test_openai_transformation.py | 103 ++++++++++++++- .../decisions_endpoints/test_endpoints.py | 117 ++++++++++++++---- 11 files changed, 440 insertions(+), 35 deletions(-) create mode 100644 litellm/llms/openai/decisions/transformation.py diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index da037f1d8cb..09464c8884d 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT from litellm.llms.custom_httpx.http_handler import _get_httpx_client, get_async_httpx_client +from litellm.llms.openai.decisions.transformation import OPENAI_DECISIONS_ENDPOINT from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT @@ -31,6 +32,7 @@ DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxy "openrouter": OPENROUTER_DECISIONS_ENDPOINT, "cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT, "strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT, + "openai": OPENAI_DECISIONS_ENDPOINT, } ) @@ -147,16 +149,7 @@ def _prepare_request( "Content-Type": "application/json", } ) - body: Final = MappingProxyType( - { - "model": endpoint.request_model(canonical_model), - "state": validated_request.state, - "questions": { - name: question.model_dump(mode="json", exclude_none=True) - for name, question in validated_request.questions.items() - }, - } - ) + body: Final = MappingProxyType(endpoint.request_body(endpoint.request_model(canonical_model), validated_request)) return _PreparedDecisionsRequest( config=endpoint, provider=provider, diff --git a/litellm/decisions/openai_transformation.py b/litellm/decisions/openai_transformation.py index af2c838b9ab..bdbfd7df0a1 100644 --- a/litellm/decisions/openai_transformation.py +++ b/litellm/decisions/openai_transformation.py @@ -1,14 +1,21 @@ +import json from collections.abc import Mapping, Sequence from typing import Final +from pydantic import TypeAdapter from typing_extensions import assert_never from litellm.types.decisions import ( ChoiceAnswer, + ChoiceQuestion, DecisionAnswer, + DecisionQuestion, + DecisionsJSON, + DecisionsRequestBody, DecisionsResponse, DecisionsUsage, NoulAnswer, + NoulQuestion, OpenAIChoiceAnswer, OpenAIChoiceProbability, OpenAIChoiceQuestion, @@ -26,10 +33,12 @@ from litellm.types.decisions import ( OpenAIScoreProbability, OpenAIScoreQuestion, ScoreAnswer, + ScoreQuestion, systemone_choice_key, ) _OPENAI_ONLY_FIELDS: Final = frozenset({"input", "questions", "safety_identifier"}) +_OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) def _message_text(message: OpenAIDecisionInputMessage) -> str: @@ -125,3 +134,92 @@ def to_openai_response( total_tokens=usage.input_tokens + usage.output_tokens, ), ) + + +def _text(value: DecisionsJSON) -> str: + return value if isinstance(value, str) else json.dumps(value) + + +def _instructions(instructions: DecisionsJSON | None) -> Mapping[str, str]: + return {} if instructions is None else {"instructions": _text(instructions)} + + +def _predicate_instructions(question: NoulQuestion) -> str: + instructions: Final = () if question.instructions is None else (_text(question.instructions),) + criteria: Final = tuple( + f"Answer {answer} when: {_text(rule)}" for answer, rule in (question.criteria or {}).items() if rule is not None + ) + return "\n\n".join((*instructions, *criteria)) + + +def _choice(value: str, description: DecisionsJSON | None) -> Mapping[str, str]: + return {"value": value} if description is None else {"value": value, "description": _text(description)} + + +def _openai_question(name: str, question: DecisionQuestion) -> Mapping[str, object]: + match question: + case NoulQuestion(): + return {"type": "predicate", "name": name, "instructions": _predicate_instructions(question)} + case ChoiceQuestion(): + return { + "type": "choice", + "name": name, + **_instructions(question.instructions), + "choices": [_choice(value, description) for value, description in question.criteria.items()], + } + case ScoreQuestion(): + return { + "type": "score", + "name": name, + **_instructions(question.instructions), + "levels": [{"label": _text(criterion)} for criterion in question.criteria], + } + case _: + assert_never(question) + + +def to_openai_request(model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return { + "model": model, + "input": _text(request.state), + "questions": [_openai_question(name, question) for name, question in request.questions.items()], + } + + +def _systemone_answer(answer: OpenAIDecisionAnswer) -> DecisionAnswer | None: + match answer: + case OpenAIPredicateAnswer(): + return NoulAnswer(type="noul", noul=answer.probability) + case OpenAIChoiceAnswer(): + return ChoiceAnswer( + type="choice", + choice=systemone_choice_key(answer.choice), + confidence=answer.confidence, + probabilities={systemone_choice_key(item.value): item.probability for item in answer.probabilities}, + ) + case OpenAIScoreAnswer(): + return ScoreAnswer( + type="score", + score=answer.score, + confidence=answer.confidence, + legend={str(item.value): item.label for item in answer.probabilities}, + probabilities={str(item.value): item.probability for item in answer.probabilities}, + ) + case OpenAIRefusalAnswer(): + return None + case _: + assert_never(answer) + + +def _named_answers(answers: Sequence[OpenAIDecisionAnswer]) -> Mapping[str, DecisionAnswer]: + translated: Final = ((answer.name, _systemone_answer(answer)) for answer in answers) + return {name: answer for name, answer in translated if name is not None and answer is not None} + + +def to_systemone_response(payload: object) -> DecisionsResponse: + response: Final = _OPENAI_RESPONSE_ADAPTER.validate_python(payload) + return DecisionsResponse( + model=response.model, + answers=_named_answers(response.answers), + usage=DecisionsUsage(input_tokens=response.usage.input_tokens, output_tokens=response.usage.output_tokens), + ) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index d4fcea24793..b001942a8ba 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -1,6 +1,19 @@ +from collections.abc import Mapping from dataclasses import dataclass from typing import Protocol +from litellm.types.decisions import DecisionsRequestBody + + +def systemone_request_body(model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return { + "model": model, + "state": request.state, + "questions": { + name: question.model_dump(mode="json", exclude_none=True) for name, question in request.questions.items() + }, + } + @dataclass(frozen=True, slots=True) class JevCompatibleDecisionsEndpoint: @@ -25,6 +38,9 @@ class JevCompatibleDecisionsEndpoint: def endpoint_url(self, api_base: str, model: str) -> str: return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" + def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return systemone_request_body(model, request) + def unwrap_response(self, payload: object) -> object: return payload @@ -49,4 +65,6 @@ class DecisionsProviderConfig(Protocol): def endpoint_url(self, api_base: str, model: str) -> str: ... + def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: ... + def unwrap_response(self, payload: object) -> object: ... diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 6e8b2999778..82c457630e6 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -4,10 +4,12 @@ from typing import Final from pydantic import TypeAdapter +from litellm.llms.base_llm.decisions.transformation import systemone_request_body from litellm.secret_managers.main import ( get_secret_str, normalize_nonempty_secret_str, ) +from litellm.types.decisions import DecisionsRequestBody _RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) @@ -43,6 +45,9 @@ class CloudflareDecisionsEndpoint: return f"{normalized_api_base}/{model}" return f"{normalized_api_base}/ai/run/{model}" + def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return systemone_request_body(model, request) + def unwrap_response(self, payload: object) -> object: if not isinstance(payload, Mapping): return payload diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py new file mode 100644 index 00000000000..7c7bee2b099 --- /dev/null +++ b/litellm/llms/openai/decisions/transformation.py @@ -0,0 +1,37 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from litellm.decisions.openai_transformation import to_openai_request, to_systemone_response +from litellm.types.decisions import DecisionsRequestBody, DecisionsResponse + + +@dataclass(frozen=True, slots=True) +class OpenAIDecisionsEndpoint: + api_key_env: tuple[str, ...] = ("OPENAI_API_KEY",) + api_base_env: str = "OPENAI_BASE_URL" + api_key_required: bool = True + + def default_api_base(self) -> str | None: + return "https://api.openai.com" + + def missing_api_base_message(self, provider: str) -> str: + return f"api_base is required for Decisions provider '{provider}'" + + def canonical_model(self, model: str) -> str: + return model + + def request_model(self, model: str) -> str: + return model + + def endpoint_url(self, api_base: str, model: str) -> str: + return f"{api_base.rstrip('/').removesuffix('/v1')}/v1/decisions" + + def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return to_openai_request(model, request) + + def unwrap_response(self, payload: object) -> DecisionsResponse: + return to_systemone_response(payload) + + +OPENAI_DECISIONS_ENDPOINT: Final[OpenAIDecisionsEndpoint] = OpenAIDecisionsEndpoint() diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1565f9d83c7..b48a842eade 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -33557,7 +33557,8 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", - "/v1/responses" + "/v1/responses", + "/v1/decisions" ], "supported_modalities": [ "text", diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 22d47ccda2e..dc19881c857 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -256,7 +256,10 @@ class OpenAIRefusalAnswer(LiteLLMPydanticObjectBase): model_config = ConfigDict(frozen=True) -OpenAIDecisionAnswer: TypeAlias = OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIRefusalAnswer +OpenAIDecisionAnswer: TypeAlias = Annotated[ + OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIRefusalAnswer, + Field(discriminator="type"), +] class OpenAIDecisionInputTokensDetails(LiteLLMPydanticObjectBase): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1565f9d83c7..b48a842eade 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -33557,7 +33557,8 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", - "/v1/responses" + "/v1/responses", + "/v1/decisions" ], "supported_modalities": [ "text", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 106710d328f..77f23e5761e 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -79,6 +79,21 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( ), ) +_OPENAI_RESPONSE: Final[Mapping[str, object]] = { + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + {"type": "refusal", "name": "sentiment"}, + ], + "usage": { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + }, +} + class _RecordingLogger(CustomLogger): def __init__(self) -> None: @@ -590,3 +605,61 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( assert provider_resolution[:2] == ("strands-decider-2B-hobson-v19", "strands_decider") assert route.called assert response.model == _STRANDS_RESPONSE["model"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api_base", "url"), + ( + (None, "https://api.openai.com/v1/decisions"), + ("https://gateway.example/v1", "https://gateway.example/v1/decisions"), + ), +) +async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_and_back( + api_base: str | None, + url: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE) + + response: Final = await litellm.adecisions( + model="openai/gpt-6-luna", + state="The package arrived broken.", + questions={ + "is_defect": {"type": "noul", "instructions": "Is this a defect?"}, + "sentiment": { + "type": "choice", + "instructions": "How does the customer feel?", + "criteria": {"positive": None, "negative": "unhappy"}, + }, + }, + api_key="caller-key", + api_base=api_base, + ) + + assert route.called + request: Final = respx_mock.calls[0].request + assert request.headers["authorization"] == "Bearer caller-key" + assert json.loads(request.content) == { + "model": "gpt-6-luna", + "input": "The package arrived broken.", + "questions": [ + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "sentiment", + "instructions": "How does the customer feel?", + "choices": [{"value": "positive"}, {"value": "negative", "description": "unhappy"}], + }, + ], + } + assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} + assert response.hidden_params["custom_llm_provider"] == "openai" + luna_cost: Final = litellm.model_cost["gpt-6-luna"] + expected_cost: Final = _INPUT_TOKENS * float(luna_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + luna_cost["output_cost_per_token"] + ) + assert expected_cost > 0 + assert litellm.completion_cost(completion_response=response) == pytest.approx(expected_cost) diff --git a/tests/unit/decisions/test_openai_transformation.py b/tests/unit/decisions/test_openai_transformation.py index a3cc1583d40..381c2aac643 100644 --- a/tests/unit/decisions/test_openai_transformation.py +++ b/tests/unit/decisions/test_openai_transformation.py @@ -4,7 +4,7 @@ from typing import Final import pytest from pydantic import TypeAdapter, ValidationError -from litellm.decisions.openai_transformation import to_systemone_request +from litellm.decisions.openai_transformation import to_openai_request, to_systemone_request, to_systemone_response from litellm.types.decisions import MAX_DECISION_QUESTIONS, DecisionsRequestBody, OpenAIDecisionRequestBody _OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) @@ -30,3 +30,104 @@ def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_req def test_an_openai_request_with_more_questions_than_systemone_takes_is_rejected_before_translation() -> None: with pytest.raises(ValidationError, match="questions"): _OPENAI_BODY.validate_python(_openai_request(MAX_DECISION_QUESTIONS + 1)) + + +def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_their_keys() -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": {"ticket": 1234, "text": "Screen cracked"}, + "questions": { + "damaged": { + "type": "noul", + "instructions": "Is the item damaged?", + "criteria": {"true": "Visible damage", "false": None}, + "provider_field": "dropped", + }, + "rubric_only": {"type": "noul", "criteria": {"true": {"signal": "refund"}}}, + "action": { + "type": "choice", + "instructions": {"policy": "refund-v2"}, + "criteria": {"refund": "Within 30 days", "escalate": None}, + }, + "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, + }, + } + ) + + assert to_openai_request("gpt-6-luna", request) == { + "model": "gpt-6-luna", + "input": '{"ticket": 1234, "text": "Screen cracked"}', + "questions": [ + { + "type": "predicate", + "name": "damaged", + "instructions": "Is the item damaged?\n\nAnswer true when: Visible damage", + }, + {"type": "predicate", "name": "rubric_only", "instructions": 'Answer true when: {"signal": "refund"}'}, + { + "type": "choice", + "name": "action", + "instructions": '{"policy": "refund-v2"}', + "choices": [{"value": "refund", "description": "Within 30 days"}, {"value": "escalate"}], + }, + {"type": "score", "name": "severity", "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}]}, + ], + } + + +def test_an_openai_response_becomes_systemone_answers_keyed_by_question_name_without_refusals() -> None: + response: Final = to_systemone_response( + { + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "damaged", "probability": 0.95}, + { + "type": "choice", + "name": "action", + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 1.72, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.0}, + {"value": 1, "label": "major", "probability": 0.28}, + {"value": 2, "label": "critical", "probability": 0.72}, + ], + "confidence": 0.58, + }, + {"type": "refusal", "name": "fraud"}, + ], + "usage": { + "input_tokens": 383, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 385, + }, + } + ) + + assert response.model_dump(mode="json") == { + "model": "gpt-6-luna", + "answers": { + "damaged": {"type": "noul", "noul": 0.95}, + "action": { + "type": "choice", + "choice": "true", + "confidence": 0.8, + "probabilities": {"true": 0.9, "escalate": 0.1}, + }, + "severity": { + "type": "score", + "score": 1.72, + "confidence": 0.58, + "legend": {"0": "minor", "1": "major", "2": "critical"}, + "probabilities": {"0": 0.0, "1": 0.28, "2": 0.72}, + }, + }, + "usage": {"input_tokens": 383, "output_tokens": 2}, + } diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 0a3ee9d1a4c..a92059772ea 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -319,6 +319,28 @@ _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST: Final[Mapping[str, object]] = { "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, } +_OPENAI_FORMAT_ANSWERS: Final = [ + {"type": "predicate", "name": "damaged", "probability": 0.95}, + { + "type": "choice", + "name": None, + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": "major", "probability": 0.7}, + ], + "confidence": 0.6, + }, + {"type": "refusal", "name": "fraud"}, +] + @pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions")) def test_openai_format_decisions_translate_through_systemone( @@ -353,27 +375,7 @@ def test_openai_format_decisions_translate_through_systemone( } body: Final = response.json() assert body["model"] == _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST["model"] - assert body["answers"] == [ - {"type": "predicate", "name": "damaged", "probability": 0.95}, - { - "type": "choice", - "name": None, - "choice": True, - "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], - "confidence": 0.8, - }, - { - "type": "score", - "name": "severity", - "score": 0.7, - "probabilities": [ - {"value": 0, "label": "minor", "probability": 0.3}, - {"value": 1, "label": "major", "probability": 0.7}, - ], - "confidence": 0.6, - }, - {"type": "refusal", "name": "fraud"}, - ] + assert body["answers"] == _OPENAI_FORMAT_ANSWERS assert body["usage"] == { "input_tokens": _INPUT_TOKENS, "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, @@ -438,6 +440,79 @@ def test_a_body_that_is_not_json_is_a_client_error( assert not upstream.called +def test_openai_format_decisions_reach_an_openai_deployment_with_the_callers_questions( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[{"model_name": "decider", "litellm_params": {"model": "openai/gpt-6-luna", "api_key": "k"}}] + ), + ) + upstream: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond( + json={ + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "0", "probability": 0.95}, + { + "type": "choice", + "name": "1", + "choice": "true", + "probabilities": [{"value": "true", "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "2", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": "major: Product unusable", "probability": 0.7}, + ], + "confidence": 0.6, + }, + {"type": "refusal", "name": "3"}, + ], + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS, "total_tokens": 370}, + } + ) + + response: Final = client.post("/v1/decisions", json=_OPENAI_FORMAT_REQUEST) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "The package arrived with a broken screen.\n\nI want a refund.\n\nOrder 1234.", + "questions": [ + {"type": "predicate", "name": "0", "instructions": "Does the customer report a damaged item?"}, + { + "type": "choice", + "name": "1", + "instructions": "Should we refund?", + "choices": [{"value": "true", "description": "Refund now"}, {"value": "escalate"}], + }, + { + "type": "score", + "name": "2", + "instructions": "How severe is the issue?", + "levels": [{"label": "minor"}, {"label": "major: Product unusable"}], + }, + {"type": "predicate", "name": "3", "instructions": "Is this fraud?"}, + ], + } + assert response.json()["answers"] == _OPENAI_FORMAT_ANSWERS + luna_cost: Final = litellm.model_cost["gpt-6-luna"] + expected_cost: Final = _INPUT_TOKENS * float(luna_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + luna_cost["output_cost_per_token"] + ) + assert expected_cost > 0 + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) + + def _serving_endpoint(bare: FastAPI, path: str) -> object: scope: Final = {"type": "http", "method": "POST", "path": path, "root_path": "", "query_string": b"", "headers": ()} return next( From fbcc73b57992ad7a2d1f91325a807f6339f6c191 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:47:54 -0700 Subject: [PATCH 08/17] refactor(decisions): move the OpenAI wire translation into llms/openai (cherry picked from commit d209a85076d61511a011494916eee4b212b4d1ff) --- litellm/decisions/openai_transformation.py | 98 -------------- .../llms/openai/decisions/transformation.py | 120 +++++++++++++++++- .../decisions/test_openai_transformation.py | 103 +-------------- tests/unit/llms/openai/decisions/__init__.py | 0 .../test_openai_decisions_transformation.py | 109 ++++++++++++++++ 5 files changed, 227 insertions(+), 203 deletions(-) create mode 100644 tests/unit/llms/openai/decisions/__init__.py create mode 100644 tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py diff --git a/litellm/decisions/openai_transformation.py b/litellm/decisions/openai_transformation.py index bdbfd7df0a1..af2c838b9ab 100644 --- a/litellm/decisions/openai_transformation.py +++ b/litellm/decisions/openai_transformation.py @@ -1,21 +1,14 @@ -import json from collections.abc import Mapping, Sequence from typing import Final -from pydantic import TypeAdapter from typing_extensions import assert_never from litellm.types.decisions import ( ChoiceAnswer, - ChoiceQuestion, DecisionAnswer, - DecisionQuestion, - DecisionsJSON, - DecisionsRequestBody, DecisionsResponse, DecisionsUsage, NoulAnswer, - NoulQuestion, OpenAIChoiceAnswer, OpenAIChoiceProbability, OpenAIChoiceQuestion, @@ -33,12 +26,10 @@ from litellm.types.decisions import ( OpenAIScoreProbability, OpenAIScoreQuestion, ScoreAnswer, - ScoreQuestion, systemone_choice_key, ) _OPENAI_ONLY_FIELDS: Final = frozenset({"input", "questions", "safety_identifier"}) -_OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) def _message_text(message: OpenAIDecisionInputMessage) -> str: @@ -134,92 +125,3 @@ def to_openai_response( total_tokens=usage.input_tokens + usage.output_tokens, ), ) - - -def _text(value: DecisionsJSON) -> str: - return value if isinstance(value, str) else json.dumps(value) - - -def _instructions(instructions: DecisionsJSON | None) -> Mapping[str, str]: - return {} if instructions is None else {"instructions": _text(instructions)} - - -def _predicate_instructions(question: NoulQuestion) -> str: - instructions: Final = () if question.instructions is None else (_text(question.instructions),) - criteria: Final = tuple( - f"Answer {answer} when: {_text(rule)}" for answer, rule in (question.criteria or {}).items() if rule is not None - ) - return "\n\n".join((*instructions, *criteria)) - - -def _choice(value: str, description: DecisionsJSON | None) -> Mapping[str, str]: - return {"value": value} if description is None else {"value": value, "description": _text(description)} - - -def _openai_question(name: str, question: DecisionQuestion) -> Mapping[str, object]: - match question: - case NoulQuestion(): - return {"type": "predicate", "name": name, "instructions": _predicate_instructions(question)} - case ChoiceQuestion(): - return { - "type": "choice", - "name": name, - **_instructions(question.instructions), - "choices": [_choice(value, description) for value, description in question.criteria.items()], - } - case ScoreQuestion(): - return { - "type": "score", - "name": name, - **_instructions(question.instructions), - "levels": [{"label": _text(criterion)} for criterion in question.criteria], - } - case _: - assert_never(question) - - -def to_openai_request(model: str, request: DecisionsRequestBody) -> Mapping[str, object]: - return { - "model": model, - "input": _text(request.state), - "questions": [_openai_question(name, question) for name, question in request.questions.items()], - } - - -def _systemone_answer(answer: OpenAIDecisionAnswer) -> DecisionAnswer | None: - match answer: - case OpenAIPredicateAnswer(): - return NoulAnswer(type="noul", noul=answer.probability) - case OpenAIChoiceAnswer(): - return ChoiceAnswer( - type="choice", - choice=systemone_choice_key(answer.choice), - confidence=answer.confidence, - probabilities={systemone_choice_key(item.value): item.probability for item in answer.probabilities}, - ) - case OpenAIScoreAnswer(): - return ScoreAnswer( - type="score", - score=answer.score, - confidence=answer.confidence, - legend={str(item.value): item.label for item in answer.probabilities}, - probabilities={str(item.value): item.probability for item in answer.probabilities}, - ) - case OpenAIRefusalAnswer(): - return None - case _: - assert_never(answer) - - -def _named_answers(answers: Sequence[OpenAIDecisionAnswer]) -> Mapping[str, DecisionAnswer]: - translated: Final = ((answer.name, _systemone_answer(answer)) for answer in answers) - return {name: answer for name, answer in translated if name is not None and answer is not None} - - -def to_systemone_response(payload: object) -> DecisionsResponse: - response: Final = _OPENAI_RESPONSE_ADAPTER.validate_python(payload) - return DecisionsResponse( - model=response.model, - answers=_named_answers(response.answers), - usage=DecisionsUsage(input_tokens=response.usage.input_tokens, output_tokens=response.usage.output_tokens), - ) diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 7c7bee2b099..ea07a8a5b89 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -1,9 +1,123 @@ -from collections.abc import Mapping +import json +from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Final -from litellm.decisions.openai_transformation import to_openai_request, to_systemone_response -from litellm.types.decisions import DecisionsRequestBody, DecisionsResponse +from pydantic import TypeAdapter +from typing_extensions import assert_never + +from litellm.types.decisions import ( + ChoiceAnswer, + ChoiceQuestion, + DecisionAnswer, + DecisionQuestion, + DecisionsJSON, + DecisionsRequestBody, + DecisionsResponse, + DecisionsUsage, + NoulAnswer, + NoulQuestion, + OpenAIChoiceAnswer, + OpenAIDecisionAnswer, + OpenAIDecisionResponse, + OpenAIPredicateAnswer, + OpenAIRefusalAnswer, + OpenAIScoreAnswer, + ScoreAnswer, + ScoreQuestion, + systemone_choice_key, +) + +_OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) + + +def _text(value: DecisionsJSON) -> str: + return value if isinstance(value, str) else json.dumps(value) + + +def _instructions(instructions: DecisionsJSON | None) -> Mapping[str, str]: + return {} if instructions is None else {"instructions": _text(instructions)} + + +def _predicate_instructions(question: NoulQuestion) -> str: + instructions: Final = () if question.instructions is None else (_text(question.instructions),) + criteria: Final = tuple( + f"Answer {answer} when: {_text(rule)}" for answer, rule in (question.criteria or {}).items() if rule is not None + ) + return "\n\n".join((*instructions, *criteria)) + + +def _choice(value: str, description: DecisionsJSON | None) -> Mapping[str, str]: + return {"value": value} if description is None else {"value": value, "description": _text(description)} + + +def _openai_question(name: str, question: DecisionQuestion) -> Mapping[str, object]: + match question: + case NoulQuestion(): + return {"type": "predicate", "name": name, "instructions": _predicate_instructions(question)} + case ChoiceQuestion(): + return { + "type": "choice", + "name": name, + **_instructions(question.instructions), + "choices": [_choice(value, description) for value, description in question.criteria.items()], + } + case ScoreQuestion(): + return { + "type": "score", + "name": name, + **_instructions(question.instructions), + "levels": [{"label": _text(criterion)} for criterion in question.criteria], + } + case _: + assert_never(question) + + +def to_openai_request(model: str, request: DecisionsRequestBody) -> Mapping[str, object]: + return { + "model": model, + "input": _text(request.state), + "questions": [_openai_question(name, question) for name, question in request.questions.items()], + } + + +def _systemone_answer(answer: OpenAIDecisionAnswer) -> DecisionAnswer | None: + match answer: + case OpenAIPredicateAnswer(): + return NoulAnswer(type="noul", noul=answer.probability) + case OpenAIChoiceAnswer(): + return ChoiceAnswer( + type="choice", + choice=systemone_choice_key(answer.choice), + confidence=answer.confidence, + probabilities={systemone_choice_key(item.value): item.probability for item in answer.probabilities}, + ) + case OpenAIScoreAnswer(): + return ScoreAnswer( + type="score", + score=answer.score, + confidence=answer.confidence, + legend={str(item.value): item.label for item in answer.probabilities}, + probabilities={str(item.value): item.probability for item in answer.probabilities}, + ) + case OpenAIRefusalAnswer(): + return None + case _: + assert_never(answer) + + +def _named_answers(answers: Sequence[OpenAIDecisionAnswer]) -> Mapping[str, DecisionAnswer]: + translated: Final = ((answer.name, _systemone_answer(answer)) for answer in answers) + return {name: answer for name, answer in translated if name is not None and answer is not None} + + +def to_systemone_response(payload: object) -> DecisionsResponse: + response: Final = _OPENAI_RESPONSE_ADAPTER.validate_python(payload) + return DecisionsResponse( + model=response.model, + answers=_named_answers(response.answers), + usage=DecisionsUsage(input_tokens=response.usage.input_tokens, output_tokens=response.usage.output_tokens), + ) @dataclass(frozen=True, slots=True) diff --git a/tests/unit/decisions/test_openai_transformation.py b/tests/unit/decisions/test_openai_transformation.py index 381c2aac643..a3cc1583d40 100644 --- a/tests/unit/decisions/test_openai_transformation.py +++ b/tests/unit/decisions/test_openai_transformation.py @@ -4,7 +4,7 @@ from typing import Final import pytest from pydantic import TypeAdapter, ValidationError -from litellm.decisions.openai_transformation import to_openai_request, to_systemone_request, to_systemone_response +from litellm.decisions.openai_transformation import to_systemone_request from litellm.types.decisions import MAX_DECISION_QUESTIONS, DecisionsRequestBody, OpenAIDecisionRequestBody _OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) @@ -30,104 +30,3 @@ def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_req def test_an_openai_request_with_more_questions_than_systemone_takes_is_rejected_before_translation() -> None: with pytest.raises(ValidationError, match="questions"): _OPENAI_BODY.validate_python(_openai_request(MAX_DECISION_QUESTIONS + 1)) - - -def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_their_keys() -> None: - request: Final = _SYSTEMONE_BODY.validate_python( - { - "state": {"ticket": 1234, "text": "Screen cracked"}, - "questions": { - "damaged": { - "type": "noul", - "instructions": "Is the item damaged?", - "criteria": {"true": "Visible damage", "false": None}, - "provider_field": "dropped", - }, - "rubric_only": {"type": "noul", "criteria": {"true": {"signal": "refund"}}}, - "action": { - "type": "choice", - "instructions": {"policy": "refund-v2"}, - "criteria": {"refund": "Within 30 days", "escalate": None}, - }, - "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, - }, - } - ) - - assert to_openai_request("gpt-6-luna", request) == { - "model": "gpt-6-luna", - "input": '{"ticket": 1234, "text": "Screen cracked"}', - "questions": [ - { - "type": "predicate", - "name": "damaged", - "instructions": "Is the item damaged?\n\nAnswer true when: Visible damage", - }, - {"type": "predicate", "name": "rubric_only", "instructions": 'Answer true when: {"signal": "refund"}'}, - { - "type": "choice", - "name": "action", - "instructions": '{"policy": "refund-v2"}', - "choices": [{"value": "refund", "description": "Within 30 days"}, {"value": "escalate"}], - }, - {"type": "score", "name": "severity", "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}]}, - ], - } - - -def test_an_openai_response_becomes_systemone_answers_keyed_by_question_name_without_refusals() -> None: - response: Final = to_systemone_response( - { - "model": "gpt-6-luna", - "answers": [ - {"type": "predicate", "name": "damaged", "probability": 0.95}, - { - "type": "choice", - "name": "action", - "choice": True, - "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], - "confidence": 0.8, - }, - { - "type": "score", - "name": "severity", - "score": 1.72, - "probabilities": [ - {"value": 0, "label": "minor", "probability": 0.0}, - {"value": 1, "label": "major", "probability": 0.28}, - {"value": 2, "label": "critical", "probability": 0.72}, - ], - "confidence": 0.58, - }, - {"type": "refusal", "name": "fraud"}, - ], - "usage": { - "input_tokens": 383, - "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, - "output_tokens": 2, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 385, - }, - } - ) - - assert response.model_dump(mode="json") == { - "model": "gpt-6-luna", - "answers": { - "damaged": {"type": "noul", "noul": 0.95}, - "action": { - "type": "choice", - "choice": "true", - "confidence": 0.8, - "probabilities": {"true": 0.9, "escalate": 0.1}, - }, - "severity": { - "type": "score", - "score": 1.72, - "confidence": 0.58, - "legend": {"0": "minor", "1": "major", "2": "critical"}, - "probabilities": {"0": 0.0, "1": 0.28, "2": 0.72}, - }, - }, - "usage": {"input_tokens": 383, "output_tokens": 2}, - } diff --git a/tests/unit/llms/openai/decisions/__init__.py b/tests/unit/llms/openai/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py new file mode 100644 index 00000000000..56c950cc8d0 --- /dev/null +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -0,0 +1,109 @@ +from typing import Final + +from pydantic import TypeAdapter + +from litellm.llms.openai.decisions.transformation import to_openai_request, to_systemone_response +from litellm.types.decisions import DecisionsRequestBody + +_SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) + + +def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_their_keys() -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": {"ticket": 1234, "text": "Screen cracked"}, + "questions": { + "damaged": { + "type": "noul", + "instructions": "Is the item damaged?", + "criteria": {"true": "Visible damage", "false": None}, + "provider_field": "dropped", + }, + "rubric_only": {"type": "noul", "criteria": {"true": {"signal": "refund"}}}, + "action": { + "type": "choice", + "instructions": {"policy": "refund-v2"}, + "criteria": {"refund": "Within 30 days", "escalate": None}, + }, + "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, + }, + } + ) + + assert to_openai_request("gpt-6-luna", request) == { + "model": "gpt-6-luna", + "input": '{"ticket": 1234, "text": "Screen cracked"}', + "questions": [ + { + "type": "predicate", + "name": "damaged", + "instructions": "Is the item damaged?\n\nAnswer true when: Visible damage", + }, + {"type": "predicate", "name": "rubric_only", "instructions": 'Answer true when: {"signal": "refund"}'}, + { + "type": "choice", + "name": "action", + "instructions": '{"policy": "refund-v2"}', + "choices": [{"value": "refund", "description": "Within 30 days"}, {"value": "escalate"}], + }, + {"type": "score", "name": "severity", "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}]}, + ], + } + + +def test_an_openai_response_becomes_systemone_answers_keyed_by_question_name_without_refusals() -> None: + response: Final = to_systemone_response( + { + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "damaged", "probability": 0.95}, + { + "type": "choice", + "name": "action", + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 1.72, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.0}, + {"value": 1, "label": "major", "probability": 0.28}, + {"value": 2, "label": "critical", "probability": 0.72}, + ], + "confidence": 0.58, + }, + {"type": "refusal", "name": "fraud"}, + ], + "usage": { + "input_tokens": 383, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 385, + }, + } + ) + + assert response.model_dump(mode="json") == { + "model": "gpt-6-luna", + "answers": { + "damaged": {"type": "noul", "noul": 0.95}, + "action": { + "type": "choice", + "choice": "true", + "confidence": 0.8, + "probabilities": {"true": 0.9, "escalate": 0.1}, + }, + "severity": { + "type": "score", + "score": 1.72, + "confidence": 0.58, + "legend": {"0": "minor", "1": "major", "2": "critical"}, + "probabilities": {"0": 0.0, "1": 0.28, "2": 0.72}, + }, + }, + "usage": {"input_tokens": 383, "output_tokens": 2}, + } From 637cc3f5b17baba725b2c8fa6fe1346b04b10d8c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:47:23 -0700 Subject: [PATCH 09/17] feat(decisions): route both request formats through a shared decisions IR System One and OpenAI-format bodies now convert to one internal representation, and each provider translates it to its own wire format. OpenAI deployments receive the caller's messages, images, names, typed choice values, level descriptions and safety_identifier unchanged. System One providers return a 400 for image input. Cached and cache-write tokens are priced on both response shapes, and provider response extras survive the round trip. (cherry picked from commit 0dd207fde1c557f8eb1c3e6dd6833e16f94fb6c5) --- litellm/cost_calculator.py | 34 +- litellm/decisions/main.py | 131 ++++++-- litellm/decisions/openai_transformation.py | 127 -------- litellm/litellm_core_utils/litellm_logging.py | 3 +- .../llms/base_llm/decisions/transformation.py | 298 +++++++++++++++++- .../cloudflare/decisions/transformation.py | 32 +- .../llms/openai/decisions/transformation.py | 285 +++++++++++++---- .../proxy/decisions_endpoints/endpoints.py | 42 +-- litellm/types/decisions.py | 159 +++++++++- litellm/utils.py | 4 +- tests/unit/decisions/test_main.py | 91 +++++- .../decisions/test_openai_transformation.py | 32 -- .../unit/llms/base_llm/decisions/__init__.py | 0 .../test_base_decisions_transformation.py | 178 +++++++++++ .../test_openai_decisions_transformation.py | 183 ++++++++--- .../decisions_endpoints/test_endpoints.py | 79 ++--- tests/unit/test_utils.py | 2 + 17 files changed, 1271 insertions(+), 409 deletions(-) delete mode 100644 litellm/decisions/openai_transformation.py delete mode 100644 tests/unit/decisions/test_openai_transformation.py create mode 100644 tests/unit/llms/base_llm/decisions/__init__.py create mode 100644 tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index fb2c23be50c..130fe939e33 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -98,7 +98,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse -from litellm.types.decisions import DecisionsResponse, DecisionsUsage +from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( HttpxBinaryResponseContent, @@ -1012,12 +1012,24 @@ def get_usage_object( return None +def _decisions_usage( + *, input_tokens: int, output_tokens: int, cached_tokens: int, cache_write_tokens: int +) -> dict[str, object]: + return { + "prompt_tokens": input_tokens, + "completion_tokens": output_tokens, + "cache_read_input_tokens": cached_tokens, + "cache_creation_input_tokens": cache_write_tokens, + } + + def _is_known_usage_objects(usage_obj): """Returns True if the usage obj is a known Usage type""" return ( isinstance(usage_obj, litellm.Usage) or isinstance(usage_obj, ResponseAPIUsage) or isinstance(usage_obj, DecisionsUsage) + or isinstance(usage_obj, OpenAIDecisionUsage) or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj) ) @@ -1436,10 +1448,19 @@ def completion_cost( litellm.Usage(**_usage_for_dump.model_dump()), ) if isinstance(usage_obj, DecisionsUsage): - _usage = { - "prompt_tokens": usage_obj.input_tokens, - "completion_tokens": usage_obj.output_tokens, - } + _usage = _decisions_usage( + input_tokens=usage_obj.input_tokens, + output_tokens=usage_obj.output_tokens, + cached_tokens=usage_obj.cached_tokens, + cache_write_tokens=usage_obj.cache_write_tokens, + ) + elif isinstance(usage_obj, OpenAIDecisionUsage): + _usage = _decisions_usage( + input_tokens=usage_obj.input_tokens, + output_tokens=usage_obj.output_tokens, + cached_tokens=usage_obj.input_tokens_details.cached_tokens, + cache_write_tokens=usage_obj.input_tokens_details.cache_write_tokens, + ) elif usage_obj is None: _usage = {} elif isinstance(usage_obj, BaseModel): @@ -1941,7 +1962,8 @@ def response_cost_calculator( | OpenAIModerationResponse | Response | SearchResponse - | DecisionsResponse, + | DecisionsResponse + | OpenAIDecisionResponse, model: str, custom_llm_provider: str | None, call_type: Literal[ diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 09464c8884d..1b973adae72 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,17 +1,26 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias import httpx from pydantic import TypeAdapter, ValidationError +from typing_extensions import assert_never import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig +from litellm.llms.base_llm.decisions.transformation import ( + DecisionsProviderConfig, + ir_to_systemone_response, + systemone_request_to_ir, +) from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT from litellm.llms.custom_httpx.http_handler import _get_httpx_client, get_async_httpx_client -from litellm.llms.openai.decisions.transformation import OPENAI_DECISIONS_ENDPOINT +from litellm.llms.openai.decisions.transformation import ( + OPENAI_DECISIONS_ENDPOINT, + ir_to_openai_response, + openai_request_to_ir, +) from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT @@ -19,9 +28,16 @@ from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_EN from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( DecisionQuestion, + DecisionsIRRequest, + DecisionsIRResponse, DecisionsJSON, - DecisionsRequest, + DecisionsRequestBody, DecisionsResponse, + OpenAIDecisionInput, + OpenAIDecisionQuestion, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, + UnsupportedDecisionsRequest, ) from litellm.utils import client @@ -36,9 +52,14 @@ DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxy } ) -_DECISIONS_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequest]] = TypeAdapter(DecisionsRequest) +DecisionsQuestions: TypeAlias = ( + Mapping[str, DecisionQuestion | Mapping[str, object]] | Sequence[OpenAIDecisionQuestion | Mapping[str, object]] +) +DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody + +_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) _DECISIONS_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) -_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) @dataclass(frozen=True, slots=True, repr=False) @@ -50,6 +71,8 @@ class _PreparedDecisionsRequest: api_key: str | None = field(repr=False) headers: Mapping[str, str] = field(repr=False) body: Mapping[str, object] = field(repr=False) + request: DecisionsRequestFormat = field(repr=False) + ir_request: DecisionsIRRequest = field(repr=False) def _resolve_provider_model(model: str, custom_llm_provider: str | None) -> tuple[str, str]: @@ -97,20 +120,52 @@ def _resolve_api_key( return server_api_key +def _validate_request( + *, + state: DecisionsJSON | None, + questions: DecisionsQuestions | None, + decision_input: OpenAIDecisionInput | None, + safety_identifier: str | None, +) -> DecisionsRequestFormat: + if decision_input is None: + return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": state, "questions": questions}) + return _OPENAI_REQUEST_ADAPTER.validate_python( + {"input": decision_input, "questions": questions, "safety_identifier": safety_identifier} + ) + + +def _ir_request(request: DecisionsRequestFormat) -> DecisionsIRRequest: + match request: + case DecisionsRequestBody(): + return systemone_request_to_ir(request) + case OpenAIDecisionRequestBody(): + return openai_request_to_ir(request) + case _: + assert_never(request) + + def _prepare_request( *, model: str, - state: DecisionsJSON, - questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + state: DecisionsJSON | None, + questions: DecisionsQuestions | None, + decision_input: OpenAIDecisionInput | None, + safety_identifier: str | None, api_key: str | None, api_base: str | None, custom_llm_provider: str | None, extra_headers: Mapping[str, str] | None, ) -> _PreparedDecisionsRequest: provider, upstream_model = _resolve_provider_model(model, custom_llm_provider) + if state is not None and decision_input is not None: + raise litellm.BadRequestError( + message="Pass either state (System One format) or input (OpenAI format) to the Decisions API, not both", + model=model, + llm_provider=provider, + ) try: - validated_request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python( - {"model": model, "state": state, "questions": questions} + validated_request: Final = _validate_request( + state=state, questions=questions, decision_input=decision_input, safety_identifier=safety_identifier ) except ValidationError as error: raise litellm.BadRequestError( @@ -149,7 +204,14 @@ def _prepare_request( "Content-Type": "application/json", } ) - body: Final = MappingProxyType(endpoint.request_body(endpoint.request_model(canonical_model), validated_request)) + ir_request: Final = _ir_request(validated_request) + body: Final = endpoint.request_body(endpoint.request_model(canonical_model), ir_request) + if isinstance(body, UnsupportedDecisionsRequest): + raise litellm.BadRequestError( + message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}", + model=model, + llm_provider=provider, + ) return _PreparedDecisionsRequest( config=endpoint, provider=provider, @@ -157,7 +219,9 @@ def _prepare_request( url=endpoint.endpoint_url(resolved_api_base, canonical_model), api_key=resolved_api_key, headers=outbound_headers, - body=body, + body=MappingProxyType(body), + request=validated_request, + ir_request=ir_request, ) @@ -192,13 +256,26 @@ def _log_request( return logging_obj +def _format_response( + response: DecisionsIRResponse, prepared: _PreparedDecisionsRequest, model: str +) -> DecisionsResponse | OpenAIDecisionResponse: + match prepared.request: + case DecisionsRequestBody(): + return ir_to_systemone_response(response, prepared.ir_request) + case OpenAIDecisionRequestBody(): + return ir_to_openai_response(response, prepared.ir_request, model) + case _: + assert_never(prepared.request) + + def _parse_response( response: httpx.Response, prepared: _PreparedDecisionsRequest, -) -> DecisionsResponse: + model: str, +) -> DecisionsResponse | OpenAIDecisionResponse: response.raise_for_status() payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content) - result: Final = _DECISIONS_RESPONSE_ADAPTER.validate_python(prepared.config.unwrap_response(payload)) + result: Final = _format_response(prepared.config.parse_response(payload, prepared.ir_request), prepared, model) result._hidden_params.update( { "model": f"{prepared.provider}/{prepared.upstream_model}", @@ -220,19 +297,23 @@ def _map_upstream_exception(error: Exception, prepared: _PreparedDecisionsReques @client async def adecisions( model: str, - state: DecisionsJSON, - questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + state: DecisionsJSON | None = None, + questions: DecisionsQuestions | None = None, api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, extra_headers: Mapping[str, str] | None = None, + input: OpenAIDecisionInput | None = None, + safety_identifier: str | None = None, **kwargs: object, -) -> DecisionsResponse: +) -> DecisionsResponse | OpenAIDecisionResponse: prepared: Final = _prepare_request( model=model, state=state, questions=questions, + decision_input=input, + safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, custom_llm_provider=custom_llm_provider, @@ -248,7 +329,7 @@ async def adecisions( timeout=timeout, logging_obj=logging_obj, ) - return _parse_response(response=response, prepared=prepared) + return _parse_response(response=response, prepared=prepared, model=model) except Exception as error: raise _map_upstream_exception(error, prepared) from error @@ -256,19 +337,23 @@ async def adecisions( @client def decisions( model: str, - state: DecisionsJSON, - questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + state: DecisionsJSON | None = None, + questions: DecisionsQuestions | None = None, api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, extra_headers: Mapping[str, str] | None = None, + input: OpenAIDecisionInput | None = None, + safety_identifier: str | None = None, **kwargs: object, -) -> DecisionsResponse: +) -> DecisionsResponse | OpenAIDecisionResponse: prepared: Final = _prepare_request( model=model, state=state, questions=questions, + decision_input=input, + safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, custom_llm_provider=custom_llm_provider, @@ -284,7 +369,7 @@ def decisions( timeout=timeout, logging_obj=logging_obj, ) - return _parse_response(response=response, prepared=prepared) + return _parse_response(response=response, prepared=prepared, model=model) except Exception as error: raise _map_upstream_exception(error, prepared) from error diff --git a/litellm/decisions/openai_transformation.py b/litellm/decisions/openai_transformation.py deleted file mode 100644 index af2c838b9ab..00000000000 --- a/litellm/decisions/openai_transformation.py +++ /dev/null @@ -1,127 +0,0 @@ -from collections.abc import Mapping, Sequence -from typing import Final - -from typing_extensions import assert_never - -from litellm.types.decisions import ( - ChoiceAnswer, - DecisionAnswer, - DecisionsResponse, - DecisionsUsage, - NoulAnswer, - OpenAIChoiceAnswer, - OpenAIChoiceProbability, - OpenAIChoiceQuestion, - OpenAIDecisionAnswer, - OpenAIDecisionInputMessage, - OpenAIDecisionQuestion, - OpenAIDecisionRequestBody, - OpenAIDecisionResponse, - OpenAIDecisionUsage, - OpenAIPredicateAnswer, - OpenAIPredicateQuestion, - OpenAIRefusalAnswer, - OpenAIScoreAnswer, - OpenAIScoreLevel, - OpenAIScoreProbability, - OpenAIScoreQuestion, - ScoreAnswer, - systemone_choice_key, -) - -_OPENAI_ONLY_FIELDS: Final = frozenset({"input", "questions", "safety_identifier"}) - - -def _message_text(message: OpenAIDecisionInputMessage) -> str: - if isinstance(message.content, str): - return message.content - return "\n\n".join(part.text for part in message.content) - - -def _state(decision_input: str | Sequence[OpenAIDecisionInputMessage]) -> str: - if isinstance(decision_input, str): - return decision_input - return "\n\n".join(_message_text(message) for message in decision_input) - - -def _level_criterion(level: OpenAIScoreLevel) -> str: - return level.label if level.description is None else f"{level.label}: {level.description}" - - -def _systemone_question(question: OpenAIDecisionQuestion) -> Mapping[str, object]: - match question: - case OpenAIPredicateQuestion(): - return {"type": "noul", "instructions": question.instructions} - case OpenAIChoiceQuestion(): - return { - "type": "choice", - "instructions": question.instructions, - "criteria": {systemone_choice_key(option.value): option.description for option in question.choices}, - } - case OpenAIScoreQuestion(): - return { - "type": "score", - "instructions": question.instructions, - "criteria": [_level_criterion(level) for level in question.levels], - } - case _: - assert_never(question) - - -def to_systemone_request(request_data: Mapping[str, object], body: OpenAIDecisionRequestBody) -> Mapping[str, object]: - return { - **{key: value for key, value in request_data.items() if key not in _OPENAI_ONLY_FIELDS}, - "state": _state(body.input), - "questions": {str(index): _systemone_question(question) for index, question in enumerate(body.questions)}, - } - - -def _openai_answer(question: OpenAIDecisionQuestion, answer: DecisionAnswer | None) -> OpenAIDecisionAnswer: - match question, answer: - case OpenAIPredicateQuestion(), NoulAnswer(): - return OpenAIPredicateAnswer(name=question.name, probability=answer.noul) - case OpenAIChoiceQuestion(), ChoiceAnswer(): - typed_values: Final = {systemone_choice_key(option.value): option.value for option in question.choices} - return OpenAIChoiceAnswer( - name=question.name, - choice=typed_values.get(answer.choice, answer.choice), - probabilities=tuple( - OpenAIChoiceProbability( - value=option.value, - probability=answer.probabilities.get(systemone_choice_key(option.value), 0.0), - ) - for option in question.choices - ), - confidence=answer.confidence, - ) - case OpenAIScoreQuestion(), ScoreAnswer(): - return OpenAIScoreAnswer( - name=question.name, - score=answer.score, - probabilities=tuple( - OpenAIScoreProbability( - value=index, label=level.label, probability=answer.probabilities.get(str(index), 0.0) - ) - for index, level in enumerate(question.levels) - ), - confidence=answer.confidence, - ) - case _: - return OpenAIRefusalAnswer(name=question.name) - - -def to_openai_response( - response: DecisionsResponse, questions: Sequence[OpenAIDecisionQuestion], requested_model: str -) -> OpenAIDecisionResponse: - usage: Final = response.usage or DecisionsUsage() - return OpenAIDecisionResponse( - model=response.model or requested_model, - answers=tuple( - _openai_answer(question, response.answers.get(str(index))) for index, question in enumerate(questions) - ), - usage=OpenAIDecisionUsage( - input_tokens=usage.input_tokens, - output_tokens=usage.output_tokens, - total_tokens=usage.input_tokens + usage.output_tokens, - ), - ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9f210c2cc78..f601503824d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -118,7 +118,7 @@ from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject -from litellm.types.decisions import DecisionsResponse +from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse from litellm.types.interactions import ( InteractionsAPIResponse, InteractionsAPIStreamingResponse, @@ -2595,6 +2595,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, OCRResponse) # OCR or isinstance(logging_result, SearchResponse) # Search API or isinstance(logging_result, DecisionsResponse) + or isinstance(logging_result, OpenAIDecisionResponse) or ( isinstance(logging_result, InteractionsAPIResponse) and logging_result.usage is not None diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index b001942a8ba..23a00d60ba0 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -1,20 +1,286 @@ -from collections.abc import Mapping +import json +from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Protocol +from types import MappingProxyType +from typing import Final, Protocol -from litellm.types.decisions import DecisionsRequestBody +from pydantic import TypeAdapter +from typing_extensions import assert_never + +from litellm.types.decisions import ( + ChoiceAnswer, + ChoiceQuestion, + DecisionAnswer, + DecisionQuestion, + DecisionsIRAnswer, + DecisionsIRChoiceAnswer, + DecisionsIRChoiceOption, + DecisionsIRChoiceProbability, + DecisionsIRChoiceQuestion, + DecisionsIRMessages, + DecisionsIRPredicateAnswer, + DecisionsIRPredicateQuestion, + DecisionsIRQuestion, + DecisionsIRRefusal, + DecisionsIRRequest, + DecisionsIRResponse, + DecisionsIRScoreAnswer, + DecisionsIRScoreLevel, + DecisionsIRScoreProbability, + DecisionsIRScoreQuestion, + DecisionsIRState, + DecisionsIRUsage, + DecisionsJSON, + DecisionsRequestBody, + DecisionsResponse, + DecisionsUsage, + NoulAnswer, + NoulQuestion, + OpenAIDecisionInputImage, + OpenAIDecisionInputMessage, + OpenAIDecisionInputText, + ScoreAnswer, + ScoreQuestion, + UnsupportedDecisionsRequest, + systemone_choice_key, +) + +_SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) +_TEXT_ONLY: Final = UnsupportedDecisionsRequest( + reason="input_image content parts are not supported because System One providers accept text input only" +) -def systemone_request_body(model: str, request: DecisionsRequestBody) -> Mapping[str, object]: +def decisions_text(value: DecisionsJSON) -> str: + return value if isinstance(value, str) else json.dumps(value) + + +def systemone_keys(questions: Sequence[DecisionsIRQuestion]) -> tuple[str, ...]: + names: Final = tuple(question.name for question in questions if question.name is not None) + if len(frozenset(names)) == len(questions): + return names + return tuple(str(index) for index in range(len(questions))) + + +def _ir_question(name: str, question: DecisionQuestion) -> DecisionsIRQuestion: + extra: Final = MappingProxyType(question.model_extra or {}) + match question: + case NoulQuestion(): + return DecisionsIRPredicateQuestion( + name=name, instructions=question.instructions, criteria=question.criteria, extra=extra + ) + case ChoiceQuestion(): + return DecisionsIRChoiceQuestion( + name=name, + instructions=question.instructions, + choices=tuple( + DecisionsIRChoiceOption(value=value, description=description) + for value, description in question.criteria.items() + ), + extra=extra, + ) + case ScoreQuestion(): + return DecisionsIRScoreQuestion( + name=name, + instructions=question.instructions, + levels=tuple( + DecisionsIRScoreLevel(label=criterion, description=None) for criterion in question.criteria + ), + extra=extra, + ) + case _: + assert_never(question) + + +def systemone_request_to_ir(request: DecisionsRequestBody) -> DecisionsIRRequest: + return DecisionsIRRequest( + input=DecisionsIRState(state=request.state), + questions=tuple(_ir_question(name, question) for name, question in request.questions.items()), + ) + + +def _has_image(message: OpenAIDecisionInputMessage) -> bool: + return not isinstance(message.content, str) and any( + isinstance(part, OpenAIDecisionInputImage) for part in message.content + ) + + +def _message_text(message: OpenAIDecisionInputMessage) -> str: + if isinstance(message.content, str): + return message.content + return "\n\n".join(part.text for part in message.content if isinstance(part, OpenAIDecisionInputText)) + + +def _systemone_state( + decision_input: DecisionsIRState | DecisionsIRMessages, +) -> DecisionsJSON | UnsupportedDecisionsRequest: + match decision_input: + case DecisionsIRState(): + return decision_input.state + case DecisionsIRMessages(): + if any(_has_image(message) for message in decision_input.messages): + return _TEXT_ONLY + return "\n\n".join(_message_text(message) for message in decision_input.messages) + case _: + assert_never(decision_input) + + +def _optional(key: str, value: object) -> Mapping[str, object]: + return {} if value is None else {key: value} + + +def _level_criterion(level: DecisionsIRScoreLevel) -> DecisionsJSON: + return level.label if level.description is None else f"{decisions_text(level.label)}: {level.description}" + + +def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]: + match question: + case DecisionsIRPredicateQuestion(): + return { + "type": "noul", + **_optional("instructions", question.instructions), + **_optional("criteria", question.criteria), + **question.extra, + } + case DecisionsIRChoiceQuestion(): + return { + "type": "choice", + **_optional("instructions", question.instructions), + "criteria": {systemone_choice_key(option.value): option.description for option in question.choices}, + **question.extra, + } + case DecisionsIRScoreQuestion(): + return { + "type": "score", + **_optional("instructions", question.instructions), + "criteria": [_level_criterion(level) for level in question.levels], + **question.extra, + } + case _: + assert_never(question) + + +def ir_to_systemone_request( + model: str, request: DecisionsIRRequest +) -> Mapping[str, object] | UnsupportedDecisionsRequest: + state: Final = _systemone_state(request.input) + if isinstance(state, UnsupportedDecisionsRequest): + return state + keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True) return { "model": model, - "state": request.state, - "questions": { - name: question.model_dump(mode="json", exclude_none=True) for name, question in request.questions.items() - }, + "state": state, + "questions": {key: _systemone_question(question) for key, question in keyed_questions}, } +def _ir_choice_answer(question: DecisionsIRChoiceQuestion, answer: ChoiceAnswer) -> DecisionsIRChoiceAnswer: + typed_values: Final = {systemone_choice_key(option.value): option.value for option in question.choices} + return DecisionsIRChoiceAnswer( + choice=typed_values.get(answer.choice, answer.choice), + confidence=answer.confidence, + probabilities=tuple( + DecisionsIRChoiceProbability(value=typed_values.get(key, key), probability=probability) + for key, probability in answer.probabilities.items() + ), + ) + + +def _ir_score_answer(question: DecisionsIRScoreQuestion, answer: ScoreAnswer) -> DecisionsIRScoreAnswer: + return DecisionsIRScoreAnswer( + score=answer.score, + confidence=answer.confidence, + probabilities=tuple( + DecisionsIRScoreProbability( + value=index, + label=answer.legend.get(str(index), level.label), + probability=answer.probabilities.get(str(index), 0.0), + ) + for index, level in enumerate(question.levels) + ), + ) + + +def _ir_answer(question: DecisionsIRQuestion, answer: DecisionAnswer | None) -> DecisionsIRAnswer: + match question, answer: + case DecisionsIRPredicateQuestion(), NoulAnswer(): + return DecisionsIRPredicateAnswer(probability=answer.noul) + case DecisionsIRChoiceQuestion(), ChoiceAnswer(): + return _ir_choice_answer(question, answer) + case DecisionsIRScoreQuestion(), ScoreAnswer(): + return _ir_score_answer(question, answer) + case _: + return DecisionsIRRefusal() + + +def systemone_response_to_ir(response: DecisionsResponse, request: DecisionsIRRequest) -> DecisionsIRResponse: + usage: Final = response.usage or DecisionsUsage() + keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True) + return DecisionsIRResponse( + model=response.model, + answers=tuple(_ir_answer(question, response.answers.get(key)) for key, question in keyed_questions), + usage=DecisionsIRUsage( + input_tokens=usage.input_tokens, + output_tokens=usage.output_tokens, + cached_tokens=usage.cached_tokens, + cache_write_tokens=usage.cache_write_tokens, + extra=MappingProxyType(usage.model_extra or {}), + ), + extra=MappingProxyType(response.model_extra or {}), + ) + + +def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(payload), request) + + +def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None: + match answer: + case DecisionsIRPredicateAnswer(): + return NoulAnswer(type="noul", noul=answer.probability) + case DecisionsIRChoiceAnswer(): + return ChoiceAnswer( + type="choice", + choice=systemone_choice_key(answer.choice), + confidence=answer.confidence, + probabilities={systemone_choice_key(item.value): item.probability for item in answer.probabilities}, + ) + case DecisionsIRScoreAnswer(): + return ScoreAnswer( + type="score", + score=answer.score, + confidence=answer.confidence, + legend={str(item.value): item.label for item in answer.probabilities}, + probabilities={str(item.value): item.probability for item in answer.probabilities}, + ) + case DecisionsIRRefusal(): + return None + case _: + assert_never(answer) + + +def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIRRequest) -> DecisionsResponse: + keyed_answers: Final = zip( + systemone_keys(request.questions), (_systemone_answer(answer) for answer in response.answers), strict=True + ) + return DecisionsResponse.model_validate( + { + **response.extra, + "model": response.model, + "answers": {key: answer for key, answer in keyed_answers if answer is not None}, + "usage": DecisionsUsage.model_validate( + { + **response.usage.extra, + "input_tokens": response.usage.input_tokens, + "output_tokens": response.usage.output_tokens, + "cached_tokens": response.usage.cached_tokens, + "cache_write_tokens": response.usage.cache_write_tokens, + } + ), + } + ) + + @dataclass(frozen=True, slots=True) class JevCompatibleDecisionsEndpoint: default_api_base_value: str | None @@ -38,11 +304,13 @@ class JevCompatibleDecisionsEndpoint: def endpoint_url(self, api_base: str, model: str) -> str: return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" - def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: - return systemone_request_body(model, request) + def request_body( + self, model: str, request: DecisionsIRRequest + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + return ir_to_systemone_request(model, request) - def unwrap_response(self, payload: object) -> object: - return payload + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + return parse_systemone_response(payload, request) class DecisionsProviderConfig(Protocol): @@ -65,6 +333,8 @@ class DecisionsProviderConfig(Protocol): def endpoint_url(self, api_base: str, model: str) -> str: ... - def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: ... + def request_body( + self, model: str, request: DecisionsIRRequest + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: ... - def unwrap_response(self, payload: object) -> object: ... + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: ... diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 82c457630e6..a325689b0c9 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -4,12 +4,12 @@ from typing import Final from pydantic import TypeAdapter -from litellm.llms.base_llm.decisions.transformation import systemone_request_body +from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_request, parse_systemone_response from litellm.secret_managers.main import ( get_secret_str, normalize_nonempty_secret_str, ) -from litellm.types.decisions import DecisionsRequestBody +from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse, UnsupportedDecisionsRequest _RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) @@ -45,19 +45,25 @@ class CloudflareDecisionsEndpoint: return f"{normalized_api_base}/{model}" return f"{normalized_api_base}/ai/run/{model}" - def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: - return systemone_request_body(model, request) + def request_body( + self, model: str, request: DecisionsIRRequest + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + return ir_to_systemone_request(model, request) - def unwrap_response(self, payload: object) -> object: - if not isinstance(payload, Mapping): - return payload - response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) - if "answers" in response_mapping: - return payload - result: Final = response_mapping.get("result") - if isinstance(result, Mapping): - return result + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + return parse_systemone_response(_unwrap_result(payload), request) + + +def _unwrap_result(payload: object) -> object: + if not isinstance(payload, Mapping): return payload + response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) + if "answers" in response_mapping: + return payload + result: Final = response_mapping.get("result") + if isinstance(result, Mapping): + return result + return payload CLOUDFLARE_DECISIONS_ENDPOINT: Final[CloudflareDecisionsEndpoint] = CloudflareDecisionsEndpoint() diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index ea07a8a5b89..b7f5a95732c 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -1,4 +1,3 @@ -import json from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Final @@ -6,117 +5,267 @@ from typing import Final from pydantic import TypeAdapter from typing_extensions import assert_never +from litellm.llms.base_llm.decisions.transformation import decisions_text from litellm.types.decisions import ( - ChoiceAnswer, - ChoiceQuestion, - DecisionAnswer, - DecisionQuestion, + DecisionsIRAnswer, + DecisionsIRChoiceAnswer, + DecisionsIRChoiceOption, + DecisionsIRChoiceProbability, + DecisionsIRChoiceQuestion, + DecisionsIRMessages, + DecisionsIRPredicateAnswer, + DecisionsIRPredicateQuestion, + DecisionsIRQuestion, + DecisionsIRRefusal, + DecisionsIRRequest, + DecisionsIRResponse, + DecisionsIRScoreAnswer, + DecisionsIRScoreLevel, + DecisionsIRScoreProbability, + DecisionsIRScoreQuestion, + DecisionsIRState, + DecisionsIRUsage, DecisionsJSON, - DecisionsRequestBody, - DecisionsResponse, - DecisionsUsage, - NoulAnswer, - NoulQuestion, OpenAIChoiceAnswer, + OpenAIChoiceProbability, + OpenAIChoiceQuestion, OpenAIDecisionAnswer, + OpenAIDecisionInput, + OpenAIDecisionInputTokensDetails, + OpenAIDecisionOutputTokensDetails, + OpenAIDecisionQuestion, + OpenAIDecisionRequestBody, OpenAIDecisionResponse, + OpenAIDecisionUsage, OpenAIPredicateAnswer, + OpenAIPredicateQuestion, OpenAIRefusalAnswer, OpenAIScoreAnswer, - ScoreAnswer, - ScoreQuestion, + OpenAIScoreProbability, + OpenAIScoreQuestion, systemone_choice_key, ) _OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) -def _text(value: DecisionsJSON) -> str: - return value if isinstance(value, str) else json.dumps(value) +def _ir_input(decision_input: OpenAIDecisionInput) -> DecisionsIRState | DecisionsIRMessages: + if isinstance(decision_input, str): + return DecisionsIRState(state=decision_input) + return DecisionsIRMessages(messages=tuple(decision_input)) -def _instructions(instructions: DecisionsJSON | None) -> Mapping[str, str]: - return {} if instructions is None else {"instructions": _text(instructions)} +def _ir_question(question: OpenAIDecisionQuestion) -> DecisionsIRQuestion: + match question: + case OpenAIPredicateQuestion(): + return DecisionsIRPredicateQuestion(name=question.name, instructions=question.instructions) + case OpenAIChoiceQuestion(): + return DecisionsIRChoiceQuestion( + name=question.name, + instructions=question.instructions, + choices=tuple( + DecisionsIRChoiceOption(value=option.value, description=option.description) + for option in question.choices + ), + ) + case OpenAIScoreQuestion(): + return DecisionsIRScoreQuestion( + name=question.name, + instructions=question.instructions, + levels=tuple( + DecisionsIRScoreLevel(label=level.label, description=level.description) for level in question.levels + ), + ) + case _: + assert_never(question) -def _predicate_instructions(question: NoulQuestion) -> str: - instructions: Final = () if question.instructions is None else (_text(question.instructions),) +def openai_request_to_ir(request: OpenAIDecisionRequestBody) -> DecisionsIRRequest: + return DecisionsIRRequest( + input=_ir_input(request.input), + questions=tuple(_ir_question(question) for question in request.questions), + safety_identifier=request.safety_identifier, + ) + + +def _text_field(key: str, value: DecisionsJSON | None) -> Mapping[str, str]: + return {} if value is None else {key: decisions_text(value)} + + +def _predicate_instructions(question: DecisionsIRPredicateQuestion) -> str: + instructions: Final = () if question.instructions is None else (decisions_text(question.instructions),) criteria: Final = tuple( - f"Answer {answer} when: {_text(rule)}" for answer, rule in (question.criteria or {}).items() if rule is not None + f"Answer {answer} when: {decisions_text(rule)}" + for answer, rule in (question.criteria or {}).items() + if rule is not None ) return "\n\n".join((*instructions, *criteria)) -def _choice(value: str, description: DecisionsJSON | None) -> Mapping[str, str]: - return {"value": value} if description is None else {"value": value, "description": _text(description)} - - -def _openai_question(name: str, question: DecisionQuestion) -> Mapping[str, object]: +def _openai_question(question: DecisionsIRQuestion) -> Mapping[str, object]: match question: - case NoulQuestion(): - return {"type": "predicate", "name": name, "instructions": _predicate_instructions(question)} - case ChoiceQuestion(): + case DecisionsIRPredicateQuestion(): + return { + "type": "predicate", + **_text_field("name", question.name), + "instructions": _predicate_instructions(question), + } + case DecisionsIRChoiceQuestion(): return { "type": "choice", - "name": name, - **_instructions(question.instructions), - "choices": [_choice(value, description) for value, description in question.criteria.items()], + **_text_field("name", question.name), + **_text_field("instructions", question.instructions), + "choices": [ + {"value": option.value, **_text_field("description", option.description)} + for option in question.choices + ], } - case ScoreQuestion(): + case DecisionsIRScoreQuestion(): return { "type": "score", - "name": name, - **_instructions(question.instructions), - "levels": [{"label": _text(criterion)} for criterion in question.criteria], + **_text_field("name", question.name), + **_text_field("instructions", question.instructions), + "levels": [ + {"label": decisions_text(level.label), **_text_field("description", level.description)} + for level in question.levels + ], } case _: assert_never(question) -def to_openai_request(model: str, request: DecisionsRequestBody) -> Mapping[str, object]: +def _openai_input(decision_input: DecisionsIRState | DecisionsIRMessages) -> str | Sequence[Mapping[str, object]]: + match decision_input: + case DecisionsIRState(): + return decisions_text(decision_input.state) + case DecisionsIRMessages(): + return [message.model_dump(mode="json", exclude_none=True) for message in decision_input.messages] + case _: + assert_never(decision_input) + + +def ir_to_openai_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]: return { "model": model, - "input": _text(request.state), - "questions": [_openai_question(name, question) for name, question in request.questions.items()], + "input": _openai_input(request.input), + "questions": [_openai_question(question) for question in request.questions], + **_text_field("safety_identifier", request.safety_identifier), } -def _systemone_answer(answer: OpenAIDecisionAnswer) -> DecisionAnswer | None: - match answer: - case OpenAIPredicateAnswer(): - return NoulAnswer(type="noul", noul=answer.probability) - case OpenAIChoiceAnswer(): - return ChoiceAnswer( - type="choice", - choice=systemone_choice_key(answer.choice), +def _level_label(question: DecisionsIRScoreQuestion, probability: OpenAIScoreProbability) -> DecisionsJSON: + if 0 <= probability.value < len(question.levels): + return question.levels[probability.value].label + return probability.label + + +def _ir_answer(question: DecisionsIRQuestion, answer: OpenAIDecisionAnswer | None) -> DecisionsIRAnswer: + match question, answer: + case DecisionsIRPredicateQuestion(), OpenAIPredicateAnswer(): + return DecisionsIRPredicateAnswer(probability=answer.probability) + case DecisionsIRChoiceQuestion(), OpenAIChoiceAnswer(): + return DecisionsIRChoiceAnswer( + choice=answer.choice, confidence=answer.confidence, - probabilities={systemone_choice_key(item.value): item.probability for item in answer.probabilities}, + probabilities=tuple( + DecisionsIRChoiceProbability(value=item.value, probability=item.probability) + for item in answer.probabilities + ), ) - case OpenAIScoreAnswer(): - return ScoreAnswer( - type="score", + case DecisionsIRScoreQuestion(), OpenAIScoreAnswer(): + return DecisionsIRScoreAnswer( score=answer.score, confidence=answer.confidence, - legend={str(item.value): item.label for item in answer.probabilities}, - probabilities={str(item.value): item.probability for item in answer.probabilities}, + probabilities=tuple( + DecisionsIRScoreProbability( + value=item.value, label=_level_label(question, item), probability=item.probability + ) + for item in answer.probabilities + ), ) - case OpenAIRefusalAnswer(): - return None case _: - assert_never(answer) + return DecisionsIRRefusal() -def _named_answers(answers: Sequence[OpenAIDecisionAnswer]) -> Mapping[str, DecisionAnswer]: - translated: Final = ((answer.name, _systemone_answer(answer)) for answer in answers) - return {name: answer for name, answer in translated if name is not None and answer is not None} - - -def to_systemone_response(payload: object) -> DecisionsResponse: - response: Final = _OPENAI_RESPONSE_ADAPTER.validate_python(payload) - return DecisionsResponse( +def openai_response_to_ir(response: OpenAIDecisionResponse, request: DecisionsIRRequest) -> DecisionsIRResponse: + answers: Final = response.answers + return DecisionsIRResponse( model=response.model, - answers=_named_answers(response.answers), - usage=DecisionsUsage(input_tokens=response.usage.input_tokens, output_tokens=response.usage.output_tokens), + answers=tuple( + _ir_answer(question, answers[index] if index < len(answers) else None) + for index, question in enumerate(request.questions) + ), + usage=DecisionsIRUsage( + input_tokens=response.usage.input_tokens, + output_tokens=response.usage.output_tokens, + cached_tokens=response.usage.input_tokens_details.cached_tokens, + cache_write_tokens=response.usage.input_tokens_details.cache_write_tokens, + reasoning_tokens=response.usage.output_tokens_details.reasoning_tokens, + ), + ) + + +def _openai_choice_answer(question: DecisionsIRChoiceQuestion, answer: DecisionsIRChoiceAnswer) -> OpenAIChoiceAnswer: + probabilities: Final = {systemone_choice_key(item.value): item.probability for item in answer.probabilities} + return OpenAIChoiceAnswer( + name=question.name, + choice=answer.choice, + probabilities=tuple( + OpenAIChoiceProbability( + value=option.value, probability=probabilities.get(systemone_choice_key(option.value), 0.0) + ) + for option in question.choices + ), + confidence=answer.confidence, + ) + + +def _openai_score_answer(question: DecisionsIRScoreQuestion, answer: DecisionsIRScoreAnswer) -> OpenAIScoreAnswer: + probabilities: Final = {item.value: item.probability for item in answer.probabilities} + return OpenAIScoreAnswer( + name=question.name, + score=answer.score, + probabilities=tuple( + OpenAIScoreProbability( + value=index, label=decisions_text(level.label), probability=probabilities.get(index, 0.0) + ) + for index, level in enumerate(question.levels) + ), + confidence=answer.confidence, + ) + + +def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) -> OpenAIDecisionAnswer: + match question, answer: + case DecisionsIRPredicateQuestion(), DecisionsIRPredicateAnswer(): + return OpenAIPredicateAnswer(name=question.name, probability=answer.probability) + case DecisionsIRChoiceQuestion(), DecisionsIRChoiceAnswer(): + return _openai_choice_answer(question, answer) + case DecisionsIRScoreQuestion(), DecisionsIRScoreAnswer(): + return _openai_score_answer(question, answer) + case _: + return OpenAIRefusalAnswer(name=question.name) + + +def ir_to_openai_response( + response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str +) -> OpenAIDecisionResponse: + return OpenAIDecisionResponse( + model=response.model or requested_model, + answers=tuple( + _openai_answer(question, answer) + for question, answer in zip(request.questions, response.answers, strict=True) + ), + usage=OpenAIDecisionUsage( + input_tokens=response.usage.input_tokens, + input_tokens_details=OpenAIDecisionInputTokensDetails( + cached_tokens=response.usage.cached_tokens, + cache_write_tokens=response.usage.cache_write_tokens, + ), + output_tokens=response.usage.output_tokens, + output_tokens_details=OpenAIDecisionOutputTokensDetails(reasoning_tokens=response.usage.reasoning_tokens), + total_tokens=response.usage.input_tokens + response.usage.output_tokens, + ), ) @@ -141,11 +290,11 @@ class OpenAIDecisionsEndpoint: def endpoint_url(self, api_base: str, model: str) -> str: return f"{api_base.rstrip('/').removesuffix('/v1')}/v1/decisions" - def request_body(self, model: str, request: DecisionsRequestBody) -> Mapping[str, object]: - return to_openai_request(model, request) + def request_body(self, model: str, request: DecisionsIRRequest) -> Mapping[str, object]: + return ir_to_openai_request(model, request) - def unwrap_response(self, payload: object) -> DecisionsResponse: - return to_systemone_response(payload) + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + return openai_response_to_ir(_OPENAI_RESPONSE_ADAPTER.validate_python(payload), request) OPENAI_DECISIONS_ENDPOINT: Final[OpenAIDecisionsEndpoint] = OpenAIDecisionsEndpoint() diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 4170e3a13c3..86fc4f9da2c 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -5,11 +5,10 @@ from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract from pydantic import TypeAdapter, ValidationError -from litellm.decisions.openai_transformation import to_openai_response, to_systemone_request from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.decisions import DecisionsRequestBody, DecisionsResponse, OpenAIDecisionRequestBody +from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) @@ -17,7 +16,6 @@ _DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = Type _OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter( OpenAIDecisionRequestBody ) -_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) _OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) @@ -48,12 +46,11 @@ async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> raise await _invalid_request(raw_data={}, error=error, user_api_key_dict=user_api_key_dict) -async def _process_systemone( +async def _process_decisions( request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, - raw_data: Mapping[str, object], - openai_body: OpenAIDecisionRequestBody | None, + body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody], ) -> object: from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, @@ -76,15 +73,18 @@ async def _process_systemone( user_temperature as proxy_user_temperature, ) - data: Final = dict(raw_data if openai_body is None else to_systemone_request(raw_data, openai_body)) + data: Final = await _request_data(request, user_api_key_dict) + try: + body_adapter.validate_python(data) + except ValidationError as error: + raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict) general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings) user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base) user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model) user_temperature: Final = _OPTIONAL_FLOAT_ADAPTER.validate_python(proxy_user_temperature) processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - _DECISIONS_REQUEST_BODY_ADAPTER.validate_python(data) - result: Final[object] = await processor.base_process_llm_request( + return await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -102,15 +102,6 @@ async def _process_systemone( user_api_base=user_api_base, version=version, ) - if openai_body is None or isinstance(result, Response): - return result - return to_openai_response( - _DECISIONS_RESPONSE_ADAPTER.validate_python(result), - openai_body.questions, - str(data.get("model", "")), - ) - except ValidationError as error: - raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict) except Exception as error: raise await processor._handle_llm_api_exception( e=error, @@ -137,12 +128,11 @@ async def systemone( fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): - return await _process_systemone( + return await _process_decisions( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, - raw_data=await _request_data(request, user_api_key_dict), - openai_body=None, + body_adapter=_DECISIONS_REQUEST_BODY_ADAPTER, ) @@ -163,15 +153,9 @@ async def decisions( fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): - raw_data: Final = await _request_data(request, user_api_key_dict) - try: - openai_body: Final = _OPENAI_DECISION_REQUEST_BODY_ADAPTER.validate_python(raw_data) - except ValidationError as error: - raise await _invalid_request(raw_data=raw_data, error=error, user_api_key_dict=user_api_key_dict) - return await _process_systemone( + return await _process_decisions( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, - raw_data=raw_data, - openai_body=openai_body, + body_adapter=_OPENAI_DECISION_REQUEST_BODY_ADAPTER, ) diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index dc19881c857..ba3c93a7d41 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -1,4 +1,6 @@ from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType from typing import Annotated, Final, Literal, TypeAlias from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config @@ -110,19 +112,23 @@ DecisionAnswer: TypeAlias = Annotated[ class DecisionsUsage(LiteLLMPydanticObjectBase): input_tokens: int = 0 output_tokens: int = 0 + cached_tokens: Annotated[int, Field(exclude=True)] = 0 + cache_write_tokens: Annotated[int, Field(exclude=True)] = 0 model_config = ConfigDict(extra="allow", frozen=True) -class DecisionsResponse(LiteLLMPydanticObjectBase): +class _HiddenParamsResponse(LiteLLMPydanticObjectBase): + _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) + + +class DecisionsResponse(_HiddenParamsResponse): model: str | None = None answers: Mapping[str, DecisionAnswer] usage: DecisionsUsage | None = None model_config = ConfigDict(extra="allow", frozen=True) - _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) - class OpenAIDecisionInputText(LiteLLMPydanticObjectBase): type: Literal["input_text"] @@ -131,14 +137,31 @@ class OpenAIDecisionInputText(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="forbid", frozen=True) +class OpenAIDecisionInputImage(LiteLLMPydanticObjectBase): + type: Literal["input_image"] + image_url: str + detail: str | None = None + + model_config = ConfigDict(extra="forbid", frozen=True) + + +OpenAIDecisionContentPart: TypeAlias = Annotated[ + OpenAIDecisionInputText | OpenAIDecisionInputImage, + Field(discriminator="type"), +] + + class OpenAIDecisionInputMessage(LiteLLMPydanticObjectBase): role: Literal["user"] = "user" type: Literal["message"] = "message" - content: str | Sequence[OpenAIDecisionInputText] + content: str | Sequence[OpenAIDecisionContentPart] model_config = ConfigDict(extra="forbid", frozen=True) +OpenAIDecisionInput: TypeAlias = str | Sequence[OpenAIDecisionInputMessage] + + class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase): type: Literal["predicate"] name: str | None = None @@ -199,7 +222,7 @@ OpenAIDecisionQuestion: TypeAlias = Annotated[ class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): - input: str | Sequence[OpenAIDecisionInputMessage] + input: OpenAIDecisionInput questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)] safety_identifier: str | None = None @@ -285,9 +308,133 @@ class OpenAIDecisionUsage(LiteLLMPydanticObjectBase): model_config = ConfigDict(frozen=True) -class OpenAIDecisionResponse(LiteLLMPydanticObjectBase): +class OpenAIDecisionResponse(_HiddenParamsResponse): model: str answers: tuple[OpenAIDecisionAnswer, ...] usage: OpenAIDecisionUsage model_config = ConfigDict(frozen=True) + + +_NO_EXTRA: Final[Mapping[str, object]] = MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRState: + state: DecisionsJSON + + +@dataclass(frozen=True, slots=True) +class DecisionsIRMessages: + messages: tuple[OpenAIDecisionInputMessage, ...] + + +@dataclass(frozen=True, slots=True) +class DecisionsIRPredicateQuestion: + name: str | None + instructions: DecisionsJSON | None + criteria: NoulCriteria | None = None + extra: Mapping[str, object] = field(default=_NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRChoiceOption: + value: str | bool + description: DecisionsJSON | None + + +@dataclass(frozen=True, slots=True) +class DecisionsIRChoiceQuestion: + name: str | None + instructions: DecisionsJSON | None + choices: tuple[DecisionsIRChoiceOption, ...] + extra: Mapping[str, object] = field(default=_NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRScoreLevel: + label: DecisionsJSON + description: str | None + + +@dataclass(frozen=True, slots=True) +class DecisionsIRScoreQuestion: + name: str | None + instructions: DecisionsJSON | None + levels: tuple[DecisionsIRScoreLevel, ...] + extra: Mapping[str, object] = field(default=_NO_EXTRA) + + +DecisionsIRQuestion: TypeAlias = DecisionsIRPredicateQuestion | DecisionsIRChoiceQuestion | DecisionsIRScoreQuestion + + +@dataclass(frozen=True, slots=True) +class DecisionsIRRequest: + input: DecisionsIRState | DecisionsIRMessages + questions: tuple[DecisionsIRQuestion, ...] + safety_identifier: str | None = None + + +@dataclass(frozen=True, slots=True) +class DecisionsIRPredicateAnswer: + probability: float + + +@dataclass(frozen=True, slots=True) +class DecisionsIRChoiceProbability: + value: str | bool + probability: float + + +@dataclass(frozen=True, slots=True) +class DecisionsIRChoiceAnswer: + choice: str | bool + confidence: float + probabilities: tuple[DecisionsIRChoiceProbability, ...] + + +@dataclass(frozen=True, slots=True) +class DecisionsIRScoreProbability: + value: int + label: DecisionsJSON + probability: float + + +@dataclass(frozen=True, slots=True) +class DecisionsIRScoreAnswer: + score: float + confidence: float + probabilities: tuple[DecisionsIRScoreProbability, ...] + + +@dataclass(frozen=True, slots=True) +class DecisionsIRRefusal: + pass + + +DecisionsIRAnswer: TypeAlias = ( + DecisionsIRPredicateAnswer | DecisionsIRChoiceAnswer | DecisionsIRScoreAnswer | DecisionsIRRefusal +) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRUsage: + input_tokens: int = 0 + output_tokens: int = 0 + cached_tokens: int = 0 + cache_write_tokens: int = 0 + reasoning_tokens: int = 0 + extra: Mapping[str, object] = field(default=_NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRResponse: + model: str | None + answers: tuple[DecisionsIRAnswer, ...] + usage: DecisionsIRUsage + extra: Mapping[str, object] = field(default=_NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class UnsupportedDecisionsRequest: + reason: str diff --git a/litellm/utils.py b/litellm/utils.py index 731f1e9b3fd..cb0920b0694 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1201,8 +1201,8 @@ def function_setup( else search_query ) elif call_type in (CallTypes.decisions.value, CallTypes.adecisions.value): - decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state", "") - messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state) + decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state") or kwargs.get("input") or "" + messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state, default=str) elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): messages = args[1] if len(args) > 1 else kwargs.get("prompt") elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value): diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 77f23e5761e..7b269148448 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -17,6 +17,7 @@ from litellm.types.decisions import ( DecisionsResponse, DecisionsUsage, NoulAnswer, + OpenAIDecisionResponse, ScoreAnswer, ) @@ -29,6 +30,8 @@ _QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( ) _INPUT_TOKENS: Final[int] = 367 _OUTPUT_TOKENS: Final[int] = 3 +_CACHED_TOKENS: Final[int] = 256 +_CACHE_WRITE_TOKENS: Final[int] = 64 _RESPONSE: Final[Mapping[str, object]] = { "model": "jev-1.13", "answers": { @@ -87,7 +90,7 @@ _OPENAI_RESPONSE: Final[Mapping[str, object]] = { ], "usage": { "input_tokens": _INPUT_TOKENS, - "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "input_tokens_details": {"cached_tokens": _CACHED_TOKENS, "cache_write_tokens": _CACHE_WRITE_TOKENS}, "output_tokens": _OUTPUT_TOKENS, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, @@ -419,7 +422,7 @@ async def test_cloudflare_clef_resolves_model_and_response_envelope( "state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } - assert response.answers == DecisionsResponse.model_validate(_RESPONSE).answers + assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} assert response._hidden_params["model"] == "cloudflare/@cf/cloudflare/clef" @@ -658,8 +661,88 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_ assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} assert response.hidden_params["custom_llm_provider"] == "openai" luna_cost: Final = litellm.model_cost["gpt-6-luna"] - expected_cost: Final = _INPUT_TOKENS * float(luna_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( - luna_cost["output_cost_per_token"] + expected_cost: Final = ( + (_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * float(luna_cost["input_cost_per_token"]) + + _CACHED_TOKENS * float(luna_cost["cache_read_input_token_cost"]) + + _CACHE_WRITE_TOKENS * float(luna_cost["cache_creation_input_token_cost"]) + + _OUTPUT_TOKENS * float(luna_cost["output_cost_per_token"]) ) assert expected_cost > 0 assert litellm.completion_cost(completion_response=response) == pytest.approx(expected_cost) + + +@pytest.mark.asyncio +async def test_openai_format_calls_to_a_systemone_provider_get_openai_format_answers( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="typesafe/jev-1.13", + input="review", + questions=[ + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "sentiment", + "instructions": "Tone?", + "choices": [{"value": "positive"}, {"value": "negative"}], + }, + ], + api_key="caller-key", + ) + + assert route.called + assert tuple(json.loads(respx_mock.calls[0].request.content)["questions"]) == ("is_defect", "sentiment") + assert isinstance(response, OpenAIDecisionResponse) + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + { + "type": "choice", + "name": "sentiment", + "choice": "positive", + "probabilities": [{"value": "positive", "probability": 0.8}, {"value": "negative", "probability": 0.2}], + "confidence": 0.8, + }, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "request_kwargs", "message"), + ( + ( + "typesafe/jev-1.13", + { + "input": [ + { + "role": "user", + "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}], + } + ], + "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], + }, + "cannot serve this request", + ), + ( + "openai/gpt-6-luna", + { + "state": "review", + "input": "review", + "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], + }, + "not both", + ), + ), + ids=("image_to_systemone_provider", "state_and_input"), +) +async def test_requests_a_provider_cannot_serve_are_rejected_before_http( + respx_mock: respx.MockRouter, + model: str, + request_kwargs: Mapping[str, object], + message: str, +) -> None: + with pytest.raises(litellm.BadRequestError, match=message): + await litellm.adecisions(model=model, api_key="caller-key", **request_kwargs) + + assert len(respx_mock.calls) == 0 diff --git a/tests/unit/decisions/test_openai_transformation.py b/tests/unit/decisions/test_openai_transformation.py deleted file mode 100644 index a3cc1583d40..00000000000 --- a/tests/unit/decisions/test_openai_transformation.py +++ /dev/null @@ -1,32 +0,0 @@ -from collections.abc import Mapping -from typing import Final - -import pytest -from pydantic import TypeAdapter, ValidationError - -from litellm.decisions.openai_transformation import to_systemone_request -from litellm.types.decisions import MAX_DECISION_QUESTIONS, DecisionsRequestBody, OpenAIDecisionRequestBody - -_OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) -_SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) - - -def _openai_request(question_count: int) -> Mapping[str, object]: - return { - "model": "decider", - "input": "The package arrived with a broken screen.", - "questions": [{"type": "predicate", "instructions": f"Question {index}?"} for index in range(question_count)], - } - - -def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_request() -> None: - raw: Final = _openai_request(MAX_DECISION_QUESTIONS) - - translated: Final = _SYSTEMONE_BODY.validate_python(to_systemone_request(raw, _OPENAI_BODY.validate_python(raw))) - - assert len(translated.questions) == MAX_DECISION_QUESTIONS - - -def test_an_openai_request_with_more_questions_than_systemone_takes_is_rejected_before_translation() -> None: - with pytest.raises(ValidationError, match="questions"): - _OPENAI_BODY.validate_python(_openai_request(MAX_DECISION_QUESTIONS + 1)) diff --git a/tests/unit/llms/base_llm/decisions/__init__.py b/tests/unit/llms/base_llm/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py new file mode 100644 index 00000000000..9c4d8a27260 --- /dev/null +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -0,0 +1,178 @@ +from collections.abc import Mapping +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm.llms.base_llm.decisions.transformation import ( + ir_to_systemone_request, + ir_to_systemone_response, + parse_systemone_response, + systemone_request_to_ir, +) +from litellm.llms.openai.decisions.transformation import openai_request_to_ir +from litellm.types.decisions import ( + MAX_DECISION_QUESTIONS, + DecisionsIRRefusal, + DecisionsIRRequest, + DecisionsRequestBody, + OpenAIDecisionRequestBody, + UnsupportedDecisionsRequest, +) + +_SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) + +_SYSTEMONE_REQUEST: Final[Mapping[str, object]] = { + "state": {"ticket": 1234, "text": "Screen cracked"}, + "questions": { + "damaged": { + "type": "noul", + "instructions": "Is the item damaged?", + "criteria": {"true": "Visible damage", "false": None}, + "provider_field": "dropped", + }, + "rubric_only": {"type": "noul", "criteria": {"true": {"signal": "refund"}}}, + "action": { + "type": "choice", + "instructions": {"policy": "refund-v2"}, + "criteria": {"refund": "Within 30 days", "escalate": None}, + }, + "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, + }, +} + + +def _openai_ir(raw: Mapping[str, object]) -> DecisionsIRRequest: + return openai_request_to_ir(_OPENAI_BODY.validate_python(raw)) + + +def test_a_systemone_request_reaches_a_systemone_provider_unchanged() -> None: + ir: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python(_SYSTEMONE_REQUEST)) + + assert ir_to_systemone_request("jev-latest", ir) == {"model": "jev-latest", **_SYSTEMONE_REQUEST} + + +@pytest.mark.parametrize( + ("names", "keys"), + ( + (("damaged", "action"), ("damaged", "action")), + (("damaged", None), ("0", "1")), + (("damaged", "damaged"), ("0", "1")), + ), + ids=("all_named", "one_unnamed", "duplicate_names"), +) +def test_openai_questions_are_keyed_by_name_only_when_every_name_is_unique( + names: tuple[str | None, str | None], keys: tuple[str, str] +) -> None: + questions: Final = [ + {"type": "predicate", "instructions": f"Question {index}?", **({} if name is None else {"name": name})} + for index, name in enumerate(names) + ] + + body: Final = ir_to_systemone_request("jev-latest", _openai_ir({"input": "review", "questions": questions})) + + assert tuple(_SYSTEMONE_BODY.validate_python(body).questions) == keys + + +def test_openai_messages_become_systemone_state_text() -> None: + ir: Final = _openai_ir( + { + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "The package arrived broken."}, + {"type": "input_text", "text": "I want a refund."}, + ], + }, + {"role": "user", "content": "Order 1234."}, + ], + "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], + } + ) + + body: Final = ir_to_systemone_request("jev-latest", ir) + + assert isinstance(body, Mapping) + assert body["state"] == "The package arrived broken.\n\nI want a refund.\n\nOrder 1234." + + +def test_image_input_is_unsupported_by_systemone_providers() -> None: + ir: Final = _openai_ir( + { + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Is the screen cracked?"}, + {"type": "input_image", "image_url": "data:image/png;base64,AA=="}, + ], + } + ], + "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], + } + ) + + assert isinstance(ir_to_systemone_request("jev-latest", ir), UnsupportedDecisionsRequest) + + +def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_request() -> None: + ir: Final = _openai_ir( + { + "input": "review", + "questions": [ + {"type": "predicate", "instructions": f"Question {index}?"} for index in range(MAX_DECISION_QUESTIONS) + ], + } + ) + + translated: Final = _SYSTEMONE_BODY.validate_python(ir_to_systemone_request("jev-latest", ir)) + + assert len(translated.questions) == MAX_DECISION_QUESTIONS + + +def test_a_systemone_response_reaches_the_caller_unchanged_with_provider_extras() -> None: + payload: Final = { + "model": "jev-latest", + "answers": { + "damaged": {"type": "noul", "noul": 0.95}, + "action": { + "type": "choice", + "choice": "refund", + "confidence": 0.8, + "probabilities": {"refund": 0.9, "escalate": 0.1}, + }, + "severity": { + "type": "score", + "score": 0.4, + "confidence": 0.6, + "legend": {"0": "minor", "1": {"label": "major"}}, + "probabilities": {"0": 0.6, "1": 0.4}, + }, + }, + "usage": {"input_tokens": 383, "output_tokens": 2, "cost": 0.25}, + "latency_ms": 3722.17, + } + ir: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python(_SYSTEMONE_REQUEST)) + + response: Final = ir_to_systemone_response(parse_systemone_response(payload, ir), ir) + + assert response.model_dump(mode="json") == payload + + +def test_missing_or_mismatched_systemone_answers_are_refusals_left_out_of_the_response() -> None: + ir: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python(_SYSTEMONE_REQUEST)) + payload: Final = { + "model": "jev-latest", + "answers": { + "damaged": {"type": "noul", "noul": 0.95}, + "action": {"type": "noul", "noul": 0.5}, + }, + "usage": {"input_tokens": 10, "output_tokens": 1}, + } + + parsed: Final = parse_systemone_response(payload, ir) + + assert parsed.answers[1:] == (DecisionsIRRefusal(), DecisionsIRRefusal(), DecisionsIRRefusal()) + assert tuple(ir_to_systemone_response(parsed, ir).answers) == ("damaged",) diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index 56c950cc8d0..942fad1db6d 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -1,11 +1,113 @@ +from collections.abc import Mapping from typing import Final from pydantic import TypeAdapter -from litellm.llms.openai.decisions.transformation import to_openai_request, to_systemone_response -from litellm.types.decisions import DecisionsRequestBody +from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_response, systemone_request_to_ir +from litellm.llms.openai.decisions.transformation import ( + OPENAI_DECISIONS_ENDPOINT, + ir_to_openai_request, + ir_to_openai_response, + openai_request_to_ir, +) +from litellm.types.decisions import DecisionsIRRequest, DecisionsRequestBody, OpenAIDecisionRequestBody _SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) + +_OPENAI_REQUEST: Final[Mapping[str, object]] = { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "The screen is cracked."}, + {"type": "input_image", "image_url": "data:image/png;base64,AA==", "detail": "high"}, + ], + }, + {"type": "message", "role": "user", "content": "Order 1234."}, + ], + "questions": [ + {"type": "predicate", "name": "damaged", "instructions": "Is the item damaged?"}, + { + "type": "choice", + "instructions": "Should we refund?", + "choices": [{"value": True, "description": "Refund now"}, {"value": "escalate"}], + }, + { + "type": "score", + "name": "severity", + "instructions": "How severe is it?", + "levels": [{"label": "minor"}, {"label": "major", "description": "Product unusable"}], + }, + {"type": "predicate", "name": "fraud", "instructions": "Is this fraud?"}, + ], + "safety_identifier": "end-user-1", +} +_PREDICATE_ANSWER: Final[Mapping[str, object]] = {"type": "predicate", "name": "damaged", "probability": 0.95} +_CHOICE_ANSWER: Final[Mapping[str, object]] = { + "type": "choice", + "name": None, + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, +} +_REFUSAL_ANSWER: Final[Mapping[str, object]] = {"type": "refusal", "name": "fraud"} +_USAGE: Final[Mapping[str, object]] = { + "input_tokens": 383, + "input_tokens_details": {"cached_tokens": 256, "cache_write_tokens": 64}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 1}, + "total_tokens": 385, +} +_OPENAI_RESPONSE: Final[Mapping[str, object]] = { + "model": "gpt-6-luna", + "answers": [ + _PREDICATE_ANSWER, + _CHOICE_ANSWER, + { + "type": "score", + "name": "severity", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": "major", "probability": 0.7}, + ], + "confidence": 0.6, + }, + _REFUSAL_ANSWER, + ], + "usage": _USAGE, +} + + +def _openai_ir(raw: Mapping[str, object]) -> DecisionsIRRequest: + return openai_request_to_ir(_OPENAI_BODY.validate_python(raw)) + + +def test_an_openai_request_reaches_openai_unchanged() -> None: + assert ir_to_openai_request("gpt-6-luna", _openai_ir(_OPENAI_REQUEST)) == { + "model": "gpt-6-luna", + **_OPENAI_REQUEST, + } + + +def test_an_openai_response_reaches_the_caller_unchanged() -> None: + ir: Final = _openai_ir(_OPENAI_REQUEST) + + parsed: Final = OPENAI_DECISIONS_ENDPOINT.parse_response(_OPENAI_RESPONSE, ir) + + assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE + + +def test_answers_openai_did_not_return_are_refusals() -> None: + ir: Final = _openai_ir(_OPENAI_REQUEST) + payload: Final = {**_OPENAI_RESPONSE, "answers": [_PREDICATE_ANSWER]} + + response: Final = ir_to_openai_response(OPENAI_DECISIONS_ENDPOINT.parse_response(payload, ir), ir, "requested") + + assert [answer.type for answer in response.answers] == ["predicate", "refusal", "refusal", "refusal"] + assert [answer.name for answer in response.answers] == ["damaged", None, "severity", "fraud"] def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_their_keys() -> None: @@ -30,7 +132,7 @@ def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_t } ) - assert to_openai_request("gpt-6-luna", request) == { + assert ir_to_openai_request("gpt-6-luna", systemone_request_to_ir(request)) == { "model": "gpt-6-luna", "input": '{"ticket": 1234, "text": "Screen cracked"}', "questions": [ @@ -51,41 +153,40 @@ def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_t } -def test_an_openai_response_becomes_systemone_answers_keyed_by_question_name_without_refusals() -> None: - response: Final = to_systemone_response( - { - "model": "gpt-6-luna", - "answers": [ - {"type": "predicate", "name": "damaged", "probability": 0.95}, - { - "type": "choice", - "name": "action", - "choice": True, - "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], - "confidence": 0.8, +def test_an_openai_response_becomes_systemone_answers_with_the_callers_score_labels() -> None: + ir: Final = systemone_request_to_ir( + _SYSTEMONE_BODY.validate_python( + { + "state": "Screen cracked", + "questions": { + "damaged": {"type": "noul", "instructions": "Damaged?"}, + "action": {"type": "choice", "criteria": {"true": None, "escalate": None}}, + "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, + "fraud": {"type": "noul", "instructions": "Fraud?"}, }, - { - "type": "score", - "name": "severity", - "score": 1.72, - "probabilities": [ - {"value": 0, "label": "minor", "probability": 0.0}, - {"value": 1, "label": "major", "probability": 0.28}, - {"value": 2, "label": "critical", "probability": 0.72}, - ], - "confidence": 0.58, - }, - {"type": "refusal", "name": "fraud"}, - ], - "usage": { - "input_tokens": 383, - "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, - "output_tokens": 2, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 385, - }, - } + } + ) ) + payload: Final = { + **_OPENAI_RESPONSE, + "answers": [ + _PREDICATE_ANSWER, + _CHOICE_ANSWER, + { + "type": "score", + "name": "severity", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": '{"label": "major"}', "probability": 0.7}, + ], + "confidence": 0.6, + }, + _REFUSAL_ANSWER, + ], + } + + response: Final = ir_to_systemone_response(OPENAI_DECISIONS_ENDPOINT.parse_response(payload, ir), ir) assert response.model_dump(mode="json") == { "model": "gpt-6-luna", @@ -99,11 +200,13 @@ def test_an_openai_response_becomes_systemone_answers_keyed_by_question_name_wit }, "severity": { "type": "score", - "score": 1.72, - "confidence": 0.58, - "legend": {"0": "minor", "1": "major", "2": "critical"}, - "probabilities": {"0": 0.0, "1": 0.28, "2": 0.72}, + "score": 0.7, + "confidence": 0.6, + "legend": {"0": "minor", "1": {"label": "major"}}, + "probabilities": {"0": 0.3, "1": 0.7}, }, }, "usage": {"input_tokens": 383, "output_tokens": 2}, } + assert response.usage is not None + assert (response.usage.cached_tokens, response.usage.cache_write_tokens) == (256, 64) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index a92059772ea..295a10a3d9b 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -440,7 +440,7 @@ def test_a_body_that_is_not_json_is_a_client_error( assert not upstream.called -def test_openai_format_decisions_reach_an_openai_deployment_with_the_callers_questions( +def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_images( client: TestClient, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, @@ -453,61 +453,52 @@ def test_openai_format_decisions_reach_an_openai_deployment_with_the_callers_que model_list=[{"model_name": "decider", "litellm_params": {"model": "openai/gpt-6-luna", "api_key": "k"}}] ), ) + image_message: Final = { + "type": "message", + "role": "user", + "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA==", "detail": "low"}], + } + request_body: Final = {**_OPENAI_FORMAT_REQUEST, "input": [*_OPENAI_FORMAT_REQUEST["input"], image_message]} + cached_tokens: Final = 128 + upstream_usage: Final = { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": cached_tokens, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + } upstream: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond( - json={ - "model": "gpt-6-luna", - "answers": [ - {"type": "predicate", "name": "0", "probability": 0.95}, - { - "type": "choice", - "name": "1", - "choice": "true", - "probabilities": [{"value": "true", "probability": 0.9}, {"value": "escalate", "probability": 0.1}], - "confidence": 0.8, - }, - { - "type": "score", - "name": "2", - "score": 0.7, - "probabilities": [ - {"value": 0, "label": "minor", "probability": 0.3}, - {"value": 1, "label": "major: Product unusable", "probability": 0.7}, - ], - "confidence": 0.6, - }, - {"type": "refusal", "name": "3"}, - ], - "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS, "total_tokens": 370}, - } + json={"model": "gpt-6-luna", "answers": _OPENAI_FORMAT_ANSWERS, "usage": upstream_usage} ) - response: Final = client.post("/v1/decisions", json=_OPENAI_FORMAT_REQUEST) + response: Final = client.post("/v1/decisions", json=request_body) assert response.status_code == 200, response.text assert json.loads(upstream.calls[0].request.content) == { "model": "gpt-6-luna", - "input": "The package arrived with a broken screen.\n\nI want a refund.\n\nOrder 1234.", - "questions": [ - {"type": "predicate", "name": "0", "instructions": "Does the customer report a damaged item?"}, + "input": [ { - "type": "choice", - "name": "1", - "instructions": "Should we refund?", - "choices": [{"value": "true", "description": "Refund now"}, {"value": "escalate"}], + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "The package arrived with a broken screen."}, + {"type": "input_text", "text": "I want a refund."}, + ], }, - { - "type": "score", - "name": "2", - "instructions": "How severe is the issue?", - "levels": [{"label": "minor"}, {"label": "major: Product unusable"}], - }, - {"type": "predicate", "name": "3", "instructions": "Is this fraud?"}, + {"type": "message", "role": "user", "content": "Order 1234."}, + image_message, ], + "questions": _OPENAI_FORMAT_REQUEST["questions"], + "safety_identifier": "end-user-1", } - assert response.json()["answers"] == _OPENAI_FORMAT_ANSWERS + body: Final = response.json() + assert body["answers"] == _OPENAI_FORMAT_ANSWERS + assert body["usage"] == upstream_usage luna_cost: Final = litellm.model_cost["gpt-6-luna"] - expected_cost: Final = _INPUT_TOKENS * float(luna_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( - luna_cost["output_cost_per_token"] + expected_cost: Final = ( + (_INPUT_TOKENS - cached_tokens) * float(luna_cost["input_cost_per_token"]) + + cached_tokens * float(luna_cost["cache_read_input_token_cost"]) + + _OUTPUT_TOKENS * float(luna_cost["output_cost_per_token"]) ) assert expected_cost > 0 assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 44f198386c5..a934f57f3d3 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1002,6 +1002,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "/v1/videos", "/vertex_ai/live", "/v1/listen", + "/v1/systemone", + "/v1/decisions", "/v1beta/interactions", ], }, From df8f46c4eaa1b48ddb2515f95fa5d1dbcad2d432 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:52:49 -0700 Subject: [PATCH 10/17] test(decisions): read _hidden_params in the OpenAI provider test on stable/1.104.x The public hidden_params accessor comes from #44668, which is not on stable/1.104.x, so the backported test reads the private attribute like the rest of the file --- tests/unit/decisions/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 7b269148448..e6600839002 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -659,7 +659,7 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_ ], } assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} - assert response.hidden_params["custom_llm_provider"] == "openai" + assert response._hidden_params["custom_llm_provider"] == "openai" luna_cost: Final = litellm.model_cost["gpt-6-luna"] expected_cost: Final = ( (_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * float(luna_cost["input_cost_per_token"]) From 29765950630f8680a6d7be16009d8542b38c8cc8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:04:39 -0700 Subject: [PATCH 11/17] fix(decisions): keep provider fields inside System One answers Backport of #45214 to stable/1.104.x. Cherry-picked from 1348de864f (litellm_openai_decisions_provider). (cherry picked from commit 1348de864fed181306fc1d67a5af3d3aa37aa8f0) --- .../llms/base_llm/decisions/transformation.py | 36 ++++++++++++------- litellm/types/decisions.py | 3 ++ .../test_base_decisions_transformation.py | 4 ++- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 23a00d60ba0..ab827ae5dcf 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -183,6 +183,7 @@ def _ir_choice_answer(question: DecisionsIRChoiceQuestion, answer: ChoiceAnswer) DecisionsIRChoiceProbability(value=typed_values.get(key, key), probability=probability) for key, probability in answer.probabilities.items() ), + extra=MappingProxyType(answer.model_extra or {}), ) @@ -198,13 +199,14 @@ def _ir_score_answer(question: DecisionsIRScoreQuestion, answer: ScoreAnswer) -> ) for index, level in enumerate(question.levels) ), + extra=MappingProxyType(answer.model_extra or {}), ) def _ir_answer(question: DecisionsIRQuestion, answer: DecisionAnswer | None) -> DecisionsIRAnswer: match question, answer: case DecisionsIRPredicateQuestion(), NoulAnswer(): - return DecisionsIRPredicateAnswer(probability=answer.noul) + return DecisionsIRPredicateAnswer(probability=answer.noul, extra=MappingProxyType(answer.model_extra or {})) case DecisionsIRChoiceQuestion(), ChoiceAnswer(): return _ir_choice_answer(question, answer) case DecisionsIRScoreQuestion(), ScoreAnswer(): @@ -237,21 +239,29 @@ def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> De def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None: match answer: case DecisionsIRPredicateAnswer(): - return NoulAnswer(type="noul", noul=answer.probability) + return NoulAnswer.model_validate({**answer.extra, "type": "noul", "noul": answer.probability}) case DecisionsIRChoiceAnswer(): - return ChoiceAnswer( - type="choice", - choice=systemone_choice_key(answer.choice), - confidence=answer.confidence, - probabilities={systemone_choice_key(item.value): item.probability for item in answer.probabilities}, + return ChoiceAnswer.model_validate( + { + **answer.extra, + "type": "choice", + "choice": systemone_choice_key(answer.choice), + "confidence": answer.confidence, + "probabilities": { + systemone_choice_key(item.value): item.probability for item in answer.probabilities + }, + } ) case DecisionsIRScoreAnswer(): - return ScoreAnswer( - type="score", - score=answer.score, - confidence=answer.confidence, - legend={str(item.value): item.label for item in answer.probabilities}, - probabilities={str(item.value): item.probability for item in answer.probabilities}, + return ScoreAnswer.model_validate( + { + **answer.extra, + "type": "score", + "score": answer.score, + "confidence": answer.confidence, + "legend": {str(item.value): item.label for item in answer.probabilities}, + "probabilities": {str(item.value): item.probability for item in answer.probabilities}, + } ) case DecisionsIRRefusal(): return None diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index ba3c93a7d41..5fbbd905123 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -378,6 +378,7 @@ class DecisionsIRRequest: @dataclass(frozen=True, slots=True) class DecisionsIRPredicateAnswer: probability: float + extra: Mapping[str, object] = field(default=_NO_EXTRA) @dataclass(frozen=True, slots=True) @@ -391,6 +392,7 @@ class DecisionsIRChoiceAnswer: choice: str | bool confidence: float probabilities: tuple[DecisionsIRChoiceProbability, ...] + extra: Mapping[str, object] = field(default=_NO_EXTRA) @dataclass(frozen=True, slots=True) @@ -405,6 +407,7 @@ class DecisionsIRScoreAnswer: score: float confidence: float probabilities: tuple[DecisionsIRScoreProbability, ...] + extra: Mapping[str, object] = field(default=_NO_EXTRA) @dataclass(frozen=True, slots=True) diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index 9c4d8a27260..a452a621d2e 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -136,12 +136,13 @@ def test_a_systemone_response_reaches_the_caller_unchanged_with_provider_extras( payload: Final = { "model": "jev-latest", "answers": { - "damaged": {"type": "noul", "noul": 0.95}, + "damaged": {"type": "noul", "noul": 0.95, "rationale": "crack visible"}, "action": { "type": "choice", "choice": "refund", "confidence": 0.8, "probabilities": {"refund": 0.9, "escalate": 0.1}, + "calibrated": True, }, "severity": { "type": "score", @@ -149,6 +150,7 @@ def test_a_systemone_response_reaches_the_caller_unchanged_with_provider_extras( "confidence": 0.6, "legend": {"0": "minor", "1": {"label": "major"}}, "probabilities": {"0": 0.6, "1": 0.4}, + "raw_logits": [0.1, 0.2], }, }, "usage": {"input_tokens": 383, "output_tokens": 2, "cost": 0.25}, From 36894a174329715fe5104b1e57b53fa5a444786c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:16:36 -0700 Subject: [PATCH 12/17] fix(decisions): give IR extra fields a default_factory so Python 3.11 imports litellm Python 3.11 rejects an unhashable MappingProxyType as a dataclass default, so import litellm raised ValueError there. The same change needs to land on #45214. --- litellm/types/decisions.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 5fbbd905123..c0b948b4f28 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -316,7 +316,8 @@ class OpenAIDecisionResponse(_HiddenParamsResponse): model_config = ConfigDict(frozen=True) -_NO_EXTRA: Final[Mapping[str, object]] = MappingProxyType({}) +def _no_extra() -> Mapping[str, object]: + return MappingProxyType({}) @dataclass(frozen=True, slots=True) @@ -334,7 +335,7 @@ class DecisionsIRPredicateQuestion: name: str | None instructions: DecisionsJSON | None criteria: NoulCriteria | None = None - extra: Mapping[str, object] = field(default=_NO_EXTRA) + extra: Mapping[str, object] = field(default_factory=_no_extra) @dataclass(frozen=True, slots=True) @@ -348,7 +349,7 @@ class DecisionsIRChoiceQuestion: name: str | None instructions: DecisionsJSON | None choices: tuple[DecisionsIRChoiceOption, ...] - extra: Mapping[str, object] = field(default=_NO_EXTRA) + extra: Mapping[str, object] = field(default_factory=_no_extra) @dataclass(frozen=True, slots=True) @@ -362,7 +363,7 @@ class DecisionsIRScoreQuestion: name: str | None instructions: DecisionsJSON | None levels: tuple[DecisionsIRScoreLevel, ...] - extra: Mapping[str, object] = field(default=_NO_EXTRA) + extra: Mapping[str, object] = field(default_factory=_no_extra) DecisionsIRQuestion: TypeAlias = DecisionsIRPredicateQuestion | DecisionsIRChoiceQuestion | DecisionsIRScoreQuestion @@ -378,7 +379,7 @@ class DecisionsIRRequest: @dataclass(frozen=True, slots=True) class DecisionsIRPredicateAnswer: probability: float - extra: Mapping[str, object] = field(default=_NO_EXTRA) + extra: Mapping[str, object] = field(default_factory=_no_extra) @dataclass(frozen=True, slots=True) @@ -392,7 +393,7 @@ class DecisionsIRChoiceAnswer: choice: str | bool confidence: float probabilities: tuple[DecisionsIRChoiceProbability, ...] - extra: Mapping[str, object] = field(default=_NO_EXTRA) + extra: Mapping[str, object] = field(default_factory=_no_extra) @dataclass(frozen=True, slots=True) @@ -407,7 +408,7 @@ class DecisionsIRScoreAnswer: score: float confidence: float probabilities: tuple[DecisionsIRScoreProbability, ...] - extra: Mapping[str, object] = field(default=_NO_EXTRA) + extra: Mapping[str, object] = field(default_factory=_no_extra) @dataclass(frozen=True, slots=True) @@ -427,7 +428,7 @@ class DecisionsIRUsage: cached_tokens: int = 0 cache_write_tokens: int = 0 reasoning_tokens: int = 0 - extra: Mapping[str, object] = field(default=_NO_EXTRA) + extra: Mapping[str, object] = field(default_factory=_no_extra) @dataclass(frozen=True, slots=True) @@ -435,7 +436,7 @@ class DecisionsIRResponse: model: str | None answers: tuple[DecisionsIRAnswer, ...] usage: DecisionsIRUsage - extra: Mapping[str, object] = field(default=_NO_EXTRA) + extra: Mapping[str, object] = field(default_factory=_no_extra) @dataclass(frozen=True, slots=True) From 7df9c0620f18d926a8ef4deef03f1c9ed9439710 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:36:04 -0700 Subject: [PATCH 13/17] fix(decisions): load on Python 3.10 and 3.11 and bill cached tokens once under custom pricing Backport of 39012f1c22 from #45214 to stable/1.104.x. The IR extra default now mirrors #45214 (shared _NO_EXTRA via default_factory). --- litellm/cost_calculator.py | 51 ++++++++++++++++--------------- litellm/types/decisions.py | 19 ++++++------ tests/unit/decisions/test_main.py | 40 ++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 34 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 130fe939e33..33c41b01419 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast from httpx import Response from pydantic import BaseModel -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict, assert_never import litellm import litellm._logging @@ -982,6 +982,8 @@ def get_usage_object( ), ) + if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)): + return None if completion_response.usage is None else _decisions_usage(completion_response.usage) if usage_obj is None: return None if isinstance(usage_obj, Usage): @@ -1012,15 +1014,28 @@ def get_usage_object( return None -def _decisions_usage( - *, input_tokens: int, output_tokens: int, cached_tokens: int, cache_write_tokens: int -) -> dict[str, object]: - return { - "prompt_tokens": input_tokens, - "completion_tokens": output_tokens, - "cache_read_input_tokens": cached_tokens, - "cache_creation_input_tokens": cache_write_tokens, - } +def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper: + match usage: + case DecisionsUsage(): + return PromptTokensDetailsWrapper( + cached_tokens=usage.cached_tokens, cache_write_tokens=usage.cache_write_tokens + ) + case OpenAIDecisionUsage(): + return PromptTokensDetailsWrapper( + cached_tokens=usage.input_tokens_details.cached_tokens, + cache_write_tokens=usage.input_tokens_details.cache_write_tokens, + ) + case _: + assert_never(usage) + + +def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage: + return Usage( + prompt_tokens=usage.input_tokens, + completion_tokens=usage.output_tokens, + total_tokens=usage.input_tokens + usage.output_tokens, + prompt_tokens_details=_decisions_prompt_tokens_details(usage), + ) def _is_known_usage_objects(usage_obj): @@ -1447,20 +1462,8 @@ def completion_cost( "usage", litellm.Usage(**_usage_for_dump.model_dump()), ) - if isinstance(usage_obj, DecisionsUsage): - _usage = _decisions_usage( - input_tokens=usage_obj.input_tokens, - output_tokens=usage_obj.output_tokens, - cached_tokens=usage_obj.cached_tokens, - cache_write_tokens=usage_obj.cache_write_tokens, - ) - elif isinstance(usage_obj, OpenAIDecisionUsage): - _usage = _decisions_usage( - input_tokens=usage_obj.input_tokens, - output_tokens=usage_obj.output_tokens, - cached_tokens=usage_obj.input_tokens_details.cached_tokens, - cache_write_tokens=usage_obj.input_tokens_details.cache_write_tokens, - ) + if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)): + _usage = _decisions_usage(usage_obj).model_dump() elif usage_obj is None: _usage = {} elif isinstance(usage_obj, BaseModel): diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index c0b948b4f28..c81f1d76a3a 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -316,8 +316,7 @@ class OpenAIDecisionResponse(_HiddenParamsResponse): model_config = ConfigDict(frozen=True) -def _no_extra() -> Mapping[str, object]: - return MappingProxyType({}) +_NO_EXTRA: Final[Mapping[str, object]] = MappingProxyType({}) @dataclass(frozen=True, slots=True) @@ -335,7 +334,7 @@ class DecisionsIRPredicateQuestion: name: str | None instructions: DecisionsJSON | None criteria: NoulCriteria | None = None - extra: Mapping[str, object] = field(default_factory=_no_extra) + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) @dataclass(frozen=True, slots=True) @@ -349,7 +348,7 @@ class DecisionsIRChoiceQuestion: name: str | None instructions: DecisionsJSON | None choices: tuple[DecisionsIRChoiceOption, ...] - extra: Mapping[str, object] = field(default_factory=_no_extra) + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) @dataclass(frozen=True, slots=True) @@ -363,7 +362,7 @@ class DecisionsIRScoreQuestion: name: str | None instructions: DecisionsJSON | None levels: tuple[DecisionsIRScoreLevel, ...] - extra: Mapping[str, object] = field(default_factory=_no_extra) + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) DecisionsIRQuestion: TypeAlias = DecisionsIRPredicateQuestion | DecisionsIRChoiceQuestion | DecisionsIRScoreQuestion @@ -379,7 +378,7 @@ class DecisionsIRRequest: @dataclass(frozen=True, slots=True) class DecisionsIRPredicateAnswer: probability: float - extra: Mapping[str, object] = field(default_factory=_no_extra) + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) @dataclass(frozen=True, slots=True) @@ -393,7 +392,7 @@ class DecisionsIRChoiceAnswer: choice: str | bool confidence: float probabilities: tuple[DecisionsIRChoiceProbability, ...] - extra: Mapping[str, object] = field(default_factory=_no_extra) + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) @dataclass(frozen=True, slots=True) @@ -408,7 +407,7 @@ class DecisionsIRScoreAnswer: score: float confidence: float probabilities: tuple[DecisionsIRScoreProbability, ...] - extra: Mapping[str, object] = field(default_factory=_no_extra) + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) @dataclass(frozen=True, slots=True) @@ -428,7 +427,7 @@ class DecisionsIRUsage: cached_tokens: int = 0 cache_write_tokens: int = 0 reasoning_tokens: int = 0 - extra: Mapping[str, object] = field(default_factory=_no_extra) + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) @dataclass(frozen=True, slots=True) @@ -436,7 +435,7 @@ class DecisionsIRResponse: model: str | None answers: tuple[DecisionsIRAnswer, ...] usage: DecisionsIRUsage - extra: Mapping[str, object] = field(default_factory=_no_extra) + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) @dataclass(frozen=True, slots=True) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index e6600839002..faefbe61e3c 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -285,6 +285,46 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None: assert cost == pytest.approx(expected_cost) +@pytest.mark.parametrize( + "response", + ( + DecisionsResponse( + model="jev-latest", + answers={}, + usage=DecisionsUsage( + input_tokens=_INPUT_TOKENS, + output_tokens=_OUTPUT_TOKENS, + cached_tokens=_CACHED_TOKENS, + cache_write_tokens=_CACHE_WRITE_TOKENS, + ), + ), + OpenAIDecisionResponse.model_validate(_OPENAI_RESPONSE), + ), + ids=("systemone", "openai"), +) +def test_custom_token_pricing_bills_cached_decisions_input_tokens_once( + response: DecisionsResponse | OpenAIDecisionResponse, +) -> None: + cost: Final = litellm.completion_cost( + completion_response=response, + model="gpt-6-luna", + custom_llm_provider="openai", + custom_cost_per_token={ + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "cache_read_input_token_cost": 0.1, + "cache_creation_input_token_cost": 1.25, + }, + ) + + assert cost == pytest.approx( + (_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * 1.0 + + _CACHED_TOKENS * 0.1 + + _CACHE_WRITE_TOKENS * 1.25 + + _OUTPUT_TOKENS * 2.0 + ) + + @pytest.mark.asyncio async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.MockRouter) -> None: respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) From c65d369958699838d95104e1bcdb7d9c4ea56332 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:49:50 -0700 Subject: [PATCH 14/17] fix(decisions): honor litellm OpenAI key and base settings for the openai provider Backport of #45214 to stable/1.104.x. Cherry-picked from e465919004 (litellm_openai_decisions_provider). --- litellm/decisions/main.py | 28 ++++------ .../llms/base_llm/decisions/transformation.py | 18 +++---- .../cloudflare/decisions/transformation.py | 10 ++-- .../llms/openai/decisions/transformation.py | 16 ++++-- tests/unit/decisions/test_main.py | 53 +++++++++++++++++++ 5 files changed, 91 insertions(+), 34 deletions(-) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 1b973adae72..01fc149da3c 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -25,7 +25,6 @@ from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISION from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_ENDPOINT -from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( DecisionQuestion, DecisionsIRRequest, @@ -103,21 +102,16 @@ def _resolve_api_key( ) -> str | None: if api_key is not None: return api_key - - server_api_key: Final = next( - (key for key in (get_secret_str(name) for name in endpoint.api_key_env) if key), - None, + configured_api_key: Final = endpoint.configured_api_key() + if configured_api_key: + return configured_api_key + if not endpoint.api_key_required: + return None + raise litellm.AuthenticationError( + message=f"Missing API key for Decisions provider '{provider}'", + model=model, + llm_provider=provider, ) - if server_api_key is None: - if not endpoint.api_key_required: - return None - raise litellm.AuthenticationError( - message=f"Missing API key for Decisions provider '{provider}'", - model=model, - llm_provider=provider, - ) - - return server_api_key def _validate_request( @@ -175,9 +169,7 @@ def _prepare_request( ) from error endpoint: Final = DECISIONS_ENDPOINTS[provider] - env_api_base: Final = get_secret_str(endpoint.api_base_env) - default_api_base: Final = endpoint.default_api_base() - resolved_api_base: Final = api_base or env_api_base or default_api_base + resolved_api_base: Final = api_base or endpoint.configured_api_base() if resolved_api_base is None: raise litellm.BadRequestError( message=endpoint.missing_api_base_message(provider), diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index ab827ae5dcf..14a955053da 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -7,6 +7,7 @@ from typing import Final, Protocol from pydantic import TypeAdapter from typing_extensions import assert_never +from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( ChoiceAnswer, ChoiceQuestion, @@ -299,8 +300,11 @@ class JevCompatibleDecisionsEndpoint: api_base_env: str api_key_required: bool = True - def default_api_base(self) -> str | None: - return self.default_api_base_value + def configured_api_key(self) -> str | None: + return next((key for key in (get_secret_str(name) for name in self.api_key_env) if key), None) + + def configured_api_base(self) -> str | None: + return get_secret_str(self.api_base_env) or self.default_api_base_value def missing_api_base_message(self, provider: str) -> str: return f"api_base is required for Decisions provider '{provider}'" @@ -324,16 +328,12 @@ class JevCompatibleDecisionsEndpoint: class DecisionsProviderConfig(Protocol): - @property - def api_key_env(self) -> tuple[str, ...]: ... - - @property - def api_base_env(self) -> str: ... - @property def api_key_required(self) -> bool: ... - def default_api_base(self) -> str | None: ... + def configured_api_key(self) -> str | None: ... + + def configured_api_base(self) -> str | None: ... def missing_api_base_message(self, provider: str) -> str: ... diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index a325689b0c9..83805e9d931 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -16,11 +16,15 @@ _RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapte @dataclass(frozen=True, slots=True) class CloudflareDecisionsEndpoint: - api_key_env: tuple[str, ...] = ("CLOUDFLARE_API_KEY",) - api_base_env: str = "CLOUDFLARE_API_BASE" api_key_required: bool = True - def default_api_base(self) -> str | None: + def configured_api_key(self) -> str | None: + return get_secret_str("CLOUDFLARE_API_KEY") or None + + def configured_api_base(self) -> str | None: + env_api_base: Final = get_secret_str("CLOUDFLARE_API_BASE") + if env_api_base: + return env_api_base account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID")) if account_id is None: return None diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index b7f5a95732c..fcae4bcc49c 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -5,7 +5,9 @@ from typing import Final from pydantic import TypeAdapter from typing_extensions import assert_never +import litellm from litellm.llms.base_llm.decisions.transformation import decisions_text +from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( DecisionsIRAnswer, DecisionsIRChoiceAnswer, @@ -271,12 +273,18 @@ def ir_to_openai_response( @dataclass(frozen=True, slots=True) class OpenAIDecisionsEndpoint: - api_key_env: tuple[str, ...] = ("OPENAI_API_KEY",) - api_base_env: str = "OPENAI_BASE_URL" api_key_required: bool = True - def default_api_base(self) -> str | None: - return "https://api.openai.com" + def configured_api_key(self) -> str | None: + return litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + + def configured_api_base(self) -> str | None: + return ( + litellm.api_base + or get_secret_str("OPENAI_BASE_URL") + or get_secret_str("OPENAI_API_BASE") + or "https://api.openai.com" + ) def missing_api_base_message(self, provider: str) -> str: return f"api_base is required for Decisions provider '{provider}'" diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index faefbe61e3c..cbc8f7f6d01 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -664,7 +664,9 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_ monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, ) -> None: + monkeypatch.setattr(litellm, "api_base", None) monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE) response: Final = await litellm.adecisions( @@ -711,6 +713,57 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_ assert litellm.completion_cost(completion_response=response) == pytest.approx(expected_cost) +@pytest.mark.parametrize( + ("settings", "env", "url", "authorization"), + ( + ({"openai_key": "sdk-key"}, {}, "https://api.openai.com/v1/decisions", "Bearer sdk-key"), + ( + {"api_key": "global-key", "openai_key": "sdk-key"}, + {"OPENAI_API_KEY": "env-key"}, + "https://api.openai.com/v1/decisions", + "Bearer global-key", + ), + ( + {}, + {"OPENAI_API_KEY": "env-key", "OPENAI_API_BASE": "https://legacy.example/v1"}, + "https://legacy.example/v1/decisions", + "Bearer env-key", + ), + ( + {"api_base": "https://sdk.example"}, + {"OPENAI_API_KEY": "env-key", "OPENAI_BASE_URL": "https://env.example"}, + "https://sdk.example/v1/decisions", + "Bearer env-key", + ), + ), + ids=("openai_key", "api_key_before_env", "openai_api_base_env", "api_base_before_env"), +) +def test_openai_decisions_use_the_same_settings_as_other_openai_calls( + settings: Mapping[str, str], + env: Mapping[str, str], + url: str, + authorization: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + for name in ("api_key", "openai_key", "api_base"): + monkeypatch.setattr(litellm, name, settings.get(name)) + for name in ("OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_API_BASE"): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE) + + litellm.decisions( + model="openai/gpt-6-luna", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == authorization + + @pytest.mark.asyncio async def test_openai_format_calls_to_a_systemone_provider_get_openai_format_answers( respx_mock: respx.MockRouter, From 5d2cc109c842e6ebfe490846b94f652a11995bce Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:01:16 -0700 Subject: [PATCH 15/17] test(decisions): clear every OpenAI base setting in the proxy OpenAI deployment test Backport of #45214 to stable/1.104.x. Cherry-picked from a2951b318d (litellm_openai_decisions_provider). --- tests/unit/proxy/decisions_endpoints/test_endpoints.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 295a10a3d9b..812f4556b85 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -445,7 +445,9 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_ monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, ) -> None: + monkeypatch.setattr(litellm, "api_base", None) monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) monkeypatch.setattr( litellm.proxy.proxy_server, "llm_router", From f0cb1d57572958cbd1f5f95167e28fe2f2cb642c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:09:37 -0700 Subject: [PATCH 16/17] fix(decisions): send OpenAI instructions for System One questions without them and reject single-option questions --- .../llms/openai/decisions/transformation.py | 32 +++++++++++-- .../test_openai_decisions_transformation.py | 48 ++++++++++++++++++- 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index fcae4bcc49c..003fccb7a8d 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -45,10 +45,14 @@ from litellm.types.decisions import ( OpenAIScoreAnswer, OpenAIScoreProbability, OpenAIScoreQuestion, + UnsupportedDecisionsRequest, systemone_choice_key, ) _OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) +_SINGLE_OPTION: Final = UnsupportedDecisionsRequest( + reason="OpenAI needs at least 2 choices or levels on every choice or score question" +) def _ir_input(decision_input: OpenAIDecisionInput) -> DecisionsIRState | DecisionsIRMessages: @@ -94,6 +98,10 @@ def _text_field(key: str, value: DecisionsJSON | None) -> Mapping[str, str]: return {} if value is None else {key: decisions_text(value)} +def _instructions(value: DecisionsJSON | None, default: str) -> str: + return default if value is None else decisions_text(value) + + def _predicate_instructions(question: DecisionsIRPredicateQuestion) -> str: instructions: Final = () if question.instructions is None else (decisions_text(question.instructions),) criteria: Final = tuple( @@ -110,13 +118,13 @@ def _openai_question(question: DecisionsIRQuestion) -> Mapping[str, object]: return { "type": "predicate", **_text_field("name", question.name), - "instructions": _predicate_instructions(question), + "instructions": _predicate_instructions(question) or "Is this true of the input?", } case DecisionsIRChoiceQuestion(): return { "type": "choice", **_text_field("name", question.name), - **_text_field("instructions", question.instructions), + "instructions": _instructions(question.instructions, "Which choice best fits the input?"), "choices": [ {"value": option.value, **_text_field("description", option.description)} for option in question.choices @@ -126,7 +134,7 @@ def _openai_question(question: DecisionsIRQuestion) -> Mapping[str, object]: return { "type": "score", **_text_field("name", question.name), - **_text_field("instructions", question.instructions), + "instructions": _instructions(question.instructions, "Which level best fits the input?"), "levels": [ {"label": decisions_text(level.label), **_text_field("description", level.description)} for level in question.levels @@ -146,7 +154,19 @@ def _openai_input(decision_input: DecisionsIRState | DecisionsIRMessages) -> str assert_never(decision_input) -def ir_to_openai_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]: +def _has_one_option(question: DecisionsIRQuestion) -> bool: + match question: + case DecisionsIRChoiceQuestion(): + return len(question.choices) < 2 + case DecisionsIRScoreQuestion(): + return len(question.levels) < 2 + case _: + return False + + +def ir_to_openai_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if any(_has_one_option(question) for question in request.questions): + return _SINGLE_OPTION return { "model": model, "input": _openai_input(request.input), @@ -298,7 +318,9 @@ class OpenAIDecisionsEndpoint: def endpoint_url(self, api_base: str, model: str) -> str: return f"{api_base.rstrip('/').removesuffix('/v1')}/v1/decisions" - def request_body(self, model: str, request: DecisionsIRRequest) -> Mapping[str, object]: + def request_body( + self, model: str, request: DecisionsIRRequest + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: return ir_to_openai_request(model, request) def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index 942fad1db6d..249a1a7640a 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -1,6 +1,7 @@ from collections.abc import Mapping from typing import Final +import pytest from pydantic import TypeAdapter from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_response, systemone_request_to_ir @@ -10,7 +11,12 @@ from litellm.llms.openai.decisions.transformation import ( ir_to_openai_response, openai_request_to_ir, ) -from litellm.types.decisions import DecisionsIRRequest, DecisionsRequestBody, OpenAIDecisionRequestBody +from litellm.types.decisions import ( + DecisionsIRRequest, + DecisionsRequestBody, + OpenAIDecisionRequestBody, + UnsupportedDecisionsRequest, +) _SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) _OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) @@ -148,11 +154,49 @@ def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_t "instructions": '{"policy": "refund-v2"}', "choices": [{"value": "refund", "description": "Within 30 days"}, {"value": "escalate"}], }, - {"type": "score", "name": "severity", "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}]}, + { + "type": "score", + "name": "severity", + "instructions": "Which level best fits the input?", + "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}], + }, ], } +def test_systemone_questions_without_instructions_become_valid_openai_questions() -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": "Screen cracked", + "questions": { + "damaged": {"type": "noul", "criteria": {"false": None}}, + "action": {"type": "choice", "criteria": {"refund": None, "escalate": None}}, + "severity": {"type": "score", "criteria": ["minor", "major"]}, + }, + } + ) + + body: Final = ir_to_openai_request("gpt-6-luna", systemone_request_to_ir(request)) + + assert all(question.instructions for question in _OPENAI_BODY.validate_python(body).questions) + + +@pytest.mark.parametrize( + "question", + ({"type": "choice", "criteria": {"refund": None}}, {"type": "score", "criteria": ["minor"]}), + ids=("choice", "score"), +) +def test_a_systemone_question_with_one_option_is_unsupported_by_openai(question: Mapping[str, object]) -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": "Screen cracked", + "questions": {"damaged": {"type": "noul", "instructions": "Damaged?"}, "q": question}, + } + ) + + assert isinstance(ir_to_openai_request("gpt-6-luna", systemone_request_to_ir(request)), UnsupportedDecisionsRequest) + + def test_an_openai_response_becomes_systemone_answers_with_the_callers_score_labels() -> None: ir: Final = systemone_request_to_ir( _SYSTEMONE_BODY.validate_python( From 7446e026907de8c0126008f43cdcefde93ae0fdc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:20:56 -0700 Subject: [PATCH 17/17] fix(decisions): return guardrail_information on OpenAI-format decisions when requested Backport of the #45184 and #45214 guardrail fix. Cherry-picked from 855f0bb765 (litellm_systemone_endpoint) as merged into #45214 at c19ada66ed (litellm_openai_decisions_provider). --- litellm/types/decisions.py | 2 +- .../decisions_endpoints/test_endpoints.py | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index c81f1d76a3a..6488ef45308 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -313,7 +313,7 @@ class OpenAIDecisionResponse(_HiddenParamsResponse): answers: tuple[OpenAIDecisionAnswer, ...] usage: OpenAIDecisionUsage - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(extra="allow", frozen=True) _NO_EXTRA: Final[Mapping[str, object]] = MappingProxyType({}) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 812f4556b85..7d965af56b1 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -386,6 +386,29 @@ def test_openai_format_decisions_translate_through_systemone( assert float(response.headers["x-litellm-response-cost"]) > 0 +@pytest.mark.parametrize( + ("endpoint", "request_body", "upstream_response"), + ( + ("/v1/systemone", _REQUEST, _RESPONSE), + ("/v1/decisions", _OPENAI_FORMAT_REQUEST, _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST), + ), + ids=("systemone", "openai_format"), +) +def test_decisions_return_guardrail_information_when_requested( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, + request_body: Mapping[str, object], + upstream_response: Mapping[str, object], +) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=upstream_response) + + response: Final = client.post(endpoint, json={**request_body, "include_guardrail_response": True}) + + assert response.status_code == 200, response.text + assert response.json()["guardrail_information"] == [] + + @pytest.mark.parametrize( "request_body", (