feat(decisions): add unified /v1/decisions endpoint for Jev-compatible providers (#44236)

Backport of #44236 to stable/1.104.x.
Cherry-picked from 8b1990b4bc (main). Adapted to this line: the lazy feature
helpers are ported by hand, managed agent authorization and the integration suites are not on
this line, and the lazy-disabled precedence test is dropped since attach_lazy_features takes no
feature list here.
This commit is contained in:
devin-ai-integration[bot] 2026-10-07 16:00:08 -07:00 • committed by mateo-berri
parent 48963be81b
commit bc4b7fd42c
40 changed files with 2029 additions and 28 deletions

View file

@ -61,7 +61,7 @@ jobs:
- shard: core-utils
artifact-name: core-utils
test-path: ""
test-path: tests/unit/decisions
unit-flag: core-utils
workers: 2
reruns: 1
@ -130,6 +130,7 @@ jobs:
artifact-name: proxy-endpoints
test-path: >-
tests/test_litellm/proxy/analytics_endpoints
tests/unit/proxy/decisions_endpoints
tests/test_litellm/proxy/management_endpoints
tests/test_litellm/proxy/list_api
tests/test_litellm/proxy/memory

View file

@ -364,11 +364,13 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
| [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | |
| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | |
| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | |
| [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | |
| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | | | | | | | |
| [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ | | | | | | | |
| [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ | | | | | | | |
| [Typesafe Decisions API (`typesafe`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | |
| [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ | | | | | | | |
| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |

View file

@ -1,7 +1,7 @@
"""Path allowlist for the gateway component.
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image,
responses, vector stores, passthrough providers, realtime websockets, MCP
tool-call endpoints, and operational endpoints (/health, /metrics, and the
/debug/memory/summary read of the serving worker's RSS).
@ -60,6 +60,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/v1/rerank",
"/v2/rerank",
"/rerank",
"/v1/decisions",
"/decisions",
"/v1/ocr",
"/ocr",
"/v1/rag/",

View file

@ -1467,6 +1467,7 @@ from .embeddings.dispatch import *
from .rust_bridge import rust
from .rag.main import *
from .sandbox.main import *
from .decisions.main import *
from .search.main import *
from .realtime_api.main import (
_arealtime,

View file

@ -98,6 +98,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro
from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.decisions import DecisionsResponse, DecisionsUsage
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
@ -1016,6 +1017,7 @@ def _is_known_usage_objects(usage_obj):
return (
isinstance(usage_obj, litellm.Usage)
or isinstance(usage_obj, ResponseAPIUsage)
or isinstance(usage_obj, DecisionsUsage)
or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj)
)
@ -1433,7 +1435,12 @@ def completion_cost(
"usage",
litellm.Usage(**_usage_for_dump.model_dump()),
)
if usage_obj is None:
if isinstance(usage_obj, DecisionsUsage):
_usage = {
"prompt_tokens": usage_obj.input_tokens,
"completion_tokens": usage_obj.output_tokens,
}
elif usage_obj is None:
_usage = {}
elif isinstance(usage_obj, BaseModel):
_usage = cast(BaseModel, usage_obj).model_dump()
@ -1933,7 +1940,8 @@ def response_cost_calculator(
| LiteLLMRealtimeStreamLoggingObject
| OpenAIModerationResponse
| Response
| SearchResponse,
| SearchResponse
| DecisionsResponse,
model: str,
custom_llm_provider: str | None,
call_type: Literal[
@ -1955,6 +1963,8 @@ def response_cost_calculator(
"arerank",
"search",
"asearch",
"decisions",
"adecisions",
],
optional_params: dict,
cache_hit: bool | None = None,

View file

@ -0,0 +1,3 @@
from litellm.decisions.main import adecisions, decisions
__all__ = ["adecisions", "decisions"]

299
litellm/decisions/main.py Normal file
View file

@ -0,0 +1,299 @@
from collections.abc import Mapping
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig
from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT
from litellm.llms.custom_httpx.http_handler import _get_httpx_client, get_async_httpx_client
from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT
from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT
from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT
from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_ENDPOINT
from litellm.secret_managers.main import get_secret_str
from litellm.types.decisions import (
DecisionQuestion,
DecisionsJSON,
DecisionsRequest,
DecisionsResponse,
)
from litellm.utils import client
DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxyType(
{
"perplexity": PERPLEXITY_DECISIONS_ENDPOINT,
"typesafe": TYPESAFE_DECISIONS_ENDPOINT,
"openrouter": OPENROUTER_DECISIONS_ENDPOINT,
"cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT,
"strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT,
}
)
_DECISIONS_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequest]] = TypeAdapter(DecisionsRequest)
_DECISIONS_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse)
@dataclass(frozen=True, slots=True, repr=False)
class _PreparedDecisionsRequest:
config: DecisionsProviderConfig
provider: str
upstream_model: str
url: str
api_key: str | None = field(repr=False)
headers: Mapping[str, str] = field(repr=False)
body: Mapping[str, object] = field(repr=False)
def _resolve_provider_model(model: str, custom_llm_provider: str | None) -> tuple[str, str]:
provider: Final = model.partition("/")[0] if custom_llm_provider is None else custom_llm_provider
if provider not in DECISIONS_ENDPOINTS:
supported: Final = ", ".join(DECISIONS_ENDPOINTS)
raise litellm.BadRequestError(
message=f"Unknown Decisions provider '{provider}'. Supported providers: {supported}",
model=model,
llm_provider=provider,
)
upstream_model: Final = model.removeprefix(f"{provider}/") if model.startswith(f"{provider}/") else model
if not upstream_model:
raise litellm.BadRequestError(
message="A model name is required for the Decisions API",
model=model,
llm_provider=provider,
)
return provider, upstream_model
def _resolve_api_key(
*,
provider: str,
model: str,
endpoint: DecisionsProviderConfig,
api_key: str | None,
) -> str | None:
if api_key is not None:
return api_key
server_api_key: Final = next(
(key for key in (get_secret_str(name) for name in endpoint.api_key_env) if key),
None,
)
if server_api_key is None:
if not endpoint.api_key_required:
return None
raise litellm.AuthenticationError(
message=f"Missing API key for Decisions provider '{provider}'",
model=model,
llm_provider=provider,
)
return server_api_key
def _prepare_request(
*,
model: str,
state: DecisionsJSON,
questions: Mapping[str, DecisionQuestion | Mapping[str, object]],
api_key: str | None,
api_base: str | None,
custom_llm_provider: str | None,
extra_headers: Mapping[str, str] | None,
) -> _PreparedDecisionsRequest:
provider, upstream_model = _resolve_provider_model(model, custom_llm_provider)
try:
validated_request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python(
{"model": model, "state": state, "questions": questions}
)
except ValidationError as error:
raise litellm.BadRequestError(
message=f"Invalid Decisions request: {error}",
model=model,
llm_provider=provider,
) from error
endpoint: Final = DECISIONS_ENDPOINTS[provider]
env_api_base: Final = get_secret_str(endpoint.api_base_env)
default_api_base: Final = endpoint.default_api_base()
resolved_api_base: Final = api_base or env_api_base or default_api_base
if resolved_api_base is None:
raise litellm.BadRequestError(
message=endpoint.missing_api_base_message(provider),
model=model,
llm_provider=provider,
)
resolved_api_key: Final = _resolve_api_key(
provider=provider,
model=model,
endpoint=endpoint,
api_key=api_key,
)
canonical_model: Final = endpoint.canonical_model(upstream_model)
outbound_headers: Final = MappingProxyType(
{
**{
name: value
for name, value in (extra_headers or {}).items()
if name.lower() not in {"authorization", "content-type"}
},
**({"Authorization": f"Bearer {resolved_api_key}"} if resolved_api_key is not None else {}),
"Content-Type": "application/json",
}
)
body: Final = MappingProxyType(
{
"model": endpoint.request_model(canonical_model),
"state": validated_request.state,
"questions": {
name: question.model_dump(mode="json", exclude_none=True)
for name, question in validated_request.questions.items()
},
}
)
return _PreparedDecisionsRequest(
config=endpoint,
provider=provider,
upstream_model=canonical_model,
url=endpoint.endpoint_url(resolved_api_base, canonical_model),
api_key=resolved_api_key,
headers=outbound_headers,
body=body,
)
def _log_request(
prepared: _PreparedDecisionsRequest,
kwargs: Mapping[str, object],
) -> LiteLLMLoggingObj | None:
logging_obj: Final = kwargs.get("litellm_logging_obj")
if not isinstance(logging_obj, LiteLLMLoggingObj):
return None
logging_obj.update_from_kwargs(
kwargs=dict(kwargs),
model=prepared.upstream_model,
litellm_params={
"litellm_call_id": kwargs.get("litellm_call_id"),
"api_base": prepared.url,
},
custom_llm_provider=prepared.provider,
)
request_body: Final = dict(prepared.body)
request_headers: Final = dict(prepared.headers)
logging_obj.pre_call(
input=request_body,
api_key=prepared.api_key,
model=prepared.upstream_model,
additional_args={
"api_base": prepared.url,
"complete_input_dict": request_body,
"headers": request_headers,
},
)
return logging_obj
def _parse_response(
response: httpx.Response,
prepared: _PreparedDecisionsRequest,
) -> DecisionsResponse:
response.raise_for_status()
payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content)
result: Final = _DECISIONS_RESPONSE_ADAPTER.validate_python(prepared.config.unwrap_response(payload))
result._hidden_params.update(
{
"model": f"{prepared.provider}/{prepared.upstream_model}",
"custom_llm_provider": prepared.provider,
"provider_response_model": f"{prepared.provider}/{prepared.upstream_model}",
}
)
return result
def _map_upstream_exception(error: Exception, prepared: _PreparedDecisionsRequest) -> Exception:
return litellm.exception_type(
model=f"{prepared.provider}/{prepared.upstream_model}",
custom_llm_provider=prepared.provider,
original_exception=error,
)
@client
async def adecisions(
model: str,
state: DecisionsJSON,
questions: Mapping[str, DecisionQuestion | Mapping[str, object]],
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: Mapping[str, str] | None = None,
**kwargs: object,
) -> DecisionsResponse:
prepared: Final = _prepare_request(
model=model,
state=state,
questions=questions,
api_key=api_key,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
)
logging_obj: Final = _log_request(prepared, kwargs)
try:
handler: Final = get_async_httpx_client(llm_provider=prepared.provider)
response: Final = await handler.post(
prepared.url,
json=dict(prepared.body),
headers=dict(prepared.headers),
timeout=timeout,
logging_obj=logging_obj,
)
return _parse_response(response=response, prepared=prepared)
except Exception as error:
raise _map_upstream_exception(error, prepared) from error
@client
def decisions(
model: str,
state: DecisionsJSON,
questions: Mapping[str, DecisionQuestion | Mapping[str, object]],
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: Mapping[str, str] | None = None,
**kwargs: object,
) -> DecisionsResponse:
prepared: Final = _prepare_request(
model=model,
state=state,
questions=questions,
api_key=api_key,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
)
logging_obj: Final = _log_request(prepared, kwargs)
try:
handler: Final = _get_httpx_client()
response: Final = handler.post(
prepared.url,
json=dict(prepared.body),
headers=dict(prepared.headers),
timeout=timeout,
logging_obj=logging_obj,
)
return _parse_response(response=response, prepared=prepared)
except Exception as error:
raise _map_upstream_exception(error, prepared) from error
__all__ = ["DECISIONS_ENDPOINTS", "adecisions", "decisions"]

View file

@ -170,6 +170,7 @@ class HealthCheckHelpers:
"batch",
"responses",
"ocr",
"evaluation",
],
Callable,
]:
@ -192,7 +193,7 @@ class HealthCheckHelpers:
from litellm.litellm_core_utils.audio_utils.utils import (
get_audio_file_for_health_check,
)
from litellm.litellm_core_utils.health_check_utils import _filter_model_params
from litellm.litellm_core_utils.health_check_utils import DECISIONS_CALL_PARAMS, _filter_model_params
from litellm.realtime_api.main import _realtime_health_check
return {
@ -259,4 +260,13 @@ class HealthCheckHelpers:
**_filter_model_params(model_params=model_params),
document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider),
),
"evaluation": lambda: litellm.adecisions(
**DECISIONS_CALL_PARAMS.validate_python(
{
"state": prompt or "health check",
"questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}},
**_filter_model_params(model_params=model_params),
}
)
),
}

View file

@ -4,6 +4,12 @@ Utils used for litellm.ahealth_check()
from typing import Final
from pydantic import TypeAdapter
from litellm.types.decisions import DecisionsCallParams
DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams)
def _filter_model_params(model_params: dict) -> dict:
"""Remove 'messages' param from model params."""

View file

@ -118,6 +118,7 @@ from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.containers.main import ContainerObject
from litellm.types.decisions import DecisionsResponse
from litellm.types.interactions import (
InteractionsAPIResponse,
InteractionsAPIStreamingResponse,
@ -1815,6 +1816,7 @@ class Logging(LiteLLMLoggingBaseClass):
LiteLLMRealtimeStreamLoggingObject,
OpenAIModerationResponse,
"SearchResponse",
DecisionsResponse,
dict,
list,
],
@ -2592,6 +2594,7 @@ class Logging(LiteLLMLoggingBaseClass):
or isinstance(logging_result, OpenAIModerationResponse)
or isinstance(logging_result, OCRResponse) # OCR
or isinstance(logging_result, SearchResponse) # Search API
or isinstance(logging_result, DecisionsResponse)
or (
isinstance(logging_result, InteractionsAPIResponse)
and logging_result.usage is not None

View file

@ -0,0 +1,3 @@
from .transformation import DecisionsProviderConfig, JevCompatibleDecisionsEndpoint
__all__ = ["DecisionsProviderConfig", "JevCompatibleDecisionsEndpoint"]

View file

@ -0,0 +1,52 @@
from dataclasses import dataclass
from typing import Protocol
@dataclass(frozen=True, slots=True)
class JevCompatibleDecisionsEndpoint:
default_api_base_value: str | None
path: str
api_key_env: tuple[str, ...]
api_base_env: str
api_key_required: bool = True
def default_api_base(self) -> str | None:
return self.default_api_base_value
def missing_api_base_message(self, provider: str) -> str:
return f"api_base is required for Decisions provider '{provider}'"
def canonical_model(self, model: str) -> str:
return model
def request_model(self, model: str) -> str:
return model
def endpoint_url(self, api_base: str, model: str) -> str:
return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}"
def unwrap_response(self, payload: object) -> object:
return payload
class DecisionsProviderConfig(Protocol):
@property
def api_key_env(self) -> tuple[str, ...]: ...
@property
def api_base_env(self) -> str: ...
@property
def api_key_required(self) -> bool: ...
def default_api_base(self) -> str | None: ...
def missing_api_base_message(self, provider: str) -> str: ...
def canonical_model(self, model: str) -> str: ...
def request_model(self, model: str) -> str: ...
def endpoint_url(self, api_base: str, model: str) -> str: ...
def unwrap_response(self, payload: object) -> object: ...

View file

@ -0,0 +1,58 @@
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final
from pydantic import TypeAdapter
from litellm.secret_managers.main import (
get_secret_str,
normalize_nonempty_secret_str,
)
_RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
@dataclass(frozen=True, slots=True)
class CloudflareDecisionsEndpoint:
api_key_env: tuple[str, ...] = ("CLOUDFLARE_API_KEY",)
api_base_env: str = "CLOUDFLARE_API_BASE"
api_key_required: bool = True
def default_api_base(self) -> str | None:
account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID"))
if account_id is None:
return None
return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run"
def missing_api_base_message(self, provider: str) -> str:
return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly"
def canonical_model(self, model: str) -> str:
if model.startswith("@cf/"):
return model
return f"@cf/cloudflare/{model}"
def request_model(self, model: str) -> str:
return model.rsplit("/", maxsplit=1)[-1]
def endpoint_url(self, api_base: str, model: str) -> str:
normalized_api_base: Final = api_base.rstrip("/")
if normalized_api_base.endswith("/ai/v1"):
return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}"
if normalized_api_base.endswith("/ai/run"):
return f"{normalized_api_base}/{model}"
return f"{normalized_api_base}/ai/run/{model}"
def unwrap_response(self, payload: object) -> object:
if not isinstance(payload, Mapping):
return payload
response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload)
if "answers" in response_mapping:
return payload
result: Final = response_mapping.get("result")
if isinstance(result, Mapping):
return result
return payload
CLOUDFLARE_DECISIONS_ENDPOINT: Final[CloudflareDecisionsEndpoint] = CloudflareDecisionsEndpoint()

View file

@ -0,0 +1,10 @@
from typing import Final
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
OPENROUTER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
default_api_base_value="https://openrouter.ai/api",
path="/alpha/decisions",
api_key_env=("OPENROUTER_API_KEY",),
api_base_env="OPENROUTER_API_BASE",
)

View file

@ -0,0 +1,10 @@
from typing import Final
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
PERPLEXITY_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
default_api_base_value="https://api.perplexity.ai",
path="/v1/decisions",
api_key_env=("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"),
api_base_env="PERPLEXITY_API_BASE",
)

View file

@ -0,0 +1,11 @@
from typing import Final
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
STRANDS_DECIDER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
default_api_base_value=None,
path="/v1/systemone",
api_key_env=("STRANDS_DECIDER_API_KEY",),
api_base_env="STRANDS_DECIDER_API_BASE",
api_key_required=False,
)

View file

@ -0,0 +1,10 @@
from typing import Final
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
TYPESAFE_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
default_api_base_value="https://api.typesafe.ai",
path="/v1/systemone",
api_key_env=("TYPESAFE_API_KEY",),
api_base_env="TYPESAFE_API_BASE",
)

View file

@ -77427,5 +77427,55 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_tool_choice": false
},
"cloudflare/clef": {
"input_cost_per_token": 2.4e-07,
"litellm_provider": "cloudflare",
"max_input_tokens": 65536,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
},
"cloudflare/clef-flash": {
"input_cost_per_token": 9e-08,
"litellm_provider": "cloudflare",
"max_input_tokens": 65536,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
},
"cloudflare/@cf/cloudflare/clef": {
"input_cost_per_token": 2.4e-07,
"litellm_provider": "cloudflare",
"max_input_tokens": 65536,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
},
"cloudflare/@cf/cloudflare/clef-flash": {
"input_cost_per_token": 9e-08,
"litellm_provider": "cloudflare",
"max_input_tokens": 65536,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
},
"perplexity/pplx-decider-v1-27b": {
"input_cost_per_token": 4e-08,
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
},
"strands_decider/strands-decider-2B-hobson-v19": {
"input_cost_per_token": 0.0,
"litellm_provider": "strands_decider",
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19",
"metadata": {
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
}
}
}

View file

@ -11,6 +11,7 @@ import importlib
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Set as AbstractSet
from dataclasses import dataclass, field
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
@ -251,6 +252,11 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
module_path="litellm.proxy.openai_evals_endpoints.endpoints",
path_prefixes=("/v1/evals", "/evals"),
),
LazyFeature(
name="decisions",
module_path="litellm.proxy.decisions_endpoints.endpoints",
path_prefixes=("/v1/decisions", "/decisions"),
),
LazyFeature(
name="claude_code_marketplace",
module_path="litellm.proxy.anthropic_endpoints.claude_code_endpoints",
@ -361,6 +367,10 @@ def _lazy_slots(app: "FastAPI") -> Mapping[str, BaseRoute | None]:
return app.state.lazy_slots if hasattr(app.state, "lazy_slots") else MappingProxyType({})
def _lazy_routes(app: "FastAPI") -> Mapping[str, tuple[BaseRoute, ...]]:
return app.state.lazy_routes if hasattr(app.state, "lazy_routes") else MappingProxyType({})
def reserve_lazy_slot(app: "FastAPI", name: str, features: tuple[LazyFeature, ...] = LAZY_FEATURES) -> None:
"""Record the route the feature's router used to be included after, so its routes
are spliced back in there once it loads and keep the same precedence. Anchoring on
@ -512,6 +522,13 @@ def _make_warmup_router(app: "FastAPI") -> "APIRouter":
return router
def lazy_owned_routes(app: "FastAPI") -> frozenset[int]:
"""ids of the routes lazy features have registered on this app. A route added later at
one of their paths (a config pass-through at /v1/decisions) goes ahead of them, the
precedence lazy mode gives it when the feature has not loaded by the time the config is read."""
return frozenset(id(route) for route in chain.from_iterable(_lazy_routes(app).values()))
def loaded_lazy_modules(app: "FastAPI") -> frozenset[str]:
"""The set of lazy feature modules whose routers are actually registered
on this app (tracked by _force_load), empty before the middleware ever ran.

View file

@ -8062,6 +8062,61 @@
}
}
},
"decisions": {
"components": {
"schemas": {}
},
"paths": {
"/decisions": {
"post": {
"operationId": "decisions_decisions_post",
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Decisions",
"tags": [
"decisions"
]
}
},
"/v1/decisions": {
"post": {
"operationId": "decisions_v1_decisions_post",
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Decisions",
"tags": [
"decisions"
]
}
}
}
},
"evals": {
"components": {
"schemas": {

View file

@ -466,6 +466,8 @@ class LiteLLMRoutes(enum.Enum):
"/v1/search",
"/search/{search_tool_name}",
"/v1/search/{search_tool_name}",
"/decisions",
"/v1/decisions",
# OCR
"/ocr",
"/v1/ocr",

View file

@ -168,6 +168,7 @@ ProxyRouteType: TypeAlias = Literal[
"avector_store_file_delete",
"aocr",
"asearch",
"adecisions",
"avideo_generation",
"avideo_list",
"avideo_status",
@ -1942,6 +1943,7 @@ class ProxyBaseLLMRequestProcessing:
"avector_store_file_delete",
"aocr",
"asearch",
"adecisions",
"avideo_generation",
"avideo_list",
"avideo_status",

View file

@ -0,0 +1 @@
__all__ = ()

View file

@ -0,0 +1,102 @@
from typing import Annotated, Final
from fastapi import APIRouter, Depends, Request, Response
from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract
from pydantic import TypeAdapter, ValidationError
from litellm.exceptions import BadRequestError
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.types.decisions import DecisionsRequestBody
router: Final = APIRouter()
_REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
_DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody)
_GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
_OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
_OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
@router.post(
"/v1/decisions",
dependencies=[Depends(user_api_key_auth)],
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
tags=["decisions"],
)
@router.post(
"/decisions",
dependencies=[Depends(user_api_key_auth)],
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
tags=["decisions"],
)
async def decisions(
request: Request,
fastapi_response: Response,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
from litellm.proxy.proxy_server import (
general_settings as proxy_general_settings,
)
from litellm.proxy.proxy_server import (
llm_router,
proxy_config,
proxy_logging_obj,
user_max_tokens,
user_request_timeout,
version,
)
from litellm.proxy.proxy_server import (
user_api_base as proxy_user_api_base,
)
from litellm.proxy.proxy_server import (
user_model as proxy_user_model,
)
from litellm.proxy.proxy_server import (
user_temperature as proxy_user_temperature,
)
data: Final = _REQUEST_DATA_ADAPTER.validate_json(await request.body())
general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings)
user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base)
user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model)
user_temperature: Final = _OPTIONAL_FLOAT_ADAPTER.validate_python(proxy_user_temperature)
processor: Final = ProxyBaseLLMRequestProcessing(data=data)
try:
_DECISIONS_REQUEST_BODY_ADAPTER.validate_python(data)
return await processor.base_process_llm_request(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
route_type="adecisions",
proxy_logging_obj=proxy_logging_obj,
llm_router=llm_router,
general_settings=general_settings,
proxy_config=proxy_config,
select_data_generator=None,
model=None,
user_model=user_model,
user_temperature=user_temperature,
user_request_timeout=user_request_timeout,
user_max_tokens=user_max_tokens,
user_api_base=user_api_base,
version=version,
)
except ValidationError as error:
bad_request_error: Final = BadRequestError(
message=f"Invalid Decisions request: {error}",
model=str(data.get("model", "")),
llm_provider="",
)
raise await processor._handle_llm_api_exception(
e=bad_request_error,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
version=version,
)
except Exception as error:
raise await processor._handle_llm_api_exception(
e=error,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
version=version,
)

View file

@ -27,6 +27,7 @@ from fastapi import (
)
from fastapi.responses import StreamingResponse
from starlette.datastructures import UploadFile as StarletteUploadFile
from starlette.routing import BaseRoute, Route
from starlette.websockets import WebSocketState
from websockets.asyncio.client import connect
from websockets.exceptions import (
@ -65,6 +66,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.passthrough import BasePassthroughUtils
from litellm.proxy._lazy_features import lazy_owned_routes
from litellm.proxy._types import (
ConfigFieldInfo,
ConfigFieldUpdate,
@ -2868,35 +2870,34 @@ def _extract_model_from_vertex_ai_setup(setup_response: Mapping[str, object]) ->
return None
def _placed_ahead(routes: Sequence[BaseRoute], moving: BaseRoute, before: BaseRoute) -> tuple[BaseRoute, ...]:
kept: Final = tuple(route for route in routes if route is not moving)
at: Final = next(index for index, route in enumerate(kept) if route is before)
return (*kept[:at], moving, *kept[at:])
class SafeRouteAdder:
"""
Wrapper class for adding routes to FastAPI app.
Only adds routes if they don't already exist on the app.
Only adds routes if they don't already exist on the app. A route a lazy feature registered
does not count: a route added at its path goes ahead of it, the precedence a config
pass-through at /v1/decisions gets in lazy mode, where the feature has not loaded yet.
"""
@staticmethod
def _colliding_routes(app: FastAPI, path: str, methods: Sequence[str]) -> tuple[Route, ...]:
wanted: Final = frozenset(methods)
return tuple(
route
for route in app.routes
if isinstance(route, Route) and route.path == path and not wanted.isdisjoint(route.methods or ())
)
@staticmethod
def _is_path_registered(app: FastAPI, path: str, methods: list[str]) -> bool:
"""
Check if a path with any of the specified methods is already registered on the app.
Args:
app: The FastAPI application instance
path: The path to check (e.g., "/v1/chat/completions")
methods: List of HTTP methods to check (e.g., ["GET", "POST"])
Returns:
True if the path is already registered with any of the methods, False otherwise
"""
for route in app.routes:
# Use getattr to safely access route attributes
route_path = getattr(route, "path", None)
route_methods = getattr(route, "methods", None)
if route_path == path and route_methods is not None:
# Check if any of the methods overlap
if any(method in route_methods for method in methods):
return True
return False
"""True when a route the app itself defines already serves the path with one of the methods."""
lazy_owned: Final = lazy_owned_routes(app)
return any(id(route) not in lazy_owned for route in SafeRouteAdder._colliding_routes(app, path, methods))
@staticmethod
def add_api_route_if_not_exists(
@ -2927,12 +2928,17 @@ class SafeRouteAdder:
)
return False
shadowed: Final = SafeRouteAdder._colliding_routes(app, path, methods)
app.add_api_route(
path=path,
endpoint=endpoint,
methods=methods,
dependencies=dependencies,
)
if shadowed:
app.router.routes[:] = _placed_ahead( # rebind-ok: the app owns its route table
app.router.routes, app.router.routes[-1], shadowed[0]
)
verbose_proxy_logger.debug(
"Successfully added route: %s with methods %s",
path,

View file

@ -90,6 +90,7 @@ ROUTE_ENDPOINT_MAPPING: Final = {
"acompact_responses": "/responses/compact",
"aocr": "/ocr",
"asearch": "/search",
"adecisions": "/decisions",
"avideo_generation": "/videos",
"avideo_list": "/videos",
"avideo_status": "/videos/{video_id}",
@ -372,6 +373,7 @@ RouteType = Literal[
"avector_store_file_delete",
"aocr",
"asearch",
"adecisions",
"avideo_generation",
"avideo_list",
"avideo_status",

View file

@ -2047,6 +2047,11 @@ class Router:
self.asearch = self.factory_function(asearch, call_type="asearch")
self.search = self.factory_function(search, call_type="search")
from litellm.decisions import adecisions, decisions
self.adecisions = self.factory_function(adecisions, call_type="adecisions")
self.decisions = self.factory_function(decisions, call_type="decisions")
def _initialize_video_endpoints(self):
"""Initialize video endpoints."""
from litellm.videos import (
@ -6556,6 +6561,8 @@ class Router:
"ocr",
"asearch",
"search",
"adecisions",
"decisions",
"aadapter_generate_content",
"avideo_generation",
"video_generation",
@ -6629,6 +6636,7 @@ class Router:
"generate_content_stream",
"ocr",
"search",
"decisions",
"video_generation",
"video_list",
"video_status",
@ -6796,6 +6804,7 @@ class Router:
"agenerate_content_stream",
"aocr",
"ocr",
"adecisions",
"avideo_generation",
"avideo_list",
"avideo_status",

123
litellm/types/decisions.py Normal file
View file

@ -0,0 +1,123 @@
from collections.abc import Mapping, Sequence
from typing import Annotated, Literal, TypeAlias
from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config
from typing_extensions import ReadOnly, Required, TypedDict
from litellm.types.llms.base import LiteLLMPydanticObjectBase
DecisionsJSON: TypeAlias = str | Mapping[str, object] | Sequence[object]
NoulCriteria: TypeAlias = Mapping[Literal["true", "false"], DecisionsJSON | None]
class NoulQuestion(LiteLLMPydanticObjectBase):
type: Literal["noul"]
instructions: DecisionsJSON | None = None
criteria: NoulCriteria | None = None
model_config = ConfigDict(extra="allow", frozen=True)
@model_validator(mode="after")
def require_instructions_or_criteria(self) -> "NoulQuestion":
if self.instructions is None and self.criteria is None:
raise ValueError("A noul question requires instructions or criteria")
return self
class ChoiceQuestion(LiteLLMPydanticObjectBase):
type: Literal["choice"]
instructions: DecisionsJSON | None = None
criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)]
model_config = ConfigDict(extra="allow", frozen=True)
class ScoreQuestion(LiteLLMPydanticObjectBase):
type: Literal["score"]
instructions: DecisionsJSON | None = None
criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)]
model_config = ConfigDict(extra="allow", frozen=True)
DecisionQuestion: TypeAlias = Annotated[
NoulQuestion | ChoiceQuestion | ScoreQuestion,
Field(discriminator="type"),
]
DecisionQuestionMap: TypeAlias = Annotated[
Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion],
Field(min_length=1, max_length=128),
]
class DecisionsRequestBody(LiteLLMPydanticObjectBase):
state: DecisionsJSON
questions: DecisionQuestionMap
model_config = ConfigDict(extra="allow", frozen=True)
class DecisionsRequest(DecisionsRequestBody):
model: str
@with_config(ConfigDict(extra="allow"))
class DecisionsCallParams(TypedDict, total=False):
model: Required[ReadOnly[str]]
state: Required[ReadOnly[DecisionsJSON]]
questions: Required[ReadOnly[DecisionQuestionMap]]
api_key: ReadOnly[str | None]
api_base: ReadOnly[str | None]
timeout: ReadOnly[float | None]
custom_llm_provider: ReadOnly[str | None]
extra_headers: ReadOnly[Mapping[str, str] | None]
class NoulAnswer(LiteLLMPydanticObjectBase):
type: Literal["noul"]
noul: float
model_config = ConfigDict(extra="allow", frozen=True)
class ChoiceAnswer(LiteLLMPydanticObjectBase):
type: Literal["choice"]
choice: str
confidence: float
probabilities: Mapping[str, float]
model_config = ConfigDict(extra="allow", frozen=True)
class ScoreAnswer(LiteLLMPydanticObjectBase):
type: Literal["score"]
score: float
confidence: float
legend: Mapping[str, DecisionsJSON]
probabilities: Mapping[str, float]
model_config = ConfigDict(extra="allow", frozen=True)
DecisionAnswer: TypeAlias = Annotated[
NoulAnswer | ChoiceAnswer | ScoreAnswer,
Field(discriminator="type"),
]
class DecisionsUsage(LiteLLMPydanticObjectBase):
input_tokens: int = 0
output_tokens: int = 0
model_config = ConfigDict(extra="allow", frozen=True)
class DecisionsResponse(LiteLLMPydanticObjectBase):
model: str | None = None
answers: Mapping[str, DecisionAnswer]
usage: DecisionsUsage | None = None
model_config = ConfigDict(extra="allow", frozen=True)
_hidden_params: dict[str, object] = PrivateAttr(default_factory=dict)

View file

@ -455,6 +455,8 @@ class CallTypes(str, Enum):
arerank = "arerank"
search = "search"
asearch = "asearch"
decisions = "decisions"
adecisions = "adecisions"
arealtime = "_arealtime"
aresponses_websocket = "_aresponses_websocket"
create_batch = "create_batch"
@ -641,6 +643,8 @@ CallTypesLiteral = Literal[
"arerank",
"search",
"asearch",
"decisions",
"adecisions",
"_arealtime",
"_aresponses_websocket",
"create_batch",
@ -746,6 +750,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
# Search
"/search": [CallTypes.asearch, CallTypes.search],
"/v1/search": [CallTypes.asearch, CallTypes.search],
"/decisions": [CallTypes.adecisions, CallTypes.decisions],
"/v1/decisions": [CallTypes.adecisions, CallTypes.decisions],
# Batches
"/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
"/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
@ -4009,6 +4015,8 @@ class LlmProviders(str, Enum):
OLLAMA_CHAT = "ollama_chat"
DEEPINFRA = "deepinfra"
PERPLEXITY = "perplexity"
TYPESAFE = "typesafe"
STRANDS_DECIDER = "strands_decider"
MISTRAL = "mistral"
MILVUS = "milvus"
GROQ = "groq"

View file

@ -1200,6 +1200,9 @@ def function_setup(
if isinstance(search_query, list)
else search_query
)
elif call_type in (CallTypes.decisions.value, CallTypes.adecisions.value):
decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state", "")
messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state)
elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value):
messages = args[1] if len(args) > 1 else kwargs.get("prompt")
elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value):

View file

@ -77427,5 +77427,55 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_tool_choice": false
},
"cloudflare/clef": {
"input_cost_per_token": 2.4e-07,
"litellm_provider": "cloudflare",
"max_input_tokens": 65536,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
},
"cloudflare/clef-flash": {
"input_cost_per_token": 9e-08,
"litellm_provider": "cloudflare",
"max_input_tokens": 65536,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
},
"cloudflare/@cf/cloudflare/clef": {
"input_cost_per_token": 2.4e-07,
"litellm_provider": "cloudflare",
"max_input_tokens": 65536,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
},
"cloudflare/@cf/cloudflare/clef-flash": {
"input_cost_per_token": 9e-08,
"litellm_provider": "cloudflare",
"max_input_tokens": 65536,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
},
"perplexity/pplx-decider-v1-27b": {
"input_cost_per_token": 4e-08,
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
},
"strands_decider/strands-decider-2B-hobson-v19": {
"input_cost_per_token": 0.0,
"litellm_provider": "strands_decider",
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19",
"metadata": {
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
}
}
}

View file

@ -2499,6 +2499,20 @@
"image_variations": true
}
},
"typesafe": {
"display_name": "TypeSafe (`typesafe`)",
"url": "https://docs.typesafe.ai/models",
"endpoints": {
"systemone": true
}
},
"strands_decider": {
"display_name": "Strands Decider (`strands_decider`)",
"url": "https://docs.litellm.ai/docs/providers",
"endpoints": {
"systemone": true
}
},
"tavily": {
"display_name": "Tavily (`tavily`)",
"url": "https://docs.litellm.ai/docs/search/tavily",

View file

@ -411,11 +411,13 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset(
"sagemaker_nova",
"scaleway",
"stability",
"strands_decider",
"synthetic",
"tencent",
"tensormesh",
"text-completion-inception",
"transcribe",
"typesafe",
"valkey",
"xiaomi_mimo",
"zai",

View file

View file

@ -0,0 +1,592 @@
from __future__ import annotations
import asyncio
import json
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import pytest
import respx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.decisions import (
ChoiceAnswer,
DecisionsResponse,
DecisionsUsage,
NoulAnswer,
ScoreAnswer,
)
_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType(
{
"is_defect": {"type": "noul", "instructions": "Is this a defect?", "provider_field": "kept"},
"sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}},
"severity": {"type": "score", "criteria": ["none", "low", "high"]},
}
)
_INPUT_TOKENS: Final[int] = 367
_OUTPUT_TOKENS: Final[int] = 3
_RESPONSE: Final[Mapping[str, object]] = {
"model": "jev-1.13",
"answers": {
"is_defect": {"type": "noul", "noul": 0.9},
"sentiment": {
"type": "choice",
"choice": "positive",
"confidence": 0.8,
"probabilities": {"positive": 0.8, "negative": 0.2},
},
"severity": {
"type": "score",
"score": 1,
"confidence": 0.7,
"legend": {"0": "none", "1": "low", "2": "high"},
"probabilities": {"0": 0.1, "1": 0.8, "2": 0.1},
},
},
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS},
}
_STRANDS_RESPONSE: Final[Mapping[str, object]] = {
"model": "strands-decider-2B-hobson-v19",
"answers": {
"severity": {
"type": "score",
"score": 1,
"confidence": 0.7,
"legend": {"0": "none", "1": "low", "2": "high"},
"probabilities": {"0": 0.1, "1": 0.8, "2": 0.1},
}
},
"usage": {"input_tokens": 216, "output_tokens": 3},
"latency_ms": 3722.17,
}
_PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = (
(
"perplexity",
"perplexity/pplx-decider-v1-27b",
"https://api.perplexity.ai/v1/decisions",
"pplx-decider-v1-27b",
),
("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"),
(
"openrouter",
"openrouter/typesafe/jev-1.13",
"https://openrouter.ai/api/alpha/decisions",
"typesafe/jev-1.13",
),
)
class _RecordingLogger(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.standard_logging_object: Mapping[str, object] | None = None
async def async_log_success_event(
self,
kwargs: Mapping[str, object],
response_obj: object,
start_time: object,
end_time: object,
) -> None:
standard_logging_object: Final = kwargs.get("standard_logging_object")
if isinstance(standard_logging_object, dict):
self.standard_logging_object = standard_logging_object
async def _drain_logging_worker() -> None:
await asyncio.sleep(0)
GLOBAL_LOGGING_WORKER.start()
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0)
@pytest.fixture(autouse=True)
def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
@pytest.mark.asyncio
@pytest.mark.parametrize(("provider", "model", "url", "upstream_model"), _PROVIDERS)
async def test_adecisions_sends_the_provider_wire_contract(
provider: str,
model: str,
url: str,
upstream_model: str,
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
response: Final = await litellm.adecisions(
model=model,
state={"source": "unit-test"},
questions=_QUESTIONS,
api_key="caller-key",
extra_headers={
"x-request-tag": "decisions-test",
"AUTHORIZATION": "attacker-key",
"Content-Type": "text/plain",
},
internal_kwarg="must-not-leak",
)
assert route.called
assert len(respx_mock.calls) == 1
request: Final = respx_mock.calls[0].request
assert request.headers["authorization"] == "Bearer caller-key"
assert request.headers["content-type"] == "application/json"
assert request.headers["x-request-tag"] == "decisions-test"
assert json.loads(request.content) == {
"model": upstream_model,
"state": {"source": "unit-test"},
"questions": {
"is_defect": {
"type": "noul",
"instructions": "Is this a defect?",
"provider_field": "kept",
},
"sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}},
"severity": {"type": "score", "criteria": ["none", "low", "high"]},
},
}
assert isinstance(response.answers["is_defect"], NoulAnswer)
assert isinstance(response.answers["sentiment"], ChoiceAnswer)
assert isinstance(response.answers["severity"], ScoreAnswer)
assert response._hidden_params["custom_llm_provider"] == provider
@pytest.mark.asyncio
async def test_router_dispatches_typesafe_decisions_without_api_base(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
monkeypatch.delenv("TYPESAFE_API_BASE", raising=False)
provider_resolution: Final = litellm.get_llm_provider("typesafe/jev-latest")
assert provider_resolution[:2] == ("jev-latest", "typesafe")
router: Final = litellm.Router(
model_list=[
{
"model_name": "jev",
"litellm_params": {
"model": "typesafe/jev-latest",
"api_key": "k",
},
}
]
)
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE)
response: Final = await router.adecisions(
model="jev",
state="router-test",
questions={
"sentiment": {
"type": "choice",
"criteria": {"positive": None, "negative": "unhappy"},
}
},
)
assert upstream.called
assert len(respx_mock.calls) == 1
assert json.loads(respx_mock.calls[0].request.content) == {
"model": "jev-latest",
"state": "router-test",
"questions": {
"sentiment": {
"type": "choice",
"criteria": {"positive": None, "negative": "unhappy"},
}
},
}
assert respx_mock.calls[0].request.headers["authorization"] == "Bearer k"
assert isinstance(response.answers["sentiment"], ChoiceAnswer)
assert response.answers["sentiment"].choice == "positive"
def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
response: Final = litellm.decisions(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
api_key="caller-key",
)
assert route.called
assert response.model == "jev-1.13"
def test_openrouter_response_keeps_provider_fields(respx_mock: respx.MockRouter) -> None:
payload: Final = {
**_RESPONSE,
"id": "decision-1",
"provider": "typesafe",
"usage": {**_RESPONSE["usage"], "cost": 0.25},
}
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=payload)
response: Final = litellm.decisions(
model="openrouter/typesafe/jev-1.13",
state="review",
questions=_QUESTIONS,
api_key="caller-key",
)
assert response.model_extra["id"] == "decision-1"
assert response.model_extra["provider"] == "typesafe"
assert response.usage is not None
assert response.usage.model_extra["cost"] == 0.25
def test_decisions_cost_uses_litellm_token_pricing() -> None:
response: Final = DecisionsResponse(
model="pplx-decider-v1-27b",
answers={},
usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS),
)
response._hidden_params = {
"model": "perplexity/pplx-decider-v1-27b",
"custom_llm_provider": "perplexity",
}
cost: Final = litellm.completion_cost(completion_response=response)
perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"]
expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float(
perplexity_cost["output_cost_per_token"]
)
assert expected_cost > 0
assert cost == pytest.approx(expected_cost)
@pytest.mark.asyncio
async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.MockRouter) -> None:
respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
recording_logger: Final = _RecordingLogger()
original_callbacks: Final = litellm.callbacks
litellm.callbacks = [recording_logger]
try:
await litellm.adecisions(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
api_key="caller-key",
)
await _drain_logging_worker()
finally:
litellm.callbacks = original_callbacks
assert recording_logger.standard_logging_object is not None
perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"]
expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float(
perplexity_cost["output_cost_per_token"]
)
assert expected_cost > 0
assert recording_logger.standard_logging_object["response_cost"] == pytest.approx(expected_cost)
assert recording_logger.standard_logging_object["prompt_tokens"] == _INPUT_TOKENS
assert recording_logger.standard_logging_object["completion_tokens"] == _OUTPUT_TOKENS
@pytest.mark.asyncio
async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
with pytest.raises(litellm.BadRequestError, match="Supported providers"):
await litellm.adecisions(
model="unknown/jev-1.13",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
api_key="caller-key",
)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
async def test_empty_custom_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
with pytest.raises(litellm.BadRequestError, match="Supported providers"):
await litellm.adecisions(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
api_key="caller-key",
custom_llm_provider="",
)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
async def test_invalid_question_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"):
await litellm.adecisions(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"sentiment": {"type": "choice"}},
api_key="caller-key",
)
assert len(respx_mock.calls) == 0
def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter) -> None:
respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(
status_code=400,
json={"error": {"message": "invalid decision"}},
)
with pytest.raises(litellm.BadRequestError):
litellm.decisions(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
api_key="caller-key",
)
def test_server_key_is_sent_to_an_explicit_api_base(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("PERPLEXITYAI_API_KEY", "server-key")
monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False)
route: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE)
litellm.decisions(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
api_base="https://egress.example/perplexity",
)
assert route.call_count == 1
assert route.calls[0].request.headers["authorization"] == "Bearer server-key"
@pytest.mark.asyncio
@pytest.mark.parametrize("model", ("cloudflare/clef", "cloudflare/@cf/cloudflare/clef"))
@pytest.mark.parametrize("wrapped", (False, True))
async def test_cloudflare_clef_resolves_model_and_response_envelope(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
model: str,
wrapped: bool,
) -> None:
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
response_body: Final[Mapping[str, object]] = (
{"result": _RESPONSE, "success": True, "errors": [], "messages": []} if wrapped else _RESPONSE
)
route: Final = respx_mock.post(
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef"
).respond(json=response_body)
response: Final = await litellm.adecisions(
model=model,
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
)
assert route.called
request: Final = respx_mock.calls[0].request
assert request.headers["authorization"] == "Bearer cloudflare-key"
assert json.loads(request.content) == {
"model": "clef",
"state": "review",
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
}
assert response.answers == DecisionsResponse.model_validate(_RESPONSE).answers
assert response._hidden_params["model"] == "cloudflare/@cf/cloudflare/clef"
@pytest.mark.asyncio
async def test_cloudflare_clef_flash_uses_flash_endpoint_and_request_model(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
route: Final = respx_mock.post(
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-flash"
).respond(json=_RESPONSE)
await litellm.adecisions(
model="cloudflare/clef-flash",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
)
assert route.called
assert json.loads(respx_mock.calls[0].request.content)["model"] == "clef-flash"
@pytest.mark.asyncio
async def test_cloudflare_api_base_from_env_uses_workers_ai_run_path(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("CLOUDFLARE_API_BASE", "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1")
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False)
route: Final = respx_mock.post(
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef"
).respond(json=_RESPONSE)
await litellm.adecisions(
model="cloudflare/clef",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
)
assert route.called
@pytest.mark.asyncio
async def test_cloudflare_requires_account_id_or_api_base_before_http(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False)
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
with pytest.raises(litellm.BadRequestError, match="Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID"):
await litellm.adecisions(
model="cloudflare/clef",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
async def test_cloudflare_clef_cost_uses_the_model_cost_map(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef").respond(
json=_RESPONSE
)
response: Final = await litellm.adecisions(
model="cloudflare/clef",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
)
cost: Final = litellm.completion_cost(completion_response=response)
clef_cost: Final = litellm.model_cost["cloudflare/@cf/cloudflare/clef"]
expected_cost: Final = _INPUT_TOKENS * float(clef_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float(
clef_cost["output_cost_per_token"]
)
assert expected_cost > 0
assert cost == pytest.approx(expected_cost)
@pytest.mark.asyncio
async def test_strands_decider_requires_api_base_before_http(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False)
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
with pytest.raises(litellm.BadRequestError, match="api_base is required"):
await litellm.adecisions(
model="strands_decider/strands-decider-2B-hobson-v19",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
async def test_strands_decider_without_key_preserves_response_extras(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False)
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
response: Final = await litellm.adecisions(
model="strands_decider/strands-decider-2B-hobson-v19",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
api_base="https://strands.example",
)
assert route.called
assert "authorization" not in respx_mock.calls[0].request.headers
assert response.model_extra["latency_ms"] == _STRANDS_RESPONSE["latency_ms"]
severity: Final = response.answers["severity"]
assert isinstance(severity, ScoreAnswer)
assert severity.legend == {"0": "none", "1": "low", "2": "high"}
@pytest.mark.asyncio
async def test_strands_decider_uses_key_from_matching_environment_base(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("STRANDS_DECIDER_API_BASE", "https://strands.example")
monkeypatch.setenv("STRANDS_DECIDER_API_KEY", "strands-key")
route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
await litellm.adecisions(
model="strands_decider/strands-decider-2B-hobson-v19",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
api_base="https://strands.example",
)
assert route.called
assert respx_mock.calls[0].request.headers["authorization"] == "Bearer strands-key"
@pytest.mark.asyncio
async def test_strands_decider_provider_resolution_and_router_dispatch(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False)
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
provider_resolution: Final = litellm.get_llm_provider("strands_decider/strands-decider-2B-hobson-v19")
router: Final = litellm.Router(
model_list=[
{
"model_name": "strands",
"litellm_params": {
"model": "strands_decider/strands-decider-2B-hobson-v19",
"api_base": "https://strands.example",
},
}
]
)
route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
response: Final = await router.adecisions(
model="strands",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
)
assert provider_resolution[:2] == ("strands-decider-2B-hobson-v19", "strands_decider")
assert route.called
assert response.model == _STRANDS_RESPONSE["model"]

View file

@ -1,5 +1,6 @@
"""Test health check helper functions"""
import json
import socket
import struct
import zlib
@ -8,6 +9,7 @@ from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import respx
import litellm
from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME
@ -528,3 +530,100 @@ def test_ocr_health_check_document_raises_without_the_extension():
_ocr_health_check_document(model="mistral/mistral-ocr-latest", custom_llm_provider="mistral")
finally:
NATIVE_OCR_HEALTH_CHECK_DOCUMENT.reset()
@pytest.mark.parametrize(
("model", "upstream_url"),
(
("perplexity/pplx-decider-v1-27b", "https://api.perplexity.ai/v1/decisions"),
("cloudflare/clef", "https://api.cloudflare.com/client/v4/accounts/acct-1/ai/run/@cf/cloudflare/clef"),
),
)
async def test_ahealth_check_probes_evaluation_models_through_the_decisions_api(
model: str,
upstream_url: str,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct-1")
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post(upstream_url).respond(
json={
"model": model,
"answers": {"reachable": {"type": "noul", "noul": 1.0}},
"usage": {"input_tokens": 12, "output_tokens": 1},
}
)
result: Final = await ahealth_check({"model": model, "api_key": "sk-test"}, mode=None)
assert "error" not in result, result
assert upstream.called
sent: Final = json.loads(upstream.calls[0].request.content)
assert sent["state"] == "health check"
assert sent["questions"]["reachable"]["type"] == "noul"
@pytest.mark.asyncio
async def test_ahealth_check_evaluation_uses_configured_probe_state_and_questions(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(
json={
"model": "perplexity/pplx-decider-v1-27b",
"answers": {"ok": {"type": "noul", "noul": 1.0}},
"usage": {"input_tokens": 12, "output_tokens": 1},
}
)
result: Final = await ahealth_check(
{
"model": "perplexity/pplx-decider-v1-27b",
"api_key": "sk-test",
"state": "custom probe",
"questions": {"ok": {"type": "noul", "instructions": "Is it ok?"}},
},
mode=None,
)
assert "error" not in result, result
assert upstream.called
sent: Final = json.loads(upstream.calls[0].request.content)
assert sent["state"] == "custom probe"
assert set(sent["questions"]) == {"ok"}
@pytest.mark.asyncio
async def test_ahealth_check_probes_strands_through_decisions_without_mode(
local_model_cost_map: None,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("http://strands.local:8080/v1/systemone").respond(
json={
"model": "strands-decider-2B-hobson-v19",
"answers": {"reachable": {"type": "noul", "noul": 1.0}},
"usage": {"input_tokens": 12, "output_tokens": 1},
}
)
result: Final = await ahealth_check(
{
"model": "strands_decider/strands-decider-2B-hobson-v19",
"api_base": "http://strands.local:8080",
},
mode=None,
)
assert "error" not in result, result
assert upstream.called
assert "authorization" not in upstream.calls[0].request.headers

View file

@ -470,6 +470,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client():
"allm_passthrough_route",
"llm_passthrough_route",
"asearch",
"adecisions",
"avector_store_create",
"avector_store_search",
"acreate_skill",

View file

@ -0,0 +1,298 @@
from __future__ import annotations
import asyncio
import json
from collections.abc import Iterator, Mapping
from pathlib import Path
from typing import Final
import pytest
import respx
from fastapi import FastAPI
from fastapi.routing import APIRoute
from fastapi.testclient import TestClient
from starlette.routing import Match
import litellm
from litellm.proxy._lazy_features import attach_lazy_features
from litellm.proxy.decisions_endpoints.endpoints import decisions
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder
from litellm.proxy.proxy_server import (
app,
cleanup_router_config_variables,
initialize,
)
_INPUT_TOKENS: Final[int] = 367
_OUTPUT_TOKENS: Final[int] = 3
_RESPONSE: Final[Mapping[str, object]] = {
"model": "pplx-decider-v1-27b",
"answers": {
"is_defect": {"type": "noul", "noul": 0.9},
},
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS},
}
_STRANDS_RESPONSE: Final[Mapping[str, object]] = {
"model": "strands-decider-2B-hobson-v19",
"answers": {
"is_defect": {"type": "noul", "noul": 0.9},
},
"usage": {"input_tokens": 216, "output_tokens": 3},
"latency_ms": 3722.17,
}
_REQUEST: Final[Mapping[str, object]] = {
"model": "decider",
"state": {"source": "proxy-test"},
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
}
@pytest.fixture
def client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
monkeypatch.setenv("OPENAI_API_KEY", "fake-openai-key")
monkeypatch.setenv("OPENAI_API_BASE", "https://fake-openai.example")
monkeypatch.setenv("REDIS_HOST", "localhost")
cleanup_router_config_variables()
config_path: Final = Path(__file__).parents[1] / "test_configs" / "test_config_no_auth.yaml"
asyncio.run(initialize(config=str(config_path), debug=True))
monkeypatch.setattr(
litellm.proxy.proxy_server,
"llm_router",
litellm.Router(
model_list=[
{
"model_name": "decider",
"litellm_params": {
"model": "perplexity/pplx-decider-v1-27b",
"api_key": "test-key",
},
}
]
),
)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
yield TestClient(app)
litellm.in_memory_llm_clients_cache.flush_cache()
@pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions"))
def test_proxy_decisions_route_returns_answers_and_cost(
client: TestClient,
respx_mock: respx.MockRouter,
endpoint: str,
) -> None:
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
response: Final = client.post(endpoint, json=_REQUEST)
assert response.status_code == 200, response.text
assert response.json()["answers"] == _RESPONSE["answers"]
assert "_hidden_params" not in response.json()
perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"]
expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float(
perplexity_cost["output_cost_per_token"]
)
assert expected_cost > 0
assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost)
assert upstream.called
assert json.loads(upstream.calls[0].request.content) == {
"model": "pplx-decider-v1-27b",
"state": {"source": "proxy-test"},
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
}
assert upstream.calls[0].request.headers["authorization"] == "Bearer test-key"
def test_proxy_decisions_dispatches_typesafe_deployment(
client: TestClient,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
router: Final = litellm.Router(
model_list=[
{
"model_name": "jev",
"litellm_params": {
"model": "typesafe/jev-latest",
"api_key": "k",
},
}
]
)
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE)
response: Final = client.post(
"/v1/decisions",
json={
"model": "jev",
"state": {"source": "proxy-test"},
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
},
)
assert response.status_code == 200, response.text
assert response.json()["answers"] == _RESPONSE["answers"]
assert upstream.called
assert json.loads(upstream.calls[0].request.content) == {
"model": "jev-latest",
"state": {"source": "proxy-test"},
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
}
def test_proxy_decisions_sends_the_env_key_to_the_deployment_api_base(
client: TestClient,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("PERPLEXITYAI_API_KEY", "server-key")
monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False)
router: Final = litellm.Router(
model_list=[
{
"model_name": "decider",
"litellm_params": {
"model": "perplexity/pplx-decider-v1-27b",
"api_base": "https://egress.example/perplexity",
},
}
]
)
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
upstream: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE)
response: Final = client.post("/v1/decisions", json=_REQUEST)
assert response.status_code == 200, response.text
assert upstream.call_count == 1
assert upstream.calls[0].request.headers["authorization"] == "Bearer server-key"
def test_proxy_decisions_unknown_model_is_a_client_error(
client: TestClient,
respx_mock: respx.MockRouter,
) -> None:
response: Final = client.post(
"/v1/decisions",
json={
"model": "missing-model",
"state": "review",
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
},
)
assert 400 <= response.status_code < 500, response.text
assert len(respx_mock.calls) == 0
@pytest.mark.parametrize(
"request_body",
(
{
"model": "decider",
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
},
{
"model": "decider",
"state": {"source": "proxy-test"},
},
),
ids=("missing_state", "missing_questions"),
)
def test_proxy_decisions_missing_required_field_is_a_client_error(
client: TestClient,
respx_mock: respx.MockRouter,
request_body: Mapping[str, object],
) -> None:
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
response: Final = client.post("/v1/decisions", json=request_body)
assert response.status_code == 400, response.text
assert not upstream.called
def test_proxy_decisions_dispatches_strands_decider(
client: TestClient,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False)
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
router: Final = litellm.Router(
model_list=[
{
"model_name": "strands",
"litellm_params": {
"model": "strands_decider/strands-decider-2B-hobson-v19",
"api_base": "https://strands.example",
},
}
]
)
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
upstream: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
response: Final = client.post(
"/v1/decisions",
json={
"model": "strands",
"state": {"source": "proxy-test"},
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
},
)
assert response.status_code == 200, response.text
assert response.json()["answers"] == _STRANDS_RESPONSE["answers"]
assert upstream.called
assert json.loads(upstream.calls[0].request.content) == {
"model": "strands-decider-2B-hobson-v19",
"state": {"source": "proxy-test"},
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
}
assert "authorization" not in upstream.calls[0].request.headers
def test_proxy_decisions_without_model_uses_the_proxy_default_model(
client: TestClient,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setattr(litellm.proxy.proxy_server, "user_model", "decider")
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
response: Final = client.post(
"/v1/decisions", json={key: value for key, value in _REQUEST.items() if key != "model"}
)
assert response.status_code == 200, response.text
assert response.json()["answers"] == _RESPONSE["answers"]
assert upstream.called
assert json.loads(upstream.calls[0].request.content)["model"] == "pplx-decider-v1-27b"
def _serving_endpoint(bare: FastAPI, path: str) -> object:
scope: Final = {"type": "http", "method": "POST", "path": path, "root_path": "", "query_string": b"", "headers": ()}
return next(
route.endpoint for route in bare.routes if isinstance(route, APIRoute) and route.matches(scope)[0] is Match.FULL
)
def test_a_config_pass_through_at_v1_decisions_keeps_its_route_and_the_native_api_serves_decisions(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("LITELLM_DISABLE_LAZY_ROUTES", raising=False)
async def pass_through() -> dict[str, str]:
return {"served_by": "pass-through"}
bare: Final = FastAPI()
attach_lazy_features(bare)
SafeRouteAdder.add_api_route_if_not_exists(bare, "/v1/decisions", pass_through, ["POST"])
with TestClient(bare) as client:
assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"}
assert _serving_endpoint(bare, "/v1/decisions") is pass_through
assert _serving_endpoint(bare, "/decisions") is decisions

View file

@ -4398,6 +4398,23 @@ export interface paths {
patch?: never;
trace?: never;
};
"/decisions": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/** Decisions */
post: operations["decisions_decisions_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/delete/allowed_ip": {
parameters: {
query?: never;
@ -18644,6 +18661,23 @@ export interface paths {
patch?: never;
trace?: never;
};
"/v1/decisions": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/** Decisions */
post: operations["decisions_v1_decisions_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/v1/embeddings": {
parameters: {
query?: never;
@ -26667,7 +26701,7 @@ export interface components {
* CallTypes
* @enum {string}
*/
CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill";
CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill";
/** CallbackDelete */
CallbackDelete: {
/** Callback Name */
@ -53175,6 +53209,26 @@ export interface operations {
};
};
};
decisions_decisions_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
};
};
delete_allowed_ip_delete_allowed_ip_post: {
parameters: {
query?: never;
@ -70747,6 +70801,26 @@ export interface operations {
};
};
};
decisions_v1_decisions_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
};
};
embeddings_v1_embeddings_post: {
parameters: {
query?: never;