diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f55e186e3b2..a4b80aca1dc 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -61,7 +61,7 @@ jobs: - shard: core-utils artifact-name: core-utils - test-path: "" + test-path: tests/unit/decisions unit-flag: core-utils workers: 2 reruns: 1 @@ -130,6 +130,7 @@ jobs: artifact-name: proxy-endpoints test-path: >- tests/test_litellm/proxy/analytics_endpoints + tests/unit/proxy/decisions_endpoints tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/list_api tests/test_litellm/proxy/memory diff --git a/README.md b/README.md index e927c80b8b4..80642882faf 100644 --- a/README.md +++ b/README.md @@ -364,11 +364,13 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | | | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | | | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | | +| [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | | | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | | [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | | | [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | | | | | | | | | [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ | | | | | | | | | [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ | | | | | | | | +| [Typesafe Decisions API (`typesafe`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | | | [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ | | | | | | | | | [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | | | [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index c4a3d3f7473..1999452d5db 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -1,7 +1,7 @@ """Path allowlist for the gateway component. The gateway exposes the LLM data-plane surface: chat/completions, embeddings, -audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image, +audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image, responses, vector stores, passthrough providers, realtime websockets, MCP tool-call endpoints, and operational endpoints (/health, /metrics, and the /debug/memory/summary read of the serving worker's RSS). @@ -60,6 +60,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/v1/rerank", "/v2/rerank", "/rerank", + "/v1/decisions", + "/decisions", "/v1/ocr", "/ocr", "/v1/rag/", diff --git a/litellm/__init__.py b/litellm/__init__.py index e334fbe8ca8..c484037ff7e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1467,6 +1467,7 @@ from .embeddings.dispatch import * from .rust_bridge import rust from .rag.main import * from .sandbox.main import * +from .decisions.main import * from .search.main import * from .realtime_api.main import ( _arealtime, diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a279b9f0903..fb2c23be50c 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -98,6 +98,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse +from litellm.types.decisions import DecisionsResponse, DecisionsUsage from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( HttpxBinaryResponseContent, @@ -1016,6 +1017,7 @@ def _is_known_usage_objects(usage_obj): return ( isinstance(usage_obj, litellm.Usage) or isinstance(usage_obj, ResponseAPIUsage) + or isinstance(usage_obj, DecisionsUsage) or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj) ) @@ -1433,7 +1435,12 @@ def completion_cost( "usage", litellm.Usage(**_usage_for_dump.model_dump()), ) - if usage_obj is None: + if isinstance(usage_obj, DecisionsUsage): + _usage = { + "prompt_tokens": usage_obj.input_tokens, + "completion_tokens": usage_obj.output_tokens, + } + elif usage_obj is None: _usage = {} elif isinstance(usage_obj, BaseModel): _usage = cast(BaseModel, usage_obj).model_dump() @@ -1933,7 +1940,8 @@ def response_cost_calculator( | LiteLLMRealtimeStreamLoggingObject | OpenAIModerationResponse | Response - | SearchResponse, + | SearchResponse + | DecisionsResponse, model: str, custom_llm_provider: str | None, call_type: Literal[ @@ -1955,6 +1963,8 @@ def response_cost_calculator( "arerank", "search", "asearch", + "decisions", + "adecisions", ], optional_params: dict, cache_hit: bool | None = None, diff --git a/litellm/decisions/__init__.py b/litellm/decisions/__init__.py new file mode 100644 index 00000000000..40bddd200a9 --- /dev/null +++ b/litellm/decisions/__init__.py @@ -0,0 +1,3 @@ +from litellm.decisions.main import adecisions, decisions + +__all__ = ["adecisions", "decisions"] diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py new file mode 100644 index 00000000000..da037f1d8cb --- /dev/null +++ b/litellm/decisions/main.py @@ -0,0 +1,299 @@ +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Final + +import httpx +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig +from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT +from litellm.llms.custom_httpx.http_handler import _get_httpx_client, get_async_httpx_client +from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT +from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT +from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT +from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_ENDPOINT +from litellm.secret_managers.main import get_secret_str +from litellm.types.decisions import ( + DecisionQuestion, + DecisionsJSON, + DecisionsRequest, + DecisionsResponse, +) +from litellm.utils import client + +DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxyType( + { + "perplexity": PERPLEXITY_DECISIONS_ENDPOINT, + "typesafe": TYPESAFE_DECISIONS_ENDPOINT, + "openrouter": OPENROUTER_DECISIONS_ENDPOINT, + "cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT, + "strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT, + } +) + +_DECISIONS_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequest]] = TypeAdapter(DecisionsRequest) +_DECISIONS_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) +_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) + + +@dataclass(frozen=True, slots=True, repr=False) +class _PreparedDecisionsRequest: + config: DecisionsProviderConfig + provider: str + upstream_model: str + url: str + api_key: str | None = field(repr=False) + headers: Mapping[str, str] = field(repr=False) + body: Mapping[str, object] = field(repr=False) + + +def _resolve_provider_model(model: str, custom_llm_provider: str | None) -> tuple[str, str]: + provider: Final = model.partition("/")[0] if custom_llm_provider is None else custom_llm_provider + if provider not in DECISIONS_ENDPOINTS: + supported: Final = ", ".join(DECISIONS_ENDPOINTS) + raise litellm.BadRequestError( + message=f"Unknown Decisions provider '{provider}'. Supported providers: {supported}", + model=model, + llm_provider=provider, + ) + upstream_model: Final = model.removeprefix(f"{provider}/") if model.startswith(f"{provider}/") else model + if not upstream_model: + raise litellm.BadRequestError( + message="A model name is required for the Decisions API", + model=model, + llm_provider=provider, + ) + return provider, upstream_model + + +def _resolve_api_key( + *, + provider: str, + model: str, + endpoint: DecisionsProviderConfig, + api_key: str | None, +) -> str | None: + if api_key is not None: + return api_key + + server_api_key: Final = next( + (key for key in (get_secret_str(name) for name in endpoint.api_key_env) if key), + None, + ) + if server_api_key is None: + if not endpoint.api_key_required: + return None + raise litellm.AuthenticationError( + message=f"Missing API key for Decisions provider '{provider}'", + model=model, + llm_provider=provider, + ) + + return server_api_key + + +def _prepare_request( + *, + model: str, + state: DecisionsJSON, + questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str | None, + extra_headers: Mapping[str, str] | None, +) -> _PreparedDecisionsRequest: + provider, upstream_model = _resolve_provider_model(model, custom_llm_provider) + try: + validated_request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python( + {"model": model, "state": state, "questions": questions} + ) + except ValidationError as error: + raise litellm.BadRequestError( + message=f"Invalid Decisions request: {error}", + model=model, + llm_provider=provider, + ) from error + + endpoint: Final = DECISIONS_ENDPOINTS[provider] + env_api_base: Final = get_secret_str(endpoint.api_base_env) + default_api_base: Final = endpoint.default_api_base() + resolved_api_base: Final = api_base or env_api_base or default_api_base + if resolved_api_base is None: + raise litellm.BadRequestError( + message=endpoint.missing_api_base_message(provider), + model=model, + llm_provider=provider, + ) + + resolved_api_key: Final = _resolve_api_key( + provider=provider, + model=model, + endpoint=endpoint, + api_key=api_key, + ) + + canonical_model: Final = endpoint.canonical_model(upstream_model) + outbound_headers: Final = MappingProxyType( + { + **{ + name: value + for name, value in (extra_headers or {}).items() + if name.lower() not in {"authorization", "content-type"} + }, + **({"Authorization": f"Bearer {resolved_api_key}"} if resolved_api_key is not None else {}), + "Content-Type": "application/json", + } + ) + body: Final = MappingProxyType( + { + "model": endpoint.request_model(canonical_model), + "state": validated_request.state, + "questions": { + name: question.model_dump(mode="json", exclude_none=True) + for name, question in validated_request.questions.items() + }, + } + ) + return _PreparedDecisionsRequest( + config=endpoint, + provider=provider, + upstream_model=canonical_model, + url=endpoint.endpoint_url(resolved_api_base, canonical_model), + api_key=resolved_api_key, + headers=outbound_headers, + body=body, + ) + + +def _log_request( + prepared: _PreparedDecisionsRequest, + kwargs: Mapping[str, object], +) -> LiteLLMLoggingObj | None: + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, LiteLLMLoggingObj): + return None + logging_obj.update_from_kwargs( + kwargs=dict(kwargs), + model=prepared.upstream_model, + litellm_params={ + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": prepared.url, + }, + custom_llm_provider=prepared.provider, + ) + request_body: Final = dict(prepared.body) + request_headers: Final = dict(prepared.headers) + logging_obj.pre_call( + input=request_body, + api_key=prepared.api_key, + model=prepared.upstream_model, + additional_args={ + "api_base": prepared.url, + "complete_input_dict": request_body, + "headers": request_headers, + }, + ) + return logging_obj + + +def _parse_response( + response: httpx.Response, + prepared: _PreparedDecisionsRequest, +) -> DecisionsResponse: + response.raise_for_status() + payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content) + result: Final = _DECISIONS_RESPONSE_ADAPTER.validate_python(prepared.config.unwrap_response(payload)) + result._hidden_params.update( + { + "model": f"{prepared.provider}/{prepared.upstream_model}", + "custom_llm_provider": prepared.provider, + "provider_response_model": f"{prepared.provider}/{prepared.upstream_model}", + } + ) + return result + + +def _map_upstream_exception(error: Exception, prepared: _PreparedDecisionsRequest) -> Exception: + return litellm.exception_type( + model=f"{prepared.provider}/{prepared.upstream_model}", + custom_llm_provider=prepared.provider, + original_exception=error, + ) + + +@client +async def adecisions( + model: str, + state: DecisionsJSON, + questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: Mapping[str, str] | None = None, + **kwargs: object, +) -> DecisionsResponse: + prepared: Final = _prepare_request( + model=model, + state=state, + questions=questions, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + ) + logging_obj: Final = _log_request(prepared, kwargs) + try: + handler: Final = get_async_httpx_client(llm_provider=prepared.provider) + response: Final = await handler.post( + prepared.url, + json=dict(prepared.body), + headers=dict(prepared.headers), + timeout=timeout, + logging_obj=logging_obj, + ) + return _parse_response(response=response, prepared=prepared) + except Exception as error: + raise _map_upstream_exception(error, prepared) from error + + +@client +def decisions( + model: str, + state: DecisionsJSON, + questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: Mapping[str, str] | None = None, + **kwargs: object, +) -> DecisionsResponse: + prepared: Final = _prepare_request( + model=model, + state=state, + questions=questions, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + ) + logging_obj: Final = _log_request(prepared, kwargs) + try: + handler: Final = _get_httpx_client() + response: Final = handler.post( + prepared.url, + json=dict(prepared.body), + headers=dict(prepared.headers), + timeout=timeout, + logging_obj=logging_obj, + ) + return _parse_response(response=response, prepared=prepared) + except Exception as error: + raise _map_upstream_exception(error, prepared) from error + + +__all__ = ["DECISIONS_ENDPOINTS", "adecisions", "decisions"] diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 22b8d850c83..0870a5b30ee 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -170,6 +170,7 @@ class HealthCheckHelpers: "batch", "responses", "ocr", + "evaluation", ], Callable, ]: @@ -192,7 +193,7 @@ class HealthCheckHelpers: from litellm.litellm_core_utils.audio_utils.utils import ( get_audio_file_for_health_check, ) - from litellm.litellm_core_utils.health_check_utils import _filter_model_params + from litellm.litellm_core_utils.health_check_utils import DECISIONS_CALL_PARAMS, _filter_model_params from litellm.realtime_api.main import _realtime_health_check return { @@ -259,4 +260,13 @@ class HealthCheckHelpers: **_filter_model_params(model_params=model_params), document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider), ), + "evaluation": lambda: litellm.adecisions( + **DECISIONS_CALL_PARAMS.validate_python( + { + "state": prompt or "health check", + "questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}}, + **_filter_model_params(model_params=model_params), + } + ) + ), } diff --git a/litellm/litellm_core_utils/health_check_utils.py b/litellm/litellm_core_utils/health_check_utils.py index ae56ae8f899..7fe2d830f1e 100644 --- a/litellm/litellm_core_utils/health_check_utils.py +++ b/litellm/litellm_core_utils/health_check_utils.py @@ -4,6 +4,12 @@ Utils used for litellm.ahealth_check() from typing import Final +from pydantic import TypeAdapter + +from litellm.types.decisions import DecisionsCallParams + +DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams) + def _filter_model_params(model_params: dict) -> dict: """Remove 'messages' param from model params.""" diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f3ff4dbc52a..9f210c2cc78 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -118,6 +118,7 @@ from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject +from litellm.types.decisions import DecisionsResponse from litellm.types.interactions import ( InteractionsAPIResponse, InteractionsAPIStreamingResponse, @@ -1815,6 +1816,7 @@ class Logging(LiteLLMLoggingBaseClass): LiteLLMRealtimeStreamLoggingObject, OpenAIModerationResponse, "SearchResponse", + DecisionsResponse, dict, list, ], @@ -2592,6 +2594,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, OpenAIModerationResponse) or isinstance(logging_result, OCRResponse) # OCR or isinstance(logging_result, SearchResponse) # Search API + or isinstance(logging_result, DecisionsResponse) or ( isinstance(logging_result, InteractionsAPIResponse) and logging_result.usage is not None diff --git a/litellm/llms/base_llm/decisions/__init__.py b/litellm/llms/base_llm/decisions/__init__.py new file mode 100644 index 00000000000..c18ac9b00f2 --- /dev/null +++ b/litellm/llms/base_llm/decisions/__init__.py @@ -0,0 +1,3 @@ +from .transformation import DecisionsProviderConfig, JevCompatibleDecisionsEndpoint + +__all__ = ["DecisionsProviderConfig", "JevCompatibleDecisionsEndpoint"] diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py new file mode 100644 index 00000000000..d4fcea24793 --- /dev/null +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -0,0 +1,52 @@ +from dataclasses import dataclass +from typing import Protocol + + +@dataclass(frozen=True, slots=True) +class JevCompatibleDecisionsEndpoint: + default_api_base_value: str | None + path: str + api_key_env: tuple[str, ...] + api_base_env: str + api_key_required: bool = True + + def default_api_base(self) -> str | None: + return self.default_api_base_value + + def missing_api_base_message(self, provider: str) -> str: + return f"api_base is required for Decisions provider '{provider}'" + + def canonical_model(self, model: str) -> str: + return model + + def request_model(self, model: str) -> str: + return model + + def endpoint_url(self, api_base: str, model: str) -> str: + return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" + + def unwrap_response(self, payload: object) -> object: + return payload + + +class DecisionsProviderConfig(Protocol): + @property + def api_key_env(self) -> tuple[str, ...]: ... + + @property + def api_base_env(self) -> str: ... + + @property + def api_key_required(self) -> bool: ... + + def default_api_base(self) -> str | None: ... + + def missing_api_base_message(self, provider: str) -> str: ... + + def canonical_model(self, model: str) -> str: ... + + def request_model(self, model: str) -> str: ... + + def endpoint_url(self, api_base: str, model: str) -> str: ... + + def unwrap_response(self, payload: object) -> object: ... diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py new file mode 100644 index 00000000000..6e8b2999778 --- /dev/null +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -0,0 +1,58 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from pydantic import TypeAdapter + +from litellm.secret_managers.main import ( + get_secret_str, + normalize_nonempty_secret_str, +) + +_RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +@dataclass(frozen=True, slots=True) +class CloudflareDecisionsEndpoint: + api_key_env: tuple[str, ...] = ("CLOUDFLARE_API_KEY",) + api_base_env: str = "CLOUDFLARE_API_BASE" + api_key_required: bool = True + + def default_api_base(self) -> str | None: + account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID")) + if account_id is None: + return None + return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run" + + def missing_api_base_message(self, provider: str) -> str: + return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly" + + def canonical_model(self, model: str) -> str: + if model.startswith("@cf/"): + return model + return f"@cf/cloudflare/{model}" + + def request_model(self, model: str) -> str: + return model.rsplit("/", maxsplit=1)[-1] + + def endpoint_url(self, api_base: str, model: str) -> str: + normalized_api_base: Final = api_base.rstrip("/") + if normalized_api_base.endswith("/ai/v1"): + return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}" + if normalized_api_base.endswith("/ai/run"): + return f"{normalized_api_base}/{model}" + return f"{normalized_api_base}/ai/run/{model}" + + def unwrap_response(self, payload: object) -> object: + if not isinstance(payload, Mapping): + return payload + response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) + if "answers" in response_mapping: + return payload + result: Final = response_mapping.get("result") + if isinstance(result, Mapping): + return result + return payload + + +CLOUDFLARE_DECISIONS_ENDPOINT: Final[CloudflareDecisionsEndpoint] = CloudflareDecisionsEndpoint() diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py new file mode 100644 index 00000000000..7a7466b1239 --- /dev/null +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -0,0 +1,10 @@ +from typing import Final + +from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint + +OPENROUTER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( + default_api_base_value="https://openrouter.ai/api", + path="/alpha/decisions", + api_key_env=("OPENROUTER_API_KEY",), + api_base_env="OPENROUTER_API_BASE", +) diff --git a/litellm/llms/perplexity/decisions/transformation.py b/litellm/llms/perplexity/decisions/transformation.py new file mode 100644 index 00000000000..69a4753f4a3 --- /dev/null +++ b/litellm/llms/perplexity/decisions/transformation.py @@ -0,0 +1,10 @@ +from typing import Final + +from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint + +PERPLEXITY_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( + default_api_base_value="https://api.perplexity.ai", + path="/v1/decisions", + api_key_env=("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"), + api_base_env="PERPLEXITY_API_BASE", +) diff --git a/litellm/llms/strands_decider/decisions/transformation.py b/litellm/llms/strands_decider/decisions/transformation.py new file mode 100644 index 00000000000..265afb2b148 --- /dev/null +++ b/litellm/llms/strands_decider/decisions/transformation.py @@ -0,0 +1,11 @@ +from typing import Final + +from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint + +STRANDS_DECIDER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( + default_api_base_value=None, + path="/v1/systemone", + api_key_env=("STRANDS_DECIDER_API_KEY",), + api_base_env="STRANDS_DECIDER_API_BASE", + api_key_required=False, +) diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py new file mode 100644 index 00000000000..17fb24ce443 --- /dev/null +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -0,0 +1,10 @@ +from typing import Final + +from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint + +TYPESAFE_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( + default_api_base_value="https://api.typesafe.ai", + path="/v1/systemone", + api_key_env=("TYPESAFE_API_KEY",), + api_base_env="TYPESAFE_API_BASE", +) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 46263a71971..1565f9d83c7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -77427,5 +77427,55 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_tool_choice": false + }, + "cloudflare/clef": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/clef-flash": { + "input_cost_per_token": 9e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/@cf/cloudflare/clef": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/@cf/cloudflare/clef-flash": { + "input_cost_per_token": 9e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "perplexity/pplx-decider-v1-27b": { + "input_cost_per_token": 4e-08, + "litellm_provider": "perplexity", + "max_input_tokens": 262144, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.perplexity.ai/api-reference/decisions-post" + }, + "strands_decider/strands-decider-2B-hobson-v19": { + "input_cost_per_token": 0.0, + "litellm_provider": "strands_decider", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19", + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } } } diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 5be87a8bf4d..810a0e18723 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -11,6 +11,7 @@ import importlib from collections.abc import Callable, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass, field +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Final @@ -251,6 +252,11 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( module_path="litellm.proxy.openai_evals_endpoints.endpoints", path_prefixes=("/v1/evals", "/evals"), ), + LazyFeature( + name="decisions", + module_path="litellm.proxy.decisions_endpoints.endpoints", + path_prefixes=("/v1/decisions", "/decisions"), + ), LazyFeature( name="claude_code_marketplace", module_path="litellm.proxy.anthropic_endpoints.claude_code_endpoints", @@ -361,6 +367,10 @@ def _lazy_slots(app: "FastAPI") -> Mapping[str, BaseRoute | None]: return app.state.lazy_slots if hasattr(app.state, "lazy_slots") else MappingProxyType({}) +def _lazy_routes(app: "FastAPI") -> Mapping[str, tuple[BaseRoute, ...]]: + return app.state.lazy_routes if hasattr(app.state, "lazy_routes") else MappingProxyType({}) + + def reserve_lazy_slot(app: "FastAPI", name: str, features: tuple[LazyFeature, ...] = LAZY_FEATURES) -> None: """Record the route the feature's router used to be included after, so its routes are spliced back in there once it loads and keep the same precedence. Anchoring on @@ -512,6 +522,13 @@ def _make_warmup_router(app: "FastAPI") -> "APIRouter": return router +def lazy_owned_routes(app: "FastAPI") -> frozenset[int]: + """ids of the routes lazy features have registered on this app. A route added later at + one of their paths (a config pass-through at /v1/decisions) goes ahead of them, the + precedence lazy mode gives it when the feature has not loaded by the time the config is read.""" + return frozenset(id(route) for route in chain.from_iterable(_lazy_routes(app).values())) + + def loaded_lazy_modules(app: "FastAPI") -> frozenset[str]: """The set of lazy feature modules whose routers are actually registered on this app (tracked by _force_load), empty before the middleware ever ran. diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 48ce63c4ef4..0270be5720b 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -8062,6 +8062,61 @@ } } }, + "decisions": { + "components": { + "schemas": {} + }, + "paths": { + "/decisions": { + "post": { + "operationId": "decisions_decisions_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Decisions", + "tags": [ + "decisions" + ] + } + }, + "/v1/decisions": { + "post": { + "operationId": "decisions_v1_decisions_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Decisions", + "tags": [ + "decisions" + ] + } + } + } + }, "evals": { "components": { "schemas": { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 344f83fe52a..b2716389dba 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -466,6 +466,8 @@ class LiteLLMRoutes(enum.Enum): "/v1/search", "/search/{search_tool_name}", "/v1/search/{search_tool_name}", + "/decisions", + "/v1/decisions", # OCR "/ocr", "/v1/ocr", diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7e1989b0aab..a5b107412cf 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -168,6 +168,7 @@ ProxyRouteType: TypeAlias = Literal[ "avector_store_file_delete", "aocr", "asearch", + "adecisions", "avideo_generation", "avideo_list", "avideo_status", @@ -1942,6 +1943,7 @@ class ProxyBaseLLMRequestProcessing: "avector_store_file_delete", "aocr", "asearch", + "adecisions", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/proxy/decisions_endpoints/__init__.py b/litellm/proxy/decisions_endpoints/__init__.py new file mode 100644 index 00000000000..ea9b7835485 --- /dev/null +++ b/litellm/proxy/decisions_endpoints/__init__.py @@ -0,0 +1 @@ +__all__ = () diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py new file mode 100644 index 00000000000..7dba64ee791 --- /dev/null +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -0,0 +1,102 @@ +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, Request, Response +from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract +from pydantic import TypeAdapter, ValidationError + +from litellm.exceptions import BadRequestError +from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.types.decisions import DecisionsRequestBody + +router: Final = APIRouter() +_REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) +_DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) +_OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) +_OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) + + +@router.post( + "/v1/decisions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +@router.post( + "/decisions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +async def decisions( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + from litellm.proxy.proxy_server import ( + general_settings as proxy_general_settings, + ) + from litellm.proxy.proxy_server import ( + llm_router, + proxy_config, + proxy_logging_obj, + user_max_tokens, + user_request_timeout, + version, + ) + from litellm.proxy.proxy_server import ( + user_api_base as proxy_user_api_base, + ) + from litellm.proxy.proxy_server import ( + user_model as proxy_user_model, + ) + from litellm.proxy.proxy_server import ( + user_temperature as proxy_user_temperature, + ) + + data: Final = _REQUEST_DATA_ADAPTER.validate_json(await request.body()) + general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings) + user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base) + user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model) + user_temperature: Final = _OPTIONAL_FLOAT_ADAPTER.validate_python(proxy_user_temperature) + processor: Final = ProxyBaseLLMRequestProcessing(data=data) + try: + _DECISIONS_REQUEST_BODY_ADAPTER.validate_python(data) + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="adecisions", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=None, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except ValidationError as error: + bad_request_error: Final = BadRequestError( + message=f"Invalid Decisions request: {error}", + model=str(data.get("model", "")), + llm_provider="", + ) + raise await processor._handle_llm_api_exception( + e=bad_request_error, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + except Exception as error: + raise await processor._handle_llm_api_exception( + e=error, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index e0a4184291e..35d4f18d47a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -27,6 +27,7 @@ from fastapi import ( ) from fastapi.responses import StreamingResponse from starlette.datastructures import UploadFile as StarletteUploadFile +from starlette.routing import BaseRoute, Route from starlette.websockets import WebSocketState from websockets.asyncio.client import connect from websockets.exceptions import ( @@ -65,6 +66,7 @@ from litellm.llms.base_llm.managed_resources.utils import ( ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.passthrough import BasePassthroughUtils +from litellm.proxy._lazy_features import lazy_owned_routes from litellm.proxy._types import ( ConfigFieldInfo, ConfigFieldUpdate, @@ -2868,35 +2870,34 @@ def _extract_model_from_vertex_ai_setup(setup_response: Mapping[str, object]) -> return None +def _placed_ahead(routes: Sequence[BaseRoute], moving: BaseRoute, before: BaseRoute) -> tuple[BaseRoute, ...]: + kept: Final = tuple(route for route in routes if route is not moving) + at: Final = next(index for index, route in enumerate(kept) if route is before) + return (*kept[:at], moving, *kept[at:]) + + class SafeRouteAdder: """ Wrapper class for adding routes to FastAPI app. - Only adds routes if they don't already exist on the app. + Only adds routes if they don't already exist on the app. A route a lazy feature registered + does not count: a route added at its path goes ahead of it, the precedence a config + pass-through at /v1/decisions gets in lazy mode, where the feature has not loaded yet. """ + @staticmethod + def _colliding_routes(app: FastAPI, path: str, methods: Sequence[str]) -> tuple[Route, ...]: + wanted: Final = frozenset(methods) + return tuple( + route + for route in app.routes + if isinstance(route, Route) and route.path == path and not wanted.isdisjoint(route.methods or ()) + ) + @staticmethod def _is_path_registered(app: FastAPI, path: str, methods: list[str]) -> bool: - """ - Check if a path with any of the specified methods is already registered on the app. - - Args: - app: The FastAPI application instance - path: The path to check (e.g., "/v1/chat/completions") - methods: List of HTTP methods to check (e.g., ["GET", "POST"]) - - Returns: - True if the path is already registered with any of the methods, False otherwise - """ - for route in app.routes: - # Use getattr to safely access route attributes - route_path = getattr(route, "path", None) - route_methods = getattr(route, "methods", None) - - if route_path == path and route_methods is not None: - # Check if any of the methods overlap - if any(method in route_methods for method in methods): - return True - return False + """True when a route the app itself defines already serves the path with one of the methods.""" + lazy_owned: Final = lazy_owned_routes(app) + return any(id(route) not in lazy_owned for route in SafeRouteAdder._colliding_routes(app, path, methods)) @staticmethod def add_api_route_if_not_exists( @@ -2927,12 +2928,17 @@ class SafeRouteAdder: ) return False + shadowed: Final = SafeRouteAdder._colliding_routes(app, path, methods) app.add_api_route( path=path, endpoint=endpoint, methods=methods, dependencies=dependencies, ) + if shadowed: + app.router.routes[:] = _placed_ahead( # rebind-ok: the app owns its route table + app.router.routes, app.router.routes[-1], shadowed[0] + ) verbose_proxy_logger.debug( "Successfully added route: %s with methods %s", path, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 536c58df65a..030fae5fa62 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -90,6 +90,7 @@ ROUTE_ENDPOINT_MAPPING: Final = { "acompact_responses": "/responses/compact", "aocr": "/ocr", "asearch": "/search", + "adecisions": "/decisions", "avideo_generation": "/videos", "avideo_list": "/videos", "avideo_status": "/videos/{video_id}", @@ -372,6 +373,7 @@ RouteType = Literal[ "avector_store_file_delete", "aocr", "asearch", + "adecisions", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/router.py b/litellm/router.py index 023b99cd64e..d34d0b21a5a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2047,6 +2047,11 @@ class Router: self.asearch = self.factory_function(asearch, call_type="asearch") self.search = self.factory_function(search, call_type="search") + from litellm.decisions import adecisions, decisions + + self.adecisions = self.factory_function(adecisions, call_type="adecisions") + self.decisions = self.factory_function(decisions, call_type="decisions") + def _initialize_video_endpoints(self): """Initialize video endpoints.""" from litellm.videos import ( @@ -6556,6 +6561,8 @@ class Router: "ocr", "asearch", "search", + "adecisions", + "decisions", "aadapter_generate_content", "avideo_generation", "video_generation", @@ -6629,6 +6636,7 @@ class Router: "generate_content_stream", "ocr", "search", + "decisions", "video_generation", "video_list", "video_status", @@ -6796,6 +6804,7 @@ class Router: "agenerate_content_stream", "aocr", "ocr", + "adecisions", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py new file mode 100644 index 00000000000..80db26f201d --- /dev/null +++ b/litellm/types/decisions.py @@ -0,0 +1,123 @@ +from collections.abc import Mapping, Sequence +from typing import Annotated, Literal, TypeAlias + +from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config +from typing_extensions import ReadOnly, Required, TypedDict + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +DecisionsJSON: TypeAlias = str | Mapping[str, object] | Sequence[object] +NoulCriteria: TypeAlias = Mapping[Literal["true", "false"], DecisionsJSON | None] + + +class NoulQuestion(LiteLLMPydanticObjectBase): + type: Literal["noul"] + instructions: DecisionsJSON | None = None + criteria: NoulCriteria | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + @model_validator(mode="after") + def require_instructions_or_criteria(self) -> "NoulQuestion": + if self.instructions is None and self.criteria is None: + raise ValueError("A noul question requires instructions or criteria") + return self + + +class ChoiceQuestion(LiteLLMPydanticObjectBase): + type: Literal["choice"] + instructions: DecisionsJSON | None = None + criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)] + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ScoreQuestion(LiteLLMPydanticObjectBase): + type: Literal["score"] + instructions: DecisionsJSON | None = None + criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)] + + model_config = ConfigDict(extra="allow", frozen=True) + + +DecisionQuestion: TypeAlias = Annotated[ + NoulQuestion | ChoiceQuestion | ScoreQuestion, + Field(discriminator="type"), +] + +DecisionQuestionMap: TypeAlias = Annotated[ + Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion], + Field(min_length=1, max_length=128), +] + + +class DecisionsRequestBody(LiteLLMPydanticObjectBase): + state: DecisionsJSON + questions: DecisionQuestionMap + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionsRequest(DecisionsRequestBody): + model: str + + +@with_config(ConfigDict(extra="allow")) +class DecisionsCallParams(TypedDict, total=False): + model: Required[ReadOnly[str]] + state: Required[ReadOnly[DecisionsJSON]] + questions: Required[ReadOnly[DecisionQuestionMap]] + api_key: ReadOnly[str | None] + api_base: ReadOnly[str | None] + timeout: ReadOnly[float | None] + custom_llm_provider: ReadOnly[str | None] + extra_headers: ReadOnly[Mapping[str, str] | None] + + +class NoulAnswer(LiteLLMPydanticObjectBase): + type: Literal["noul"] + noul: float + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ChoiceAnswer(LiteLLMPydanticObjectBase): + type: Literal["choice"] + choice: str + confidence: float + probabilities: Mapping[str, float] + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ScoreAnswer(LiteLLMPydanticObjectBase): + type: Literal["score"] + score: float + confidence: float + legend: Mapping[str, DecisionsJSON] + probabilities: Mapping[str, float] + + model_config = ConfigDict(extra="allow", frozen=True) + + +DecisionAnswer: TypeAlias = Annotated[ + NoulAnswer | ChoiceAnswer | ScoreAnswer, + Field(discriminator="type"), +] + + +class DecisionsUsage(LiteLLMPydanticObjectBase): + input_tokens: int = 0 + output_tokens: int = 0 + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionsResponse(LiteLLMPydanticObjectBase): + model: str | None = None + answers: Mapping[str, DecisionAnswer] + usage: DecisionsUsage | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 15b34285c5e..791541f9804 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -455,6 +455,8 @@ class CallTypes(str, Enum): arerank = "arerank" search = "search" asearch = "asearch" + decisions = "decisions" + adecisions = "adecisions" arealtime = "_arealtime" aresponses_websocket = "_aresponses_websocket" create_batch = "create_batch" @@ -641,6 +643,8 @@ CallTypesLiteral = Literal[ "arerank", "search", "asearch", + "decisions", + "adecisions", "_arealtime", "_aresponses_websocket", "create_batch", @@ -746,6 +750,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { # Search "/search": [CallTypes.asearch, CallTypes.search], "/v1/search": [CallTypes.asearch, CallTypes.search], + "/decisions": [CallTypes.adecisions, CallTypes.decisions], + "/v1/decisions": [CallTypes.adecisions, CallTypes.decisions], # Batches "/batches": [CallTypes.acreate_batch, CallTypes.create_batch], "/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch], @@ -4009,6 +4015,8 @@ class LlmProviders(str, Enum): OLLAMA_CHAT = "ollama_chat" DEEPINFRA = "deepinfra" PERPLEXITY = "perplexity" + TYPESAFE = "typesafe" + STRANDS_DECIDER = "strands_decider" MISTRAL = "mistral" MILVUS = "milvus" GROQ = "groq" diff --git a/litellm/utils.py b/litellm/utils.py index 40636b15f8b..731f1e9b3fd 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1200,6 +1200,9 @@ def function_setup( if isinstance(search_query, list) else search_query ) + elif call_type in (CallTypes.decisions.value, CallTypes.adecisions.value): + decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state", "") + messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state) elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): messages = args[1] if len(args) > 1 else kwargs.get("prompt") elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 46263a71971..1565f9d83c7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -77427,5 +77427,55 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_tool_choice": false + }, + "cloudflare/clef": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/clef-flash": { + "input_cost_per_token": 9e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/@cf/cloudflare/clef": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "cloudflare/@cf/cloudflare/clef-flash": { + "input_cost_per_token": 9e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, + "perplexity/pplx-decider-v1-27b": { + "input_cost_per_token": 4e-08, + "litellm_provider": "perplexity", + "max_input_tokens": 262144, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.perplexity.ai/api-reference/decisions-post" + }, + "strands_decider/strands-decider-2B-hobson-v19": { + "input_cost_per_token": 0.0, + "litellm_provider": "strands_decider", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19", + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e6cb0592a15..54e08240c90 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2499,6 +2499,20 @@ "image_variations": true } }, + "typesafe": { + "display_name": "TypeSafe (`typesafe`)", + "url": "https://docs.typesafe.ai/models", + "endpoints": { + "systemone": true + } + }, + "strands_decider": { + "display_name": "Strands Decider (`strands_decider`)", + "url": "https://docs.litellm.ai/docs/providers", + "endpoints": { + "systemone": true + } + }, "tavily": { "display_name": "Tavily (`tavily`)", "url": "https://docs.litellm.ai/docs/search/tavily", diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 0dec44af402..a747271019f 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -411,11 +411,13 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( "sagemaker_nova", "scaleway", "stability", + "strands_decider", "synthetic", "tencent", "tensormesh", "text-completion-inception", "transcribe", + "typesafe", "valkey", "xiaomi_mimo", "zai", diff --git a/tests/unit/decisions/__init__.py b/tests/unit/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py new file mode 100644 index 00000000000..106710d328f --- /dev/null +++ b/tests/unit/decisions/test_main.py @@ -0,0 +1,592 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import pytest +import respx + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.decisions import ( + ChoiceAnswer, + DecisionsResponse, + DecisionsUsage, + NoulAnswer, + ScoreAnswer, +) + +_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( + { + "is_defect": {"type": "noul", "instructions": "Is this a defect?", "provider_field": "kept"}, + "sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}}, + "severity": {"type": "score", "criteria": ["none", "low", "high"]}, + } +) +_INPUT_TOKENS: Final[int] = 367 +_OUTPUT_TOKENS: Final[int] = 3 +_RESPONSE: Final[Mapping[str, object]] = { + "model": "jev-1.13", + "answers": { + "is_defect": {"type": "noul", "noul": 0.9}, + "sentiment": { + "type": "choice", + "choice": "positive", + "confidence": 0.8, + "probabilities": {"positive": 0.8, "negative": 0.2}, + }, + "severity": { + "type": "score", + "score": 1, + "confidence": 0.7, + "legend": {"0": "none", "1": "low", "2": "high"}, + "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}, + }, + }, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, +} +_STRANDS_RESPONSE: Final[Mapping[str, object]] = { + "model": "strands-decider-2B-hobson-v19", + "answers": { + "severity": { + "type": "score", + "score": 1, + "confidence": 0.7, + "legend": {"0": "none", "1": "low", "2": "high"}, + "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}, + } + }, + "usage": {"input_tokens": 216, "output_tokens": 3}, + "latency_ms": 3722.17, +} +_PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( + ( + "perplexity", + "perplexity/pplx-decider-v1-27b", + "https://api.perplexity.ai/v1/decisions", + "pplx-decider-v1-27b", + ), + ("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"), + ( + "openrouter", + "openrouter/typesafe/jev-1.13", + "https://openrouter.ai/api/alpha/decisions", + "typesafe/jev-1.13", + ), +) + + +class _RecordingLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.standard_logging_object: Mapping[str, object] | None = None + + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, + ) -> None: + standard_logging_object: Final = kwargs.get("standard_logging_object") + if isinstance(standard_logging_object, dict): + self.standard_logging_object = standard_logging_object + + +async def _drain_logging_worker() -> None: + await asyncio.sleep(0) + GLOBAL_LOGGING_WORKER.start() + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + +@pytest.fixture(autouse=True) +def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("provider", "model", "url", "upstream_model"), _PROVIDERS) +async def test_adecisions_sends_the_provider_wire_contract( + provider: str, + model: str, + url: str, + upstream_model: str, + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url).respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model=model, + state={"source": "unit-test"}, + questions=_QUESTIONS, + api_key="caller-key", + extra_headers={ + "x-request-tag": "decisions-test", + "AUTHORIZATION": "attacker-key", + "Content-Type": "text/plain", + }, + internal_kwarg="must-not-leak", + ) + + assert route.called + assert len(respx_mock.calls) == 1 + request: Final = respx_mock.calls[0].request + assert request.headers["authorization"] == "Bearer caller-key" + assert request.headers["content-type"] == "application/json" + assert request.headers["x-request-tag"] == "decisions-test" + assert json.loads(request.content) == { + "model": upstream_model, + "state": {"source": "unit-test"}, + "questions": { + "is_defect": { + "type": "noul", + "instructions": "Is this a defect?", + "provider_field": "kept", + }, + "sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}}, + "severity": {"type": "score", "criteria": ["none", "low", "high"]}, + }, + } + assert isinstance(response.answers["is_defect"], NoulAnswer) + assert isinstance(response.answers["sentiment"], ChoiceAnswer) + assert isinstance(response.answers["severity"], ScoreAnswer) + assert response._hidden_params["custom_llm_provider"] == provider + + +@pytest.mark.asyncio +async def test_router_dispatches_typesafe_decisions_without_api_base( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) + monkeypatch.delenv("TYPESAFE_API_BASE", raising=False) + provider_resolution: Final = litellm.get_llm_provider("typesafe/jev-latest") + + assert provider_resolution[:2] == ("jev-latest", "typesafe") + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": { + "model": "typesafe/jev-latest", + "api_key": "k", + }, + } + ] + ) + upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) + + response: Final = await router.adecisions( + model="jev", + state="router-test", + questions={ + "sentiment": { + "type": "choice", + "criteria": {"positive": None, "negative": "unhappy"}, + } + }, + ) + + assert upstream.called + assert len(respx_mock.calls) == 1 + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "jev-latest", + "state": "router-test", + "questions": { + "sentiment": { + "type": "choice", + "criteria": {"positive": None, "negative": "unhappy"}, + } + }, + } + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer k" + assert isinstance(response.answers["sentiment"], ChoiceAnswer) + assert response.answers["sentiment"].choice == "positive" + + +def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + assert route.called + assert response.model == "jev-1.13" + + +def test_openrouter_response_keeps_provider_fields(respx_mock: respx.MockRouter) -> None: + payload: Final = { + **_RESPONSE, + "id": "decision-1", + "provider": "typesafe", + "usage": {**_RESPONSE["usage"], "cost": 0.25}, + } + respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=payload) + + response: Final = litellm.decisions( + model="openrouter/typesafe/jev-1.13", + state="review", + questions=_QUESTIONS, + api_key="caller-key", + ) + + assert response.model_extra["id"] == "decision-1" + assert response.model_extra["provider"] == "typesafe" + assert response.usage is not None + assert response.usage.model_extra["cost"] == 0.25 + + +def test_decisions_cost_uses_litellm_token_pricing() -> None: + response: Final = DecisionsResponse( + model="pplx-decider-v1-27b", + answers={}, + usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS), + ) + response._hidden_params = { + "model": "perplexity/pplx-decider-v1-27b", + "custom_llm_provider": "perplexity", + } + + cost: Final = litellm.completion_cost(completion_response=response) + perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] + expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + perplexity_cost["output_cost_per_token"] + ) + + assert expected_cost > 0 + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.asyncio +async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + recording_logger: Final = _RecordingLogger() + original_callbacks: Final = litellm.callbacks + litellm.callbacks = [recording_logger] + + try: + await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + await _drain_logging_worker() + finally: + litellm.callbacks = original_callbacks + + assert recording_logger.standard_logging_object is not None + perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] + expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + perplexity_cost["output_cost_per_token"] + ) + + assert expected_cost > 0 + assert recording_logger.standard_logging_object["response_cost"] == pytest.approx(expected_cost) + assert recording_logger.standard_logging_object["prompt_tokens"] == _INPUT_TOKENS + assert recording_logger.standard_logging_object["completion_tokens"] == _OUTPUT_TOKENS + + +@pytest.mark.asyncio +async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="Supported providers"): + await litellm.adecisions( + model="unknown/jev-1.13", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_empty_custom_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="Supported providers"): + await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + custom_llm_provider="", + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_invalid_question_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): + await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"sentiment": {"type": "choice"}}, + api_key="caller-key", + ) + + assert len(respx_mock.calls) == 0 + + +def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + status_code=400, + json={"error": {"message": "invalid decision"}}, + ) + + with pytest.raises(litellm.BadRequestError): + litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + +def test_server_key_is_sent_to_an_explicit_api_base( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("PERPLEXITYAI_API_KEY", "server-key") + monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False) + route: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE) + + litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_base="https://egress.example/perplexity", + ) + + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == "Bearer server-key" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("cloudflare/clef", "cloudflare/@cf/cloudflare/clef")) +@pytest.mark.parametrize("wrapped", (False, True)) +async def test_cloudflare_clef_resolves_model_and_response_envelope( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + model: str, + wrapped: bool, +) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + response_body: Final[Mapping[str, object]] = ( + {"result": _RESPONSE, "success": True, "errors": [], "messages": []} if wrapped else _RESPONSE + ) + route: Final = respx_mock.post( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" + ).respond(json=response_body) + + response: Final = await litellm.adecisions( + model=model, + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert route.called + request: Final = respx_mock.calls[0].request + assert request.headers["authorization"] == "Bearer cloudflare-key" + assert json.loads(request.content) == { + "model": "clef", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers == DecisionsResponse.model_validate(_RESPONSE).answers + assert response._hidden_params["model"] == "cloudflare/@cf/cloudflare/clef" + + +@pytest.mark.asyncio +async def test_cloudflare_clef_flash_uses_flash_endpoint_and_request_model( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + route: Final = respx_mock.post( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-flash" + ).respond(json=_RESPONSE) + + await litellm.adecisions( + model="cloudflare/clef-flash", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content)["model"] == "clef-flash" + + +@pytest.mark.asyncio +async def test_cloudflare_api_base_from_env_uses_workers_ai_run_path( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("CLOUDFLARE_API_BASE", "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1") + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False) + route: Final = respx_mock.post( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" + ).respond(json=_RESPONSE) + + await litellm.adecisions( + model="cloudflare/clef", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert route.called + + +@pytest.mark.asyncio +async def test_cloudflare_requires_account_id_or_api_base_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False) + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + + with pytest.raises(litellm.BadRequestError, match="Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID"): + await litellm.adecisions( + model="cloudflare/clef", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_cloudflare_clef_cost_uses_the_model_cost_map( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef").respond( + json=_RESPONSE + ) + + response: Final = await litellm.adecisions( + model="cloudflare/clef", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + cost: Final = litellm.completion_cost(completion_response=response) + clef_cost: Final = litellm.model_cost["cloudflare/@cf/cloudflare/clef"] + expected_cost: Final = _INPUT_TOKENS * float(clef_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + clef_cost["output_cost_per_token"] + ) + + assert expected_cost > 0 + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.asyncio +async def test_strands_decider_requires_api_base_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + + with pytest.raises(litellm.BadRequestError, match="api_base is required"): + await litellm.adecisions( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_strands_decider_without_key_preserves_response_extras( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + + response: Final = await litellm.adecisions( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base="https://strands.example", + ) + + assert route.called + assert "authorization" not in respx_mock.calls[0].request.headers + assert response.model_extra["latency_ms"] == _STRANDS_RESPONSE["latency_ms"] + severity: Final = response.answers["severity"] + assert isinstance(severity, ScoreAnswer) + assert severity.legend == {"0": "none", "1": "low", "2": "high"} + + +@pytest.mark.asyncio +async def test_strands_decider_uses_key_from_matching_environment_base( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("STRANDS_DECIDER_API_BASE", "https://strands.example") + monkeypatch.setenv("STRANDS_DECIDER_API_KEY", "strands-key") + route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + + await litellm.adecisions( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base="https://strands.example", + ) + + assert route.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer strands-key" + + +@pytest.mark.asyncio +async def test_strands_decider_provider_resolution_and_router_dispatch( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + provider_resolution: Final = litellm.get_llm_provider("strands_decider/strands-decider-2B-hobson-v19") + router: Final = litellm.Router( + model_list=[ + { + "model_name": "strands", + "litellm_params": { + "model": "strands_decider/strands-decider-2B-hobson-v19", + "api_base": "https://strands.example", + }, + } + ] + ) + route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + + response: Final = await router.adecisions( + model="strands", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + ) + + assert provider_resolution[:2] == ("strands-decider-2B-hobson-v19", "strands_decider") + assert route.called + assert response.model == _STRANDS_RESPONSE["model"] diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index ad0f82109f3..853fe9ddfad 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1,5 +1,6 @@ """Test health check helper functions""" +import json import socket import struct import zlib @@ -8,6 +9,7 @@ from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +import respx import litellm from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME @@ -528,3 +530,100 @@ def test_ocr_health_check_document_raises_without_the_extension(): _ocr_health_check_document(model="mistral/mistral-ocr-latest", custom_llm_provider="mistral") finally: NATIVE_OCR_HEALTH_CHECK_DOCUMENT.reset() + + +@pytest.mark.parametrize( + ("model", "upstream_url"), + ( + ("perplexity/pplx-decider-v1-27b", "https://api.perplexity.ai/v1/decisions"), + ("cloudflare/clef", "https://api.cloudflare.com/client/v4/accounts/acct-1/ai/run/@cf/cloudflare/clef"), + ), +) +async def test_ahealth_check_probes_evaluation_models_through_the_decisions_api( + model: str, + upstream_url: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct-1") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(upstream_url).respond( + json={ + "model": model, + "answers": {"reachable": {"type": "noul", "noul": 1.0}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check({"model": model, "api_key": "sk-test"}, mode=None) + + assert "error" not in result, result + assert upstream.called + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["state"] == "health check" + assert sent["questions"]["reachable"]["type"] == "noul" + + +@pytest.mark.asyncio +async def test_ahealth_check_evaluation_uses_configured_probe_state_and_questions( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={ + "model": "perplexity/pplx-decider-v1-27b", + "answers": {"ok": {"type": "noul", "noul": 1.0}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check( + { + "model": "perplexity/pplx-decider-v1-27b", + "api_key": "sk-test", + "state": "custom probe", + "questions": {"ok": {"type": "noul", "instructions": "Is it ok?"}}, + }, + mode=None, + ) + + assert "error" not in result, result + assert upstream.called + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["state"] == "custom probe" + assert set(sent["questions"]) == {"ok"} + + +@pytest.mark.asyncio +async def test_ahealth_check_probes_strands_through_decisions_without_mode( + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("http://strands.local:8080/v1/systemone").respond( + json={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"reachable": {"type": "noul", "noul": 1.0}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check( + { + "model": "strands_decider/strands-decider-2B-hobson-v19", + "api_base": "http://strands.local:8080", + }, + mode=None, + ) + + assert "error" not in result, result + assert upstream.called + assert "authorization" not in upstream.calls[0].request.headers diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index caf941ebd19..5f254a3bc6c 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -470,6 +470,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "allm_passthrough_route", "llm_passthrough_route", "asearch", + "adecisions", "avector_store_create", "avector_store_search", "acreate_skill", diff --git a/tests/unit/proxy/decisions_endpoints/__init__.py b/tests/unit/proxy/decisions_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py new file mode 100644 index 00000000000..d5f57d85ffd --- /dev/null +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import Iterator, Mapping +from pathlib import Path +from typing import Final + +import pytest +import respx +from fastapi import FastAPI +from fastapi.routing import APIRoute +from fastapi.testclient import TestClient +from starlette.routing import Match + +import litellm +from litellm.proxy._lazy_features import attach_lazy_features +from litellm.proxy.decisions_endpoints.endpoints import decisions +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder +from litellm.proxy.proxy_server import ( + app, + cleanup_router_config_variables, + initialize, +) + +_INPUT_TOKENS: Final[int] = 367 +_OUTPUT_TOKENS: Final[int] = 3 +_RESPONSE: Final[Mapping[str, object]] = { + "model": "pplx-decider-v1-27b", + "answers": { + "is_defect": {"type": "noul", "noul": 0.9}, + }, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, +} +_STRANDS_RESPONSE: Final[Mapping[str, object]] = { + "model": "strands-decider-2B-hobson-v19", + "answers": { + "is_defect": {"type": "noul", "noul": 0.9}, + }, + "usage": {"input_tokens": 216, "output_tokens": 3}, + "latency_ms": 3722.17, +} +_REQUEST: Final[Mapping[str, object]] = { + "model": "decider", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, +} + + +@pytest.fixture +def client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + monkeypatch.setenv("OPENAI_API_KEY", "fake-openai-key") + monkeypatch.setenv("OPENAI_API_BASE", "https://fake-openai.example") + monkeypatch.setenv("REDIS_HOST", "localhost") + cleanup_router_config_variables() + config_path: Final = Path(__file__).parents[1] / "test_configs" / "test_config_no_auth.yaml" + asyncio.run(initialize(config=str(config_path), debug=True)) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "decider", + "litellm_params": { + "model": "perplexity/pplx-decider-v1-27b", + "api_key": "test-key", + }, + } + ] + ), + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield TestClient(app) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions")) +def test_proxy_decisions_route_returns_answers_and_cost( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post(endpoint, json=_REQUEST) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == _RESPONSE["answers"] + assert "_hidden_params" not in response.json() + perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] + expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + perplexity_cost["output_cost_per_token"] + ) + + assert expected_cost > 0 + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) + assert upstream.called + assert json.loads(upstream.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert upstream.calls[0].request.headers["authorization"] == "Bearer test-key" + + +def test_proxy_decisions_dispatches_typesafe_deployment( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": { + "model": "typesafe/jev-latest", + "api_key": "k", + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) + + response: Final = client.post( + "/v1/decisions", + json={ + "model": "jev", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == _RESPONSE["answers"] + assert upstream.called + assert json.loads(upstream.calls[0].request.content) == { + "model": "jev-latest", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + +def test_proxy_decisions_sends_the_env_key_to_the_deployment_api_base( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("PERPLEXITYAI_API_KEY", "server-key") + monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "decider", + "litellm_params": { + "model": "perplexity/pplx-decider-v1-27b", + "api_base": "https://egress.example/perplexity", + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + upstream: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post("/v1/decisions", json=_REQUEST) + + assert response.status_code == 200, response.text + assert upstream.call_count == 1 + assert upstream.calls[0].request.headers["authorization"] == "Bearer server-key" + + +def test_proxy_decisions_unknown_model_is_a_client_error( + client: TestClient, + respx_mock: respx.MockRouter, +) -> None: + response: Final = client.post( + "/v1/decisions", + json={ + "model": "missing-model", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + ) + + assert 400 <= response.status_code < 500, response.text + assert len(respx_mock.calls) == 0 + + +@pytest.mark.parametrize( + "request_body", + ( + { + "model": "decider", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + { + "model": "decider", + "state": {"source": "proxy-test"}, + }, + ), + ids=("missing_state", "missing_questions"), +) +def test_proxy_decisions_missing_required_field_is_a_client_error( + client: TestClient, + respx_mock: respx.MockRouter, + request_body: Mapping[str, object], +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post("/v1/decisions", json=request_body) + + assert response.status_code == 400, response.text + assert not upstream.called + + +def test_proxy_decisions_dispatches_strands_decider( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "strands", + "litellm_params": { + "model": "strands_decider/strands-decider-2B-hobson-v19", + "api_base": "https://strands.example", + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + upstream: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + + response: Final = client.post( + "/v1/decisions", + json={ + "model": "strands", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == _STRANDS_RESPONSE["answers"] + assert upstream.called + assert json.loads(upstream.calls[0].request.content) == { + "model": "strands-decider-2B-hobson-v19", + "state": {"source": "proxy-test"}, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert "authorization" not in upstream.calls[0].request.headers + + +def test_proxy_decisions_without_model_uses_the_proxy_default_model( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm.proxy.proxy_server, "user_model", "decider") + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post( + "/v1/decisions", json={key: value for key, value in _REQUEST.items() if key != "model"} + ) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == _RESPONSE["answers"] + assert upstream.called + assert json.loads(upstream.calls[0].request.content)["model"] == "pplx-decider-v1-27b" + + +def _serving_endpoint(bare: FastAPI, path: str) -> object: + scope: Final = {"type": "http", "method": "POST", "path": path, "root_path": "", "query_string": b"", "headers": ()} + return next( + route.endpoint for route in bare.routes if isinstance(route, APIRoute) and route.matches(scope)[0] is Match.FULL + ) + + +def test_a_config_pass_through_at_v1_decisions_keeps_its_route_and_the_native_api_serves_decisions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LITELLM_DISABLE_LAZY_ROUTES", raising=False) + + async def pass_through() -> dict[str, str]: + return {"served_by": "pass-through"} + + bare: Final = FastAPI() + attach_lazy_features(bare) + SafeRouteAdder.add_api_route_if_not_exists(bare, "/v1/decisions", pass_through, ["POST"]) + with TestClient(bare) as client: + assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} + assert _serving_endpoint(bare, "/v1/decisions") is pass_through + assert _serving_endpoint(bare, "/decisions") is decisions diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 30b322f7b73..637cfb00d84 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4398,6 +4398,23 @@ export interface paths { patch?: never; trace?: never; }; + "/decisions": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Decisions */ + post: operations["decisions_decisions_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/delete/allowed_ip": { parameters: { query?: never; @@ -18644,6 +18661,23 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/decisions": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Decisions */ + post: operations["decisions_v1_decisions_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/embeddings": { parameters: { query?: never; @@ -26667,7 +26701,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ @@ -53175,6 +53209,26 @@ export interface operations { }; }; }; + decisions_decisions_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; delete_allowed_ip_delete_allowed_ip_post: { parameters: { query?: never; @@ -70747,6 +70801,26 @@ export interface operations { }; }; }; + decisions_v1_decisions_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; embeddings_v1_embeddings_post: { parameters: { query?: never;