mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
feat(decisions): add unified /v1/decisions endpoint for Jev-compatible providers (#44236)
Backport of #44236 to stable/1.104.x.
Cherry-picked from 8b1990b4bc (main). Adapted to this line: the lazy feature
helpers are ported by hand, managed agent authorization and the integration suites are not on
this line, and the lazy-disabled precedence test is dropped since attach_lazy_features takes no
feature list here.
This commit is contained in:
parent
48963be81b
commit
bc4b7fd42c
40 changed files with 2029 additions and 28 deletions
3
.github/workflows/test-unit.yml
vendored
3
.github/workflows/test-unit.yml
vendored
|
|
@ -61,7 +61,7 @@ jobs:
|
|||
|
||||
- shard: core-utils
|
||||
artifact-name: core-utils
|
||||
test-path: ""
|
||||
test-path: tests/unit/decisions
|
||||
unit-flag: core-utils
|
||||
workers: 2
|
||||
reruns: 1
|
||||
|
|
@ -130,6 +130,7 @@ jobs:
|
|||
artifact-name: proxy-endpoints
|
||||
test-path: >-
|
||||
tests/test_litellm/proxy/analytics_endpoints
|
||||
tests/unit/proxy/decisions_endpoints
|
||||
tests/test_litellm/proxy/management_endpoints
|
||||
tests/test_litellm/proxy/list_api
|
||||
tests/test_litellm/proxy/memory
|
||||
|
|
|
|||
|
|
@ -364,11 +364,13 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
|
|||
| [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | |
|
||||
| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
|
||||
| [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Typesafe Decisions API (`typesafe`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | |
|
||||
| [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""Path allowlist for the gateway component.
|
||||
|
||||
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
|
||||
audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
|
||||
audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image,
|
||||
responses, vector stores, passthrough providers, realtime websockets, MCP
|
||||
tool-call endpoints, and operational endpoints (/health, /metrics, and the
|
||||
/debug/memory/summary read of the serving worker's RSS).
|
||||
|
|
@ -60,6 +60,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
|||
"/v1/rerank",
|
||||
"/v2/rerank",
|
||||
"/rerank",
|
||||
"/v1/decisions",
|
||||
"/decisions",
|
||||
"/v1/ocr",
|
||||
"/ocr",
|
||||
"/v1/rag/",
|
||||
|
|
|
|||
|
|
@ -1467,6 +1467,7 @@ from .embeddings.dispatch import *
|
|||
from .rust_bridge import rust
|
||||
from .rag.main import *
|
||||
from .sandbox.main import *
|
||||
from .decisions.main import *
|
||||
from .search.main import *
|
||||
from .realtime_api.main import (
|
||||
_arealtime,
|
||||
|
|
|
|||
|
|
@ -98,6 +98,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro
|
|||
from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.agents import LiteLLMSendMessageResponse
|
||||
from litellm.types.decisions import DecisionsResponse, DecisionsUsage
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.openai import (
|
||||
HttpxBinaryResponseContent,
|
||||
|
|
@ -1016,6 +1017,7 @@ def _is_known_usage_objects(usage_obj):
|
|||
return (
|
||||
isinstance(usage_obj, litellm.Usage)
|
||||
or isinstance(usage_obj, ResponseAPIUsage)
|
||||
or isinstance(usage_obj, DecisionsUsage)
|
||||
or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj)
|
||||
)
|
||||
|
||||
|
|
@ -1433,7 +1435,12 @@ def completion_cost(
|
|||
"usage",
|
||||
litellm.Usage(**_usage_for_dump.model_dump()),
|
||||
)
|
||||
if usage_obj is None:
|
||||
if isinstance(usage_obj, DecisionsUsage):
|
||||
_usage = {
|
||||
"prompt_tokens": usage_obj.input_tokens,
|
||||
"completion_tokens": usage_obj.output_tokens,
|
||||
}
|
||||
elif usage_obj is None:
|
||||
_usage = {}
|
||||
elif isinstance(usage_obj, BaseModel):
|
||||
_usage = cast(BaseModel, usage_obj).model_dump()
|
||||
|
|
@ -1933,7 +1940,8 @@ def response_cost_calculator(
|
|||
| LiteLLMRealtimeStreamLoggingObject
|
||||
| OpenAIModerationResponse
|
||||
| Response
|
||||
| SearchResponse,
|
||||
| SearchResponse
|
||||
| DecisionsResponse,
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
call_type: Literal[
|
||||
|
|
@ -1955,6 +1963,8 @@ def response_cost_calculator(
|
|||
"arerank",
|
||||
"search",
|
||||
"asearch",
|
||||
"decisions",
|
||||
"adecisions",
|
||||
],
|
||||
optional_params: dict,
|
||||
cache_hit: bool | None = None,
|
||||
|
|
|
|||
3
litellm/decisions/__init__.py
Normal file
3
litellm/decisions/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from litellm.decisions.main import adecisions, decisions
|
||||
|
||||
__all__ = ["adecisions", "decisions"]
|
||||
299
litellm/decisions/main.py
Normal file
299
litellm/decisions/main.py
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig
|
||||
from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT
|
||||
from litellm.llms.custom_httpx.http_handler import _get_httpx_client, get_async_httpx_client
|
||||
from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT
|
||||
from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT
|
||||
from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT
|
||||
from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_ENDPOINT
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.decisions import (
|
||||
DecisionQuestion,
|
||||
DecisionsJSON,
|
||||
DecisionsRequest,
|
||||
DecisionsResponse,
|
||||
)
|
||||
from litellm.utils import client
|
||||
|
||||
DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxyType(
|
||||
{
|
||||
"perplexity": PERPLEXITY_DECISIONS_ENDPOINT,
|
||||
"typesafe": TYPESAFE_DECISIONS_ENDPOINT,
|
||||
"openrouter": OPENROUTER_DECISIONS_ENDPOINT,
|
||||
"cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT,
|
||||
"strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT,
|
||||
}
|
||||
)
|
||||
|
||||
_DECISIONS_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequest]] = TypeAdapter(DecisionsRequest)
|
||||
_DECISIONS_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
|
||||
_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, repr=False)
|
||||
class _PreparedDecisionsRequest:
|
||||
config: DecisionsProviderConfig
|
||||
provider: str
|
||||
upstream_model: str
|
||||
url: str
|
||||
api_key: str | None = field(repr=False)
|
||||
headers: Mapping[str, str] = field(repr=False)
|
||||
body: Mapping[str, object] = field(repr=False)
|
||||
|
||||
|
||||
def _resolve_provider_model(model: str, custom_llm_provider: str | None) -> tuple[str, str]:
|
||||
provider: Final = model.partition("/")[0] if custom_llm_provider is None else custom_llm_provider
|
||||
if provider not in DECISIONS_ENDPOINTS:
|
||||
supported: Final = ", ".join(DECISIONS_ENDPOINTS)
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Unknown Decisions provider '{provider}'. Supported providers: {supported}",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
upstream_model: Final = model.removeprefix(f"{provider}/") if model.startswith(f"{provider}/") else model
|
||||
if not upstream_model:
|
||||
raise litellm.BadRequestError(
|
||||
message="A model name is required for the Decisions API",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
return provider, upstream_model
|
||||
|
||||
|
||||
def _resolve_api_key(
|
||||
*,
|
||||
provider: str,
|
||||
model: str,
|
||||
endpoint: DecisionsProviderConfig,
|
||||
api_key: str | None,
|
||||
) -> str | None:
|
||||
if api_key is not None:
|
||||
return api_key
|
||||
|
||||
server_api_key: Final = next(
|
||||
(key for key in (get_secret_str(name) for name in endpoint.api_key_env) if key),
|
||||
None,
|
||||
)
|
||||
if server_api_key is None:
|
||||
if not endpoint.api_key_required:
|
||||
return None
|
||||
raise litellm.AuthenticationError(
|
||||
message=f"Missing API key for Decisions provider '{provider}'",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
|
||||
return server_api_key
|
||||
|
||||
|
||||
def _prepare_request(
|
||||
*,
|
||||
model: str,
|
||||
state: DecisionsJSON,
|
||||
questions: Mapping[str, DecisionQuestion | Mapping[str, object]],
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
custom_llm_provider: str | None,
|
||||
extra_headers: Mapping[str, str] | None,
|
||||
) -> _PreparedDecisionsRequest:
|
||||
provider, upstream_model = _resolve_provider_model(model, custom_llm_provider)
|
||||
try:
|
||||
validated_request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python(
|
||||
{"model": model, "state": state, "questions": questions}
|
||||
)
|
||||
except ValidationError as error:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Invalid Decisions request: {error}",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
) from error
|
||||
|
||||
endpoint: Final = DECISIONS_ENDPOINTS[provider]
|
||||
env_api_base: Final = get_secret_str(endpoint.api_base_env)
|
||||
default_api_base: Final = endpoint.default_api_base()
|
||||
resolved_api_base: Final = api_base or env_api_base or default_api_base
|
||||
if resolved_api_base is None:
|
||||
raise litellm.BadRequestError(
|
||||
message=endpoint.missing_api_base_message(provider),
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
|
||||
resolved_api_key: Final = _resolve_api_key(
|
||||
provider=provider,
|
||||
model=model,
|
||||
endpoint=endpoint,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
canonical_model: Final = endpoint.canonical_model(upstream_model)
|
||||
outbound_headers: Final = MappingProxyType(
|
||||
{
|
||||
**{
|
||||
name: value
|
||||
for name, value in (extra_headers or {}).items()
|
||||
if name.lower() not in {"authorization", "content-type"}
|
||||
},
|
||||
**({"Authorization": f"Bearer {resolved_api_key}"} if resolved_api_key is not None else {}),
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
)
|
||||
body: Final = MappingProxyType(
|
||||
{
|
||||
"model": endpoint.request_model(canonical_model),
|
||||
"state": validated_request.state,
|
||||
"questions": {
|
||||
name: question.model_dump(mode="json", exclude_none=True)
|
||||
for name, question in validated_request.questions.items()
|
||||
},
|
||||
}
|
||||
)
|
||||
return _PreparedDecisionsRequest(
|
||||
config=endpoint,
|
||||
provider=provider,
|
||||
upstream_model=canonical_model,
|
||||
url=endpoint.endpoint_url(resolved_api_base, canonical_model),
|
||||
api_key=resolved_api_key,
|
||||
headers=outbound_headers,
|
||||
body=body,
|
||||
)
|
||||
|
||||
|
||||
def _log_request(
|
||||
prepared: _PreparedDecisionsRequest,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> LiteLLMLoggingObj | None:
|
||||
logging_obj: Final = kwargs.get("litellm_logging_obj")
|
||||
if not isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
return None
|
||||
logging_obj.update_from_kwargs(
|
||||
kwargs=dict(kwargs),
|
||||
model=prepared.upstream_model,
|
||||
litellm_params={
|
||||
"litellm_call_id": kwargs.get("litellm_call_id"),
|
||||
"api_base": prepared.url,
|
||||
},
|
||||
custom_llm_provider=prepared.provider,
|
||||
)
|
||||
request_body: Final = dict(prepared.body)
|
||||
request_headers: Final = dict(prepared.headers)
|
||||
logging_obj.pre_call(
|
||||
input=request_body,
|
||||
api_key=prepared.api_key,
|
||||
model=prepared.upstream_model,
|
||||
additional_args={
|
||||
"api_base": prepared.url,
|
||||
"complete_input_dict": request_body,
|
||||
"headers": request_headers,
|
||||
},
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _parse_response(
|
||||
response: httpx.Response,
|
||||
prepared: _PreparedDecisionsRequest,
|
||||
) -> DecisionsResponse:
|
||||
response.raise_for_status()
|
||||
payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content)
|
||||
result: Final = _DECISIONS_RESPONSE_ADAPTER.validate_python(prepared.config.unwrap_response(payload))
|
||||
result._hidden_params.update(
|
||||
{
|
||||
"model": f"{prepared.provider}/{prepared.upstream_model}",
|
||||
"custom_llm_provider": prepared.provider,
|
||||
"provider_response_model": f"{prepared.provider}/{prepared.upstream_model}",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _map_upstream_exception(error: Exception, prepared: _PreparedDecisionsRequest) -> Exception:
|
||||
return litellm.exception_type(
|
||||
model=f"{prepared.provider}/{prepared.upstream_model}",
|
||||
custom_llm_provider=prepared.provider,
|
||||
original_exception=error,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
async def adecisions(
|
||||
model: str,
|
||||
state: DecisionsJSON,
|
||||
questions: Mapping[str, DecisionQuestion | Mapping[str, object]],
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: Mapping[str, str] | None = None,
|
||||
**kwargs: object,
|
||||
) -> DecisionsResponse:
|
||||
prepared: Final = _prepare_request(
|
||||
model=model,
|
||||
state=state,
|
||||
questions=questions,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
extra_headers=extra_headers,
|
||||
)
|
||||
logging_obj: Final = _log_request(prepared, kwargs)
|
||||
try:
|
||||
handler: Final = get_async_httpx_client(llm_provider=prepared.provider)
|
||||
response: Final = await handler.post(
|
||||
prepared.url,
|
||||
json=dict(prepared.body),
|
||||
headers=dict(prepared.headers),
|
||||
timeout=timeout,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return _parse_response(response=response, prepared=prepared)
|
||||
except Exception as error:
|
||||
raise _map_upstream_exception(error, prepared) from error
|
||||
|
||||
|
||||
@client
|
||||
def decisions(
|
||||
model: str,
|
||||
state: DecisionsJSON,
|
||||
questions: Mapping[str, DecisionQuestion | Mapping[str, object]],
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: Mapping[str, str] | None = None,
|
||||
**kwargs: object,
|
||||
) -> DecisionsResponse:
|
||||
prepared: Final = _prepare_request(
|
||||
model=model,
|
||||
state=state,
|
||||
questions=questions,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
extra_headers=extra_headers,
|
||||
)
|
||||
logging_obj: Final = _log_request(prepared, kwargs)
|
||||
try:
|
||||
handler: Final = _get_httpx_client()
|
||||
response: Final = handler.post(
|
||||
prepared.url,
|
||||
json=dict(prepared.body),
|
||||
headers=dict(prepared.headers),
|
||||
timeout=timeout,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return _parse_response(response=response, prepared=prepared)
|
||||
except Exception as error:
|
||||
raise _map_upstream_exception(error, prepared) from error
|
||||
|
||||
|
||||
__all__ = ["DECISIONS_ENDPOINTS", "adecisions", "decisions"]
|
||||
|
|
@ -170,6 +170,7 @@ class HealthCheckHelpers:
|
|||
"batch",
|
||||
"responses",
|
||||
"ocr",
|
||||
"evaluation",
|
||||
],
|
||||
Callable,
|
||||
]:
|
||||
|
|
@ -192,7 +193,7 @@ class HealthCheckHelpers:
|
|||
from litellm.litellm_core_utils.audio_utils.utils import (
|
||||
get_audio_file_for_health_check,
|
||||
)
|
||||
from litellm.litellm_core_utils.health_check_utils import _filter_model_params
|
||||
from litellm.litellm_core_utils.health_check_utils import DECISIONS_CALL_PARAMS, _filter_model_params
|
||||
from litellm.realtime_api.main import _realtime_health_check
|
||||
|
||||
return {
|
||||
|
|
@ -259,4 +260,13 @@ class HealthCheckHelpers:
|
|||
**_filter_model_params(model_params=model_params),
|
||||
document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider),
|
||||
),
|
||||
"evaluation": lambda: litellm.adecisions(
|
||||
**DECISIONS_CALL_PARAMS.validate_python(
|
||||
{
|
||||
"state": prompt or "health check",
|
||||
"questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}},
|
||||
**_filter_model_params(model_params=model_params),
|
||||
}
|
||||
)
|
||||
),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,12 @@ Utils used for litellm.ahealth_check()
|
|||
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.decisions import DecisionsCallParams
|
||||
|
||||
DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams)
|
||||
|
||||
|
||||
def _filter_model_params(model_params: dict) -> dict:
|
||||
"""Remove 'messages' param from model params."""
|
||||
|
|
|
|||
|
|
@ -118,6 +118,7 @@ from litellm.llms.base_llm.search.transformation import SearchResponse
|
|||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.agents import LiteLLMSendMessageResponse
|
||||
from litellm.types.containers.main import ContainerObject
|
||||
from litellm.types.decisions import DecisionsResponse
|
||||
from litellm.types.interactions import (
|
||||
InteractionsAPIResponse,
|
||||
InteractionsAPIStreamingResponse,
|
||||
|
|
@ -1815,6 +1816,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
LiteLLMRealtimeStreamLoggingObject,
|
||||
OpenAIModerationResponse,
|
||||
"SearchResponse",
|
||||
DecisionsResponse,
|
||||
dict,
|
||||
list,
|
||||
],
|
||||
|
|
@ -2592,6 +2594,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
or isinstance(logging_result, OpenAIModerationResponse)
|
||||
or isinstance(logging_result, OCRResponse) # OCR
|
||||
or isinstance(logging_result, SearchResponse) # Search API
|
||||
or isinstance(logging_result, DecisionsResponse)
|
||||
or (
|
||||
isinstance(logging_result, InteractionsAPIResponse)
|
||||
and logging_result.usage is not None
|
||||
|
|
|
|||
3
litellm/llms/base_llm/decisions/__init__.py
Normal file
3
litellm/llms/base_llm/decisions/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from .transformation import DecisionsProviderConfig, JevCompatibleDecisionsEndpoint
|
||||
|
||||
__all__ = ["DecisionsProviderConfig", "JevCompatibleDecisionsEndpoint"]
|
||||
52
litellm/llms/base_llm/decisions/transformation.py
Normal file
52
litellm/llms/base_llm/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class JevCompatibleDecisionsEndpoint:
|
||||
default_api_base_value: str | None
|
||||
path: str
|
||||
api_key_env: tuple[str, ...]
|
||||
api_base_env: str
|
||||
api_key_required: bool = True
|
||||
|
||||
def default_api_base(self) -> str | None:
|
||||
return self.default_api_base_value
|
||||
|
||||
def missing_api_base_message(self, provider: str) -> str:
|
||||
return f"api_base is required for Decisions provider '{provider}'"
|
||||
|
||||
def canonical_model(self, model: str) -> str:
|
||||
return model
|
||||
|
||||
def request_model(self, model: str) -> str:
|
||||
return model
|
||||
|
||||
def endpoint_url(self, api_base: str, model: str) -> str:
|
||||
return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}"
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
return payload
|
||||
|
||||
|
||||
class DecisionsProviderConfig(Protocol):
|
||||
@property
|
||||
def api_key_env(self) -> tuple[str, ...]: ...
|
||||
|
||||
@property
|
||||
def api_base_env(self) -> str: ...
|
||||
|
||||
@property
|
||||
def api_key_required(self) -> bool: ...
|
||||
|
||||
def default_api_base(self) -> str | None: ...
|
||||
|
||||
def missing_api_base_message(self, provider: str) -> str: ...
|
||||
|
||||
def canonical_model(self, model: str) -> str: ...
|
||||
|
||||
def request_model(self, model: str) -> str: ...
|
||||
|
||||
def endpoint_url(self, api_base: str, model: str) -> str: ...
|
||||
|
||||
def unwrap_response(self, payload: object) -> object: ...
|
||||
58
litellm/llms/cloudflare/decisions/transformation.py
Normal file
58
litellm/llms/cloudflare/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.secret_managers.main import (
|
||||
get_secret_str,
|
||||
normalize_nonempty_secret_str,
|
||||
)
|
||||
|
||||
_RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CloudflareDecisionsEndpoint:
|
||||
api_key_env: tuple[str, ...] = ("CLOUDFLARE_API_KEY",)
|
||||
api_base_env: str = "CLOUDFLARE_API_BASE"
|
||||
api_key_required: bool = True
|
||||
|
||||
def default_api_base(self) -> str | None:
|
||||
account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID"))
|
||||
if account_id is None:
|
||||
return None
|
||||
return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run"
|
||||
|
||||
def missing_api_base_message(self, provider: str) -> str:
|
||||
return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly"
|
||||
|
||||
def canonical_model(self, model: str) -> str:
|
||||
if model.startswith("@cf/"):
|
||||
return model
|
||||
return f"@cf/cloudflare/{model}"
|
||||
|
||||
def request_model(self, model: str) -> str:
|
||||
return model.rsplit("/", maxsplit=1)[-1]
|
||||
|
||||
def endpoint_url(self, api_base: str, model: str) -> str:
|
||||
normalized_api_base: Final = api_base.rstrip("/")
|
||||
if normalized_api_base.endswith("/ai/v1"):
|
||||
return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}"
|
||||
if normalized_api_base.endswith("/ai/run"):
|
||||
return f"{normalized_api_base}/{model}"
|
||||
return f"{normalized_api_base}/ai/run/{model}"
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
if not isinstance(payload, Mapping):
|
||||
return payload
|
||||
response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload)
|
||||
if "answers" in response_mapping:
|
||||
return payload
|
||||
result: Final = response_mapping.get("result")
|
||||
if isinstance(result, Mapping):
|
||||
return result
|
||||
return payload
|
||||
|
||||
|
||||
CLOUDFLARE_DECISIONS_ENDPOINT: Final[CloudflareDecisionsEndpoint] = CloudflareDecisionsEndpoint()
|
||||
10
litellm/llms/openrouter/decisions/transformation.py
Normal file
10
litellm/llms/openrouter/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
|
||||
|
||||
OPENROUTER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
|
||||
default_api_base_value="https://openrouter.ai/api",
|
||||
path="/alpha/decisions",
|
||||
api_key_env=("OPENROUTER_API_KEY",),
|
||||
api_base_env="OPENROUTER_API_BASE",
|
||||
)
|
||||
10
litellm/llms/perplexity/decisions/transformation.py
Normal file
10
litellm/llms/perplexity/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
|
||||
|
||||
PERPLEXITY_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
|
||||
default_api_base_value="https://api.perplexity.ai",
|
||||
path="/v1/decisions",
|
||||
api_key_env=("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"),
|
||||
api_base_env="PERPLEXITY_API_BASE",
|
||||
)
|
||||
11
litellm/llms/strands_decider/decisions/transformation.py
Normal file
11
litellm/llms/strands_decider/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
|
||||
|
||||
STRANDS_DECIDER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
|
||||
default_api_base_value=None,
|
||||
path="/v1/systemone",
|
||||
api_key_env=("STRANDS_DECIDER_API_KEY",),
|
||||
api_base_env="STRANDS_DECIDER_API_BASE",
|
||||
api_key_required=False,
|
||||
)
|
||||
10
litellm/llms/typesafe/decisions/transformation.py
Normal file
10
litellm/llms/typesafe/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
|
||||
|
||||
TYPESAFE_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
|
||||
default_api_base_value="https://api.typesafe.ai",
|
||||
path="/v1/systemone",
|
||||
api_key_env=("TYPESAFE_API_KEY",),
|
||||
api_base_env="TYPESAFE_API_BASE",
|
||||
)
|
||||
|
|
@ -77427,5 +77427,55 @@
|
|||
"source": "https://aws.amazon.com/bedrock/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": false
|
||||
},
|
||||
"cloudflare/clef": {
|
||||
"input_cost_per_token": 2.4e-07,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef": {
|
||||
"input_cost_per_token": 2.4e-07,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"perplexity/pplx-decider-v1-27b": {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
|
||||
},
|
||||
"strands_decider/strands-decider-2B-hobson-v19": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "strands_decider",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19",
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import importlib
|
|||
from collections.abc import Callable, Mapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
|
|
@ -251,6 +252,11 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
|||
module_path="litellm.proxy.openai_evals_endpoints.endpoints",
|
||||
path_prefixes=("/v1/evals", "/evals"),
|
||||
),
|
||||
LazyFeature(
|
||||
name="decisions",
|
||||
module_path="litellm.proxy.decisions_endpoints.endpoints",
|
||||
path_prefixes=("/v1/decisions", "/decisions"),
|
||||
),
|
||||
LazyFeature(
|
||||
name="claude_code_marketplace",
|
||||
module_path="litellm.proxy.anthropic_endpoints.claude_code_endpoints",
|
||||
|
|
@ -361,6 +367,10 @@ def _lazy_slots(app: "FastAPI") -> Mapping[str, BaseRoute | None]:
|
|||
return app.state.lazy_slots if hasattr(app.state, "lazy_slots") else MappingProxyType({})
|
||||
|
||||
|
||||
def _lazy_routes(app: "FastAPI") -> Mapping[str, tuple[BaseRoute, ...]]:
|
||||
return app.state.lazy_routes if hasattr(app.state, "lazy_routes") else MappingProxyType({})
|
||||
|
||||
|
||||
def reserve_lazy_slot(app: "FastAPI", name: str, features: tuple[LazyFeature, ...] = LAZY_FEATURES) -> None:
|
||||
"""Record the route the feature's router used to be included after, so its routes
|
||||
are spliced back in there once it loads and keep the same precedence. Anchoring on
|
||||
|
|
@ -512,6 +522,13 @@ def _make_warmup_router(app: "FastAPI") -> "APIRouter":
|
|||
return router
|
||||
|
||||
|
||||
def lazy_owned_routes(app: "FastAPI") -> frozenset[int]:
|
||||
"""ids of the routes lazy features have registered on this app. A route added later at
|
||||
one of their paths (a config pass-through at /v1/decisions) goes ahead of them, the
|
||||
precedence lazy mode gives it when the feature has not loaded by the time the config is read."""
|
||||
return frozenset(id(route) for route in chain.from_iterable(_lazy_routes(app).values()))
|
||||
|
||||
|
||||
def loaded_lazy_modules(app: "FastAPI") -> frozenset[str]:
|
||||
"""The set of lazy feature modules whose routers are actually registered
|
||||
on this app (tracked by _force_load), empty before the middleware ever ran.
|
||||
|
|
|
|||
|
|
@ -8062,6 +8062,61 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"decisions": {
|
||||
"components": {
|
||||
"schemas": {}
|
||||
},
|
||||
"paths": {
|
||||
"/decisions": {
|
||||
"post": {
|
||||
"operationId": "decisions_decisions_post",
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Decisions",
|
||||
"tags": [
|
||||
"decisions"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/v1/decisions": {
|
||||
"post": {
|
||||
"operationId": "decisions_v1_decisions_post",
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Decisions",
|
||||
"tags": [
|
||||
"decisions"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"evals": {
|
||||
"components": {
|
||||
"schemas": {
|
||||
|
|
|
|||
|
|
@ -466,6 +466,8 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/v1/search",
|
||||
"/search/{search_tool_name}",
|
||||
"/v1/search/{search_tool_name}",
|
||||
"/decisions",
|
||||
"/v1/decisions",
|
||||
# OCR
|
||||
"/ocr",
|
||||
"/v1/ocr",
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@ ProxyRouteType: TypeAlias = Literal[
|
|||
"avector_store_file_delete",
|
||||
"aocr",
|
||||
"asearch",
|
||||
"adecisions",
|
||||
"avideo_generation",
|
||||
"avideo_list",
|
||||
"avideo_status",
|
||||
|
|
@ -1942,6 +1943,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"avector_store_file_delete",
|
||||
"aocr",
|
||||
"asearch",
|
||||
"adecisions",
|
||||
"avideo_generation",
|
||||
"avideo_list",
|
||||
"avideo_status",
|
||||
|
|
|
|||
1
litellm/proxy/decisions_endpoints/__init__.py
Normal file
1
litellm/proxy/decisions_endpoints/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
__all__ = ()
|
||||
102
litellm/proxy/decisions_endpoints/endpoints.py
Normal file
102
litellm/proxy/decisions_endpoints/endpoints.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
from typing import Annotated, Final
|
||||
|
||||
from fastapi import APIRouter, Depends, Request, Response
|
||||
from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.types.decisions import DecisionsRequestBody
|
||||
|
||||
router: Final = APIRouter()
|
||||
_REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||
_DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody)
|
||||
_GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||
_OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
|
||||
_OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/decisions",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||
tags=["decisions"],
|
||||
)
|
||||
@router.post(
|
||||
"/decisions",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||
tags=["decisions"],
|
||||
)
|
||||
async def decisions(
|
||||
request: Request,
|
||||
fastapi_response: Response,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
):
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings as proxy_general_settings,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
llm_router,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
user_max_tokens,
|
||||
user_request_timeout,
|
||||
version,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
user_api_base as proxy_user_api_base,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
user_model as proxy_user_model,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
user_temperature as proxy_user_temperature,
|
||||
)
|
||||
|
||||
data: Final = _REQUEST_DATA_ADAPTER.validate_json(await request.body())
|
||||
general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings)
|
||||
user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base)
|
||||
user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model)
|
||||
user_temperature: Final = _OPTIONAL_FLOAT_ADAPTER.validate_python(proxy_user_temperature)
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data=data)
|
||||
try:
|
||||
_DECISIONS_REQUEST_BODY_ADAPTER.validate_python(data)
|
||||
return await processor.base_process_llm_request(
|
||||
request=request,
|
||||
fastapi_response=fastapi_response,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
route_type="adecisions",
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
proxy_config=proxy_config,
|
||||
select_data_generator=None,
|
||||
model=None,
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
except ValidationError as error:
|
||||
bad_request_error: Final = BadRequestError(
|
||||
message=f"Invalid Decisions request: {error}",
|
||||
model=str(data.get("model", "")),
|
||||
llm_provider="",
|
||||
)
|
||||
raise await processor._handle_llm_api_exception(
|
||||
e=bad_request_error,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
version=version,
|
||||
)
|
||||
except Exception as error:
|
||||
raise await processor._handle_llm_api_exception(
|
||||
e=error,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
version=version,
|
||||
)
|
||||
|
|
@ -27,6 +27,7 @@ from fastapi import (
|
|||
)
|
||||
from fastapi.responses import StreamingResponse
|
||||
from starlette.datastructures import UploadFile as StarletteUploadFile
|
||||
from starlette.routing import BaseRoute, Route
|
||||
from starlette.websockets import WebSocketState
|
||||
from websockets.asyncio.client import connect
|
||||
from websockets.exceptions import (
|
||||
|
|
@ -65,6 +66,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
|
|||
)
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.passthrough import BasePassthroughUtils
|
||||
from litellm.proxy._lazy_features import lazy_owned_routes
|
||||
from litellm.proxy._types import (
|
||||
ConfigFieldInfo,
|
||||
ConfigFieldUpdate,
|
||||
|
|
@ -2868,35 +2870,34 @@ def _extract_model_from_vertex_ai_setup(setup_response: Mapping[str, object]) ->
|
|||
return None
|
||||
|
||||
|
||||
def _placed_ahead(routes: Sequence[BaseRoute], moving: BaseRoute, before: BaseRoute) -> tuple[BaseRoute, ...]:
|
||||
kept: Final = tuple(route for route in routes if route is not moving)
|
||||
at: Final = next(index for index, route in enumerate(kept) if route is before)
|
||||
return (*kept[:at], moving, *kept[at:])
|
||||
|
||||
|
||||
class SafeRouteAdder:
|
||||
"""
|
||||
Wrapper class for adding routes to FastAPI app.
|
||||
Only adds routes if they don't already exist on the app.
|
||||
Only adds routes if they don't already exist on the app. A route a lazy feature registered
|
||||
does not count: a route added at its path goes ahead of it, the precedence a config
|
||||
pass-through at /v1/decisions gets in lazy mode, where the feature has not loaded yet.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _colliding_routes(app: FastAPI, path: str, methods: Sequence[str]) -> tuple[Route, ...]:
|
||||
wanted: Final = frozenset(methods)
|
||||
return tuple(
|
||||
route
|
||||
for route in app.routes
|
||||
if isinstance(route, Route) and route.path == path and not wanted.isdisjoint(route.methods or ())
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_path_registered(app: FastAPI, path: str, methods: list[str]) -> bool:
|
||||
"""
|
||||
Check if a path with any of the specified methods is already registered on the app.
|
||||
|
||||
Args:
|
||||
app: The FastAPI application instance
|
||||
path: The path to check (e.g., "/v1/chat/completions")
|
||||
methods: List of HTTP methods to check (e.g., ["GET", "POST"])
|
||||
|
||||
Returns:
|
||||
True if the path is already registered with any of the methods, False otherwise
|
||||
"""
|
||||
for route in app.routes:
|
||||
# Use getattr to safely access route attributes
|
||||
route_path = getattr(route, "path", None)
|
||||
route_methods = getattr(route, "methods", None)
|
||||
|
||||
if route_path == path and route_methods is not None:
|
||||
# Check if any of the methods overlap
|
||||
if any(method in route_methods for method in methods):
|
||||
return True
|
||||
return False
|
||||
"""True when a route the app itself defines already serves the path with one of the methods."""
|
||||
lazy_owned: Final = lazy_owned_routes(app)
|
||||
return any(id(route) not in lazy_owned for route in SafeRouteAdder._colliding_routes(app, path, methods))
|
||||
|
||||
@staticmethod
|
||||
def add_api_route_if_not_exists(
|
||||
|
|
@ -2927,12 +2928,17 @@ class SafeRouteAdder:
|
|||
)
|
||||
return False
|
||||
|
||||
shadowed: Final = SafeRouteAdder._colliding_routes(app, path, methods)
|
||||
app.add_api_route(
|
||||
path=path,
|
||||
endpoint=endpoint,
|
||||
methods=methods,
|
||||
dependencies=dependencies,
|
||||
)
|
||||
if shadowed:
|
||||
app.router.routes[:] = _placed_ahead( # rebind-ok: the app owns its route table
|
||||
app.router.routes, app.router.routes[-1], shadowed[0]
|
||||
)
|
||||
verbose_proxy_logger.debug(
|
||||
"Successfully added route: %s with methods %s",
|
||||
path,
|
||||
|
|
|
|||
|
|
@ -90,6 +90,7 @@ ROUTE_ENDPOINT_MAPPING: Final = {
|
|||
"acompact_responses": "/responses/compact",
|
||||
"aocr": "/ocr",
|
||||
"asearch": "/search",
|
||||
"adecisions": "/decisions",
|
||||
"avideo_generation": "/videos",
|
||||
"avideo_list": "/videos",
|
||||
"avideo_status": "/videos/{video_id}",
|
||||
|
|
@ -372,6 +373,7 @@ RouteType = Literal[
|
|||
"avector_store_file_delete",
|
||||
"aocr",
|
||||
"asearch",
|
||||
"adecisions",
|
||||
"avideo_generation",
|
||||
"avideo_list",
|
||||
"avideo_status",
|
||||
|
|
|
|||
|
|
@ -2047,6 +2047,11 @@ class Router:
|
|||
self.asearch = self.factory_function(asearch, call_type="asearch")
|
||||
self.search = self.factory_function(search, call_type="search")
|
||||
|
||||
from litellm.decisions import adecisions, decisions
|
||||
|
||||
self.adecisions = self.factory_function(adecisions, call_type="adecisions")
|
||||
self.decisions = self.factory_function(decisions, call_type="decisions")
|
||||
|
||||
def _initialize_video_endpoints(self):
|
||||
"""Initialize video endpoints."""
|
||||
from litellm.videos import (
|
||||
|
|
@ -6556,6 +6561,8 @@ class Router:
|
|||
"ocr",
|
||||
"asearch",
|
||||
"search",
|
||||
"adecisions",
|
||||
"decisions",
|
||||
"aadapter_generate_content",
|
||||
"avideo_generation",
|
||||
"video_generation",
|
||||
|
|
@ -6629,6 +6636,7 @@ class Router:
|
|||
"generate_content_stream",
|
||||
"ocr",
|
||||
"search",
|
||||
"decisions",
|
||||
"video_generation",
|
||||
"video_list",
|
||||
"video_status",
|
||||
|
|
@ -6796,6 +6804,7 @@ class Router:
|
|||
"agenerate_content_stream",
|
||||
"aocr",
|
||||
"ocr",
|
||||
"adecisions",
|
||||
"avideo_generation",
|
||||
"avideo_list",
|
||||
"avideo_status",
|
||||
|
|
|
|||
123
litellm/types/decisions.py
Normal file
123
litellm/types/decisions.py
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Annotated, Literal, TypeAlias
|
||||
|
||||
from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config
|
||||
from typing_extensions import ReadOnly, Required, TypedDict
|
||||
|
||||
from litellm.types.llms.base import LiteLLMPydanticObjectBase
|
||||
|
||||
DecisionsJSON: TypeAlias = str | Mapping[str, object] | Sequence[object]
|
||||
NoulCriteria: TypeAlias = Mapping[Literal["true", "false"], DecisionsJSON | None]
|
||||
|
||||
|
||||
class NoulQuestion(LiteLLMPydanticObjectBase):
|
||||
type: Literal["noul"]
|
||||
instructions: DecisionsJSON | None = None
|
||||
criteria: NoulCriteria | None = None
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_instructions_or_criteria(self) -> "NoulQuestion":
|
||||
if self.instructions is None and self.criteria is None:
|
||||
raise ValueError("A noul question requires instructions or criteria")
|
||||
return self
|
||||
|
||||
|
||||
class ChoiceQuestion(LiteLLMPydanticObjectBase):
|
||||
type: Literal["choice"]
|
||||
instructions: DecisionsJSON | None = None
|
||||
criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)]
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
||||
class ScoreQuestion(LiteLLMPydanticObjectBase):
|
||||
type: Literal["score"]
|
||||
instructions: DecisionsJSON | None = None
|
||||
criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)]
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
||||
DecisionQuestion: TypeAlias = Annotated[
|
||||
NoulQuestion | ChoiceQuestion | ScoreQuestion,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
DecisionQuestionMap: TypeAlias = Annotated[
|
||||
Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion],
|
||||
Field(min_length=1, max_length=128),
|
||||
]
|
||||
|
||||
|
||||
class DecisionsRequestBody(LiteLLMPydanticObjectBase):
|
||||
state: DecisionsJSON
|
||||
questions: DecisionQuestionMap
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
||||
class DecisionsRequest(DecisionsRequestBody):
|
||||
model: str
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class DecisionsCallParams(TypedDict, total=False):
|
||||
model: Required[ReadOnly[str]]
|
||||
state: Required[ReadOnly[DecisionsJSON]]
|
||||
questions: Required[ReadOnly[DecisionQuestionMap]]
|
||||
api_key: ReadOnly[str | None]
|
||||
api_base: ReadOnly[str | None]
|
||||
timeout: ReadOnly[float | None]
|
||||
custom_llm_provider: ReadOnly[str | None]
|
||||
extra_headers: ReadOnly[Mapping[str, str] | None]
|
||||
|
||||
|
||||
class NoulAnswer(LiteLLMPydanticObjectBase):
|
||||
type: Literal["noul"]
|
||||
noul: float
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
||||
class ChoiceAnswer(LiteLLMPydanticObjectBase):
|
||||
type: Literal["choice"]
|
||||
choice: str
|
||||
confidence: float
|
||||
probabilities: Mapping[str, float]
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
||||
class ScoreAnswer(LiteLLMPydanticObjectBase):
|
||||
type: Literal["score"]
|
||||
score: float
|
||||
confidence: float
|
||||
legend: Mapping[str, DecisionsJSON]
|
||||
probabilities: Mapping[str, float]
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
||||
DecisionAnswer: TypeAlias = Annotated[
|
||||
NoulAnswer | ChoiceAnswer | ScoreAnswer,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class DecisionsUsage(LiteLLMPydanticObjectBase):
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
||||
class DecisionsResponse(LiteLLMPydanticObjectBase):
|
||||
model: str | None = None
|
||||
answers: Mapping[str, DecisionAnswer]
|
||||
usage: DecisionsUsage | None = None
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
_hidden_params: dict[str, object] = PrivateAttr(default_factory=dict)
|
||||
|
|
@ -455,6 +455,8 @@ class CallTypes(str, Enum):
|
|||
arerank = "arerank"
|
||||
search = "search"
|
||||
asearch = "asearch"
|
||||
decisions = "decisions"
|
||||
adecisions = "adecisions"
|
||||
arealtime = "_arealtime"
|
||||
aresponses_websocket = "_aresponses_websocket"
|
||||
create_batch = "create_batch"
|
||||
|
|
@ -641,6 +643,8 @@ CallTypesLiteral = Literal[
|
|||
"arerank",
|
||||
"search",
|
||||
"asearch",
|
||||
"decisions",
|
||||
"adecisions",
|
||||
"_arealtime",
|
||||
"_aresponses_websocket",
|
||||
"create_batch",
|
||||
|
|
@ -746,6 +750,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
|
|||
# Search
|
||||
"/search": [CallTypes.asearch, CallTypes.search],
|
||||
"/v1/search": [CallTypes.asearch, CallTypes.search],
|
||||
"/decisions": [CallTypes.adecisions, CallTypes.decisions],
|
||||
"/v1/decisions": [CallTypes.adecisions, CallTypes.decisions],
|
||||
# Batches
|
||||
"/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
|
||||
"/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
|
||||
|
|
@ -4009,6 +4015,8 @@ class LlmProviders(str, Enum):
|
|||
OLLAMA_CHAT = "ollama_chat"
|
||||
DEEPINFRA = "deepinfra"
|
||||
PERPLEXITY = "perplexity"
|
||||
TYPESAFE = "typesafe"
|
||||
STRANDS_DECIDER = "strands_decider"
|
||||
MISTRAL = "mistral"
|
||||
MILVUS = "milvus"
|
||||
GROQ = "groq"
|
||||
|
|
|
|||
|
|
@ -1200,6 +1200,9 @@ def function_setup(
|
|||
if isinstance(search_query, list)
|
||||
else search_query
|
||||
)
|
||||
elif call_type in (CallTypes.decisions.value, CallTypes.adecisions.value):
|
||||
decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state", "")
|
||||
messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state)
|
||||
elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value):
|
||||
messages = args[1] if len(args) > 1 else kwargs.get("prompt")
|
||||
elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value):
|
||||
|
|
|
|||
|
|
@ -77427,5 +77427,55 @@
|
|||
"source": "https://aws.amazon.com/bedrock/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": false
|
||||
},
|
||||
"cloudflare/clef": {
|
||||
"input_cost_per_token": 2.4e-07,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef": {
|
||||
"input_cost_per_token": 2.4e-07,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"perplexity/pplx-decider-v1-27b": {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
|
||||
},
|
||||
"strands_decider/strands-decider-2B-hobson-v19": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "strands_decider",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19",
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2499,6 +2499,20 @@
|
|||
"image_variations": true
|
||||
}
|
||||
},
|
||||
"typesafe": {
|
||||
"display_name": "TypeSafe (`typesafe`)",
|
||||
"url": "https://docs.typesafe.ai/models",
|
||||
"endpoints": {
|
||||
"systemone": true
|
||||
}
|
||||
},
|
||||
"strands_decider": {
|
||||
"display_name": "Strands Decider (`strands_decider`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers",
|
||||
"endpoints": {
|
||||
"systemone": true
|
||||
}
|
||||
},
|
||||
"tavily": {
|
||||
"display_name": "Tavily (`tavily`)",
|
||||
"url": "https://docs.litellm.ai/docs/search/tavily",
|
||||
|
|
|
|||
|
|
@ -411,11 +411,13 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset(
|
|||
"sagemaker_nova",
|
||||
"scaleway",
|
||||
"stability",
|
||||
"strands_decider",
|
||||
"synthetic",
|
||||
"tencent",
|
||||
"tensormesh",
|
||||
"text-completion-inception",
|
||||
"transcribe",
|
||||
"typesafe",
|
||||
"valkey",
|
||||
"xiaomi_mimo",
|
||||
"zai",
|
||||
|
|
|
|||
0
tests/unit/decisions/__init__.py
Normal file
0
tests/unit/decisions/__init__.py
Normal file
592
tests/unit/decisions/test_main.py
Normal file
592
tests/unit/decisions/test_main.py
Normal file
|
|
@ -0,0 +1,592 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.types.decisions import (
|
||||
ChoiceAnswer,
|
||||
DecisionsResponse,
|
||||
DecisionsUsage,
|
||||
NoulAnswer,
|
||||
ScoreAnswer,
|
||||
)
|
||||
|
||||
_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType(
|
||||
{
|
||||
"is_defect": {"type": "noul", "instructions": "Is this a defect?", "provider_field": "kept"},
|
||||
"sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}},
|
||||
"severity": {"type": "score", "criteria": ["none", "low", "high"]},
|
||||
}
|
||||
)
|
||||
_INPUT_TOKENS: Final[int] = 367
|
||||
_OUTPUT_TOKENS: Final[int] = 3
|
||||
_RESPONSE: Final[Mapping[str, object]] = {
|
||||
"model": "jev-1.13",
|
||||
"answers": {
|
||||
"is_defect": {"type": "noul", "noul": 0.9},
|
||||
"sentiment": {
|
||||
"type": "choice",
|
||||
"choice": "positive",
|
||||
"confidence": 0.8,
|
||||
"probabilities": {"positive": 0.8, "negative": 0.2},
|
||||
},
|
||||
"severity": {
|
||||
"type": "score",
|
||||
"score": 1,
|
||||
"confidence": 0.7,
|
||||
"legend": {"0": "none", "1": "low", "2": "high"},
|
||||
"probabilities": {"0": 0.1, "1": 0.8, "2": 0.1},
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS},
|
||||
}
|
||||
_STRANDS_RESPONSE: Final[Mapping[str, object]] = {
|
||||
"model": "strands-decider-2B-hobson-v19",
|
||||
"answers": {
|
||||
"severity": {
|
||||
"type": "score",
|
||||
"score": 1,
|
||||
"confidence": 0.7,
|
||||
"legend": {"0": "none", "1": "low", "2": "high"},
|
||||
"probabilities": {"0": 0.1, "1": 0.8, "2": 0.1},
|
||||
}
|
||||
},
|
||||
"usage": {"input_tokens": 216, "output_tokens": 3},
|
||||
"latency_ms": 3722.17,
|
||||
}
|
||||
_PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = (
|
||||
(
|
||||
"perplexity",
|
||||
"perplexity/pplx-decider-v1-27b",
|
||||
"https://api.perplexity.ai/v1/decisions",
|
||||
"pplx-decider-v1-27b",
|
||||
),
|
||||
("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"),
|
||||
(
|
||||
"openrouter",
|
||||
"openrouter/typesafe/jev-1.13",
|
||||
"https://openrouter.ai/api/alpha/decisions",
|
||||
"typesafe/jev-1.13",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class _RecordingLogger(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.standard_logging_object: Mapping[str, object] | None = None
|
||||
|
||||
async def async_log_success_event(
|
||||
self,
|
||||
kwargs: Mapping[str, object],
|
||||
response_obj: object,
|
||||
start_time: object,
|
||||
end_time: object,
|
||||
) -> None:
|
||||
standard_logging_object: Final = kwargs.get("standard_logging_object")
|
||||
if isinstance(standard_logging_object, dict):
|
||||
self.standard_logging_object = standard_logging_object
|
||||
|
||||
|
||||
async def _drain_logging_worker() -> None:
|
||||
await asyncio.sleep(0)
|
||||
GLOBAL_LOGGING_WORKER.start()
|
||||
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("provider", "model", "url", "upstream_model"), _PROVIDERS)
|
||||
async def test_adecisions_sends_the_provider_wire_contract(
|
||||
provider: str,
|
||||
model: str,
|
||||
url: str,
|
||||
upstream_model: str,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model=model,
|
||||
state={"source": "unit-test"},
|
||||
questions=_QUESTIONS,
|
||||
api_key="caller-key",
|
||||
extra_headers={
|
||||
"x-request-tag": "decisions-test",
|
||||
"AUTHORIZATION": "attacker-key",
|
||||
"Content-Type": "text/plain",
|
||||
},
|
||||
internal_kwarg="must-not-leak",
|
||||
)
|
||||
|
||||
assert route.called
|
||||
assert len(respx_mock.calls) == 1
|
||||
request: Final = respx_mock.calls[0].request
|
||||
assert request.headers["authorization"] == "Bearer caller-key"
|
||||
assert request.headers["content-type"] == "application/json"
|
||||
assert request.headers["x-request-tag"] == "decisions-test"
|
||||
assert json.loads(request.content) == {
|
||||
"model": upstream_model,
|
||||
"state": {"source": "unit-test"},
|
||||
"questions": {
|
||||
"is_defect": {
|
||||
"type": "noul",
|
||||
"instructions": "Is this a defect?",
|
||||
"provider_field": "kept",
|
||||
},
|
||||
"sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}},
|
||||
"severity": {"type": "score", "criteria": ["none", "low", "high"]},
|
||||
},
|
||||
}
|
||||
assert isinstance(response.answers["is_defect"], NoulAnswer)
|
||||
assert isinstance(response.answers["sentiment"], ChoiceAnswer)
|
||||
assert isinstance(response.answers["severity"], ScoreAnswer)
|
||||
assert response._hidden_params["custom_llm_provider"] == provider
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_dispatches_typesafe_decisions_without_api_base(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
|
||||
monkeypatch.delenv("TYPESAFE_API_BASE", raising=False)
|
||||
provider_resolution: Final = litellm.get_llm_provider("typesafe/jev-latest")
|
||||
|
||||
assert provider_resolution[:2] == ("jev-latest", "typesafe")
|
||||
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "jev",
|
||||
"litellm_params": {
|
||||
"model": "typesafe/jev-latest",
|
||||
"api_key": "k",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = await router.adecisions(
|
||||
model="jev",
|
||||
state="router-test",
|
||||
questions={
|
||||
"sentiment": {
|
||||
"type": "choice",
|
||||
"criteria": {"positive": None, "negative": "unhappy"},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert upstream.called
|
||||
assert len(respx_mock.calls) == 1
|
||||
assert json.loads(respx_mock.calls[0].request.content) == {
|
||||
"model": "jev-latest",
|
||||
"state": "router-test",
|
||||
"questions": {
|
||||
"sentiment": {
|
||||
"type": "choice",
|
||||
"criteria": {"positive": None, "negative": "unhappy"},
|
||||
}
|
||||
},
|
||||
}
|
||||
assert respx_mock.calls[0].request.headers["authorization"] == "Bearer k"
|
||||
assert isinstance(response.answers["sentiment"], ChoiceAnswer)
|
||||
assert response.answers["sentiment"].choice == "positive"
|
||||
|
||||
|
||||
def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = litellm.decisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
assert route.called
|
||||
assert response.model == "jev-1.13"
|
||||
|
||||
|
||||
def test_openrouter_response_keeps_provider_fields(respx_mock: respx.MockRouter) -> None:
|
||||
payload: Final = {
|
||||
**_RESPONSE,
|
||||
"id": "decision-1",
|
||||
"provider": "typesafe",
|
||||
"usage": {**_RESPONSE["usage"], "cost": 0.25},
|
||||
}
|
||||
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=payload)
|
||||
|
||||
response: Final = litellm.decisions(
|
||||
model="openrouter/typesafe/jev-1.13",
|
||||
state="review",
|
||||
questions=_QUESTIONS,
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
assert response.model_extra["id"] == "decision-1"
|
||||
assert response.model_extra["provider"] == "typesafe"
|
||||
assert response.usage is not None
|
||||
assert response.usage.model_extra["cost"] == 0.25
|
||||
|
||||
|
||||
def test_decisions_cost_uses_litellm_token_pricing() -> None:
|
||||
response: Final = DecisionsResponse(
|
||||
model="pplx-decider-v1-27b",
|
||||
answers={},
|
||||
usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS),
|
||||
)
|
||||
response._hidden_params = {
|
||||
"model": "perplexity/pplx-decider-v1-27b",
|
||||
"custom_llm_provider": "perplexity",
|
||||
}
|
||||
|
||||
cost: Final = litellm.completion_cost(completion_response=response)
|
||||
perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"]
|
||||
expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float(
|
||||
perplexity_cost["output_cost_per_token"]
|
||||
)
|
||||
|
||||
assert expected_cost > 0
|
||||
assert cost == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.MockRouter) -> None:
|
||||
respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
recording_logger: Final = _RecordingLogger()
|
||||
original_callbacks: Final = litellm.callbacks
|
||||
litellm.callbacks = [recording_logger]
|
||||
|
||||
try:
|
||||
await litellm.adecisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_key="caller-key",
|
||||
)
|
||||
await _drain_logging_worker()
|
||||
finally:
|
||||
litellm.callbacks = original_callbacks
|
||||
|
||||
assert recording_logger.standard_logging_object is not None
|
||||
perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"]
|
||||
expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float(
|
||||
perplexity_cost["output_cost_per_token"]
|
||||
)
|
||||
|
||||
assert expected_cost > 0
|
||||
assert recording_logger.standard_logging_object["response_cost"] == pytest.approx(expected_cost)
|
||||
assert recording_logger.standard_logging_object["prompt_tokens"] == _INPUT_TOKENS
|
||||
assert recording_logger.standard_logging_object["completion_tokens"] == _OUTPUT_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
with pytest.raises(litellm.BadRequestError, match="Supported providers"):
|
||||
await litellm.adecisions(
|
||||
model="unknown/jev-1.13",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_custom_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
with pytest.raises(litellm.BadRequestError, match="Supported providers"):
|
||||
await litellm.adecisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_key="caller-key",
|
||||
custom_llm_provider="",
|
||||
)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_question_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"):
|
||||
await litellm.adecisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
state="review",
|
||||
questions={"sentiment": {"type": "choice"}},
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
||||
|
||||
def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter) -> None:
|
||||
respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(
|
||||
status_code=400,
|
||||
json={"error": {"message": "invalid decision"}},
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
litellm.decisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
|
||||
def test_server_key_is_sent_to_an_explicit_api_base(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("PERPLEXITYAI_API_KEY", "server-key")
|
||||
monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False)
|
||||
route: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE)
|
||||
|
||||
litellm.decisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_base="https://egress.example/perplexity",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert route.calls[0].request.headers["authorization"] == "Bearer server-key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", ("cloudflare/clef", "cloudflare/@cf/cloudflare/clef"))
|
||||
@pytest.mark.parametrize("wrapped", (False, True))
|
||||
async def test_cloudflare_clef_resolves_model_and_response_envelope(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
model: str,
|
||||
wrapped: bool,
|
||||
) -> None:
|
||||
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
|
||||
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
|
||||
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
|
||||
response_body: Final[Mapping[str, object]] = (
|
||||
{"result": _RESPONSE, "success": True, "errors": [], "messages": []} if wrapped else _RESPONSE
|
||||
)
|
||||
route: Final = respx_mock.post(
|
||||
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef"
|
||||
).respond(json=response_body)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model=model,
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
)
|
||||
|
||||
assert route.called
|
||||
request: Final = respx_mock.calls[0].request
|
||||
assert request.headers["authorization"] == "Bearer cloudflare-key"
|
||||
assert json.loads(request.content) == {
|
||||
"model": "clef",
|
||||
"state": "review",
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
assert response.answers == DecisionsResponse.model_validate(_RESPONSE).answers
|
||||
assert response._hidden_params["model"] == "cloudflare/@cf/cloudflare/clef"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cloudflare_clef_flash_uses_flash_endpoint_and_request_model(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
|
||||
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
|
||||
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
|
||||
route: Final = respx_mock.post(
|
||||
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-flash"
|
||||
).respond(json=_RESPONSE)
|
||||
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef-flash",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
)
|
||||
|
||||
assert route.called
|
||||
assert json.loads(respx_mock.calls[0].request.content)["model"] == "clef-flash"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cloudflare_api_base_from_env_uses_workers_ai_run_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("CLOUDFLARE_API_BASE", "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1")
|
||||
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
|
||||
monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False)
|
||||
route: Final = respx_mock.post(
|
||||
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef"
|
||||
).respond(json=_RESPONSE)
|
||||
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
)
|
||||
|
||||
assert route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cloudflare_requires_account_id_or_api_base_before_http(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False)
|
||||
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
|
||||
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID"):
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cloudflare_clef_cost_uses_the_model_cost_map(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
|
||||
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
|
||||
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
|
||||
respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef").respond(
|
||||
json=_RESPONSE
|
||||
)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model="cloudflare/clef",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
)
|
||||
|
||||
cost: Final = litellm.completion_cost(completion_response=response)
|
||||
clef_cost: Final = litellm.model_cost["cloudflare/@cf/cloudflare/clef"]
|
||||
expected_cost: Final = _INPUT_TOKENS * float(clef_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float(
|
||||
clef_cost["output_cost_per_token"]
|
||||
)
|
||||
|
||||
assert expected_cost > 0
|
||||
assert cost == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strands_decider_requires_api_base_before_http(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False)
|
||||
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="api_base is required"):
|
||||
await litellm.adecisions(
|
||||
model="strands_decider/strands-decider-2B-hobson-v19",
|
||||
state="review",
|
||||
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
|
||||
)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strands_decider_without_key_preserves_response_extras(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False)
|
||||
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
|
||||
route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model="strands_decider/strands-decider-2B-hobson-v19",
|
||||
state="review",
|
||||
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
|
||||
api_base="https://strands.example",
|
||||
)
|
||||
|
||||
assert route.called
|
||||
assert "authorization" not in respx_mock.calls[0].request.headers
|
||||
assert response.model_extra["latency_ms"] == _STRANDS_RESPONSE["latency_ms"]
|
||||
severity: Final = response.answers["severity"]
|
||||
assert isinstance(severity, ScoreAnswer)
|
||||
assert severity.legend == {"0": "none", "1": "low", "2": "high"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strands_decider_uses_key_from_matching_environment_base(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("STRANDS_DECIDER_API_BASE", "https://strands.example")
|
||||
monkeypatch.setenv("STRANDS_DECIDER_API_KEY", "strands-key")
|
||||
route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
|
||||
|
||||
await litellm.adecisions(
|
||||
model="strands_decider/strands-decider-2B-hobson-v19",
|
||||
state="review",
|
||||
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
|
||||
api_base="https://strands.example",
|
||||
)
|
||||
|
||||
assert route.called
|
||||
assert respx_mock.calls[0].request.headers["authorization"] == "Bearer strands-key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strands_decider_provider_resolution_and_router_dispatch(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False)
|
||||
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
|
||||
provider_resolution: Final = litellm.get_llm_provider("strands_decider/strands-decider-2B-hobson-v19")
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "strands",
|
||||
"litellm_params": {
|
||||
"model": "strands_decider/strands-decider-2B-hobson-v19",
|
||||
"api_base": "https://strands.example",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
|
||||
|
||||
response: Final = await router.adecisions(
|
||||
model="strands",
|
||||
state="review",
|
||||
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
|
||||
)
|
||||
|
||||
assert provider_resolution[:2] == ("strands-decider-2B-hobson-v19", "strands_decider")
|
||||
assert route.called
|
||||
assert response.model == _STRANDS_RESPONSE["model"]
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
"""Test health check helper functions"""
|
||||
|
||||
import json
|
||||
import socket
|
||||
import struct
|
||||
import zlib
|
||||
|
|
@ -8,6 +9,7 @@ from typing import Final
|
|||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME
|
||||
|
|
@ -528,3 +530,100 @@ def test_ocr_health_check_document_raises_without_the_extension():
|
|||
_ocr_health_check_document(model="mistral/mistral-ocr-latest", custom_llm_provider="mistral")
|
||||
finally:
|
||||
NATIVE_OCR_HEALTH_CHECK_DOCUMENT.reset()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "upstream_url"),
|
||||
(
|
||||
("perplexity/pplx-decider-v1-27b", "https://api.perplexity.ai/v1/decisions"),
|
||||
("cloudflare/clef", "https://api.cloudflare.com/client/v4/accounts/acct-1/ai/run/@cf/cloudflare/clef"),
|
||||
),
|
||||
)
|
||||
async def test_ahealth_check_probes_evaluation_models_through_the_decisions_api(
|
||||
model: str,
|
||||
upstream_url: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct-1")
|
||||
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post(upstream_url).respond(
|
||||
json={
|
||||
"model": model,
|
||||
"answers": {"reachable": {"type": "noul", "noul": 1.0}},
|
||||
"usage": {"input_tokens": 12, "output_tokens": 1},
|
||||
}
|
||||
)
|
||||
|
||||
result: Final = await ahealth_check({"model": model, "api_key": "sk-test"}, mode=None)
|
||||
|
||||
assert "error" not in result, result
|
||||
assert upstream.called
|
||||
sent: Final = json.loads(upstream.calls[0].request.content)
|
||||
assert sent["state"] == "health check"
|
||||
assert sent["questions"]["reachable"]["type"] == "noul"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ahealth_check_evaluation_uses_configured_probe_state_and_questions(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(
|
||||
json={
|
||||
"model": "perplexity/pplx-decider-v1-27b",
|
||||
"answers": {"ok": {"type": "noul", "noul": 1.0}},
|
||||
"usage": {"input_tokens": 12, "output_tokens": 1},
|
||||
}
|
||||
)
|
||||
|
||||
result: Final = await ahealth_check(
|
||||
{
|
||||
"model": "perplexity/pplx-decider-v1-27b",
|
||||
"api_key": "sk-test",
|
||||
"state": "custom probe",
|
||||
"questions": {"ok": {"type": "noul", "instructions": "Is it ok?"}},
|
||||
},
|
||||
mode=None,
|
||||
)
|
||||
|
||||
assert "error" not in result, result
|
||||
assert upstream.called
|
||||
sent: Final = json.loads(upstream.calls[0].request.content)
|
||||
assert sent["state"] == "custom probe"
|
||||
assert set(sent["questions"]) == {"ok"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ahealth_check_probes_strands_through_decisions_without_mode(
|
||||
local_model_cost_map: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
|
||||
monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post("http://strands.local:8080/v1/systemone").respond(
|
||||
json={
|
||||
"model": "strands-decider-2B-hobson-v19",
|
||||
"answers": {"reachable": {"type": "noul", "noul": 1.0}},
|
||||
"usage": {"input_tokens": 12, "output_tokens": 1},
|
||||
}
|
||||
)
|
||||
|
||||
result: Final = await ahealth_check(
|
||||
{
|
||||
"model": "strands_decider/strands-decider-2B-hobson-v19",
|
||||
"api_base": "http://strands.local:8080",
|
||||
},
|
||||
mode=None,
|
||||
)
|
||||
|
||||
assert "error" not in result, result
|
||||
assert upstream.called
|
||||
assert "authorization" not in upstream.calls[0].request.headers
|
||||
|
|
|
|||
|
|
@ -470,6 +470,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client():
|
|||
"allm_passthrough_route",
|
||||
"llm_passthrough_route",
|
||||
"asearch",
|
||||
"adecisions",
|
||||
"avector_store_create",
|
||||
"avector_store_search",
|
||||
"acreate_skill",
|
||||
|
|
|
|||
0
tests/unit/proxy/decisions_endpoints/__init__.py
Normal file
0
tests/unit/proxy/decisions_endpoints/__init__.py
Normal file
298
tests/unit/proxy/decisions_endpoints/test_endpoints.py
Normal file
298
tests/unit/proxy/decisions_endpoints/test_endpoints.py
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Iterator, Mapping
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi import FastAPI
|
||||
from fastapi.routing import APIRoute
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.routing import Match
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._lazy_features import attach_lazy_features
|
||||
from litellm.proxy.decisions_endpoints.endpoints import decisions
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder
|
||||
from litellm.proxy.proxy_server import (
|
||||
app,
|
||||
cleanup_router_config_variables,
|
||||
initialize,
|
||||
)
|
||||
|
||||
_INPUT_TOKENS: Final[int] = 367
|
||||
_OUTPUT_TOKENS: Final[int] = 3
|
||||
_RESPONSE: Final[Mapping[str, object]] = {
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"answers": {
|
||||
"is_defect": {"type": "noul", "noul": 0.9},
|
||||
},
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS},
|
||||
}
|
||||
_STRANDS_RESPONSE: Final[Mapping[str, object]] = {
|
||||
"model": "strands-decider-2B-hobson-v19",
|
||||
"answers": {
|
||||
"is_defect": {"type": "noul", "noul": 0.9},
|
||||
},
|
||||
"usage": {"input_tokens": 216, "output_tokens": 3},
|
||||
"latency_ms": 3722.17,
|
||||
}
|
||||
_REQUEST: Final[Mapping[str, object]] = {
|
||||
"model": "decider",
|
||||
"state": {"source": "proxy-test"},
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "fake-openai-key")
|
||||
monkeypatch.setenv("OPENAI_API_BASE", "https://fake-openai.example")
|
||||
monkeypatch.setenv("REDIS_HOST", "localhost")
|
||||
cleanup_router_config_variables()
|
||||
config_path: Final = Path(__file__).parents[1] / "test_configs" / "test_config_no_auth.yaml"
|
||||
asyncio.run(initialize(config=str(config_path), debug=True))
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "decider",
|
||||
"litellm_params": {
|
||||
"model": "perplexity/pplx-decider-v1-27b",
|
||||
"api_key": "test-key",
|
||||
},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
yield TestClient(app)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions"))
|
||||
def test_proxy_decisions_route_returns_answers_and_cost(
|
||||
client: TestClient,
|
||||
respx_mock: respx.MockRouter,
|
||||
endpoint: str,
|
||||
) -> None:
|
||||
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post(endpoint, json=_REQUEST)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["answers"] == _RESPONSE["answers"]
|
||||
assert "_hidden_params" not in response.json()
|
||||
perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"]
|
||||
expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float(
|
||||
perplexity_cost["output_cost_per_token"]
|
||||
)
|
||||
|
||||
assert expected_cost > 0
|
||||
assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost)
|
||||
assert upstream.called
|
||||
assert json.loads(upstream.calls[0].request.content) == {
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"state": {"source": "proxy-test"},
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
assert upstream.calls[0].request.headers["authorization"] == "Bearer test-key"
|
||||
|
||||
|
||||
def test_proxy_decisions_dispatches_typesafe_deployment(
|
||||
client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "jev",
|
||||
"litellm_params": {
|
||||
"model": "typesafe/jev-latest",
|
||||
"api_key": "k",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
|
||||
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post(
|
||||
"/v1/decisions",
|
||||
json={
|
||||
"model": "jev",
|
||||
"state": {"source": "proxy-test"},
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["answers"] == _RESPONSE["answers"]
|
||||
assert upstream.called
|
||||
assert json.loads(upstream.calls[0].request.content) == {
|
||||
"model": "jev-latest",
|
||||
"state": {"source": "proxy-test"},
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
|
||||
|
||||
def test_proxy_decisions_sends_the_env_key_to_the_deployment_api_base(
|
||||
client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("PERPLEXITYAI_API_KEY", "server-key")
|
||||
monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "decider",
|
||||
"litellm_params": {
|
||||
"model": "perplexity/pplx-decider-v1-27b",
|
||||
"api_base": "https://egress.example/perplexity",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
|
||||
upstream: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post("/v1/decisions", json=_REQUEST)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert upstream.call_count == 1
|
||||
assert upstream.calls[0].request.headers["authorization"] == "Bearer server-key"
|
||||
|
||||
|
||||
def test_proxy_decisions_unknown_model_is_a_client_error(
|
||||
client: TestClient,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
response: Final = client.post(
|
||||
"/v1/decisions",
|
||||
json={
|
||||
"model": "missing-model",
|
||||
"state": "review",
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
},
|
||||
)
|
||||
|
||||
assert 400 <= response.status_code < 500, response.text
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_body",
|
||||
(
|
||||
{
|
||||
"model": "decider",
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
},
|
||||
{
|
||||
"model": "decider",
|
||||
"state": {"source": "proxy-test"},
|
||||
},
|
||||
),
|
||||
ids=("missing_state", "missing_questions"),
|
||||
)
|
||||
def test_proxy_decisions_missing_required_field_is_a_client_error(
|
||||
client: TestClient,
|
||||
respx_mock: respx.MockRouter,
|
||||
request_body: Mapping[str, object],
|
||||
) -> None:
|
||||
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post("/v1/decisions", json=request_body)
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert not upstream.called
|
||||
|
||||
|
||||
def test_proxy_decisions_dispatches_strands_decider(
|
||||
client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False)
|
||||
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "strands",
|
||||
"litellm_params": {
|
||||
"model": "strands_decider/strands-decider-2B-hobson-v19",
|
||||
"api_base": "https://strands.example",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
|
||||
upstream: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
|
||||
|
||||
response: Final = client.post(
|
||||
"/v1/decisions",
|
||||
json={
|
||||
"model": "strands",
|
||||
"state": {"source": "proxy-test"},
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["answers"] == _STRANDS_RESPONSE["answers"]
|
||||
assert upstream.called
|
||||
assert json.loads(upstream.calls[0].request.content) == {
|
||||
"model": "strands-decider-2B-hobson-v19",
|
||||
"state": {"source": "proxy-test"},
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
assert "authorization" not in upstream.calls[0].request.headers
|
||||
|
||||
|
||||
def test_proxy_decisions_without_model_uses_the_proxy_default_model(
|
||||
client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "user_model", "decider")
|
||||
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = client.post(
|
||||
"/v1/decisions", json={key: value for key, value in _REQUEST.items() if key != "model"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["answers"] == _RESPONSE["answers"]
|
||||
assert upstream.called
|
||||
assert json.loads(upstream.calls[0].request.content)["model"] == "pplx-decider-v1-27b"
|
||||
|
||||
|
||||
def _serving_endpoint(bare: FastAPI, path: str) -> object:
|
||||
scope: Final = {"type": "http", "method": "POST", "path": path, "root_path": "", "query_string": b"", "headers": ()}
|
||||
return next(
|
||||
route.endpoint for route in bare.routes if isinstance(route, APIRoute) and route.matches(scope)[0] is Match.FULL
|
||||
)
|
||||
|
||||
|
||||
def test_a_config_pass_through_at_v1_decisions_keeps_its_route_and_the_native_api_serves_decisions(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("LITELLM_DISABLE_LAZY_ROUTES", raising=False)
|
||||
|
||||
async def pass_through() -> dict[str, str]:
|
||||
return {"served_by": "pass-through"}
|
||||
|
||||
bare: Final = FastAPI()
|
||||
attach_lazy_features(bare)
|
||||
SafeRouteAdder.add_api_route_if_not_exists(bare, "/v1/decisions", pass_through, ["POST"])
|
||||
with TestClient(bare) as client:
|
||||
assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"}
|
||||
assert _serving_endpoint(bare, "/v1/decisions") is pass_through
|
||||
assert _serving_endpoint(bare, "/decisions") is decisions
|
||||
76
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
76
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -4398,6 +4398,23 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/decisions": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Decisions */
|
||||
post: operations["decisions_decisions_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/delete/allowed_ip": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -18644,6 +18661,23 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/decisions": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Decisions */
|
||||
post: operations["decisions_v1_decisions_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/embeddings": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -26667,7 +26701,7 @@ export interface components {
|
|||
* CallTypes
|
||||
* @enum {string}
|
||||
*/
|
||||
CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill";
|
||||
CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill";
|
||||
/** CallbackDelete */
|
||||
CallbackDelete: {
|
||||
/** Callback Name */
|
||||
|
|
@ -53175,6 +53209,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
decisions_decisions_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
delete_allowed_ip_delete_allowed_ip_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -70747,6 +70801,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
decisions_v1_decisions_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
embeddings_v1_embeddings_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue