diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 5842cf6f1ac..14eecf01461 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -14,8 +14,9 @@ import argparse import json import subprocess import sys -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Final from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors @@ -25,6 +26,7 @@ BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json" GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" +EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({}) CostMap = dict[str, object] @@ -51,7 +53,31 @@ def _rendered_schema(cost_map: CostMap) -> str: return str(error) -def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]: +def _evaluation_output_price_failure( + key: str, entry: object, evaluation_output_price_sources: Mapping[str, str] +) -> str | None: + if not isinstance(entry, dict) or entry.get("mode") != "evaluation": + return None + output_cost: Final = entry.get("output_cost_per_token") + if ( + not isinstance(output_cost, (int, float)) + or isinstance(output_cost, bool) + or output_cost <= 0 + or key in evaluation_output_price_sources + ): + return None + return ( + f"{key}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor " + "prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)" + ) + + +def _file_failures( + head: Snapshot, + head_map: CostMap, + *, + evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES, +) -> tuple[str, ...]: schema_text: Final = _rendered_schema(head_map) if not schema_text.startswith("{"): return (schema_text,) @@ -75,6 +101,16 @@ def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]: f"{COST_MAP_PATH} does not validate against its schema: {error}" for error in validation_errors(head_map, json.loads(schema_text))[:20] ), + *( + failure + for key, entry in head_map.items() + if ( + failure := _evaluation_output_price_failure( + key, entry, evaluation_output_price_sources + ) + ) + is not None + ), ) @@ -121,13 +157,25 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str: return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched" -def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]: +def guard_failures( + base: Snapshot, + head: Snapshot, + changed_files: Sequence[str], + bot: bool, + *, + evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES, +) -> tuple[str, ...]: if not bot and not touches_cost_map(changed_files): return () head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH) if isinstance(head_map, str): return (head_map,) - return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ())) + return ( + *_file_failures( + head, head_map, evaluation_output_price_sources=evaluation_output_price_sources + ), + *(_bot_failures(base, head_map, changed_files) if bot else ()), + ) def _git(*args: str) -> str | None: diff --git a/litellm/__init__.py b/litellm/__init__.py index d576ea03c94..81fd6ca6ca5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1474,6 +1474,7 @@ from .rust_bridge import rust from .rag.main import * from .sandbox.main import * from .decisions.main import * +from .systemone.main import * from .search.main import * from .realtime_api.main import ( _arealtime, @@ -1654,6 +1655,30 @@ if TYPE_CHECKING: from .llms.hosted_vllm.rerank.transformation import ( HostedVLLMRerankConfig as HostedVLLMRerankConfig, ) + from .llms.perplexity.decisions.transformation import ( + PerplexityDecisionsConfig as PerplexityDecisionsConfig, + ) + from .llms.typesafe.decisions.transformation import ( + TypeSafeDecisionsConfig as TypeSafeDecisionsConfig, + ) + from .llms.openrouter.decisions.transformation import ( + OpenRouterDecisionsConfig as OpenRouterDecisionsConfig, + ) + from .llms.cloudflare.decisions.transformation import ( + CloudflareDecisionsConfig as CloudflareDecisionsConfig, + ) + from .llms.strands_decider.decisions.transformation import ( + StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig, + ) + from .llms.hosted_vllm.decisions.transformation import ( + HostedVLLMDecisionsConfig as HostedVLLMDecisionsConfig, + ) + from .llms.openai.decisions.transformation import ( + OpenAIDecisionsConfig as OpenAIDecisionsConfig, + ) + from .llms.azure_ai.decisions.transformation import ( + AzureAIDecisionsConfig as AzureAIDecisionsConfig, + ) from .llms.nvidia_nim.rerank.transformation import ( NvidiaNimRerankConfig as NvidiaNimRerankConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index ffcbffb05b6..257e748ab98 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -153,6 +153,14 @@ LLM_CONFIG_NAMES: Final = ( "JinaAIRerankConfig", "DeepinfraRerankConfig", "HostedVLLMRerankConfig", + "PerplexityDecisionsConfig", + "TypeSafeDecisionsConfig", + "OpenRouterDecisionsConfig", + "CloudflareDecisionsConfig", + "StrandsDeciderDecisionsConfig", + "HostedVLLMDecisionsConfig", + "OpenAIDecisionsConfig", + "AzureAIDecisionsConfig", "NvidiaNimRerankConfig", "NvidiaNimRankingConfig", "VertexAIRerankConfig", @@ -696,6 +704,20 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.hosted_vllm.rerank.transformation", "HostedVLLMRerankConfig", ), + "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"), + "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"), + "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), + "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"), + "StrandsDeciderDecisionsConfig": ( + ".llms.strands_decider.decisions.transformation", + "StrandsDeciderDecisionsConfig", + ), + "HostedVLLMDecisionsConfig": ( + ".llms.hosted_vllm.decisions.transformation", + "HostedVLLMDecisionsConfig", + ), + "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), + "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), "NvidiaNimRerankConfig": ( ".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig", diff --git a/litellm/constants.py b/litellm/constants.py index af4d1268c03..9ee73ec7b98 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -266,6 +266,7 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int( # Provider-specific API base URLs XAI_API_BASE: Final = "https://api.x.ai/v1" +PLACEHOLDER_API_KEY: Final = "fake-api-key" OPEN_SANDBOX_API_BASE_ENV_VAR: Final = "OPEN_SANDBOX_API_BASE" OPEN_SANDBOX_API_KEY_ENV_VAR: Final = "OPEN_SANDBOX_API_KEY" OPEN_SANDBOX_DEFAULT_TEMPLATE: Final = "opensandbox/code-interpreter:v1.1.0" diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9dd6cc0f1f9..2a254cfec1f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2014,6 +2014,8 @@ def response_cost_calculator( "asearch", "decisions", "adecisions", + "systemone", + "asystemone", ], optional_params: dict, cache_hit: bool | None = None, diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py new file mode 100644 index 00000000000..1ad2f006736 --- /dev/null +++ b/litellm/decisions/call.py @@ -0,0 +1,347 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field, replace +from typing import Final, TypeAlias + +import httpx +from pydantic import ConfigDict, TypeAdapter, ValidationError +from typing_extensions import assert_never + +import litellm +from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.decisions.transformation import ( + BaseDecisionsConfig, + ir_to_systemone_response, + systemone_request_to_ir, +) +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir +from litellm.types.decisions import ( + DecisionQuestion, + DecisionsIRRequest, + DecisionsIRResponse, + DecisionsJSON, + DecisionsRequestBody, + DecisionsResponse, + OpenAIDecisionInput, + OpenAIDecisionQuestion, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, + UnsupportedDecisionsRequest, +) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]] +OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]] +DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody + +_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) +_SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( + str | None, config=ConfigDict(title="safety_identifier") +) +_HANDLER: Final = BaseLLMHTTPHandler() + + +@dataclass(frozen=True, slots=True) +class SystemOneRequestFields: + state: DecisionsJSON | None + questions: SystemOneQuestions | None + + def validate(self, safety_identifier: str | None) -> DecisionsRequestBody: + return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": self.state, "questions": self.questions}) + + +@dataclass(frozen=True, slots=True) +class OpenAIDecisionRequestFields: + input: OpenAIDecisionInput | None + questions: OpenAIDecisionQuestions | None + + def validate(self, safety_identifier: str | None) -> OpenAIDecisionRequestBody: + return _OPENAI_REQUEST_ADAPTER.validate_python( + {"input": self.input, "questions": self.questions, "safety_identifier": safety_identifier} + ) + + +RequestFields: TypeAlias = SystemOneRequestFields | OpenAIDecisionRequestFields + + +@dataclass(frozen=True, slots=True, repr=False) +class DecisionsCall: + model: str + requested_model: str + custom_llm_provider: str + provider_config: BaseDecisionsConfig + request: DecisionsRequestFormat + ir_request: DecisionsIRRequest + body: Mapping[str, object] = field(repr=False) + api_base: str + api_key: str | None = field(repr=False) + logging_obj: LiteLLMLoggingObj | None + headers: Mapping[str, str] + timeout: float | httpx.Timeout | None + litellm_params: Mapping[str, object] = field(repr=False) + + +def supported_decisions_providers() -> tuple[str, ...]: + return tuple( + provider.value + for provider in LlmProviders + if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None + ) + + +def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig: + provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None) + provider_config: Final = ( + None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider) + ) + if provider_config is None: + supported: Final = ", ".join(supported_decisions_providers()) + raise litellm.BadRequestError( + message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}", + model=model, + llm_provider=custom_llm_provider, + ) + return provider_config + + +def _ir_request(request: DecisionsRequestFormat, safety_identifier: str | None) -> DecisionsIRRequest: + match request: + case DecisionsRequestBody(): + return replace(systemone_request_to_ir(request), safety_identifier=safety_identifier) + case OpenAIDecisionRequestBody(): + return openai_request_to_ir(request) + case _: + assert_never(request) + + +def _drops_params(kwargs: Mapping[str, object]) -> bool: + return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True + + +def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, object]) -> str | None: + if isinstance(safety_identifier, str) or not _drops_params(kwargs): + return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier) + return None + + +@dataclass(frozen=True, slots=True) +class _UnsupportedSafetyIdentifier: + pass + + +def _provider_ir_request( + request: DecisionsRequestFormat, + *, + safety_identifier: str | None, + provider_config: BaseDecisionsConfig, + kwargs: Mapping[str, object], +) -> DecisionsIRRequest | _UnsupportedSafetyIdentifier: + ir_request: Final = _ir_request(request, safety_identifier) + if ir_request.safety_identifier is None or provider_config.supports_safety_identifier: + return ir_request + if _drops_params(kwargs): + return replace(ir_request, safety_identifier=None) + return _UnsupportedSafetyIdentifier() + + +def prepare_call( + *, + model: str, + fields: RequestFields, + safety_identifier: str | None, + api_key: str | None, + api_base: str | None, + timeout: float | httpx.Timeout | None, + custom_llm_provider: str | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> DecisionsCall: + upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + provider_config: Final = _provider_config(upstream_model, provider) + if not upstream_model: + raise litellm.BadRequestError( + message="A model name is required for the Decisions API", + model=model, + llm_provider=provider, + ) + try: + canonical_model: Final = provider_config.canonical_model(upstream_model) + except ValueError as error: + raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error + try: + request_safety_identifier: Final = _request_safety_identifier(safety_identifier, kwargs) + request: Final = fields.validate(request_safety_identifier) + except ValidationError as error: + raise litellm.BadRequestError( + message=f"Invalid Decisions request: {error}", + model=model, + llm_provider=provider, + ) from error + + resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base) + if resolved_api_base is None: + raise litellm.BadRequestError( + message=provider_config.missing_api_base_message(provider), + model=model, + llm_provider=provider, + ) + resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key) + if resolved_api_key is None and provider_config.api_key_required: + raise litellm.AuthenticationError( + message=f"Missing API key for Decisions provider '{provider}'", + model=model, + llm_provider=provider, + ) + + ir_request: Final = _provider_ir_request( + request, + safety_identifier=request_safety_identifier, + provider_config=provider_config, + kwargs=kwargs, + ) + if isinstance(ir_request, _UnsupportedSafetyIdentifier): + raise litellm.UnsupportedParamsError( + message=( + f"{provider} does not support parameters: ['safety_identifier'], for model={model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ), + model=model, + llm_provider=provider, + ) + body: Final = provider_config.transform_decisions_request( + model=canonical_model, request=ir_request, custom_llm_provider=provider + ) + if isinstance(body, UnsupportedDecisionsRequest): + raise litellm.BadRequestError( + message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}", + model=model, + llm_provider=provider, + ) + + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.update_from_kwargs( + kwargs=dict(kwargs), + model=canonical_model, + litellm_params={ + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), + }, + custom_llm_provider=provider, + ) + return DecisionsCall( + model=canonical_model, + requested_model=model, + custom_llm_provider=provider, + provider_config=provider_config, + request=request, + ir_request=ir_request, + body=body, + api_base=resolved_api_base, + api_key=resolved_api_key, + logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, + headers=extra_headers or {}, + timeout=timeout, + litellm_params=kwargs, + ) + + +def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + formatted: Final = _formatted_response(response, call) + provider_cost: Final = call.provider_config.provider_reported_cost(response) + formatted.set_hidden_params( + { + "model": f"{call.custom_llm_provider}/{call.model}", + "custom_llm_provider": call.custom_llm_provider, + "provider_response_model": f"{call.custom_llm_provider}/{call.model}", + **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}), + } + ) + return formatted + + +def _formatted_response( + response: DecisionsIRResponse, call: DecisionsCall +) -> DecisionsResponse | OpenAIDecisionResponse: + match call.request: + case DecisionsRequestBody(): + return ir_to_systemone_response(response, call.ir_request) + case OpenAIDecisionRequestBody(): + return ir_to_openai_response(response, call.ir_request, call.requested_model) + case _: + assert_never(call.request) + + +def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception: + if isinstance(error, BaseLLMException) and error.status_code_is_synthesized: + provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception" + return litellm.APIConnectionError( + message=f"{provider_label} - {error.message}", + llm_provider=call.custom_llm_provider, + model=f"{call.custom_llm_provider}/{call.model}", + ) + return litellm.exception_type( + model=f"{call.custom_llm_provider}/{call.model}", + custom_llm_provider=call.custom_llm_provider, + original_exception=error, + ) + + +async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + try: + response: Final = await _HANDLER.adecisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.ir_request, + body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + ) + except Exception as error: + raise _map_upstream_exception(error, call) from error + return _format_response(response, call) + + +def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + try: + response: Final = _HANDLER.decisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.ir_request, + body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + ) + except Exception as error: + raise _map_upstream_exception(error, call) from error + return _format_response(response, call) + + +def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name: str, other_function: str) -> None: + if field_name not in kwargs: + return + raise litellm.BadRequestError( + message=f"Requests with '{field_name}' go through litellm.{other_function}", + model=model, + llm_provider="", + ) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 33d7fee408f..8b13d04770a 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,373 +1,131 @@ -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field -from types import MappingProxyType -from typing import Final, TypeAlias +import warnings +from collections.abc import Mapping +from typing import Final import httpx -from pydantic import TypeAdapter, ValidationError -from typing_extensions import assert_never +from pydantic import TypeAdapter -import litellm -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.base_llm.decisions.transformation import ( - DecisionsProviderConfig, - ir_to_systemone_response, - systemone_request_to_ir, -) -from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT -from litellm.llms.custom_httpx.http_handler import _get_httpx_client, get_async_httpx_client -from litellm.llms.openai.decisions.transformation import ( - OPENAI_DECISIONS_ENDPOINT, - ir_to_openai_response, - openai_request_to_ir, -) -from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT -from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT -from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT -from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_ENDPOINT -from litellm.types.decisions import ( - DecisionQuestion, - DecisionsIRRequest, - DecisionsIRResponse, - DecisionsJSON, - DecisionsRequestBody, - DecisionsResponse, - OpenAIDecisionInput, - OpenAIDecisionQuestion, - OpenAIDecisionRequestBody, - OpenAIDecisionResponse, - UnsupportedDecisionsRequest, +from litellm.decisions.call import ( + DecisionsCall, + OpenAIDecisionQuestions, + OpenAIDecisionRequestFields, + RequestFields, + SystemOneQuestions, + SystemOneRequestFields, + asend, + prepare_call, + send, ) +from litellm.types.decisions import DecisionsJSON, DecisionsResponse, OpenAIDecisionInput, OpenAIDecisionResponse from litellm.utils import client -DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxyType( - { - "perplexity": PERPLEXITY_DECISIONS_ENDPOINT, - "typesafe": TYPESAFE_DECISIONS_ENDPOINT, - "openrouter": OPENROUTER_DECISIONS_ENDPOINT, - "cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT, - "strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT, - "openai": OPENAI_DECISIONS_ENDPOINT, - } +_STATE_ADAPTER: Final[TypeAdapter[DecisionsJSON | None]] = TypeAdapter(DecisionsJSON | None) +_SYSTEMONE_QUESTIONS_ADAPTER: Final[TypeAdapter[SystemOneQuestions | None]] = TypeAdapter(SystemOneQuestions | None) +_OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter( + OpenAIDecisionQuestions | None ) -def supported_decisions_providers() -> tuple[str, ...]: - return tuple(DECISIONS_ENDPOINTS) - -DecisionsQuestions: TypeAlias = ( - Mapping[str, DecisionQuestion | Mapping[str, object]] | Sequence[OpenAIDecisionQuestion | Mapping[str, object]] -) -DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody - -_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) -_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) -_DECISIONS_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) - - -@dataclass(frozen=True, slots=True, repr=False) -class _PreparedDecisionsRequest: - config: DecisionsProviderConfig - provider: str - upstream_model: str - url: str - api_key: str | None = field(repr=False) - headers: Mapping[str, str] = field(repr=False) - body: Mapping[str, object] = field(repr=False) - request: DecisionsRequestFormat = field(repr=False) - ir_request: DecisionsIRRequest = field(repr=False) - - -def _resolve_provider_model(model: str, custom_llm_provider: str | None) -> tuple[str, str]: - provider: Final = model.partition("/")[0] if custom_llm_provider is None else custom_llm_provider - if provider not in DECISIONS_ENDPOINTS: - supported: Final = ", ".join(supported_decisions_providers()) - raise litellm.BadRequestError( - message=f"Unknown Decisions provider '{provider}'. Supported providers: {supported}", - model=model, - llm_provider=provider, +def _request_fields( + decision_input: OpenAIDecisionInput | None, + questions: OpenAIDecisionQuestions | SystemOneQuestions | None, + kwargs: Mapping[str, object], +) -> RequestFields: + if "state" not in kwargs: + return OpenAIDecisionRequestFields( + input=decision_input, questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions) ) - upstream_model: Final = model.removeprefix(f"{provider}/") if model.startswith(f"{provider}/") else model - if not upstream_model: - raise litellm.BadRequestError( - message="A model name is required for the Decisions API", - model=model, - llm_provider=provider, - ) - return provider, upstream_model - - -def _resolve_api_key( - *, - provider: str, - model: str, - endpoint: DecisionsProviderConfig, - api_key: str | None, -) -> str | None: - if api_key is not None: - return api_key - configured_api_key: Final = endpoint.configured_api_key() - if configured_api_key: - return configured_api_key - if not endpoint.api_key_required: - return None - raise litellm.AuthenticationError( - message=f"Missing API key for Decisions provider '{provider}'", - model=model, - llm_provider=provider, + warnings.warn( + "litellm.decisions(state=...) is deprecated, call litellm.systemone for the System One request format", + UserWarning, + stacklevel=5, + ) + return SystemOneRequestFields( + state=_STATE_ADAPTER.validate_python(kwargs["state"]), + questions=_SYSTEMONE_QUESTIONS_ADAPTER.validate_python(questions), ) -def _validate_request( - *, - state: DecisionsJSON | None, - questions: DecisionsQuestions | None, - decision_input: OpenAIDecisionInput | None, - safety_identifier: str | None, -) -> DecisionsRequestFormat: - if decision_input is None: - return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": state, "questions": questions}) - return _OPENAI_REQUEST_ADAPTER.validate_python( - {"input": decision_input, "questions": questions, "safety_identifier": safety_identifier} - ) - - -def _ir_request(request: DecisionsRequestFormat) -> DecisionsIRRequest: - match request: - case DecisionsRequestBody(): - return systemone_request_to_ir(request) - case OpenAIDecisionRequestBody(): - return openai_request_to_ir(request) - case _: - assert_never(request) - - -def _prepare_request( +def _prepare_decisions_call( *, model: str, - state: DecisionsJSON | None, - questions: DecisionsQuestions | None, decision_input: OpenAIDecisionInput | None, + questions: OpenAIDecisionQuestions | SystemOneQuestions | None, safety_identifier: str | None, api_key: str | None, api_base: str | None, + timeout: float | httpx.Timeout | None, custom_llm_provider: str | None, extra_headers: Mapping[str, str] | None, -) -> _PreparedDecisionsRequest: - provider, upstream_model = _resolve_provider_model(model, custom_llm_provider) - if state is not None and decision_input is not None: - raise litellm.BadRequestError( - message="Pass either state (System One format) or input (OpenAI format) to the Decisions API, not both", - model=model, - llm_provider=provider, - ) - try: - validated_request: Final = _validate_request( - state=state, questions=questions, decision_input=decision_input, safety_identifier=safety_identifier - ) - except ValidationError as error: - raise litellm.BadRequestError( - message=f"Invalid Decisions request: {error}", - model=model, - llm_provider=provider, - ) from error - - endpoint: Final = DECISIONS_ENDPOINTS[provider] - resolved_api_base: Final = api_base or endpoint.configured_api_base() - if resolved_api_base is None: - raise litellm.BadRequestError( - message=endpoint.missing_api_base_message(provider), - model=model, - llm_provider=provider, - ) - - resolved_api_key: Final = _resolve_api_key( - provider=provider, - model=model, - endpoint=endpoint, - api_key=api_key, - ) - - canonical_model: Final = endpoint.canonical_model(upstream_model) - outbound_headers: Final = MappingProxyType( - { - **{ - name: value - for name, value in (extra_headers or {}).items() - if name.lower() not in {"authorization", "content-type"} - }, - **({"Authorization": f"Bearer {resolved_api_key}"} if resolved_api_key is not None else {}), - "Content-Type": "application/json", - } - ) - ir_request: Final = _ir_request(validated_request) - body: Final = endpoint.request_body(endpoint.request_model(canonical_model), ir_request) - if isinstance(body, UnsupportedDecisionsRequest): - raise litellm.BadRequestError( - message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}", - model=model, - llm_provider=provider, - ) - return _PreparedDecisionsRequest( - config=endpoint, - provider=provider, - upstream_model=canonical_model, - url=endpoint.endpoint_url(resolved_api_base, canonical_model), - api_key=resolved_api_key, - headers=outbound_headers, - body=MappingProxyType(body), - request=validated_request, - ir_request=ir_request, - ) - - -def _log_request( - prepared: _PreparedDecisionsRequest, kwargs: Mapping[str, object], -) -> LiteLLMLoggingObj | None: - logging_obj: Final = kwargs.get("litellm_logging_obj") - if not isinstance(logging_obj, LiteLLMLoggingObj): - return None - logging_obj.update_from_kwargs( - kwargs=dict(kwargs), - model=prepared.upstream_model, - litellm_params={ - "litellm_call_id": kwargs.get("litellm_call_id"), - "api_base": prepared.url, - }, - custom_llm_provider=prepared.provider, - ) - request_body: Final = dict(prepared.body) - request_headers: Final = dict(prepared.headers) - logging_obj.pre_call( - input=request_body, - api_key=prepared.api_key, - model=prepared.upstream_model, - additional_args={ - "api_base": prepared.url, - "complete_input_dict": request_body, - "headers": request_headers, - }, - ) - return logging_obj - - -def _format_response( - response: DecisionsIRResponse, prepared: _PreparedDecisionsRequest, model: str -) -> DecisionsResponse | OpenAIDecisionResponse: - match prepared.request: - case DecisionsRequestBody(): - return ir_to_systemone_response(response, prepared.ir_request) - case OpenAIDecisionRequestBody(): - return ir_to_openai_response(response, prepared.ir_request, model) - case _: - assert_never(prepared.request) - - -def _parse_response( - response: httpx.Response, - prepared: _PreparedDecisionsRequest, - model: str, -) -> DecisionsResponse | OpenAIDecisionResponse: - response.raise_for_status() - payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content) - result: Final = _format_response(prepared.config.parse_response(payload, prepared.ir_request), prepared, model) - result._hidden_params.update( - { - "model": f"{prepared.provider}/{prepared.upstream_model}", - "custom_llm_provider": prepared.provider, - "provider_response_model": f"{prepared.provider}/{prepared.upstream_model}", - } - ) - return result - - -def _map_upstream_exception(error: Exception, prepared: _PreparedDecisionsRequest) -> Exception: - return litellm.exception_type( - model=f"{prepared.provider}/{prepared.upstream_model}", - custom_llm_provider=prepared.provider, - original_exception=error, +) -> DecisionsCall: + return prepare_call( + model=model, + fields=_request_fields(decision_input, questions, kwargs), + safety_identifier=safety_identifier, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs={key: value for key, value in kwargs.items() if key != "state"}, ) @client async def adecisions( model: str, - state: DecisionsJSON | None = None, - questions: DecisionsQuestions | None = None, + input: OpenAIDecisionInput | None = None, + questions: OpenAIDecisionQuestions | SystemOneQuestions | None = None, + safety_identifier: str | None = None, api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, extra_headers: Mapping[str, str] | None = None, - input: OpenAIDecisionInput | None = None, - safety_identifier: str | None = None, **kwargs: object, -) -> DecisionsResponse | OpenAIDecisionResponse: - prepared: Final = _prepare_request( +) -> OpenAIDecisionResponse | DecisionsResponse: + call: Final = _prepare_decisions_call( model=model, - state=state, - questions=questions, decision_input=input, + questions=questions, safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, + timeout=timeout, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + kwargs=kwargs, ) - logging_obj: Final = _log_request(prepared, kwargs) - try: - handler: Final = get_async_httpx_client(llm_provider=prepared.provider) - response: Final = await handler.post( - prepared.url, - json=dict(prepared.body), - headers=dict(prepared.headers), - timeout=timeout, - logging_obj=logging_obj, - ) - return _parse_response(response=response, prepared=prepared, model=model) - except Exception as error: - raise _map_upstream_exception(error, prepared) from error + return await asend(call) @client def decisions( model: str, - state: DecisionsJSON | None = None, - questions: DecisionsQuestions | None = None, + input: OpenAIDecisionInput | None = None, + questions: OpenAIDecisionQuestions | SystemOneQuestions | None = None, + safety_identifier: str | None = None, api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, extra_headers: Mapping[str, str] | None = None, - input: OpenAIDecisionInput | None = None, - safety_identifier: str | None = None, **kwargs: object, -) -> DecisionsResponse | OpenAIDecisionResponse: - prepared: Final = _prepare_request( +) -> OpenAIDecisionResponse | DecisionsResponse: + call: Final = _prepare_decisions_call( model=model, - state=state, - questions=questions, decision_input=input, + questions=questions, safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, + timeout=timeout, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + kwargs=kwargs, ) - logging_obj: Final = _log_request(prepared, kwargs) - try: - handler: Final = _get_httpx_client() - response: Final = handler.post( - prepared.url, - json=dict(prepared.body), - headers=dict(prepared.headers), - timeout=timeout, - logging_obj=logging_obj, - ) - return _parse_response(response=response, prepared=prepared, model=model) - except Exception as error: - raise _map_upstream_exception(error, prepared) from error + return send(call) -__all__ = ["DECISIONS_ENDPOINTS", "adecisions", "decisions"] +__all__ = ["adecisions", "decisions"] diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 0fdfb301291..9fb7f233242 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -2146,7 +2146,10 @@ def _map_openrouter_exception( exception_provider: str, extra_information: str, ) -> None: - if hasattr(original_exception, "status_code"): + received_status: Final = hasattr(original_exception, "status_code") and not getattr( + original_exception, "status_code_is_synthesized", False + ) + if received_status: if original_exception.status_code == 400: raise BadRequestError( message=f"{exception_provider} - {error_str}", diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 41965404351..9080796fbf6 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -3,7 +3,7 @@ Helper functions for health check calls. """ import base64 -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from typing import TYPE_CHECKING, Final, Literal from litellm.llms.base_llm.ocr.transformation import DocumentType @@ -30,6 +30,21 @@ def get_image_file_for_health_check() -> bytes: return base64.b64decode(TEST_IMAGE_BASE64) +def _decisions_health_check_questions(model: str, custom_llm_provider: str) -> Mapping[str, Mapping[str, object]]: + import litellm + from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + from litellm.utils import ProviderConfigManager + + provider: Final = next((member for member in litellm.LlmProviders if member.value == custom_llm_provider), None) + config: Final = ( + None + if provider is None + else ProviderConfigManager.get_provider_decisions_config(model=model, provider=provider) + ) + questions: Final = BaseDecisionsConfig.health_check_questions if config is None else config.health_check_questions + return {name: dict(question) for name, question in questions.items()} + + def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType: native: Final = NATIVE_OCR_HEALTH_CHECK_DOCUMENT.load() if native is None: @@ -258,11 +273,13 @@ class HealthCheckHelpers: **_filter_model_params(model_params=model_params), document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider), ), - "evaluation": lambda: litellm.adecisions( + "evaluation": lambda: litellm.asystemone( **DECISIONS_CALL_PARAMS.validate_python( { "state": prompt or "health check", - "questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}}, + "questions": _decisions_health_check_questions( + model=model, custom_llm_provider=custom_llm_provider + ), **_filter_model_params(model_params=model_params), } ) diff --git a/litellm/llms/azure_ai/decisions/transformation.py b/litellm/llms/azure_ai/decisions/transformation.py new file mode 100644 index 00000000000..9b10f393c7c --- /dev/null +++ b/litellm/llms/azure_ai/decisions/transformation.py @@ -0,0 +1,24 @@ +import re +from typing import Final + +import httpx + +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + +_FOUNDRY_ROUTE_SUFFIX: Final = re.compile(r"(/api/projects/[^/]+)?(/openai/v1|/openai|/models|/v1)?$") + + +class AzureAIDecisionsConfig(BaseDecisionsConfig): + """Microsoft Foundry decision models (Microsoft-Decision-1), where model is the deployment name""" + + path = "/providers/microsoft/v1/systemone" + api_key_env = ("AZURE_AI_API_KEY",) + api_base_env = ("AZURE_AI_API_BASE",) + + def missing_api_base_message(self, custom_llm_provider: str) -> str: + return "Missing AZURE_AI_API_BASE - set AZURE_AI_API_BASE or pass api_base, e.g. https://.services.ai.azure.com" + + def get_complete_url(self, api_base: str, model: str) -> str: + url: Final = httpx.URL(api_base) + resource_path: Final = _FOUNDRY_ROUTE_SUFFIX.sub("", url.path.rstrip("/")) + return str(url.copy_with(path=f"{resource_path}{self.path}", query=None, fragment=None)) diff --git a/litellm/llms/base_llm/decisions/__init__.py b/litellm/llms/base_llm/decisions/__init__.py index c18ac9b00f2..313e57e9762 100644 --- a/litellm/llms/base_llm/decisions/__init__.py +++ b/litellm/llms/base_llm/decisions/__init__.py @@ -1,3 +1,3 @@ -from .transformation import DecisionsProviderConfig, JevCompatibleDecisionsEndpoint +from .transformation import BaseDecisionsConfig -__all__ = ["DecisionsProviderConfig", "JevCompatibleDecisionsEndpoint"] +__all__ = ["BaseDecisionsConfig"] diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 14a955053da..724ca23c2ab 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -1,12 +1,14 @@ import json +from abc import ABC from collections.abc import Mapping, Sequence -from dataclasses import dataclass from types import MappingProxyType -from typing import Final, Protocol +from typing import Final -from pydantic import TypeAdapter +import httpx +from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( ChoiceAnswer, @@ -46,7 +48,9 @@ from litellm.types.decisions import ( systemone_choice_key, ) +_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) +_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) _TEXT_ONLY: Final = UnsupportedDecisionsRequest( reason="input_image content parts are not supported because System One providers accept text input only" ) @@ -292,22 +296,34 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR ) -@dataclass(frozen=True, slots=True) -class JevCompatibleDecisionsEndpoint: - default_api_base_value: str | None - path: str - api_key_env: tuple[str, ...] - api_base_env: str +class BaseDecisionsConfig(ABC): + path: str = "/v1/systemone" + api_key_env: tuple[str, ...] = () + api_base_env: tuple[str, ...] = () api_key_required: bool = True + supports_safety_identifier: bool = False + health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType( + {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})} + ) - def configured_api_key(self) -> str | None: - return next((key for key in (get_secret_str(name) for name in self.api_key_env) if key), None) + def get_default_api_base(self) -> str | None: + return None - def configured_api_base(self) -> str | None: - return get_secret_str(self.api_base_env) or self.default_api_base_value + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: + return None - def missing_api_base_message(self, provider: str) -> str: - return f"api_base is required for Decisions provider '{provider}'" + def missing_api_base_message(self, custom_llm_provider: str) -> str: + return f"api_base is required for Decisions provider '{custom_llm_provider}'" + + def resolve_api_base(self, api_base: str | None) -> str | None: + return api_base or self._first_secret(self.api_base_env) or self.get_default_api_base() + + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key or self._first_secret(self.api_key_env) + + @staticmethod + def _first_secret(names: tuple[str, ...]) -> str | None: + return next((value for value in (get_secret_str(name) for name in names) if value), None) def canonical_model(self, model: str) -> str: return model @@ -315,36 +331,64 @@ class JevCompatibleDecisionsEndpoint: def request_model(self, model: str) -> str: return model - def endpoint_url(self, api_base: str, model: str) -> str: + def validate_environment(self, headers: Mapping[str, str], model: str, api_key: str | None) -> dict[str, str]: + return { + **{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS}, + **({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}), + "Content-Type": "application/json", + } + + def get_complete_url(self, api_base: str, model: str) -> str: return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" - def request_body( - self, model: str, request: DecisionsIRRequest + def signs_with_aws(self, api_base: str) -> bool: + return False + + def sign_request( + self, + headers: Mapping[str, str], + url: str, + api_base: str, + body: Mapping[str, object], + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> tuple[Mapping[str, str], bytes | None]: + return headers, None + + def transform_decisions_request( + self, + model: str, + request: DecisionsIRRequest, + custom_llm_provider: str, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_systemone_request(model, request) + return ir_to_systemone_request(self.request_model(model), request) + + def unwrap_response(self, payload: object) -> object: + return payload def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: - return parse_systemone_response(payload, request) + return parse_systemone_response(self.unwrap_response(payload), request) + def transform_decisions_response( + self, + model: str, + custom_llm_provider: str, + raw_response: httpx.Response, + request: DecisionsIRRequest, + ) -> DecisionsIRResponse: + payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content) + try: + return self.parse_response(payload, request) + except ValidationError as error: + raise BaseLLMException( + status_code=500, + message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: {error}", + ) from error -class DecisionsProviderConfig(Protocol): - @property - def api_key_required(self) -> bool: ... - - def configured_api_key(self) -> str | None: ... - - def configured_api_base(self) -> str | None: ... - - def missing_api_base_message(self, provider: str) -> str: ... - - def canonical_model(self, model: str) -> str: ... - - def request_model(self, model: str) -> str: ... - - def endpoint_url(self, api_base: str, model: str) -> str: ... - - def request_body( - self, model: str, request: DecisionsIRRequest - ) -> Mapping[str, object] | UnsupportedDecisionsRequest: ... - - def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: ... + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, + ) -> BaseLLMException: + return BaseLLMException(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 83805e9d931..fa8c68ad3a4 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -1,36 +1,28 @@ from collections.abc import Mapping -from dataclasses import dataclass from typing import Final -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError -from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_request, parse_systemone_response +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.secret_managers.main import ( get_secret_str, normalize_nonempty_secret_str, ) -from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse, UnsupportedDecisionsRequest _RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) -@dataclass(frozen=True, slots=True) -class CloudflareDecisionsEndpoint: - api_key_required: bool = True +class CloudflareDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("CLOUDFLARE_API_KEY",) + api_base_env = ("CLOUDFLARE_API_BASE",) - def configured_api_key(self) -> str | None: - return get_secret_str("CLOUDFLARE_API_KEY") or None - - def configured_api_base(self) -> str | None: - env_api_base: Final = get_secret_str("CLOUDFLARE_API_BASE") - if env_api_base: - return env_api_base + def get_default_api_base(self) -> str | None: account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID")) if account_id is None: return None return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run" - def missing_api_base_message(self, provider: str) -> str: + def missing_api_base_message(self, custom_llm_provider: str) -> str: return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly" def canonical_model(self, model: str) -> str: @@ -41,7 +33,7 @@ class CloudflareDecisionsEndpoint: def request_model(self, model: str) -> str: return model.rsplit("/", maxsplit=1)[-1] - def endpoint_url(self, api_base: str, model: str) -> str: + def get_complete_url(self, api_base: str, model: str) -> str: normalized_api_base: Final = api_base.rstrip("/") if normalized_api_base.endswith("/ai/v1"): return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}" @@ -49,25 +41,14 @@ class CloudflareDecisionsEndpoint: return f"{normalized_api_base}/{model}" return f"{normalized_api_base}/ai/run/{model}" - def request_body( - self, model: str, request: DecisionsIRRequest - ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_systemone_request(model, request) - - def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: - return parse_systemone_response(_unwrap_result(payload), request) - - -def _unwrap_result(payload: object) -> object: - if not isinstance(payload, Mapping): - return payload - response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) - if "answers" in response_mapping: - return payload - result: Final = response_mapping.get("result") - if isinstance(result, Mapping): - return result - return payload - - -CLOUDFLARE_DECISIONS_ENDPOINT: Final[CloudflareDecisionsEndpoint] = CloudflareDecisionsEndpoint() + def unwrap_response(self, payload: object) -> object: + try: + response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) + except ValidationError: + return payload + if "answers" in response_mapping: + return response_mapping + try: + return _RESPONSE_MAPPING_ADAPTER.validate_python(response_mapping.get("result")) + except ValidationError: + return response_mapping diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index dd97db45a88..8383078b7ba 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -3,7 +3,7 @@ import json import ssl from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager -from functools import lru_cache +from functools import lru_cache, partial from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -68,6 +68,7 @@ from litellm.llms.base_llm.base_model_iterator import ( from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.base_llm.containers.transformation import BaseContainerConfig +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.base_llm.files.transformation import ( @@ -120,6 +121,7 @@ from litellm.types.containers.main import ( ContainerObject, DeleteContainerResult, ) +from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig from litellm.types.integrations.custom_logger import ( NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, @@ -1413,6 +1415,132 @@ class BaseLLMHTTPHandler: request_data=request_data, ) + def _prepare_decisions_request( + self, + model: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + ) -> tuple[str, dict[str, str], dict[str, object]]: + outbound_headers: Final = provider_config.validate_environment(headers=headers, model=model, api_key=api_key) + url: Final = provider_config.get_complete_url(api_base=api_base, model=model) + data: Final = dict(body) + if logging_obj is not None: + logging_obj.pre_call( + input=data, + api_key=api_key, + model=model, + additional_args={"api_base": url, "complete_input_dict": data, "headers": outbound_headers}, + ) + return url, outbound_headers, data + + def decisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsIRRequest, + body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + litellm_params: Mapping[str, object], + client: HTTPHandler | None = None, + ) -> DecisionsIRResponse: + url, outbound_headers, data = self._prepare_decisions_request( + model=model, + logging_obj=logging_obj, + provider_config=provider_config, + body=body, + api_base=api_base, + api_key=api_key, + headers=headers, + ) + signed_headers, signed_body = provider_config.sign_request( + headers=outbound_headers, + url=url, + api_base=api_base, + body=data, + api_key=api_key, + litellm_params=litellm_params, + ) + sync_httpx_client: Final = client if client is not None else _get_httpx_client() + try: + response: Final = sync_httpx_client.post( + url, + content=signed_body, + json=data if signed_body is None else None, + headers=dict(signed_headers), + timeout=timeout, + logging_obj=logging_obj, + ) + except httpx.HTTPError as e: + raise self._handle_error(e=e, provider_config=provider_config) + return provider_config.transform_decisions_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request + ) + + async def adecisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsIRRequest, + body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + litellm_params: Mapping[str, object], + client: AsyncHTTPHandler | None = None, + ) -> DecisionsIRResponse: + url, outbound_headers, data = self._prepare_decisions_request( + model=model, + logging_obj=logging_obj, + provider_config=provider_config, + body=body, + api_base=api_base, + api_key=api_key, + headers=headers, + ) + sign: Final = partial( + provider_config.sign_request, + headers=outbound_headers, + url=url, + api_base=api_base, + body=data, + api_key=api_key, + litellm_params=litellm_params, + ) + signed_headers, signed_body = ( + await run_aws_signing(sign) if provider_config.signs_with_aws(api_base) else sign() + ) + async_httpx_client: Final = ( + client + if client is not None + else get_async_httpx_client(llm_provider=litellm.LlmProviders(custom_llm_provider)) + ) + try: + response: Final = await async_httpx_client.post( + url, + content=signed_body, + json=data if signed_body is None else None, + headers=dict(signed_headers), + timeout=timeout, + logging_obj=logging_obj, + ) + except httpx.HTTPError as e: + raise self._handle_error(e=e, provider_config=provider_config) + return provider_config.transform_decisions_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request + ) + def _prepare_audio_transcription_request( self, model: str, @@ -5926,6 +6054,7 @@ class BaseLLMHTTPHandler: e: Exception, provider_config: Union[ BaseConfig, + BaseDecisionsConfig, BaseRerankConfig, BaseResponsesAPIConfig, BaseImageEditConfig, diff --git a/litellm/llms/hosted_vllm/decisions/transformation.py b/litellm/llms/hosted_vllm/decisions/transformation.py new file mode 100644 index 00000000000..ef99f2de173 --- /dev/null +++ b/litellm/llms/hosted_vllm/decisions/transformation.py @@ -0,0 +1,26 @@ +from types import MappingProxyType +from typing import Final + +from litellm.constants import PLACEHOLDER_API_KEY +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class HostedVLLMDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("HOSTED_VLLM_API_KEY",) + api_base_env = ("HOSTED_VLLM_API_BASE",) + api_key_required = False + health_check_questions = MappingProxyType( + { + "reachable": MappingProxyType( + { + "type": "choice", + "instructions": "Is the service reachable?", + "criteria": {"yes": None, "no": None}, + } + ) + } + ) + + def resolve_api_key(self, api_key: str | None) -> str | None: + resolved: Final = super().resolve_api_key(api_key) + return None if resolved == PLACEHOLDER_API_KEY else resolved diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 003fccb7a8d..317e5abeeb5 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -1,12 +1,11 @@ from collections.abc import Mapping, Sequence -from dataclasses import dataclass from typing import Final from pydantic import TypeAdapter from typing_extensions import assert_never import litellm -from litellm.llms.base_llm.decisions.transformation import decisions_text +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( DecisionsIRAnswer, @@ -291,40 +290,32 @@ def ir_to_openai_response( ) -@dataclass(frozen=True, slots=True) -class OpenAIDecisionsEndpoint: - api_key_required: bool = True +class OpenAIDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" + supports_safety_identifier = True - def configured_api_key(self) -> str | None: - return litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + def get_default_api_base(self) -> str | None: + return "https://api.openai.com" - def configured_api_base(self) -> str | None: + def resolve_api_base(self, api_base: str | None) -> str | None: return ( - litellm.api_base + api_base + or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE") - or "https://api.openai.com" + or self.get_default_api_base() ) - def missing_api_base_message(self, provider: str) -> str: - return f"api_base is required for Decisions provider '{provider}'" + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") - def canonical_model(self, model: str) -> str: - return model - - def request_model(self, model: str) -> str: - return model - - def endpoint_url(self, api_base: str, model: str) -> str: - return f"{api_base.rstrip('/').removesuffix('/v1')}/v1/decisions" - - def request_body( - self, model: str, request: DecisionsIRRequest + def transform_decisions_request( + self, + model: str, + request: DecisionsIRRequest, + custom_llm_provider: str, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_openai_request(model, request) + return ir_to_openai_request(self.request_model(model), request) def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: return openai_response_to_ir(_OPENAI_RESPONSE_ADAPTER.validate_python(payload), request) - - -OPENAI_DECISIONS_ENDPOINT: Final[OpenAIDecisionsEndpoint] = OpenAIDecisionsEndpoint() diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 08d43c169f4..71e36a3abbe 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx -import litellm from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -40,18 +39,10 @@ class CacheControlSupportedModels(str, Enum): class OpenrouterConfig(OpenAIGPTConfig): def get_supported_openai_params(self, model: str) -> list: """ - Allow reasoning parameters for models flagged as reasoning-capable. + Include reasoning parameters regardless of model-map capabilities. """ supported_params: Final = super().get_supported_openai_params(model=model) - try: - if litellm.supports_reasoning(model=model, custom_llm_provider="openrouter") or litellm.supports_reasoning( - model=model - ): - supported_params.append("reasoning_effort") - supported_params.append("thinking") - except Exception: - pass - return list(dict.fromkeys(supported_params)) + return list(dict.fromkeys([*supported_params, "reasoning_effort", "thinking"])) def map_openai_params( self, diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 7a7466b1239..0fa5c7ca7cb 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,10 +1,19 @@ from typing import Final -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.types.decisions import DecisionsIRResponse -OPENROUTER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://openrouter.ai/api", - path="/alpha/decisions", - api_key_env=("OPENROUTER_API_KEY",), - api_base_env="OPENROUTER_API_BASE", -) + +class OpenRouterDecisionsConfig(BaseDecisionsConfig): + path = "/alpha/decisions" + api_key_env = ("OPENROUTER_API_KEY",) + api_base_env = ("OPENROUTER_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://openrouter.ai/api" + + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: + cost: Final = response.usage.extra.get("cost") + if not isinstance(cost, (int, float)): + return None + return float(cost) diff --git a/litellm/llms/perplexity/decisions/transformation.py b/litellm/llms/perplexity/decisions/transformation.py index 69a4753f4a3..11e2a38f06e 100644 --- a/litellm/llms/perplexity/decisions/transformation.py +++ b/litellm/llms/perplexity/decisions/transformation.py @@ -1,10 +1,10 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -PERPLEXITY_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://api.perplexity.ai", - path="/v1/decisions", - api_key_env=("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"), - api_base_env="PERPLEXITY_API_BASE", -) +class PerplexityDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" + api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY") + api_base_env = ("PERPLEXITY_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://api.perplexity.ai" diff --git a/litellm/llms/strands_decider/decisions/transformation.py b/litellm/llms/strands_decider/decisions/transformation.py index 265afb2b148..fd8a643be2a 100644 --- a/litellm/llms/strands_decider/decisions/transformation.py +++ b/litellm/llms/strands_decider/decisions/transformation.py @@ -1,11 +1,161 @@ +import hashlib +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from functools import partial +from types import MappingProxyType from typing import Final +from urllib.parse import quote, unquote -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint +import httpx +from pydantic import BaseModel, ConfigDict, ValidationError -STRANDS_DECIDER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value=None, - path="/v1/systemone", - api_key_env=("STRANDS_DECIDER_API_KEY",), - api_base_env="STRANDS_DECIDER_API_BASE", - api_key_required=False, +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, sign_aws_json_post +from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse +from litellm.types.llms.bedrock import AwsAuthParams + +AGENTCORE_SESSION_HEADER: Final = "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id" +_RESERVED_HEADER_PREFIXES: Final = ("x-amzn-bedrock-agentcore-runtime-", "x-amz-") +_RUNTIME_ARN: Final = re.compile( + r"\Aarn:aws(?:-[a-z]+)*:bedrock-agentcore:(?P[a-z0-9-]+):[0-9]{12}:runtime/[A-Za-z0-9_-]+\Z" ) +_INVOCATIONS_PATH: Final = re.compile(r"\A/runtimes/(?P[^/]+)/invocations\Z") +_RUNTIME_ERROR_STATUS: Final = MappingProxyType({"bad_request": 400, "invalid_request": 400, "loading": 503}) + + +@dataclass(frozen=True, slots=True) +class AgentCoreRuntime: + arn: str + region: str + + @property + def invocations_url(self) -> str: + host: Final = f"bedrock-agentcore.{self.region}.{get_aws_dns_suffix(self.region)}" + return f"https://{host}/runtimes/{quote(self.arn, safe='')}/invocations" + + @property + def default_session_id(self) -> str: + return f"litellm-decider-{hashlib.sha256(self.arn.encode()).hexdigest()}" + + +def agentcore_runtime(api_base: str) -> AgentCoreRuntime | None: + match: Final = _RUNTIME_ARN.match(api_base) + return None if match is None else AgentCoreRuntime(arn=api_base, region=match["region"]) + + +def invoked_runtime(url: httpx.URL) -> AgentCoreRuntime | None: + match: Final = _INVOCATIONS_PATH.match(url.raw_path.decode("ascii")) + runtime: Final = None if match is None else agentcore_runtime(unquote(match["arn"])) + return runtime if runtime is not None and runtime.invocations_url == str(url) else None + + +class _RuntimeError(BaseModel): + model_config = ConfigDict(frozen=True) + + code: str + message: str = "" + + +class _RuntimeReply(BaseModel): + model_config = ConfigDict(frozen=True) + + error: _RuntimeError | None = None + answers: object = None + + +def _runtime_error(content: bytes) -> _RuntimeError | None: + try: + reply: Final = _RuntimeReply.model_validate_json(content) + except ValidationError: + return None + return reply.error if reply.answers is None else None + + +class _SessionParams(BaseModel): + model_config = ConfigDict(frozen=True) + + agentcore_runtime_session_id: str | None = None + + +def _is_reserved_header(name: str) -> bool: + lowered: Final = name.lower() + return lowered == "host" or lowered.startswith(_RESERVED_HEADER_PREFIXES) + + +def _runtime_headers( + headers: Mapping[str, str], runtime: AgentCoreRuntime, litellm_params: Mapping[str, object] +) -> Mapping[str, str]: + dropped: Final = sorted(name for name in headers if _is_reserved_header(name)) + if dropped: + verbose_logger.warning("Strands Decider: not forwarding reserved AgentCore header(s) %s", dropped) + configured: Final = _SessionParams.model_validate(litellm_params).agentcore_runtime_session_id + return { + **{name: value for name, value in headers.items() if not _is_reserved_header(name)}, + AGENTCORE_SESSION_HEADER: runtime.default_session_id if configured is None else configured, + } + + +class StrandsDeciderDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("STRANDS_DECIDER_API_KEY",) + api_base_env = ("STRANDS_DECIDER_API_BASE",) + api_key_required = False + + def __init__(self, aws: BaseAWSLLM | None = None) -> None: + self._aws: Final = aws if aws is not None else BaseAWSLLM() + + def get_complete_url(self, api_base: str, model: str) -> str: + runtime: Final = agentcore_runtime(api_base) + return super().get_complete_url(api_base, model) if runtime is None else runtime.invocations_url + + def signs_with_aws(self, api_base: str) -> bool: + return agentcore_runtime(api_base) is not None + + def sign_request( + self, + headers: Mapping[str, str], + url: str, + api_base: str, + body: Mapping[str, object], + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> tuple[Mapping[str, str], bytes | None]: + runtime: Final = agentcore_runtime(api_base) + if runtime is None: + return headers, None + session_headers: Final = _runtime_headers(headers, runtime, litellm_params) + if api_key is not None: + return session_headers, None + payload: Final = json.dumps(body) + signed: Final = sign_aws_json_post( + get_credentials=partial( + self._aws.resolve_credentials, AwsAuthParams.model_validate(litellm_params), runtime.region + ), + service_name="bedrock-agentcore", + aws_region_name=runtime.region, + url=url, + body=payload, + headers=session_headers, + ) + return dict(signed.headers.items()), payload.encode() + + def transform_decisions_response( + self, + model: str, + custom_llm_provider: str, + raw_response: httpx.Response, + request: DecisionsIRRequest, + ) -> DecisionsIRResponse: + runtime_error: Final = ( + None if invoked_runtime(raw_response.request.url) is None else _runtime_error(raw_response.content) + ) + if runtime_error is not None: + raise BaseLLMException( + status_code=_RUNTIME_ERROR_STATUS.get(runtime_error.code, 500), + message=f"Strands Decider runtime error '{runtime_error.code}': {runtime_error.message}", + ) + return super().transform_decisions_response(model, custom_llm_provider, raw_response, request) diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py index 17fb24ce443..e71c526a840 100644 --- a/litellm/llms/typesafe/decisions/transformation.py +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -1,10 +1,9 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -TYPESAFE_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://api.typesafe.ai", - path="/v1/systemone", - api_key_env=("TYPESAFE_API_KEY",), - api_base_env="TYPESAFE_API_BASE", -) +class TypeSafeDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("TYPESAFE_API_KEY",) + api_base_env = ("TYPESAFE_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://api.typesafe.ai" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 94ebc0bd735..d42e66605d7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -79765,5 +79765,12 @@ "metadata": { "notes": "Self-hosted decision model; infrastructure costs are paid separately" } + }, + "azure_ai/Microsoft-Decision-1": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "azure_ai", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" } } diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index ed697c2525c..23beced550d 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -9494,17 +9494,878 @@ }, "decisions": { "components": { - "schemas": {} + "schemas": { + "ChoiceAnswer": { + "additionalProperties": true, + "properties": { + "choice": { + "title": "Choice", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "confidence", + "probabilities" + ], + "title": "ChoiceAnswer", + "type": "object" + }, + "DecisionsResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "additionalProperties": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/ChoiceAnswer", + "noul": "#/components/schemas/NoulAnswer", + "score": "#/components/schemas/ScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/NoulAnswer" + }, + { + "$ref": "#/components/schemas/ChoiceAnswer" + }, + { + "$ref": "#/components/schemas/ScoreAnswer" + } + ] + }, + "title": "Answers", + "type": "object" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/components/schemas/DecisionsUsage" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "answers" + ], + "title": "DecisionsResponse", + "type": "object" + }, + "DecisionsUsage": { + "additionalProperties": true, + "properties": { + "input_tokens": { + "default": 0, + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "default": 0, + "title": "Output Tokens", + "type": "integer" + } + }, + "title": "DecisionsUsage", + "type": "object" + }, + "NoulAnswer": { + "additionalProperties": true, + "properties": { + "noul": { + "title": "Noul", + "type": "number" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "title": "NoulAnswer", + "type": "object" + }, + "OpenAIChoiceAnswer": { + "properties": { + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Choice" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probabilities": { + "items": { + "$ref": "#/components/schemas/OpenAIChoiceProbability" + }, + "title": "Probabilities", + "type": "array" + }, + "type": { + "const": "choice", + "default": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "choice", + "probabilities", + "confidence" + ], + "title": "OpenAIChoiceAnswer", + "type": "object" + }, + "OpenAIChoiceProbability": { + "properties": { + "probability": { + "title": "Probability", + "type": "number" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value", + "probability" + ], + "title": "OpenAIChoiceProbability", + "type": "object" + }, + "OpenAIDecisionInputTokensDetails": { + "properties": { + "cache_write_tokens": { + "default": 0, + "title": "Cache Write Tokens", + "type": "integer" + }, + "cached_tokens": { + "default": 0, + "title": "Cached Tokens", + "type": "integer" + } + }, + "title": "OpenAIDecisionInputTokensDetails", + "type": "object" + }, + "OpenAIDecisionOutputTokensDetails": { + "properties": { + "reasoning_tokens": { + "default": 0, + "title": "Reasoning Tokens", + "type": "integer" + } + }, + "title": "OpenAIDecisionOutputTokensDetails", + "type": "object" + }, + "OpenAIDecisionResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "items": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/OpenAIChoiceAnswer", + "predicate": "#/components/schemas/OpenAIPredicateAnswer", + "refusal": "#/components/schemas/OpenAIRefusalAnswer", + "score": "#/components/schemas/OpenAIScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/OpenAIPredicateAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIChoiceAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIScoreAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIRefusalAnswer" + } + ] + }, + "title": "Answers", + "type": "array" + }, + "model": { + "title": "Model", + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/OpenAIDecisionUsage" + } + }, + "required": [ + "model", + "answers", + "usage" + ], + "title": "OpenAIDecisionResponse", + "type": "object" + }, + "OpenAIDecisionUsage": { + "properties": { + "input_tokens": { + "title": "Input Tokens", + "type": "integer" + }, + "input_tokens_details": { + "$ref": "#/components/schemas/OpenAIDecisionInputTokensDetails", + "default": { + "cache_write_tokens": 0, + "cached_tokens": 0 + } + }, + "output_tokens": { + "title": "Output Tokens", + "type": "integer" + }, + "output_tokens_details": { + "$ref": "#/components/schemas/OpenAIDecisionOutputTokensDetails", + "default": { + "reasoning_tokens": 0 + } + }, + "total_tokens": { + "title": "Total Tokens", + "type": "integer" + } + }, + "required": [ + "input_tokens", + "output_tokens", + "total_tokens" + ], + "title": "OpenAIDecisionUsage", + "type": "object" + }, + "OpenAIPredicateAnswer": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probability": { + "title": "Probability", + "type": "number" + }, + "type": { + "const": "predicate", + "default": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "probability" + ], + "title": "OpenAIPredicateAnswer", + "type": "object" + }, + "OpenAIRefusalAnswer": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "type": { + "const": "refusal", + "default": "refusal", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name" + ], + "title": "OpenAIRefusalAnswer", + "type": "object" + }, + "OpenAIScoreAnswer": { + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probabilities": { + "items": { + "$ref": "#/components/schemas/OpenAIScoreProbability" + }, + "title": "Probabilities", + "type": "array" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "default": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "score", + "probabilities", + "confidence" + ], + "title": "OpenAIScoreAnswer", + "type": "object" + }, + "OpenAIScoreProbability": { + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "probability": { + "title": "Probability", + "type": "number" + }, + "value": { + "title": "Value", + "type": "integer" + } + }, + "required": [ + "value", + "label", + "probability" + ], + "title": "OpenAIScoreProbability", + "type": "object" + }, + "ScoreAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "legend": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "title": "Legend", + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "score", + "confidence", + "legend", + "probabilities" + ], + "title": "ScoreAnswer", + "type": "object" + } + } }, "paths": { "/decisions": { "post": { + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_decisions_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "model": "luna", + "questions": [ + { + "instructions": "Is the customer asking for a refund?", + "name": "is_refund_request", + "type": "predicate" + }, + { + "choices": [ + { + "description": "can wait a week", + "value": "low" + }, + { + "description": "needs action today", + "value": "high" + } + ], + "instructions": "How urgent is this?", + "name": "urgency", + "type": "choice" + }, + { + "instructions": "How frustrated is the customer?", + "levels": [ + { + "label": "calm" + }, + { + "label": "mildly annoyed" + }, + { + "label": "angry" + } + ], + "name": "frustration", + "type": "score" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "input_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "OpenAIDecisionInputText", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "image_url": { + "title": "Image Url", + "type": "string" + }, + "type": { + "const": "input_image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "OpenAIDecisionInputImage", + "type": "object" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "default": "user", + "title": "Role", + "type": "string" + }, + "type": { + "const": "message", + "default": "message", + "title": "Type", + "type": "string" + } + }, + "required": [ + "content" + ], + "title": "OpenAIDecisionInputMessage", + "type": "object" + }, + "type": "array" + } + ], + "description": "The text being judged, or user messages whose content mixes input_text and input_image parts", + "title": "Input" + }, + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "description": "Predicate, choice and score questions. Answers come back in the same order", + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The yes/no question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIPredicateQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "choices": { + "description": "The options the model picks from. Values must be unique", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "OpenAIChoiceOption", + "type": "object" + }, + "maxItems": 255, + "minItems": 2, + "title": "Choices", + "type": "array" + }, + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "choices" + ], + "title": "OpenAIChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "levels": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label" + ], + "title": "OpenAIScoreLevel", + "type": "object" + }, + "maxItems": 10, + "minItems": 2, + "title": "Levels", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "levels" + ], + "title": "OpenAIScoreQuestion", + "type": "object" + } + ] + }, + "maxItems": 128, + "minItems": 1, + "title": "Questions", + "type": "array" + }, + "safety_identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", + "title": "Safety Identifier" + } + }, + "required": [ + "input", + "questions", + "model" + ], + "title": "OpenAIDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/OpenAIDecisionResponse" + } } }, "description": "Successful Response" @@ -9515,7 +10376,7 @@ "APIKeyHeader": [] } ], - "summary": "Decisions", + "summary": "Ask questions about an input (OpenAI Decisions format)", "tags": [ "decisions" ] @@ -9523,12 +10384,290 @@ }, "/systemone": { "post": { + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "jev", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "urgency": { + "criteria": { + "high": "needs action today", + "low": "can wait a week" + }, + "instructions": "How urgent is this?", + "type": "choice" + } + }, + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + }, + "schema": { + "additionalProperties": true, + "properties": { + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": true, + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what makes the answer true and what makes it false", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The yes/no question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "NoulQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "description": "Candidate labels mapped to an optional description of each", + "maxProperties": 255, + "minProperties": 1, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "maxItems": 10, + "minItems": 1, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ScoreQuestion", + "type": "object" + } + ] + }, + "description": "Named noul, choice and score questions. Each key becomes a key in the answers", + "maxProperties": 128, + "minProperties": 1, + "propertyNames": { + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The thing being judged, such as a support ticket, a document or a chat transcript", + "title": "State" + } + }, + "required": [ + "state", + "questions", + "model" + ], + "title": "DecisionsRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/DecisionsResponse" + } } }, "description": "Successful Response" @@ -9539,7 +10678,7 @@ "APIKeyHeader": [] } ], - "summary": "Systemone", + "summary": "Ask named questions about a state (System One format)", "tags": [ "decisions" ] @@ -9547,12 +10686,382 @@ }, "/v1/decisions": { "post": { + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_v1_decisions_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "model": "luna", + "questions": [ + { + "instructions": "Is the customer asking for a refund?", + "name": "is_refund_request", + "type": "predicate" + }, + { + "choices": [ + { + "description": "can wait a week", + "value": "low" + }, + { + "description": "needs action today", + "value": "high" + } + ], + "instructions": "How urgent is this?", + "name": "urgency", + "type": "choice" + }, + { + "instructions": "How frustrated is the customer?", + "levels": [ + { + "label": "calm" + }, + { + "label": "mildly annoyed" + }, + { + "label": "angry" + } + ], + "name": "frustration", + "type": "score" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "input_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "OpenAIDecisionInputText", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "image_url": { + "title": "Image Url", + "type": "string" + }, + "type": { + "const": "input_image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "OpenAIDecisionInputImage", + "type": "object" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "default": "user", + "title": "Role", + "type": "string" + }, + "type": { + "const": "message", + "default": "message", + "title": "Type", + "type": "string" + } + }, + "required": [ + "content" + ], + "title": "OpenAIDecisionInputMessage", + "type": "object" + }, + "type": "array" + } + ], + "description": "The text being judged, or user messages whose content mixes input_text and input_image parts", + "title": "Input" + }, + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "description": "Predicate, choice and score questions. Answers come back in the same order", + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The yes/no question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIPredicateQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "choices": { + "description": "The options the model picks from. Values must be unique", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "OpenAIChoiceOption", + "type": "object" + }, + "maxItems": 255, + "minItems": 2, + "title": "Choices", + "type": "array" + }, + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "choices" + ], + "title": "OpenAIChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "levels": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label" + ], + "title": "OpenAIScoreLevel", + "type": "object" + }, + "maxItems": 10, + "minItems": 2, + "title": "Levels", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "levels" + ], + "title": "OpenAIScoreQuestion", + "type": "object" + } + ] + }, + "maxItems": 128, + "minItems": 1, + "title": "Questions", + "type": "array" + }, + "safety_identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", + "title": "Safety Identifier" + } + }, + "required": [ + "input", + "questions", + "model" + ], + "title": "OpenAIDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/OpenAIDecisionResponse" + } } }, "description": "Successful Response" @@ -9563,7 +11072,7 @@ "APIKeyHeader": [] } ], - "summary": "Decisions", + "summary": "Ask questions about an input (OpenAI Decisions format)", "tags": [ "decisions" ] @@ -9571,12 +11080,290 @@ }, "/v1/systemone": { "post": { + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_v1_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "jev", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "urgency": { + "criteria": { + "high": "needs action today", + "low": "can wait a week" + }, + "instructions": "How urgent is this?", + "type": "choice" + } + }, + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + }, + "schema": { + "additionalProperties": true, + "properties": { + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": true, + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what makes the answer true and what makes it false", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The yes/no question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "NoulQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "description": "Candidate labels mapped to an optional description of each", + "maxProperties": 255, + "minProperties": 1, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "maxItems": 10, + "minItems": 1, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ScoreQuestion", + "type": "object" + } + ] + }, + "description": "Named noul, choice and score questions. Each key becomes a key in the answers", + "maxProperties": 128, + "minProperties": 1, + "propertyNames": { + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The thing being judged, such as a support ticket, a document or a chat transcript", + "title": "State" + } + }, + "required": [ + "state", + "questions", + "model" + ], + "title": "DecisionsRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/DecisionsResponse" + } } }, "description": "Successful Response" @@ -9587,7 +11374,7 @@ "APIKeyHeader": [] } ], - "summary": "Systemone", + "summary": "Ask named questions about a state (System One format)", "tags": [ "decisions" ] diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 813d72ed7ae..f20bdada863 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -343,6 +343,8 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( "anthropic_workspace_id", "anthropic-workspace-id", "bedrock_tags", + # Deployment-pinned AgentCore session: every new caller-chosen value provisions another microVM. + "agentcore_runtime_session_id", # Provider-specific endpoint overrides that flow into the outbound # request via ``optional_params``. Same threat as ``api_base``: # ``s3_endpoint_url`` redirects Bedrock file uploads to attacker diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index ae989bb6a00..31e84fd4d9e 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -174,6 +174,7 @@ ProxyRouteType: TypeAlias = Literal[ "aocr", "asearch", "adecisions", + "asystemone", "avideo_generation", "avideo_list", "avideo_status", @@ -1960,6 +1961,7 @@ class ProxyBaseLLMRequestProcessing: "aocr", "asearch", "adecisions", + "asystemone", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index c1d4442893a..45ca509b7b7 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -437,3 +437,41 @@ class CustomOpenAPISpec: # Add responses API request schema return CustomOpenAPISpec.add_responses_api_request_schema(with_embeddings) + + +_MAX_INLINE_DEPTH: Final = 32 + + +def _inlined(node: JsonValue, defs: Mapping[str, JsonValue], depth: int = 0) -> JsonValue: + if depth >= _MAX_INLINE_DEPTH or not isinstance(node, (list, dict)): + return node + if isinstance(node, list): + return [_inlined(item, defs, depth + 1) for item in node] + ref: Final = node.get("$ref") + if isinstance(ref, str) and ref.startswith("#/$defs/"): + target: Final = _inlined(defs[ref.removeprefix("#/$defs/")], defs, depth + 1) + siblings: Final = {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$ref"} + return {**target, **siblings} if isinstance(target, dict) else siblings + if "propertyName" in node and "mapping" in node: + return {"propertyName": node["propertyName"]} + return {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$defs"} + + +def inline_request_body(model_class: type, example: Mapping[str, object]) -> JsonObject: + """ + Build an ``openapi_extra["requestBody"]`` for a route that reads its body with ``request.body()``. + + Swagger UI resolves ``$ref`` against the whole document, so a schema placed inline in an operation + cannot keep Pydantic's ``#/$defs/...`` references. Every reference is expanded in place instead, + which also keeps the operation self-contained inside the lazy OpenAPI snapshot. + """ + schema: Final = CustomOpenAPISpec.get_pydantic_schema(model_class) + media: Final[JsonObject] = {"example": cast(JsonValue, dict(example))} # cast-ok: JSON example literal + if schema is None: + return {"required": True, "content": {"application/json": media}} + raw_defs: Final = schema.get("$defs") + defs: Final[Mapping[str, JsonValue]] = raw_defs if isinstance(raw_defs, dict) else MappingProxyType({}) + return { + "required": True, + "content": {"application/json": {"schema": _inlined(schema, defs), **media}}, + } diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 86fc4f9da2c..ac93d2178b9 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -1,5 +1,6 @@ from collections.abc import Mapping -from typing import Annotated, Final +from types import MappingProxyType +from typing import Annotated, Final, Literal from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract @@ -8,7 +9,15 @@ from pydantic import TypeAdapter, ValidationError from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body +from litellm.types.decisions import ( + DecisionsRequest, + DecisionsRequestBody, + DecisionsResponse, + OpenAIDecisionRequest, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, +) router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) @@ -19,6 +28,51 @@ _OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBo _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) _OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) +_SYSTEMONE_REQUEST_BODY: Final = inline_request_body( + DecisionsRequest, + { + "model": "jev", + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "questions": { + "is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "urgency": { + "type": "choice", + "instructions": "How urgent is this?", + "criteria": {"low": "can wait a week", "high": "needs action today"}, + }, + "frustration": { + "type": "score", + "instructions": "How frustrated is the customer?", + "criteria": ["calm", "mildly annoyed", "angry"], + }, + }, + }, +) +_DECISIONS_REQUEST_BODY: Final = inline_request_body( + OpenAIDecisionRequest, + { + "model": "luna", + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "questions": [ + {"type": "predicate", "name": "is_refund_request", "instructions": "Is the customer asking for a refund?"}, + { + "type": "choice", + "name": "urgency", + "instructions": "How urgent is this?", + "choices": [ + {"value": "low", "description": "can wait a week"}, + {"value": "high", "description": "needs action today"}, + ], + }, + { + "type": "score", + "name": "frustration", + "instructions": "How frustrated is the customer?", + "levels": [{"label": "calm"}, {"label": "mildly annoyed"}, {"label": "angry"}], + }, + ], + }, +) async def _invalid_request( @@ -38,6 +92,10 @@ async def _invalid_request( ) +def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType({key: value for key, value in data.items() if key != "safety_identifier"}) + + async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: body: Final = await request.body() try: @@ -51,6 +109,7 @@ async def _process_decisions( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody], + route_type: Literal["adecisions", "asystemone"], ) -> object: from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, @@ -75,7 +134,7 @@ async def _process_decisions( data: Final = await _request_data(request, user_api_key_dict) try: - body_adapter.validate_python(data) + body_adapter.validate_python(_fields_checked_before_routing(data)) except ValidationError as error: raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict) general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings) @@ -88,7 +147,7 @@ async def _process_decisions( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, - route_type="adecisions", + route_type=route_type, proxy_logging_obj=proxy_logging_obj, llm_router=llm_router, general_settings=general_settings, @@ -116,23 +175,41 @@ async def _process_decisions( dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask named questions about a state (System One format)", + responses={200: {"model": DecisionsResponse}}, + openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY}, ) @router.post( "/systemone", dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask named questions about a state (System One format)", + responses={200: {"model": DecisionsResponse}}, + openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY}, ) async def systemone( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): + """ + Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + + Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + + [Docs](https://docs.litellm.ai/docs/decisions) + """ return await _process_decisions( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, body_adapter=_DECISIONS_REQUEST_BODY_ADAPTER, + route_type="asystemone", ) @@ -141,21 +218,39 @@ async def systemone( dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask questions about an input (OpenAI Decisions format)", + responses={200: {"model": OpenAIDecisionResponse}}, + openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY}, ) @router.post( "/decisions", dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask questions about an input (OpenAI Decisions format)", + responses={200: {"model": OpenAIDecisionResponse}}, + openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY}, ) async def decisions( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): + """ + Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + + Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + model list. System One decision models accept this format too, LiteLLM translates the request and the + answers. Streaming is not supported. + + [Docs](https://docs.litellm.ai/docs/decisions) + """ return await _process_decisions( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, body_adapter=_OPENAI_DECISION_REQUEST_BODY_ADAPTER, + route_type="adecisions", ) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index c7a6a4f30b0..2552b274427 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -1405,7 +1405,7 @@ async def get_guardrail_ui_settings(): - PII entity categories for UI organization - Content filter settings (patterns and categories) """ - from litellm.decisions.main import supported_decisions_providers + from litellm.decisions.call import supported_decisions_providers from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.patterns import ( PATTERN_CATEGORIES, get_available_content_categories, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py index 3ad92acb48a..e13c9d0c1ae 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py @@ -17,6 +17,7 @@ from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject class _TypeSafeUsage(BaseModel): input_tokens: int = 0 output_tokens: int = 0 + cost: float | None = None class _TypeSafeResponse(BaseModel): @@ -82,7 +83,9 @@ class TypeSafePassthroughLoggingHandler: ) pricing: Final = _pricing_for(candidate_model_keys) response_cost: Final = ( - input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token + usage.cost + if custom_llm_provider == "openrouter" and usage.cost is not None + else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token ) usage_object: Final = Usage( prompt_tokens=input_tokens, diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 11d2ff61b95..bdfeddba3b3 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3161,6 +3161,34 @@ ], "default_model_placeholder": "soniox/stt-async-v5" }, + { + "provider": "StrandsDecider", + "provider_display_name": "Strands Decider", + "litellm_provider": "strands_decider", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "http://localhost:8000", + "tooltip": "URL of your self-hosted Strands Decider server", + "required": true, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "strands_decider/strands-decider-2B-hobson-v19" + }, { "provider": "TEXT_COMPLETION_CODESTRAL", "provider_display_name": "Text-Completion-Codestral", @@ -3263,6 +3291,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "TypeSafe", + "provider_display_name": "TypeSafe", + "litellm_provider": "typesafe", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.typesafe.ai", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "typesafe/jev-latest" + }, { "provider": "V0", "provider_display_name": "V0", diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index a2741c9bb8f..743f3003c63 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -91,6 +91,7 @@ ROUTE_ENDPOINT_MAPPING: Final = { "aocr": "/ocr", "asearch": "/search", "adecisions": "/decisions", + "asystemone": "/systemone", "avideo_generation": "/videos", "avideo_list": "/videos", "avideo_status": "/videos/{video_id}", @@ -375,6 +376,7 @@ RouteType = Literal[ "aocr", "asearch", "adecisions", + "asystemone", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/router.py b/litellm/router.py index 76abe7e2bab..1ba2544cdb8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2091,6 +2091,11 @@ class Router: self.adecisions = self.factory_function(adecisions, call_type="adecisions") self.decisions = self.factory_function(decisions, call_type="decisions") + from litellm.systemone import asystemone, systemone + + self.asystemone = self.factory_function(asystemone, call_type="asystemone") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable + self.systemone = self.factory_function(systemone, call_type="systemone") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable + def _initialize_video_endpoints(self): """Initialize video endpoints.""" from litellm.videos import ( @@ -6646,6 +6651,8 @@ class Router: "search", "adecisions", "decisions", + "asystemone", + "systemone", "aadapter_generate_content", "avideo_generation", "video_generation", @@ -6720,6 +6727,7 @@ class Router: "ocr", "search", "decisions", + "systemone", "video_generation", "video_list", "video_status", @@ -6888,6 +6896,7 @@ class Router: "aocr", "ocr", "adecisions", + "asystemone", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/systemone/__init__.py b/litellm/systemone/__init__.py new file mode 100644 index 00000000000..eb4971a16e4 --- /dev/null +++ b/litellm/systemone/__init__.py @@ -0,0 +1,3 @@ +from litellm.systemone.main import asystemone, systemone + +__all__ = ["asystemone", "systemone"] diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py new file mode 100644 index 00000000000..754b69142f9 --- /dev/null +++ b/litellm/systemone/main.py @@ -0,0 +1,107 @@ +from collections.abc import Mapping +from typing import Final + +import httpx + +from litellm.decisions.call import ( + DecisionsCall, + SystemOneQuestions, + SystemOneRequestFields, + asend, + prepare_call, + reject_other_format, + send, +) +from litellm.types.decisions import DecisionsJSON, DecisionsResponse +from litellm.utils import client # pyright: ignore[reportUnknownVariableType] # client is an untyped decorator + + +def _prepare_systemone_call( + *, + model: str, + state: DecisionsJSON | None, + questions: SystemOneQuestions | None, + safety_identifier: str | None, + api_key: str | None, + api_base: str | None, + timeout: float | httpx.Timeout | None, + custom_llm_provider: str | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> DecisionsCall: + reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions") + return prepare_call( + model=model, + fields=SystemOneRequestFields(state=state, questions=questions), + safety_identifier=safety_identifier, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) + + +def _systemone_response(response: object) -> DecisionsResponse: + assert isinstance(response, DecisionsResponse) + return response + + +@client +async def asystemone( + model: str, + state: DecisionsJSON | None = None, + questions: SystemOneQuestions | None = None, + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: Mapping[str, str] | None = None, + safety_identifier: str | None = None, + **kwargs: object, # kwargs-ok: litellm_params forwarded by @client +) -> DecisionsResponse: + call: Final = _prepare_systemone_call( + model=model, + state=state, + questions=questions, + safety_identifier=safety_identifier, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) + return _systemone_response(await asend(call)) + + +@client +def systemone( + model: str, + state: DecisionsJSON | None = None, + questions: SystemOneQuestions | None = None, + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: Mapping[str, str] | None = None, + safety_identifier: str | None = None, + **kwargs: object, # kwargs-ok: litellm_params forwarded by @client +) -> DecisionsResponse: + call: Final = _prepare_systemone_call( + model=model, + state=state, + questions=questions, + safety_identifier=safety_identifier, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) + return _systemone_response(send(call)) + + +__all__ = ["asystemone", "systemone"] diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 6488ef45308..3bc90a5109a 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -15,8 +15,13 @@ MAX_DECISION_QUESTIONS: Final = 128 class NoulQuestion(LiteLLMPydanticObjectBase): type: Literal["noul"] - instructions: DecisionsJSON | None = None - criteria: NoulCriteria | None = None + instructions: Annotated[DecisionsJSON | None, Field(description="The yes/no question to ask about the state")] = ( + None + ) + criteria: Annotated[ + NoulCriteria | None, + Field(description="Optional descriptions of what makes the answer true and what makes it false"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) @@ -29,16 +34,24 @@ class NoulQuestion(LiteLLMPydanticObjectBase): class ChoiceQuestion(LiteLLMPydanticObjectBase): type: Literal["choice"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)] + instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None + criteria: Annotated[ + Mapping[str, DecisionsJSON | None], + Field(min_length=1, max_length=255, description="Candidate labels mapped to an optional description of each"), + ] model_config = ConfigDict(extra="allow", frozen=True) class ScoreQuestion(LiteLLMPydanticObjectBase): type: Literal["score"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)] + instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None + criteria: Annotated[ + Sequence[DecisionsJSON], + Field( + min_length=1, max_length=10, description="Scale levels from lowest to highest. The score is a level index" + ), + ] model_config = ConfigDict(extra="allow", frozen=True) @@ -55,14 +68,20 @@ DecisionQuestionMap: TypeAlias = Annotated[ class DecisionsRequestBody(LiteLLMPydanticObjectBase): - state: DecisionsJSON - questions: DecisionQuestionMap + state: Annotated[ + DecisionsJSON, + Field(description="The thing being judged, such as a support ticket, a document or a chat transcript"), + ] + questions: Annotated[ + DecisionQuestionMap, + Field(description="Named noul, choice and score questions. Each key becomes a key in the answers"), + ] model_config = ConfigDict(extra="allow", frozen=True) class DecisionsRequest(DecisionsRequestBody): - model: str + model: Annotated[str, Field(description="A decision model from the proxy model_list")] @with_config(ConfigDict(extra="allow")) @@ -121,6 +140,13 @@ class DecisionsUsage(LiteLLMPydanticObjectBase): class _HiddenParamsResponse(LiteLLMPydanticObjectBase): _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) + @property + def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation + return self._hidden_params + + def set_hidden_params(self, params: Mapping[str, object]) -> None: + self._hidden_params.update(params) + class DecisionsResponse(_HiddenParamsResponse): model: str | None = None @@ -164,8 +190,8 @@ OpenAIDecisionInput: TypeAlias = str | Sequence[OpenAIDecisionInputMessage] class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase): type: Literal["predicate"] - name: str | None = None - instructions: str + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The yes/no question to ask about the input")] model_config = ConfigDict(extra="forbid", frozen=True) @@ -185,9 +211,12 @@ def systemone_choice_key(value: str | bool) -> str: class OpenAIChoiceQuestion(LiteLLMPydanticObjectBase): type: Literal["choice"] - name: str | None = None - instructions: str - choices: Annotated[Sequence[OpenAIChoiceOption], Field(min_length=2, max_length=255)] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The question to ask about the input")] + choices: Annotated[ + Sequence[OpenAIChoiceOption], + Field(min_length=2, max_length=255, description="The options the model picks from. Values must be unique"), + ] model_config = ConfigDict(extra="forbid", frozen=True) @@ -208,9 +237,14 @@ class OpenAIScoreLevel(LiteLLMPydanticObjectBase): class OpenAIScoreQuestion(LiteLLMPydanticObjectBase): type: Literal["score"] - name: str | None = None - instructions: str - levels: Annotated[Sequence[OpenAIScoreLevel], Field(min_length=2, max_length=10)] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The question to ask about the input")] + levels: Annotated[ + Sequence[OpenAIScoreLevel], + Field( + min_length=2, max_length=10, description="Scale levels from lowest to highest. The score is a level index" + ), + ] model_config = ConfigDict(extra="forbid", frozen=True) @@ -222,13 +256,32 @@ OpenAIDecisionQuestion: TypeAlias = Annotated[ class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): - input: OpenAIDecisionInput - questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)] - safety_identifier: str | None = None + input: Annotated[ + OpenAIDecisionInput, + Field( + description="The text being judged, or user messages whose content mixes input_text and input_image parts" + ), + ] + questions: Annotated[ + Sequence[OpenAIDecisionQuestion], + Field( + min_length=1, + max_length=MAX_DECISION_QUESTIONS, + description="Predicate, choice and score questions. Answers come back in the same order", + ), + ] + safety_identifier: Annotated[ + str | None, + Field(description="A stable id for your end user. Sent to OpenAI and dropped for other providers"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) +class OpenAIDecisionRequest(OpenAIDecisionRequestBody): + model: Annotated[str, Field(description="A decision model from the proxy model_list")] + + class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase): type: Literal["predicate"] = "predicate" name: str | None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 561965633b5..a673db3db86 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -469,6 +469,8 @@ class CallTypes(str, Enum): asearch = "asearch" decisions = "decisions" adecisions = "adecisions" + systemone = "systemone" + asystemone = "asystemone" arealtime = "_arealtime" aresponses_websocket = "_aresponses_websocket" create_batch = "create_batch" @@ -657,6 +659,8 @@ CallTypesLiteral = Literal[ "asearch", "decisions", "adecisions", + "systemone", + "asystemone", "_arealtime", "_aresponses_websocket", "create_batch", @@ -768,8 +772,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/search": [CallTypes.asearch, CallTypes.search], "/decisions": [CallTypes.adecisions, CallTypes.decisions], "/v1/decisions": [CallTypes.adecisions, CallTypes.decisions], - "/systemone": [CallTypes.adecisions, CallTypes.decisions], - "/v1/systemone": [CallTypes.adecisions, CallTypes.decisions], + "/systemone": [CallTypes.asystemone, CallTypes.systemone], + "/v1/systemone": [CallTypes.asystemone, CallTypes.systemone], # Batches "/batches": [CallTypes.acreate_batch, CallTypes.create_batch], "/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch], diff --git a/litellm/utils.py b/litellm/utils.py index f2da0aa42ab..0c829520c72 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -381,6 +381,7 @@ if TYPE_CHECKING: # Type stubs for lazy-loaded config classes and types from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.containers.transformation import BaseContainerConfig + from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.files.transformation import BaseFilesConfig from litellm.llms.base_llm.google_genai.transformation import ( @@ -971,6 +972,12 @@ def _ocr_document_summary(document: object) -> str: # Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc. + +_DECISION_CALL_TYPES: Final[frozenset[str]] = frozenset( + (CallTypes.decisions.value, CallTypes.adecisions.value, CallTypes.systemone.value, CallTypes.asystemone.value) +) + + def function_setup( original_function: str, rules_obj: Rules, @@ -1217,7 +1224,7 @@ def function_setup( if isinstance(search_query, list) else search_query ) - elif call_type in (CallTypes.decisions.value, CallTypes.adecisions.value): + elif call_type in _DECISION_CALL_TYPES: decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state") or kwargs.get("input") or "" messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state, default=str) elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): @@ -8897,6 +8904,26 @@ class ProviderConfigManager: return get_dashscope_family_rerank_config(provider.value) return litellm.CohereRerankConfig() + @staticmethod + def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None: + if provider == LlmProviders.PERPLEXITY: + return litellm.PerplexityDecisionsConfig() + if provider == LlmProviders.TYPESAFE: + return litellm.TypeSafeDecisionsConfig() + if provider == LlmProviders.OPENROUTER: + return litellm.OpenRouterDecisionsConfig() + if provider == LlmProviders.CLOUDFLARE: + return litellm.CloudflareDecisionsConfig() + if provider == LlmProviders.STRANDS_DECIDER: + return litellm.StrandsDeciderDecisionsConfig() + if provider == LlmProviders.HOSTED_VLLM: + return litellm.HostedVLLMDecisionsConfig() + if provider == LlmProviders.OPENAI: + return litellm.OpenAIDecisionsConfig() + if provider == LlmProviders.AZURE_AI: + return litellm.AzureAIDecisionsConfig() + return None + @staticmethod def get_provider_anthropic_messages_config( model: str, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 94ebc0bd735..d42e66605d7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -79765,5 +79765,12 @@ "metadata": { "notes": "Self-hosted decision model; infrastructure costs are paid separately" } + }, + "azure_ai/Microsoft-Decision-1": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "azure_ai", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 7eefae0f748..4ce6924edb0 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1369,7 +1369,8 @@ "rerank": true, "a2a": true, "interactions": true, - "video_generations": true + "video_generations": true, + "systemone": true } }, "huggingface": { diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 863934d76f7..d664c7771cc 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -53,6 +53,7 @@ IGNORE_FUNCTIONS = [ "sanitize_oci_schema", # OCI: bounded by JSON-schema tree depth. "_freeze_for_dedupe", # OTEL: max depth set (default 16, _FREEZE_MAX_DEPTH); fails closed by returning repr(value) at the cap. "apply_json_merge_patch", # max depth set (_MAX_MERGE_DEPTH=64); fails closed by raising ValueError at the cap. + "_inlined", # max depth set (_MAX_INLINE_DEPTH=32); passes the schema node through untouched at the cap. Walks a Pydantic JSON schema once at import time. "_filter_argument_value", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by blocking the tool call at the cap. "_redact_scanned_content", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by returning "[REDACTED]" at the cap. "replace_ciphertexts", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); walks stored JSON, which has no cycles, and leaves values below the cap untouched. diff --git a/tests/integration/_support/forward_proxy.py b/tests/integration/_support/forward_proxy.py new file mode 100644 index 00000000000..0d92c3d0719 --- /dev/null +++ b/tests/integration/_support/forward_proxy.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import contextlib +import socket +import threading +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from queue import SimpleQueue +from typing import Final + + +@dataclass(frozen=True, slots=True) +class Tunnel: + method: str + target: str + headers: Mapping[str, str] + + +@dataclass(frozen=True, slots=True) +class ForwardProxy: + url: str + port: int + received: SimpleQueue[Tunnel] + + def drain(self) -> tuple[Tunnel, ...]: + return tuple(self.received.get_nowait() for _ in range(self.received.qsize())) + + def targets(self) -> tuple[str, ...]: + return tuple(tunnel.target for tunnel in self.drain()) + + +@contextmanager +def refusing_forward_proxy(port: int = 0) -> Generator[ForwardProxy, None, None]: + """Owned HTTP forward proxy: records the host each CONNECT asks for and refuses the tunnel with 403. + + A litellm proxy booted with ``HTTPS_PROXY`` pointed here makes the host it dials for a provider + observable without a byte leaving the box; the caller sees litellm's own connection error. + """ + received: Final[SimpleQueue[Tunnel]] = SimpleQueue() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + timeout = 5 + + def refuse(self) -> None: + received.put(Tunnel(self.command, self.path, {name.lower(): value for name, value in self.headers.items()})) + self.send_response(403) + self.send_header("content-length", "0") + self.send_header("connection", "close") + self.end_headers() + self.close_connection = True + + do_CONNECT = refuse + do_GET = refuse + do_POST = refuse + do_PUT = refuse + do_DELETE = refuse + + def log_message(self, format: str, *args: object) -> None: + pass + + with ThreadingHTTPServer(("127.0.0.1", port), Handler) as server: + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) + thread.start() + try: + yield ForwardProxy(f"http://127.0.0.1:{server.server_port}", server.server_port, received) + finally: + server.shutdown() + thread.join(timeout=6) + assert not thread.is_alive(), "Owned forward proxy survived cleanup" + server.server_close() + + +def _pipe(source: socket.socket, sink: socket.socket) -> None: + with contextlib.suppress(OSError): + for chunk in iter(lambda: source.recv(65536), b""): + sink.sendall(chunk) + with contextlib.suppress(OSError): + sink.shutdown(socket.SHUT_WR) + + +@contextmanager +def tunnelling_forward_proxy(destination_port: int, port: int = 0) -> Generator[ForwardProxy, None, None]: + received: Final[SimpleQueue[Tunnel]] = SimpleQueue() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + rbufsize = 0 + timeout = 30 + request: socket.socket + + def do_CONNECT(self) -> None: + received.put(Tunnel(self.command, self.path, {name.lower(): value for name, value in self.headers.items()})) + self.close_connection = True + with socket.create_connection(("127.0.0.1", destination_port), timeout=30) as destination: + self.send_response(200, "Connection established") + self.end_headers() + outbound: Final = threading.Thread(target=_pipe, args=(self.request, destination)) + outbound.start() + _pipe(destination, self.request) + outbound.join(timeout=35) + + def log_message(self, format: str, *args: object) -> None: + pass + + with ThreadingHTTPServer(("127.0.0.1", port), Handler) as server: + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) + thread.start() + try: + yield ForwardProxy(f"http://127.0.0.1:{server.server_port}", server.server_port, received) + finally: + server.shutdown() + thread.join(timeout=6) + assert not thread.is_alive(), "Owned forward proxy survived cleanup" + server.server_close() diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 17bd1c2ebfe..11b902b540a 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -24,6 +24,16 @@ _CONFIGURED_PROBE_REPLY: Final = JsonResponse( "usage": {"input_tokens": 10, "output_tokens": 1}, }, ) +_VLLM_PROBE_REPLY: Final = JsonResponse( + content_type="application/json", + body={ + "model": "Qwen/Qwen3-0.6B", + "answers": { + "reachable": {"type": "choice", "choice": "yes", "confidence": 1.0, "probabilities": {"yes": 1.0, "no": 0.0}} + }, + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, +) _STRANDS_PROBE_REPLY: Final = JsonResponse( content_type="application/json", body={ @@ -147,3 +157,32 @@ def test_evaluation_mode_health_check_of_the_self_hosted_strands_model_resolves_ }, ) ] + + +def test_evaluation_mode_health_check_of_hosted_vllm_sends_a_choice_probe(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _VLLM_PROBE_REPLY) + scenario.cleanups.callback(delete_scenario, handle) + model: Final = scenario.model( + model="hosted_vllm/Qwen/Qwen3-0.6B", + api_base=handle.api_base(), + model_info={"mode": "evaluation"}, + ) + report: Final = _health_report(gateway, model) + assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report + assert _probes_sent_to(gateway, handle) == [ + ( + f"/{handle.scenario_id}/v1/systemone", + { + "model": "Qwen/Qwen3-0.6B", + "state": os.environ.get("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"), + "questions": { + "reachable": { + "type": "choice", + "instructions": "Is the service reachable?", + "criteria": {"yes": None, "no": None}, + } + }, + }, + ) + ] diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py new file mode 100644 index 00000000000..1104a6151db --- /dev/null +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -0,0 +1,593 @@ +import math +import uuid +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario +from integration.cost_calculation.cost_tracking_case import JsonResponse +from pydantic import JsonValue, TypeAdapter + +import litellm + +_API_KEY: Final = "synthetic-decisions-key" +_SAFETY_IDENTIFIER: Final = "end-user-7" +_DROPPING_DEPLOYMENT: Final = "decisions-under-litellm-settings-drop-params" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_INPUT: Final = "Ticket (billing): The export job hangs at 99%" +_FOLLOW_UP: Final = "Customer: still stuck after retrying" +_QUESTIONS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [{"value": "low", "description": "cosmetic"}, {"value": "high", "description": "blocks users"}], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, +] +_SYSTEM_ONE_QUESTIONS: Final[dict[str, JsonValue]] = { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, +} +_SYSTEM_ONE_ANSWERS: Final[dict[str, JsonValue]] = { + "defect": {"type": "noul", "noul": 0.93}, + "severity": {"type": "choice", "choice": "high", "confidence": 0.8, "probabilities": {"low": 0.2, "high": 0.8}}, + "confidence": { + "type": "score", + "score": 1.0, + "confidence": 0.7, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.3, "1": 0.7}, + }, +} +_ANSWERS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "defect", "probability": 0.93}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.2}, {"value": "high", "probability": 0.8}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "confidence", + "score": 1.0, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.3}, + {"value": 1, "label": "sure", "probability": 0.7}, + ], + "confidence": 0.7, + }, +] +_INPUT_TOKENS: Final = 367 +_OUTPUT_TOKENS: Final = 3 +_SYSTEM_ONE_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS} +_USAGE: Final[dict[str, JsonValue]] = { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, +} +_SDK_QUESTIONS: Final = TypeAdapter(list[dict[str, object]]).validate_python(_QUESTIONS) +_SPEND_QUERY: Final = ( + "SELECT spend, status, call_type, model_group, custom_llm_provider, api_base, prompt_tokens, completion_tokens, " + 'request_tags FROM "LiteLLM_SpendLogs" WHERE request_id = %s' +) + + +@dataclass(frozen=True, slots=True) +class _Provider: + name: str + model: str + path: str + body_model: str + api_key: str | None + wraps_result: bool + cost_map_key: str | None + speaks_openai: bool = False + provider_reported_cost: float | None = None + + def upstream_body(self) -> dict[str, JsonValue]: + if self.speaks_openai: + return {"model": self.body_model, "input": _INPUT, "questions": _QUESTIONS} + return {"model": self.body_model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS} + + def _with_reported_cost(self, usage: dict[str, JsonValue]) -> dict[str, JsonValue]: + if self.provider_reported_cost is None: + return usage + return {**usage, "cost": self.provider_reported_cost} + + def upstream_reply(self) -> dict[str, JsonValue]: + if self.speaks_openai: + return {"model": self.body_model, "answers": _ANSWERS, "usage": self._with_reported_cost(_USAGE)} + answer: Final[dict[str, JsonValue]] = { + "model": self.body_model, + "answers": _SYSTEM_ONE_ANSWERS, + "usage": self._with_reported_cost(_SYSTEM_ONE_USAGE), + } + return {"result": answer, "success": True} if self.wraps_result else answer + + def litellm_response(self) -> dict[str, JsonValue]: + return {"model": self.body_model, "answers": _ANSWERS, "usage": _USAGE} + + +_PROVIDERS: Final = ( + _Provider( + "perplexity", + "perplexity/pplx-decider-v1-27b", + "/v1/decisions", + "pplx-decider-v1-27b", + _API_KEY, + False, + "perplexity/pplx-decider-v1-27b", + ), + _Provider("typesafe", "typesafe/jev-1.13.0", "/v1/systemone", "jev-1.13.0", _API_KEY, False, "typesafe/jev-1.13.0"), + _Provider( + "openrouter", + "openrouter/typesafe/jev-1.13", + "/alpha/decisions", + "typesafe/jev-1.13", + _API_KEY, + False, + None, + provider_reported_cost=1.5834e-5, + ), + _Provider( + "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None + ), + _Provider( + "cloudflare", + "cloudflare/clef", + "/ai/run/@cf/cloudflare/clef", + "clef", + _API_KEY, + True, + "cloudflare/@cf/cloudflare/clef", + ), + _Provider("hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None), + _Provider( + "databricks", + "databricks/databricks-openjev-qwen35-4b", + "/databricks-openjev-qwen35-4b/invocations", + "databricks-openjev-qwen35-4b", + _API_KEY, + False, + None, + ), + _Provider( + "azure_ai", "azure_ai/decision-1", "/providers/microsoft/v1/systemone", "decision-1", _API_KEY, False, None + ), + _Provider("openai", "openai/gpt-6-luna", "/v1/decisions", "gpt-6-luna", _API_KEY, False, "gpt-6-luna", True), +) +_SYSTEM_ONE_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if not provider.speaks_openai) +_PERPLEXITY: Final = _PROVIDERS[0] +_TYPESAFE: Final = _PROVIDERS[1] +_OPENAI: Final = next(provider for provider in _PROVIDERS if provider.speaks_openai) +_PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"} +_INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( + ("missing questions", {"input": _INPUT}), + ("missing input", {"questions": _QUESTIONS}), + ("numeric input", {"input": 5, "questions": _QUESTIONS}), + ("assistant message input", {"input": [{"role": "assistant", "content": "hi"}], "questions": _QUESTIONS}), + ("empty questions", {"input": _INPUT, "questions": []}), + ("questions as a map", {"input": _INPUT, "questions": {"q": _PREDICATE}}), + ("predicate without instructions", {"input": _INPUT, "questions": [{"type": "predicate", "name": "q"}]}), + ( + "choice with one choice", + {"input": _INPUT, "questions": [{**_PREDICATE, "type": "choice", "choices": [{"value": "only"}]}]}, + ), + ( + "score with one level", + {"input": _INPUT, "questions": [{**_PREDICATE, "type": "score", "levels": [{"label": "only"}]}]}, + ), + ("unknown question type", {"input": _INPUT, "questions": [{**_PREDICATE, "type": "ranking"}]}), + ("numeric safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": 7}), + ("list safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": [_SAFETY_IDENTIFIER]}), +) +_IMAGE_INPUT: Final[list[JsonValue]] = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": _INPUT}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, + ], + } +] +_OPENAI_IMAGE_MESSAGES: Final[list[JsonValue]] = [ + { + "role": "user", + "type": "message", + "content": [ + {"type": "input_text", "text": _INPUT}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, + ], + } +] +_MESSAGE_LIST_INPUT: Final[list[JsonValue]] = [ + {"role": "user", "content": _INPUT}, + {"role": "user", "content": [{"type": "input_text", "text": _FOLLOW_UP}]}, +] + + +def _response_cost(response: httpx.Response) -> float: + return float(response.headers["x-litellm-response-cost"]) if "x-litellm-response-cost" in response.headers else 0.0 + + +def _provider_id(provider: _Provider) -> str: + return provider.name + + +def _number(value: JsonValue) -> float: + assert isinstance(value, (int, float)) and not isinstance(value, bool), value + return float(value) + + +def _expected_spend(provider: _Provider) -> float: + if provider.provider_reported_cost is not None: + return provider.provider_reported_cost + if provider.cost_map_key is None: + return 0.0 + cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes()) + prices: Final = object_value(cost_map[provider.cost_map_key]) + return _INPUT_TOKENS * _number(prices["input_cost_per_token"]) + _OUTPUT_TOKENS * _number( + prices["output_cost_per_token"] + ) + + +def _register(scenario: Scenario, body: dict[str, JsonValue], *, status: int = 200) -> ScenarioHandle: + handle: Final = register_scenario( + f"decisions-{uuid.uuid4().hex[:12]}", JsonResponse(content_type="application/json", body=body, status=status) + ) + scenario.cleanups.callback(delete_scenario, handle) + return handle + + +def _deployment(scenario: Scenario, handle: ScenarioHandle, provider: _Provider, *, drop_params: bool = False) -> str: + if drop_params: + return scenario.model( + model=provider.model, api_base=handle.api_base(), api_key=provider.api_key, drop_params=True + ) + return scenario.model(model=provider.model, api_base=handle.api_base(), api_key=provider.api_key) + + +def _decide(gateway: Gateway, model: str, *, route: str = "/v1/decisions", **extra: JsonValue) -> httpx.Response: + return gateway.request("POST", route, {"model": model, "input": _INPUT, "questions": _QUESTIONS, **extra}) + + +def _decide_in_system_one_format(gateway: Gateway, model: str, **extra: JsonValue) -> httpx.Response: + return gateway.request( + "POST", "/v1/systemone", {"model": model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS, **extra} + ) + + +def _observed_requests(gateway: Gateway) -> tuple[dict[str, JsonValue], ...]: + with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream: + observations: Final = _JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"] + assert isinstance(observations, list), observations + return tuple(map(object_value, observations)) + + +def _calls_to(requests: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> list[dict[str, JsonValue]]: + return [request for request in requests if string_value(request["path"]).startswith(f"/{handle.scenario_id}/")] + + +def _upstream_calls(gateway: Gateway, handle: ScenarioHandle) -> list[dict[str, JsonValue]]: + return _calls_to(_observed_requests(gateway), handle) + + +def _spend_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70) + return rows[0] + + +def _assert_refused_as_invalid(gateway: Gateway, model: str, label: str, body: dict[str, JsonValue]) -> None: + response: Final = gateway.request("POST", "/v1/decisions", {"model": model, **body}) + assert response.status_code == 400, (label, response.text) + assert "Invalid Decisions request" in response.text, (label, response.text) + + +def _assert_refused_for_safety_identifier(response: httpx.Response) -> None: + assert response.status_code == 400, response.text + assert "safety_identifier" in response.text and "drop_params" in response.text, response.text + + +def _drop_params_config(directory: Path, api_base: str) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = { + **base, + "litellm_settings": {**object_value(base["litellm_settings"]), "drop_params": True}, + "model_list": [ + { + "model_name": _DROPPING_DEPLOYMENT, + "litellm_params": {"model": _PERPLEXITY.model, "api_base": api_base, "api_key": _API_KEY}, + } + ], + } + path: Final = directory / "decisions-drop-params.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.mark.parametrize("provider", _PROVIDERS, ids=_provider_id) +def test_each_provider_gets_its_own_path_key_and_body_and_is_billed(gateway: Gateway, provider: _Provider) -> None: + expected_spend: Final = _expected_spend(provider) + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + model: Final = _deployment(scenario, handle, provider) + response: Final = _decide(gateway, model) + assert response.status_code == 200, response.text + assert response.json() == provider.litellm_response() + assert response.headers["x-litellm-model-group"] == model + assert math.isclose(_response_cost(response), expected_spend, rel_tol=1e-9) + (call,) = _upstream_calls(gateway, handle) + assert call["path"] == f"/{handle.scenario_id}{provider.path}" + assert call["authorization"] == (f"Bearer {provider.api_key}" if provider.api_key else "") + assert call["body"] == provider.upstream_body() + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + assert ( + row["status"], + row["call_type"], + row["custom_llm_provider"], + row["model_group"], + row["api_base"], + row["prompt_tokens"], + row["completion_tokens"], + ) == ( + "success", + "adecisions", + provider.name, + model, + f"{handle.api_base()}{provider.path}", + _INPUT_TOKENS, + _OUTPUT_TOKENS, + ) + assert math.isclose(_number(row["spend"]), expected_spend, rel_tol=1e-9), row + + +def test_the_unversioned_alias_serves_the_same_request(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide(gateway, model, route="/decisions") + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_repeated_identical_requests_each_reach_the_upstream_and_are_each_billed(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + responses: Final = tuple(_decide(gateway, model) for _ in range(2)) + assert [response.status_code for response in responses] == [200, 200], [r.text for r in responses] + call_ids: Final = tuple(response.headers["x-litellm-call-id"] for response in responses) + assert len(set(call_ids)) == 2, call_ids + assert len(_upstream_calls(gateway, handle)) == 2 + for call_id in call_ids: + assert _spend_row(call_id)["status"] == "success" + + +async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _TYPESAFE.upstream_reply()) + synchronous: Final = litellm.decisions( + model=_TYPESAFE.model, input=_INPUT, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + ) + asynchronous: Final = await litellm.adecisions( + model=_TYPESAFE.model, input=_INPUT, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + ) + for response in (synchronous, asynchronous): + assert response.model_dump(mode="json") == _TYPESAFE.litellm_response() + calls: Final = _upstream_calls(gateway, handle) + assert len(calls) == 2, calls + for call in calls: + assert call["path"] == f"/{handle.scenario_id}{_TYPESAFE.path}" + assert call["authorization"] == f"Bearer {_API_KEY}" + assert call["body"] == _TYPESAFE.upstream_body() + + +def test_gateway_only_fields_stay_at_the_gateway_and_tags_reach_the_spend_log(gateway: Gateway) -> None: + tag: Final = f"decisions-openai-format-{uuid.uuid4().hex[:8]}" + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide( + gateway, model, user="auditor", num_retries=0, temperature=0.2, metadata={"tags": [tag]} + ) + assert response.status_code == 200, response.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + tags: Final = row["request_tags"] + assert isinstance(tags, list) and tag in tags, row + + +def test_invalid_bodies_are_refused_at_the_gateway_without_an_upstream_call(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + for label, body in _INVALID_BODIES: + _assert_refused_as_invalid(gateway, model, label, body) + assert _upstream_calls(gateway, handle) == [] + + +@pytest.mark.parametrize("provider", _SYSTEM_ONE_PROVIDERS, ids=_provider_id) +def test_system_one_providers_refuse_images_at_the_gateway_without_an_upstream_call( + gateway: Gateway, provider: _Provider +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + model: Final = _deployment(scenario, handle, provider) + response: Final = _decide(gateway, model, input=_IMAGE_INPUT) + assert response.status_code == 400, response.text + assert "input_image" in response.text + assert _upstream_calls(gateway, handle) == [] + + +def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + model: Final = _deployment(scenario, handle, _OPENAI) + response: Final = _decide(gateway, model, input=_IMAGE_INPUT) + assert response.status_code == 200, response.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == {"model": _OPENAI.body_model, "input": _OPENAI_IMAGE_MESSAGES, "questions": _QUESTIONS} + + +def test_a_message_list_input_reaches_a_system_one_provider_as_its_flattened_text(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide(gateway, model, input=_MESSAGE_LIST_INPUT) + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == { + "model": _PERPLEXITY.body_model, + "state": f"{_INPUT}\n\n{_FOLLOW_UP}", + "questions": _SYSTEM_ONE_QUESTIONS, + } + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + assert (row["status"], row["call_type"], row["prompt_tokens"], row["completion_tokens"]) == ( + "success", + "adecisions", + _INPUT_TOKENS, + _OUTPUT_TOKENS, + ) + + +@pytest.mark.parametrize("provider", _SYSTEM_ONE_PROVIDERS, ids=_provider_id) +def test_safety_identifier_is_refused_by_system_one_providers_unless_the_deployment_drops_params( + gateway: Gateway, provider: _Provider +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + strict: Final = _deployment(scenario, handle, provider) + refused: Final = _decide(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER) + _assert_refused_for_safety_identifier(refused) + assert _upstream_calls(gateway, handle) == [] + + dropping: Final = _deployment(scenario, handle, provider, drop_params=True) + accepted: Final = _decide(gateway, dropping, safety_identifier=_SAFETY_IDENTIFIER) + assert accepted.status_code == 200, accepted.text + assert accepted.json() == provider.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == provider.upstream_body() + row: Final = _spend_row(accepted.headers["x-litellm-call-id"]) + assert (row["status"], row["call_type"], row["model_group"]) == ("success", "adecisions", dropping) + + +@pytest.mark.parametrize("value", ("", "x" * 5000), ids=("empty", "5kb")) +def test_every_string_safety_identifier_is_refused_or_dropped_like_the_usual_one(gateway: Gateway, value: str) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + strict: Final = _deployment(scenario, handle, _PERPLEXITY) + _assert_refused_for_safety_identifier(_decide(gateway, strict, safety_identifier=value)) + dropping: Final = _deployment(scenario, handle, _PERPLEXITY, drop_params=True) + accepted: Final = _decide(gateway, dropping, safety_identifier=value) + assert accepted.status_code == 200, accepted.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_a_request_body_drop_params_drops_the_safety_identifier_like_chat(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + strict: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER, drop_params=True) + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_openai_keeps_the_safety_identifier_on_the_wire_without_drop_params(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + model: Final = _deployment(scenario, handle, _OPENAI) + response: Final = _decide(gateway, model, safety_identifier=_SAFETY_IDENTIFIER) + assert response.status_code == 200, response.text + assert response.json() == _OPENAI.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == {**_OPENAI.upstream_body(), "safety_identifier": _SAFETY_IDENTIFIER} + + +def test_a_system_one_format_safety_identifier_is_refused_unless_the_deployment_drops_params(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + strict: Final = _deployment(scenario, handle, _PERPLEXITY) + refused: Final = _decide_in_system_one_format(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER) + _assert_refused_for_safety_identifier(refused) + assert _upstream_calls(gateway, handle) == [] + + dropping: Final = _deployment(scenario, handle, _PERPLEXITY, drop_params=True) + accepted: Final = _decide_in_system_one_format(gateway, dropping, safety_identifier=_SAFETY_IDENTIFIER) + assert accepted.status_code == 200, accepted.text + assert accepted.json()["answers"] == _SYSTEM_ONE_ANSWERS + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_openai_keeps_a_system_one_format_safety_identifier_on_the_wire(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + model: Final = _deployment(scenario, handle, _OPENAI) + response: Final = _decide_in_system_one_format(gateway, model, safety_identifier=_SAFETY_IDENTIFIER) + assert response.status_code == 200, response.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == {**_OPENAI.upstream_body(), "safety_identifier": _SAFETY_IDENTIFIER} + + +@pytest.mark.parametrize("decide", (_decide, _decide_in_system_one_format), ids=("openai_format", "system_one_format")) +@pytest.mark.parametrize("value", (7, [_SAFETY_IDENTIFIER]), ids=("numeric", "list")) +def test_a_non_string_safety_identifier_is_refused_as_invalid_unless_the_deployment_drops_params( + gateway: Gateway, value: JsonValue, decide: Callable[..., httpx.Response] +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + strict: Final = _deployment(scenario, handle, _OPENAI) + refused: Final = decide(gateway, strict, safety_identifier=value) + assert refused.status_code == 400, refused.text + assert "Invalid Decisions request" in refused.text, refused.text + assert _upstream_calls(gateway, handle) == [] + + dropping: Final = _deployment(scenario, handle, _OPENAI, drop_params=True) + accepted: Final = decide(gateway, dropping, safety_identifier=value) + assert accepted.status_code == 200, accepted.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _OPENAI.upstream_body() + + +def test_litellm_settings_drop_params_drops_the_safety_identifier_for_a_strict_deployment( + gateway: Gateway, tmp_path: Path +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + config: Final = _drop_params_config(tmp_path, handle.api_base()) + with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: + response: Final = _decide(owned.gateway, _DROPPING_DEPLOYMENT, safety_identifier=_SAFETY_IDENTIFIER) + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 709ea42b76e..55fc142163c 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -25,6 +25,7 @@ _PASS_THROUGH_MODEL: Final = "gpt-6-luna" _PASS_THROUGH_AUTHORIZATION: Final = "Bearer customer-held-upstream-key" _PASS_THROUGH_NEIGHBOUR: Final = "decisions-beside-a-pass-through" _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_QUESTION_MAPPINGS: Final = TypeAdapter(dict[str, dict[str, object]]) _USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 367, "output_tokens": 3} _STATE: Final[dict[str, JsonValue]] = {"ticket": "The export job hangs at 99%", "component": "billing"} _QUESTIONS: Final[dict[str, JsonValue]] = { @@ -32,6 +33,7 @@ _QUESTIONS: Final[dict[str, JsonValue]] = { "severity": {"type": "choice", "criteria": {"low": "cosmetic", "high": "blocks users"}, "weight": 2}, "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, } +_SDK_QUESTIONS: Final = _QUESTION_MAPPINGS.validate_python(_QUESTIONS) _ANSWERS: Final[dict[str, JsonValue]] = { "defect": {"type": "noul", "noul": 0.93}, "severity": {"type": "choice", "choice": "high", "confidence": 0.8, "probabilities": {"low": 0.2, "high": 0.8}}, @@ -67,6 +69,7 @@ class _Provider: api_key: str | None wraps_result: bool cost_map_key: str | None + provider_reported_cost: float | None = None _PROVIDERS: Final = ( @@ -87,11 +90,15 @@ _PROVIDERS: Final = ( "typesafe/jev-1.13", _API_KEY, False, - "openrouter/typesafe/jev-1.13", + None, + 1.5834e-5, ), _Provider( "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None ), + _Provider( + "hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None + ), _Provider( "cloudflare", "cloudflare/clef", @@ -101,8 +108,16 @@ _PROVIDERS: Final = ( True, "cloudflare/@cf/cloudflare/clef", ), + _Provider( + "azure_ai", "azure_ai/decision-1", "/providers/microsoft/v1/systemone", "decision-1", _API_KEY, False, None + ), ) _PERPLEXITY: Final = _PROVIDERS[0] +_OPENROUTER: Final = _PROVIDERS[2] +_OPENROUTER_CHAT_MODEL: Final = "openrouter/openai/gpt-5-mini" +_UNSUPPORTED_PROVIDER_MODEL: Final = "anthropic/claude-opus-5-5" +_CONNECTION_ERROR: Final = "litellm.APIConnectionError" +_GENERIC_API_ERROR: Final = "litellm.APIError" _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( ("missing questions", {"state": _STATE}), ("missing state", {"questions": _QUESTIONS}), @@ -123,17 +138,32 @@ def _number(value: JsonValue) -> float: return float(value) -def _expected_spend(cost_map_key: str | None) -> float: - if cost_map_key is None: +def _expected_spend(provider: _Provider) -> float: + if provider.provider_reported_cost is not None: + return provider.provider_reported_cost + if provider.cost_map_key is None: return 0.0 - prices: Final = object_value(json.loads(Path("model_prices_and_context_window.json").read_text())[cost_map_key]) + prices: Final = object_value( + json.loads(Path("model_prices_and_context_window.json").read_text())[provider.cost_map_key] + ) return _number(_USAGE["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number( _USAGE["output_tokens"] ) * _number(prices["output_cost_per_token"]) +def _usage(provider: _Provider) -> dict[str, JsonValue]: + return { + **_USAGE, + **({"cost": provider.provider_reported_cost} if provider.provider_reported_cost is not None else {}), + } + + def _answer_body(provider: _Provider) -> dict[str, JsonValue]: - answer: Final[dict[str, JsonValue]] = {"model": provider.body_model, "answers": _ANSWERS, "usage": _USAGE} + answer: Final[dict[str, JsonValue]] = { + "model": provider.body_model, + "answers": _ANSWERS, + "usage": _usage(provider), + } return {"result": answer, "success": True} if provider.wraps_result else answer @@ -177,6 +207,12 @@ def _spend_row(call_id: str) -> dict[str, JsonValue]: return rows[0] +def _assert_connection_error(response: httpx.Response) -> None: + assert 500 <= response.status_code < 600, response.text + assert _CONNECTION_ERROR in response.text, response.text + assert _GENERIC_API_ERROR not in response.text, response.text + + def _free_closed_port() -> int: with socket.socket() as probe: probe.bind(("127.0.0.1", 0)) @@ -210,16 +246,14 @@ def _pass_through_config(directory: Path, pass_through_target: str, native_api_b @pytest.mark.parametrize("provider", _PROVIDERS, ids=lambda provider: provider.name) -def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cost_map( - gateway: Gateway, provider: _Provider -) -> None: - expected_spend: Final = _expected_spend(provider.cost_map_key) +def test_each_provider_gets_its_own_path_key_and_body_and_is_billed(gateway: Gateway, provider: _Provider) -> None: + expected_spend: Final = _expected_spend(provider) with gateway.scenario() as scenario: handle: Final = _register(scenario, _answer_body(provider)) model: Final = _deployment(scenario, handle, provider) response: Final = _decide(gateway, model) assert response.status_code == 200, response.text - assert response.json() == {"model": provider.body_model, "answers": _ANSWERS, "usage": _USAGE} + assert response.json() == {"model": provider.body_model, "answers": _ANSWERS, "usage": _usage(provider)} assert response.headers["x-litellm-model-group"] == model assert math.isclose(float(response.headers.get("x-litellm-response-cost", "0")), expected_spend, rel_tol=1e-9) (call,) = _upstream_calls(gateway, handle) @@ -235,7 +269,7 @@ def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cos row["api_base"], row["prompt_tokens"], row["completion_tokens"], - ) == ("success", "adecisions", provider.name, model, f"{handle.api_base()}{provider.path}", 367, 3) + ) == ("success", "asystemone", provider.name, model, f"{handle.api_base()}{provider.path}", 367, 3) assert math.isclose(_number(row["spend"]), expected_spend, rel_tol=1e-9), row @@ -257,10 +291,10 @@ async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway with gateway.scenario() as scenario: handle: Final = _register(scenario, _answer_body(provider)) synchronous: Final = litellm.decisions( - model=provider.model, state=_STATE, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY ) - asynchronous: Final = await litellm.adecisions( - model=provider.model, state=_STATE, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + asynchronous: Final = await litellm.asystemone( + model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY ) for response in (synchronous, asynchronous): assert response.model_dump(mode="json") == { @@ -427,7 +461,7 @@ def test_upstream_errors_keep_their_status_and_log_an_unbilled_failure(gateway: row: Final = _spend_row(response.headers["x-litellm-call-id"]) assert (row["status"], row["call_type"], row["model_group"], _number(row["spend"])) == ( "failure", - "adecisions", + "asystemone", model, 0.0, ) @@ -443,16 +477,75 @@ def test_upstream_success_without_answers_is_a_gateway_side_server_error(gateway assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" -def test_unreachable_upstream_fails_only_its_own_deployment(gateway: Gateway) -> None: +@pytest.mark.parametrize("provider", _PROVIDERS, ids=lambda provider: provider.name) +def test_unreachable_upstream_fails_only_its_own_deployment(gateway: Gateway, provider: _Provider) -> None: with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) - healthy: Final = _deployment(scenario, handle, _PERPLEXITY) + handle: Final = _register(scenario, _answer_body(provider)) + healthy: Final = _deployment(scenario, handle, provider) dead: Final = scenario.model( - model=_PERPLEXITY.model, api_base=f"http://127.0.0.1:{_free_closed_port()}", api_key=_API_KEY + model=provider.model, api_base=f"http://127.0.0.1:{_free_closed_port()}", api_key=provider.api_key ) failed: Final = _decide(gateway, dead, num_retries=0) - assert 500 <= failed.status_code < 600, failed.text + _assert_connection_error(failed) assert _spend_row(failed.headers["x-litellm-call-id"])["status"] == "failure" served: Final = _decide(gateway, healthy) assert served.status_code == 200, served.text assert len(_upstream_calls(gateway, handle)) == 1 + + +def test_an_unreachable_openrouter_deployment_reports_a_connection_error_on_chat_embeddings_and_decisions( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + dead_api_base: Final = f"http://127.0.0.1:{_free_closed_port()}" + chat_model: Final = scenario.model(model=_OPENROUTER_CHAT_MODEL, api_base=dead_api_base, api_key=_API_KEY) + decisions_model: Final = scenario.model(model=_OPENROUTER.model, api_base=dead_api_base, api_key=_API_KEY) + chat: Final = _chat(gateway, chat_model, num_retries=0) + streamed: Final = _chat(gateway, chat_model, num_retries=0, stream=True) + embeddings: Final = gateway.request( + "POST", "/v1/embeddings", {"model": chat_model, "input": "hi", "num_retries": 0} + ) + decisions: Final = _decide(gateway, decisions_model, num_retries=0) + for response in (chat, streamed, embeddings, decisions): + _assert_connection_error(response) + for response in (chat, decisions): + assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" + + +async def test_sdk_openrouter_connection_failures_raise_a_connection_error(gateway: Gateway) -> None: + dead_api_base: Final = f"http://127.0.0.1:{_free_closed_port()}" + with pytest.raises(litellm.APIConnectionError): + litellm.completion( + model=_OPENROUTER_CHAT_MODEL, + messages=[{"role": "user", "content": "hi"}], + api_base=dead_api_base, + api_key=_API_KEY, + ) + with pytest.raises(litellm.APIConnectionError): + await litellm.acompletion( + model=_OPENROUTER_CHAT_MODEL, + messages=[{"role": "user", "content": "hi"}], + api_base=dead_api_base, + api_key=_API_KEY, + ) + with pytest.raises(litellm.APIConnectionError): + litellm.decisions( + model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY + ) + with pytest.raises(litellm.APIConnectionError): + await litellm.asystemone( + model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY + ) + + +def test_a_deployment_whose_provider_has_no_decisions_support_is_refused_naming_every_supported_provider( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) + model: Final = scenario.model(model=_UNSUPPORTED_PROVIDER_MODEL, api_base=handle.api_base(), api_key=_API_KEY) + response: Final = _decide(gateway, model) + assert response.status_code == 400, response.text + for provider in _PROVIDERS: + assert provider.name in response.text, response.text + assert _upstream_calls(gateway, handle) == [] diff --git a/tests/integration/providers/test_strands_decider_agentcore_wire.py b/tests/integration/providers/test_strands_decider_agentcore_wire.py new file mode 100644 index 00000000000..493d8c238fa --- /dev/null +++ b/tests/integration/providers/test_strands_decider_agentcore_wire.py @@ -0,0 +1,827 @@ +import hashlib +import json +import os +import re +import threading +import time +import uuid +from collections.abc import Callable, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, replace +from functools import partial +from pathlib import Path +from typing import Final +from urllib.parse import quote, unquote + +import httpx +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, gateway_from_environment +from integration._support.database import read_rows +from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy +from integration._support.process import owned_proxy +from integration._support.sigv4 import encoded_path, signature +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm.llms.strands_decider.decisions.transformation import StrandsDeciderDecisionsConfig + +_ACCOUNT: Final = "123456789012" +_ACCESS_KEY: Final = "AKIAAGENTCOREAUDIT01" +_SECRET_KEY: Final = "synthetic-agentcore-audit-secret" +_SIGNING: Final[Mapping[str, JsonValue]] = {"aws_access_key_id": _ACCESS_KEY, "aws_secret_access_key": _SECRET_KEY} +_JWT: Final = "synthetic-agentcore-jwt" +_ENV_JWT: Final = "synthetic-agentcore-env-jwt" +_US_EAST: Final = "us-east-1" +_EU_WEST: Final = "eu-west-1" +_MODEL: Final = "strands_decider/systemone-decider" +_BODY_MODEL: Final = "systemone-decider" +_SESSION_HEADER: Final = "x-amzn-bedrock-agentcore-runtime-session-id" +_CONFIGURED_SESSION: Final = "agentcore-audit-configured-session-0000000001" +_STATE: Final[dict[str, JsonValue]] = {"ticket": "Refund requested twice for one order", "queue": "payments"} +_QUESTIONS: Final[dict[str, JsonValue]] = { + "duplicate": {"type": "noul", "instructions": "Is this a duplicate refund request?"}, + "urgency": {"type": "choice", "criteria": {"low": "can wait", "high": "money at risk"}}, +} +_SDK_QUESTIONS: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python(_QUESTIONS) +_ANSWERS: Final[dict[str, JsonValue]] = { + "duplicate": {"type": "noul", "noul": 0.88}, + "urgency": {"type": "choice", "choice": "high", "confidence": 0.75, "probabilities": {"low": 0.25, "high": 0.75}}, +} +_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 211, "output_tokens": 2} +_UPSTREAM_BODY: Final[dict[str, JsonValue]] = {"model": _BODY_MODEL, "state": _STATE, "questions": _QUESTIONS} +_INPUT: Final = "Refund requested twice for one order" +_OPENAI_QUESTIONS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "duplicate", "instructions": "Is this a duplicate refund request?"}, + { + "type": "choice", + "name": "urgency", + "instructions": "How urgent is it?", + "choices": [{"value": "low", "description": "can wait"}, {"value": "high", "description": "money at risk"}], + }, +] +_OPENAI_UPSTREAM_BODY: Final[dict[str, JsonValue]] = { + "model": _BODY_MODEL, + "state": _INPUT, + "questions": { + "duplicate": {"type": "noul", "instructions": "Is this a duplicate refund request?"}, + "urgency": { + "type": "choice", + "instructions": "How urgent is it?", + "criteria": {"low": "can wait", "high": "money at risk"}, + }, + }, +} +_OPENAI_ANSWER: Final[dict[str, JsonValue]] = { + "model": _BODY_MODEL, + "answers": [ + {"type": "predicate", "name": "duplicate", "probability": 0.88}, + { + "type": "choice", + "name": "urgency", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.25}, {"value": "high", "probability": 0.75}], + "confidence": 0.75, + }, + ], + "usage": { + "input_tokens": 211, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 213, + }, +} + + +@dataclass(frozen=True, slots=True) +class _Route: + path: str + body: Mapping[str, JsonValue] + upstream: Mapping[str, JsonValue] + answer: Mapping[str, JsonValue] + + +_SYSTEMONE: Final = _Route( + "/v1/systemone", + {"state": _STATE, "questions": _QUESTIONS}, + _UPSTREAM_BODY, + {"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE}, +) +_OPENAI: Final = _Route( + "/v1/decisions", {"input": _INPUT, "questions": _OPENAI_QUESTIONS}, _OPENAI_UPSTREAM_BODY, _OPENAI_ANSWER +) +_ROUTES: Final = (_SYSTEMONE, _OPENAI, replace(_SYSTEMONE, path="/systemone"), replace(_OPENAI, path="/decisions")) +_SPEND_QUERY: Final = ( + "SELECT status, call_type, custom_llm_provider, model_group, api_base, prompt_tokens, completion_tokens " + 'FROM "LiteLLM_SpendLogs" WHERE request_id = %s' +) +_RUNTIME_TARGET: Final = re.compile( + r"\A/runtimes/arn:aws:bedrock-agentcore:[a-z0-9-]+:[0-9]{12}:runtime/(?P[A-Za-z0-9_-]+)/invocations\Z" +) +_ERROR_STATUS: Final = {"bad_request": 400, "invalid_request": 400, "loading": 503, "inference_error": 500} +_HTTP_ERRORS: Final = { + 403: "AccessDeniedException", + 404: "ResourceNotFoundException", + 429: "ThrottlingException", +} +_MALFORMED_SESSIONS: Final[Mapping[str, JsonValue]] = {"int": 1234567, "list": ["a" * 40], "empty": ""} +_LONG_SESSION: Final = "s" * 5120 +_NEAR_MISS_ARNS: Final = { + "endpoint": f"arn:aws:bedrock-agentcore:us-east-1:{_ACCOUNT}:runtime/audit_qualified/runtime-endpoint/DEFAULT", + "account": "arn:aws:bedrock-agentcore:us-east-1:12345:runtime/audit_short_account", + "service": f"arn:aws:bedrock:us-east-1:{_ACCOUNT}:runtime/audit_other_service", + "host": f"arn:aws:bedrock-agentcore:us-east-1.example.com:{_ACCOUNT}:runtime/audit_crafted_host", +} +_DEPLOYMENT_HEADERS: Final[dict[str, JsonValue]] = { + "x-amz-target": "AmazonBedrockAgentCore.Forged", + "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id": "deployment-forged-session-00000000000001", + "Host": "bedrock-agentcore.us-west-2.amazonaws.com", + "x-audit-trace": "deployment-trace", +} +_SIGNED: Final = "agentcore-audit-signed" +_REGION: Final = "agentcore-audit-region" +_SESSION: Final = "agentcore-audit-session" +_LONG: Final = "agentcore-audit-session-long" +_BEARER: Final = "agentcore-audit-bearer" +_HEADERS: Final = "agentcore-audit-headers" +_OPT_IN: Final = "agentcore-audit-opt-in" +_HEALTH: Final = "agentcore-audit-health" +_DROPPED: Final = "agentcore-audit-dropped" +_OUTAGE: Final = "agentcore-audit-outage" +_ENV: Final = "agentcore-audit-env" +_UNSIGNED: Final = "agentcore-audit-unsigned" +_SCRUBBED_PREFIXES: Final = ("AWS_", "STRANDS_DECIDER_") +_SCRUBBED_NAMES: Final = frozenset( + {"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY", "SSL_VERIFY", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE"} +) +_load_yaml: Final[Callable[[str], object]] = yaml.safe_load +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_OWNED_PROXY_CELL_SECONDS: Final = 2 * max(30.0, float(os.environ.get("INTEGRATION_PROXY_READY_SECONDS") or 70)) + 120 +pytestmark: Final = pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) + + +def _host(region: str) -> str: + return f"bedrock-agentcore.{region}.amazonaws.com" + + +def _arn(runtime: str, region: str = _US_EAST) -> str: + return f"arn:aws:bedrock-agentcore:{region}:{_ACCOUNT}:runtime/{runtime}" + + +def _invocations_url(runtime: str, region: str = _US_EAST) -> str: + return f"https://{_host(region)}/runtimes/{quote(_arn(runtime, region), safe='')}/invocations" + + +def _default_session(runtime: str, region: str = _US_EAST) -> str: + return f"litellm-decider-{hashlib.sha256(_arn(runtime, region).encode()).hexdigest()}" + + +def _runtime_of(request: Request) -> str | None: + match: Final = _RUNTIME_TARGET.match(unquote(request.target)) + return None if match is None else match["runtime"] + + +def _json_reply(body: Mapping[str, JsonValue], status: int = 200, **headers: str) -> Reply: + return Reply(status=status, body=json.dumps(body).encode(), headers=headers) + + +def _respond(outage: threading.Event, request: Request) -> Reply: + runtime: Final = _runtime_of(request) or "" + if runtime == "audit_dropped" or (runtime == "audit_outage" and outage.is_set()): + return Reply(drop_connection=True) + if runtime.startswith("audit_error_"): + code: Final = runtime.removeprefix("audit_error_") + return _json_reply({"error": {"code": code, "message": f"runtime refused with {code}"}}) + if runtime.startswith("audit_status_"): + status: Final = int(runtime.removeprefix("audit_status_")) + return _json_reply( + {"message": f"scripted {_HTTP_ERRORS[status]}"}, status, **{"x-amzn-errortype": _HTTP_ERRORS[status]} + ) + if not runtime: + return _json_reply({"message": f"unknown target {request.target}"}, 404) + return _json_reply({"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE}) + + +@dataclass(frozen=True, slots=True) +class _Runtime: + wire: Wire + tunnel: ForwardProxy + cert: Path + outage: threading.Event + + def reset(self) -> None: + self.outage.clear() + self.wire.drain() + self.tunnel.drain() + + def calls(self) -> tuple[Request, ...]: + return self.wire.drain() + + +def _calls_to(calls: Sequence[Request], runtime: str) -> tuple[Request, ...]: + return tuple(call for call in calls if _runtime_of(call) == runtime) + + +def _deployment(name: str, **litellm_params: JsonValue) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": {"model": _MODEL, **litellm_params}, + "model_info": {"id": name, "mode": "evaluation"}, + } + + +def _signed_deployment(name: str, runtime: str, region: str = _US_EAST, **extra: JsonValue) -> dict[str, JsonValue]: + return _deployment(name, api_base=_arn(runtime, region), **_SIGNING, **extra) + + +def _deployments() -> list[dict[str, JsonValue]]: + return [ + _signed_deployment(_SIGNED, "audit_signed", aws_region_name=_US_EAST), + _signed_deployment(_REGION, "audit_region", _EU_WEST, aws_region_name=_US_EAST), + _signed_deployment(_SESSION, "audit_session", agentcore_runtime_session_id=_CONFIGURED_SESSION), + _signed_deployment(_LONG, "audit_session_long", agentcore_runtime_session_id=_LONG_SESSION), + _deployment(_BEARER, api_base=_arn("audit_bearer"), api_key=_JWT), + _signed_deployment(_HEADERS, "audit_headers", extra_headers=_DEPLOYMENT_HEADERS), + _signed_deployment( + _OPT_IN, "audit_opt_in", configurable_clientside_auth_params=["agentcore_runtime_session_id"] + ), + _signed_deployment(_HEALTH, "audit_health"), + _signed_deployment(_DROPPED, "audit_dropped"), + _signed_deployment(_OUTAGE, "audit_outage"), + _deployment(_ENV, **_SIGNING), + _deployment(_UNSIGNED, api_base=_arn("audit_unsigned")), + *(_signed_deployment(f"agentcore-audit-error-{code}", f"audit_error_{code}") for code in _ERROR_STATUS), + *(_signed_deployment(f"agentcore-audit-status-{status}", f"audit_status_{status}") for status in _HTTP_ERRORS), + *( + _signed_deployment( + f"agentcore-audit-session-{label}", f"audit_session_{label}", agentcore_runtime_session_id=value + ) + for label, value in _MALFORMED_SESSIONS.items() + ), + *( + _deployment(f"agentcore-audit-near-miss-{label}", api_base=value, **_SIGNING) + for label, value in _NEAR_MISS_ARNS.items() + ), + ] + + +def _write_config(directory: Path) -> Path: + base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text())) + configuration: Final = { + **base, + "model_list": _deployments(), + "router_settings": {"disable_cooldowns": True, "num_retries": 0}, + } + path: Final = directory / "agentcore.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +def _scrubbed() -> tuple[str, ...]: + return tuple( + name for name in os.environ if name.upper().startswith(_SCRUBBED_PREFIXES) or name.upper() in _SCRUBBED_NAMES + ) + + +def _isolated_aws(directory: Path) -> dict[str, str]: + return { + "AWS_EC2_METADATA_DISABLED": "true", + "AWS_SHARED_CREDENTIALS_FILE": str(directory / "absent-credentials"), + "AWS_CONFIG_FILE": str(directory / "absent-config"), + } + + +def _proxy_environment(runtime: _Runtime, directory: Path) -> dict[str, str]: + return { + "HTTPS_PROXY": runtime.tunnel.url, + "NO_PROXY": "127.0.0.1,localhost", + "SSL_CERT_FILE": str(runtime.cert), + "STRANDS_DECIDER_API_BASE": _arn("audit_env"), + **_isolated_aws(directory), + } + + +@pytest.fixture(scope="module") +def module_gateway() -> Iterator[Gateway]: + with gateway_from_environment() as value: + yield value + + +@pytest.fixture(scope="module") +def runtime(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Runtime]: + cert, key = write_self_signed_cert(tmp_path_factory.mktemp("agentcore-cert"), (_host(_US_EAST), _host(_EU_WEST))) + outage: Final = threading.Event() + with wire_server(partial(_respond, outage), tls=server_context(cert, key)) as wire: + with tunnelling_forward_proxy(int(wire.url.rsplit(":", 1)[1])) as tunnel: + yield _Runtime(wire, tunnel, cert, outage) + + +@pytest.fixture(scope="module") +def agentcore( + module_gateway: Gateway, runtime: _Runtime, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("agentcore-proxy") + with owned_proxy( + module_gateway, + directory, + _proxy_environment(runtime, directory), + config=_write_config(directory), + remove_environment=_scrubbed(), + workers=2, + ) as candidate: + yield candidate + + +def _decide(gateway: Gateway, model: str, route: _Route = _SYSTEMONE, **extra: JsonValue) -> httpx.Response: + return gateway.request("POST", route.path, {"model": model, **route.body, **extra}) + + +def _authorization_fields(request: Request) -> dict[str, str]: + authorization: Final = request.headers.get("authorization", "") + assert authorization.startswith("AWS4-HMAC-SHA256 "), dict(request.headers) + return dict(part.split("=", 1) for part in authorization.removeprefix("AWS4-HMAC-SHA256 ").split(", ")) + + +def _assert_signed(request: Request, region: str = _US_EAST) -> None: + fields: Final = _authorization_fields(request) + access_key, scope = fields["Credential"].split("/", 1) + assert access_key == _ACCESS_KEY, fields + assert scope == f"{request.headers['x-amz-date'][:8]}/{region}/bedrock-agentcore/aws4_request", fields + signed: Final = fields["SignedHeaders"] + assert {"host", "x-amz-date", _SESSION_HEADER}.issubset(signed.split(";")), signed + expected: Final = signature( + "POST", encoded_path(request.target), request.headers, signed, request.body, _SECRET_KEY, scope + ) + assert fields["Signature"] == expected[1], (request.target, fields) + + +def _assert_answered(response: httpx.Response, model: str, route: _Route = _SYSTEMONE) -> None: + assert response.status_code == 200, response.text + assert response.json() == route.answer, response.text + assert response.headers["x-litellm-model-group"] == model, dict(response.headers) + + +def _assert_runtime_target(request: Request, runtime: str, region: str = _US_EAST) -> None: + assert request.method == "POST", request.method + assert request.target == f"/runtimes/{quote(_arn(runtime, region), safe='')}/invocations", request.target + assert request.headers["host"] == _host(region), dict(request.headers) + + +def _assert_runtime_call(request: Request, runtime: str, region: str = _US_EAST, route: _Route = _SYSTEMONE) -> None: + _assert_runtime_target(request, runtime, region) + assert json.loads(request.body) == route.upstream, request.body + + +def _canonical(body: JsonValue | Mapping[str, JsonValue]) -> str: + return json.dumps(body, sort_keys=True) + + +def _spend_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70) + return rows[0] + + +def _spend_rows(call_ids: Sequence[str]) -> list[dict[str, JsonValue]]: + query: Final = ( + "SELECT request_id, status FROM \"LiteLLM_SpendLogs\" WHERE request_id = ANY(string_to_array(%s, ','))" + ) + return eventually( + lambda: read_rows(query, (",".join(call_ids),)), lambda found: len(found) >= len(call_ids), seconds=70 + ) + + +def _connect_targets(runtime: _Runtime) -> frozenset[str]: + return frozenset(runtime.tunnel.targets()) + + +def test_an_arn_deployment_answers_on_every_decisions_route_signed_for_bedrock_agentcore( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + responses: Final = tuple(_decide(agentcore, _SIGNED, route) for route in _ROUTES) + for response, route in zip(responses, _ROUTES, strict=True): + _assert_answered(response, _SIGNED, route) + assert _connect_targets(runtime) == {f"{_host(_US_EAST)}:443"} + calls: Final = runtime.calls() + assert len(calls) == len(_ROUTES), calls + for call, route in zip(calls, _ROUTES, strict=True): + _assert_runtime_call(call, "audit_signed", route=route) + assert call.headers[_SESSION_HEADER] == _default_session("audit_signed"), dict(call.headers) + _assert_signed(call) + for response in responses: + row = _spend_row(response.headers["x-litellm-call-id"]) + assert row == { + "status": "success", + "call_type": "asystemone", + "custom_llm_provider": "strands_decider", + "model_group": _SIGNED, + "api_base": _invocations_url("audit_signed"), + "prompt_tokens": 211, + "completion_tokens": 2, + }, row + + +def test_the_arn_region_picks_the_host_and_signing_scope_over_aws_region_name( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + _assert_answered(_decide(agentcore, _REGION), _REGION) + assert _connect_targets(runtime) == {f"{_host(_EU_WEST)}:443"} + (call,) = runtime.calls() + _assert_runtime_call(call, "audit_region", _EU_WEST) + _assert_signed(call, _EU_WEST) + + +def test_a_configured_session_id_is_sent_as_written_and_signed(agentcore: Gateway, runtime: _Runtime) -> None: + runtime.reset() + _assert_answered(_decide(agentcore, _SESSION), _SESSION) + _assert_answered(_decide(agentcore, _LONG), _LONG) + calls: Final = runtime.calls() + (configured,) = _calls_to(calls, "audit_session") + (long,) = _calls_to(calls, "audit_session_long") + assert configured.headers[_SESSION_HEADER] == _CONFIGURED_SESSION, dict(configured.headers) + assert long.headers[_SESSION_HEADER] == _LONG_SESSION + _assert_signed(configured) + _assert_signed(long) + + +def test_a_deployment_api_key_sends_bearer_and_the_session_without_sigv4(agentcore: Gateway, runtime: _Runtime) -> None: + runtime.reset() + _assert_answered(_decide(agentcore, _BEARER), _BEARER) + (call,) = runtime.calls() + _assert_runtime_call(call, "audit_bearer") + assert call.headers["authorization"] == f"Bearer {_JWT}", dict(call.headers) + assert call.headers[_SESSION_HEADER] == _default_session("audit_bearer"), dict(call.headers) + assert "x-amz-date" not in call.headers, dict(call.headers) + + +def test_reserved_extra_headers_are_dropped_and_the_rest_are_kept(agentcore: Gateway, runtime: _Runtime) -> None: + runtime.reset() + _assert_answered(_decide(agentcore, _HEADERS), _HEADERS) + caller_headers: Final[dict[str, JsonValue]] = { + "x-amz-security-token": "caller-forged-token", + "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id": "caller-forged-session-0000000000000000001", + "x-audit-caller": "caller-trace", + } + _assert_answered(_decide(agentcore, _HEADERS, extra_headers=caller_headers), _HEADERS) + deployment, caller = runtime.calls() + for call in (deployment, caller): + _assert_runtime_call(call, "audit_headers") + assert call.headers[_SESSION_HEADER] == _default_session("audit_headers"), dict(call.headers) + assert "x-amz-target" not in call.headers, dict(call.headers) + assert "x-amz-security-token" not in call.headers, dict(call.headers) + _assert_signed(call) + assert deployment.headers["x-audit-trace"] == "deployment-trace", dict(deployment.headers) + assert caller.headers["x-audit-caller"] == "caller-trace", dict(caller.headers) + + +def test_an_opted_in_caller_session_id_becomes_the_header_and_stays_out_of_the_body( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + chosen: Final = f"agentcore-audit-caller-session-{uuid.uuid4().hex}" + _assert_answered(_decide(agentcore, _OPT_IN, agentcore_runtime_session_id=chosen), _OPT_IN) + _assert_answered(_decide(agentcore, _OPT_IN), _OPT_IN) + first, second = runtime.calls() + for call in (first, second): + _assert_runtime_call(call, "audit_opt_in") + _assert_signed(call) + assert first.headers[_SESSION_HEADER] == chosen, dict(first.headers) + assert second.headers[_SESSION_HEADER] == _default_session("audit_opt_in"), dict(second.headers) + + +def test_an_arn_deployment_without_api_base_takes_the_runtime_from_the_environment( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + response: Final = _decide(agentcore, _ENV) + _assert_answered(response, _ENV) + (call,) = runtime.calls() + _assert_runtime_call(call, "audit_env") + _assert_signed(call) + assert _spend_row(response.headers["x-litellm-call-id"])["api_base"] == _invocations_url("audit_env") + + +def test_an_arn_deployment_added_through_the_management_api_answers_signed( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + with agentcore.scenario() as scenario: + model: Final = scenario.model( + model_info=None, + model=_MODEL, + api_base=_arn("audit_stored"), + api_key=None, + aws_access_key_id=_ACCESS_KEY, + aws_secret_access_key=_SECRET_KEY, + ) + _assert_answered(_decide(agentcore, model), model) + (call,) = runtime.calls() + _assert_runtime_call(call, "audit_stored") + _assert_signed(call) + + +def test_health_and_test_connection_probe_the_runtime_signed(agentcore: Gateway, runtime: _Runtime) -> None: + runtime.reset() + health: Final = agentcore.request("GET", "/health", params={"model": _HEALTH}) + assert health.status_code == 200, health.text + assert (health.json()["healthy_count"], health.json()["unhealthy_count"]) == (1, 0), health.text + (probe,) = runtime.calls() + assert probe.target == f"/runtimes/{quote(_arn('audit_health'), safe='')}/invocations", probe.target + _assert_signed(probe) + connection: Final = agentcore.request( + "POST", + "/health/test_connection", + { + "litellm_params": {"model": _MODEL, "api_base": _arn("audit_test_connection"), **_SIGNING}, + "mode": "evaluation", + }, + ) + assert connection.status_code == 200, connection.text + assert connection.json()["status"] == "success", connection.text + (tested,) = runtime.calls() + assert _runtime_of(tested) == "audit_test_connection", tested.target + _assert_signed(tested) + + +@pytest.mark.parametrize(("code", "status"), tuple(_ERROR_STATUS.items()), ids=tuple(_ERROR_STATUS)) +def test_a_runtime_error_reply_maps_to_its_status_with_the_runtime_message( + agentcore: Gateway, runtime: _Runtime, code: str, status: int +) -> None: + runtime.reset() + model: Final = f"agentcore-audit-error-{code}" + response: Final = _decide(agentcore, model) + assert response.status_code == status, response.text + assert f"Strands Decider runtime error '{code}': runtime refused with {code}" in response.text, response.text + assert len(_calls_to(runtime.calls(), f"audit_error_{code}")) == 1 + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + assert (row["status"], row["model_group"]) == ("failure", model), row + + +@pytest.mark.parametrize("status", tuple(_HTTP_ERRORS), ids=tuple(str(status) for status in _HTTP_ERRORS)) +def test_a_runtime_http_error_keeps_its_status_and_message(agentcore: Gateway, runtime: _Runtime, status: int) -> None: + runtime.reset() + model: Final = f"agentcore-audit-status-{status}" + response: Final = _decide(agentcore, model) + assert response.status_code == status, response.text + assert f"scripted {_HTTP_ERRORS[status]}" in response.text, response.text + assert len(_calls_to(runtime.calls(), f"audit_status_{status}")) == 1 + assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" + + +@pytest.mark.parametrize("label", tuple(_MALFORMED_SESSIONS)) +def test_a_malformed_configured_session_id_fails_only_its_own_deployment( + agentcore: Gateway, runtime: _Runtime, label: str +) -> None: + runtime.reset() + model: Final = f"agentcore-audit-session-{label}" + response: Final = _decide(agentcore, model) + calls: Final = _calls_to(runtime.calls(), f"audit_session_{label}") + if label == "empty": + _assert_answered(response, model) + (call,) = calls + assert call.headers[_SESSION_HEADER] == "", dict(call.headers) + else: + assert response.status_code == 500, response.text + assert "agentcore_runtime_session_id" in response.text, response.text + assert calls == () + _assert_answered(_decide(agentcore, _SIGNED), _SIGNED) + + +def test_a_deployment_without_aws_credentials_fails_before_dialing_the_runtime( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + response: Final = _decide(agentcore, _UNSIGNED) + assert response.status_code == 500, response.text + assert "credentials" in response.text.lower(), response.text + assert runtime.calls() == () + assert _connect_targets(runtime) == frozenset() + _assert_answered(_decide(agentcore, _SIGNED), _SIGNED) + + +@pytest.mark.parametrize("label", tuple(_NEAR_MISS_ARNS)) +def test_a_near_miss_arn_keeps_the_plain_api_base_path(agentcore: Gateway, runtime: _Runtime, label: str) -> None: + runtime.reset() + response: Final = _decide(agentcore, f"agentcore-audit-near-miss-{label}") + assert 500 <= response.status_code < 600, response.text + assert "litellm.APIConnectionError" in response.text, response.text + assert _connect_targets(runtime) == frozenset() + assert runtime.calls() == () + + +def test_a_dropped_connection_is_a_connection_error_after_one_attempt(agentcore: Gateway, runtime: _Runtime) -> None: + runtime.reset() + response: Final = _decide(agentcore, _DROPPED) + assert response.status_code == 500, response.text + assert "litellm.APIConnectionError" in response.text, response.text + assert len(_calls_to(runtime.calls(), "audit_dropped")) == 1 + assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" + + +def _burst_routes(size: int) -> tuple[_Route, ...]: + return tuple(_ROUTES[index % len(_ROUTES)] for index in range(size)) + + +def _burst(gateway: Gateway, model: str, size: int) -> tuple[httpx.Response, ...]: + with ThreadPoolExecutor(max_workers=size) as pool: + return tuple(pool.map(partial(_decide, gateway, model), _burst_routes(size))) + + +def _health_counts(gateway: Gateway, model: str) -> tuple[int, int, int]: + response: Final = gateway.request("GET", "/health", params={"model": model}) + body: Final = JSON_OBJECT.validate_json(response.content) + healthy: Final = body["healthy_count"] + unhealthy: Final = body["unhealthy_count"] + assert isinstance(healthy, int) and isinstance(unhealthy, int), body + return response.status_code, healthy, unhealthy + + +def test_a_runtime_outage_mid_burst_fails_each_call_once_and_recovers_signed( + agentcore: Gateway, runtime: _Runtime +) -> None: + runtime.reset() + runtime.outage.set() + failed: Final = _burst(agentcore, _OUTAGE, 10) + assert [response.status_code for response in failed] == [500] * 10, [response.text for response in failed] + assert _health_counts(agentcore, _OUTAGE) == (503, 0, 1) + runtime.outage.clear() + runtime.calls() + served: Final = _burst(agentcore, _OUTAGE, 20) + for response, route in zip(served, _burst_routes(20), strict=True): + _assert_answered(response, _OUTAGE, route) + calls: Final = runtime.calls() + assert len(calls) == 20, calls + for call in calls: + _assert_runtime_target(call, "audit_outage") + _assert_signed(call) + assert sorted(_canonical(_JSON.validate_json(call.body)) for call in calls) == sorted( + _canonical(route.upstream) for route in _burst_routes(20) + ) + assert _health_counts(agentcore, _OUTAGE) == (200, 1, 0) + expected: Final = { + **{response.headers["x-litellm-call-id"]: "failure" for response in failed}, + **{response.headers["x-litellm-call-id"]: "success" for response in served}, + } + assert len(expected) == 30, expected + rows: Final = _spend_rows(tuple(expected)) + assert sorted((str(row["request_id"]), row["status"]) for row in rows) == sorted(expected.items()), rows + + +async def test_sdk_sync_and_async_calls_reach_the_runtime_signed( + runtime: _Runtime, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + runtime.reset() + for name in _scrubbed(): + monkeypatch.delenv(name) + for name, value in { + "HTTPS_PROXY": runtime.tunnel.url, + "NO_PROXY": "127.0.0.1,localhost", + "SSL_CERT_FILE": str(runtime.cert), + **_isolated_aws(tmp_path), + }.items(): + monkeypatch.setenv(name, value) + litellm.in_memory_llm_clients_cache.flush_cache() + try: + synchronous: Final = litellm.decisions( + model=_MODEL, + state=_STATE, + questions=_SDK_QUESTIONS, + api_base=_arn("audit_sdk"), + aws_access_key_id=_ACCESS_KEY, + aws_secret_access_key=_SECRET_KEY, + ) + asynchronous: Final = await litellm.asystemone( + model=_MODEL, + state=_STATE, + questions=_SDK_QUESTIONS, + api_base=_arn("audit_sdk"), + aws_access_key_id=_ACCESS_KEY, + aws_secret_access_key=_SECRET_KEY, + ) + monkeypatch.setenv("STRANDS_DECIDER_API_KEY", _ENV_JWT) + bearer: Final = await litellm.asystemone( + model=_MODEL, state=_STATE, questions=_SDK_QUESTIONS, api_base=_arn("audit_sdk_env_key") + ) + finally: + litellm.in_memory_llm_clients_cache.flush_cache() + for response in (synchronous, asynchronous, bearer): + assert response.model_dump(mode="json") == {"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE} + calls: Final = runtime.calls() + signed: Final = _calls_to(calls, "audit_sdk") + assert len(signed) == 2, calls + for call in signed: + _assert_runtime_call(call, "audit_sdk") + assert call.headers[_SESSION_HEADER] == _default_session("audit_sdk"), dict(call.headers) + _assert_signed(call) + (env_key,) = _calls_to(calls, "audit_sdk_env_key") + assert env_key.headers["authorization"] == f"Bearer {_ENV_JWT}", dict(env_key.headers) + assert "x-amz-date" not in env_key.headers, dict(env_key.headers) + + +def _answering(request: Request) -> Reply: + return _json_reply({"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE}) + + +def _envelope(request: Request) -> Reply: + return _json_reply({"error": {"code": "bad_request", "message": "plain server refused"}}) + + +def _plain_model(scenario: Scenario, wire: Wire, **extra: JsonValue) -> str: + return scenario.model(model_info=None, model=_MODEL, api_base=wire.url, api_key=None, **extra) + + +def test_a_request_body_session_id_is_refused_on_every_route_without_an_upstream_call(gateway: Gateway) -> None: + with wire_server(_answering) as wire, gateway.scenario() as scenario: + model: Final = _plain_model(scenario, wire) + responses: Final = tuple( + _decide(gateway, model, route, agentcore_runtime_session_id=f"caller-{uuid.uuid4().hex}") + for route in _ROUTES + ) + for response in responses: + assert response.status_code == 401, response.text + assert "agentcore_runtime_session_id is not allowed in request body" in response.text, response.text + assert wire.drain() == () + + +def test_a_plain_api_base_opted_into_the_session_param_sends_no_session_header_or_body_field( + gateway: Gateway, +) -> None: + with wire_server(_answering) as wire, gateway.scenario() as scenario: + model: Final = _plain_model( + scenario, wire, configurable_clientside_auth_params=["agentcore_runtime_session_id"] + ) + _assert_answered(_decide(gateway, model, agentcore_runtime_session_id=f"caller-{uuid.uuid4().hex}"), model) + (call,) = wire.drain() + assert call.target == "/v1/systemone", call.target + assert _SESSION_HEADER not in call.headers, dict(call.headers) + assert json.loads(call.body) == _UPSTREAM_BODY, call.body + + +def test_a_plain_api_base_error_envelope_keeps_the_unexpected_response_server_error(gateway: Gateway) -> None: + with wire_server(_envelope) as wire, gateway.scenario() as scenario: + model: Final = _plain_model(scenario, wire) + response: Final = _decide(gateway, model) + assert response.status_code == 500, response.text + assert "Strands Decider runtime error" not in response.text, response.text + assert len(wire.drain()) == 1 + + +def test_a_plain_api_base_forwards_amazon_shaped_extra_headers_unchanged(gateway: Gateway) -> None: + headers: Final[dict[str, JsonValue]] = { + "x-amz-target": "plain-target", + "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id": "plain-session-kept", + "x-audit-trace": "plain-trace", + } + with wire_server(_answering) as wire, gateway.scenario() as scenario: + model: Final = _plain_model(scenario, wire, extra_headers=headers) + _assert_answered(_decide(gateway, model), model) + (call,) = wire.drain() + assert (call.headers["x-amz-target"], call.headers[_SESSION_HEADER], call.headers["x-audit-trace"]) == ( + "plain-target", + "plain-session-kept", + "plain-trace", + ), dict(call.headers) + assert "authorization" not in call.headers, dict(call.headers) + assert json.loads(call.body) == _UPSTREAM_BODY, call.body + + +def _worst_case_api_base() -> str: + return "arn:aws" + "-a" * 131072 + "!" + + +def test_a_worst_case_runtime_arn_pattern_input_is_classified_in_linear_time( + gateway: Gateway, record_property: Callable[[str, object], None] +) -> None: + worst: Final = _worst_case_api_base() + started: Final = time.perf_counter() + url: Final = StrandsDeciderDecisionsConfig().get_complete_url(worst, _BODY_MODEL) + classified: Final = time.perf_counter() - started + record_property("classifier_seconds", f"{classified:.6f}") + assert url == f"{worst}/v1/systemone" + assert classified < 0.5, classified + with wire_server(_answering) as wire, gateway.scenario() as scenario: + model: Final = _plain_model(scenario, wire, configurable_clientside_auth_params=["api_base"]) + with ThreadPoolExecutor(max_workers=2) as pool: + timed: Final = pool.submit(_timed_decide, gateway, model, worst) + liveliness_started: Final = time.perf_counter() + liveliness: Final = gateway.request("GET", "/health/liveliness") + liveliness_seconds: Final = time.perf_counter() - liveliness_started + response, decide_seconds = timed.result() + record_property("decide_seconds", f"{decide_seconds:.6f}") + record_property("liveliness_seconds", f"{liveliness_seconds:.6f}") + assert liveliness.status_code == 200, liveliness.text + assert 500 <= response.status_code < 600, response.text + assert "litellm.APIConnectionError" in response.text, response.text[:2000] + assert decide_seconds < 5, decide_seconds + assert liveliness_seconds < 2, liveliness_seconds + assert wire.drain() == () + + +def _timed_decide(gateway: Gateway, model: str, api_base: str) -> tuple[httpx.Response, float]: + started: Final = time.perf_counter() + response: Final = _decide(gateway, model, api_base=api_base) + return response, time.perf_counter() - started diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 085adea81ac..8ed5237868a 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -1,27 +1,36 @@ -model_list: [] -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - database_url: os.environ/DATABASE_URL - store_model_in_db: true - disable_spend_logs: false - proxy_batch_write_at: 1 - proxy_batch_polling_interval: 1 -litellm_settings: - enable_redis_auth_cache: true - cache: true - cache_params: - type: redis - host: os.environ/REDIS_HOST - port: os.environ/REDIS_PORT -router_settings: - disable_cooldowns: true -vector_store_registry: - - vector_store_name: integration-config-store +model_list: + - model_name: perplexity/pplx-decider-v1-27b litellm_params: - vector_store_id: vs_integration_config_store - custom_llm_provider: openai - api_base: os.environ/INTEGRATION_UPSTREAM_URL - api_key: integration-provider-key - vector_store_description: declared in tests/integration/proxy_config.yaml -environment_variables: - PERPLEXITYAI_API_KEY: synthetic-decisions-env-key + model: perplexity/pplx-decider-v1-27b + api_base: http://127.0.0.1:8191 + api_key: synthetic-perplexity-key + - model_name: typesafe/jev-1.13.0 + litellm_params: + model: typesafe/jev-1.13.0 + api_base: http://127.0.0.1:8191 + api_key: synthetic-typesafe-key + - model_name: openrouter/typesafe/jev-1.13 + litellm_params: + model: openrouter/typesafe/jev-1.13 + api_base: http://127.0.0.1:8191 + api_key: synthetic-openrouter-key + - model_name: cloudflare/@cf/cloudflare/clef + litellm_params: + model: cloudflare/@cf/cloudflare/clef + api_base: http://127.0.0.1:8191 + api_key: synthetic-cloudflare-key + - model_name: strands_decider/strands-decider-2B-hobson-v19 + litellm_params: + model: strands_decider/strands-decider-2B-hobson-v19 + api_base: http://127.0.0.1:8191 + - model_name: hosted_vllm/Qwen/Qwen3-0.6B + litellm_params: + model: hosted_vllm/Qwen/Qwen3-0.6B + api_base: http://127.0.0.1:8191 + - model_name: azure_ai/decision-1 + litellm_params: + model: azure_ai/decision-1 + api_base: http://127.0.0.1:8191 + api_key: synthetic-azure-ai-key + model_info: + base_model: azure_ai/Microsoft-Decision-1 diff --git a/tests/integration/translation/decisions/__init__.py b/tests/integration/translation/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/decisions/bases/__init__.py b/tests/integration/translation/decisions/bases/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/decisions/bases/azure_ai.py b/tests/integration/translation/decisions/bases/azure_ai.py new file mode 100644 index 00000000000..27e968c55cb --- /dev/null +++ b/tests/integration/translation/decisions/bases/azure_ai.py @@ -0,0 +1,79 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from the Microsoft Foundry Decision playground (POST /providers/microsoft/v1/systemone, model is the deployment name). Mock reply captured live from a Microsoft-Decision-1 deployment on 2026-10-09. +""" +MICROSOFT_DECISION_1_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "azure_ai/decision-1", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/providers/microsoft/v1/systemone", + expected_provider_headers={"authorization": "Bearer synthetic-azure-ai-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "decision-1", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "microsoft-decision-1", + "answers": { + "confidence": { + "confidence": 0.6351489346076444, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.1824255326961778, "1": 0.8175744673038222}, + "score": 0.8175744673038222, + "type": "score", + }, + "defect": {"noul": 0.982013764002278, "type": "noul"}, + "severity": { + "choice": "high", + "confidence": 0.9866142978108987, + "probabilities": {"high": 0.9933071489054494, "low": 0.006692851094550702}, + "type": "choice", + }, + }, + "usage": {"input_tokens": 87, "output_tokens": 3}, + }, + expected_litellm_response={ + "model": "microsoft-decision-1", + "answers": { + "defect": {"type": "noul", "noul": 0.982013764002278}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9866142978108987, + "probabilities": {"high": 0.9933071489054494, "low": 0.006692851094550702}, + }, + "confidence": { + "type": "score", + "score": 0.8175744673038222, + "confidence": 0.6351489346076444, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.1824255326961778, "1": 0.8175744673038222}, + }, + }, + "usage": {"input_tokens": 87, "output_tokens": 3}, + }, +) diff --git a/tests/integration/translation/decisions/bases/cloudflare.py b/tests/integration/translation/decisions/bases/cloudflare.py new file mode 100644 index 00000000000..3efc96eeb87 --- /dev/null +++ b/tests/integration/translation/decisions/bases/cloudflare.py @@ -0,0 +1,181 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://developers.cloudflare.com/workers-ai (POST /ai/run/@cf/cloudflare/clef, reply wrapped in result). Mock reply captured live on 2026-10-06. +""" +CLEF_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "cloudflare/@cf/cloudflare/clef", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/ai/run/@cf/cloudflare/clef", + expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "result": { + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + }, + expected_litellm_response={ + "model": "clef", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.9345}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.0509}, {"value": "high", "probability": 0.9491}], + "confidence": 0.8067, + }, + { + "type": "score", + "name": "confidence", + "score": 0.9036, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.0964}, + {"value": 1, "label": "sure", "probability": 0.9036}, + ], + "confidence": 0.6515, + }, + ], + "usage": { + "input_tokens": 290, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 290, + }, + }, +) +CLEF_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "cloudflare/@cf/cloudflare/clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/ai/run/@cf/cloudflare/clef", + expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "result": { + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + }, + expected_litellm_response={ + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, +) diff --git a/tests/integration/translation/decisions/bases/hosted_vllm.py b/tests/integration/translation/decisions/bases/hosted_vllm.py new file mode 100644 index 00000000000..2da37f83a86 --- /dev/null +++ b/tests/integration/translation/decisions/bases/hosted_vllm.py @@ -0,0 +1,67 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from a real vLLM serve (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-09 from a vLLM CPU build of vllm main 702ce313f serving Qwen/Qwen3-0.6B. +""" +QWEN3_0_6B_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "hosted_vllm/Qwen/Qwen3-0.6B", + "state": "Help! My payouts have been failing for 3 days!", + "questions": { + "is_urgent": { + "type": "choice", + "instructions": "Does this convey urgency?", + "criteria": {"yes": "the message conveys urgency", "no": "the message does not convey urgency"}, + } + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"content-type": "application/json"}, + expected_provider_request={ + "model": "Qwen/Qwen3-0.6B", + "state": "Help! My payouts have been failing for 3 days!", + "questions": { + "is_urgent": { + "type": "choice", + "instructions": "Does this convey urgency?", + "criteria": {"yes": "the message conveys urgency", "no": "the message does not convey urgency"}, + } + }, + }, + mock_provider_response={ + "id": "decision-9c26c028d7f4457e", + "object": "structured_decision", + "created": 1791517765, + "model": "Qwen/Qwen3-0.6B", + "answers": { + "is_urgent": { + "type": "choice", + "choice": "yes", + "probabilities": {"yes": 0.9669140238756562, "no": 0.03308597612434374}, + "confidence": 0.9649145337527952, + } + }, + "usage": {"input_tokens": 61, "output_tokens": 1}, + "diagnostics": {"is_urgent": {"label_mass": 0.9979320910923947, "argmax_is_label": True}}, + }, + expected_litellm_response={ + "id": "decision-9c26c028d7f4457e", + "object": "structured_decision", + "created": 1791517765, + "model": "Qwen/Qwen3-0.6B", + "answers": { + "is_urgent": { + "type": "choice", + "choice": "yes", + "probabilities": {"yes": 0.9669140238756562, "no": 0.03308597612434374}, + "confidence": 0.9649145337527952, + } + }, + "usage": {"input_tokens": 61, "output_tokens": 1}, + "diagnostics": {"is_urgent": {"label_mass": 0.9979320910923947, "argmax_is_label": True}}, + }, +) diff --git a/tests/integration/translation/decisions/bases/openrouter.py b/tests/integration/translation/decisions/bases/openrouter.py new file mode 100644 index 00000000000..ed39bc1a55b --- /dev/null +++ b/tests/integration/translation/decisions/bases/openrouter.py @@ -0,0 +1,177 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://openrouter.ai/docs (POST /api/alpha/decisions). Mock reply captured live on 2026-10-06. +""" +TYPESAFE_JEV_1_13_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "openrouter/typesafe/jev-1.13", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/alpha/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-openrouter-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, + expected_litellm_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.81}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.01}, {"value": "high", "probability": 0.99}], + "confidence": 0.99, + }, + { + "type": "score", + "name": "confidence", + "score": 0.5, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.5}, + {"value": 1, "label": "sure", "probability": 0.5}, + ], + "confidence": 0, + }, + ], + "usage": { + "input_tokens": 377, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 62, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 439, + }, + }, +) +TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "openrouter/typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/alpha/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-openrouter-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, + expected_litellm_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, +) diff --git a/tests/integration/translation/decisions/bases/perplexity.py b/tests/integration/translation/decisions/bases/perplexity.py new file mode 100644 index 00000000000..9e29ceab8cc --- /dev/null +++ b/tests/integration/translation/decisions/bases/perplexity.py @@ -0,0 +1,174 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://docs.perplexity.ai (POST /v1/decisions). Mock reply captured live on 2026-10-06. +""" +PPLX_DECIDER_V1_27B_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "perplexity/pplx-decider-v1-27b", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-perplexity-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, + expected_litellm_response={ + "model": "pplx-decider-v1-27b", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.9989100737587077}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [ + {"value": "low", "probability": 0.0017684392321610232}, + {"value": "high", "probability": 0.9982315607678389}, + ], + "confidence": 0.9964631215356778, + }, + { + "type": "score", + "name": "confidence", + "score": 0.07367392327139817, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.9263260767286018}, + {"value": 1, "label": "sure", "probability": 0.07367392327139817}, + ], + "confidence": 0.8526521534572037, + }, + ], + "usage": { + "input_tokens": 318, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 321, + }, + }, +) +PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "perplexity/pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-perplexity-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, + expected_litellm_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, +) diff --git a/tests/integration/translation/decisions/bases/strands_decider.py b/tests/integration/translation/decisions/bases/strands_decider.py new file mode 100644 index 00000000000..2950b090737 --- /dev/null +++ b/tests/integration/translation/decisions/bases/strands_decider.py @@ -0,0 +1,69 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://github.com/strands-agents/decider (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-06. +""" +STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "strands_decider/strands-decider-2B-hobson-v19", + "input": "Help! My payouts have been failing for 3 days!", + "questions": [{"type": "predicate", "name": "is_urgent", "instructions": "Does this convey urgency?"}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"content-type": "application/json"}, + expected_provider_request={ + "model": "strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + }, + mock_provider_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, + expected_litellm_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": [{"type": "predicate", "name": "is_urgent", "probability": 0.8277}], + "usage": { + "input_tokens": 86, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 87, + }, + }, +) +STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "strands_decider/strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"content-type": "application/json"}, + expected_provider_request={ + "model": "strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + }, + mock_provider_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, + expected_litellm_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, +) diff --git a/tests/integration/translation/decisions/bases/typesafe.py b/tests/integration/translation/decisions/bases/typesafe.py new file mode 100644 index 00000000000..7d47ce6b082 --- /dev/null +++ b/tests/integration/translation/decisions/bases/typesafe.py @@ -0,0 +1,171 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://docs.typesafe.ai (POST /v1/systemone). Mock reply captured live on 2026-10-06. +""" +JEV_1_13_0_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "typesafe/jev-1.13.0", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"authorization": "Bearer synthetic-typesafe-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, + expected_litellm_response={ + "model": "jev-1.13.0", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.78}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.01}, {"value": "high", "probability": 0.99}], + "confidence": 0.99, + }, + { + "type": "score", + "name": "confidence", + "score": 0.51, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.49}, + {"value": 1, "label": "sure", "probability": 0.51}, + ], + "confidence": 0.03, + }, + ], + "usage": { + "input_tokens": 377, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 62, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 439, + }, + }, +) +JEV_1_13_0_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "typesafe/jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"authorization": "Bearer synthetic-typesafe-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, + expected_litellm_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, +) diff --git a/tests/integration/translation/decisions/basic/__init__.py b/tests/integration/translation/decisions/basic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_azure_ai.py b/tests/integration/translation/decisions/basic/test_decisions_basic_azure_ai.py new file mode 100644 index 00000000000..1d6b5cc4af1 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_azure_ai.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.azure_ai import MICROSOFT_DECISION_1_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [MICROSOFT_DECISION_1_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_azure_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py new file mode 100644 index 00000000000..7209c255efb --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.cloudflare import CLEF_SYSTEMONE_TEST_CASE, CLEF_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [CLEF_TEST_CASE, CLEF_SYSTEMONE_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_cloudflare(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py b/tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py new file mode 100644 index 00000000000..f52d9147273 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.hosted_vllm import QWEN3_0_6B_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [QWEN3_0_6B_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_hosted_vllm(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py new file mode 100644 index 00000000000..0e658c3ce32 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py @@ -0,0 +1,16 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.openrouter import ( + TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE, + TYPESAFE_JEV_1_13_TEST_CASE, +) +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize( + "case", [TYPESAFE_JEV_1_13_TEST_CASE, TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE], ids=lambda case: case.id +) +def test_decisions_basic_openrouter(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py new file mode 100644 index 00000000000..1b3cec8495d --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py @@ -0,0 +1,16 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.perplexity import ( + PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE, + PPLX_DECIDER_V1_27B_TEST_CASE, +) +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize( + "case", [PPLX_DECIDER_V1_27B_TEST_CASE, PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE], ids=lambda case: case.id +) +def test_decisions_basic_perplexity(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py new file mode 100644 index 00000000000..d13fd932961 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py @@ -0,0 +1,18 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.strands_decider import ( + STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE, + STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE, +) +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize( + "case", + [STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE, STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE], + ids=lambda case: case.id, +) +def test_decisions_basic_strands_decider(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py new file mode 100644 index 00000000000..79a017d6be5 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.typesafe import JEV_1_13_0_SYSTEMONE_TEST_CASE, JEV_1_13_0_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE, JEV_1_13_0_SYSTEMONE_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_typesafe(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 2578cb7d78a..2e1c67c3d91 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -45,6 +45,7 @@ import litellm.router as litellm_router_module # noqa: E402 # same import-time import litellm.utils as litellm_utils_module # noqa: E402 # same import-time dependency from litellm._logging import ALL_LOGGERS # noqa: E402 # same import-time dependency from litellm.anthropic_beta_headers_manager import reload_beta_headers_config # noqa: E402 # same import-time dependency +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER # noqa: E402 # same import-time dependency from litellm.litellm_core_utils.prompt_templates import factory as prompt_factory_module # noqa: E402 # same import-time dependency from litellm.litellm_core_utils.prompt_templates import ( # noqa: E402 # same import-time dependency image_handling as image_handling_module, @@ -157,6 +158,31 @@ def _run_coroutine_if_needed(result: object) -> None: loop.create_task(coroutine) +async def _complete_test_logging() -> None: + await GLOBAL_LOGGING_WORKER.flush() + await GLOBAL_LOGGING_WORKER.stop() + + +def _flush_completed_test_logging() -> None: + queue: Final = GLOBAL_LOGGING_WORKER._queue + if queue is None or queue._unfinished_tasks == 0: + return + worker_loop: Final = GLOBAL_LOGGING_WORKER._bound_loop + if worker_loop is None or worker_loop.is_closed(): + _run_coroutine_if_needed(_complete_test_logging()) + return + if worker_loop.is_running(): + asyncio.run_coroutine_threadsafe(_complete_test_logging(), worker_loop).result() + return + worker_loop.run_until_complete(_complete_test_logging()) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_call() -> Iterator[None]: + yield + _flush_completed_test_logging() + + def _close_handler_if_needed(handler: object) -> None: close: Final = getattr(handler, "close", None) if not callable(close): diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index cbc8f7f6d01..8bb201a2750 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -2,16 +2,25 @@ from __future__ import annotations import asyncio import json +import threading from collections.abc import Mapping +from concurrent.futures import wait from types import MappingProxyType from typing import Final +import httpx import pytest import respx +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials import litellm +from litellm.constants import AWS_SIGNING_MAX_THREADS +from litellm.cost_calculator import get_response_cost_from_hidden_params from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR from litellm.types.decisions import ( ChoiceAnswer, DecisionsResponse, @@ -20,6 +29,8 @@ from litellm.types.decisions import ( OpenAIDecisionResponse, ScoreAnswer, ) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager _QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( { @@ -138,7 +149,7 @@ async def test_adecisions_sends_the_provider_wire_contract( ) -> None: route: Final = respx_mock.post(url).respond(json=_RESPONSE) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model=model, state={"source": "unit-test"}, questions=_QUESTIONS, @@ -200,7 +211,7 @@ async def test_router_dispatches_typesafe_decisions_without_api_base( ) upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) - response: Final = await router.adecisions( + response: Final = await router.asystemone( model="jev", state="router-test", questions={ @@ -231,7 +242,7 @@ async def test_router_dispatches_typesafe_decisions_without_api_base( def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) - response: Final = litellm.decisions( + response: Final = litellm.systemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -251,7 +262,7 @@ def test_openrouter_response_keeps_provider_fields(respx_mock: respx.MockRouter) } respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=payload) - response: Final = litellm.decisions( + response: Final = litellm.systemone( model="openrouter/typesafe/jev-1.13", state="review", questions=_QUESTIONS, @@ -270,10 +281,7 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None: answers={}, usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS), ) - response._hidden_params = { - "model": "perplexity/pplx-decider-v1-27b", - "custom_llm_provider": "perplexity", - } + response.set_hidden_params({"model": "perplexity/pplx-decider-v1-27b", "custom_llm_provider": "perplexity"}) cost: Final = litellm.completion_cost(completion_response=response) perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] @@ -325,6 +333,15 @@ def test_custom_token_pricing_bills_cached_decisions_input_tokens_once( ) +def test_decisions_response_hidden_params_getter_preserves_mutable_identity() -> None: + response: Final = DecisionsResponse(model="decider", answers={}, usage=None) + + assert response.hidden_params is response._hidden_params + + response.hidden_params["mutation"] = "visible" + assert response._hidden_params["mutation"] == "visible" + + @pytest.mark.asyncio async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.MockRouter) -> None: respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) @@ -333,7 +350,7 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo litellm.callbacks = [recording_logger] try: - await litellm.adecisions( + await litellm.asystemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -357,8 +374,8 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo @pytest.mark.asyncio async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: - with pytest.raises(litellm.BadRequestError, match="Supported providers"): - await litellm.adecisions( + with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"): + await litellm.asystemone( model="unknown/jev-1.13", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -369,23 +386,85 @@ async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRo @pytest.mark.asyncio -async def test_empty_custom_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: - with pytest.raises(litellm.BadRequestError, match="Supported providers"): - await litellm.adecisions( - model="perplexity/pplx-decider-v1-27b", +async def test_provider_without_decisions_support_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match=r"Unknown Decisions provider 'anthropic'\. Supported providers"): + await litellm.asystemone( + model="anthropic/claude-sonnet-4-5", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, api_key="caller-key", - custom_llm_provider="", ) assert len(respx_mock.calls) == 0 +@pytest.mark.asyncio +async def test_empty_custom_provider_falls_back_to_the_model_prefix(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = await litellm.asystemone( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + custom_llm_provider="", + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content)["model"] == "pplx-decider-v1-27b" + assert response._hidden_params["custom_llm_provider"] == "perplexity" + + +@pytest.mark.asyncio +async def test_upstream_reply_without_answers_is_a_server_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={"model": "pplx-decider-v1-27b", "usage": {"input_tokens": 10, "output_tokens": 0}} + ) + + with pytest.raises(litellm.InternalServerError, match="unexpected response"): + await litellm.asystemone( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + +@pytest.mark.asyncio +async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_already_resolved( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": { + "model": "openrouter/typesafe/jev-1.13", + "api_key": "deployment-key", + "api_base": "https://egress.example/openrouter", + }, + } + ] + ) + upstream: Final = respx_mock.post("https://egress.example/openrouter/alpha/decisions").respond(json=_RESPONSE) + + await router.asystemone( + model="jev", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert upstream.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer deployment-key" + assert json.loads(respx_mock.calls[0].request.content)["model"] == "typesafe/jev-1.13" + + @pytest.mark.asyncio async def test_invalid_question_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): - await litellm.adecisions( + await litellm.asystemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"sentiment": {"type": "choice"}}, @@ -402,7 +481,21 @@ def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter ) with pytest.raises(litellm.BadRequestError): - litellm.decisions( + litellm.systemone( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + +def test_unreachable_upstream_maps_to_a_connection_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").mock( + side_effect=httpx.ConnectError("Cannot connect to host api.perplexity.ai:443") + ) + + with pytest.raises(litellm.APIConnectionError, match="PerplexityException - Cannot connect to host"): + litellm.systemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -418,7 +511,7 @@ def test_server_key_is_sent_to_an_explicit_api_base( monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False) route: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE) - litellm.decisions( + litellm.systemone( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -448,7 +541,7 @@ async def test_cloudflare_clef_resolves_model_and_response_envelope( "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" ).respond(json=response_body) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model=model, state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -478,7 +571,7 @@ async def test_cloudflare_clef_flash_uses_flash_endpoint_and_request_model( "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-flash" ).respond(json=_RESPONSE) - await litellm.adecisions( + await litellm.asystemone( model="cloudflare/clef-flash", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -500,7 +593,7 @@ async def test_cloudflare_api_base_from_env_uses_workers_ai_run_path( "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" ).respond(json=_RESPONSE) - await litellm.adecisions( + await litellm.asystemone( model="cloudflare/clef", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -519,7 +612,7 @@ async def test_cloudflare_requires_account_id_or_api_base_before_http( monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") with pytest.raises(litellm.BadRequestError, match="Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID"): - await litellm.adecisions( + await litellm.asystemone( model="cloudflare/clef", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -540,7 +633,7 @@ async def test_cloudflare_clef_cost_uses_the_model_cost_map( json=_RESPONSE ) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model="cloudflare/clef", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -565,7 +658,7 @@ async def test_strands_decider_requires_api_base_before_http( monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) with pytest.raises(litellm.BadRequestError, match="api_base is required"): - await litellm.adecisions( + await litellm.asystemone( model="strands_decider/strands-decider-2B-hobson-v19", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -583,7 +676,7 @@ async def test_strands_decider_without_key_preserves_response_extras( monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model="strands_decider/strands-decider-2B-hobson-v19", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -598,6 +691,33 @@ async def test_strands_decider_without_key_preserves_response_extras( assert severity.legend == {"0": "none", "1": "low", "2": "high"} +@pytest.mark.asyncio +async def test_strands_decider_plain_api_base_answers_while_every_aws_signing_thread_is_busy( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + release: Final = threading.Event() + busy: Final = tuple(AWS_SIGNING_EXECUTOR.submit(release.wait) for _ in range(AWS_SIGNING_MAX_THREADS)) + + try: + response: Final = await asyncio.wait_for( + litellm.asystemone( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base="https://strands.example", + ), + timeout=5, + ) + finally: + release.set() + wait(busy) + + assert isinstance(response.answers["severity"], ScoreAnswer) + + @pytest.mark.asyncio async def test_strands_decider_uses_key_from_matching_environment_base( monkeypatch: pytest.MonkeyPatch, @@ -607,7 +727,7 @@ async def test_strands_decider_uses_key_from_matching_environment_base( monkeypatch.setenv("STRANDS_DECIDER_API_KEY", "strands-key") route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) - await litellm.adecisions( + await litellm.asystemone( model="strands_decider/strands-decider-2B-hobson-v19", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -639,7 +759,7 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( ) route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) - response: Final = await router.adecisions( + response: Final = await router.asystemone( model="strands", state="review", questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, @@ -650,6 +770,329 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( assert response.model == _STRANDS_RESPONSE["model"] +_RUNTIME_ARN: Final = "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/qa_decider-AbC123xyz0" +_RUNTIME_URL: Final = ( + "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/" + "arn%3Aaws%3Abedrock-agentcore%3Aus-east-1%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0/invocations" +) +_RUNTIME_CREDENTIALS: Final = Credentials(access_key="AKIDEXAMPLE", secret_key="example-secret") +_SESSION_HEADER: Final = "x-amzn-bedrock-agentcore-runtime-session-id" + + +def _sigv4_verifies(request: httpx.Request) -> bool: + authorization: Final = request.headers["authorization"] + signed_names: Final = authorization.split("SignedHeaders=")[1].split(",")[0].split(";") + replayed: Final = AWSRequest( + method=request.method, + url=str(request.url), + data=request.content, + headers={name: request.headers[name] for name in signed_names}, + ) + replayed.context["timestamp"] = request.headers["x-amz-date"] + auth: Final = SigV4Auth(_RUNTIME_CREDENTIALS, "bedrock-agentcore", "us-east-1") + string_to_sign: Final = auth.string_to_sign(replayed, auth.canonical_request(replayed)) + return authorization.endswith(f"Signature={auth.signature(string_to_sign, replayed)}") + + +@pytest.mark.parametrize("use_async", [True, False]) +@pytest.mark.asyncio +async def test_strands_decider_runtime_arn_sends_a_sigv4_signed_invoke_agent_runtime_call( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + use_async: bool, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + route: Final = respx_mock.post(_RUNTIME_URL).respond(json=_STRANDS_RESPONSE) + call_args: Final[Mapping[str, object]] = { + "model": "strands_decider/strands-decider-2B-hobson-v19", + "state": "review", + "questions": {"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + "api_base": _RUNTIME_ARN, + "aws_access_key_id": _RUNTIME_CREDENTIALS.access_key, + "aws_secret_access_key": _RUNTIME_CREDENTIALS.secret_key, + } + + response: Final = await litellm.asystemone(**call_args) if use_async else litellm.systemone(**call_args) + + assert route.call_count == 1 + sent: Final = route.calls[0].request + assert _sigv4_verifies(sent) + assert json.loads(sent.content)["model"] == "strands-decider-2B-hobson-v19" + assert sent.headers[_SESSION_HEADER].startswith("litellm-decider-") + assert response.model == _STRANDS_RESPONSE["model"] + + +@pytest.mark.asyncio +async def test_strands_decider_runtime_arn_router_deployment_answers_openai_format_calls( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "strands", + "litellm_params": { + "model": "strands_decider/strands-decider-2B-hobson-v19", + "api_base": _RUNTIME_ARN, + "aws_access_key_id": _RUNTIME_CREDENTIALS.access_key, + "aws_secret_access_key": _RUNTIME_CREDENTIALS.secret_key, + "aws_region_name": "eu-west-1", + "agentcore_runtime_session_id": "pooled-session-0123456789abcdef0123456", + }, + } + ] + ) + route: Final = respx_mock.post(_RUNTIME_URL).respond(json=_STRANDS_RESPONSE) + + response: Final = await router.adecisions( + model="strands", + input="review", + extra_headers={_SESSION_HEADER: "caller-session-0123456789abcdef01234567"}, + questions=[ + { + "type": "score", + "name": "severity", + "instructions": "How bad?", + "levels": [{"label": "none"}, {"label": "low"}, {"label": "high"}], + } + ], + ) + + sent: Final = route.calls[0].request + assert _sigv4_verifies(sent) + assert sent.headers[_SESSION_HEADER] == "pooled-session-0123456789abcdef0123456" + assert isinstance(response, OpenAIDecisionResponse) + + +@pytest.mark.asyncio +async def test_strands_decider_runtime_arn_with_api_key_sends_the_bearer_token_unsigned( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + route: Final = respx_mock.post(_RUNTIME_URL).respond(json=_STRANDS_RESPONSE) + + await litellm.asystemone( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base=_RUNTIME_ARN, + api_key="runtime-jwt", + ) + + sent: Final = route.calls[0].request + assert sent.headers["authorization"] == "Bearer runtime-jwt" + assert "x-amz-date" not in sent.headers + assert _SESSION_HEADER in sent.headers + + +@pytest.mark.parametrize( + ("code", "error_class"), + [ + ("bad_request", litellm.BadRequestError), + ("loading", litellm.ServiceUnavailableError), + ("inference_error", litellm.InternalServerError), + ], +) +@pytest.mark.asyncio +async def test_strands_decider_runtime_error_envelope_raises_instead_of_answering( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + code: str, + error_class: type[Exception], +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + respx_mock.post(_RUNTIME_URL).respond(json={"error": {"code": code, "message": "at most 16 questions"}}) + + with pytest.raises(error_class, match="at most 16 questions"): + await litellm.asystemone( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base=_RUNTIME_ARN, + aws_access_key_id=_RUNTIME_CREDENTIALS.access_key, + aws_secret_access_key=_RUNTIME_CREDENTIALS.secret_key, + ) + + +@pytest.mark.asyncio +async def test_strands_decider_plain_api_base_error_envelope_stays_an_unexpected_response( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) + respx_mock.post("https://strands.example/v1/systemone").respond( + json={"error": {"code": "bad_request", "message": "at most 16 questions"}} + ) + + with pytest.raises(litellm.InternalServerError, match="returned an unexpected response"): + await litellm.asystemone( + model="strands_decider/strands-decider-2B-hobson-v19", + state="review", + questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + api_base="https://strands.example", + ) + + +_VLLM_RESPONSE: Final[Mapping[str, object]] = { + "id": "systemone-4af3d2c1", + "object": "structured_decision", + "created": 1760012345, + "model": "Qwen/Qwen3-0.6B", + "answers": { + "is_defect": { + "type": "choice", + "choice": "yes", + "confidence": 0.91, + "probabilities": {"yes": 0.91, "no": 0.09}, + } + }, + "usage": {"input_tokens": 154, "output_tokens": 2}, + "diagnostics": {"engine": "vllm", "logprobs_mode": "raw_logprobs"}, +} + + +@pytest.mark.asyncio +async def test_hosted_vllm_requires_api_base_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + + with pytest.raises(litellm.BadRequestError, match="api_base is required"): + await litellm.asystemone( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="review", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_hosted_vllm_without_key_sends_no_authorization_and_preserves_response_extras( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + response: Final = await litellm.asystemone( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + api_base="http://vllm.local:8000", + ) + + assert route.called + assert "authorization" not in respx_mock.calls[0].request.headers + assert response.model_extra["id"] == _VLLM_RESPONSE["id"] + assert response.model_extra["object"] == _VLLM_RESPONSE["object"] + assert response.model_extra["diagnostics"] == _VLLM_RESPONSE["diagnostics"] + + +@pytest.mark.asyncio +async def test_hosted_vllm_uses_key_and_base_from_environment( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://vllm.local:8000/v1") + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "vllm-key") + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + await litellm.asystemone( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + ) + + assert route.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer vllm-key" + + +@pytest.mark.asyncio +async def test_hosted_vllm_api_base_with_v1_suffix_posts_to_v1_systemone( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + await litellm.asystemone( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + api_base="http://vllm.local:8000/v1", + ) + + assert route.called + + +@pytest.mark.asyncio +async def test_hosted_vllm_choice_question_goes_out_as_a_jev_choice_with_the_served_model_name( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + await litellm.asystemone( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={ + "is_defect": { + "type": "choice", + "instructions": "Is this a defect?", + "criteria": {"yes": "defective", "no": "intact"}, + } + }, + api_base="http://vllm.local:8000", + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "Qwen/Qwen3-0.6B", + "state": "The package arrived broken.", + "questions": { + "is_defect": { + "type": "choice", + "instructions": "Is this a defect?", + "criteria": {"yes": "defective", "no": "intact"}, + } + }, + } + + +_PROVIDERS_WHOSE_CHAT_MODELS_ALSO_DECIDE: Final = frozenset( + {LlmProviders.OPENAI.value, LlmProviders.OPENROUTER.value, LlmProviders.HOSTED_VLLM.value} +) +_DEDICATED_DECIDER_PROVIDERS: Final[tuple[str, ...]] = tuple( + sorted( + provider.value + for provider in LlmProviders + if provider.value not in _PROVIDERS_WHOSE_CHAT_MODELS_ALSO_DECIDE + and ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None + ) +) + + +@pytest.mark.parametrize("provider", _DEDICATED_DECIDER_PROVIDERS) +def test_every_dedicated_decider_provider_ships_an_evaluation_mode_cost_map_entry(provider: str) -> None: + evaluation_entries: Final = tuple( + name + for name, info in litellm.model_cost.items() + if isinstance(info, Mapping) and info.get("litellm_provider") == provider and info.get("mode") == "evaluation" + ) + + assert evaluation_entries, f"a {provider} decider would be health-checked as chat without a mode: evaluation entry" + + @pytest.mark.asyncio @pytest.mark.parametrize( ("api_base", "url"), @@ -669,7 +1112,7 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_ monkeypatch.delenv("OPENAI_API_BASE", raising=False) route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE) - response: Final = await litellm.adecisions( + response: Final = await litellm.asystemone( model="openai/gpt-6-luna", state="The package arrived broken.", questions={ @@ -701,7 +1144,7 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_ ], } assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} - assert response._hidden_params["custom_llm_provider"] == "openai" + assert response.hidden_params["custom_llm_provider"] == "openai" luna_cost: Final = litellm.model_cost["gpt-6-luna"] expected_cost: Final = ( (_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * float(luna_cost["input_cost_per_token"]) @@ -754,7 +1197,7 @@ def test_openai_decisions_use_the_same_settings_as_other_openai_calls( monkeypatch.setenv(name, value) route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE) - litellm.decisions( + litellm.systemone( model="openai/gpt-6-luna", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, @@ -817,17 +1260,8 @@ async def test_openai_format_calls_to_a_systemone_provider_get_openai_format_ans }, "cannot serve this request", ), - ( - "openai/gpt-6-luna", - { - "state": "review", - "input": "review", - "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], - }, - "not both", - ), ), - ids=("image_to_systemone_provider", "state_and_input"), + ids=("image_to_systemone_provider",), ) async def test_requests_a_provider_cannot_serve_are_rejected_before_http( respx_mock: respx.MockRouter, @@ -839,3 +1273,257 @@ async def test_requests_a_provider_cannot_serve_are_rejected_before_http( await litellm.adecisions(model=model, api_key="caller-key", **request_kwargs) assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_openrouter_decisions_uses_provider_reported_cost_without_cost_map( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delitem(litellm.model_cost, "openrouter/typesafe/jev-1.13", raising=False) + cost: Final = 1.1802e-5 + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond( + json={ + "model": "typesafe/jev-1.13", + "answers": _RESPONSE["answers"], + "usage": { + "input_tokens": _INPUT_TOKENS, + "output_tokens": _OUTPUT_TOKENS, + "cost": cost, + }, + } + ) + + response: Final = await litellm.asystemone( + model="openrouter/typesafe/jev-1.13", + state={"source": "unit-test"}, + questions=_QUESTIONS, + api_key="caller-key", + ) + + assert route.called + assert get_response_cost_from_hidden_params(response.hidden_params) == cost + + +_SAFETY_IDENTIFIER: Final = "end-user-7" +_PREDICATE_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, +) +_PREDICATE_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( + ("decisions", MappingProxyType({"input": "review", "questions": _PREDICATE_QUESTIONS})), + ( + "systemone", + MappingProxyType( + {"state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}} + ), + ), +) +_PREDICATE_REQUEST_IDS: Final = ("decisions", "systemone") + + +@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +def test_safety_identifier_is_refused_before_http_when_the_provider_cannot_take_it( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"safety_identifier.*drop_params") as caught: + getattr(litellm, api)( + model="perplexity/pplx-decider-v1-27b", + safety_identifier=_SAFETY_IDENTIFIER, + api_key="caller-key", + **request_kwargs, + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope", ("call", "global")) +async def test_safety_identifier_is_dropped_from_the_wire_under_drop_params( + scope: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", scope == "global") + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + input="review", + questions=_PREDICATE_QUESTIONS, + safety_identifier=_SAFETY_IDENTIFIER, + api_key="caller-key", + **({"drop_params": True} if scope == "call" else {}), + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert isinstance(response, OpenAIDecisionResponse) + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": "is_defect", "probability": 0.9} + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params")) +async def test_safety_identifier_reaches_openai_whether_or_not_params_are_dropped( + drop_params: bool, + api: str, + request_kwargs: Mapping[str, object], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + + await getattr(litellm, f"a{api}")( + model="openai/gpt-6-luna", + safety_identifier=_SAFETY_IDENTIFIER, + api_key="caller-key", + **request_kwargs, + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + "safety_identifier": _SAFETY_IDENTIFIER, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +async def test_non_string_safety_identifier_is_rejected_before_http( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + invalid_safety_identifier: Final[Mapping[str, object]] = MappingProxyType({"safety_identifier": 7}) + + with pytest.raises(litellm.BadRequestError, match=r"(?s)Invalid Decisions request.*safety_identifier"): + await getattr(litellm, f"a{api}")( + model="openai/gpt-6-luna", api_key="caller-key", **request_kwargs, **invalid_safety_identifier + ) + + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_params( + api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + dropped_safety_identifier: Final[Mapping[str, object]] = MappingProxyType( + {"safety_identifier": 7, "drop_params": True} + ) + + await getattr(litellm, f"a{api}")( + model="openai/gpt-6-luna", api_key="caller-key", **request_kwargs, **dropped_safety_identifier + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "api_base", + ( + "https://res.services.ai.azure.com", + "https://res.services.ai.azure.com/", + "https://res.services.ai.azure.com/models", + "https://res.services.ai.azure.com/openai/v1", + "https://res.services.ai.azure.com/api/projects/proj", + "https://res.services.ai.azure.com/api/projects/proj/openai/v1", + "https://res.services.ai.azure.com/api/projects/proj/models", + "https://res.services.ai.azure.com/models?api-version=2024-05-01-preview", + ), +) +async def test_azure_ai_decision_posts_deployment_to_foundry_systemone_route( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + api_base: str, +) -> None: + monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) + monkeypatch.delenv("AZURE_AI_API_KEY", raising=False) + route: Final = respx_mock.post("https://res.services.ai.azure.com/providers/microsoft/v1/systemone").respond( + json={**_RESPONSE, "model": "microsoft-decision-1"} + ) + + response: Final = await litellm.asystemone( + model="azure_ai/decision-1", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_base=api_base, + api_key="foundry-key", + ) + + assert route.called + request: Final = respx_mock.calls[0].request + assert request.headers["authorization"] == "Bearer foundry-key" + assert "api-key" not in request.headers + assert json.loads(request.content) == { + "model": "decision-1", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} + assert response._hidden_params["model"] == "azure_ai/decision-1" + + +@pytest.mark.asyncio +async def test_azure_ai_decision_reads_foundry_env_and_keeps_gateway_path_prefix( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("AZURE_AI_API_BASE", "https://gateway.example.com/foundry/models") + monkeypatch.setenv("AZURE_AI_API_KEY", "env-foundry-key") + route: Final = respx_mock.post("https://gateway.example.com/foundry/providers/microsoft/v1/systemone").respond( + json=_RESPONSE + ) + + await litellm.asystemone( + model="azure_ai/decision-1", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert route.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer env-foundry-key" + + +@pytest.mark.asyncio +async def test_azure_ai_decision_requires_api_base_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) + monkeypatch.setenv("AZURE_AI_API_KEY", "foundry-key") + + with pytest.raises(litellm.BadRequestError, match="Missing AZURE_AI_API_BASE"): + await litellm.asystemone( + model="azure_ai/decision-1", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert len(respx_mock.calls) == 0 diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 9de768ea47b..20ccd49973e 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1195,6 +1195,34 @@ def test_branchless_provider_transport_error_maps_to_api_connection_error(): ) +def test_openrouter_transport_error_maps_to_api_connection_error(): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + original_exception = BaseLLMException(status_code=500, message="[Errno 111] Connection refused") + original_exception.status_code_is_synthesized = True + + with pytest.raises(litellm.APIConnectionError): + exception_type( + model="typesafe/jev-1.13", + original_exception=original_exception, + custom_llm_provider="openrouter", + ) + + +def test_openrouter_upstream_500_still_maps_to_api_error(): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + original_exception = BaseLLMException(status_code=500, message="upstream exploded") + + with pytest.raises(litellm.APIError) as excinfo: + exception_type( + model="typesafe/jev-1.13", + original_exception=original_exception, + custom_llm_provider="openrouter", + ) + assert excinfo.value.status_code == 500 + + def test_branchless_provider_upstream_500_still_maps_to_internal_server_error(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 941e44feb26..72d4ed0ffde 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -647,3 +647,78 @@ async def test_ahealth_check_probes_strands_through_decisions_without_mode( assert "error" not in result, result assert upstream.called assert "authorization" not in upstream.calls[0].request.headers + + +@pytest.mark.asyncio +async def test_ahealth_check_probes_hosted_vllm_with_a_choice_question( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond( + json={ + "model": "Qwen/Qwen3-0.6B", + "answers": { + "reachable": { + "type": "choice", + "choice": "yes", + "confidence": 1.0, + "probabilities": {"yes": 1.0, "no": 0.0}, + } + }, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check( + { + "model": "hosted_vllm/Qwen/Qwen3-0.6B", + "api_base": "http://vllm.local:8000", + }, + mode="evaluation", + ) + + assert "error" not in result, result + assert upstream.called + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["questions"] == { + "reachable": { + "type": "choice", + "instructions": "Is the service reachable?", + "criteria": {"yes": None, "no": None}, + } + } + + +@pytest.mark.asyncio +async def test_ahealth_check_keeps_the_noul_probe_for_other_decisions_providers( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond( + json={ + "model": "jev-1.13", + "answers": {"reachable": {"type": "noul", "noul": 1.0}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check( + { + "model": "typesafe/jev-1.13", + "api_key": "sk-test", + }, + mode="evaluation", + ) + + assert "error" not in result, result + assert upstream.called + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["questions"] == { + "reachable": {"type": "noul", "instructions": "Is the service reachable?"} + } diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 5f254a3bc6c..7268748b7e9 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -471,6 +471,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "llm_passthrough_route", "asearch", "adecisions", + "asystemone", "avector_store_create", "avector_store_search", "acreate_skill", diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index 249a1a7640a..e73c5b92ab6 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -6,7 +6,7 @@ from pydantic import TypeAdapter from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_response, systemone_request_to_ir from litellm.llms.openai.decisions.transformation import ( - OPENAI_DECISIONS_ENDPOINT, + OpenAIDecisionsConfig, ir_to_openai_request, ir_to_openai_response, openai_request_to_ir, @@ -101,7 +101,7 @@ def test_an_openai_request_reaches_openai_unchanged() -> None: def test_an_openai_response_reaches_the_caller_unchanged() -> None: ir: Final = _openai_ir(_OPENAI_REQUEST) - parsed: Final = OPENAI_DECISIONS_ENDPOINT.parse_response(_OPENAI_RESPONSE, ir) + parsed: Final = OpenAIDecisionsConfig().parse_response(_OPENAI_RESPONSE, ir) assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE @@ -110,7 +110,7 @@ def test_answers_openai_did_not_return_are_refusals() -> None: ir: Final = _openai_ir(_OPENAI_REQUEST) payload: Final = {**_OPENAI_RESPONSE, "answers": [_PREDICATE_ANSWER]} - response: Final = ir_to_openai_response(OPENAI_DECISIONS_ENDPOINT.parse_response(payload, ir), ir, "requested") + response: Final = ir_to_openai_response(OpenAIDecisionsConfig().parse_response(payload, ir), ir, "requested") assert [answer.type for answer in response.answers] == ["predicate", "refusal", "refusal", "refusal"] assert [answer.name for answer in response.answers] == ["damaged", None, "severity", "fraud"] @@ -230,7 +230,7 @@ def test_an_openai_response_becomes_systemone_answers_with_the_callers_score_lab ], } - response: Final = ir_to_systemone_response(OPENAI_DECISIONS_ENDPOINT.parse_response(payload, ir), ir) + response: Final = ir_to_systemone_response(OpenAIDecisionsConfig().parse_response(payload, ir), ir) assert response.model_dump(mode="json") == { "model": "gpt-6-luna", diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index b177b80aed1..beffcf9b69c 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -96,6 +96,15 @@ def test_openrouter_extra_body_transformation(): ] +def test_openrouter_supported_params_include_reasoning_for_unmapped_model(): + supported_params = OpenrouterConfig().get_supported_openai_params( + "openrouter/unmapped-model" + ) + + assert "reasoning_effort" in supported_params + assert "thinking" in supported_params + + def test_openrouter_cache_control_flag_removal(): transformed_request = OpenrouterConfig().transform_request( model="openrouter/deepseek/deepseek-chat", @@ -537,17 +546,15 @@ def test_openrouter_reasoning_models_allow_reasoning_effort_param(): assert supported_params.count("reasoning_effort") == 1 -def test_openrouter_non_reasoning_models_do_not_add_reasoning_effort(): - """ - Models without reasoning support should not gain reasoning-specific params. - """ +def test_openrouter_non_reasoning_models_support_reasoning_params(): config = OpenrouterConfig() supported_params = config.get_supported_openai_params( model="openrouter/anthropic/claude-3-5-haiku" ) - assert "reasoning_effort" not in supported_params + assert "reasoning_effort" in supported_params + assert "thinking" in supported_params def test_openrouter_reasoning_effort_max_maps_to_xhigh(): diff --git a/tests/unit/llms/strands_decider/__init__.py b/tests/unit/llms/strands_decider/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/strands_decider/decisions/__init__.py b/tests/unit/llms/strands_decider/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/strands_decider/decisions/test_strands_decider_decisions_transformation.py b/tests/unit/llms/strands_decider/decisions/test_strands_decider_decisions_transformation.py new file mode 100644 index 00000000000..4d249e59233 --- /dev/null +++ b/tests/unit/llms/strands_decider/decisions/test_strands_decider_decisions_transformation.py @@ -0,0 +1,256 @@ +import json +from collections.abc import Mapping +from typing import Final + +import httpx +import pytest +from botocore.credentials import Credentials +from pydantic import TypeAdapter + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.decisions.transformation import systemone_request_to_ir +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.strands_decider.decisions.transformation import ( + AGENTCORE_SESSION_HEADER, + StrandsDeciderDecisionsConfig, +) +from litellm.types.decisions import DecisionsIRResponse, DecisionsRequestBody +from litellm.types.llms.bedrock import AwsAuthParams + +_ARN: Final = "arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/qa_decider-AbC123xyz0" +_BODY: Final[Mapping[str, object]] = { + "model": "strands-decider-2B-hobson-v21", + "state": "My order arrived broken.", + "questions": {"is_complaint": {"type": "noul", "instructions": "Is this a complaint?"}}, +} +_IR_REQUEST: Final = systemone_request_to_ir( + TypeAdapter(DecisionsRequestBody).validate_python({"state": _BODY["state"], "questions": _BODY["questions"]}) +) + + +class _RecordingAWS(BaseAWSLLM): + def __init__(self) -> None: + super().__init__() + self.resolved: tuple[tuple[AwsAuthParams, str | None], ...] = () + + def resolve_credentials(self, auth_params: AwsAuthParams, aws_region_name: str | None) -> Credentials: + self.resolved = (*self.resolved, (auth_params, aws_region_name)) + return Credentials(access_key="AKIDEXAMPLE", secret_key="example-secret") + + +def _sign( + config: StrandsDeciderDecisionsConfig, + api_base: str, + headers: Mapping[str, str], + api_key: str | None = None, + litellm_params: Mapping[str, object] | None = None, +) -> tuple[Mapping[str, str], bytes | None]: + return config.sign_request( + headers=headers, + url=config.get_complete_url(api_base=api_base, model="strands-decider-2B-hobson-v21"), + api_base=api_base, + body=_BODY, + api_key=api_key, + litellm_params=litellm_params or {}, + ) + + +@pytest.mark.parametrize( + ("arn", "expected_url"), + [ + ( + _ARN, + "https://bedrock-agentcore.us-west-2.amazonaws.com/runtimes/" + "arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0/invocations", + ), + ( + "arn:aws-cn:bedrock-agentcore:cn-north-1:123456789012:runtime/qa_decider-AbC123xyz0", + "https://bedrock-agentcore.cn-north-1.amazonaws.com.cn/runtimes/" + "arn%3Aaws-cn%3Abedrock-agentcore%3Acn-north-1%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0" + "/invocations", + ), + ( + "arn:aws-us-gov:bedrock-agentcore:us-gov-west-1:123456789012:runtime/qa_decider-AbC123xyz0", + "https://bedrock-agentcore.us-gov-west-1.amazonaws.com/runtimes/" + "arn%3Aaws-us-gov%3Abedrock-agentcore%3Aus-gov-west-1%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0" + "/invocations", + ), + ], +) +def test_runtime_arn_targets_the_regional_invoke_agent_runtime_url(arn: str, expected_url: str) -> None: + assert StrandsDeciderDecisionsConfig().get_complete_url(api_base=arn, model="any") == expected_url + + +@pytest.mark.parametrize( + "api_base", + [ + "https://strands.example", + "arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/qa_decider-AbC123xyz0/runtime-endpoint/DEFAULT", + "arn:aws:bedrock-agentcore:evil.example/x:123456789012:runtime/qa_decider-AbC123xyz0", + "arn:aws:bedrock:us-west-2:123456789012:runtime/qa_decider-AbC123xyz0", + ], +) +def test_other_api_bases_keep_the_plain_systemone_request(api_base: str) -> None: + aws: Final = _RecordingAWS() + config: Final = StrandsDeciderDecisionsConfig(aws=aws) + headers: Final = {"Content-Type": "application/json"} + + assert config.get_complete_url(api_base=api_base, model="any") == f"{api_base}/v1/systemone" + assert _sign(config, api_base, headers) == (headers, None) + assert aws.resolved == () + + +def test_sign_request_signs_the_body_for_the_arn_region_with_deployment_credentials() -> None: + aws: Final = _RecordingAWS() + config: Final = StrandsDeciderDecisionsConfig(aws=aws) + + signed_headers, signed_body = _sign( + config, + _ARN, + {"Content-Type": "application/json"}, + litellm_params={"aws_profile_name": "qa", "aws_role_name": "decider-role", "aws_region_name": "us-east-1"}, + ) + + assert signed_body == json.dumps(_BODY).encode() + assert len(aws.resolved) == 1 + auth_params, region = aws.resolved[0] + assert (auth_params.aws_profile_name, auth_params.aws_role_name, region) == ("qa", "decider-role", "us-west-2") + authorization: Final = signed_headers["Authorization"] + assert authorization.startswith("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/") + assert "/us-west-2/bedrock-agentcore/aws4_request" in authorization + assert AGENTCORE_SESSION_HEADER.lower() in authorization + + +def test_default_session_id_is_stable_per_runtime_and_within_the_agentcore_length_bounds() -> None: + other_arn: Final = _ARN.replace("AbC123xyz0", "ZyX987cba1") + + first, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), _ARN, {}) + again, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), _ARN, {}) + other, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), other_arn, {}) + + session_id: Final = first[AGENTCORE_SESSION_HEADER] + assert session_id == again[AGENTCORE_SESSION_HEADER] + assert session_id != other[AGENTCORE_SESSION_HEADER] + assert 33 <= len(session_id) <= 256 + + +_CALLER_RESERVED_HEADERS: Final = { + AGENTCORE_SESSION_HEADER.lower(): "caller-session-0123456789abcdef0123456789", + "X-Amzn-Bedrock-AgentCore-Runtime-User-Id": "someone-else", + "X-Amz-Security-Token": "caller-token", + "Host": "evil.example", +} + + +@pytest.mark.parametrize("api_key", [None, "runtime-jwt"]) +def test_caller_reserved_agentcore_headers_never_reach_the_runtime(api_key: str | None) -> None: + signed_headers, _ = _sign( + StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), + _ARN, + {**_CALLER_RESERVED_HEADERS, "X-Request-Source": "batch"}, + api_key=api_key, + ) + default_headers, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), _ARN, {}, api_key=api_key) + + forwarded: Final = {name.lower(): value for name, value in signed_headers.items()} + assert forwarded[AGENTCORE_SESSION_HEADER.lower()] == default_headers[AGENTCORE_SESSION_HEADER] + assert not any(value in forwarded.values() for value in _CALLER_RESERVED_HEADERS.values()) + assert "x-amzn-bedrock-agentcore-runtime-user-id" not in forwarded + assert "host" not in forwarded + assert forwarded["x-request-source"] == "batch" + + +def test_deployment_session_param_picks_the_session_over_the_default_and_any_caller_header() -> None: + replica_session: Final = "strands-decider-replica-a-0000000000" + + signed_headers, _ = _sign( + StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), + _ARN, + _CALLER_RESERVED_HEADERS, + litellm_params={"agentcore_runtime_session_id": replica_session}, + ) + + session_headers: Final = [ + value for name, value in signed_headers.items() if name.lower() == "x-amzn-bedrock-agentcore-runtime-session-id" + ] + assert session_headers == [replica_session] + assert AGENTCORE_SESSION_HEADER.lower() in signed_headers["Authorization"] + + +def test_api_key_on_a_runtime_arn_sends_the_bearer_token_without_sigv4() -> None: + aws: Final = _RecordingAWS() + + signed_headers, signed_body = _sign( + StrandsDeciderDecisionsConfig(aws=aws), _ARN, {"Authorization": "Bearer runtime-jwt"}, api_key="runtime-jwt" + ) + + assert signed_body is None + assert signed_headers["Authorization"] == "Bearer runtime-jwt" + assert AGENTCORE_SESSION_HEADER in signed_headers + assert aws.resolved == () + + +def _response(url: str, payload: object) -> httpx.Response: + return httpx.Response(200, json=payload, request=httpx.Request("POST", url)) + + +def _transform(api_base: str, payload: object) -> DecisionsIRResponse: + config: Final = StrandsDeciderDecisionsConfig() + url: Final = config.get_complete_url(api_base=api_base, model="strands-decider-2B-hobson-v21") + return config.transform_decisions_response( + model="strands-decider-2B-hobson-v21", + custom_llm_provider="strands_decider", + raw_response=_response(url, payload), + request=_IR_REQUEST, + ) + + +@pytest.mark.parametrize( + ("code", "status_code"), + [("bad_request", 400), ("invalid_request", 400), ("loading", 503), ("inference_error", 500), ("new_code", 500)], +) +def test_runtime_error_envelope_raises_with_the_mapped_status(code: str, status_code: int) -> None: + with pytest.raises(BaseLLMException) as raised: + _transform(_ARN, {"error": {"code": code, "message": "at most 16 questions"}}) + + assert raised.value.status_code == status_code + assert code in raised.value.message + assert "at most 16 questions" in raised.value.message + + +@pytest.mark.parametrize("code", ["bad_request", "loading", "inference_error"]) +def test_plain_api_base_error_envelope_keeps_the_unexpected_response_error(code: str) -> None: + with pytest.raises(BaseLLMException) as raised: + _transform("https://strands.example", {"error": {"code": code, "message": "at most 16 questions"}}) + + assert raised.value.status_code == 500 + assert raised.value.message.startswith("Decisions provider 'strands_decider' returned an unexpected response") + + +@pytest.mark.parametrize( + "url", + [ + "https://strands.example/runtimes/arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A123456789012%3Aruntime" + "%2Fqa_decider-AbC123xyz0/invocations", + "https://bedrock-agentcore.us-west-2.amazonaws.com/runtimes/not-an-arn/invocations", + ], +) +def test_error_envelope_from_a_url_that_is_not_the_runtime_keeps_the_unexpected_response_error(url: str) -> None: + with pytest.raises(BaseLLMException) as raised: + StrandsDeciderDecisionsConfig().transform_decisions_response( + model="strands-decider-2B-hobson-v21", + custom_llm_provider="strands_decider", + raw_response=_response(url, {"error": {"code": "bad_request", "message": "at most 16 questions"}}), + request=_IR_REQUEST, + ) + + assert raised.value.status_code == 500 + + +@pytest.mark.parametrize("payload", [["not", "an", "object"], {"error": "unstructured"}]) +def test_runtime_reply_that_is_not_an_error_envelope_keeps_the_unexpected_response_error(payload: object) -> None: + with pytest.raises(BaseLLMException) as raised: + _transform(_ARN, payload) + + assert raised.value.status_code == 500 + assert raised.value.message.startswith("Decisions provider 'strands_decider' returned an unexpected response") diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 83ac56c4c85..aadb5881ce0 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -4,7 +4,7 @@ Unit tests for auth_utils functions related to rate limiting and customer ID ext import base64 import logging -from typing import Optional +from typing import Final, Optional from unittest.mock import MagicMock, patch import pytest @@ -16,8 +16,6 @@ from litellm.proxy.auth.auth_utils import ( abbreviate_api_key, check_complete_credentials, custom_auth_common_checks_warning, - log_once_if_budget_reservation_disabled, - warn_once_if_custom_auth_skips_common_checks, get_end_user_id_from_request_body, get_key_mcp_rpm_limit, get_key_model_rpm_limit, @@ -29,6 +27,8 @@ from litellm.proxy.auth.auth_utils import ( get_project_model_tpm_limit, get_request_route_template, is_request_body_safe, + log_once_if_budget_reservation_disabled, + warn_once_if_custom_auth_skips_common_checks, ) @@ -2727,6 +2727,22 @@ class TestIsRequestBodySafeBlocksBedrockProjectOverride: ) +def test_agentcore_runtime_session_id_in_request_body_is_rejected_without_admin_opt_in(): + body: Final = {"model": "strands-decider", "agentcore_runtime_session_id": "caller-session-0123456789abcdef0123"} + + with pytest.raises(ValueError, match="agentcore_runtime_session_id"): + is_request_body_safe(request_body=body, general_settings={}, llm_router=None, model="strands-decider") + assert ( + is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="strands-decider", + ) + is True + ) + + class TestIsRequestBodySafeBlocksClaudePlatformWorkspaceOverride: @pytest.mark.parametrize( "alias", ["workspace_id", "aws_workspace_id", "anthropic_workspace_id", "anthropic-workspace-id"] diff --git a/tests/unit/proxy/common_utils/test_custom_openapi_spec.py b/tests/unit/proxy/common_utils/test_custom_openapi_spec.py index 485dbcef2f6..41ab613e7ea 100644 --- a/tests/unit/proxy/common_utils/test_custom_openapi_spec.py +++ b/tests/unit/proxy/common_utils/test_custom_openapi_spec.py @@ -4,11 +4,14 @@ Simple unit tests for CustomOpenAPISpec class. Tests basic functionality of OpenAPI schema generation. """ -from unittest.mock import Mock, patch +import json +from typing import Annotated, Literal +from unittest.mock import patch import pytest +from pydantic import BaseModel, ConfigDict, Field -from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec +from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec, inline_request_body class TestCustomOpenAPISpec: @@ -27,19 +30,13 @@ class TestCustomOpenAPISpec: }, } - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) - def test_add_chat_completion_request_schema( - self, mock_add_schema, base_openapi_schema - ): + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") + def test_add_chat_completion_request_schema(self, mock_add_schema, base_openapi_schema): """Test that chat completion schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema with patch("litellm.proxy._types.ProxyChatCompletionRequest") as mock_model: - result = CustomOpenAPISpec.add_chat_completion_request_schema( - base_openapi_schema - ) + result = CustomOpenAPISpec.add_chat_completion_request_schema(base_openapi_schema) mock_add_schema.assert_called_once_with( openapi_schema=base_openapi_schema, @@ -50,9 +47,7 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") def test_add_embedding_request_schema(self, mock_add_schema, base_openapi_schema): """Test that embedding schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema @@ -69,19 +64,13 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) - def test_add_responses_api_request_schema( - self, mock_add_schema, base_openapi_schema - ): + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") + def test_add_responses_api_request_schema(self, mock_add_schema, base_openapi_schema): """Test that responses API schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema with patch("litellm.types.llms.openai.ResponsesAPIRequestParams") as mock_model: - result = CustomOpenAPISpec.add_responses_api_request_schema( - base_openapi_schema - ) + result = CustomOpenAPISpec.add_responses_api_request_schema(base_openapi_schema) mock_add_schema.assert_called_once_with( openapi_schema=base_openapi_schema, @@ -123,15 +112,11 @@ def test_defs_rewritten_in_add_schema_to_components(): ) assert "$defs" not in openapi_schema assert ( - openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][ - "items" - ]["anyOf"][0]["$ref"] + openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" ) assert ( - openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][ - "items" - ]["anyOf"][1]["$ref"] + openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" ) @@ -188,14 +173,8 @@ def test_rewrite_defs_refs(): rewritten = CustomOpenAPISpec._rewrite_defs_refs(schema=schema, renames={}) assert "$defs" not in rewritten - assert ( - rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] - == "#/components/schemas/UserMessage" - ) - assert ( - rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] - == "#/components/schemas/AssistantMessage" - ) + assert rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" + assert rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" def test_get_pydantic_schema_generates_schema_for_responses_request_typed_dict(): @@ -390,3 +369,42 @@ def test_add_schema_to_components_renames_def_with_different_required_set(): assert schemas["Block"]["required"] == ["type", "x", "y"] assert schemas["Req_Block"]["required"] == ["keys", "type", "x", "y"] assert schemas["Req"]["properties"]["b"]["$ref"] == "#/components/schemas/Req_Block" + + +class _Cat(BaseModel): + kind: Literal["cat"] + lives: int = 9 + + model_config = ConfigDict(frozen=True) + + +class _Dog(BaseModel): + kind: Literal["dog"] + good: bool = True + + model_config = ConfigDict(frozen=True) + + +class _Shelter(BaseModel): + pets: dict[str, Annotated[_Cat | _Dog, Field(discriminator="kind")]] + favorite: _Cat | None = None + + model_config = ConfigDict(frozen=True) + + +def test_inline_request_body_expands_every_ref_so_the_operation_stands_alone(): + body = inline_request_body(_Shelter, {"pets": {"tom": {"kind": "cat"}}}) + + media = body["content"]["application/json"] + assert body["required"] is True + assert media["example"] == {"pets": {"tom": {"kind": "cat"}}} + assert "$ref" not in json.dumps(media["schema"]) + assert "$defs" not in media["schema"] + pet_schema = media["schema"]["properties"]["pets"]["additionalProperties"] + variants = {variant["properties"]["kind"]["const"]: variant for variant in pet_schema["oneOf"]} + assert variants["cat"]["properties"]["lives"]["default"] == 9 + assert variants["dog"]["properties"]["good"]["type"] == "boolean" + assert pet_schema["discriminator"] == {"propertyName": "kind"} + favorite = media["schema"]["properties"]["favorite"] + assert favorite["default"] is None + assert any(option.get("properties", {}).get("lives") for option in favorite["anyOf"]) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index ccfa12e10d6..e92ebc0eec9 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -12,17 +12,19 @@ import respx from fastapi import FastAPI from fastapi.routing import APIRoute from fastapi.testclient import TestClient +from pydantic import TypeAdapter from starlette.routing import Match import litellm from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features -from litellm.proxy.decisions_endpoints.endpoints import decisions, systemone +from litellm.proxy.decisions_endpoints.endpoints import decisions, router, systemone from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder from litellm.proxy.proxy_server import ( app, cleanup_router_config_variables, initialize, ) +from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody _INPUT_TOKENS: Final[int] = 367 _OUTPUT_TOKENS: Final[int] = 3 @@ -530,6 +532,68 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_ assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) +@pytest.mark.parametrize("safety_identifier", (7, ["end-user-1"]), ids=("numeric", "list")) +@pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params")) +def test_a_non_string_safety_identifier_is_refused_unless_the_deployment_drops_params( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + safety_identifier: object, + deployment_drops_params: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "decider", + "litellm_params": { + "model": "openai/gpt-6-luna", + "api_key": "k", + "drop_params": deployment_drops_params, + }, + } + ] + ), + ) + upstream: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond( + json={ + "model": "gpt-6-luna", + "answers": _OPENAI_FORMAT_ANSWERS[:1], + "usage": { + "input_tokens": _INPUT_TOKENS, + "output_tokens": _OUTPUT_TOKENS, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + }, + } + ) + request_body: Final = { + "model": "decider", + "input": "The package arrived with a broken screen.", + "questions": _OPENAI_FORMAT_REQUEST["questions"][:1], + "safety_identifier": safety_identifier, + } + + response: Final = client.post("/v1/decisions", json=request_body) + + if not deployment_drops_params: + assert response.status_code == 400, response.text + assert "safety_identifier" in response.json()["error"]["message"] + assert not upstream.called + return + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "The package arrived with a broken screen.", + "questions": _OPENAI_FORMAT_REQUEST["questions"][:1], + } + + def _decisions_feature() -> LazyFeature: return next(feature for feature in LAZY_FEATURES if feature.name == "decisions") @@ -581,3 +645,63 @@ def test_with_lazy_routes_disabled_a_config_pass_through_at_v1_decisions_still_w assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} assert _serving_endpoint(bare, "/v1/decisions") is pass_through assert _serving_endpoint(bare, "/decisions") is decisions + + +@pytest.fixture(scope="module") +def decisions_openapi() -> dict[str, object]: + app = FastAPI() + app.include_router(router) + return app.openapi() + + +@pytest.mark.parametrize( + ("path", "body_adapter", "response_schema", "required"), + ( + ("/v1/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}), + ("/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}), + ( + "/v1/decisions", + TypeAdapter(OpenAIDecisionRequestBody), + "OpenAIDecisionResponse", + {"model", "input", "questions"}, + ), + ( + "/decisions", + TypeAdapter(OpenAIDecisionRequestBody), + "OpenAIDecisionResponse", + {"model", "input", "questions"}, + ), + ), +) +def test_swagger_documents_the_request_format_and_response_of_each_decisions_route( + decisions_openapi: dict[str, object], + path: str, + body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody], + response_schema: str, + required: set[str], +) -> None: + operation = decisions_openapi["paths"][path]["post"] + media = operation["requestBody"]["content"]["application/json"] + + assert "Decisions" in operation["summary"] or "System One" in operation["summary"] + assert "docs.litellm.ai/docs/decisions" in operation["description"] + assert required <= set(media["schema"]["required"]) + assert required <= set(media["schema"]["properties"]) + assert "$ref" not in json.dumps(media["schema"]) + assert media["schema"]["properties"]["model"]["description"] + + example = {key: value for key, value in media["example"].items() if key != "model"} + assert len(body_adapter.validate_python(example).questions) == 3 + + response_ref = operation["responses"]["200"]["content"]["application/json"]["schema"]["$ref"] + assert response_ref == f"#/components/schemas/{response_schema}" + assert "answers" in decisions_openapi["components"]["schemas"][response_schema]["properties"] + + +def test_the_two_decisions_routes_document_different_request_formats(decisions_openapi: dict[str, object]) -> None: + def properties(path: str) -> set[str]: + operation = decisions_openapi["paths"][path]["post"] + return set(operation["requestBody"]["content"]["application/json"]["schema"]["properties"]) + + assert "state" in properties("/v1/systemone") and "state" not in properties("/v1/decisions") + assert "input" in properties("/v1/decisions") and "input" not in properties("/v1/systemone") diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_decision_model.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_decision_model.py index 1c2b297f238..54a73690840 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_decision_model.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_decision_model.py @@ -414,7 +414,7 @@ def test_initialize_rejects_invalid_streaming_scan_settings(mock_mgr, overrides, def test_add_guardrail_settings_returns_decisions_providers(): import asyncio - from litellm.decisions.main import supported_decisions_providers + from litellm.decisions.call import supported_decisions_providers from litellm.proxy.guardrails.guardrail_endpoints import get_guardrail_ui_settings settings: Final = asyncio.run(get_guardrail_ui_settings()) diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index e0a5ef063e8..b76cb17e5bd 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Final from unittest.mock import MagicMock import httpx @@ -12,7 +13,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndp @pytest.fixture(autouse=True) -def local_model_cost_map(monkeypatch: pytest.MonkeyPatch): +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) @@ -137,14 +138,17 @@ def test_success_handler_dispatches_to_typesafe_handler(): assert normalized["kwargs"]["model"] == "typesafe/jev-1.13.0" -def test_openrouter_decisions_response_is_priced_from_request_model_registry_row(): +def test_openrouter_decisions_response_uses_provider_cost_without_cost_map_row( + monkeypatch: pytest.MonkeyPatch, +): logging_obj = _logging_obj() - model_cost = litellm.model_cost["openrouter/typesafe/jev-1.13"] + cost: Final = 1.5834e-5 + monkeypatch.delitem(litellm.model_cost, "openrouter/typesafe/jev-1.13", raising=False) response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( httpx_response=_response(), response_body={ "model": "typesafe/jev-1.13-20260917", - "usage": {"input_tokens": 282, "output_tokens": 20}, + "usage": {"input_tokens": 282, "output_tokens": 20, "cost": cost}, }, logging_obj=logging_obj, url_route="https://openrouter.ai/api/alpha/decisions", @@ -156,10 +160,9 @@ def test_openrouter_decisions_response_is_priced_from_request_model_registry_row custom_llm_provider="openrouter", ) - expected_cost = 282 * model_cost["input_cost_per_token"] + 20 * model_cost["output_cost_per_token"] assert response["kwargs"]["model"] == "openrouter/typesafe/jev-1.13-20260917" assert response["kwargs"]["custom_llm_provider"] == "openrouter" - assert response["kwargs"]["response_cost"] == pytest.approx(expected_cost) + assert response["kwargs"]["response_cost"] == cost assert response["kwargs"]["combined_usage_object"].prompt_tokens == 282 assert response["kwargs"]["combined_usage_object"].completion_tokens == 20 assert response["kwargs"]["combined_usage_object"].total_tokens == 302 diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 85b667e1be3..90db461efd3 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -9,6 +9,7 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from pydantic import TypeAdapter from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.public_endpoints import router @@ -16,6 +17,7 @@ from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_pres from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, ) +from litellm.types.proxy.public_endpoints.public_endpoints import ProviderCreateInfo from litellm.types.utils import LlmProviders @@ -377,6 +379,53 @@ def test_chatgpt_provider_fields(): assert chatgpt["credential_fields"] == [] +def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: + app_instance: Final = FastAPI() + app_instance.include_router(router) + test_client: Final = TestClient(app_instance) + + response: Final = test_client.get("/public/providers/fields") + assert response.status_code == 200 + providers: Final = TypeAdapter(list[ProviderCreateInfo]).validate_python(response.json()) + entry: Final = next((p for p in providers if p.provider == provider), None) + assert entry is not None, f"{provider} provider entry not found" + return entry + + +def test_typesafe_provider_fields(): + typesafe: Final = _decisions_provider_entry("TypeSafe") + + assert typesafe.provider_display_name == "TypeSafe" + assert typesafe.litellm_provider == LlmProviders.TYPESAFE.value + assert typesafe.default_model_placeholder is not None + assert typesafe.default_model_placeholder.startswith("typesafe/") + + fields_by_key: Final = {f.key: f for f in typesafe.credential_fields} + + assert fields_by_key["api_key"].required is True + assert fields_by_key["api_key"].field_type == "password" + + assert fields_by_key["api_base"].required is False + assert fields_by_key["api_base"].field_type == "text" + + +def test_strands_decider_provider_fields(): + strands: Final = _decisions_provider_entry("StrandsDecider") + + assert strands.provider_display_name == "Strands Decider" + assert strands.litellm_provider == LlmProviders.STRANDS_DECIDER.value + assert strands.default_model_placeholder is not None + assert strands.default_model_placeholder.startswith("strands_decider/") + + fields_by_key: Final = {f.key: f for f in strands.credential_fields} + + assert fields_by_key["api_base"].required is True + assert fields_by_key["api_base"].field_type == "text" + + assert fields_by_key["api_key"].required is False + assert fields_by_key["api_key"].field_type == "password" + + ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( { "a2a", @@ -411,13 +460,11 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( "sagemaker_nova", "scaleway", "stability", - "strands_decider", "synthetic", "tencent", "tensormesh", "text-completion-inception", "transcribe", - "typesafe", "valkey", "xiaomi_mimo", "zai", diff --git a/tests/unit/systemone/__init__.py b/tests/unit/systemone/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/systemone/test_main.py b/tests/unit/systemone/test_main.py new file mode 100644 index 00000000000..342b7f1ef97 --- /dev/null +++ b/tests/unit/systemone/test_main.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import pytest +import respx +from respx.models import Call + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse + +_STATE: Final = "Customer asked for a refund" +_SYSTEMONE_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( + {"is_refund": {"type": "noul", "instructions": "Is this a refund request?"}} +) +_OPENAI_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_refund", "instructions": "Is this a refund request?"}, +) +_SYSTEMONE_RESPONSE: Final[Mapping[str, object]] = { + "model": "jev-1.13", + "answers": {"is_refund": {"type": "noul", "noul": 0.93}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, +} +_TYPESAFE_URL: Final = "https://api.typesafe.ai/v1/systemone" + + +@pytest.fixture(autouse=True) +def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +class _CallTypeRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.call_types: list[str] = [] # mutable-ok: test recorder collects logger callbacks + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.call_types.append(kwargs["call_type"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", (True, False), ids=("asystemone", "systemone")) +async def test_systemone_sends_the_systemone_wire_format_and_returns_a_systemone_response( + use_async: bool, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + kwargs: Final[Mapping[str, object]] = MappingProxyType( + {"model": "typesafe/jev-1.13", "state": _STATE, "questions": _SYSTEMONE_QUESTIONS, "api_key": "caller-key"} + ) + + response: Final = await litellm.asystemone(**kwargs) if use_async else litellm.systemone(**kwargs) + + assert route.called + call: Final[Call] = respx_mock.calls.last + assert json.loads(call.request.content) == { + "model": "jev-1.13", + "state": _STATE, + "questions": {"is_refund": {"type": "noul", "instructions": "Is this a refund request?"}}, + } + assert isinstance(response, DecisionsResponse) + assert response.answers["is_refund"].model_dump(mode="json") == {"type": "noul", "noul": 0.93} + + +@pytest.mark.asyncio +async def test_systemone_rejects_openai_input_and_points_at_decisions(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match=r"litellm\.decisions") as caught: + await litellm.asystemone( + model="typesafe/jev-1.13", input=_STATE, questions=_SYSTEMONE_QUESTIONS, api_key="caller-key" + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", (True, False), ids=("adecisions", "decisions")) +async def test_decisions_with_state_still_sends_the_systemone_format_and_warns( + use_async: bool, respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + kwargs: Final[Mapping[str, object]] = MappingProxyType( + {"model": "typesafe/jev-1.13", "state": _STATE, "questions": _SYSTEMONE_QUESTIONS, "api_key": "caller-key"} + ) + + with pytest.warns(UserWarning, match=r"litellm\.systemone"): + response: Final = await litellm.adecisions(**kwargs) if use_async else litellm.decisions(**kwargs) + + assert route.called + call: Final[Call] = respx_mock.calls.last + assert json.loads(call.request.content) == { + "model": "jev-1.13", + "state": _STATE, + "questions": {"is_refund": {"type": "noul", "instructions": "Is this a refund request?"}}, + } + assert isinstance(response, DecisionsResponse) + assert response.answers["is_refund"].model_dump(mode="json") == {"type": "noul", "noul": 0.93} + + +@pytest.mark.asyncio +async def test_systemone_rejects_openai_question_lists_before_http(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + + with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): + await litellm.asystemone( + model="typesafe/jev-1.13", state=_STATE, questions=_OPENAI_QUESTIONS, api_key="caller-key" + ) + + assert not route.called + + +@pytest.mark.asyncio +async def test_router_asystemone_logs_the_systemone_call_type( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE) + recorder: Final = _CallTypeRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + router: Final = litellm.Router( + model_list=[{"model_name": "jev", "litellm_params": {"model": "typesafe/jev-1.13", "api_key": "caller-key"}}] + ) + + response: Final = await router.asystemone(model="jev", state=_STATE, questions=_SYSTEMONE_QUESTIONS) + await asyncio.sleep(0) + GLOBAL_LOGGING_WORKER.start() + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + assert isinstance(response, DecisionsResponse) + assert recorder.call_types == ["asystemone"] + assert not isinstance(response, OpenAIDecisionResponse) diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index 1595bd9864f..e2c54d381d8 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -2,6 +2,7 @@ import importlib.util import json import subprocess import sys +from collections.abc import Mapping from pathlib import Path from types import ModuleType from typing import Final @@ -62,8 +63,19 @@ def _snapshot(cost_map: dict[str, object], backup: str | None = None, schema: st BASE: Final = _snapshot(BASE_MAP) -def _failures(head: object, changed_files: tuple[str, ...] = MAP_FILES, bot: bool = True) -> tuple[str, ...]: - return guard.guard_failures(BASE, head, changed_files, bot) +def _failures( + head: object, + changed_files: tuple[str, ...] = MAP_FILES, + bot: bool = True, + evaluation_output_price_sources: Mapping[str, str] = guard.EVALUATION_OUTPUT_PRICE_SOURCES, +) -> tuple[str, ...]: + return guard.guard_failures( + BASE, + head, + changed_files, + bot, + evaluation_output_price_sources=evaluation_output_price_sources, + ) def test_in_sync_files_pass_for_humans_and_bots() -> None: @@ -71,6 +83,52 @@ def test_in_sync_files_pass_for_humans_and_bots() -> None: assert _failures(BASE, bot=True) == () +EVALUATION_MODEL: Final = "azure_ai/Microsoft-Decision-1" +EVALUATION_OUTPUT_PRICE_FAILURE: Final = ( + f"{EVALUATION_MODEL}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor " + "prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)" +) + + +@pytest.mark.parametrize("bot", (True, False)) +def test_evaluation_rows_with_output_price_fail_for_humans_and_bots(bot: bool) -> None: + head: Final = _snapshot( + {**BASE_MAP, EVALUATION_MODEL: _entry(mode="evaluation", output_cost_per_token=4.2e-08)} + ) + assert _failures(head, bot=bot) == (EVALUATION_OUTPUT_PRICE_FAILURE,) + + +def test_evaluation_rows_with_zero_or_missing_output_price_pass() -> None: + zero_output: Final = _snapshot( + {**BASE_MAP, EVALUATION_MODEL: _entry(mode="evaluation", output_cost_per_token=0.0)} + ) + missing_output_entry: Final = { + key: value + for key, value in _entry(mode="evaluation").items() + if key != "output_cost_per_token" + } + missing_output: Final = _snapshot({**BASE_MAP, EVALUATION_MODEL: missing_output_entry}) + + assert _failures(zero_output) == () + assert _failures(missing_output) == () + + +def test_chat_rows_with_output_price_pass() -> None: + head: Final = _snapshot( + {**BASE_MAP, "openrouter/chat-with-output": _entry(output_cost_per_token=4.2e-08)} + ) + assert _failures(head) == () + + +def test_evaluation_output_price_source_allowlist_passes() -> None: + head: Final = _snapshot( + {**BASE_MAP, EVALUATION_MODEL: _entry(mode="evaluation", output_cost_per_token=4.2e-08)} + ) + source: Final = 'https://vendor.example/pricing "Output tokens are charged per token."' + + assert _failures(head, evaluation_output_price_sources={EVALUATION_MODEL: source}) == () + + def test_bot_may_add_and_reprice_models() -> None: head = _snapshot({**BASE_MAP, "openrouter/a": _entry(9e-06, supports_vision=True), "openrouter/c": _entry()}) assert _failures(head) == () diff --git a/ui/litellm-dashboard/public/assets/logos/strands.svg b/ui/litellm-dashboard/public/assets/logos/strands.svg new file mode 100644 index 00000000000..466fb64465e --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/strands.svg @@ -0,0 +1,4 @@ + + + + diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 29ce2d4865a..3337a7f9ffd 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -209,6 +209,26 @@ describe("provider_info_helpers", () => { const { logo } = getProviderLogoAndName("openai"); expect(logo).toContain("openai_small"); }); + + it("should resolve the typesafe slug and TypeSafe enum key to the TypeSafe name and bundled logo", () => { + const fromSlug = getProviderLogoAndName("typesafe"); + expect(fromSlug.displayName).toBe(Providers.TypeSafe); + expect(fromSlug.logo).toContain("typesafe"); + + const fromEnumKey = getProviderLogoAndName("TypeSafe"); + expect(fromEnumKey.displayName).toBe(Providers.TypeSafe); + expect(fromEnumKey.logo).toBe(fromSlug.logo); + }); + + it("should resolve the strands_decider slug and StrandsDecider enum key to the Strands Decider name and bundled logo", () => { + const fromSlug = getProviderLogoAndName("strands_decider"); + expect(fromSlug.displayName).toBe("Strands Decider"); + expect(fromSlug.logo).toContain("strands"); + + const fromEnumKey = getProviderLogoAndName("StrandsDecider"); + expect(fromEnumKey.displayName).toBe(Providers.StrandsDecider); + expect(fromEnumKey.logo).toBe(fromSlug.logo); + }); }); describe("getPlaceholder", () => { @@ -309,6 +329,11 @@ describe("provider_info_helpers", () => { expect(getPlaceholder("CHATGPT")).toBe("chatgpt/gpt-5.4"); }); + it("should return decision model placeholders for the TypeSafe and StrandsDecider dropdown keys", () => { + expect(getPlaceholder("TypeSafe")).toBe("typesafe/jev-latest"); + expect(getPlaceholder("StrandsDecider")).toBe("strands_decider/strands-decider-2B-hobson-v19"); + }); + it("should return default gpt-3.5-turbo placeholder for unknown provider", () => { expect(getPlaceholder("UnknownProvider" as any)).toBe("gpt-3.5-turbo"); }); @@ -412,6 +437,30 @@ describe("provider_info_helpers", () => { expect(getProviderModels("Sail" as Providers, modelMap)).toEqual(["sail/openai/gpt-oss-120b"]); }); + it("should list only typesafe decision models for the 'TypeSafe' provider key, not the OpenRouter-hosted one", () => { + const modelMap = { + "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, + "typesafe/jev-preview": { litellm_provider: "typesafe", mode: "evaluation" }, + "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" }, + "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" }, + }; + expect(getProviderModels("TypeSafe" as Providers, modelMap)).toEqual([ + "typesafe/jev-latest", + "typesafe/jev-preview", + ]); + }); + + it("should list only strands_decider models for the 'StrandsDecider' provider key", () => { + const modelMap = { + "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" }, + "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, + "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" }, + }; + expect(getProviderModels("StrandsDecider" as Providers, modelMap)).toEqual([ + "strands_decider/strands-decider-2B-hobson-v19", + ]); + }); + it("should include bedrock converse but exclude standalone bedrock_mantle when called with 'Bedrock' provider key", () => { const modelMap = { "bedrock-base": { litellm_provider: "bedrock" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index b0e33338bab..a88615df185 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -55,8 +55,10 @@ import sapLogo from "../../public/assets/logos/sap.png"; import scxAiLogo from "../../public/assets/logos/scx_ai.svg"; import snowflakeLogo from "../../public/assets/logos/snowflake.svg"; import sonioxLogo from "../../public/assets/logos/soniox.svg"; +import strandsLogo from "../../public/assets/logos/strands.svg"; import togetheraiLogo from "../../public/assets/logos/togetherai.svg"; import topazLogo from "../../public/assets/logos/topaz.svg"; +import typesafeLogo from "../../public/assets/logos/typesafe.png"; import v0Logo from "../../public/assets/logos/v0.svg"; import vercelLogo from "../../public/assets/logos/vercel.svg"; import vllmLogo from "../../public/assets/logos/vllm.png"; @@ -166,10 +168,12 @@ export enum Providers { SCX_AI = "SCX.ai", Snowflake = "Snowflake", Soniox = "Soniox", + StrandsDecider = "Strands Decider", TEXT_COMPLETION_CODESTRAL = "Text-Completion-Codestral", TogetherAI = "TogetherAI", TOPAZ = "Topaz", Triton = "Triton", + TypeSafe = "TypeSafe", V0 = "V0", VERCEL_AI_GATEWAY = "Vercel Ai Gateway", Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)", @@ -285,10 +289,12 @@ export const provider_map: Record = { SCX_AI: "scx-ai", Snowflake: "snowflake", Soniox: "soniox", + StrandsDecider: "strands_decider", TEXT_COMPLETION_CODESTRAL: "text-completion-codestral", TogetherAI: "together_ai", TOPAZ: "topaz", Triton: "triton", + TypeSafe: "typesafe", V0: "v0", VERCEL_AI_GATEWAY: "vercel_ai_gateway", Vertex_AI: "vertex_ai", @@ -384,10 +390,12 @@ export const providerLogoMap: Partial> = { [Providers.SCX_AI]: scxAiLogo.src, [Providers.Snowflake]: snowflakeLogo.src, [Providers.Soniox]: sonioxLogo.src, + [Providers.StrandsDecider]: strandsLogo.src, [Providers.TEXT_COMPLETION_CODESTRAL]: mistralLogo.src, [Providers.TogetherAI]: togetheraiLogo.src, [Providers.TOPAZ]: topazLogo.src, [Providers.Triton]: nvidiaTritonLogo.src, + [Providers.TypeSafe]: typesafeLogo.src, [Providers.V0]: v0Logo.src, [Providers.VERCEL_AI_GATEWAY]: vercelLogo.src, [Providers.Vertex_AI]: googleLogo.src, @@ -453,6 +461,8 @@ const providerPlaceholderMap: Partial> = { [Providers.Sail]: "sail/openai/gpt-oss-120b", [Providers.SCX_AI]: "scx-ai/GLM-5.2", [Providers.Snowflake]: "snowflake/mistral-7b", + [Providers.StrandsDecider]: "strands_decider/strands-decider-2B-hobson-v19", + [Providers.TypeSafe]: "typesafe/jev-latest", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", [Providers.Voyage]: "voyage/", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5aaf42f30d4..6fef4a1e03e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1189,60 +1189,6 @@ export interface paths { patch?: never; trace?: never; }; - "/audit": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get Audit Logs - * @description Get all audit logs with filtering and pagination. - * - * Returns a paginated response of audit logs matching the specified filters. - * - * Note: object_team_id and object_key_hash use Prisma JSON path filtering, - * which requires PostgreSQL. - */ - get: operations["get_audit_logs_audit_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/audit/{id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get Audit Log By Id - * @description Get detailed information about a specific audit log entry by its ID. - * - * Args: - * id (str): The unique identifier of the audit log entry - * - * Returns: - * AuditLogResponse: Detailed information about the audit log entry - * - * Raises: - * HTTPException: If the audit log is not found or if there's a database connection error - */ - get: operations["get_audit_log_by_id_audit__id__get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/authorize": { parameters: { query?: never; @@ -4577,7 +4523,18 @@ export interface paths { }; get?: never; put?: never; - /** Decisions */ + /** + * Ask questions about an input (OpenAI Decisions format) + * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * + * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + * `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + * rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + * model list. System One decision models accept this format too, LiteLLM translates the request and the + * answers. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["decisions_decisions_post"]; delete?: never; options?: never; @@ -4602,50 +4559,6 @@ export interface paths { patch?: never; trace?: never; }; - "/email/event_settings": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get Email Event Settings - * @description Get all email event settings - */ - get: operations["get_email_event_settings_email_event_settings_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - /** - * Update Event Settings - * @description Update the settings for email events - */ - patch: operations["update_event_settings_email_event_settings_patch"]; - trace?: never; - }; - "/email/event_settings/reset": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** - * Reset Event Settings - * @description Reset all email event settings to default (new user invitations on, virtual key creation off) - */ - post: operations["reset_event_settings_email_event_settings_reset_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/embeddings": { parameters: { query?: never; @@ -12729,240 +12642,6 @@ export interface paths { patch?: never; trace?: never; }; - "/project/delete": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - post?: never; - /** - * Delete Project - * @description Delete projects - * - * Parameters: - * - project_ids: *List[str]* - List of project ids to delete - * - * Example: - * ```bash - * curl --location --request DELETE 'http://0.0.0.0:4000/project/delete' \ - * --header 'Authorization: Bearer sk-1234' \ - * --header 'Content-Type: application/json' \ - * --data '{ - * "project_ids": ["project-123", "project-456"] - * }' - * ``` - */ - delete: operations["delete_project_project_delete_delete"]; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/project/info": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Project Info - * @description Get information about a specific project - * - * Parameters: - * - project_id: *str* - The project id to fetch info for - * - * Example: - * ```bash - * curl --location 'http://0.0.0.0:4000/project/info?project_id=project-123' \ - * --header 'Authorization: Bearer sk-1234' - * ``` - */ - get: operations["project_info_project_info_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/project/list": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * List Projects - * @description List all projects that the user has access to - * - * Example: - * ```bash - * curl --location 'http://0.0.0.0:4000/project/list' \ - * --header 'Authorization: Bearer sk-1234' - * ``` - */ - get: operations["list_projects_project_list_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/project/new": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** - * New Project - * @description Create a new project. Projects sit between teams and keys in the hierarchy. - * - * Only admins or team admins can create projects. - * - * # Parameters - * - * - project_alias: *Optional[str]* - The name of the project. - * - description: *Optional[str]* - Description of the project's purpose and use case. - * - team_id: *str* - The team id that this project belongs to. Required. - * - models: *List* - The models the project has access to. - * - budget_id: *Optional[str]* - The id for a budget (tpm/rpm/max budget) for the project. - * ### IF NO BUDGET ID - CREATE ONE WITH THESE PARAMS ### - * - max_budget: *Optional[float]* - Max budget for project - * - tpm_limit: *Optional[int]* - Max tpm limit for project - * - rpm_limit: *Optional[int]* - Max rpm limit for project - * - max_parallel_requests: *Optional[int]* - Max parallel requests for project - * - soft_budget: *Optional[float]* - Get a slack alert when this soft budget is reached. Don't block requests. - * - model_max_budget: *Optional[dict]* - Max budget for a specific model. Example: {"gpt-4": 100.0, "gpt-3.5-turbo": 50.0} - * - model_rpm_limit: *Optional[dict]* - RPM limits per model. Example: {"gpt-4": 1000, "gpt-3.5-turbo": 5000} - * - model_tpm_limit: *Optional[dict]* - TPM limits per model. Example: {"gpt-4": 50000, "gpt-3.5-turbo": 100000} - * - budget_duration: *Optional[str]* - Frequency of reseting project budget - * - metadata: *Optional[dict]* - Metadata for project, store information for project. Example metadata - {"use_case_id": "SNOW-12345", "responsible_ai_id": "RAI-67890"} - * - tags: *Optional[list]* - Tags for the project. Example: ["production", "api"] - * - blocked: *bool* - Flag indicating if the project is blocked or not - will stop all calls from keys with this project_id. - * - object_permission: Optional[LiteLLM_ObjectPermissionBase] - project-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"]}. IF null or {} then no object permission. - * - * Example 1: Create new project **without** a budget_id, with model-specific limits - * - * ```bash - * curl --location 'http://0.0.0.0:4000/project/new' \ - * --header 'Authorization: Bearer sk-1234' \ - * --header 'Content-Type: application/json' \ - * --data '{ - * "project_alias": "flight-search-assistant", - * "description": "AI-powered flight search and booking assistant", - * "team_id": "team-123", - * "models": ["gpt-4", "gpt-3.5-turbo"], - * "max_budget": 100, - * "model_rpm_limit": { - * "gpt-4": 1000, - * "gpt-3.5-turbo": 5000 - * }, - * "model_tpm_limit": { - * "gpt-4": 50000, - * "gpt-3.5-turbo": 100000 - * }, - * "metadata": { - * "use_case_id": "SNOW-12345", - * "responsible_ai_id": "RAI-67890" - * } - * }' - * ``` - * - * Example 2: Create new project **with** a budget_id - * - * ```bash - * curl --location 'http://0.0.0.0:4000/project/new' \ - * --header 'Authorization: Bearer sk-1234' \ - * --header 'Content-Type: application/json' \ - * --data '{ - * "project_alias": "hotel-recommendations", - * "description": "Personalized hotel recommendation engine", - * "team_id": "team-123", - * "models": ["claude-3-sonnet"], - * "budget_id": "428eeaa8-f3ac-4e85-a8fb-7dc8d7aa8689", - * "metadata": { - * "use_case_id": "SNOW-54321" - * } - * }' - * ``` - */ - post: operations["new_project_project_new_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/project/update": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** - * Update Project - * @description Update a project - * - * Parameters: - * - project_id: *str* - The project id to update. Required. - * - project_alias: *Optional[str]* - Updated name for the project - * - description: *Optional[str]* - Updated description for the project - * - team_id: *Optional[str]* - Updated team_id for the project - * - metadata: *Optional[dict]* - Updated metadata for project - * - models: *Optional[list]* - Updated list of models for the project - * - blocked: *Optional[bool]* - Updated blocked status - * - max_budget: *Optional[float]* - Updated max budget - * - tpm_limit: *Optional[int]* - Updated tpm limit - * - rpm_limit: *Optional[int]* - Updated rpm limit - * - model_rpm_limit: *Optional[dict]* - Updated RPM limits per model - * - model_tpm_limit: *Optional[dict]* - Updated TPM limits per model - * - budget_duration: *Optional[str]* - Updated budget duration - * - tags: *Optional[list]* - Updated list of tags for the project - * - object_permission: Optional[LiteLLM_ObjectPermissionBase] - Updated object permission - * - * Example: - * ```bash - * curl --location 'http://0.0.0.0:4000/project/update' \ - * --header 'Authorization: Bearer sk-1234' \ - * --header 'Content-Type: application/json' \ - * --data '{ - * "project_id": "project-123", - * "description": "Updated flight search system with enhanced capabilities", - * "max_budget": 200, - * "model_rpm_limit": { - * "gpt-4": 2000, - * "gpt-3.5-turbo": 10000 - * }, - * "metadata": { - * "use_case_id": "SNOW-12345", - * "status": "active" - * } - * }' - * ``` - */ - post: operations["update_project_project_update_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/prompts": { parameters: { query?: never; @@ -14219,27 +13898,6 @@ export interface paths { patch?: never; trace?: never; }; - "/robots.txt": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get Robots - * @description Block all web crawlers from indexing the proxy server endpoints - * This is useful for ensuring that the API endpoints aren't indexed by search engines - */ - get: operations["get_robots_robots_txt_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/roi-calculator/connections/test": { parameters: { query?: never; @@ -16015,7 +15673,18 @@ export interface paths { }; get?: never; put?: never; - /** Systemone */ + /** + * Ask named questions about a state (System One format) + * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + * + * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + * `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + * `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + * is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + * too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["systemone_systemone_post"]; delete?: never; options?: never; @@ -18233,26 +17902,6 @@ export interface paths { patch?: never; trace?: never; }; - "/user/available_users": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Available Enterprise Users - * @description For keys with `max_users` set, return the list of users that are allowed to use the key. - */ - get: operations["available_enterprise_users_user_available_users_get"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/user/bulk_update": { parameters: { query?: never; @@ -19798,7 +19447,18 @@ export interface paths { }; get?: never; put?: never; - /** Decisions */ + /** + * Ask questions about an input (OpenAI Decisions format) + * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * + * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + * `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + * rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + * model list. System One decision models accept this format too, LiteLLM translates the request and the + * answers. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["decisions_v1_decisions_post"]; delete?: never; options?: never; @@ -22121,7 +21781,18 @@ export interface paths { }; get?: never; put?: never; - /** Systemone */ + /** + * Ask named questions about a state (System One format) + * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + * + * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + * `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + * `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + * is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + * too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["systemone_v1_systemone_post"]; delete?: never; options?: never; @@ -26147,37 +25818,6 @@ export interface components { */ unnamed_teams_count: number; }; - /** - * AuditLogResponse - * @description Response model for a single audit log entry - */ - AuditLogResponse: { - /** Action */ - action: string; - /** Before Value */ - before_value?: { - [key: string]: unknown; - } | null; - /** Changed By */ - changed_by: string; - /** Changed By Api Key */ - changed_by_api_key: string; - /** Id */ - id: string; - /** Object Id */ - object_id: string; - /** Table Name */ - table_name: string; - /** - * Updated At - * Format: date-time - */ - updated_at: string; - /** Updated Values */ - updated_values?: { - [key: string]: unknown; - } | null; - }; /** AutoRouterAllowance */ AutoRouterAllowance: { /** @@ -28100,7 +27740,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ @@ -28733,6 +28373,24 @@ export interface components { /** Instruction */ instruction: string; }; + /** ChoiceAnswer */ + ChoiceAnswer: { + /** Choice */ + choice: string; + /** Confidence */ + confidence: number; + /** Probabilities */ + probabilities: { + [key: string]: number; + }; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "choice"; + } & { + [key: string]: unknown; + }; /** ChoiceLogprobs */ ChoiceLogprobs: { /** Content */ @@ -31099,6 +30757,33 @@ export interface components { */ threshold: number; }; + /** DecisionsResponse */ + DecisionsResponse: { + /** Answers */ + answers: { + [key: string]: components["schemas"]["NoulAnswer"] | components["schemas"]["ChoiceAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + /** Model */ + model?: string | null; + usage?: components["schemas"]["DecisionsUsage"] | null; + } & { + [key: string]: unknown; + }; + /** DecisionsUsage */ + DecisionsUsage: { + /** + * Input Tokens + * @default 0 + */ + input_tokens: number; + /** + * Output Tokens + * @default 0 + */ + output_tokens: number; + } & { + [key: string]: unknown; + }; /** * DefaultInternalUserParams * @description Default parameters to apply when a new user signs in via SSO or is created on the /user/new API endpoint @@ -31246,14 +30931,6 @@ export interface components { /** Organization Ids */ organization_ids: string[]; }; - /** - * DeleteProjectRequest - * @description Request model for DELETE /project/delete - */ - DeleteProjectRequest: { - /** Project Ids */ - project_ids: string[]; - }; /** * DeleteResponseResult * @description Result of a delete response request @@ -31497,27 +31174,6 @@ export interface components { */ type?: "message"; }; - /** - * EmailEvent - * @enum {string} - */ - EmailEvent: "Virtual Key Created" | "New User Invitation" | "Virtual Key Rotated" | "Soft Budget Crossed" | "Max Budget Alert"; - /** EmailEventSettings */ - EmailEventSettings: { - /** Enabled */ - enabled: boolean; - event: components["schemas"]["EmailEvent"]; - }; - /** EmailEventSettingsResponse */ - EmailEventSettingsResponse: { - /** Settings */ - settings: components["schemas"]["EmailEventSettings"][]; - }; - /** EmailEventSettingsUpdateRequest */ - EmailEventSettingsUpdateRequest: { - /** Settings */ - settings: components["schemas"]["EmailEventSettings"][]; - }; /** EmbeddingRequest */ EmbeddingRequest: { /** @@ -35313,65 +34969,6 @@ export interface components { } & { [key: string]: unknown; }; - /** - * LiteLLM_ProjectTable - * @description Database model representation for project - */ - LiteLLM_ProjectTable: { - /** - * Blocked - * @default false - */ - blocked: boolean; - /** Budget Id */ - budget_id?: string | null; - /** Created At */ - created_at?: string | null; - /** Created By */ - created_by?: string | null; - /** Description */ - description?: string | null; - litellm_budget_table?: components["schemas"]["LiteLLM_BudgetTable"] | null; - /** Metadata */ - metadata?: { - [key: string]: unknown; - } | null; - /** Model Rpm Limit */ - model_rpm_limit?: { - [key: string]: unknown; - } | null; - /** Model Spend */ - model_spend?: { - [key: string]: unknown; - } | null; - /** Model Tpm Limit */ - model_tpm_limit?: { - [key: string]: unknown; - } | null; - /** - * Models - * @default [] - */ - models: string[]; - object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null; - /** Object Permission Id */ - object_permission_id?: string | null; - /** Project Alias */ - project_alias?: string | null; - /** Project Id */ - project_id: string; - /** - * Spend - * @default 0 - */ - spend: number; - /** Team Id */ - team_id?: string | null; - /** Updated At */ - updated_at?: string | null; - /** Updated By */ - updated_by?: string | null; - }; /** LiteLLM_ProxyModelTable */ LiteLLM_ProxyModelTable: { /** @@ -38615,148 +38212,6 @@ export interface components { /** Users */ users?: components["schemas"]["LiteLLM_UserTable"][] | null; }; - /** - * NewProjectRequest - * @description Request model for POST /project/new - */ - NewProjectRequest: { - /** Allowed Models */ - allowed_models?: string[] | null; - /** - * Blocked - * @default false - */ - blocked: boolean; - /** Budget Duration */ - budget_duration?: string | null; - /** Budget Id */ - budget_id?: string | null; - /** Description */ - description?: string | null; - /** Guardrails */ - guardrails?: string[] | null; - /** Max Budget */ - max_budget?: number | null; - /** Max Parallel Requests */ - max_parallel_requests?: number | null; - /** Metadata */ - metadata?: { - [key: string]: unknown; - } | null; - /** Model Itpm Limit */ - model_itpm_limit?: { - [key: string]: number; - } | null; - /** Model Max Budget */ - model_max_budget?: { - [key: string]: unknown; - } | null; - /** Model Otpm Limit */ - model_otpm_limit?: { - [key: string]: number; - } | null; - /** Model Rpm Limit */ - model_rpm_limit?: { - [key: string]: unknown; - } | null; - /** Model Tpm Limit */ - model_tpm_limit?: { - [key: string]: unknown; - } | null; - /** - * Models - * @default [] - */ - models: string[]; - object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null; - /** Policies */ - policies?: string[] | null; - /** Project Alias */ - project_alias?: string | null; - /** Project Id */ - project_id?: string | null; - /** Rpm Limit */ - rpm_limit?: number | null; - /** Soft Budget */ - soft_budget?: number | null; - /** Tags */ - tags?: string[] | null; - /** Team Id */ - team_id: string; - /** Temp Budget Expiry */ - temp_budget_expiry?: string | null; - /** Temp Budget Increase */ - temp_budget_increase?: number | null; - /** Tpd Limit */ - tpd_limit?: number | null; - /** Tpm Limit */ - tpm_limit?: number | null; - }; - /** - * NewProjectResponse - * @description Response model for POST /project/new - */ - NewProjectResponse: { - /** - * Blocked - * @default false - */ - blocked: boolean; - /** Budget Id */ - budget_id?: string | null; - /** - * Created At - * Format: date-time - */ - created_at: string; - /** Created By */ - created_by?: string | null; - /** Description */ - description?: string | null; - litellm_budget_table?: components["schemas"]["LiteLLM_BudgetTable"] | null; - /** Metadata */ - metadata?: { - [key: string]: unknown; - } | null; - /** Model Rpm Limit */ - model_rpm_limit?: { - [key: string]: unknown; - } | null; - /** Model Spend */ - model_spend?: { - [key: string]: unknown; - } | null; - /** Model Tpm Limit */ - model_tpm_limit?: { - [key: string]: unknown; - } | null; - /** - * Models - * @default [] - */ - models: string[]; - object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null; - /** Object Permission Id */ - object_permission_id?: string | null; - /** Project Alias */ - project_alias?: string | null; - /** Project Id */ - project_id: string; - /** - * Spend - * @default 0 - */ - spend: number; - /** Team Id */ - team_id?: string | null; - /** - * Updated At - * Format: date-time - */ - updated_at: string; - /** Updated By */ - updated_by?: string | null; - }; /** NewTeamRequest */ NewTeamRequest: { /** Access Group Ids */ @@ -39198,6 +38653,18 @@ export interface components { /** User Role */ user_role?: ("proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer") | null; }; + /** NoulAnswer */ + NoulAnswer: { + /** Noul */ + noul: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "noul"; + } & { + [key: string]: unknown; + }; /** * OAuth2SecurityScheme * @description Defines a security scheme using OAuth 2.0. @@ -39236,6 +38703,129 @@ export interface components { [key: string]: unknown; } | null; }; + /** OpenAIChoiceAnswer */ + OpenAIChoiceAnswer: { + /** Choice */ + choice: string | boolean; + /** Confidence */ + confidence: number; + /** Name */ + name: string | null; + /** Probabilities */ + probabilities: components["schemas"]["OpenAIChoiceProbability"][]; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "choice"; + }; + /** OpenAIChoiceProbability */ + OpenAIChoiceProbability: { + /** Probability */ + probability: number; + /** Value */ + value: string | boolean; + }; + /** OpenAIDecisionInputTokensDetails */ + OpenAIDecisionInputTokensDetails: { + /** + * Cache Write Tokens + * @default 0 + */ + cache_write_tokens: number; + /** + * Cached Tokens + * @default 0 + */ + cached_tokens: number; + }; + /** OpenAIDecisionOutputTokensDetails */ + OpenAIDecisionOutputTokensDetails: { + /** + * Reasoning Tokens + * @default 0 + */ + reasoning_tokens: number; + }; + /** OpenAIDecisionResponse */ + OpenAIDecisionResponse: { + /** Answers */ + answers: (components["schemas"]["OpenAIPredicateAnswer"] | components["schemas"]["OpenAIChoiceAnswer"] | components["schemas"]["OpenAIScoreAnswer"] | components["schemas"]["OpenAIRefusalAnswer"])[]; + /** Model */ + model: string; + usage: components["schemas"]["OpenAIDecisionUsage"]; + } & { + [key: string]: unknown; + }; + /** OpenAIDecisionUsage */ + OpenAIDecisionUsage: { + /** Input Tokens */ + input_tokens: number; + /** + * @default { + * "cache_write_tokens": 0, + * "cached_tokens": 0 + * } + */ + input_tokens_details: components["schemas"]["OpenAIDecisionInputTokensDetails"]; + /** Output Tokens */ + output_tokens: number; + /** + * @default { + * "reasoning_tokens": 0 + * } + */ + output_tokens_details: components["schemas"]["OpenAIDecisionOutputTokensDetails"]; + /** Total Tokens */ + total_tokens: number; + }; + /** OpenAIPredicateAnswer */ + OpenAIPredicateAnswer: { + /** Name */ + name: string | null; + /** Probability */ + probability: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "predicate"; + }; + /** OpenAIRefusalAnswer */ + OpenAIRefusalAnswer: { + /** Name */ + name: string | null; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "refusal"; + }; + /** OpenAIScoreAnswer */ + OpenAIScoreAnswer: { + /** Confidence */ + confidence: number; + /** Name */ + name: string | null; + /** Probabilities */ + probabilities: components["schemas"]["OpenAIScoreProbability"][]; + /** Score */ + score: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "score"; + }; + /** OpenAIScoreProbability */ + OpenAIScoreProbability: { + /** Label */ + label: string; + /** Probability */ + probability: number; + /** Value */ + value: number; + }; /** * OpenIdConnectSecurityScheme * @description Defines a security scheme using OpenID Connect. @@ -39632,34 +39222,6 @@ export interface components { /** Page Size */ page_size: number; }; - /** - * PaginatedAuditLogResponse - * @description Response model for paginated audit logs - */ - PaginatedAuditLogResponse: { - /** Audit Logs */ - audit_logs: components["schemas"]["AuditLogResponse"][]; - /** - * Page - * @description Current page number - */ - page: number; - /** - * Page Size - * @description Number of items per page - */ - page_size: number; - /** - * Total - * @description Total number of audit logs matching the filters - */ - total: number; - /** - * Total Pages - * @description Total number of pages - */ - total_pages: number; - }; /** PassThroughEndpointResponse */ PassThroughEndpointResponse: { /** Endpoints */ @@ -44570,6 +44132,30 @@ export interface components { */ team_id: string; }; + /** ScoreAnswer */ + ScoreAnswer: { + /** Confidence */ + confidence: number; + /** Legend */ + legend: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[]; + }; + /** Probabilities */ + probabilities: { + [key: string]: number; + }; + /** Score */ + score: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "score"; + } & { + [key: string]: unknown; + }; /** * Screenshot * @description A screenshot action. @@ -47636,77 +47222,6 @@ export interface components { */ version?: string | null; }; - /** - * UpdateProjectRequest - * @description Request model for POST /project/update - */ - UpdateProjectRequest: { - /** Allowed Models */ - allowed_models?: string[] | null; - /** Blocked */ - blocked?: boolean | null; - /** Budget Duration */ - budget_duration?: string | null; - /** Budget Id */ - budget_id?: string | null; - /** Description */ - description?: string | null; - /** Guardrails */ - guardrails?: string[] | null; - /** Max Budget */ - max_budget?: number | null; - /** Max Parallel Requests */ - max_parallel_requests?: number | null; - /** Metadata */ - metadata?: { - [key: string]: unknown; - } | null; - /** Model Itpm Limit */ - model_itpm_limit?: { - [key: string]: number; - } | null; - /** Model Max Budget */ - model_max_budget?: { - [key: string]: unknown; - } | null; - /** Model Otpm Limit */ - model_otpm_limit?: { - [key: string]: number; - } | null; - /** Model Rpm Limit */ - model_rpm_limit?: { - [key: string]: unknown; - } | null; - /** Model Tpm Limit */ - model_tpm_limit?: { - [key: string]: unknown; - } | null; - /** Models */ - models?: string[] | null; - object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null; - /** Policies */ - policies?: string[] | null; - /** Project Alias */ - project_alias?: string | null; - /** Project Id */ - project_id: string; - /** Rpm Limit */ - rpm_limit?: number | null; - /** Soft Budget */ - soft_budget?: number | null; - /** Tags */ - tags?: string[] | null; - /** Team Id */ - team_id?: string | null; - /** Temp Budget Expiry */ - temp_budget_expiry?: string | null; - /** Temp Budget Increase */ - temp_budget_increase?: number | null; - /** Tpd Limit */ - tpd_limit?: number | null; - /** Tpm Limit */ - tpm_limit?: number | null; - }; /** * UpdatePublicModelGroupsRequest * @description Request model for updating public model groups @@ -51563,107 +51078,6 @@ export interface operations { }; }; }; - get_audit_logs_audit_get: { - parameters: { - query?: { - page?: number; - page_size?: number; - /** @description Filter by user or system that performed the action */ - changed_by?: string | null; - /** @description Filter by API key hash that performed the action */ - changed_by_api_key?: string | null; - /** @description Filter by action type (create, update, delete) */ - action?: string | null; - /** @description Filter by table name that was modified */ - table_name?: string | null; - /** @description Filter by ID of the object that was modified */ - object_id?: string | null; - /** @description Filter logs after this date */ - start_date?: string | null; - /** @description Filter logs before this date */ - end_date?: string | null; - /** @description Filter by team_id present in before_value or updated_values JSON (PostgreSQL only) */ - object_team_id?: string | null; - /** @description Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only) */ - object_key_hash?: string | null; - /** @description Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value */ - search?: string | null; - /** @description Column to sort by (e.g. 'updated_at', 'action', 'table_name') */ - sort_by?: string | null; - /** @description Sort order ('asc' or 'desc') */ - sort_order?: string; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["PaginatedAuditLogResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - get_audit_log_by_id_audit__id__get: { - parameters: { - query?: never; - header?: never; - path: { - id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["AuditLogResponse"]; - }; - }; - /** @description Audit log not found */ - 404: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - /** @description Database connection error */ - 500: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - }; - }; authorize_authorize_get: { parameters: { query: { @@ -56567,7 +55981,179 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + * "model": "luna", + * "questions": [ + * { + * "instructions": "Is the customer asking for a refund?", + * "name": "is_refund_request", + * "type": "predicate" + * }, + * { + * "choices": [ + * { + * "description": "can wait a week", + * "value": "low" + * }, + * { + * "description": "needs action today", + * "value": "high" + * } + * ], + * "instructions": "How urgent is this?", + * "name": "urgency", + * "type": "choice" + * }, + * { + * "instructions": "How frustrated is the customer?", + * "levels": [ + * { + * "label": "calm" + * }, + * { + * "label": "mildly annoyed" + * }, + * { + * "label": "angry" + * } + * ], + * "name": "frustration", + * "type": "score" + * } + * ] + * } + */ + "application/json": { + /** + * Input + * @description The text being judged, or user messages whose content mixes input_text and input_image parts + */ + input: string | { + /** Content */ + content: string | ({ + /** Text */ + text: string; + /** + * Type + * @constant + */ + type: "input_text"; + } | { + /** Detail */ + detail?: string | null; + /** Image Url */ + image_url: string; + /** + * Type + * @constant + */ + type: "input_image"; + })[]; + /** + * Role + * @default user + * @constant + */ + role?: "user"; + /** + * Type + * @default message + * @constant + */ + type?: "message"; + }[]; + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Predicate, choice and score questions. Answers come back in the same order + */ + questions: ({ + /** + * Instructions + * @description The yes/no question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "predicate"; + } | { + /** + * Choices + * @description The options the model picks from. Values must be unique + */ + choices: { + /** Description */ + description?: string | null; + /** Value */ + value: string | boolean; + }[]; + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Levels + * @description Scale levels from lowest to highest. The score is a level index + */ + levels: { + /** Description */ + description?: string | null; + /** Label */ + label: string; + }[]; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "score"; + })[]; + /** + * Safety Identifier + * @description A stable id for your end user. Sent to OpenAI and dropped for other providers + */ + safety_identifier?: string | null; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -56575,7 +56161,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["OpenAIDecisionResponse"]; }; }; }; @@ -56613,79 +56199,6 @@ export interface operations { }; }; }; - get_email_event_settings_email_event_settings_get: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["EmailEventSettingsResponse"]; - }; - }; - }; - }; - update_event_settings_email_event_settings_patch: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["EmailEventSettingsUpdateRequest"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - reset_event_settings_email_event_settings_reset_post: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - }; - }; - }; - }; embeddings_embeddings_post: { parameters: { query?: never; @@ -67572,156 +67085,6 @@ export interface operations { }; }; }; - delete_project_project_delete_delete: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["DeleteProjectRequest"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["LiteLLM_ProjectTable"][]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - project_info_project_info_get: { - parameters: { - query: { - project_id: string; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["LiteLLM_ProjectTable"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - list_projects_project_list_get: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["LiteLLM_ProjectTable"][]; - }; - }; - }; - }; - new_project_project_new_post: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["NewProjectRequest"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["NewProjectResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - update_project_project_update_post: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["UpdateProjectRequest"]; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["LiteLLM_ProjectTable"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; create_prompt_prompts_post: { parameters: { query?: never; @@ -68914,26 +68277,6 @@ export interface operations { }; }; }; - get_robots_robots_txt_get: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - }; - }; - }; - }; test_roi_calculator_connections_roi_calculator_connections_test_post: { parameters: { query?: never; @@ -71141,7 +70484,132 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "jev", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "urgency": { + * "criteria": { + * "high": "needs action today", + * "low": "can wait a week" + * }, + * "instructions": "How urgent is this?", + * "type": "choice" + * } + * }, + * "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + * } + */ + "application/json": { + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Named noul, choice and score questions. Each key becomes a key in the answers + */ + questions: { + [key: string]: ({ + /** + * Criteria + * @description Optional descriptions of what makes the answer true and what makes it false + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description The yes/no question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Candidate labels mapped to an optional description of each + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Scale levels from lowest to highest. The score is a level index + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[])[]; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + } & { + [key: string]: unknown; + }); + }; + /** + * State + * @description The thing being judged, such as a support ticket, a document or a chat transcript + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -71149,7 +70617,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["DecisionsResponse"]; }; }; }; @@ -74059,26 +73527,6 @@ export interface operations { }; }; }; - available_enterprise_users_user_available_users_get: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - }; - }; - }; - }; bulk_user_update_user_bulk_update_post: { parameters: { query?: never; @@ -76262,7 +75710,179 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + * "model": "luna", + * "questions": [ + * { + * "instructions": "Is the customer asking for a refund?", + * "name": "is_refund_request", + * "type": "predicate" + * }, + * { + * "choices": [ + * { + * "description": "can wait a week", + * "value": "low" + * }, + * { + * "description": "needs action today", + * "value": "high" + * } + * ], + * "instructions": "How urgent is this?", + * "name": "urgency", + * "type": "choice" + * }, + * { + * "instructions": "How frustrated is the customer?", + * "levels": [ + * { + * "label": "calm" + * }, + * { + * "label": "mildly annoyed" + * }, + * { + * "label": "angry" + * } + * ], + * "name": "frustration", + * "type": "score" + * } + * ] + * } + */ + "application/json": { + /** + * Input + * @description The text being judged, or user messages whose content mixes input_text and input_image parts + */ + input: string | { + /** Content */ + content: string | ({ + /** Text */ + text: string; + /** + * Type + * @constant + */ + type: "input_text"; + } | { + /** Detail */ + detail?: string | null; + /** Image Url */ + image_url: string; + /** + * Type + * @constant + */ + type: "input_image"; + })[]; + /** + * Role + * @default user + * @constant + */ + role?: "user"; + /** + * Type + * @default message + * @constant + */ + type?: "message"; + }[]; + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Predicate, choice and score questions. Answers come back in the same order + */ + questions: ({ + /** + * Instructions + * @description The yes/no question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "predicate"; + } | { + /** + * Choices + * @description The options the model picks from. Values must be unique + */ + choices: { + /** Description */ + description?: string | null; + /** Value */ + value: string | boolean; + }[]; + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Levels + * @description Scale levels from lowest to highest. The score is a level index + */ + levels: { + /** Description */ + description?: string | null; + /** Label */ + label: string; + }[]; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "score"; + })[]; + /** + * Safety Identifier + * @description A stable id for your end user. Sent to OpenAI and dropped for other providers + */ + safety_identifier?: string | null; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -76270,7 +75890,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["OpenAIDecisionResponse"]; }; }; }; @@ -79414,7 +79034,132 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "jev", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "urgency": { + * "criteria": { + * "high": "needs action today", + * "low": "can wait a week" + * }, + * "instructions": "How urgent is this?", + * "type": "choice" + * } + * }, + * "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + * } + */ + "application/json": { + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Named noul, choice and score questions. Each key becomes a key in the answers + */ + questions: { + [key: string]: ({ + /** + * Criteria + * @description Optional descriptions of what makes the answer true and what makes it false + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description The yes/no question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Candidate labels mapped to an optional description of each + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Scale levels from lowest to highest. The score is a level index + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[])[]; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + } & { + [key: string]: unknown; + }); + }; + /** + * State + * @description The thing being judged, such as a support ticket, a document or a chat transcript + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -79422,7 +79167,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["DecisionsResponse"]; }; }; };