scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-17 04:34:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 172

Merge pull request #45891 from BerriAI/litellm_backport_decisions_ui_rc_1_106_0

Browse source 
feat(decisions): backport the rest of the decisions and System One launch to rc/1.106.0
This commit is contained in:
Mateo Wang 2026-10-10 22:15:21 -07:00 • committed by GitHub
commit 7b406378bd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
125 changed files with 11911 additions and 1135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/test-unit.yml vendored
View file

@ -139,6 +139,7 @@ jobs:
test-path: |-
tests/unit/decisions
tests/unit/litellm_core_utils
tests/unit/systemone
python-version: "3.12"
workers: 4
reruns: 0
@ -343,6 +344,7 @@ jobs:
tests/unit/llms/snowflake
tests/unit/llms/soniox
tests/unit/llms/stability
tests/unit/llms/strands_decider
tests/unit/llms/tavily
tests/unit/llms/tencent
tests/unit/llms/tinyfish

7
litellm/__init__.py
View file

@ -1478,6 +1478,7 @@ from .rust_bridge import rust
from .rag.main import *
from .sandbox.main import *
from .decisions.main import *
from .systemone.main import *
from .tool_loop import ToolLoopMaxRoundsExceeded, arun_tool_loop, run_tool_loop
from .search.main import *
from .realtime_api.main import (
@ -1681,6 +1682,12 @@ if TYPE_CHECKING:
from .llms.strands_decider.decisions.transformation import (
StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig,
)
from .llms.databricks.decisions.transformation import (
DatabricksDecisionsConfig as DatabricksDecisionsConfig,
)
from .llms.hosted_vllm.decisions.transformation import (
HostedVLLMDecisionsConfig as HostedVLLMDecisionsConfig,
)
from .llms.openai.decisions.transformation import (
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
)

7
litellm/_lazy_imports_registry.py
View file

@ -162,6 +162,8 @@ LLM_CONFIG_NAMES: Final = (
"OpenRouterDecisionsConfig",
"CloudflareDecisionsConfig",
"StrandsDeciderDecisionsConfig",
"DatabricksDecisionsConfig",
"HostedVLLMDecisionsConfig",
"OpenAIDecisionsConfig",
"AzureAIDecisionsConfig",
"NvidiaNimRerankConfig",
@ -724,6 +726,11 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
".llms.strands_decider.decisions.transformation",
"StrandsDeciderDecisionsConfig",
),
"DatabricksDecisionsConfig": (".llms.databricks.decisions.transformation", "DatabricksDecisionsConfig"),
"HostedVLLMDecisionsConfig": (
".llms.hosted_vllm.decisions.transformation",
"HostedVLLMDecisionsConfig",
),
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
"NvidiaNimRerankConfig": (

1
litellm/constants.py
View file

@ -285,6 +285,7 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int(
# Provider-specific API base URLs
XAI_API_BASE: Final = "https://api.x.ai/v1"
PLACEHOLDER_API_KEY: Final = "fake-api-key"
OPEN_SANDBOX_API_BASE_ENV_VAR: Final = "OPEN_SANDBOX_API_BASE"
OPEN_SANDBOX_API_KEY_ENV_VAR: Final = "OPEN_SANDBOX_API_KEY"
OPEN_SANDBOX_DEFAULT_TEMPLATE: Final = "opensandbox/code-interpreter:v1.1.0"

2
litellm/cost_calculator.py
View file

@ -2026,6 +2026,8 @@ def response_cost_calculator(
"asearch",
"decisions",
"adecisions",
"systemone",
"asystemone",
],
optional_params: dict,
cache_hit: bool | None = None,

347
litellm/decisions/call.py Normal file
View file

@ -0,0 +1,347 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field, replace
from typing import Final, TypeAlias
import httpx
from pydantic import ConfigDict, TypeAdapter, ValidationError
from typing_extensions import assert_never
import litellm
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
BaseDecisionsConfig,
ir_to_systemone_response,
systemone_request_to_ir,
)
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir
from litellm.types.decisions import (
DecisionQuestion,
DecisionsIRRequest,
DecisionsIRResponse,
DecisionsJSON,
DecisionsRequestBody,
DecisionsResponse,
OpenAIDecisionInput,
OpenAIDecisionQuestion,
OpenAIDecisionRequestBody,
OpenAIDecisionResponse,
UnsupportedDecisionsRequest,
)
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody
_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody)
_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody)
_SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(
str | None, config=ConfigDict(title="safety_identifier")
)
_HANDLER: Final = BaseLLMHTTPHandler()
@dataclass(frozen=True, slots=True)
class SystemOneRequestFields:
state: DecisionsJSON | None
questions: SystemOneQuestions | None
def validate(self, safety_identifier: str | None) -> DecisionsRequestBody:
return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": self.state, "questions": self.questions})
@dataclass(frozen=True, slots=True)
class OpenAIDecisionRequestFields:
input: OpenAIDecisionInput | None
questions: OpenAIDecisionQuestions | None
def validate(self, safety_identifier: str | None) -> OpenAIDecisionRequestBody:
return _OPENAI_REQUEST_ADAPTER.validate_python(
{"input": self.input, "questions": self.questions, "safety_identifier": safety_identifier}
)
RequestFields: TypeAlias = SystemOneRequestFields | OpenAIDecisionRequestFields
@dataclass(frozen=True, slots=True, repr=False)
class DecisionsCall:
model: str
requested_model: str
custom_llm_provider: str
provider_config: BaseDecisionsConfig
request: DecisionsRequestFormat
ir_request: DecisionsIRRequest
body: Mapping[str, object] = field(repr=False)
api_base: str
api_key: str | None = field(repr=False)
logging_obj: LiteLLMLoggingObj | None
headers: Mapping[str, str]
timeout: float | httpx.Timeout | None
litellm_params: Mapping[str, object] = field(repr=False)
def supported_decisions_providers() -> tuple[str, ...]:
return tuple(
provider.value
for provider in LlmProviders
if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None
)
def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig:
provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None)
provider_config: Final = (
None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider)
)
if provider_config is None:
supported: Final = ", ".join(supported_decisions_providers())
raise litellm.BadRequestError(
message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}",
model=model,
llm_provider=custom_llm_provider,
)
return provider_config
def _ir_request(request: DecisionsRequestFormat, safety_identifier: str | None) -> DecisionsIRRequest:
match request:
case DecisionsRequestBody():
return replace(systemone_request_to_ir(request), safety_identifier=safety_identifier)
case OpenAIDecisionRequestBody():
return openai_request_to_ir(request)
case _:
assert_never(request)
def _drops_params(kwargs: Mapping[str, object]) -> bool:
return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True
def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, object]) -> str | None:
if isinstance(safety_identifier, str) or not _drops_params(kwargs):
return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier)
return None
@dataclass(frozen=True, slots=True)
class _UnsupportedSafetyIdentifier:
pass
def _provider_ir_request(
request: DecisionsRequestFormat,
*,
safety_identifier: str | None,
provider_config: BaseDecisionsConfig,
kwargs: Mapping[str, object],
) -> DecisionsIRRequest | _UnsupportedSafetyIdentifier:
ir_request: Final = _ir_request(request, safety_identifier)
if ir_request.safety_identifier is None or provider_config.supports_safety_identifier:
return ir_request
if _drops_params(kwargs):
return replace(ir_request, safety_identifier=None)
return _UnsupportedSafetyIdentifier()
def prepare_call(
*,
model: str,
fields: RequestFields,
safety_identifier: str | None,
api_key: str | None,
api_base: str | None,
timeout: float | httpx.Timeout | None,
custom_llm_provider: str | None,
extra_headers: Mapping[str, str] | None,
kwargs: Mapping[str, object],
) -> DecisionsCall:
upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
api_key=api_key,
)
provider_config: Final = _provider_config(upstream_model, provider)
if not upstream_model:
raise litellm.BadRequestError(
message="A model name is required for the Decisions API",
model=model,
llm_provider=provider,
)
try:
canonical_model: Final = provider_config.canonical_model(upstream_model)
except ValueError as error:
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
try:
request_safety_identifier: Final = _request_safety_identifier(safety_identifier, kwargs)
request: Final = fields.validate(request_safety_identifier)
except ValidationError as error:
raise litellm.BadRequestError(
message=f"Invalid Decisions request: {error}",
model=model,
llm_provider=provider,
) from error
resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base)
if resolved_api_base is None:
raise litellm.BadRequestError(
message=provider_config.missing_api_base_message(provider),
model=model,
llm_provider=provider,
)
resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key)
if resolved_api_key is None and provider_config.api_key_required:
raise litellm.AuthenticationError(
message=f"Missing API key for Decisions provider '{provider}'",
model=model,
llm_provider=provider,
)
ir_request: Final = _provider_ir_request(
request,
safety_identifier=request_safety_identifier,
provider_config=provider_config,
kwargs=kwargs,
)
if isinstance(ir_request, _UnsupportedSafetyIdentifier):
raise litellm.UnsupportedParamsError(
message=(
f"{provider} does not support parameters: ['safety_identifier'], for model={model}. "
"To drop these, set `litellm.drop_params=True` or for proxy:\n\n"
"`litellm_settings:\n drop_params: true`\n"
),
model=model,
llm_provider=provider,
)
body: Final = provider_config.transform_decisions_request(
model=canonical_model, request=ir_request, custom_llm_provider=provider
)
if isinstance(body, UnsupportedDecisionsRequest):
raise litellm.BadRequestError(
message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}",
model=model,
llm_provider=provider,
)
logging_obj: Final = kwargs.get("litellm_logging_obj")
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.update_from_kwargs(
kwargs=dict(kwargs),
model=canonical_model,
litellm_params={
"litellm_call_id": kwargs.get("litellm_call_id"),
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
},
custom_llm_provider=provider,
)
return DecisionsCall(
model=canonical_model,
requested_model=model,
custom_llm_provider=provider,
provider_config=provider_config,
request=request,
ir_request=ir_request,
body=body,
api_base=resolved_api_base,
api_key=resolved_api_key,
logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
headers=extra_headers or {},
timeout=timeout,
litellm_params=kwargs,
)
def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
formatted: Final = _formatted_response(response, call)
provider_cost: Final = call.provider_config.provider_reported_cost(response)
formatted.set_hidden_params(
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": f"{call.custom_llm_provider}/{call.model}",
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
}
)
return formatted
def _formatted_response(
response: DecisionsIRResponse, call: DecisionsCall
) -> DecisionsResponse | OpenAIDecisionResponse:
match call.request:
case DecisionsRequestBody():
return ir_to_systemone_response(response, call.ir_request)
case OpenAIDecisionRequestBody():
return ir_to_openai_response(response, call.ir_request, call.requested_model)
case _:
assert_never(call.request)
def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception:
if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
return litellm.APIConnectionError(
message=f"{provider_label} - {error.message}",
llm_provider=call.custom_llm_provider,
model=f"{call.custom_llm_provider}/{call.model}",
)
return litellm.exception_type(
model=f"{call.custom_llm_provider}/{call.model}",
custom_llm_provider=call.custom_llm_provider,
original_exception=error,
)
async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
try:
response: Final = await _HANDLER.adecisions(
model=call.model,
custom_llm_provider=call.custom_llm_provider,
logging_obj=call.logging_obj,
provider_config=call.provider_config,
request=call.ir_request,
body=call.body,
api_base=call.api_base,
api_key=call.api_key,
headers=call.headers,
timeout=call.timeout,
litellm_params=call.litellm_params,
)
except Exception as error:
raise _map_upstream_exception(error, call) from error
return _format_response(response, call)
def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
try:
response: Final = _HANDLER.decisions(
model=call.model,
custom_llm_provider=call.custom_llm_provider,
logging_obj=call.logging_obj,
provider_config=call.provider_config,
request=call.ir_request,
body=call.body,
api_base=call.api_base,
api_key=call.api_key,
headers=call.headers,
timeout=call.timeout,
litellm_params=call.litellm_params,
)
except Exception as error:
raise _map_upstream_exception(error, call) from error
return _format_response(response, call)
def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name: str, other_function: str) -> None:
if field_name not in kwargs:
return
raise litellm.BadRequestError(
message=f"Requests with '{field_name}' go through litellm.{other_function}",
model=model,
llm_provider="",
)

326
litellm/decisions/main.py
View file

@ -1,116 +1,56 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import Final, TypeAlias
import warnings
from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
from typing_extensions import assert_never
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
BaseDecisionsConfig,
ir_to_systemone_response,
systemone_request_to_ir,
from litellm.decisions.call import (
DecisionsCall,
OpenAIDecisionQuestions,
OpenAIDecisionRequestFields,
RequestFields,
SystemOneQuestions,
SystemOneRequestFields,
asend,
prepare_call,
send,
)
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir
from litellm.types.decisions import (
DecisionQuestion,
DecisionsIRRequest,
DecisionsIRResponse,
DecisionsJSON,
DecisionsRequestBody,
DecisionsResponse,
OpenAIDecisionInput,
OpenAIDecisionQuestion,
OpenAIDecisionRequestBody,
OpenAIDecisionResponse,
UnsupportedDecisionsRequest,
from litellm.types.decisions import DecisionsJSON, DecisionsResponse, OpenAIDecisionInput, OpenAIDecisionResponse
from litellm.utils import client
_STATE_ADAPTER: Final[TypeAdapter[DecisionsJSON | None]] = TypeAdapter(DecisionsJSON | None)
_SYSTEMONE_QUESTIONS_ADAPTER: Final[TypeAdapter[SystemOneQuestions | None]] = TypeAdapter(SystemOneQuestions | None)
_OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter(
OpenAIDecisionQuestions | None
)
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, client
DecisionsQuestions: TypeAlias = (
Mapping[str, DecisionQuestion | Mapping[str, object]] | Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
)
DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody
_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody)
_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody)
_HANDLER: Final = BaseLLMHTTPHandler()
@dataclass(frozen=True, slots=True, repr=False)
class _DecisionsCall:
model: str
requested_model: str
custom_llm_provider: str
provider_config: BaseDecisionsConfig
request: DecisionsRequestFormat
ir_request: DecisionsIRRequest
body: Mapping[str, object] = field(repr=False)
api_base: str
api_key: str | None = field(repr=False)
logging_obj: LiteLLMLoggingObj | None
headers: Mapping[str, str]
timeout: float | httpx.Timeout | None
def supported_decisions_providers() -> tuple[str, ...]:
return tuple(
provider.value
for provider in LlmProviders
if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None
)
def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig:
provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None)
provider_config: Final = (
None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider)
)
if provider_config is None:
supported: Final = ", ".join(supported_decisions_providers())
raise litellm.BadRequestError(
message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}",
model=model,
llm_provider=custom_llm_provider,
)
return provider_config
def _validate_request(
*,
state: DecisionsJSON | None,
questions: DecisionsQuestions | None,
def _request_fields(
decision_input: OpenAIDecisionInput | None,
safety_identifier: str | None,
) -> DecisionsRequestFormat:
if decision_input is None:
return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": state, "questions": questions})
return _OPENAI_REQUEST_ADAPTER.validate_python(
{"input": decision_input, "questions": questions, "safety_identifier": safety_identifier}
questions: OpenAIDecisionQuestions | SystemOneQuestions | None,
kwargs: Mapping[str, object],
) -> RequestFields:
if "state" not in kwargs:
return OpenAIDecisionRequestFields(
input=decision_input, questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions)
)
warnings.warn(
"litellm.decisions(state=...) is deprecated, call litellm.systemone for the System One request format",
UserWarning,
stacklevel=5,
)
return SystemOneRequestFields(
state=_STATE_ADAPTER.validate_python(kwargs["state"]),
questions=_SYSTEMONE_QUESTIONS_ADAPTER.validate_python(questions),
)
def _ir_request(request: DecisionsRequestFormat) -> DecisionsIRRequest:
match request:
case DecisionsRequestBody():
return systemone_request_to_ir(request)
case OpenAIDecisionRequestBody():
return openai_request_to_ir(request)
case _:
assert_never(request)
def _prepare_call(
def _prepare_decisions_call(
*,
model: str,
state: DecisionsJSON | None,
questions: DecisionsQuestions | None,
decision_input: OpenAIDecisionInput | None,
questions: OpenAIDecisionQuestions | SystemOneQuestions | None,
safety_identifier: str | None,
api_key: str | None,
api_base: str | None,
@ -118,149 +58,37 @@ def _prepare_call(
custom_llm_provider: str | None,
extra_headers: Mapping[str, str] | None,
kwargs: Mapping[str, object],
) -> _DecisionsCall:
upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider(
) -> DecisionsCall:
return prepare_call(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
fields=_request_fields(decision_input, questions, kwargs),
safety_identifier=safety_identifier,
api_key=api_key,
)
provider_config: Final = _provider_config(upstream_model, provider)
canonical_model: Final = provider_config.canonical_model(upstream_model)
if not upstream_model:
raise litellm.BadRequestError(
message="A model name is required for the Decisions API",
model=model,
llm_provider=provider,
)
if state is not None and decision_input is not None:
raise litellm.BadRequestError(
message="Pass either state (System One format) or input (OpenAI format) to the Decisions API, not both",
model=model,
llm_provider=provider,
)
try:
request: Final = _validate_request(
state=state, questions=questions, decision_input=decision_input, safety_identifier=safety_identifier
)
except ValidationError as error:
raise litellm.BadRequestError(
message=f"Invalid Decisions request: {error}",
model=model,
llm_provider=provider,
) from error
resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base)
if resolved_api_base is None:
raise litellm.BadRequestError(
message=provider_config.missing_api_base_message(provider),
model=model,
llm_provider=provider,
)
resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key)
if resolved_api_key is None and provider_config.api_key_required:
raise litellm.AuthenticationError(
message=f"Missing API key for Decisions provider '{provider}'",
model=model,
llm_provider=provider,
)
ir_request: Final = _ir_request(request)
body: Final = provider_config.transform_decisions_request(
model=canonical_model, request=ir_request, custom_llm_provider=provider
)
if isinstance(body, UnsupportedDecisionsRequest):
raise litellm.BadRequestError(
message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}",
model=model,
llm_provider=provider,
)
logging_obj: Final = kwargs.get("litellm_logging_obj")
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.update_from_kwargs(
kwargs=dict(kwargs),
model=canonical_model,
litellm_params={
"litellm_call_id": kwargs.get("litellm_call_id"),
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
},
custom_llm_provider=provider,
)
return _DecisionsCall(
model=canonical_model,
requested_model=model,
custom_llm_provider=provider,
provider_config=provider_config,
request=request,
ir_request=ir_request,
body=body,
api_base=resolved_api_base,
api_key=resolved_api_key,
logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
headers=extra_headers or {},
api_base=api_base,
timeout=timeout,
)
def _format_response(response: DecisionsIRResponse, call: _DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
formatted: Final = _formatted_response(response, call)
formatted.set_hidden_params(
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": f"{call.custom_llm_provider}/{call.model}",
}
)
return formatted
def _formatted_response(
response: DecisionsIRResponse, call: _DecisionsCall
) -> DecisionsResponse | OpenAIDecisionResponse:
match call.request:
case DecisionsRequestBody():
return ir_to_systemone_response(response, call.ir_request)
case OpenAIDecisionRequestBody():
return ir_to_openai_response(response, call.ir_request, call.requested_model)
case _:
assert_never(call.request)
def _map_upstream_exception(error: Exception, call: _DecisionsCall) -> Exception:
if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
return litellm.APIConnectionError(
message=f"{provider_label} - {error.message}",
llm_provider=call.custom_llm_provider,
model=f"{call.custom_llm_provider}/{call.model}",
)
return litellm.exception_type(
model=f"{call.custom_llm_provider}/{call.model}",
custom_llm_provider=call.custom_llm_provider,
original_exception=error,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs={key: value for key, value in kwargs.items() if key != "state"},
)
@client
async def adecisions(
model: str,
state: DecisionsJSON | None = None,
questions: DecisionsQuestions | None = None,
input: OpenAIDecisionInput | None = None,
questions: OpenAIDecisionQuestions | SystemOneQuestions | None = None,
safety_identifier: str | None = None,
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: Mapping[str, str] | None = None,
input: OpenAIDecisionInput | None = None,
safety_identifier: str | None = None,
**kwargs: object,
) -> DecisionsResponse | OpenAIDecisionResponse:
call: Final = _prepare_call(
) -> OpenAIDecisionResponse | DecisionsResponse:
call: Final = _prepare_decisions_call(
model=model,
state=state,
questions=questions,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
api_key=api_key,
api_base=api_base,
@ -269,43 +97,26 @@ async def adecisions(
extra_headers=extra_headers,
kwargs=kwargs,
)
try:
response: Final = await _HANDLER.adecisions(
model=call.model,
custom_llm_provider=call.custom_llm_provider,
logging_obj=call.logging_obj,
provider_config=call.provider_config,
request=call.ir_request,
body=call.body,
api_base=call.api_base,
api_key=call.api_key,
headers=call.headers,
timeout=call.timeout,
)
except Exception as error:
raise _map_upstream_exception(error, call) from error
return _format_response(response, call)
return await asend(call)
@client
def decisions(
model: str,
state: DecisionsJSON | None = None,
questions: DecisionsQuestions | None = None,
input: OpenAIDecisionInput | None = None,
questions: OpenAIDecisionQuestions | SystemOneQuestions | None = None,
safety_identifier: str | None = None,
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: Mapping[str, str] | None = None,
input: OpenAIDecisionInput | None = None,
safety_identifier: str | None = None,
**kwargs: object,
) -> DecisionsResponse | OpenAIDecisionResponse:
call: Final = _prepare_call(
) -> OpenAIDecisionResponse | DecisionsResponse:
call: Final = _prepare_decisions_call(
model=model,
state=state,
questions=questions,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
api_key=api_key,
api_base=api_base,
@ -314,22 +125,7 @@ def decisions(
extra_headers=extra_headers,
kwargs=kwargs,
)
try:
response: Final = _HANDLER.decisions(
model=call.model,
custom_llm_provider=call.custom_llm_provider,
logging_obj=call.logging_obj,
provider_config=call.provider_config,
request=call.ir_request,
body=call.body,
api_base=call.api_base,
api_key=call.api_key,
headers=call.headers,
timeout=call.timeout,
)
except Exception as error:
raise _map_upstream_exception(error, call) from error
return _format_response(response, call)
return send(call)
__all__ = ["adecisions", "decisions"]

23
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -3,7 +3,7 @@ Helper functions for health check calls.
"""
import base64
from collections.abc import Awaitable, Callable
from collections.abc import Awaitable, Callable, Mapping
from typing import TYPE_CHECKING, Final, Literal
from litellm.llms.base_llm.ocr.transformation import DocumentType
@ -56,6 +56,21 @@ def get_image_file_for_health_check() -> bytes:
return base64.b64decode(TEST_IMAGE_BASE64)
def _decisions_health_check_questions(model: str, custom_llm_provider: str) -> Mapping[str, Mapping[str, object]]:
import litellm
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
from litellm.utils import ProviderConfigManager
provider: Final = next((member for member in litellm.LlmProviders if member.value == custom_llm_provider), None)
config: Final = (
None
if provider is None
else ProviderConfigManager.get_provider_decisions_config(model=model, provider=provider)
)
questions: Final = BaseDecisionsConfig.health_check_questions if config is None else config.health_check_questions
return {name: dict(question) for name, question in questions.items()}
def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType:
native: Final = NATIVE_OCR_HEALTH_CHECK_DOCUMENT.load()
if native is None:
@ -296,11 +311,13 @@ class HealthCheckHelpers:
**_filter_model_params(model_params=model_params),
document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider),
),
"evaluation": lambda: litellm.adecisions(
"evaluation": lambda: litellm.asystemone(
**DECISIONS_CALL_PARAMS.validate_python(
{
"state": prompt or "health check",
"questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}},
"questions": _decisions_health_check_questions(
model=model, custom_llm_provider=custom_llm_provider
),
**_filter_model_params(model_params=model_params),
}
)

21
litellm/llms/base_llm/decisions/transformation.py
View file

@ -301,10 +301,17 @@ class BaseDecisionsConfig(ABC):
api_key_env: tuple[str, ...] = ()
api_base_env: tuple[str, ...] = ()
api_key_required: bool = True
supports_safety_identifier: bool = False
health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
{"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})}
)
def get_default_api_base(self) -> str | None:
return None
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
return None
def missing_api_base_message(self, custom_llm_provider: str) -> str:
return f"api_base is required for Decisions provider '{custom_llm_provider}'"
@ -334,6 +341,20 @@ class BaseDecisionsConfig(ABC):
def get_complete_url(self, api_base: str, model: str) -> str:
return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}"
def signs_with_aws(self, api_base: str) -> bool:
return False
def sign_request(
self,
headers: Mapping[str, str],
url: str,
api_base: str,
body: Mapping[str, object],
api_key: str | None,
litellm_params: Mapping[str, object],
) -> tuple[Mapping[str, str], bytes | None]:
return headers, None
def transform_decisions_request(
self,
model: str,

38
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -4,7 +4,7 @@ import json
import ssl
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Coroutine, Iterator, Mapping, Sequence
from contextlib import asynccontextmanager
from functools import lru_cache
from functools import lru_cache, partial
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -1517,6 +1517,7 @@ class BaseLLMHTTPHandler:
api_key: str | None,
headers: Mapping[str, str],
timeout: float | httpx.Timeout | None,
litellm_params: Mapping[str, object],
client: HTTPHandler | None = None,
) -> DecisionsIRResponse:
url, outbound_headers, data = self._prepare_decisions_request(
@ -1528,10 +1529,23 @@ class BaseLLMHTTPHandler:
api_key=api_key,
headers=headers,
)
signed_headers, signed_body = provider_config.sign_request(
headers=outbound_headers,
url=url,
api_base=api_base,
body=data,
api_key=api_key,
litellm_params=litellm_params,
)
sync_httpx_client: Final = client if client is not None else get_httpx_client()
try:
response: Final = sync_httpx_client.post(
url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj
url,
content=signed_body,
json=data if signed_body is None else None,
headers=dict(signed_headers),
timeout=timeout,
logging_obj=logging_obj,
)
except httpx.HTTPError as e:
raise self._handle_error(e=e, provider_config=provider_config)
@ -1551,6 +1565,7 @@ class BaseLLMHTTPHandler:
api_key: str | None,
headers: Mapping[str, str],
timeout: float | httpx.Timeout | None,
litellm_params: Mapping[str, object],
client: AsyncHTTPHandler | None = None,
) -> DecisionsIRResponse:
url, outbound_headers, data = self._prepare_decisions_request(
@ -1562,6 +1577,18 @@ class BaseLLMHTTPHandler:
api_key=api_key,
headers=headers,
)
sign: Final = partial(
provider_config.sign_request,
headers=outbound_headers,
url=url,
api_base=api_base,
body=data,
api_key=api_key,
litellm_params=litellm_params,
)
signed_headers, signed_body = (
await run_aws_signing(sign) if provider_config.signs_with_aws(api_base) else sign()
)
async_httpx_client: Final = (
client
if client is not None
@ -1569,7 +1596,12 @@ class BaseLLMHTTPHandler:
)
try:
response: Final = await async_httpx_client.post(
url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj
url,
content=signed_body,
json=data if signed_body is None else None,
headers=dict(signed_headers),
timeout=timeout,
logging_obj=logging_obj,
)
except httpx.HTTPError as e:
raise self._handle_error(e=e, provider_config=provider_config)

57
litellm/llms/databricks/decisions/transformation.py Normal file
View file

@ -0,0 +1,57 @@
import re
from collections.abc import Mapping
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
_SERVING_ENDPOINT_NAME: Final = re.compile(r"[A-Za-z0-9_-][A-Za-z0-9._-]*")
def validate_serving_endpoint_name(name: str) -> str:
if _SERVING_ENDPOINT_NAME.fullmatch(name) is None:
raise ValueError(
f"Databricks serving endpoint name {name!r} must be the bare endpoint name (letters, digits, '-', '_' "
"and '.', with no '/', '?', '#', spaces, or a leading '.'), e.g. databricks-openjev-qwen35-4b"
)
return name
@dataclass(frozen=True, slots=True)
class DatabricksDecisionsConnection:
api_base: str
api_key: str = field(repr=False)
class DatabricksDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("DATABRICKS_API_KEY", "DATABRICKS_TOKEN")
api_base_env = ("DATABRICKS_API_BASE",)
def missing_api_base_message(self, custom_llm_provider: str) -> str:
return (
f"api_base is required for Decisions provider '{custom_llm_provider}': set DATABRICKS_API_BASE to "
"https://<workspace-host>/serving-endpoints"
)
def canonical_model(self, model: str) -> str:
return validate_serving_endpoint_name(model)
def get_complete_url(self, api_base: str, model: str) -> str:
return f"{api_base.rstrip('/')}/{validate_serving_endpoint_name(model)}/invocations"
def classifier_response(self, body: Mapping[str, object], requested_model: str) -> Mapping[str, object]:
return MappingProxyType({**body, "model": requested_model})
def connection(self, api_base: str | None, api_key: str | None) -> DatabricksDecisionsConnection:
base: Final = self.resolve_api_base(api_base)
key: Final = api_key if api_base is not None else self.resolve_api_key(api_key)
if not base or not key:
raise ValueError(
"Databricks requires api_key or DATABRICKS_API_KEY (or DATABRICKS_TOKEN) and api_base or "
"DATABRICKS_API_BASE pointing to https://<workspace-host>/serving-endpoints"
)
return DatabricksDecisionsConnection(api_base=base.rstrip("/"), api_key=key)
DATABRICKS_DECISIONS_CONFIG: Final[DatabricksDecisionsConfig] = DatabricksDecisionsConfig()

26
litellm/llms/hosted_vllm/decisions/transformation.py Normal file
View file

@ -0,0 +1,26 @@
from types import MappingProxyType
from typing import Final
from litellm.constants import PLACEHOLDER_API_KEY
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class HostedVLLMDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("HOSTED_VLLM_API_KEY",)
api_base_env = ("HOSTED_VLLM_API_BASE",)
api_key_required = False
health_check_questions = MappingProxyType(
{
"reachable": MappingProxyType(
{
"type": "choice",
"instructions": "Is the service reachable?",
"criteria": {"yes": None, "no": None},
}
)
}
)
def resolve_api_key(self, api_key: str | None) -> str | None:
resolved: Final = super().resolve_api_key(api_key)
return None if resolved == PLACEHOLDER_API_KEY else resolved

1
litellm/llms/openai/decisions/transformation.py
View file

@ -292,6 +292,7 @@ def ir_to_openai_response(
class OpenAIDecisionsConfig(BaseDecisionsConfig):
path = "/v1/decisions"
supports_safety_identifier = True
def get_default_api_base(self) -> str | None:
return "https://api.openai.com"

13
litellm/llms/openrouter/chat/transformation.py
View file

@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, Final, cast
import httpx
import litellm
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -41,18 +40,10 @@ class CacheControlSupportedModels(str, Enum):
class OpenrouterConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list:
"""
Allow reasoning parameters for models flagged as reasoning-capable.
Include reasoning parameters regardless of model-map capabilities.
"""
supported_params: Final = super().get_supported_openai_params(model=model)
try:
if litellm.supports_reasoning(model=model, custom_llm_provider="openrouter") or litellm.supports_reasoning(
model=model
):
supported_params.append("reasoning_effort")
supported_params.append("thinking")
except Exception:
pass
return list(dict.fromkeys(supported_params))
return list(dict.fromkeys([*supported_params, "reasoning_effort", "thinking"]))
def map_openai_params(
self,

9
litellm/llms/openrouter/decisions/transformation.py
View file

@ -1,4 +1,7 @@
from typing import Final
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
from litellm.types.decisions import DecisionsIRResponse
class OpenRouterDecisionsConfig(BaseDecisionsConfig):
@ -8,3 +11,9 @@ class OpenRouterDecisionsConfig(BaseDecisionsConfig):
def get_default_api_base(self) -> str | None:
return "https://openrouter.ai/api"
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
cost: Final = response.usage.extra.get("cost")
if not isinstance(cost, (int, float)):
return None
return float(cost)

70
litellm/llms/openrouter/responses/transformation.py
View file

@ -8,11 +8,25 @@ encrypted_content for multi-turn stateless workflows.
Docs: https://openrouter.ai/docs/api/reference/responses/overview
"""
from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER
from litellm.litellm_core_utils.hidden_params import set_hidden_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
ResponseCompletedEvent,
ResponseFailedEvent,
ResponseIncompleteEvent,
ResponsesAPIResponse,
ResponsesAPIStreamingResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
@ -33,6 +47,62 @@ class OpenRouterResponsesAPIConfig(OpenAIResponsesAPIConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.OPENROUTER
def transform_response_api_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIResponse:
response: Final = super().transform_response_api_response(
model=model,
raw_response=raw_response,
logging_obj=logging_obj,
)
self._set_provider_reported_cost(response)
return response
def transform_streaming_response(
self,
model: str,
parsed_chunk: Mapping[str, object],
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIStreamingResponse:
response_event: Final = super().transform_streaming_response(
model=model,
parsed_chunk=dict(parsed_chunk),
logging_obj=logging_obj,
)
terminal_response: Final = (
response_event.response
if isinstance(
response_event,
(ResponseCompletedEvent, ResponseFailedEvent, ResponseIncompleteEvent),
)
else None
)
if terminal_response is not None:
self._set_provider_reported_cost(terminal_response)
return response_event
@staticmethod
def _set_provider_reported_cost(response: ResponsesAPIResponse) -> None:
cost: Final = response.usage.cost if response.usage is not None else None
if cost is None:
return
additional_headers: Final = TypeAdapter(dict[str, object]).validate_python(
response.hidden_params.get("additional_headers", {})
)
set_hidden_params(
response,
{
**response.hidden_params,
"additional_headers": {
**additional_headers,
RESPONSE_COST_HEADER: float(cost),
},
},
)
def validate_environment(
self,
headers: dict,

154
litellm/llms/strands_decider/decisions/transformation.py
View file

@ -1,7 +1,161 @@
import hashlib
import json
import re
from collections.abc import Mapping
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import Final
from urllib.parse import quote, unquote
import httpx
from pydantic import BaseModel, ConfigDict, ValidationError
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, sign_aws_json_post
from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse
from litellm.types.llms.bedrock import AwsAuthParams
AGENTCORE_SESSION_HEADER: Final = "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"
_RESERVED_HEADER_PREFIXES: Final = ("x-amzn-bedrock-agentcore-runtime-", "x-amz-")
_RUNTIME_ARN: Final = re.compile(
r"\Aarn:aws(?:-[a-z]+)*:bedrock-agentcore:(?P<region>[a-z0-9-]+):[0-9]{12}:runtime/[A-Za-z0-9_-]+\Z"
)
_INVOCATIONS_PATH: Final = re.compile(r"\A/runtimes/(?P<arn>[^/]+)/invocations\Z")
_RUNTIME_ERROR_STATUS: Final = MappingProxyType({"bad_request": 400, "invalid_request": 400, "loading": 503})
@dataclass(frozen=True, slots=True)
class AgentCoreRuntime:
arn: str
region: str
@property
def invocations_url(self) -> str:
host: Final = f"bedrock-agentcore.{self.region}.{get_aws_dns_suffix(self.region)}"
return f"https://{host}/runtimes/{quote(self.arn, safe='')}/invocations"
@property
def default_session_id(self) -> str:
return f"litellm-decider-{hashlib.sha256(self.arn.encode()).hexdigest()}"
def agentcore_runtime(api_base: str) -> AgentCoreRuntime | None:
match: Final = _RUNTIME_ARN.match(api_base)
return None if match is None else AgentCoreRuntime(arn=api_base, region=match["region"])
def invoked_runtime(url: httpx.URL) -> AgentCoreRuntime | None:
match: Final = _INVOCATIONS_PATH.match(url.raw_path.decode("ascii"))
runtime: Final = None if match is None else agentcore_runtime(unquote(match["arn"]))
return runtime if runtime is not None and runtime.invocations_url == str(url) else None
class _RuntimeError(BaseModel):
model_config = ConfigDict(frozen=True)
code: str
message: str = ""
class _RuntimeReply(BaseModel):
model_config = ConfigDict(frozen=True)
error: _RuntimeError | None = None
answers: object = None
def _runtime_error(content: bytes) -> _RuntimeError | None:
try:
reply: Final = _RuntimeReply.model_validate_json(content)
except ValidationError:
return None
return reply.error if reply.answers is None else None
class _SessionParams(BaseModel):
model_config = ConfigDict(frozen=True)
agentcore_runtime_session_id: str | None = None
def _is_reserved_header(name: str) -> bool:
lowered: Final = name.lower()
return lowered == "host" or lowered.startswith(_RESERVED_HEADER_PREFIXES)
def _runtime_headers(
headers: Mapping[str, str], runtime: AgentCoreRuntime, litellm_params: Mapping[str, object]
) -> Mapping[str, str]:
dropped: Final = sorted(name for name in headers if _is_reserved_header(name))
if dropped:
verbose_logger.warning("Strands Decider: not forwarding reserved AgentCore header(s) %s", dropped)
configured: Final = _SessionParams.model_validate(litellm_params).agentcore_runtime_session_id
return {
**{name: value for name, value in headers.items() if not _is_reserved_header(name)},
AGENTCORE_SESSION_HEADER: runtime.default_session_id if configured is None else configured,
}
class StrandsDeciderDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("STRANDS_DECIDER_API_KEY",)
api_base_env = ("STRANDS_DECIDER_API_BASE",)
api_key_required = False
def __init__(self, aws: BaseAWSLLM | None = None) -> None:
self._aws: Final = aws if aws is not None else BaseAWSLLM()
def get_complete_url(self, api_base: str, model: str) -> str:
runtime: Final = agentcore_runtime(api_base)
return super().get_complete_url(api_base, model) if runtime is None else runtime.invocations_url
def signs_with_aws(self, api_base: str) -> bool:
return agentcore_runtime(api_base) is not None
def sign_request(
self,
headers: Mapping[str, str],
url: str,
api_base: str,
body: Mapping[str, object],
api_key: str | None,
litellm_params: Mapping[str, object],
) -> tuple[Mapping[str, str], bytes | None]:
runtime: Final = agentcore_runtime(api_base)
if runtime is None:
return headers, None
session_headers: Final = _runtime_headers(headers, runtime, litellm_params)
if api_key is not None:
return session_headers, None
payload: Final = json.dumps(body)
signed: Final = sign_aws_json_post(
get_credentials=partial(
self._aws.resolve_credentials, AwsAuthParams.model_validate(litellm_params), runtime.region
),
service_name="bedrock-agentcore",
aws_region_name=runtime.region,
url=url,
body=payload,
headers=session_headers,
)
return dict(signed.headers.items()), payload.encode()
def transform_decisions_response(
self,
model: str,
custom_llm_provider: str,
raw_response: httpx.Response,
request: DecisionsIRRequest,
) -> DecisionsIRResponse:
runtime_error: Final = (
None if invoked_runtime(raw_response.request.url) is None else _runtime_error(raw_response.content)
)
if runtime_error is not None:
raise BaseLLMException(
status_code=_RUNTIME_ERROR_STATUS.get(runtime_error.code, 500),
message=f"Strands Decider runtime error '{runtime_error.code}': {runtime_error.message}",
)
return super().transform_decisions_response(model, custom_llm_provider, raw_response, request)

12
litellm/model_prices_and_context_window_backup.json
View file

@ -21165,6 +21165,18 @@
"output_vector_size": 1024,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
"databricks/databricks-openjev-qwen35-4b": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 0.0,
"input_cost_per_token": 0.0,
"litellm_provider": "databricks",
"metadata": {
"notes": "Pay-per-token Foundation Model API decision endpoint (OpenJev, Qwen 3.5 4B) queried through the System One API. Databricks published no DBU rate for it as of 2026-10-07, so a deployment's model_info prices set the billed cost."
},
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models"
},
"dataforseo/search": {
"input_cost_per_query": 0.003,
"litellm_provider": "dataforseo",

1805
litellm/proxy/_lazy_openapi_snapshot.json
View file

File diff suppressed because it is too large Load diff

2
litellm/proxy/auth/auth_utils.py
View file

@ -411,6 +411,8 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
"anthropic_workspace_id",
"anthropic-workspace-id",
"bedrock_tags",
# Deployment-pinned AgentCore session: every new caller-chosen value provisions another microVM.
"agentcore_runtime_session_id",
# Provider-specific endpoint overrides that flow into the outbound
# request via ``optional_params``. Same threat as ``api_base``:
# ``s3_endpoint_url`` redirects Bedrock file uploads to attacker

2
litellm/proxy/common_request_processing.py
View file

@ -182,6 +182,7 @@ ProxyRouteType: TypeAlias = Literal[
"aocr",
"asearch",
"adecisions",
"asystemone",
"avideo_generation",
"avideo_list",
"avideo_status",
@ -1986,6 +1987,7 @@ class ProxyBaseLLMRequestProcessing:
"aocr",
"asearch",
"adecisions",
"asystemone",
"avideo_generation",
"avideo_list",
"avideo_status",

38
litellm/proxy/common_utils/custom_openapi_spec.py
View file

@ -437,3 +437,41 @@ class CustomOpenAPISpec:
# Add responses API request schema
return CustomOpenAPISpec.add_responses_api_request_schema(with_embeddings)
_MAX_INLINE_DEPTH: Final = 32
def _inlined(node: JsonValue, defs: Mapping[str, JsonValue], depth: int = 0) -> JsonValue:
if depth >= _MAX_INLINE_DEPTH or not isinstance(node, (list, dict)):
return node
if isinstance(node, list):
return [_inlined(item, defs, depth + 1) for item in node]
ref: Final = node.get("$ref")
if isinstance(ref, str) and ref.startswith("#/$defs/"):
target: Final = _inlined(defs[ref.removeprefix("#/$defs/")], defs, depth + 1)
siblings: Final = {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$ref"}
return {**target, **siblings} if isinstance(target, dict) else siblings
if "propertyName" in node and "mapping" in node:
return {"propertyName": node["propertyName"]}
return {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$defs"}
def inline_request_body(model_class: type, example: Mapping[str, object]) -> JsonObject:
"""
Build an ``openapi_extra["requestBody"]`` for a route that reads its body with ``request.body()``.
Swagger UI resolves ``$ref`` against the whole document, so a schema placed inline in an operation
cannot keep Pydantic's ``#/$defs/...`` references. Every reference is expanded in place instead,
which also keeps the operation self-contained inside the lazy OpenAPI snapshot.
"""
schema: Final = CustomOpenAPISpec.get_pydantic_schema(model_class)
media: Final[JsonObject] = {"example": cast(JsonValue, dict(example))} # cast-ok: JSON example literal
if schema is None:
return {"required": True, "content": {"application/json": media}}
raw_defs: Final = schema.get("$defs")
defs: Final[Mapping[str, JsonValue]] = raw_defs if isinstance(raw_defs, dict) else MappingProxyType({})
return {
"required": True,
"content": {"application/json": {"schema": _inlined(schema, defs), **media}},
}

34
litellm/proxy/common_utils/registry_read_through.py
View file

@ -7,18 +7,24 @@ immediately can land on a sibling that has never heard of it and fail 400/404.
On a registry miss, callers here fetch the missing row from the DB and load it
into the local registry before giving up. A short negative-result TTL per key
plus a global resync budget per window bound the DB load from lookups of
genuinely unknown names.
genuinely unknown names. An auto router's tiers are deployment rows of their
own, so a miss on an auto router reconciles every row instead of just its own,
or the first routed request on this replica would pick a tier it has not loaded.
"""
import asyncio
import time
from collections.abc import Awaitable, Callable
from collections.abc import Awaitable, Callable, Mapping, Sequence
from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
if TYPE_CHECKING:
from prisma import models as prisma_models
from prisma.types import (
LiteLLM_AgentsTableInclude,
LiteLLM_AgentsTableWhereUniqueInput,
@ -129,6 +135,11 @@ async def _resync_model_deployments(model_name: str) -> bool:
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
)
return proxy_server.llm_router is not None
if _holds_an_auto_router(rows):
await proxy_server.proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
)
return _model_is_loaded(model_name)
async with proxy_server.MODEL_RECONCILE_LOCK:
await proxy_server.proxy_config.get_credentials(prisma_client=prisma_client)
proxy_server.proxy_config._add_deployment(db_models=rows)
@ -136,6 +147,25 @@ async def _resync_model_deployments(model_name: str) -> bool:
return True
def _holds_an_auto_router(rows: Sequence["prisma_models.LiteLLM_ProxyModelTable"]) -> bool:
return any(_names_an_auto_router(row.litellm_params) for row in rows)
_STORED_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
def _names_an_auto_router(stored_litellm_params: object) -> bool:
try:
params: Final = _STORED_LITELLM_PARAMS.validate_python(stored_litellm_params)
except ValidationError:
return False
stored_model: Final = params.get("model")
if not isinstance(stored_model, str):
return False
model: Final = decrypt_value_helper(value=stored_model, key="model", return_original_value=True)
return isinstance(model, str) and model.startswith("auto_router/")
async def _resync_guardrails(guardrail_name: str) -> bool:
from litellm.proxy import proxy_server
from litellm.proxy.guardrails.guardrail_registry import (

103
litellm/proxy/decisions_endpoints/endpoints.py
View file

@ -1,5 +1,6 @@
from collections.abc import Mapping
from typing import Annotated, Final
from types import MappingProxyType
from typing import Annotated, Final, Literal
from fastapi import APIRouter, Depends, Request, Response
from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract
@ -8,7 +9,15 @@ from pydantic import TypeAdapter, ValidationError
from litellm.exceptions import BadRequestError
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody
from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body
from litellm.types.decisions import (
DecisionsRequest,
DecisionsRequestBody,
DecisionsResponse,
OpenAIDecisionRequest,
OpenAIDecisionRequestBody,
OpenAIDecisionResponse,
)
router: Final = APIRouter()
_REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
@ -19,6 +28,51 @@ _OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBo
_GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
_OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
_OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
_SYSTEMONE_REQUEST_BODY: Final = inline_request_body(
DecisionsRequest,
{
"model": "jev",
"state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.",
"questions": {
"is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"},
"urgency": {
"type": "choice",
"instructions": "How urgent is this?",
"criteria": {"low": "can wait a week", "high": "needs action today"},
},
"frustration": {
"type": "score",
"instructions": "How frustrated is the customer?",
"criteria": ["calm", "mildly annoyed", "angry"],
},
},
},
)
_DECISIONS_REQUEST_BODY: Final = inline_request_body(
OpenAIDecisionRequest,
{
"model": "luna",
"input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.",
"questions": [
{"type": "predicate", "name": "is_refund_request", "instructions": "Is the customer asking for a refund?"},
{
"type": "choice",
"name": "urgency",
"instructions": "How urgent is this?",
"choices": [
{"value": "low", "description": "can wait a week"},
{"value": "high", "description": "needs action today"},
],
},
{
"type": "score",
"name": "frustration",
"instructions": "How frustrated is the customer?",
"levels": [{"label": "calm"}, {"label": "mildly annoyed"}, {"label": "angry"}],
},
],
},
)
async def _invalid_request(
@ -38,6 +92,10 @@ async def _invalid_request(
)
def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType({key: value for key, value in data.items() if key != "safety_identifier"})
async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]:
body: Final = await request.body()
try:
@ -51,6 +109,7 @@ async def _process_decisions(
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth,
body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody],
route_type: Literal["adecisions", "asystemone"],
) -> object:
from litellm.proxy.proxy_server import (
general_settings as proxy_general_settings,
@ -75,7 +134,7 @@ async def _process_decisions(
data: Final = await _request_data(request, user_api_key_dict)
try:
body_adapter.validate_python(data)
body_adapter.validate_python(_fields_checked_before_routing(data))
except ValidationError as error:
raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict)
general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings)
@ -88,7 +147,7 @@ async def _process_decisions(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
route_type="adecisions",
route_type=route_type,
proxy_logging_obj=proxy_logging_obj,
llm_router=llm_router,
general_settings=general_settings,
@ -116,23 +175,41 @@ async def _process_decisions(
dependencies=[Depends(user_api_key_auth)],
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
tags=["decisions"],
summary="Ask named questions about a state (System One format)",
responses={200: {"model": DecisionsResponse}},
openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY},
)
@router.post(
"/systemone",
dependencies=[Depends(user_api_key_auth)],
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
tags=["decisions"],
summary="Ask named questions about a state (System One format)",
responses={200: {"model": DecisionsResponse}},
openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY},
)
async def systemone(
request: Request,
fastapi_response: Response,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""
Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,
`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider
is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format
too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.
[Docs](https://docs.litellm.ai/docs/decisions)
"""
return await _process_decisions(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
body_adapter=_DECISIONS_REQUEST_BODY_ADAPTER,
route_type="asystemone",
)
@ -141,21 +218,39 @@ async def systemone(
dependencies=[Depends(user_api_key_auth)],
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
tags=["decisions"],
summary="Ask questions about an input (OpenAI Decisions format)",
responses={200: {"model": OpenAIDecisionResponse}},
openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY},
)
@router.post(
"/decisions",
dependencies=[Depends(user_api_key_auth)],
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
tags=["decisions"],
summary="Ask questions about an input (OpenAI Decisions format)",
responses={200: {"model": OpenAIDecisionResponse}},
openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY},
)
async def decisions(
request: Request,
fastapi_response: Response,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""
Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.
Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with
`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these
rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy
model list. System One decision models accept this format too, LiteLLM translates the request and the
answers. Streaming is not supported.
[Docs](https://docs.litellm.ai/docs/decisions)
"""
return await _process_decisions(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
body_adapter=_OPENAI_DECISION_REQUEST_BODY_ADAPTER,
route_type="adecisions",
)

2
litellm/proxy/guardrails/guardrail_endpoints.py
View file

@ -1496,7 +1496,7 @@ async def get_guardrail_ui_settings() -> GuardrailUIAddGuardrailSettings:
- PII entity categories for UI organization
- Content filter settings (patterns and categories)
"""
from litellm.decisions.main import supported_decisions_providers
from litellm.decisions.call import supported_decisions_providers
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.patterns import (
PATTERN_CATEGORIES,
get_available_content_categories,

2
litellm/proxy/management_helpers/auto_router_permissions.py
View file

@ -76,7 +76,7 @@ class _MemberOpenSourceClassifierConfig(LiteLLMBaseModel):
model_config = ConfigDict(extra="forbid")
provider: Literal["jev", "laya", "bespoke"] = "jev"
provider: Literal["jev", "laya", "bespoke", "databricks"] = "jev"
model: str
api_key: None = None
api_base: None = None

5
litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py
View file

@ -21,6 +21,7 @@ if TYPE_CHECKING:
class _TypeSafeUsage(LiteLLMBaseModel):
input_tokens: int = 0
output_tokens: int = 0
cost: float | None = None
class _TypeSafeResponse(LiteLLMBaseModel):
@ -88,7 +89,9 @@ class TypeSafePassthroughLoggingHandler:
)
pricing: Final = _pricing_for(candidate_model_keys)
response_cost: Final = (
input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
usage.cost
if custom_llm_provider == "openrouter" and usage.cost is not None
else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
)
usage_object: Final = Usage(
prompt_tokens=input_tokens,

2
litellm/proxy/route_llm_request.py
View file

@ -94,6 +94,7 @@ ROUTE_ENDPOINT_MAPPING: Final = {
"aocr": "/ocr",
"asearch": "/search",
"adecisions": "/decisions",
"asystemone": "/systemone",
"avideo_generation": "/videos",
"avideo_list": "/videos",
"avideo_status": "/videos/{video_id}",
@ -519,6 +520,7 @@ RouteType = Literal[
"aocr",
"asearch",
"adecisions",
"asystemone",
"avideo_generation",
"avideo_list",
"avideo_status",

9
litellm/router.py
View file

@ -2232,6 +2232,11 @@ class Router:
self.adecisions = self.factory_function(adecisions, call_type="adecisions")
self.decisions = self.factory_function(decisions, call_type="decisions")
from litellm.systemone import asystemone, systemone
self.asystemone = self.factory_function(asystemone, call_type="asystemone") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable
self.systemone = self.factory_function(systemone, call_type="systemone") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable
def _initialize_video_endpoints(self):
"""Initialize video endpoints."""
from litellm.videos import (
@ -6972,6 +6977,8 @@ class Router:
"search",
"adecisions",
"decisions",
"asystemone",
"systemone",
"aadapter_generate_content",
"avideo_generation",
"video_generation",
@ -7046,6 +7053,7 @@ class Router:
"ocr",
"search",
"decisions",
"systemone",
"video_generation",
"video_list",
"video_status",
@ -7214,6 +7222,7 @@ class Router:
"aocr",
"ocr",
"adecisions",
"asystemone",
"avideo_generation",
"avideo_list",
"avideo_status",

9
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -59,6 +59,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in
from litellm.llms.anthropic.common_utils import is_claude_code_user_agent
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
from litellm.router_strategy.complexity_router.tier_predictor import (
@ -1342,6 +1343,14 @@ class ComplexityRouter(CustomLogger):
http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint),
provider=config.provider,
)
if config.provider == "databricks":
databricks: Final = DATABRICKS_DECISIONS_CONFIG.connection(config.api_base, config.api_key)
return HttpJevClassifierClient(
api_key=databricks.api_key,
api_base=databricks.api_base,
http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint),
provider="databricks",
)
api_key: Final = config.api_key or get_secret_str("TYPESAFE_API_KEY")
if not api_key:
raise ValueError(

23
litellm/router_strategy/complexity_router/config.py
View file

@ -705,10 +705,18 @@ def normalize_classifier_config_aliases(config: Mapping[str, object]) -> Mapping
return normalized
_ENVIRONMENT_KEY_SCOPE: Final = MappingProxyType(
{
"jev": "TYPESAFE_API_KEY is only sent to TYPESAFE_API_BASE or https://api.typesafe.ai",
"databricks": "DATABRICKS_API_KEY or DATABRICKS_TOKEN is only sent to DATABRICKS_API_BASE",
}
)
class OpenSourceClassifierConfig(LiteLLMBaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
provider: Literal["jev", "laya", "bespoke"] = "jev"
provider: Literal["jev", "laya", "bespoke", "databricks"] = "jev"
model: str = "jev-latest"
api_key: str | None = Field(default=None, description="Provider API key; optional for self-hosted providers")
api_base: str | None = Field(
@ -753,10 +761,19 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel):
if self.api_base is not None:
_ = validate_oss_api_base(self.provider, self.api_base)
return self
if self.provider == "databricks":
from litellm.llms.databricks.decisions.transformation import validate_serving_endpoint_name
if "model" not in self.model_fields_set:
raise ValueError(
"opensource_classifier_config.model is required for provider 'databricks': the serving endpoint "
"name, e.g. databricks-openjev-qwen35-4b"
)
_ = validate_serving_endpoint_name(self.model)
if self.api_base is not None and self.api_key is None:
raise ValueError(
"opensource_classifier_config.api_base requires opensource_classifier_config.api_key: TYPESAFE_API_KEY is only sent "
"to TYPESAFE_API_BASE or https://api.typesafe.ai"
"opensource_classifier_config.api_base requires opensource_classifier_config.api_key: "
f"{_ENVIRONMENT_KEY_SCOPE[self.provider]}"
)
return self

36
litellm/router_strategy/complexity_router/jev_classifier.py
View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.internal_call_metadata import (
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG
from litellm.llms.laya.common_utils import laya_response_model
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.typesafe_passthrough_logging_handler import (
TypeSafePassthroughLoggingHandler,
@ -27,6 +28,7 @@ from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN
JevProbability: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)]
ClassifierProvider: TypeAlias = Literal["typesafe", "laya", "bespoke", "databricks"]
DEFAULT_JEV_INSTRUCTIONS: Final = _DEFAULT_JEV_INSTRUCTIONS
@ -85,12 +87,12 @@ class HttpJevClassifierClient:
api_key: str | None,
api_base: str,
http_client: AsyncHTTPHandler,
provider: Literal["typesafe", "laya", "bespoke"] = "typesafe",
provider: ClassifierProvider = "typesafe",
) -> None:
self._api_key = api_key
self._api_base = api_base.rstrip("/")
self._http_client = http_client
self._provider = provider
self._provider: ClassifierProvider = provider
async def evaluate(
self,
@ -103,33 +105,43 @@ class HttpJevClassifierClient:
MappingProxyType({"Authorization": f"Bearer {self._api_key}"}) if self._api_key else MappingProxyType({})
)
response: Final = await self._http_client.post( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler has a dynamic post signature
f"{self._api_base}/v1/systemone",
self._request_url(request.model),
json=request.model_dump(mode="json"),
headers=MappingProxyType({**authorization, "Content-Type": "application/json"}), # pyright: ignore[reportArgumentType] # HTTP headers are not mutated by AsyncHTTPHandler
timeout=timeout_s,
)
response.raise_for_status()
body: Final = TypeAdapter(dict[str, object]).validate_json(response.content)
normalized_body: Final = (
MappingProxyType({**body, "model": laya_response_model(body, request.model)})
if self._provider == "laya"
else body
)
normalized_body: Final = self._normalized_body(body, request.model)
try:
self._log_response(request, response, request_kwargs, start_time)
self._log_response(request, response, normalized_body, request_kwargs, start_time)
except Exception as exc: # noqa: BLE001 # logging integrations must not discard a provider verdict
verbose_router_logger.warning("JEV response logging failed (%s)", type(exc).__name__)
return TypeAdapter(JevSystemOneResponse).validate_python(normalized_body)
def _request_url(self, model: str) -> str:
if self._provider == "databricks":
return DATABRICKS_DECISIONS_CONFIG.get_complete_url(self._api_base, model)
return f"{self._api_base}/v1/systemone"
def _normalized_body(self, body: Mapping[str, object], requested_model: str) -> Mapping[str, object]:
match self._provider:
case "laya":
return MappingProxyType({**body, "model": laya_response_model(body, requested_model)})
case "databricks":
return DATABRICKS_DECISIONS_CONFIG.classifier_response(body, requested_model)
case "typesafe" | "bespoke":
return body
def _log_response(
self,
request: JevSystemOneRequest,
response: httpx.Response,
body: Mapping[str, object],
request_kwargs: Mapping[str, object] | None,
start_time: datetime,
) -> None:
try:
body: Final = TypeAdapter(dict[str, object]).validate_json(response.content)
_ = TypeAdapter(JevUsage | None).validate_python(body.get("usage"))
except ValidationError:
return
@ -202,7 +214,7 @@ class JevVerdict(NamedTuple):
confidence: float
model: str
cost: float | None
provider: Literal["typesafe", "laya", "bespoke"] = "typesafe"
provider: ClassifierProvider = "typesafe"
class _RegistryPricing(LiteLLMBaseModel):
@ -226,7 +238,7 @@ def build_jev_request(
def jev_classifier_cost(
response: JevSystemOneResponse, configured_model: str, provider: Literal["typesafe", "laya", "bespoke"] = "typesafe"
response: JevSystemOneResponse, configured_model: str, provider: ClassifierProvider = "typesafe"
) -> float | None:
usage: Final = response.usage
if usage is None:

3
litellm/systemone/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from litellm.systemone.main import asystemone, systemone
__all__ = ["asystemone", "systemone"]

107
litellm/systemone/main.py Normal file
View file

@ -0,0 +1,107 @@
from collections.abc import Mapping
from typing import Final
import httpx
from litellm.decisions.call import (
DecisionsCall,
SystemOneQuestions,
SystemOneRequestFields,
asend,
prepare_call,
reject_other_format,
send,
)
from litellm.types.decisions import DecisionsJSON, DecisionsResponse
from litellm.utils import client # pyright: ignore[reportUnknownVariableType] # client is an untyped decorator
def _prepare_systemone_call(
*,
model: str,
state: DecisionsJSON | None,
questions: SystemOneQuestions | None,
safety_identifier: str | None,
api_key: str | None,
api_base: str | None,
timeout: float | httpx.Timeout | None,
custom_llm_provider: str | None,
extra_headers: Mapping[str, str] | None,
kwargs: Mapping[str, object],
) -> DecisionsCall:
reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions")
return prepare_call(
model=model,
fields=SystemOneRequestFields(state=state, questions=questions),
safety_identifier=safety_identifier,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
def _systemone_response(response: object) -> DecisionsResponse:
assert isinstance(response, DecisionsResponse)
return response
@client
async def asystemone(
model: str,
state: DecisionsJSON | None = None,
questions: SystemOneQuestions | None = None,
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: Mapping[str, str] | None = None,
safety_identifier: str | None = None,
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
) -> DecisionsResponse:
call: Final = _prepare_systemone_call(
model=model,
state=state,
questions=questions,
safety_identifier=safety_identifier,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
return _systemone_response(await asend(call))
@client
def systemone(
model: str,
state: DecisionsJSON | None = None,
questions: SystemOneQuestions | None = None,
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: Mapping[str, str] | None = None,
safety_identifier: str | None = None,
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
) -> DecisionsResponse:
call: Final = _prepare_systemone_call(
model=model,
state=state,
questions=questions,
safety_identifier=safety_identifier,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
return _systemone_response(send(call))
__all__ = ["asystemone", "systemone"]

86
litellm/types/decisions.py
View file

@ -15,8 +15,13 @@ MAX_DECISION_QUESTIONS: Final = 128
class NoulQuestion(LiteLLMPydanticObjectBase):
type: Literal["noul"]
instructions: DecisionsJSON | None = None
criteria: NoulCriteria | None = None
instructions: Annotated[DecisionsJSON | None, Field(description="The yes/no question to ask about the state")] = (
None
)
criteria: Annotated[
NoulCriteria | None,
Field(description="Optional descriptions of what makes the answer true and what makes it false"),
] = None
model_config = ConfigDict(extra="allow", frozen=True)
@ -29,16 +34,24 @@ class NoulQuestion(LiteLLMPydanticObjectBase):
class ChoiceQuestion(LiteLLMPydanticObjectBase):
type: Literal["choice"]
instructions: DecisionsJSON | None = None
criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)]
instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None
criteria: Annotated[
Mapping[str, DecisionsJSON | None],
Field(min_length=1, max_length=255, description="Candidate labels mapped to an optional description of each"),
]
model_config = ConfigDict(extra="allow", frozen=True)
class ScoreQuestion(LiteLLMPydanticObjectBase):
type: Literal["score"]
instructions: DecisionsJSON | None = None
criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)]
instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None
criteria: Annotated[
Sequence[DecisionsJSON],
Field(
min_length=1, max_length=10, description="Scale levels from lowest to highest. The score is a level index"
),
]
model_config = ConfigDict(extra="allow", frozen=True)
@ -55,14 +68,20 @@ DecisionQuestionMap: TypeAlias = Annotated[
class DecisionsRequestBody(LiteLLMPydanticObjectBase):
state: DecisionsJSON
questions: DecisionQuestionMap
state: Annotated[
DecisionsJSON,
Field(description="The thing being judged, such as a support ticket, a document or a chat transcript"),
]
questions: Annotated[
DecisionQuestionMap,
Field(description="Named noul, choice and score questions. Each key becomes a key in the answers"),
]
model_config = ConfigDict(extra="allow", frozen=True)
class DecisionsRequest(DecisionsRequestBody):
model: str
model: Annotated[str, Field(description="A decision model from the proxy model_list")]
@with_config(ConfigDict(extra="allow"))
@ -171,8 +190,8 @@ OpenAIDecisionInput: TypeAlias = str | Sequence[OpenAIDecisionInputMessage]
class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase):
type: Literal["predicate"]
name: str | None = None
instructions: str
name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None
instructions: Annotated[str, Field(description="The yes/no question to ask about the input")]
model_config = ConfigDict(extra="forbid", frozen=True)
@ -192,9 +211,12 @@ def systemone_choice_key(value: str | bool) -> str:
class OpenAIChoiceQuestion(LiteLLMPydanticObjectBase):
type: Literal["choice"]
name: str | None = None
instructions: str
choices: Annotated[Sequence[OpenAIChoiceOption], Field(min_length=2, max_length=255)]
name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None
instructions: Annotated[str, Field(description="The question to ask about the input")]
choices: Annotated[
Sequence[OpenAIChoiceOption],
Field(min_length=2, max_length=255, description="The options the model picks from. Values must be unique"),
]
model_config = ConfigDict(extra="forbid", frozen=True)
@ -215,9 +237,14 @@ class OpenAIScoreLevel(LiteLLMPydanticObjectBase):
class OpenAIScoreQuestion(LiteLLMPydanticObjectBase):
type: Literal["score"]
name: str | None = None
instructions: str
levels: Annotated[Sequence[OpenAIScoreLevel], Field(min_length=2, max_length=10)]
name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None
instructions: Annotated[str, Field(description="The question to ask about the input")]
levels: Annotated[
Sequence[OpenAIScoreLevel],
Field(
min_length=2, max_length=10, description="Scale levels from lowest to highest. The score is a level index"
),
]
model_config = ConfigDict(extra="forbid", frozen=True)
@ -229,13 +256,32 @@ OpenAIDecisionQuestion: TypeAlias = Annotated[
class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase):
input: OpenAIDecisionInput
questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)]
safety_identifier: str | None = None
input: Annotated[
OpenAIDecisionInput,
Field(
description="The text being judged, or user messages whose content mixes input_text and input_image parts"
),
]
questions: Annotated[
Sequence[OpenAIDecisionQuestion],
Field(
min_length=1,
max_length=MAX_DECISION_QUESTIONS,
description="Predicate, choice and score questions. Answers come back in the same order",
),
]
safety_identifier: Annotated[
str | None,
Field(description="A stable id for your end user. Sent to OpenAI and dropped for other providers"),
] = None
model_config = ConfigDict(extra="allow", frozen=True)
class OpenAIDecisionRequest(OpenAIDecisionRequestBody):
model: Annotated[str, Field(description="A decision model from the proxy model_list")]
class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase):
type: Literal["predicate"] = "predicate"
name: str | None

8
litellm/types/utils.py
View file

@ -489,6 +489,8 @@ class CallTypes(str, Enum):
asearch = "asearch"
decisions = "decisions"
adecisions = "adecisions"
systemone = "systemone"
asystemone = "asystemone"
arealtime = "_arealtime"
aresponses_websocket = "_aresponses_websocket"
create_batch = "create_batch"
@ -677,6 +679,8 @@ CallTypesLiteral = Literal[
"asearch",
"decisions",
"adecisions",
"systemone",
"asystemone",
"_arealtime",
"_aresponses_websocket",
"create_batch",
@ -788,8 +792,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
"/v1/search": [CallTypes.asearch, CallTypes.search],
"/decisions": [CallTypes.adecisions, CallTypes.decisions],
"/v1/decisions": [CallTypes.adecisions, CallTypes.decisions],
"/systemone": [CallTypes.adecisions, CallTypes.decisions],
"/v1/systemone": [CallTypes.adecisions, CallTypes.decisions],
"/systemone": [CallTypes.asystemone, CallTypes.systemone],
"/v1/systemone": [CallTypes.asystemone, CallTypes.systemone],
# Batches
"/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
"/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch],

12
litellm/utils.py
View file

@ -974,6 +974,12 @@ def _ocr_document_summary(document: object) -> str:
# Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc.
_DECISION_CALL_TYPES: Final[frozenset[str]] = frozenset(
(CallTypes.decisions.value, CallTypes.adecisions.value, CallTypes.systemone.value, CallTypes.asystemone.value)
)
def function_setup(
original_function: str,
rules_obj: Rules,
@ -1220,7 +1226,7 @@ def function_setup(
if isinstance(search_query, list)
else search_query
)
elif call_type in (CallTypes.decisions.value, CallTypes.adecisions.value):
elif call_type in _DECISION_CALL_TYPES:
decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state") or kwargs.get("input") or ""
messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state, default=str)
elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value):
@ -8989,6 +8995,10 @@ class ProviderConfigManager:
return litellm.CloudflareDecisionsConfig()
if provider == LlmProviders.STRANDS_DECIDER:
return litellm.StrandsDeciderDecisionsConfig()
if provider == LlmProviders.DATABRICKS:
return litellm.DatabricksDecisionsConfig()
if provider == LlmProviders.HOSTED_VLLM:
return litellm.HostedVLLMDecisionsConfig()
if provider == LlmProviders.OPENAI:
return litellm.OpenAIDecisionsConfig()
if provider == LlmProviders.AZURE_AI:

12
model_prices_and_context_window.json
View file

@ -21165,6 +21165,18 @@
"output_vector_size": 1024,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
"databricks/databricks-openjev-qwen35-4b": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 0.0,
"input_cost_per_token": 0.0,
"litellm_provider": "databricks",
"metadata": {
"notes": "Pay-per-token Foundation Model API decision endpoint (OpenJev, Qwen 3.5 4B) queried through the System One API. Databricks published no DBU rate for it as of 2026-10-07, so a deployment's model_info prices set the billed cost."
},
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models"
},
"dataforseo/search": {
"input_cost_per_query": 0.003,
"litellm_provider": "dataforseo",

3
provider_endpoints_support.json
View file

@ -1370,7 +1370,8 @@
"rerank": true,
"a2a": true,
"interactions": true,
"video_generations": true
"video_generations": true,
"systemone": true
}
},
"huggingface": {

1
tests/code_coverage_tests/recursive_detector.py
View file

@ -58,6 +58,7 @@ IGNORE_FUNCTIONS = [
"sanitize_oci_schema", # OCI: bounded by JSON-schema tree depth.
"_freeze_for_dedupe", # OTEL: max depth set (default 16, _FREEZE_MAX_DEPTH); fails closed by returning repr(value) at the cap.
"apply_json_merge_patch", # max depth set (_MAX_MERGE_DEPTH=64); fails closed by raising ValueError at the cap.
"_inlined", # max depth set (_MAX_INLINE_DEPTH=32); passes the schema node through untouched at the cap. Walks a Pydantic JSON schema once at import time.
"_filter_argument_value", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by blocking the tool call at the cap.
"_redact_scanned_content", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by returning "[REDACTED]" at the cap.
"replace_ciphertexts", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); walks stored JSON, which has no cycles, and leaves values below the cap untouched.

46
tests/integration/_support/forward_proxy.py
View file

@ -1,5 +1,7 @@
from __future__ import annotations
import contextlib
import socket
import threading
from collections.abc import Generator, Mapping
from contextlib import contextmanager
@ -69,3 +71,47 @@ def refusing_forward_proxy(port: int = 0) -> Generator[ForwardProxy, None, None]
thread.join(timeout=6)
assert not thread.is_alive(), "Owned forward proxy survived cleanup"
server.server_close()
def _pipe(source: socket.socket, sink: socket.socket) -> None:
with contextlib.suppress(OSError):
for chunk in iter(lambda: source.recv(65536), b""):
sink.sendall(chunk)
with contextlib.suppress(OSError):
sink.shutdown(socket.SHUT_WR)
@contextmanager
def tunnelling_forward_proxy(destination_port: int, port: int = 0) -> Generator[ForwardProxy, None, None]:
received: Final[SimpleQueue[Tunnel]] = SimpleQueue()
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
rbufsize = 0
timeout = 30
request: socket.socket
def do_CONNECT(self) -> None:
received.put(Tunnel(self.command, self.path, {name.lower(): value for name, value in self.headers.items()}))
self.close_connection = True
with socket.create_connection(("127.0.0.1", destination_port), timeout=30) as destination:
self.send_response(200, "Connection established")
self.end_headers()
outbound: Final = threading.Thread(target=_pipe, args=(self.request, destination))
outbound.start()
_pipe(destination, self.request)
outbound.join(timeout=35)
def log_message(self, format: str, *args: object) -> None:
pass
with ThreadingHTTPServer(("127.0.0.1", port), Handler) as server:
thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05})
thread.start()
try:
yield ForwardProxy(f"http://127.0.0.1:{server.server_port}", server.server_port, received)
finally:
server.shutdown()
thread.join(timeout=6)
assert not thread.is_alive(), "Owned forward proxy survived cleanup"
server.server_close()

334
tests/integration/management/test_auto_router_databricks_classifier_writes.py Normal file
View file

@ -0,0 +1,334 @@
import asyncio
import json
import uuid
from collections.abc import Callable, Mapping
from typing import Final
import httpx
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.openai_wire import answering_model_discovery, chat_reply
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue
_ENDPOINT: Final = "databricks-openjev-qwen35-4b"
_SECOND_ENDPOINT: Final = "databricks-openjev-qwen35-4b-canary"
_API_KEY: Final = "synthetic-databricks-key"
_BARE_NAME_RULE: Final = (
"must be the bare endpoint name (letters, digits, '-', '_' and '.', with no '/', '?', '#', spaces, or a "
"leading '.'), e.g. databricks-openjev-qwen35-4b"
)
_REJECTED_AT_LOAD: Final = "The router would drop this deployment at load time, so the write is rejected instead."
_MODEL_REQUIRED: Final = (
"opensource_classifier_config.model is required for provider 'databricks': the serving endpoint name, "
"e.g. databricks-openjev-qwen35-4b"
)
_BASE_NEEDS_KEY: Final = (
"opensource_classifier_config.api_base requires opensource_classifier_config.api_key: DATABRICKS_API_KEY or "
"DATABRICKS_TOKEN is only sent to DATABRICKS_API_BASE"
)
_BLANK_KEY: Final = (
"opensource_classifier_config.api_key must be non-empty; omit it to use the provider environment key"
)
_CLASSIFIER_ANSWER: Final[dict[str, JsonValue]] = {
"answers": {
"tier": {
"type": "choice",
"choice": "COMPLEX",
"confidence": 0.91,
"probabilities": {"SIMPLE": 0.03, "MEDIUM": 0.06, "COMPLEX": 0.91},
}
},
"usage": {"input_tokens": 12, "output_tokens": 1},
}
_ROWS_QUERY: Final = (
"SELECT request_id, status, metadata->>'internal_call_origin' AS origin "
'FROM "LiteLLM_SpendLogs" WHERE model_group = %s'
)
def _classifier(request: Request) -> Reply:
return Reply(body=json.dumps(_CLASSIFIER_ANSWER).encode())
def _tier(text: str) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
body: Final = json.loads(request.body)
return chat_reply(f"{text}-{uuid.uuid4().hex[:8]}", "tier-model", text, stream=bool(body.get("stream")))
return answering_model_discovery(respond)
def _classifier_config(api_base: str, **overrides: JsonValue) -> dict[str, JsonValue]:
return {
"provider": "databricks",
"model": _ENDPOINT,
"api_base": api_base,
"api_key": _API_KEY,
"timeout_ms": 20000,
"circuit_breaker_enabled": False,
**overrides,
}
def _without(config: Mapping[str, JsonValue], field: str) -> dict[str, JsonValue]:
return {name: value for name, value in config.items() if name != field}
def _router_config(classifier: Mapping[str, JsonValue], simple: str, complex_: str) -> dict[str, JsonValue]:
return {
"classifier_type": "oss_classifier",
"opensource_classifier_config": dict(classifier),
"tiers": {"SIMPLE": simple, "MEDIUM": simple, "COMPLEX": complex_, "REASONING": complex_},
}
def _validate(gateway: Gateway, config: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
response: Final = gateway.request(
"POST", "/auto_router/validate_complexity_router_config", {"complexity_router_config": dict(config)}
)
assert response.status_code == 200, response.text
return object_value(response.json())
def _create_router(scenario: Scenario, config: Mapping[str, JsonValue]) -> tuple[str, str]:
name: Final = f"router-{uuid.uuid4().hex[:10]}"
created: Final = scenario.gateway.post(
"/model/new",
{
"model_name": name,
"litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": dict(config)},
"model_info": {},
},
)
identity: Final = string_value(object_value(created["model_info"])["id"])
scenario.cleanups.callback(scenario.delete_model, identity)
return name, identity
def _stored_classifier(gateway: Gateway, identity: str) -> dict[str, JsonValue]:
stored: Final = gateway.request("GET", "/model/info", params={"litellm_model_id": identity})
assert stored.status_code == 200, stored.text
(entry,) = stored.json()["data"]
router_config: Final = object_value(object_value(object_value(entry)["litellm_params"])["complexity_router_config"])
return object_value(router_config["opensource_classifier_config"])
def _judge_targets(judge: Wire) -> tuple[str, ...]:
return tuple(request.target for request in judge.drain() if request.method == "POST")
def _chat_body(model: str) -> dict[str, JsonValue]:
return {"model": model, "messages": [{"role": "user", "content": f"design a distributed cache {uuid.uuid4().hex}"}]}
async def _burst(base_url: str, key: str, model: str, count: int) -> tuple[httpx.Response, ...]:
async with httpx.AsyncClient(
base_url=base_url, timeout=60, trust_env=False, headers={"Authorization": f"Bearer {key}"}
) as client:
return tuple(
await asyncio.gather(*(client.post("/v1/chat/completions", json=_chat_body(model)) for _ in range(count)))
)
def _rows(router: str, *, want: int) -> tuple[dict[str, JsonValue], ...]:
return tuple(eventually(lambda: read_rows(_ROWS_QUERY, (router,)), lambda found: len(found) >= want, seconds=70))
def test_validate_accepts_a_databricks_classifier_with_a_bare_endpoint_name_of_any_length(gateway: Gateway) -> None:
config: Final = _router_config(
_classifier_config("https://dbc-example.cloud.databricks.com/serving-endpoints"), "simple", "complex"
)
assert _validate(gateway, config) == {"valid": True, "error": None}
long_name: Final = _router_config(
_classifier_config("https://dbc-example.cloud.databricks.com/serving-endpoints", model="e" * 5000),
"simple",
"complex",
)
assert _validate(gateway, long_name) == {"valid": True, "error": None}
def test_validate_names_each_databricks_classifier_misconfiguration(gateway: Gateway) -> None:
base: Final = _classifier_config("https://dbc-example.cloud.databricks.com/serving-endpoints")
cases: Final[tuple[tuple[str, dict[str, JsonValue], str], ...]] = (
("model omitted", _without(base, "model"), f"at opensource_classifier_config: Value error, {_MODEL_REQUIRED}"),
(
"api_base without api_key",
_without(base, "api_key"),
f"at opensource_classifier_config: Value error, {_BASE_NEEDS_KEY}",
),
("slash in the name", {**base, "model": "a/b"}, f"Databricks serving endpoint name 'a/b' {_BARE_NAME_RULE}"),
("space in the name", {**base, "model": "a b"}, f"Databricks serving endpoint name 'a b' {_BARE_NAME_RULE}"),
("leading dot", {**base, "model": ".hidden"}, f"Databricks serving endpoint name '.hidden' {_BARE_NAME_RULE}"),
("empty name", {**base, "model": ""}, f"Databricks serving endpoint name '' {_BARE_NAME_RULE}"),
(
"blank api_key",
{**base, "api_key": ""},
f"at opensource_classifier_config.api_key: Value error, {_BLANK_KEY}",
),
("int name", {**base, "model": 5}, "at opensource_classifier_config.model: Input should be a valid string"),
(
"list name",
{**base, "model": ["a"]},
"at opensource_classifier_config.model: Input should be a valid string",
),
("null name", {**base, "model": None}, "at opensource_classifier_config.model: Input should be a valid string"),
)
for label, classifier, expected in cases:
verdict: Final = _validate(gateway, _router_config(classifier, "simple", "complex"))
assert verdict["valid"] is False, (label, verdict)
error: Final = string_value(verdict["error"])
assert error.startswith("complexity_router_config is invalid "), (label, error)
assert expected in error, (label, error)
assert error.endswith(_REJECTED_AT_LOAD), (label, error)
def test_test_routing_classifies_through_the_databricks_endpoint_without_calling_the_tier_it_picked(
gateway: Gateway,
) -> None:
prompt: Final = f"design a distributed cache {uuid.uuid4().hex}"
with (
wire_server(_classifier) as judge,
wire_server(_tier("simple answer")) as simple,
wire_server(_tier("complex answer")) as complex_tier,
gateway.scenario() as scenario,
):
simple_model: Final = scenario.model(model="openai/simple-tier", api_base=simple.url)
complex_model: Final = scenario.model(model="openai/complex-tier", api_base=complex_tier.url)
config: Final = _router_config(_classifier_config(judge.url), simple_model, complex_model)
response: Final = gateway.request(
"POST", "/auto_router/test_routing", {"prompt": prompt, "complexity_router_config": config}
)
assert response.status_code == 200, response.text
assert response.json() == {
"routed_model": complex_model,
"routed_model_configured": True,
"routing_decision": {
"router_model_name": "auto_router_routing_test",
"router_type": "complexity",
"routed_model": complex_model,
"cause": "jev_classifier",
"tier": "COMPLEX",
"signals": [
"databricks-classifier:COMPLEX",
"databricks-confidence=0.910000",
"tier-probability:SIMPLE=0.030000",
"tier-probability:MEDIUM=0.060000",
"tier-probability:COMPLEX=0.910000",
],
"classifier_model": f"databricks/{_ENDPOINT}",
"classifier_cost": 0.0,
"classifier_probabilities": {"SIMPLE": 0.03, "MEDIUM": 0.06, "COMPLEX": 0.91},
"classifier_confidence": 0.91,
"conversation_continuing": False,
},
}, response.text
(call,) = [request for request in judge.drain() if request.method == "POST"]
assert call.target == f"/{_ENDPOINT}/invocations", call.target
assert call.headers.get("authorization") == f"Bearer {_API_KEY}", call.headers
body: Final = json.loads(call.body)
assert body["model"] == _ENDPOINT, body
assert body["state"] == f"\nClassify this message:\n{prompt}", body
assert _judge_targets(simple) == () and _judge_targets(complex_tier) == ()
def test_model_new_refuses_a_databricks_classifier_without_an_endpoint_name_and_stores_nothing(
gateway: Gateway,
) -> None:
name: Final = f"router-{uuid.uuid4().hex[:10]}"
config: Final = _router_config(
_without(_classifier_config("https://dbc-example.cloud.databricks.com/serving-endpoints"), "model"),
"simple",
"complex",
)
response: Final = gateway.request(
"POST",
"/model/new",
{
"model_name": name,
"litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": config},
"model_info": {},
},
)
assert response.status_code == 400, response.text
error: Final = object_value(object_value(response.json())["error"])
assert _MODEL_REQUIRED in string_value(error["message"]), error
assert (error["type"], error["param"]) == ("validation_error", "litellm_params.model"), error
listed: Final = gateway.get("/model/info")["data"]
assert isinstance(listed, list)
assert [entry for entry in map(object_value, listed) if entry["model_name"] == name] == []
def test_model_update_refuses_dropping_the_endpoint_name_and_keeps_the_stored_classifier_serving(
gateway: Gateway,
) -> None:
with (
wire_server(_classifier) as judge,
wire_server(_tier("simple answer")) as simple,
wire_server(_tier("complex answer")) as complex_tier,
gateway.scenario() as scenario,
):
simple_model: Final = scenario.model(model="openai/simple-tier", api_base=simple.url)
complex_model: Final = scenario.model(model="openai/complex-tier", api_base=complex_tier.url)
classifier: Final = _classifier_config(judge.url)
name, identity = _create_router(scenario, _router_config(classifier, simple_model, complex_model))
dropped: Final = _router_config(_without(classifier, "model"), simple_model, complex_model)
response: Final = gateway.request(
"PATCH",
f"/model/{identity}/update",
{"litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": dropped}},
)
assert response.status_code == 400, response.text
assert _MODEL_REQUIRED in string_value(object_value(object_value(response.json())["error"])["message"]), (
response.text
)
assert _stored_classifier(gateway, identity)["model"] == _ENDPOINT
chat: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(name))
assert chat.status_code == 200, chat.text
assert chat.json()["choices"][0]["message"]["content"] == "complex answer", chat.text
assert chat.headers["x-litellm-complexity-router-cause"] == "jev_classifier", dict(chat.headers)
assert _judge_targets(judge) == (f"/{_ENDPOINT}/invocations",)
async def test_renaming_the_endpoint_under_a_burst_moves_the_classifier_and_logs_every_request_once(
gateway: Gateway,
) -> None:
base_url: Final = str(gateway.client.base_url)
with (
wire_server(_classifier) as judge,
wire_server(_tier("simple answer")) as simple,
wire_server(_tier("complex answer")) as complex_tier,
gateway.scenario() as scenario,
):
simple_model: Final = scenario.model(model="openai/simple-tier", api_base=simple.url)
complex_model: Final = scenario.model(model="openai/complex-tier", api_base=complex_tier.url)
classifier: Final = _classifier_config(judge.url)
name, identity = _create_router(scenario, _router_config(classifier, simple_model, complex_model))
burst: Final = asyncio.create_task(_burst(base_url, gateway.key, name, 24))
renamed: Final = _router_config({**classifier, "model": _SECOND_ENDPOINT}, simple_model, complex_model)
patched: Final = gateway.request(
"PATCH",
f"/model/{identity}/update",
{"litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": renamed}},
)
assert patched.status_code == 200, patched.text
during: Final = await burst
assert [response.status_code for response in during] == [200] * 24, [response.text for response in during]
assert set(_judge_targets(judge)) <= {f"/{_ENDPOINT}/invocations", f"/{_SECOND_ENDPOINT}/invocations"}
assert _stored_classifier(gateway, identity)["model"] == _SECOND_ENDPOINT
def classifier_target() -> tuple[str, ...]:
chat: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(name))
assert chat.status_code == 200, chat.text
return _judge_targets(judge)
eventually(classifier_target, lambda targets: targets == (f"/{_SECOND_ENDPOINT}/invocations",), seconds=30)
after: Final = await _burst(base_url, gateway.key, name, 6)
assert [response.status_code for response in after] == [200] * 6, [response.text for response in after]
assert set(_judge_targets(judge)) == {f"/{_SECOND_ENDPOINT}/invocations"}
identities: Final = tuple(string_value(response.json()["id"]) for response in (*during, *after))
assert len(set(identities)) == len(identities), identities
rows: Final = _rows(name, want=len(identities))
request_rows: Final = [row for row in rows if row["origin"] != "autorouter_classifier"]
assert sorted(string_value(row["request_id"]) for row in request_rows) == sorted(identities), request_rows
assert all(row["status"] == "success" for row in rows), rows

79
tests/integration/management/test_model_health_check.py
View file

@ -24,6 +24,16 @@ _CONFIGURED_PROBE_REPLY: Final = JsonResponse(
"usage": {"input_tokens": 10, "output_tokens": 1},
},
)
_VLLM_PROBE_REPLY: Final = JsonResponse(
content_type="application/json",
body={
"model": "Qwen/Qwen3-0.6B",
"answers": {
"reachable": {"type": "choice", "choice": "yes", "confidence": 1.0, "probabilities": {"yes": 1.0, "no": 0.0}}
},
"usage": {"input_tokens": 10, "output_tokens": 1},
},
)
_STRANDS_PROBE_REPLY: Final = JsonResponse(
content_type="application/json",
body={
@ -147,3 +157,72 @@ def test_evaluation_mode_health_check_of_the_self_hosted_strands_model_resolves_
},
)
]
def test_evaluation_mode_health_check_of_hosted_vllm_sends_a_choice_probe(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _VLLM_PROBE_REPLY)
scenario.cleanups.callback(delete_scenario, handle)
model: Final = scenario.model(
model="hosted_vllm/Qwen/Qwen3-0.6B",
api_base=handle.api_base(),
model_info={"mode": "evaluation"},
)
report: Final = _health_report(gateway, model)
assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report
assert _probes_sent_to(gateway, handle) == [
(
f"/{handle.scenario_id}/v1/systemone",
{
"model": "Qwen/Qwen3-0.6B",
"state": os.environ.get("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"),
"questions": {
"reachable": {
"type": "choice",
"instructions": "Is the service reachable?",
"criteria": {"yes": None, "no": None},
}
},
},
)
]
_DATABRICKS_PROBE_REPLY: Final = JsonResponse(
content_type="application/json",
body={
"model": "databricks-openjev-qwen35-4b",
"answers": {"reachable": {"type": "noul", "noul": 1.0}},
"usage": {"input_tokens": 10, "output_tokens": 1},
},
)
def test_evaluation_mode_health_check_of_a_databricks_serving_endpoint_resolves_the_mode_from_the_cost_map(
gateway: Gateway,
) -> None:
with gateway.scenario() as scenario:
handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _DATABRICKS_PROBE_REPLY)
scenario.cleanups.callback(delete_scenario, handle)
model: Final = scenario.model(
model="databricks/databricks-openjev-qwen35-4b",
api_base=handle.api_base(),
api_key="synthetic-databricks-key",
)
listed: Final = gateway.request("GET", "/v2/model/info", params={"model": model})
assert listed.status_code == 200, listed.text
assert [object_value(object_value(entry)["model_info"])["mode"] for entry in listed.json()["data"]] == [
"evaluation"
], listed.text
report: Final = _health_report(gateway, model)
assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report
assert _probes_sent_to(gateway, handle) == [
(
f"/{handle.scenario_id}/databricks-openjev-qwen35-4b/invocations",
{
"model": "databricks-openjev-qwen35-4b",
"state": os.environ.get("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"),
"questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}},
},
)
]

518
tests/integration/providers/test_databricks_decisions_wire.py Normal file
View file

@ -0,0 +1,518 @@
import threading
import time
import uuid
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Final
import httpx
import pytest
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.process import graceful_stop_seconds, owned_proxy_process
from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario
from integration.cost_calculation.cost_tracking_case import JsonResponse
from pydantic import JsonValue
_ENDPOINT: Final = "databricks-openjev-qwen35-4b"
_API_KEY: Final = "synthetic-databricks-key"
_ENV_KEY: Final = "synthetic-databricks-env-key"
_ENV_TOKEN: Final = "synthetic-databricks-env-token"
_STATE: Final[dict[str, JsonValue]] = {"ticket": "The export job hangs at 99%", "component": "billing"}
_TIER_CRITERIA: Final[dict[str, JsonValue]] = {"SIMPLE": "a lookup", "MEDIUM": "some work", "COMPLEX": "deep work"}
_QUESTIONS: Final[dict[str, JsonValue]] = {"tier": {"type": "choice", "criteria": _TIER_CRITERIA}}
_OPENAI_INPUT: Final = "The export job hangs at 99% in billing"
_OPENAI_TIER_QUESTION: Final[dict[str, JsonValue]] = {
"type": "choice",
"name": "tier",
"instructions": "Which tier?",
"choices": [{"value": name, "description": description} for name, description in _TIER_CRITERIA.items()],
}
_ANSWERS: Final[dict[str, JsonValue]] = {
"tier": {
"type": "choice",
"choice": "COMPLEX",
"confidence": 0.91,
"probabilities": {"SIMPLE": 0.03, "MEDIUM": 0.06, "COMPLEX": 0.91},
}
}
_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 367, "output_tokens": 3}
_ANSWER_BODY: Final[dict[str, JsonValue]] = {"model": _ENDPOINT, "answers": _ANSWERS, "usage": _USAGE}
_BARE_NAME_RULE: Final = (
"must be the bare endpoint name (letters, digits, '-', '_' and '.', with no '/', '?', '#', spaces, or a "
"leading '.'), e.g. databricks-openjev-qwen35-4b"
)
_MISSING_KEY: Final = "Missing API key for Decisions provider 'databricks'"
_NON_BARE_NAMES: Final[tuple[tuple[str, str], ...]] = (
("slash", "foo/bar"),
("space", "a b"),
("leading-dot", ".hidden"),
("question-mark", "a?b"),
("hash", "a#b"),
)
_DATABRICKS_ERRORS: Final[tuple[tuple[int, dict[str, JsonValue]], ...]] = (
(400, {"error_code": "BAD_REQUEST", "message": "Invalid input: questions must not be empty"}),
(404, {"error_code": "RESOURCE_DOES_NOT_EXIST", "message": f"Endpoint with name '{_ENDPOINT}' does not exist."}),
)
_SPEND_QUERY: Final = (
'SELECT spend, status, call_type, model_group, custom_llm_provider, api_base FROM "LiteLLM_SpendLogs" '
"WHERE request_id = %s"
)
_OWNED_PROXY_BUDGET: Final = int(2 * graceful_stop_seconds() + 120)
_MEGABYTE_NAME: Final = "a" * 1_000_000 + "/"
def _number(value: JsonValue) -> float:
assert isinstance(value, (int, float)) and not isinstance(value, bool), value
return float(value)
def _register(scenario: Scenario, body: dict[str, JsonValue], *, status: int = 200) -> ScenarioHandle:
handle: Final = register_scenario(
f"databricks-{uuid.uuid4().hex[:12]}", JsonResponse(content_type="application/json", body=body, status=status)
)
scenario.cleanups.callback(delete_scenario, handle)
return handle
def _decide(gateway: Gateway, model: str, *, key: str | None = None) -> httpx.Response:
return gateway.request("POST", "/v1/systemone", {"model": model, "state": _STATE, "questions": _QUESTIONS}, key=key)
def _observed(gateway: Gateway) -> tuple[dict[str, JsonValue], ...]:
with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream:
return tuple(map(object_value, upstream.get("/__observations").json()["requests"]))
def _calls_to(requests: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> list[dict[str, JsonValue]]:
return [request for request in requests if string_value(request["path"]).startswith(f"/{handle.scenario_id}/")]
def _upstream_calls(gateway: Gateway, handle: ScenarioHandle) -> list[dict[str, JsonValue]]:
return _calls_to(_observed(gateway), handle)
def _spend_row(call_id: str) -> dict[str, JsonValue]:
rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70)
return rows[0]
def _wildcard_deployment(scenario: Scenario, **parameters: JsonValue) -> str:
prefix: Final = f"databricks-{uuid.uuid4().hex[:8]}"
created: Final = scenario.gateway.post(
"/model/new",
{"model_name": f"{prefix}/*", "litellm_params": {"model": "databricks/*", **parameters}, "model_info": {}},
)
scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"]))
return prefix
def _router_config(classifier: Mapping[str, JsonValue], simple: str, complex_: str) -> dict[str, JsonValue]:
return {
"classifier_type": "oss_classifier",
"opensource_classifier_config": dict(classifier),
"tiers": {"SIMPLE": simple, "MEDIUM": simple, "COMPLEX": complex_, "REASONING": complex_},
}
def _environment_classifier() -> dict[str, JsonValue]:
return {"provider": "databricks", "model": _ENDPOINT, "timeout_ms": 20000, "circuit_breaker_enabled": False}
def _router(scenario: Scenario, classifier: Mapping[str, JsonValue]) -> str:
simple: Final = scenario.model(model="openai/simple-tier")
complex_: Final = scenario.model(model="openai/complex-tier")
name: Final = f"router-{uuid.uuid4().hex[:10]}"
created: Final = scenario.gateway.post(
"/model/new",
{
"model_name": name,
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": _router_config(classifier, simple, complex_),
},
"model_info": {},
},
)
scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"]))
return name
def _chat(gateway: Gateway, model: str, *, key: str | None = None) -> httpx.Response:
return gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": f"design a distributed cache {uuid.uuid4().hex}"}]},
key=key,
)
def _assert_classified_by_the_complex_tier(response: httpx.Response, router: str) -> None:
assert response.status_code == 200, response.text
assert response.json()["choices"][0]["message"]["content"], response.text
assert (
response.headers["x-litellm-model-group"],
response.headers["x-litellm-model-name"],
response.headers["x-litellm-complexity-router-tier"],
response.headers["x-litellm-complexity-router-cause"],
response.headers["x-litellm-classifier-cost"],
) == (router, "openai/complex-tier", "COMPLEX", "jev_classifier", "0.0"), dict(response.headers)
def _assert_classifier_call(
call: Mapping[str, JsonValue], *, path: str, authorization: str, prompt_marker: str
) -> None:
assert call["path"] == path, call
assert call["authorization"] == authorization, call
body: Final = object_value(call["body"])
assert body["model"] == _ENDPOINT, body
assert prompt_marker in string_value(body["state"]), body
assert sorted(object_value(object_value(body["questions"])["tier"])) == ["criteria", "instructions", "type"], body
def _assert_refused_before_any_upstream_call(gateway: Gateway, model: str, message: str) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _ANSWER_BODY)
deployment: Final = scenario.model(model=model, api_base=handle.api_base(), api_key=_API_KEY)
response: Final = _decide(gateway, deployment)
assert response.status_code == 400, response.text
assert message in response.text, response.text
assert _upstream_calls(gateway, handle) == []
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (row["status"], row["call_type"], _number(row["spend"])) == ("failure", "asystemone", 0.0), row
@pytest.mark.parametrize("name", [name for _, name in _NON_BARE_NAMES], ids=[label for label, _ in _NON_BARE_NAMES])
def test_a_deployment_naming_a_non_bare_endpoint_is_refused_before_any_upstream_call(
gateway: Gateway, name: str
) -> None:
_assert_refused_before_any_upstream_call(
gateway, f"databricks/{name}", f"Databricks serving endpoint name {name!r} {_BARE_NAME_RULE}"
)
def test_a_deployment_naming_no_endpoint_is_refused_before_any_upstream_call(gateway: Gateway) -> None:
_assert_refused_before_any_upstream_call(gateway, "databricks/", "A model name is required for the Decisions API")
def test_an_openai_format_request_at_v1_decisions_reaches_the_serving_endpoint_as_system_one(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _ANSWER_BODY)
deployment: Final = scenario.model(
model=f"databricks/{_ENDPOINT}", api_base=handle.api_base(), api_key=_API_KEY
)
response: Final = gateway.request(
"POST",
"/v1/decisions",
{
"model": deployment,
"input": _OPENAI_INPUT,
"questions": [_OPENAI_TIER_QUESTION],
},
)
assert response.status_code == 200, response.text
assert response.json() == {
"model": _ENDPOINT,
"answers": [
{
"type": "choice",
"name": "tier",
"choice": "COMPLEX",
"probabilities": [
{"value": "SIMPLE", "probability": 0.03},
{"value": "MEDIUM", "probability": 0.06},
{"value": "COMPLEX", "probability": 0.91},
],
"confidence": 0.91,
}
],
"usage": {
"input_tokens": 367,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens": 3,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 370,
},
}, response.text
assert response.headers["x-litellm-model-name"] == f"databricks/{_ENDPOINT}", dict(response.headers)
(call,) = _upstream_calls(gateway, handle)
assert call["path"] == f"/{handle.scenario_id}/{_ENDPOINT}/invocations", call
assert call["authorization"] == f"Bearer {_API_KEY}", call
assert call["body"] == {
"model": _ENDPOINT,
"state": _OPENAI_INPUT,
"questions": {"tier": {"type": "choice", "instructions": "Which tier?", "criteria": _TIER_CRITERIA}},
}, call
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (
row["status"],
row["call_type"],
row["custom_llm_provider"],
row["model_group"],
row["api_base"],
_number(row["spend"]),
) == ("success", "asystemone", "databricks", deployment, f"{handle.api_base()}/{_ENDPOINT}/invocations", 0.0), (
row
)
def test_a_wildcard_deployment_sends_each_request_to_the_named_endpoint_and_bills_each_once(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _ANSWER_BODY)
prefix: Final = _wildcard_deployment(scenario, api_base=handle.api_base(), api_key=_API_KEY)
model: Final = f"{prefix}/{_ENDPOINT}"
responses: Final = tuple(_decide(gateway, model) for _ in range(2))
for response in responses:
assert response.status_code == 200, response.text
assert response.json() == _ANSWER_BODY, response.text
assert response.headers["x-litellm-model-group"] == model, dict(response.headers)
assert response.headers["x-litellm-model-name"] == f"databricks/{_ENDPOINT}", dict(response.headers)
call_ids: Final = tuple(response.headers["x-litellm-call-id"] for response in responses)
assert len(set(call_ids)) == 2, call_ids
calls: Final = _upstream_calls(gateway, handle)
assert len(calls) == 2, calls
for call in calls:
assert call["path"] == f"/{handle.scenario_id}/{_ENDPOINT}/invocations", call
assert call["authorization"] == f"Bearer {_API_KEY}", call
assert call["body"] == {"model": _ENDPOINT, "state": _STATE, "questions": _QUESTIONS}, call
for call_id in call_ids:
row: Final = _spend_row(call_id)
assert (
row["status"],
row["call_type"],
row["custom_llm_provider"],
row["model_group"],
row["api_base"],
_number(row["spend"]),
) == ("success", "asystemone", "databricks", model, f"{handle.api_base()}/{_ENDPOINT}/invocations", 0.0), (
row
)
@pytest.mark.parametrize("model", (5, ["a"]), ids=("int", "list"))
def test_a_non_string_model_is_refused_at_the_gateway_without_an_upstream_call(
gateway: Gateway, model: JsonValue
) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _ANSWER_BODY)
_wildcard_deployment(scenario, api_base=handle.api_base(), api_key=_API_KEY)
response: Final = gateway.request(
"POST", "/v1/systemone", {"model": model, "state": _STATE, "questions": _QUESTIONS}
)
assert 400 <= response.status_code < 500, response.text
assert _upstream_calls(gateway, handle) == []
def test_a_five_kilobyte_endpoint_name_reaches_the_upstream_and_its_not_found_answer_is_kept(gateway: Gateway) -> None:
name: Final = "n" * 5000
with gateway.scenario() as scenario:
handle: Final = _register(
scenario,
{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": f"Endpoint with name '{name}' does not exist."},
status=404,
)
prefix: Final = _wildcard_deployment(scenario, api_base=handle.api_base(), api_key=_API_KEY)
response: Final = _decide(gateway, f"{prefix}/{name}")
assert response.status_code == 404, response.text[:500]
assert "RESOURCE_DOES_NOT_EXIST" in response.text, response.text[:500]
(call,) = _upstream_calls(gateway, handle)
assert call["path"] == f"/{handle.scenario_id}/{name}/invocations", call["path"][:200]
assert call["authorization"] == f"Bearer {_API_KEY}", call["authorization"]
assert call["body"] == {"model": name, "state": _STATE, "questions": _QUESTIONS}
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (row["status"], row["call_type"], row["model_group"], _number(row["spend"])) == (
"failure",
"asystemone",
f"{prefix}/{name}",
0.0,
), row
@pytest.mark.parametrize("status,body", _DATABRICKS_ERRORS, ids=("400", "404"))
def test_a_databricks_error_keeps_its_status_and_message_and_bills_nothing(
gateway: Gateway, status: int, body: dict[str, JsonValue]
) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, body, status=status)
model: Final = scenario.model(model=f"databricks/{_ENDPOINT}", api_base=handle.api_base(), api_key=_API_KEY)
response: Final = _decide(gateway, model)
assert response.status_code == status, response.text
assert string_value(body["message"]) in response.text, response.text
(call,) = _upstream_calls(gateway, handle)
assert call["path"] == f"/{handle.scenario_id}/{_ENDPOINT}/invocations", call
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (row["status"], row["call_type"], row["model_group"], _number(row["spend"])) == (
"failure",
"asystemone",
model,
0.0,
), row
def test_a_deployment_without_a_key_and_no_environment_key_is_refused_before_any_upstream_call(
gateway: Gateway,
) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _ANSWER_BODY)
model: Final = scenario.model(model=f"databricks/{_ENDPOINT}", api_base=handle.api_base(), api_key=None)
response: Final = _decide(gateway, model)
assert response.status_code == 401, response.text
assert _MISSING_KEY in response.text, response.text
assert _upstream_calls(gateway, handle) == []
@pytest.mark.timeout(_OWNED_PROXY_BUDGET)
def test_the_environment_key_and_base_serve_a_deployment_a_classifier_and_a_team_member_that_leave_them_unset(
gateway: Gateway, tmp_path: Path
) -> None:
with gateway.scenario() as rig_scenario:
handle: Final = _register(rig_scenario, _ANSWER_BODY)
environment_base: Final = f"{handle.api_base()}/serving-endpoints"
environment_path: Final = f"/{handle.scenario_id}/serving-endpoints/{_ENDPOINT}/invocations"
environment: Final = {
"DATABRICKS_API_KEY": _ENV_KEY,
"DATABRICKS_TOKEN": _ENV_TOKEN,
"DATABRICKS_API_BASE": environment_base,
}
with owned_proxy_process(gateway, tmp_path, environment) as owned, owned.gateway.scenario() as scenario:
candidate: Final = owned.gateway
with_base: Final = scenario.model(model=f"databricks/{_ENDPOINT}", api_base=handle.api_base(), api_key=None)
bare: Final = scenario.model(model=f"databricks/{_ENDPOINT}", api_base=None, api_key=None)
decided: Final = tuple(_decide(candidate, model) for model in (with_base, bare))
for response in decided:
assert response.status_code == 200, response.text
assert response.json() == _ANSWER_BODY, response.text
assert [(call["path"], call["authorization"]) for call in _upstream_calls(candidate, handle)] == [
(f"/{handle.scenario_id}/{_ENDPOINT}/invocations", f"Bearer {_ENV_KEY}"),
(environment_path, f"Bearer {_ENV_KEY}"),
]
for response, model in zip(decided, (with_base, bare)):
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (row["status"], row["model_group"], _number(row["spend"])) == ("success", model, 0.0), row
assert (
_spend_row(decided[1].headers["x-litellm-call-id"])["api_base"]
== f"{environment_base}/{_ENDPOINT}/invocations"
)
router: Final = _router(scenario, _environment_classifier())
routed: Final = _chat(candidate, router)
_assert_classified_by_the_complex_tier(routed, router)
(judge_call,) = _upstream_calls(candidate, handle)
_assert_classifier_call(
judge_call,
path=environment_path,
authorization=f"Bearer {_ENV_KEY}",
prompt_marker="design a distributed cache",
)
team: Final = scenario.team(team_member_permissions=["/auto_router/manage"])
member: Final = scenario.member(team)
member_key: Final = scenario.key(user_id=member, team_id=team)
simple: Final = scenario.model(model="openai/simple-tier")
complex_: Final = scenario.model(model="openai/complex-tier")
team_router: Final = f"team-router-{uuid.uuid4().hex[:8]}"
created: Final = candidate.request(
"POST",
"/model/new",
{
"model_name": team_router,
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": _router_config(_environment_classifier(), simple, complex_),
},
"model_info": {"team_id": team},
},
key=member_key,
)
assert created.status_code == 200, created.text
team_router_id: Final = string_value(object_value(object_value(created.json())["model_info"])["id"])
scenario.cleanups.callback(scenario.delete_model, team_router_id)
stored: Final = candidate.request("GET", "/model/info", params={"litellm_model_id": team_router_id})
assert stored.status_code == 200, stored.text
(entry,) = stored.json()["data"]
stored_classifier: Final = object_value(
object_value(object_value(object_value(entry)["litellm_params"])["complexity_router_config"])[
"opensource_classifier_config"
]
)
assert (stored_classifier["provider"], stored_classifier["model"]) == ("databricks", _ENDPOINT), entry
assert "api_key" not in stored_classifier and "api_base" not in stored_classifier, entry
member_chat: Final = _chat(candidate, team_router, key=member_key)
_assert_classified_by_the_complex_tier(member_chat, team_router)
(member_judge_call,) = _upstream_calls(candidate, handle)
_assert_classifier_call(
member_judge_call,
path=environment_path,
authorization=f"Bearer {_ENV_KEY}",
prompt_marker="design a distributed cache",
)
@pytest.mark.timeout(_OWNED_PROXY_BUDGET)
def test_the_environment_token_serves_a_deployment_and_a_classifier_when_no_environment_key_is_set(
gateway: Gateway, tmp_path: Path
) -> None:
with gateway.scenario() as rig_scenario:
handle: Final = _register(rig_scenario, _ANSWER_BODY)
environment_path: Final = f"/{handle.scenario_id}/serving-endpoints/{_ENDPOINT}/invocations"
environment: Final = {
"DATABRICKS_TOKEN": _ENV_TOKEN,
"DATABRICKS_API_BASE": f"{handle.api_base()}/serving-endpoints",
}
with (
owned_proxy_process(gateway, tmp_path, environment, remove_environment=("DATABRICKS_API_KEY",)) as owned,
owned.gateway.scenario() as scenario,
):
candidate: Final = owned.gateway
bare: Final = scenario.model(model=f"databricks/{_ENDPOINT}", api_base=None, api_key=None)
decided: Final = _decide(candidate, bare)
assert decided.status_code == 200, decided.text
assert decided.json() == _ANSWER_BODY, decided.text
(decision_call,) = _upstream_calls(candidate, handle)
assert (decision_call["path"], decision_call["authorization"]) == (environment_path, f"Bearer {_ENV_TOKEN}")
assert _spend_row(decided.headers["x-litellm-call-id"])["status"] == "success"
router: Final = _router(scenario, _environment_classifier())
routed: Final = _chat(candidate, router)
_assert_classified_by_the_complex_tier(routed, router)
(judge_call,) = _upstream_calls(candidate, handle)
_assert_classifier_call(
judge_call,
path=environment_path,
authorization=f"Bearer {_ENV_TOKEN}",
prompt_marker="design a distributed cache",
)
def test_a_megabyte_endpoint_name_is_refused_in_linear_time_while_liveliness_stays_fast(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _ANSWER_BODY)
prefix: Final = _wildcard_deployment(scenario, api_base=handle.api_base(), api_key=_API_KEY)
model: Final = f"{prefix}/{_MEGABYTE_NAME}"
anonymous: Final = gateway.client.post(
"/v1/systemone", json={"model": model, "state": _STATE, "questions": _QUESTIONS}
)
assert anonymous.status_code == 401, anonymous.text[:300]
liveliness: Final[list[tuple[int, float]]] = []
stop: Final = threading.Event()
def poll() -> None:
while not stop.is_set():
started: Final = time.perf_counter()
probe: Final = gateway.client.get("/health/liveliness")
liveliness.append((probe.status_code, time.perf_counter() - started))
poller: Final = threading.Thread(target=poll)
poller.start()
started: Final = time.perf_counter()
response: Final = _decide(gateway, model)
elapsed: Final = time.perf_counter() - started
stop.set()
poller.join()
assert elapsed < 3, elapsed
assert liveliness and all(status == 200 for status, _ in liveliness), liveliness
assert max(seconds for _, seconds in liveliness) < 1, liveliness
assert response.status_code == 400, response.text[:300]
assert _BARE_NAME_RULE in response.text, response.text[-400:]
assert _upstream_calls(gateway, handle) == []
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (row["status"], row["call_type"], _number(row["spend"])) == ("failure", "asystemone", 0.0), row

593
tests/integration/providers/test_decisions_openai_format_wire.py Normal file
View file

@ -0,0 +1,593 @@
import math
import uuid
from collections.abc import Callable, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Final
import httpx
import pytest
import yaml
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy_process
from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario
from integration.cost_calculation.cost_tracking_case import JsonResponse
from pydantic import JsonValue, TypeAdapter
import litellm
_API_KEY: Final = "synthetic-decisions-key"
_SAFETY_IDENTIFIER: Final = "end-user-7"
_DROPPING_DEPLOYMENT: Final = "decisions-under-litellm-settings-drop-params"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_INPUT: Final = "Ticket (billing): The export job hangs at 99%"
_FOLLOW_UP: Final = "Customer: still stuck after retrying"
_QUESTIONS: Final[list[JsonValue]] = [
{"type": "predicate", "name": "defect", "instructions": "Is this a defect?"},
{
"type": "choice",
"name": "severity",
"instructions": "How severe is it?",
"choices": [{"value": "low", "description": "cosmetic"}, {"value": "high", "description": "blocks users"}],
},
{
"type": "score",
"name": "confidence",
"instructions": "How sure are you?",
"levels": [{"label": "unsure"}, {"label": "sure"}],
},
]
_SYSTEM_ONE_QUESTIONS: Final[dict[str, JsonValue]] = {
"defect": {"type": "noul", "instructions": "Is this a defect?"},
"severity": {
"type": "choice",
"instructions": "How severe is it?",
"criteria": {"low": "cosmetic", "high": "blocks users"},
},
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
}
_SYSTEM_ONE_ANSWERS: Final[dict[str, JsonValue]] = {
"defect": {"type": "noul", "noul": 0.93},
"severity": {"type": "choice", "choice": "high", "confidence": 0.8, "probabilities": {"low": 0.2, "high": 0.8}},
"confidence": {
"type": "score",
"score": 1.0,
"confidence": 0.7,
"legend": {"0": "unsure", "1": "sure"},
"probabilities": {"0": 0.3, "1": 0.7},
},
}
_ANSWERS: Final[list[JsonValue]] = [
{"type": "predicate", "name": "defect", "probability": 0.93},
{
"type": "choice",
"name": "severity",
"choice": "high",
"probabilities": [{"value": "low", "probability": 0.2}, {"value": "high", "probability": 0.8}],
"confidence": 0.8,
},
{
"type": "score",
"name": "confidence",
"score": 1.0,
"probabilities": [
{"value": 0, "label": "unsure", "probability": 0.3},
{"value": 1, "label": "sure", "probability": 0.7},
],
"confidence": 0.7,
},
]
_INPUT_TOKENS: Final = 367
_OUTPUT_TOKENS: Final = 3
_SYSTEM_ONE_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}
_USAGE: Final[dict[str, JsonValue]] = {
"input_tokens": _INPUT_TOKENS,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens": _OUTPUT_TOKENS,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS,
}
_SDK_QUESTIONS: Final = TypeAdapter(list[dict[str, object]]).validate_python(_QUESTIONS)
_SPEND_QUERY: Final = (
"SELECT spend, status, call_type, model_group, custom_llm_provider, api_base, prompt_tokens, completion_tokens, "
'request_tags FROM "LiteLLM_SpendLogs" WHERE request_id = %s'
)
@dataclass(frozen=True, slots=True)
class _Provider:
name: str
model: str
path: str
body_model: str
api_key: str | None
wraps_result: bool
cost_map_key: str | None
speaks_openai: bool = False
provider_reported_cost: float | None = None
def upstream_body(self) -> dict[str, JsonValue]:
if self.speaks_openai:
return {"model": self.body_model, "input": _INPUT, "questions": _QUESTIONS}
return {"model": self.body_model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS}
def _with_reported_cost(self, usage: dict[str, JsonValue]) -> dict[str, JsonValue]:
if self.provider_reported_cost is None:
return usage
return {**usage, "cost": self.provider_reported_cost}
def upstream_reply(self) -> dict[str, JsonValue]:
if self.speaks_openai:
return {"model": self.body_model, "answers": _ANSWERS, "usage": self._with_reported_cost(_USAGE)}
answer: Final[dict[str, JsonValue]] = {
"model": self.body_model,
"answers": _SYSTEM_ONE_ANSWERS,
"usage": self._with_reported_cost(_SYSTEM_ONE_USAGE),
}
return {"result": answer, "success": True} if self.wraps_result else answer
def litellm_response(self) -> dict[str, JsonValue]:
return {"model": self.body_model, "answers": _ANSWERS, "usage": _USAGE}
_PROVIDERS: Final = (
_Provider(
"perplexity",
"perplexity/pplx-decider-v1-27b",
"/v1/decisions",
"pplx-decider-v1-27b",
_API_KEY,
False,
"perplexity/pplx-decider-v1-27b",
),
_Provider("typesafe", "typesafe/jev-1.13.0", "/v1/systemone", "jev-1.13.0", _API_KEY, False, "typesafe/jev-1.13.0"),
_Provider(
"openrouter",
"openrouter/typesafe/jev-1.13",
"/alpha/decisions",
"typesafe/jev-1.13",
_API_KEY,
False,
None,
provider_reported_cost=1.5834e-5,
),
_Provider(
"strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None
),
_Provider(
"cloudflare",
"cloudflare/clef",
"/ai/run/@cf/cloudflare/clef",
"clef",
_API_KEY,
True,
"cloudflare/@cf/cloudflare/clef",
),
_Provider("hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None),
_Provider(
"databricks",
"databricks/databricks-openjev-qwen35-4b",
"/databricks-openjev-qwen35-4b/invocations",
"databricks-openjev-qwen35-4b",
_API_KEY,
False,
None,
),
_Provider(
"azure_ai", "azure_ai/decision-1", "/providers/microsoft/v1/systemone", "decision-1", _API_KEY, False, None
),
_Provider("openai", "openai/gpt-6-luna", "/v1/decisions", "gpt-6-luna", _API_KEY, False, "gpt-6-luna", True),
)
_SYSTEM_ONE_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if not provider.speaks_openai)
_PERPLEXITY: Final = _PROVIDERS[0]
_TYPESAFE: Final = _PROVIDERS[1]
_OPENAI: Final = next(provider for provider in _PROVIDERS if provider.speaks_openai)
_PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"}
_INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = (
("missing questions", {"input": _INPUT}),
("missing input", {"questions": _QUESTIONS}),
("numeric input", {"input": 5, "questions": _QUESTIONS}),
("assistant message input", {"input": [{"role": "assistant", "content": "hi"}], "questions": _QUESTIONS}),
("empty questions", {"input": _INPUT, "questions": []}),
("questions as a map", {"input": _INPUT, "questions": {"q": _PREDICATE}}),
("predicate without instructions", {"input": _INPUT, "questions": [{"type": "predicate", "name": "q"}]}),
(
"choice with one choice",
{"input": _INPUT, "questions": [{**_PREDICATE, "type": "choice", "choices": [{"value": "only"}]}]},
),
(
"score with one level",
{"input": _INPUT, "questions": [{**_PREDICATE, "type": "score", "levels": [{"label": "only"}]}]},
),
("unknown question type", {"input": _INPUT, "questions": [{**_PREDICATE, "type": "ranking"}]}),
("numeric safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": 7}),
("list safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": [_SAFETY_IDENTIFIER]}),
)
_IMAGE_INPUT: Final[list[JsonValue]] = [
{
"role": "user",
"content": [
{"type": "input_text", "text": _INPUT},
{"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"},
],
}
]
_OPENAI_IMAGE_MESSAGES: Final[list[JsonValue]] = [
{
"role": "user",
"type": "message",
"content": [
{"type": "input_text", "text": _INPUT},
{"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"},
],
}
]
_MESSAGE_LIST_INPUT: Final[list[JsonValue]] = [
{"role": "user", "content": _INPUT},
{"role": "user", "content": [{"type": "input_text", "text": _FOLLOW_UP}]},
]
def _response_cost(response: httpx.Response) -> float:
return float(response.headers["x-litellm-response-cost"]) if "x-litellm-response-cost" in response.headers else 0.0
def _provider_id(provider: _Provider) -> str:
return provider.name
def _number(value: JsonValue) -> float:
assert isinstance(value, (int, float)) and not isinstance(value, bool), value
return float(value)
def _expected_spend(provider: _Provider) -> float:
if provider.provider_reported_cost is not None:
return provider.provider_reported_cost
if provider.cost_map_key is None:
return 0.0
cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes())
prices: Final = object_value(cost_map[provider.cost_map_key])
return _INPUT_TOKENS * _number(prices["input_cost_per_token"]) + _OUTPUT_TOKENS * _number(
prices["output_cost_per_token"]
)
def _register(scenario: Scenario, body: dict[str, JsonValue], *, status: int = 200) -> ScenarioHandle:
handle: Final = register_scenario(
f"decisions-{uuid.uuid4().hex[:12]}", JsonResponse(content_type="application/json", body=body, status=status)
)
scenario.cleanups.callback(delete_scenario, handle)
return handle
def _deployment(scenario: Scenario, handle: ScenarioHandle, provider: _Provider, *, drop_params: bool = False) -> str:
if drop_params:
return scenario.model(
model=provider.model, api_base=handle.api_base(), api_key=provider.api_key, drop_params=True
)
return scenario.model(model=provider.model, api_base=handle.api_base(), api_key=provider.api_key)
def _decide(gateway: Gateway, model: str, *, route: str = "/v1/decisions", **extra: JsonValue) -> httpx.Response:
return gateway.request("POST", route, {"model": model, "input": _INPUT, "questions": _QUESTIONS, **extra})
def _decide_in_system_one_format(gateway: Gateway, model: str, **extra: JsonValue) -> httpx.Response:
return gateway.request(
"POST", "/v1/systemone", {"model": model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS, **extra}
)
def _observed_requests(gateway: Gateway) -> tuple[dict[str, JsonValue], ...]:
with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream:
observations: Final = _JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"]
assert isinstance(observations, list), observations
return tuple(map(object_value, observations))
def _calls_to(requests: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> list[dict[str, JsonValue]]:
return [request for request in requests if string_value(request["path"]).startswith(f"/{handle.scenario_id}/")]
def _upstream_calls(gateway: Gateway, handle: ScenarioHandle) -> list[dict[str, JsonValue]]:
return _calls_to(_observed_requests(gateway), handle)
def _spend_row(call_id: str) -> dict[str, JsonValue]:
rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70)
return rows[0]
def _assert_refused_as_invalid(gateway: Gateway, model: str, label: str, body: dict[str, JsonValue]) -> None:
response: Final = gateway.request("POST", "/v1/decisions", {"model": model, **body})
assert response.status_code == 400, (label, response.text)
assert "Invalid Decisions request" in response.text, (label, response.text)
def _assert_refused_for_safety_identifier(response: httpx.Response) -> None:
assert response.status_code == 400, response.text
assert "safety_identifier" in response.text and "drop_params" in response.text, response.text
def _drop_params_config(directory: Path, api_base: str) -> Path:
base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
config: Final = {
**base,
"litellm_settings": {**object_value(base["litellm_settings"]), "drop_params": True},
"model_list": [
{
"model_name": _DROPPING_DEPLOYMENT,
"litellm_params": {"model": _PERPLEXITY.model, "api_base": api_base, "api_key": _API_KEY},
}
],
}
path: Final = directory / "decisions-drop-params.yaml"
path.write_text(yaml.safe_dump(config))
return path
@pytest.mark.parametrize("provider", _PROVIDERS, ids=_provider_id)
def test_each_provider_gets_its_own_path_key_and_body_and_is_billed(gateway: Gateway, provider: _Provider) -> None:
expected_spend: Final = _expected_spend(provider)
with gateway.scenario() as scenario:
handle: Final = _register(scenario, provider.upstream_reply())
model: Final = _deployment(scenario, handle, provider)
response: Final = _decide(gateway, model)
assert response.status_code == 200, response.text
assert response.json() == provider.litellm_response()
assert response.headers["x-litellm-model-group"] == model
assert math.isclose(_response_cost(response), expected_spend, rel_tol=1e-9)
(call,) = _upstream_calls(gateway, handle)
assert call["path"] == f"/{handle.scenario_id}{provider.path}"
assert call["authorization"] == (f"Bearer {provider.api_key}" if provider.api_key else "")
assert call["body"] == provider.upstream_body()
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (
row["status"],
row["call_type"],
row["custom_llm_provider"],
row["model_group"],
row["api_base"],
row["prompt_tokens"],
row["completion_tokens"],
) == (
"success",
"adecisions",
provider.name,
model,
f"{handle.api_base()}{provider.path}",
_INPUT_TOKENS,
_OUTPUT_TOKENS,
)
assert math.isclose(_number(row["spend"]), expected_spend, rel_tol=1e-9), row
def test_the_unversioned_alias_serves_the_same_request(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _PERPLEXITY.upstream_reply())
model: Final = _deployment(scenario, handle, _PERPLEXITY)
response: Final = _decide(gateway, model, route="/decisions")
assert response.status_code == 200, response.text
assert response.json() == _PERPLEXITY.litellm_response()
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == _PERPLEXITY.upstream_body()
def test_repeated_identical_requests_each_reach_the_upstream_and_are_each_billed(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _PERPLEXITY.upstream_reply())
model: Final = _deployment(scenario, handle, _PERPLEXITY)
responses: Final = tuple(_decide(gateway, model) for _ in range(2))
assert [response.status_code for response in responses] == [200, 200], [r.text for r in responses]
call_ids: Final = tuple(response.headers["x-litellm-call-id"] for response in responses)
assert len(set(call_ids)) == 2, call_ids
assert len(_upstream_calls(gateway, handle)) == 2
for call_id in call_ids:
assert _spend_row(call_id)["status"] == "success"
async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _TYPESAFE.upstream_reply())
synchronous: Final = litellm.decisions(
model=_TYPESAFE.model, input=_INPUT, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY
)
asynchronous: Final = await litellm.adecisions(
model=_TYPESAFE.model, input=_INPUT, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY
)
for response in (synchronous, asynchronous):
assert response.model_dump(mode="json") == _TYPESAFE.litellm_response()
calls: Final = _upstream_calls(gateway, handle)
assert len(calls) == 2, calls
for call in calls:
assert call["path"] == f"/{handle.scenario_id}{_TYPESAFE.path}"
assert call["authorization"] == f"Bearer {_API_KEY}"
assert call["body"] == _TYPESAFE.upstream_body()
def test_gateway_only_fields_stay_at_the_gateway_and_tags_reach_the_spend_log(gateway: Gateway) -> None:
tag: Final = f"decisions-openai-format-{uuid.uuid4().hex[:8]}"
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _PERPLEXITY.upstream_reply())
model: Final = _deployment(scenario, handle, _PERPLEXITY)
response: Final = _decide(
gateway, model, user="auditor", num_retries=0, temperature=0.2, metadata={"tags": [tag]}
)
assert response.status_code == 200, response.text
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == _PERPLEXITY.upstream_body()
row: Final = _spend_row(response.headers["x-litellm-call-id"])
tags: Final = row["request_tags"]
assert isinstance(tags, list) and tag in tags, row
def test_invalid_bodies_are_refused_at_the_gateway_without_an_upstream_call(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _PERPLEXITY.upstream_reply())
model: Final = _deployment(scenario, handle, _PERPLEXITY)
for label, body in _INVALID_BODIES:
_assert_refused_as_invalid(gateway, model, label, body)
assert _upstream_calls(gateway, handle) == []
@pytest.mark.parametrize("provider", _SYSTEM_ONE_PROVIDERS, ids=_provider_id)
def test_system_one_providers_refuse_images_at_the_gateway_without_an_upstream_call(
gateway: Gateway, provider: _Provider
) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, provider.upstream_reply())
model: Final = _deployment(scenario, handle, provider)
response: Final = _decide(gateway, model, input=_IMAGE_INPUT)
assert response.status_code == 400, response.text
assert "input_image" in response.text
assert _upstream_calls(gateway, handle) == []
def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _OPENAI.upstream_reply())
model: Final = _deployment(scenario, handle, _OPENAI)
response: Final = _decide(gateway, model, input=_IMAGE_INPUT)
assert response.status_code == 200, response.text
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == {"model": _OPENAI.body_model, "input": _OPENAI_IMAGE_MESSAGES, "questions": _QUESTIONS}
def test_a_message_list_input_reaches_a_system_one_provider_as_its_flattened_text(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _PERPLEXITY.upstream_reply())
model: Final = _deployment(scenario, handle, _PERPLEXITY)
response: Final = _decide(gateway, model, input=_MESSAGE_LIST_INPUT)
assert response.status_code == 200, response.text
assert response.json() == _PERPLEXITY.litellm_response()
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == {
"model": _PERPLEXITY.body_model,
"state": f"{_INPUT}\n\n{_FOLLOW_UP}",
"questions": _SYSTEM_ONE_QUESTIONS,
}
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (row["status"], row["call_type"], row["prompt_tokens"], row["completion_tokens"]) == (
"success",
"adecisions",
_INPUT_TOKENS,
_OUTPUT_TOKENS,
)
@pytest.mark.parametrize("provider", _SYSTEM_ONE_PROVIDERS, ids=_provider_id)
def test_safety_identifier_is_refused_by_system_one_providers_unless_the_deployment_drops_params(
gateway: Gateway, provider: _Provider
) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, provider.upstream_reply())
strict: Final = _deployment(scenario, handle, provider)
refused: Final = _decide(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER)
_assert_refused_for_safety_identifier(refused)
assert _upstream_calls(gateway, handle) == []
dropping: Final = _deployment(scenario, handle, provider, drop_params=True)
accepted: Final = _decide(gateway, dropping, safety_identifier=_SAFETY_IDENTIFIER)
assert accepted.status_code == 200, accepted.text
assert accepted.json() == provider.litellm_response()
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == provider.upstream_body()
row: Final = _spend_row(accepted.headers["x-litellm-call-id"])
assert (row["status"], row["call_type"], row["model_group"]) == ("success", "adecisions", dropping)
@pytest.mark.parametrize("value", ("", "x" * 5000), ids=("empty", "5kb"))
def test_every_string_safety_identifier_is_refused_or_dropped_like_the_usual_one(gateway: Gateway, value: str) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _PERPLEXITY.upstream_reply())
strict: Final = _deployment(scenario, handle, _PERPLEXITY)
_assert_refused_for_safety_identifier(_decide(gateway, strict, safety_identifier=value))
dropping: Final = _deployment(scenario, handle, _PERPLEXITY, drop_params=True)
accepted: Final = _decide(gateway, dropping, safety_identifier=value)
assert accepted.status_code == 200, accepted.text
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == _PERPLEXITY.upstream_body()
def test_a_request_body_drop_params_drops_the_safety_identifier_like_chat(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _PERPLEXITY.upstream_reply())
strict: Final = _deployment(scenario, handle, _PERPLEXITY)
response: Final = _decide(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER, drop_params=True)
assert response.status_code == 200, response.text
assert response.json() == _PERPLEXITY.litellm_response()
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == _PERPLEXITY.upstream_body()
def test_openai_keeps_the_safety_identifier_on_the_wire_without_drop_params(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _OPENAI.upstream_reply())
model: Final = _deployment(scenario, handle, _OPENAI)
response: Final = _decide(gateway, model, safety_identifier=_SAFETY_IDENTIFIER)
assert response.status_code == 200, response.text
assert response.json() == _OPENAI.litellm_response()
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == {**_OPENAI.upstream_body(), "safety_identifier": _SAFETY_IDENTIFIER}
def test_a_system_one_format_safety_identifier_is_refused_unless_the_deployment_drops_params(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _PERPLEXITY.upstream_reply())
strict: Final = _deployment(scenario, handle, _PERPLEXITY)
refused: Final = _decide_in_system_one_format(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER)
_assert_refused_for_safety_identifier(refused)
assert _upstream_calls(gateway, handle) == []
dropping: Final = _deployment(scenario, handle, _PERPLEXITY, drop_params=True)
accepted: Final = _decide_in_system_one_format(gateway, dropping, safety_identifier=_SAFETY_IDENTIFIER)
assert accepted.status_code == 200, accepted.text
assert accepted.json()["answers"] == _SYSTEM_ONE_ANSWERS
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == _PERPLEXITY.upstream_body()
def test_openai_keeps_a_system_one_format_safety_identifier_on_the_wire(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _OPENAI.upstream_reply())
model: Final = _deployment(scenario, handle, _OPENAI)
response: Final = _decide_in_system_one_format(gateway, model, safety_identifier=_SAFETY_IDENTIFIER)
assert response.status_code == 200, response.text
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == {**_OPENAI.upstream_body(), "safety_identifier": _SAFETY_IDENTIFIER}
@pytest.mark.parametrize("decide", (_decide, _decide_in_system_one_format), ids=("openai_format", "system_one_format"))
@pytest.mark.parametrize("value", (7, [_SAFETY_IDENTIFIER]), ids=("numeric", "list"))
def test_a_non_string_safety_identifier_is_refused_as_invalid_unless_the_deployment_drops_params(
gateway: Gateway, value: JsonValue, decide: Callable[..., httpx.Response]
) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _OPENAI.upstream_reply())
strict: Final = _deployment(scenario, handle, _OPENAI)
refused: Final = decide(gateway, strict, safety_identifier=value)
assert refused.status_code == 400, refused.text
assert "Invalid Decisions request" in refused.text, refused.text
assert _upstream_calls(gateway, handle) == []
dropping: Final = _deployment(scenario, handle, _OPENAI, drop_params=True)
accepted: Final = decide(gateway, dropping, safety_identifier=value)
assert accepted.status_code == 200, accepted.text
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == _OPENAI.upstream_body()
def test_litellm_settings_drop_params_drops_the_safety_identifier_for_a_strict_deployment(
gateway: Gateway, tmp_path: Path
) -> None:
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _PERPLEXITY.upstream_reply())
config: Final = _drop_params_config(tmp_path, handle.api_base())
with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned:
response: Final = _decide(owned.gateway, _DROPPING_DEPLOYMENT, safety_identifier=_SAFETY_IDENTIFIER)
assert response.status_code == 200, response.text
assert response.json() == _PERPLEXITY.litellm_response()
(call,) = _upstream_calls(gateway, handle)
assert call["body"] == _PERPLEXITY.upstream_body()

55
tests/integration/providers/test_decisions_wire.py
View file

@ -69,6 +69,7 @@ class _Provider:
api_key: str | None
wraps_result: bool
cost_map_key: str | None
provider_reported_cost: float | None = None
_PROVIDERS: Final = (
@ -89,11 +90,15 @@ _PROVIDERS: Final = (
"typesafe/jev-1.13",
_API_KEY,
False,
"openrouter/typesafe/jev-1.13",
None,
1.5834e-5,
),
_Provider(
"strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None
),
_Provider(
"hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None
),
_Provider(
"cloudflare",
"cloudflare/clef",
@ -106,6 +111,15 @@ _PROVIDERS: Final = (
_Provider(
"azure_ai", "azure_ai/decision-1", "/providers/microsoft/v1/systemone", "decision-1", _API_KEY, False, None
),
_Provider(
"databricks",
"databricks/databricks-openjev-qwen35-4b",
"/databricks-openjev-qwen35-4b/invocations",
"databricks-openjev-qwen35-4b",
_API_KEY,
False,
None,
),
)
_PERPLEXITY: Final = _PROVIDERS[0]
_OPENROUTER: Final = _PROVIDERS[2]
@ -133,17 +147,32 @@ def _number(value: JsonValue) -> float:
return float(value)
def _expected_spend(cost_map_key: str | None) -> float:
if cost_map_key is None:
def _expected_spend(provider: _Provider) -> float:
if provider.provider_reported_cost is not None:
return provider.provider_reported_cost
if provider.cost_map_key is None:
return 0.0
prices: Final = object_value(json.loads(Path("model_prices_and_context_window.json").read_text())[cost_map_key])
prices: Final = object_value(
json.loads(Path("model_prices_and_context_window.json").read_text())[provider.cost_map_key]
)
return _number(_USAGE["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number(
_USAGE["output_tokens"]
) * _number(prices["output_cost_per_token"])
def _usage(provider: _Provider) -> dict[str, JsonValue]:
return {
**_USAGE,
**({"cost": provider.provider_reported_cost} if provider.provider_reported_cost is not None else {}),
}
def _answer_body(provider: _Provider) -> dict[str, JsonValue]:
answer: Final[dict[str, JsonValue]] = {"model": provider.body_model, "answers": _ANSWERS, "usage": _USAGE}
answer: Final[dict[str, JsonValue]] = {
"model": provider.body_model,
"answers": _ANSWERS,
"usage": _usage(provider),
}
return {"result": answer, "success": True} if provider.wraps_result else answer
@ -226,16 +255,14 @@ def _pass_through_config(directory: Path, pass_through_target: str, native_api_b
@pytest.mark.parametrize("provider", _PROVIDERS, ids=lambda provider: provider.name)
def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cost_map(
gateway: Gateway, provider: _Provider
) -> None:
expected_spend: Final = _expected_spend(provider.cost_map_key)
def test_each_provider_gets_its_own_path_key_and_body_and_is_billed(gateway: Gateway, provider: _Provider) -> None:
expected_spend: Final = _expected_spend(provider)
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _answer_body(provider))
model: Final = _deployment(scenario, handle, provider)
response: Final = _decide(gateway, model)
assert response.status_code == 200, response.text
assert response.json() == {"model": provider.body_model, "answers": _ANSWERS, "usage": _USAGE}
assert response.json() == {"model": provider.body_model, "answers": _ANSWERS, "usage": _usage(provider)}
assert response.headers["x-litellm-model-group"] == model
assert math.isclose(float(response.headers.get("x-litellm-response-cost", "0")), expected_spend, rel_tol=1e-9)
(call,) = _upstream_calls(gateway, handle)
@ -251,7 +278,7 @@ def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cos
row["api_base"],
row["prompt_tokens"],
row["completion_tokens"],
) == ("success", "adecisions", provider.name, model, f"{handle.api_base()}{provider.path}", 367, 3)
) == ("success", "asystemone", provider.name, model, f"{handle.api_base()}{provider.path}", 367, 3)
assert math.isclose(_number(row["spend"]), expected_spend, rel_tol=1e-9), row
@ -292,7 +319,7 @@ async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway
synchronous: Final = litellm.decisions(
model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY
)
asynchronous: Final = await litellm.adecisions(
asynchronous: Final = await litellm.asystemone(
model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY
)
for response in (synchronous, asynchronous):
@ -460,7 +487,7 @@ def test_upstream_errors_keep_their_status_and_log_an_unbilled_failure(gateway:
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (row["status"], row["call_type"], row["model_group"], _number(row["spend"])) == (
"failure",
"adecisions",
"asystemone",
model,
0.0,
)
@ -532,7 +559,7 @@ async def test_sdk_openrouter_connection_failures_raise_a_connection_error(gatew
model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY
)
with pytest.raises(litellm.APIConnectionError):
await litellm.adecisions(
await litellm.asystemone(
model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY
)

827
tests/integration/providers/test_strands_decider_agentcore_wire.py Normal file
View file

@ -0,0 +1,827 @@
import hashlib
import json
import os
import re
import threading
import time
import uuid
from collections.abc import Callable, Iterator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, replace
from functools import partial
from pathlib import Path
from typing import Final
from urllib.parse import quote, unquote
import httpx
import pytest
import yaml
from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, gateway_from_environment
from integration._support.database import read_rows
from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy
from integration._support.process import owned_proxy
from integration._support.sigv4 import encoded_path, signature
from integration._support.tls import server_context, write_self_signed_cert
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.llms.strands_decider.decisions.transformation import StrandsDeciderDecisionsConfig
_ACCOUNT: Final = "123456789012"
_ACCESS_KEY: Final = "AKIAAGENTCOREAUDIT01"
_SECRET_KEY: Final = "synthetic-agentcore-audit-secret"
_SIGNING: Final[Mapping[str, JsonValue]] = {"aws_access_key_id": _ACCESS_KEY, "aws_secret_access_key": _SECRET_KEY}
_JWT: Final = "synthetic-agentcore-jwt"
_ENV_JWT: Final = "synthetic-agentcore-env-jwt"
_US_EAST: Final = "us-east-1"
_EU_WEST: Final = "eu-west-1"
_MODEL: Final = "strands_decider/systemone-decider"
_BODY_MODEL: Final = "systemone-decider"
_SESSION_HEADER: Final = "x-amzn-bedrock-agentcore-runtime-session-id"
_CONFIGURED_SESSION: Final = "agentcore-audit-configured-session-0000000001"
_STATE: Final[dict[str, JsonValue]] = {"ticket": "Refund requested twice for one order", "queue": "payments"}
_QUESTIONS: Final[dict[str, JsonValue]] = {
"duplicate": {"type": "noul", "instructions": "Is this a duplicate refund request?"},
"urgency": {"type": "choice", "criteria": {"low": "can wait", "high": "money at risk"}},
}
_SDK_QUESTIONS: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python(_QUESTIONS)
_ANSWERS: Final[dict[str, JsonValue]] = {
"duplicate": {"type": "noul", "noul": 0.88},
"urgency": {"type": "choice", "choice": "high", "confidence": 0.75, "probabilities": {"low": 0.25, "high": 0.75}},
}
_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 211, "output_tokens": 2}
_UPSTREAM_BODY: Final[dict[str, JsonValue]] = {"model": _BODY_MODEL, "state": _STATE, "questions": _QUESTIONS}
_INPUT: Final = "Refund requested twice for one order"
_OPENAI_QUESTIONS: Final[list[JsonValue]] = [
{"type": "predicate", "name": "duplicate", "instructions": "Is this a duplicate refund request?"},
{
"type": "choice",
"name": "urgency",
"instructions": "How urgent is it?",
"choices": [{"value": "low", "description": "can wait"}, {"value": "high", "description": "money at risk"}],
},
]
_OPENAI_UPSTREAM_BODY: Final[dict[str, JsonValue]] = {
"model": _BODY_MODEL,
"state": _INPUT,
"questions": {
"duplicate": {"type": "noul", "instructions": "Is this a duplicate refund request?"},
"urgency": {
"type": "choice",
"instructions": "How urgent is it?",
"criteria": {"low": "can wait", "high": "money at risk"},
},
},
}
_OPENAI_ANSWER: Final[dict[str, JsonValue]] = {
"model": _BODY_MODEL,
"answers": [
{"type": "predicate", "name": "duplicate", "probability": 0.88},
{
"type": "choice",
"name": "urgency",
"choice": "high",
"probabilities": [{"value": "low", "probability": 0.25}, {"value": "high", "probability": 0.75}],
"confidence": 0.75,
},
],
"usage": {
"input_tokens": 211,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens": 2,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 213,
},
}
@dataclass(frozen=True, slots=True)
class _Route:
path: str
body: Mapping[str, JsonValue]
upstream: Mapping[str, JsonValue]
answer: Mapping[str, JsonValue]
_SYSTEMONE: Final = _Route(
"/v1/systemone",
{"state": _STATE, "questions": _QUESTIONS},
_UPSTREAM_BODY,
{"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE},
)
_OPENAI: Final = _Route(
"/v1/decisions", {"input": _INPUT, "questions": _OPENAI_QUESTIONS}, _OPENAI_UPSTREAM_BODY, _OPENAI_ANSWER
)
_ROUTES: Final = (_SYSTEMONE, _OPENAI, replace(_SYSTEMONE, path="/systemone"), replace(_OPENAI, path="/decisions"))
_SPEND_QUERY: Final = (
"SELECT status, call_type, custom_llm_provider, model_group, api_base, prompt_tokens, completion_tokens "
'FROM "LiteLLM_SpendLogs" WHERE request_id = %s'
)
_RUNTIME_TARGET: Final = re.compile(
r"\A/runtimes/arn:aws:bedrock-agentcore:[a-z0-9-]+:[0-9]{12}:runtime/(?P<runtime>[A-Za-z0-9_-]+)/invocations\Z"
)
_ERROR_STATUS: Final = {"bad_request": 400, "invalid_request": 400, "loading": 503, "inference_error": 500}
_HTTP_ERRORS: Final = {
403: "AccessDeniedException",
404: "ResourceNotFoundException",
429: "ThrottlingException",
}
_MALFORMED_SESSIONS: Final[Mapping[str, JsonValue]] = {"int": 1234567, "list": ["a" * 40], "empty": ""}
_LONG_SESSION: Final = "s" * 5120
_NEAR_MISS_ARNS: Final = {
"endpoint": f"arn:aws:bedrock-agentcore:us-east-1:{_ACCOUNT}:runtime/audit_qualified/runtime-endpoint/DEFAULT",
"account": "arn:aws:bedrock-agentcore:us-east-1:12345:runtime/audit_short_account",
"service": f"arn:aws:bedrock:us-east-1:{_ACCOUNT}:runtime/audit_other_service",
"host": f"arn:aws:bedrock-agentcore:us-east-1.example.com:{_ACCOUNT}:runtime/audit_crafted_host",
}
_DEPLOYMENT_HEADERS: Final[dict[str, JsonValue]] = {
"x-amz-target": "AmazonBedrockAgentCore.Forged",
"X-Amzn-Bedrock-AgentCore-Runtime-Session-Id": "deployment-forged-session-00000000000001",
"Host": "bedrock-agentcore.us-west-2.amazonaws.com",
"x-audit-trace": "deployment-trace",
}
_SIGNED: Final = "agentcore-audit-signed"
_REGION: Final = "agentcore-audit-region"
_SESSION: Final = "agentcore-audit-session"
_LONG: Final = "agentcore-audit-session-long"
_BEARER: Final = "agentcore-audit-bearer"
_HEADERS: Final = "agentcore-audit-headers"
_OPT_IN: Final = "agentcore-audit-opt-in"
_HEALTH: Final = "agentcore-audit-health"
_DROPPED: Final = "agentcore-audit-dropped"
_OUTAGE: Final = "agentcore-audit-outage"
_ENV: Final = "agentcore-audit-env"
_UNSIGNED: Final = "agentcore-audit-unsigned"
_SCRUBBED_PREFIXES: Final = ("AWS_", "STRANDS_DECIDER_")
_SCRUBBED_NAMES: Final = frozenset(
{"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY", "SSL_VERIFY", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE"}
)
_load_yaml: Final[Callable[[str], object]] = yaml.safe_load
_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
_OWNED_PROXY_CELL_SECONDS: Final = 2 * max(30.0, float(os.environ.get("INTEGRATION_PROXY_READY_SECONDS") or 70)) + 120
pytestmark: Final = pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
def _host(region: str) -> str:
return f"bedrock-agentcore.{region}.amazonaws.com"
def _arn(runtime: str, region: str = _US_EAST) -> str:
return f"arn:aws:bedrock-agentcore:{region}:{_ACCOUNT}:runtime/{runtime}"
def _invocations_url(runtime: str, region: str = _US_EAST) -> str:
return f"https://{_host(region)}/runtimes/{quote(_arn(runtime, region), safe='')}/invocations"
def _default_session(runtime: str, region: str = _US_EAST) -> str:
return f"litellm-decider-{hashlib.sha256(_arn(runtime, region).encode()).hexdigest()}"
def _runtime_of(request: Request) -> str | None:
match: Final = _RUNTIME_TARGET.match(unquote(request.target))
return None if match is None else match["runtime"]
def _json_reply(body: Mapping[str, JsonValue], status: int = 200, **headers: str) -> Reply:
return Reply(status=status, body=json.dumps(body).encode(), headers=headers)
def _respond(outage: threading.Event, request: Request) -> Reply:
runtime: Final = _runtime_of(request) or ""
if runtime == "audit_dropped" or (runtime == "audit_outage" and outage.is_set()):
return Reply(drop_connection=True)
if runtime.startswith("audit_error_"):
code: Final = runtime.removeprefix("audit_error_")
return _json_reply({"error": {"code": code, "message": f"runtime refused with {code}"}})
if runtime.startswith("audit_status_"):
status: Final = int(runtime.removeprefix("audit_status_"))
return _json_reply(
{"message": f"scripted {_HTTP_ERRORS[status]}"}, status, **{"x-amzn-errortype": _HTTP_ERRORS[status]}
)
if not runtime:
return _json_reply({"message": f"unknown target {request.target}"}, 404)
return _json_reply({"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE})
@dataclass(frozen=True, slots=True)
class _Runtime:
wire: Wire
tunnel: ForwardProxy
cert: Path
outage: threading.Event
def reset(self) -> None:
self.outage.clear()
self.wire.drain()
self.tunnel.drain()
def calls(self) -> tuple[Request, ...]:
return self.wire.drain()
def _calls_to(calls: Sequence[Request], runtime: str) -> tuple[Request, ...]:
return tuple(call for call in calls if _runtime_of(call) == runtime)
def _deployment(name: str, **litellm_params: JsonValue) -> dict[str, JsonValue]:
return {
"model_name": name,
"litellm_params": {"model": _MODEL, **litellm_params},
"model_info": {"id": name, "mode": "evaluation"},
}
def _signed_deployment(name: str, runtime: str, region: str = _US_EAST, **extra: JsonValue) -> dict[str, JsonValue]:
return _deployment(name, api_base=_arn(runtime, region), **_SIGNING, **extra)
def _deployments() -> list[dict[str, JsonValue]]:
return [
_signed_deployment(_SIGNED, "audit_signed", aws_region_name=_US_EAST),
_signed_deployment(_REGION, "audit_region", _EU_WEST, aws_region_name=_US_EAST),
_signed_deployment(_SESSION, "audit_session", agentcore_runtime_session_id=_CONFIGURED_SESSION),
_signed_deployment(_LONG, "audit_session_long", agentcore_runtime_session_id=_LONG_SESSION),
_deployment(_BEARER, api_base=_arn("audit_bearer"), api_key=_JWT),
_signed_deployment(_HEADERS, "audit_headers", extra_headers=_DEPLOYMENT_HEADERS),
_signed_deployment(
_OPT_IN, "audit_opt_in", configurable_clientside_auth_params=["agentcore_runtime_session_id"]
),
_signed_deployment(_HEALTH, "audit_health"),
_signed_deployment(_DROPPED, "audit_dropped"),
_signed_deployment(_OUTAGE, "audit_outage"),
_deployment(_ENV, **_SIGNING),
_deployment(_UNSIGNED, api_base=_arn("audit_unsigned")),
*(_signed_deployment(f"agentcore-audit-error-{code}", f"audit_error_{code}") for code in _ERROR_STATUS),
*(_signed_deployment(f"agentcore-audit-status-{status}", f"audit_status_{status}") for status in _HTTP_ERRORS),
*(
_signed_deployment(
f"agentcore-audit-session-{label}", f"audit_session_{label}", agentcore_runtime_session_id=value
)
for label, value in _MALFORMED_SESSIONS.items()
),
*(
_deployment(f"agentcore-audit-near-miss-{label}", api_base=value, **_SIGNING)
for label, value in _NEAR_MISS_ARNS.items()
),
]
def _write_config(directory: Path) -> Path:
base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text()))
configuration: Final = {
**base,
"model_list": _deployments(),
"router_settings": {"disable_cooldowns": True, "num_retries": 0},
}
path: Final = directory / "agentcore.yaml"
path.write_text(yaml.safe_dump(configuration))
return path
def _scrubbed() -> tuple[str, ...]:
return tuple(
name for name in os.environ if name.upper().startswith(_SCRUBBED_PREFIXES) or name.upper() in _SCRUBBED_NAMES
)
def _isolated_aws(directory: Path) -> dict[str, str]:
return {
"AWS_EC2_METADATA_DISABLED": "true",
"AWS_SHARED_CREDENTIALS_FILE": str(directory / "absent-credentials"),
"AWS_CONFIG_FILE": str(directory / "absent-config"),
}
def _proxy_environment(runtime: _Runtime, directory: Path) -> dict[str, str]:
return {
"HTTPS_PROXY": runtime.tunnel.url,
"NO_PROXY": "127.0.0.1,localhost",
"SSL_CERT_FILE": str(runtime.cert),
"STRANDS_DECIDER_API_BASE": _arn("audit_env"),
**_isolated_aws(directory),
}
@pytest.fixture(scope="module")
def module_gateway() -> Iterator[Gateway]:
with gateway_from_environment() as value:
yield value
@pytest.fixture(scope="module")
def runtime(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Runtime]:
cert, key = write_self_signed_cert(tmp_path_factory.mktemp("agentcore-cert"), (_host(_US_EAST), _host(_EU_WEST)))
outage: Final = threading.Event()
with wire_server(partial(_respond, outage), tls=server_context(cert, key)) as wire:
with tunnelling_forward_proxy(int(wire.url.rsplit(":", 1)[1])) as tunnel:
yield _Runtime(wire, tunnel, cert, outage)
@pytest.fixture(scope="module")
def agentcore(
module_gateway: Gateway, runtime: _Runtime, tmp_path_factory: pytest.TempPathFactory
) -> Iterator[Gateway]:
directory: Final = tmp_path_factory.mktemp("agentcore-proxy")
with owned_proxy(
module_gateway,
directory,
_proxy_environment(runtime, directory),
config=_write_config(directory),
remove_environment=_scrubbed(),
workers=2,
) as candidate:
yield candidate
def _decide(gateway: Gateway, model: str, route: _Route = _SYSTEMONE, **extra: JsonValue) -> httpx.Response:
return gateway.request("POST", route.path, {"model": model, **route.body, **extra})
def _authorization_fields(request: Request) -> dict[str, str]:
authorization: Final = request.headers.get("authorization", "")
assert authorization.startswith("AWS4-HMAC-SHA256 "), dict(request.headers)
return dict(part.split("=", 1) for part in authorization.removeprefix("AWS4-HMAC-SHA256 ").split(", "))
def _assert_signed(request: Request, region: str = _US_EAST) -> None:
fields: Final = _authorization_fields(request)
access_key, scope = fields["Credential"].split("/", 1)
assert access_key == _ACCESS_KEY, fields
assert scope == f"{request.headers['x-amz-date'][:8]}/{region}/bedrock-agentcore/aws4_request", fields
signed: Final = fields["SignedHeaders"]
assert {"host", "x-amz-date", _SESSION_HEADER}.issubset(signed.split(";")), signed
expected: Final = signature(
"POST", encoded_path(request.target), request.headers, signed, request.body, _SECRET_KEY, scope
)
assert fields["Signature"] == expected[1], (request.target, fields)
def _assert_answered(response: httpx.Response, model: str, route: _Route = _SYSTEMONE) -> None:
assert response.status_code == 200, response.text
assert response.json() == route.answer, response.text
assert response.headers["x-litellm-model-group"] == model, dict(response.headers)
def _assert_runtime_target(request: Request, runtime: str, region: str = _US_EAST) -> None:
assert request.method == "POST", request.method
assert request.target == f"/runtimes/{quote(_arn(runtime, region), safe='')}/invocations", request.target
assert request.headers["host"] == _host(region), dict(request.headers)
def _assert_runtime_call(request: Request, runtime: str, region: str = _US_EAST, route: _Route = _SYSTEMONE) -> None:
_assert_runtime_target(request, runtime, region)
assert json.loads(request.body) == route.upstream, request.body
def _canonical(body: JsonValue | Mapping[str, JsonValue]) -> str:
return json.dumps(body, sort_keys=True)
def _spend_row(call_id: str) -> dict[str, JsonValue]:
rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70)
return rows[0]
def _spend_rows(call_ids: Sequence[str]) -> list[dict[str, JsonValue]]:
query: Final = (
"SELECT request_id, status FROM \"LiteLLM_SpendLogs\" WHERE request_id = ANY(string_to_array(%s, ','))"
)
return eventually(
lambda: read_rows(query, (",".join(call_ids),)), lambda found: len(found) >= len(call_ids), seconds=70
)
def _connect_targets(runtime: _Runtime) -> frozenset[str]:
return frozenset(runtime.tunnel.targets())
def test_an_arn_deployment_answers_on_every_decisions_route_signed_for_bedrock_agentcore(
agentcore: Gateway, runtime: _Runtime
) -> None:
runtime.reset()
responses: Final = tuple(_decide(agentcore, _SIGNED, route) for route in _ROUTES)
for response, route in zip(responses, _ROUTES, strict=True):
_assert_answered(response, _SIGNED, route)
assert _connect_targets(runtime) == {f"{_host(_US_EAST)}:443"}
calls: Final = runtime.calls()
assert len(calls) == len(_ROUTES), calls
for call, route in zip(calls, _ROUTES, strict=True):
_assert_runtime_call(call, "audit_signed", route=route)
assert call.headers[_SESSION_HEADER] == _default_session("audit_signed"), dict(call.headers)
_assert_signed(call)
for response in responses:
row = _spend_row(response.headers["x-litellm-call-id"])
assert row == {
"status": "success",
"call_type": "asystemone",
"custom_llm_provider": "strands_decider",
"model_group": _SIGNED,
"api_base": _invocations_url("audit_signed"),
"prompt_tokens": 211,
"completion_tokens": 2,
}, row
def test_the_arn_region_picks_the_host_and_signing_scope_over_aws_region_name(
agentcore: Gateway, runtime: _Runtime
) -> None:
runtime.reset()
_assert_answered(_decide(agentcore, _REGION), _REGION)
assert _connect_targets(runtime) == {f"{_host(_EU_WEST)}:443"}
(call,) = runtime.calls()
_assert_runtime_call(call, "audit_region", _EU_WEST)
_assert_signed(call, _EU_WEST)
def test_a_configured_session_id_is_sent_as_written_and_signed(agentcore: Gateway, runtime: _Runtime) -> None:
runtime.reset()
_assert_answered(_decide(agentcore, _SESSION), _SESSION)
_assert_answered(_decide(agentcore, _LONG), _LONG)
calls: Final = runtime.calls()
(configured,) = _calls_to(calls, "audit_session")
(long,) = _calls_to(calls, "audit_session_long")
assert configured.headers[_SESSION_HEADER] == _CONFIGURED_SESSION, dict(configured.headers)
assert long.headers[_SESSION_HEADER] == _LONG_SESSION
_assert_signed(configured)
_assert_signed(long)
def test_a_deployment_api_key_sends_bearer_and_the_session_without_sigv4(agentcore: Gateway, runtime: _Runtime) -> None:
runtime.reset()
_assert_answered(_decide(agentcore, _BEARER), _BEARER)
(call,) = runtime.calls()
_assert_runtime_call(call, "audit_bearer")
assert call.headers["authorization"] == f"Bearer {_JWT}", dict(call.headers)
assert call.headers[_SESSION_HEADER] == _default_session("audit_bearer"), dict(call.headers)
assert "x-amz-date" not in call.headers, dict(call.headers)
def test_reserved_extra_headers_are_dropped_and_the_rest_are_kept(agentcore: Gateway, runtime: _Runtime) -> None:
runtime.reset()
_assert_answered(_decide(agentcore, _HEADERS), _HEADERS)
caller_headers: Final[dict[str, JsonValue]] = {
"x-amz-security-token": "caller-forged-token",
"X-Amzn-Bedrock-AgentCore-Runtime-Session-Id": "caller-forged-session-0000000000000000001",
"x-audit-caller": "caller-trace",
}
_assert_answered(_decide(agentcore, _HEADERS, extra_headers=caller_headers), _HEADERS)
deployment, caller = runtime.calls()
for call in (deployment, caller):
_assert_runtime_call(call, "audit_headers")
assert call.headers[_SESSION_HEADER] == _default_session("audit_headers"), dict(call.headers)
assert "x-amz-target" not in call.headers, dict(call.headers)
assert "x-amz-security-token" not in call.headers, dict(call.headers)
_assert_signed(call)
assert deployment.headers["x-audit-trace"] == "deployment-trace", dict(deployment.headers)
assert caller.headers["x-audit-caller"] == "caller-trace", dict(caller.headers)
def test_an_opted_in_caller_session_id_becomes_the_header_and_stays_out_of_the_body(
agentcore: Gateway, runtime: _Runtime
) -> None:
runtime.reset()
chosen: Final = f"agentcore-audit-caller-session-{uuid.uuid4().hex}"
_assert_answered(_decide(agentcore, _OPT_IN, agentcore_runtime_session_id=chosen), _OPT_IN)
_assert_answered(_decide(agentcore, _OPT_IN), _OPT_IN)
first, second = runtime.calls()
for call in (first, second):
_assert_runtime_call(call, "audit_opt_in")
_assert_signed(call)
assert first.headers[_SESSION_HEADER] == chosen, dict(first.headers)
assert second.headers[_SESSION_HEADER] == _default_session("audit_opt_in"), dict(second.headers)
def test_an_arn_deployment_without_api_base_takes_the_runtime_from_the_environment(
agentcore: Gateway, runtime: _Runtime
) -> None:
runtime.reset()
response: Final = _decide(agentcore, _ENV)
_assert_answered(response, _ENV)
(call,) = runtime.calls()
_assert_runtime_call(call, "audit_env")
_assert_signed(call)
assert _spend_row(response.headers["x-litellm-call-id"])["api_base"] == _invocations_url("audit_env")
def test_an_arn_deployment_added_through_the_management_api_answers_signed(
agentcore: Gateway, runtime: _Runtime
) -> None:
runtime.reset()
with agentcore.scenario() as scenario:
model: Final = scenario.model(
model_info=None,
model=_MODEL,
api_base=_arn("audit_stored"),
api_key=None,
aws_access_key_id=_ACCESS_KEY,
aws_secret_access_key=_SECRET_KEY,
)
_assert_answered(_decide(agentcore, model), model)
(call,) = runtime.calls()
_assert_runtime_call(call, "audit_stored")
_assert_signed(call)
def test_health_and_test_connection_probe_the_runtime_signed(agentcore: Gateway, runtime: _Runtime) -> None:
runtime.reset()
health: Final = agentcore.request("GET", "/health", params={"model": _HEALTH})
assert health.status_code == 200, health.text
assert (health.json()["healthy_count"], health.json()["unhealthy_count"]) == (1, 0), health.text
(probe,) = runtime.calls()
assert probe.target == f"/runtimes/{quote(_arn('audit_health'), safe='')}/invocations", probe.target
_assert_signed(probe)
connection: Final = agentcore.request(
"POST",
"/health/test_connection",
{
"litellm_params": {"model": _MODEL, "api_base": _arn("audit_test_connection"), **_SIGNING},
"mode": "evaluation",
},
)
assert connection.status_code == 200, connection.text
assert connection.json()["status"] == "success", connection.text
(tested,) = runtime.calls()
assert _runtime_of(tested) == "audit_test_connection", tested.target
_assert_signed(tested)
@pytest.mark.parametrize(("code", "status"), tuple(_ERROR_STATUS.items()), ids=tuple(_ERROR_STATUS))
def test_a_runtime_error_reply_maps_to_its_status_with_the_runtime_message(
agentcore: Gateway, runtime: _Runtime, code: str, status: int
) -> None:
runtime.reset()
model: Final = f"agentcore-audit-error-{code}"
response: Final = _decide(agentcore, model)
assert response.status_code == status, response.text
assert f"Strands Decider runtime error '{code}': runtime refused with {code}" in response.text, response.text
assert len(_calls_to(runtime.calls(), f"audit_error_{code}")) == 1
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (row["status"], row["model_group"]) == ("failure", model), row
@pytest.mark.parametrize("status", tuple(_HTTP_ERRORS), ids=tuple(str(status) for status in _HTTP_ERRORS))
def test_a_runtime_http_error_keeps_its_status_and_message(agentcore: Gateway, runtime: _Runtime, status: int) -> None:
runtime.reset()
model: Final = f"agentcore-audit-status-{status}"
response: Final = _decide(agentcore, model)
assert response.status_code == status, response.text
assert f"scripted {_HTTP_ERRORS[status]}" in response.text, response.text
assert len(_calls_to(runtime.calls(), f"audit_status_{status}")) == 1
assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure"
@pytest.mark.parametrize("label", tuple(_MALFORMED_SESSIONS))
def test_a_malformed_configured_session_id_fails_only_its_own_deployment(
agentcore: Gateway, runtime: _Runtime, label: str
) -> None:
runtime.reset()
model: Final = f"agentcore-audit-session-{label}"
response: Final = _decide(agentcore, model)
calls: Final = _calls_to(runtime.calls(), f"audit_session_{label}")
if label == "empty":
_assert_answered(response, model)
(call,) = calls
assert call.headers[_SESSION_HEADER] == "", dict(call.headers)
else:
assert response.status_code == 500, response.text
assert "agentcore_runtime_session_id" in response.text, response.text
assert calls == ()
_assert_answered(_decide(agentcore, _SIGNED), _SIGNED)
def test_a_deployment_without_aws_credentials_fails_before_dialing_the_runtime(
agentcore: Gateway, runtime: _Runtime
) -> None:
runtime.reset()
response: Final = _decide(agentcore, _UNSIGNED)
assert response.status_code == 500, response.text
assert "credentials" in response.text.lower(), response.text
assert runtime.calls() == ()
assert _connect_targets(runtime) == frozenset()
_assert_answered(_decide(agentcore, _SIGNED), _SIGNED)
@pytest.mark.parametrize("label", tuple(_NEAR_MISS_ARNS))
def test_a_near_miss_arn_keeps_the_plain_api_base_path(agentcore: Gateway, runtime: _Runtime, label: str) -> None:
runtime.reset()
response: Final = _decide(agentcore, f"agentcore-audit-near-miss-{label}")
assert 500 <= response.status_code < 600, response.text
assert "litellm.APIConnectionError" in response.text, response.text
assert _connect_targets(runtime) == frozenset()
assert runtime.calls() == ()
def test_a_dropped_connection_is_a_connection_error_after_one_attempt(agentcore: Gateway, runtime: _Runtime) -> None:
runtime.reset()
response: Final = _decide(agentcore, _DROPPED)
assert response.status_code == 500, response.text
assert "litellm.APIConnectionError" in response.text, response.text
assert len(_calls_to(runtime.calls(), "audit_dropped")) == 1
assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure"
def _burst_routes(size: int) -> tuple[_Route, ...]:
return tuple(_ROUTES[index % len(_ROUTES)] for index in range(size))
def _burst(gateway: Gateway, model: str, size: int) -> tuple[httpx.Response, ...]:
with ThreadPoolExecutor(max_workers=size) as pool:
return tuple(pool.map(partial(_decide, gateway, model), _burst_routes(size)))
def _health_counts(gateway: Gateway, model: str) -> tuple[int, int, int]:
response: Final = gateway.request("GET", "/health", params={"model": model})
body: Final = JSON_OBJECT.validate_json(response.content)
healthy: Final = body["healthy_count"]
unhealthy: Final = body["unhealthy_count"]
assert isinstance(healthy, int) and isinstance(unhealthy, int), body
return response.status_code, healthy, unhealthy
def test_a_runtime_outage_mid_burst_fails_each_call_once_and_recovers_signed(
agentcore: Gateway, runtime: _Runtime
) -> None:
runtime.reset()
runtime.outage.set()
failed: Final = _burst(agentcore, _OUTAGE, 10)
assert [response.status_code for response in failed] == [500] * 10, [response.text for response in failed]
assert _health_counts(agentcore, _OUTAGE) == (503, 0, 1)
runtime.outage.clear()
runtime.calls()
served: Final = _burst(agentcore, _OUTAGE, 20)
for response, route in zip(served, _burst_routes(20), strict=True):
_assert_answered(response, _OUTAGE, route)
calls: Final = runtime.calls()
assert len(calls) == 20, calls
for call in calls:
_assert_runtime_target(call, "audit_outage")
_assert_signed(call)
assert sorted(_canonical(_JSON.validate_json(call.body)) for call in calls) == sorted(
_canonical(route.upstream) for route in _burst_routes(20)
)
assert _health_counts(agentcore, _OUTAGE) == (200, 1, 0)
expected: Final = {
**{response.headers["x-litellm-call-id"]: "failure" for response in failed},
**{response.headers["x-litellm-call-id"]: "success" for response in served},
}
assert len(expected) == 30, expected
rows: Final = _spend_rows(tuple(expected))
assert sorted((str(row["request_id"]), row["status"]) for row in rows) == sorted(expected.items()), rows
async def test_sdk_sync_and_async_calls_reach_the_runtime_signed(
runtime: _Runtime, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
runtime.reset()
for name in _scrubbed():
monkeypatch.delenv(name)
for name, value in {
"HTTPS_PROXY": runtime.tunnel.url,
"NO_PROXY": "127.0.0.1,localhost",
"SSL_CERT_FILE": str(runtime.cert),
**_isolated_aws(tmp_path),
}.items():
monkeypatch.setenv(name, value)
litellm.in_memory_llm_clients_cache.flush_cache()
try:
synchronous: Final = litellm.decisions(
model=_MODEL,
state=_STATE,
questions=_SDK_QUESTIONS,
api_base=_arn("audit_sdk"),
aws_access_key_id=_ACCESS_KEY,
aws_secret_access_key=_SECRET_KEY,
)
asynchronous: Final = await litellm.asystemone(
model=_MODEL,
state=_STATE,
questions=_SDK_QUESTIONS,
api_base=_arn("audit_sdk"),
aws_access_key_id=_ACCESS_KEY,
aws_secret_access_key=_SECRET_KEY,
)
monkeypatch.setenv("STRANDS_DECIDER_API_KEY", _ENV_JWT)
bearer: Final = await litellm.asystemone(
model=_MODEL, state=_STATE, questions=_SDK_QUESTIONS, api_base=_arn("audit_sdk_env_key")
)
finally:
litellm.in_memory_llm_clients_cache.flush_cache()
for response in (synchronous, asynchronous, bearer):
assert response.model_dump(mode="json") == {"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE}
calls: Final = runtime.calls()
signed: Final = _calls_to(calls, "audit_sdk")
assert len(signed) == 2, calls
for call in signed:
_assert_runtime_call(call, "audit_sdk")
assert call.headers[_SESSION_HEADER] == _default_session("audit_sdk"), dict(call.headers)
_assert_signed(call)
(env_key,) = _calls_to(calls, "audit_sdk_env_key")
assert env_key.headers["authorization"] == f"Bearer {_ENV_JWT}", dict(env_key.headers)
assert "x-amz-date" not in env_key.headers, dict(env_key.headers)
def _answering(request: Request) -> Reply:
return _json_reply({"model": _BODY_MODEL, "answers": _ANSWERS, "usage": _USAGE})
def _envelope(request: Request) -> Reply:
return _json_reply({"error": {"code": "bad_request", "message": "plain server refused"}})
def _plain_model(scenario: Scenario, wire: Wire, **extra: JsonValue) -> str:
return scenario.model(model_info=None, model=_MODEL, api_base=wire.url, api_key=None, **extra)
def test_a_request_body_session_id_is_refused_on_every_route_without_an_upstream_call(gateway: Gateway) -> None:
with wire_server(_answering) as wire, gateway.scenario() as scenario:
model: Final = _plain_model(scenario, wire)
responses: Final = tuple(
_decide(gateway, model, route, agentcore_runtime_session_id=f"caller-{uuid.uuid4().hex}")
for route in _ROUTES
)
for response in responses:
assert response.status_code == 401, response.text
assert "agentcore_runtime_session_id is not allowed in request body" in response.text, response.text
assert wire.drain() == ()
def test_a_plain_api_base_opted_into_the_session_param_sends_no_session_header_or_body_field(
gateway: Gateway,
) -> None:
with wire_server(_answering) as wire, gateway.scenario() as scenario:
model: Final = _plain_model(
scenario, wire, configurable_clientside_auth_params=["agentcore_runtime_session_id"]
)
_assert_answered(_decide(gateway, model, agentcore_runtime_session_id=f"caller-{uuid.uuid4().hex}"), model)
(call,) = wire.drain()
assert call.target == "/v1/systemone", call.target
assert _SESSION_HEADER not in call.headers, dict(call.headers)
assert json.loads(call.body) == _UPSTREAM_BODY, call.body
def test_a_plain_api_base_error_envelope_keeps_the_unexpected_response_server_error(gateway: Gateway) -> None:
with wire_server(_envelope) as wire, gateway.scenario() as scenario:
model: Final = _plain_model(scenario, wire)
response: Final = _decide(gateway, model)
assert response.status_code == 500, response.text
assert "Strands Decider runtime error" not in response.text, response.text
assert len(wire.drain()) == 1
def test_a_plain_api_base_forwards_amazon_shaped_extra_headers_unchanged(gateway: Gateway) -> None:
headers: Final[dict[str, JsonValue]] = {
"x-amz-target": "plain-target",
"X-Amzn-Bedrock-AgentCore-Runtime-Session-Id": "plain-session-kept",
"x-audit-trace": "plain-trace",
}
with wire_server(_answering) as wire, gateway.scenario() as scenario:
model: Final = _plain_model(scenario, wire, extra_headers=headers)
_assert_answered(_decide(gateway, model), model)
(call,) = wire.drain()
assert (call.headers["x-amz-target"], call.headers[_SESSION_HEADER], call.headers["x-audit-trace"]) == (
"plain-target",
"plain-session-kept",
"plain-trace",
), dict(call.headers)
assert "authorization" not in call.headers, dict(call.headers)
assert json.loads(call.body) == _UPSTREAM_BODY, call.body
def _worst_case_api_base() -> str:
return "arn:aws" + "-a" * 131072 + "!"
def test_a_worst_case_runtime_arn_pattern_input_is_classified_in_linear_time(
gateway: Gateway, record_property: Callable[[str, object], None]
) -> None:
worst: Final = _worst_case_api_base()
started: Final = time.perf_counter()
url: Final = StrandsDeciderDecisionsConfig().get_complete_url(worst, _BODY_MODEL)
classified: Final = time.perf_counter() - started
record_property("classifier_seconds", f"{classified:.6f}")
assert url == f"{worst}/v1/systemone"
assert classified < 0.5, classified
with wire_server(_answering) as wire, gateway.scenario() as scenario:
model: Final = _plain_model(scenario, wire, configurable_clientside_auth_params=["api_base"])
with ThreadPoolExecutor(max_workers=2) as pool:
timed: Final = pool.submit(_timed_decide, gateway, model, worst)
liveliness_started: Final = time.perf_counter()
liveliness: Final = gateway.request("GET", "/health/liveliness")
liveliness_seconds: Final = time.perf_counter() - liveliness_started
response, decide_seconds = timed.result()
record_property("decide_seconds", f"{decide_seconds:.6f}")
record_property("liveliness_seconds", f"{liveliness_seconds:.6f}")
assert liveliness.status_code == 200, liveliness.text
assert 500 <= response.status_code < 600, response.text
assert "litellm.APIConnectionError" in response.text, response.text[:2000]
assert decide_seconds < 5, decide_seconds
assert liveliness_seconds < 2, liveliness_seconds
assert wire.drain() == ()
def _timed_decide(gateway: Gateway, model: str, api_base: str) -> tuple[httpx.Response, float]:
started: Final = time.perf_counter()
response: Final = _decide(gateway, model, api_base=api_base)
return response, time.perf_counter() - started

4
tests/integration/proxy_config.yaml
View file

@ -346,6 +346,10 @@ model_list:
litellm_params:
model: strands_decider/strands-decider-2B-hobson-v19
api_base: http://127.0.0.1:8191
- model_name: hosted_vllm/Qwen/Qwen3-0.6B
litellm_params:
model: hosted_vllm/Qwen/Qwen3-0.6B
api_base: http://127.0.0.1:8191
- model_name: azure_ai/decision-1
litellm_params:
model: azure_ai/decision-1

1023
tests/integration/routing/test_complexity_router_databricks_classifier.py Normal file
View file

File diff suppressed because it is too large Load diff

4
tests/integration/translation/case.py
View file

@ -7,7 +7,8 @@ from pydantic import JsonValue
@dataclass(frozen=True, slots=True, kw_only=True)
class TranslationTestCase:
"""One request through the proxy to a deployment in `proxy_config.yaml`: what the test sends to LiteLLM, the
exact request the provider must receive, the fake provider's reply, and the exact response LiteLLM must return."""
exact request the provider must receive, the fake provider's status and reply, and the exact response LiteLLM must
return."""
scenario: str
litellm_endpoint: str
@ -15,6 +16,7 @@ class TranslationTestCase:
expected_provider_endpoint: str
expected_provider_headers: Mapping[str, str]
expected_provider_request: Mapping[str, JsonValue]
mock_provider_status_code: int = 200
mock_provider_response: Mapping[str, JsonValue]
expected_litellm_status_code: int = 200
expected_litellm_response: Mapping[str, JsonValue]

97
tests/integration/translation/decisions/bases/cloudflare.py
View file

@ -6,6 +6,103 @@ from integration.translation.case import TranslationTestCase
"""
CLEF_TEST_CASE: Final = TranslationTestCase(
scenario="basic",
litellm_endpoint="/v1/decisions",
litellm_request={
"model": "cloudflare/@cf/cloudflare/clef",
"input": "Ticket (billing): The export job hangs at 99% and never finishes",
"questions": [
{"type": "predicate", "name": "defect", "instructions": "Is this a defect?"},
{
"type": "choice",
"name": "severity",
"instructions": "How severe is it?",
"choices": [
{"value": "low", "description": "cosmetic"},
{"value": "high", "description": "blocks users"},
],
},
{
"type": "score",
"name": "confidence",
"instructions": "How sure are you?",
"levels": [{"label": "unsure"}, {"label": "sure"}],
},
],
"cache": {"no-cache": True},
},
expected_provider_endpoint="/ai/run/@cf/cloudflare/clef",
expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"},
expected_provider_request={
"model": "clef",
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
"questions": {
"defect": {"type": "noul", "instructions": "Is this a defect?"},
"severity": {
"type": "choice",
"instructions": "How severe is it?",
"criteria": {"low": "cosmetic", "high": "blocks users"},
},
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
},
},
mock_provider_response={
"result": {
"model": "clef",
"answers": {
"defect": {"type": "noul", "noul": 0.9345},
"severity": {
"type": "choice",
"choice": "high",
"confidence": 0.8067,
"probabilities": {"low": 0.0509, "high": 0.9491},
},
"confidence": {
"type": "score",
"score": 0.9036,
"confidence": 0.6515,
"legend": {"0": "unsure", "1": "sure"},
"probabilities": {"0": 0.0964, "1": 0.9036},
},
},
"usage": {"input_tokens": 290, "output_tokens": 0},
},
"success": True,
"errors": [],
"messages": [],
},
expected_litellm_response={
"model": "clef",
"answers": [
{"type": "predicate", "name": "defect", "probability": 0.9345},
{
"type": "choice",
"name": "severity",
"choice": "high",
"probabilities": [{"value": "low", "probability": 0.0509}, {"value": "high", "probability": 0.9491}],
"confidence": 0.8067,
},
{
"type": "score",
"name": "confidence",
"score": 0.9036,
"probabilities": [
{"value": 0, "label": "unsure", "probability": 0.0964},
{"value": 1, "label": "sure", "probability": 0.9036},
],
"confidence": 0.6515,
},
],
"usage": {
"input_tokens": 290,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens": 0,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 290,
},
},
)
CLEF_SYSTEMONE_TEST_CASE: Final = TranslationTestCase(
scenario="systemone",
litellm_endpoint="/v1/systemone",
litellm_request={
"model": "cloudflare/@cf/cloudflare/clef",

175
tests/integration/translation/decisions/bases/hosted_vllm.py Normal file
View file

@ -0,0 +1,175 @@
import json
from dataclasses import replace
from typing import Final
from integration.translation.case import TranslationTestCase
"""Provider request and reply shape from a real vLLM serve (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-09 from a vLLM CPU build of vllm main 702ce313f serving Qwen/Qwen3-0.6B.
"""
QWEN3_0_6B_TEST_CASE: Final = TranslationTestCase(
scenario="basic",
litellm_endpoint="/v1/systemone",
litellm_request={
"model": "hosted_vllm/Qwen/Qwen3-0.6B",
"state": "Help! My payouts have been failing for 3 days!",
"questions": {
"is_urgent": {
"type": "choice",
"instructions": "Does this convey urgency?",
"criteria": {"yes": "the message conveys urgency", "no": "the message does not convey urgency"},
}
},
"cache": {"no-cache": True},
},
expected_provider_endpoint="/v1/systemone",
expected_provider_headers={"content-type": "application/json"},
expected_provider_request={
"model": "Qwen/Qwen3-0.6B",
"state": "Help! My payouts have been failing for 3 days!",
"questions": {
"is_urgent": {
"type": "choice",
"instructions": "Does this convey urgency?",
"criteria": {"yes": "the message conveys urgency", "no": "the message does not convey urgency"},
}
},
},
mock_provider_response={
"id": "decision-9c26c028d7f4457e",
"object": "structured_decision",
"created": 1791517765,
"model": "Qwen/Qwen3-0.6B",
"answers": {
"is_urgent": {
"type": "choice",
"choice": "yes",
"probabilities": {"yes": 0.9669140238756562, "no": 0.03308597612434374},
"confidence": 0.9649145337527952,
}
},
"usage": {"input_tokens": 61, "output_tokens": 1},
"diagnostics": {"is_urgent": {"label_mass": 0.9979320910923947, "argmax_is_label": True}},
},
expected_litellm_response={
"id": "decision-9c26c028d7f4457e",
"object": "structured_decision",
"created": 1791517765,
"model": "Qwen/Qwen3-0.6B",
"answers": {
"is_urgent": {
"type": "choice",
"choice": "yes",
"probabilities": {"yes": 0.9669140238756562, "no": 0.03308597612434374},
"confidence": 0.9649145337527952,
}
},
"usage": {"input_tokens": 61, "output_tokens": 1},
"diagnostics": {"is_urgent": {"label_mass": 0.9979320910923947, "argmax_is_label": True}},
},
)
"""OpenAI Decisions client shape in, vLLM /v1/systemone body out. Mock reply captured live on 2026-10-09 from the same
vLLM build.
"""
QWEN3_0_6B_DECISIONS_TEST_CASE: Final = replace(
QWEN3_0_6B_TEST_CASE,
scenario="decisions",
litellm_endpoint="/v1/decisions",
litellm_request={
"model": "hosted_vllm/Qwen/Qwen3-0.6B",
"input": "The checkout page crashed twice today",
"questions": [
{
"type": "choice",
"name": "is_defect",
"instructions": "Is this a defect?",
"choices": [{"value": "yes"}, {"value": "no"}],
}
],
"cache": {"no-cache": True},
},
expected_provider_request={
"model": "Qwen/Qwen3-0.6B",
"state": "The checkout page crashed twice today",
"questions": {
"is_defect": {"type": "choice", "instructions": "Is this a defect?", "criteria": {"yes": None, "no": None}}
},
},
mock_provider_response={
"id": "decision-91ce41627dbf016f",
"object": "structured_decision",
"created": 1791577835,
"model": "Qwen/Qwen3-0.6B",
"answers": {
"is_defect": {
"type": "choice",
"choice": "yes",
"probabilities": {"yes": 0.8670357477770336, "no": 0.13296425222296632},
"confidence": 0.8669148648509282,
}
},
"usage": {"input_tokens": 43, "output_tokens": 1},
"diagnostics": {"is_defect": {"label_mass": 0.9998605790748358, "argmax_is_label": True}},
},
expected_litellm_response={
"model": "Qwen/Qwen3-0.6B",
"answers": [
{
"type": "choice",
"name": "is_defect",
"choice": "yes",
"probabilities": [
{"value": "yes", "probability": 0.8670357477770336},
{"value": "no", "probability": 0.13296425222296632},
],
"confidence": 0.8669148648509282,
}
],
"usage": {
"input_tokens": 43,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens": 1,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 44,
},
},
)
"""vLLM only scores `choice` questions, so an OpenAI `predicate` goes out as `noul` and comes back as vLLM's 400,
which LiteLLM passes through as a 400.
"""
VLLM_NOUL_REJECTED: Final = {
"error": {
"message": "unknown question type 'noul'; supported: ['choice']",
"type": "BadRequestError",
"param": None,
"code": 400,
}
}
QWEN3_0_6B_PREDICATE_REJECTED_TEST_CASE: Final = replace(
QWEN3_0_6B_DECISIONS_TEST_CASE,
scenario="predicate_rejected",
litellm_request={
"model": "hosted_vllm/Qwen/Qwen3-0.6B",
"input": "The checkout page crashed twice today",
"questions": [{"type": "predicate", "name": "is_bug", "instructions": "Is this a bug?"}],
"cache": {"no-cache": True},
},
expected_provider_request={
"model": "Qwen/Qwen3-0.6B",
"state": "The checkout page crashed twice today",
"questions": {"is_bug": {"type": "noul", "instructions": "Is this a bug?"}},
},
mock_provider_status_code=400,
mock_provider_response=VLLM_NOUL_REJECTED,
expected_litellm_status_code=400,
expected_litellm_response={
"error": {
"message": f"litellm.BadRequestError: Hosted_vllmException - {json.dumps(VLLM_NOUL_REJECTED)}\n\n"
"LiteLLM: model group 'hosted_vllm/Qwen/Qwen3-0.6B' failed with the error above. No fallback was attempted.",
"type": "invalid_request_error",
"param": None,
"code": "400",
}
},
)

94
tests/integration/translation/decisions/bases/openrouter.py
View file

@ -6,6 +6,100 @@ from integration.translation.case import TranslationTestCase
"""
TYPESAFE_JEV_1_13_TEST_CASE: Final = TranslationTestCase(
scenario="basic",
litellm_endpoint="/v1/decisions",
litellm_request={
"model": "openrouter/typesafe/jev-1.13",
"input": "Ticket (billing): The export job hangs at 99% and never finishes",
"questions": [
{"type": "predicate", "name": "defect", "instructions": "Is this a defect?"},
{
"type": "choice",
"name": "severity",
"instructions": "How severe is it?",
"choices": [
{"value": "low", "description": "cosmetic"},
{"value": "high", "description": "blocks users"},
],
},
{
"type": "score",
"name": "confidence",
"instructions": "How sure are you?",
"levels": [{"label": "unsure"}, {"label": "sure"}],
},
],
"cache": {"no-cache": True},
},
expected_provider_endpoint="/alpha/decisions",
expected_provider_headers={"authorization": "Bearer synthetic-openrouter-key", "content-type": "application/json"},
expected_provider_request={
"model": "typesafe/jev-1.13",
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
"questions": {
"defect": {"type": "noul", "instructions": "Is this a defect?"},
"severity": {
"type": "choice",
"instructions": "How severe is it?",
"criteria": {"low": "cosmetic", "high": "blocks users"},
},
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
},
},
mock_provider_response={
"model": "typesafe/jev-1.13-20260917",
"answers": {
"defect": {"type": "noul", "noul": 0.81},
"severity": {
"type": "choice",
"choice": "high",
"confidence": 0.99,
"probabilities": {"low": 0.01, "high": 0.99},
},
"confidence": {
"type": "score",
"score": 0.5,
"confidence": 0,
"legend": {"0": "unsure", "1": "sure"},
"probabilities": {"0": 0.5, "1": 0.5},
},
},
"usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05},
"id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki",
"provider": "TypeSafe",
},
expected_litellm_response={
"model": "typesafe/jev-1.13-20260917",
"answers": [
{"type": "predicate", "name": "defect", "probability": 0.81},
{
"type": "choice",
"name": "severity",
"choice": "high",
"probabilities": [{"value": "low", "probability": 0.01}, {"value": "high", "probability": 0.99}],
"confidence": 0.99,
},
{
"type": "score",
"name": "confidence",
"score": 0.5,
"probabilities": [
{"value": 0, "label": "unsure", "probability": 0.5},
{"value": 1, "label": "sure", "probability": 0.5},
],
"confidence": 0,
},
],
"usage": {
"input_tokens": 377,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens": 62,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 439,
},
},
)
TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE: Final = TranslationTestCase(
scenario="systemone",
litellm_endpoint="/v1/systemone",
litellm_request={
"model": "openrouter/typesafe/jev-1.13",

95
tests/integration/translation/decisions/bases/perplexity.py
View file

@ -6,6 +6,101 @@ from integration.translation.case import TranslationTestCase
"""
PPLX_DECIDER_V1_27B_TEST_CASE: Final = TranslationTestCase(
scenario="basic",
litellm_endpoint="/v1/decisions",
litellm_request={
"model": "perplexity/pplx-decider-v1-27b",
"input": "Ticket (billing): The export job hangs at 99% and never finishes",
"questions": [
{"type": "predicate", "name": "defect", "instructions": "Is this a defect?"},
{
"type": "choice",
"name": "severity",
"instructions": "How severe is it?",
"choices": [
{"value": "low", "description": "cosmetic"},
{"value": "high", "description": "blocks users"},
],
},
{
"type": "score",
"name": "confidence",
"instructions": "How sure are you?",
"levels": [{"label": "unsure"}, {"label": "sure"}],
},
],
"cache": {"no-cache": True},
},
expected_provider_endpoint="/v1/decisions",
expected_provider_headers={"authorization": "Bearer synthetic-perplexity-key", "content-type": "application/json"},
expected_provider_request={
"model": "pplx-decider-v1-27b",
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
"questions": {
"defect": {"type": "noul", "instructions": "Is this a defect?"},
"severity": {
"type": "choice",
"instructions": "How severe is it?",
"criteria": {"low": "cosmetic", "high": "blocks users"},
},
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
},
},
mock_provider_response={
"model": "pplx-decider-v1-27b",
"answers": {
"defect": {"type": "noul", "noul": 0.9989100737587077},
"severity": {
"type": "choice",
"choice": "high",
"confidence": 0.9964631215356778,
"probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389},
},
"confidence": {
"type": "score",
"score": 0.07367392327139817,
"confidence": 0.8526521534572037,
"legend": {"0": "unsure", "1": "sure"},
"probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817},
},
},
"usage": {"input_tokens": 318, "output_tokens": 3},
},
expected_litellm_response={
"model": "pplx-decider-v1-27b",
"answers": [
{"type": "predicate", "name": "defect", "probability": 0.9989100737587077},
{
"type": "choice",
"name": "severity",
"choice": "high",
"probabilities": [
{"value": "low", "probability": 0.0017684392321610232},
{"value": "high", "probability": 0.9982315607678389},
],
"confidence": 0.9964631215356778,
},
{
"type": "score",
"name": "confidence",
"score": 0.07367392327139817,
"probabilities": [
{"value": 0, "label": "unsure", "probability": 0.9263260767286018},
{"value": 1, "label": "sure", "probability": 0.07367392327139817},
],
"confidence": 0.8526521534572037,
},
],
"usage": {
"input_tokens": 318,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens": 3,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 321,
},
},
)
PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE: Final = TranslationTestCase(
scenario="systemone",
litellm_endpoint="/v1/systemone",
litellm_request={
"model": "perplexity/pplx-decider-v1-27b",

34
tests/integration/translation/decisions/bases/strands_decider.py
View file

@ -6,6 +6,40 @@ from integration.translation.case import TranslationTestCase
"""
STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE: Final = TranslationTestCase(
scenario="basic",
litellm_endpoint="/v1/decisions",
litellm_request={
"model": "strands_decider/strands-decider-2B-hobson-v19",
"input": "Help! My payouts have been failing for 3 days!",
"questions": [{"type": "predicate", "name": "is_urgent", "instructions": "Does this convey urgency?"}],
"cache": {"no-cache": True},
},
expected_provider_endpoint="/v1/systemone",
expected_provider_headers={"content-type": "application/json"},
expected_provider_request={
"model": "strands-decider-2B-hobson-v19",
"state": "Help! My payouts have been failing for 3 days!",
"questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}},
},
mock_provider_response={
"model": "strands-decider-2B-hobson-v19",
"answers": {"is_urgent": {"type": "noul", "noul": 0.8277}},
"usage": {"input_tokens": 86, "output_tokens": 1},
"latency_ms": 140.03,
},
expected_litellm_response={
"model": "strands-decider-2B-hobson-v19",
"answers": [{"type": "predicate", "name": "is_urgent", "probability": 0.8277}],
"usage": {
"input_tokens": 86,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens": 1,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 87,
},
},
)
STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE: Final = TranslationTestCase(
scenario="systemone",
litellm_endpoint="/v1/systemone",
litellm_request={
"model": "strands_decider/strands-decider-2B-hobson-v19",

92
tests/integration/translation/decisions/bases/typesafe.py
View file

@ -6,6 +6,98 @@ from integration.translation.case import TranslationTestCase
"""
JEV_1_13_0_TEST_CASE: Final = TranslationTestCase(
scenario="basic",
litellm_endpoint="/v1/decisions",
litellm_request={
"model": "typesafe/jev-1.13.0",
"input": "Ticket (billing): The export job hangs at 99% and never finishes",
"questions": [
{"type": "predicate", "name": "defect", "instructions": "Is this a defect?"},
{
"type": "choice",
"name": "severity",
"instructions": "How severe is it?",
"choices": [
{"value": "low", "description": "cosmetic"},
{"value": "high", "description": "blocks users"},
],
},
{
"type": "score",
"name": "confidence",
"instructions": "How sure are you?",
"levels": [{"label": "unsure"}, {"label": "sure"}],
},
],
"cache": {"no-cache": True},
},
expected_provider_endpoint="/v1/systemone",
expected_provider_headers={"authorization": "Bearer synthetic-typesafe-key", "content-type": "application/json"},
expected_provider_request={
"model": "jev-1.13.0",
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
"questions": {
"defect": {"type": "noul", "instructions": "Is this a defect?"},
"severity": {
"type": "choice",
"instructions": "How severe is it?",
"criteria": {"low": "cosmetic", "high": "blocks users"},
},
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
},
},
mock_provider_response={
"model": "jev-1.13.0",
"answers": {
"defect": {"type": "noul", "noul": 0.78},
"severity": {
"type": "choice",
"choice": "high",
"confidence": 0.99,
"probabilities": {"low": 0.01, "high": 0.99},
},
"confidence": {
"type": "score",
"score": 0.51,
"confidence": 0.03,
"legend": {"0": "unsure", "1": "sure"},
"probabilities": {"0": 0.49, "1": 0.51},
},
},
"usage": {"input_tokens": 377, "output_tokens": 62},
},
expected_litellm_response={
"model": "jev-1.13.0",
"answers": [
{"type": "predicate", "name": "defect", "probability": 0.78},
{
"type": "choice",
"name": "severity",
"choice": "high",
"probabilities": [{"value": "low", "probability": 0.01}, {"value": "high", "probability": 0.99}],
"confidence": 0.99,
},
{
"type": "score",
"name": "confidence",
"score": 0.51,
"probabilities": [
{"value": 0, "label": "unsure", "probability": 0.49},
{"value": 1, "label": "sure", "probability": 0.51},
],
"confidence": 0.03,
},
],
"usage": {
"input_tokens": 377,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens": 62,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 439,
},
},
)
JEV_1_13_0_SYSTEMONE_TEST_CASE: Final = TranslationTestCase(
scenario="systemone",
litellm_endpoint="/v1/systemone",
litellm_request={
"model": "typesafe/jev-1.13.0",

4
tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py
View file

@ -2,10 +2,10 @@ import pytest
from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE
from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE, CLEF_SYSTEMONE_TEST_CASE
from integration.translation.runner import assert_translation
@pytest.mark.parametrize("case", [CLEF_TEST_CASE], ids=lambda case: case.id)
@pytest.mark.parametrize("case", [CLEF_TEST_CASE, CLEF_SYSTEMONE_TEST_CASE], ids=lambda case: case.id)
def test_decisions_basic_cloudflare(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
assert_translation(case, gateway, provider)

19
tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py Normal file
View file

@ -0,0 +1,19 @@
import pytest
from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.decisions.bases.hosted_vllm import (
QWEN3_0_6B_DECISIONS_TEST_CASE,
QWEN3_0_6B_PREDICATE_REJECTED_TEST_CASE,
QWEN3_0_6B_TEST_CASE,
)
from integration.translation.runner import assert_translation
@pytest.mark.parametrize(
"case",
[QWEN3_0_6B_TEST_CASE, QWEN3_0_6B_DECISIONS_TEST_CASE, QWEN3_0_6B_PREDICATE_REJECTED_TEST_CASE],
ids=lambda case: case.id,
)
def test_decisions_basic_hosted_vllm(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
assert_translation(case, gateway, provider)

9
tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py
View file

@ -2,10 +2,15 @@ import pytest
from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.decisions.bases.openrouter import TYPESAFE_JEV_1_13_TEST_CASE
from integration.translation.decisions.bases.openrouter import (
TYPESAFE_JEV_1_13_TEST_CASE,
TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE,
)
from integration.translation.runner import assert_translation
@pytest.mark.parametrize("case", [TYPESAFE_JEV_1_13_TEST_CASE], ids=lambda case: case.id)
@pytest.mark.parametrize(
"case", [TYPESAFE_JEV_1_13_TEST_CASE, TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE], ids=lambda case: case.id
)
def test_decisions_basic_openrouter(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
assert_translation(case, gateway, provider)

9
tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py
View file

@ -2,10 +2,15 @@ import pytest
from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.decisions.bases.perplexity import PPLX_DECIDER_V1_27B_TEST_CASE
from integration.translation.decisions.bases.perplexity import (
PPLX_DECIDER_V1_27B_TEST_CASE,
PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE,
)
from integration.translation.runner import assert_translation
@pytest.mark.parametrize("case", [PPLX_DECIDER_V1_27B_TEST_CASE], ids=lambda case: case.id)
@pytest.mark.parametrize(
"case", [PPLX_DECIDER_V1_27B_TEST_CASE, PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE], ids=lambda case: case.id
)
def test_decisions_basic_perplexity(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
assert_translation(case, gateway, provider)

11
tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py
View file

@ -2,10 +2,17 @@ import pytest
from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.decisions.bases.strands_decider import STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE
from integration.translation.decisions.bases.strands_decider import (
STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE,
STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE,
)
from integration.translation.runner import assert_translation
@pytest.mark.parametrize("case", [STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE], ids=lambda case: case.id)
@pytest.mark.parametrize(
"case",
[STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE, STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE],
ids=lambda case: case.id,
)
def test_decisions_basic_strands_decider(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
assert_translation(case, gateway, provider)

4
tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py
View file

@ -2,10 +2,10 @@ import pytest
from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE
from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE, JEV_1_13_0_SYSTEMONE_TEST_CASE
from integration.translation.runner import assert_translation
@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE], ids=lambda case: case.id)
@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE, JEV_1_13_0_SYSTEMONE_TEST_CASE], ids=lambda case: case.id)
def test_decisions_basic_typesafe(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
assert_translation(case, gateway, provider)

2
tests/integration/translation/runner.py
View file

@ -10,7 +10,7 @@ TRANSPORT_HEADERS: Final = frozenset({"host", "accept", "accept-encoding", "conn
def assert_translation(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
provider.expect(Reply(body=json.dumps(case.mock_provider_response).encode()))
provider.expect(Reply(status=case.mock_provider_status_code, body=json.dumps(case.mock_provider_response).encode()))
response: Final = gateway.request("POST", case.litellm_endpoint, case.litellm_request)
received: Final = provider.received()
assert [(request.method, request.target) for request in received] == [("POST", case.expected_provider_endpoint)]

41
tests/unit/conftest.py
View file

@ -45,17 +45,25 @@ import litellm # noqa: E402 # litellm reads LITELLM_LOCAL_MODEL_COST_MAP at im
import litellm.router as litellm_router_module # noqa: E402 # same import-time dependency
import litellm.utils as litellm_utils_module # noqa: E402 # same import-time dependency
from litellm._logging import ALL_LOGGERS # noqa: E402 # same import-time dependency
from litellm.anthropic_beta_headers_manager import (
reload_beta_headers_config, # noqa: E402 # same import-time dependency
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER # noqa: E402 # same import-time dependency
from litellm.anthropic_beta_headers_manager import reload_beta_headers_config # noqa: E402 # same import-time dependency
from litellm.litellm_core_utils.prompt_templates import factory as prompt_factory_module # noqa: E402 # same import-time dependency
from litellm.litellm_core_utils.prompt_templates import (
factory as prompt_factory_module, # noqa: E402 # same import-time dependency
)
from litellm.litellm_core_utils.prompt_templates import ( # noqa: E402 # same import-time dependency
image_handling as image_handling_module,
)
from litellm.llms.gemini.chat import transformation as gemini_chat_transformation_module # noqa: E402 # same import-time dependency
from litellm.llms.custom_httpx.async_client_cleanup import ( # noqa: E402 # same import-time dependency
close_litellm_async_clients,
)
from litellm.proxy.db import tool_registry_writer as tool_registry_writer_module # noqa: E402 # same import-time dependency
from litellm.llms.gemini.chat import (
transformation as gemini_chat_transformation_module, # noqa: E402 # same import-time dependency
)
from litellm.proxy.db import (
tool_registry_writer as tool_registry_writer_module, # noqa: E402 # same import-time dependency
)
LOOPBACK_HOSTS: Final = ["127.0.0.1", "::1", "localhost"]
AMBIENT_AZURE_CREDENTIAL_ENV_VARS: Final = (
@ -159,6 +167,31 @@ def _run_coroutine_if_needed(result: object) -> None:
loop.create_task(coroutine)
async def _complete_test_logging() -> None:
await GLOBAL_LOGGING_WORKER.flush()
await GLOBAL_LOGGING_WORKER.stop()
def _flush_completed_test_logging() -> None:
queue: Final = GLOBAL_LOGGING_WORKER._queue
if queue is None or queue._unfinished_tasks == 0:
return
worker_loop: Final = GLOBAL_LOGGING_WORKER._bound_loop
if worker_loop is None or worker_loop.is_closed():
_run_coroutine_if_needed(_complete_test_logging())
return
if worker_loop.is_running():
asyncio.run_coroutine_threadsafe(_complete_test_logging(), worker_loop).result()
return
worker_loop.run_until_complete(_complete_test_logging())
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_call() -> Iterator[None]:
yield
_flush_completed_test_logging()
def _close_handler_if_needed(handler: object) -> None:
close: Final = getattr(handler, "close", None)
if not callable(close):

744
tests/unit/decisions/test_main.py
View file

@ -2,6 +2,8 @@ from __future__ import annotations
import asyncio
import json
import threading
from concurrent.futures import wait
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
@ -9,10 +11,16 @@ from typing import Final
import httpx
import pytest
import respx
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
import litellm
from litellm.cost_calculator import get_response_cost_from_hidden_params
from litellm.constants import AWS_SIGNING_MAX_THREADS
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR
from litellm.types.decisions import (
ChoiceAnswer,
DecisionsResponse,
@ -21,6 +29,8 @@ from litellm.types.decisions import (
OpenAIDecisionResponse,
ScoreAnswer,
)
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType(
{
@ -67,6 +77,20 @@ _STRANDS_RESPONSE: Final[Mapping[str, object]] = {
"usage": {"input_tokens": 216, "output_tokens": 3},
"latency_ms": 3722.17,
}
_DATABRICKS_ENDPOINT: Final = "databricks-openjev-qwen35-4b"
_DATABRICKS_RESPONSE: Final[Mapping[str, object]] = {
"model": "/mosaicml/local_model",
"answers": {
"is_defect": {"type": "noul", "noul": 0.9},
"sentiment": {
"type": "choice",
"choice": "positive",
"confidence": 0.8,
"probabilities": {"positive": 0.8, "negative": 0.2},
},
},
"usage": {"input_tokens": 214, "output_tokens": 0},
}
_PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = (
(
"perplexity",
@ -139,7 +163,7 @@ async def test_adecisions_sends_the_provider_wire_contract(
) -> None:
route: Final = respx_mock.post(url).respond(json=_RESPONSE)
response: Final = await litellm.adecisions(
response: Final = await litellm.asystemone(
model=model,
state={"source": "unit-test"},
questions=_QUESTIONS,
@ -201,7 +225,7 @@ async def test_router_dispatches_typesafe_decisions_without_api_base(
)
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE)
response: Final = await router.adecisions(
response: Final = await router.asystemone(
model="jev",
state="router-test",
questions={
@ -232,7 +256,7 @@ async def test_router_dispatches_typesafe_decisions_without_api_base(
def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
response: Final = litellm.decisions(
response: Final = litellm.systemone(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -252,7 +276,7 @@ def test_openrouter_response_keeps_provider_fields(respx_mock: respx.MockRouter)
}
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=payload)
response: Final = litellm.decisions(
response: Final = litellm.systemone(
model="openrouter/typesafe/jev-1.13",
state="review",
questions=_QUESTIONS,
@ -340,7 +364,7 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo
litellm.callbacks = [recording_logger]
try:
await litellm.adecisions(
await litellm.asystemone(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -365,7 +389,7 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo
@pytest.mark.asyncio
async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"):
await litellm.adecisions(
await litellm.asystemone(
model="unknown/jev-1.13",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -378,7 +402,7 @@ async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRo
@pytest.mark.asyncio
async def test_provider_without_decisions_support_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
with pytest.raises(litellm.BadRequestError, match=r"Unknown Decisions provider 'anthropic'\. Supported providers"):
await litellm.adecisions(
await litellm.asystemone(
model="anthropic/claude-sonnet-4-5",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -392,7 +416,7 @@ async def test_provider_without_decisions_support_is_rejected_before_http(respx_
async def test_empty_custom_provider_falls_back_to_the_model_prefix(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
response: Final = await litellm.adecisions(
response: Final = await litellm.asystemone(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -412,7 +436,7 @@ async def test_upstream_reply_without_answers_is_a_server_error(respx_mock: resp
)
with pytest.raises(litellm.InternalServerError, match="unexpected response"):
await litellm.adecisions(
await litellm.asystemone(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -440,7 +464,7 @@ async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_a
)
upstream: Final = respx_mock.post("https://egress.example/openrouter/alpha/decisions").respond(json=_RESPONSE)
await router.adecisions(
await router.asystemone(
model="jev",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -454,7 +478,7 @@ async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_a
@pytest.mark.asyncio
async def test_invalid_question_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"):
await litellm.adecisions(
await litellm.asystemone(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"sentiment": {"type": "choice"}},
@ -471,7 +495,7 @@ def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter
)
with pytest.raises(litellm.BadRequestError):
litellm.decisions(
litellm.systemone(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -485,7 +509,7 @@ def test_unreachable_upstream_maps_to_a_connection_error(respx_mock: respx.MockR
)
with pytest.raises(litellm.APIConnectionError, match="PerplexityException - Cannot connect to host"):
litellm.decisions(
litellm.systemone(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -501,7 +525,7 @@ def test_server_key_is_sent_to_an_explicit_api_base(
monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False)
route: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE)
litellm.decisions(
litellm.systemone(
model="perplexity/pplx-decider-v1-27b",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -531,7 +555,7 @@ async def test_cloudflare_clef_resolves_model_and_response_envelope(
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef"
).respond(json=response_body)
response: Final = await litellm.adecisions(
response: Final = await litellm.asystemone(
model=model,
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -561,7 +585,7 @@ async def test_cloudflare_clef_flash_uses_flash_endpoint_and_request_model(
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-flash"
).respond(json=_RESPONSE)
await litellm.adecisions(
await litellm.asystemone(
model="cloudflare/clef-flash",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -583,7 +607,7 @@ async def test_cloudflare_api_base_from_env_uses_workers_ai_run_path(
"https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef"
).respond(json=_RESPONSE)
await litellm.adecisions(
await litellm.asystemone(
model="cloudflare/clef",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -602,7 +626,7 @@ async def test_cloudflare_requires_account_id_or_api_base_before_http(
monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key")
with pytest.raises(litellm.BadRequestError, match="Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID"):
await litellm.adecisions(
await litellm.asystemone(
model="cloudflare/clef",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -623,7 +647,7 @@ async def test_cloudflare_clef_cost_uses_the_model_cost_map(
json=_RESPONSE
)
response: Final = await litellm.adecisions(
response: Final = await litellm.asystemone(
model="cloudflare/clef",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -648,7 +672,7 @@ async def test_strands_decider_requires_api_base_before_http(
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
with pytest.raises(litellm.BadRequestError, match="api_base is required"):
await litellm.adecisions(
await litellm.asystemone(
model="strands_decider/strands-decider-2B-hobson-v19",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
@ -666,7 +690,7 @@ async def test_strands_decider_without_key_preserves_response_extras(
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
response: Final = await litellm.adecisions(
response: Final = await litellm.asystemone(
model="strands_decider/strands-decider-2B-hobson-v19",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
@ -681,6 +705,33 @@ async def test_strands_decider_without_key_preserves_response_extras(
assert severity.legend == {"0": "none", "1": "low", "2": "high"}
@pytest.mark.asyncio
async def test_strands_decider_plain_api_base_answers_while_every_aws_signing_thread_is_busy(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
release: Final = threading.Event()
busy: Final = tuple(AWS_SIGNING_EXECUTOR.submit(release.wait) for _ in range(AWS_SIGNING_MAX_THREADS))
try:
response: Final = await asyncio.wait_for(
litellm.asystemone(
model="strands_decider/strands-decider-2B-hobson-v19",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
api_base="https://strands.example",
),
timeout=5,
)
finally:
release.set()
wait(busy)
assert isinstance(response.answers["severity"], ScoreAnswer)
@pytest.mark.asyncio
async def test_strands_decider_uses_key_from_matching_environment_base(
monkeypatch: pytest.MonkeyPatch,
@ -690,7 +741,7 @@ async def test_strands_decider_uses_key_from_matching_environment_base(
monkeypatch.setenv("STRANDS_DECIDER_API_KEY", "strands-key")
route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
await litellm.adecisions(
await litellm.asystemone(
model="strands_decider/strands-decider-2B-hobson-v19",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
@ -722,7 +773,7 @@ async def test_strands_decider_provider_resolution_and_router_dispatch(
)
route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE)
response: Final = await router.adecisions(
response: Final = await router.asystemone(
model="strands",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
@ -733,6 +784,463 @@ async def test_strands_decider_provider_resolution_and_router_dispatch(
assert response.model == _STRANDS_RESPONSE["model"]
_RUNTIME_ARN: Final = "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/qa_decider-AbC123xyz0"
_RUNTIME_URL: Final = (
"https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/"
"arn%3Aaws%3Abedrock-agentcore%3Aus-east-1%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0/invocations"
)
_RUNTIME_CREDENTIALS: Final = Credentials(access_key="AKIDEXAMPLE", secret_key="example-secret")
_SESSION_HEADER: Final = "x-amzn-bedrock-agentcore-runtime-session-id"
def _sigv4_verifies(request: httpx.Request) -> bool:
authorization: Final = request.headers["authorization"]
signed_names: Final = authorization.split("SignedHeaders=")[1].split(",")[0].split(";")
replayed: Final = AWSRequest(
method=request.method,
url=str(request.url),
data=request.content,
headers={name: request.headers[name] for name in signed_names},
)
replayed.context["timestamp"] = request.headers["x-amz-date"]
auth: Final = SigV4Auth(_RUNTIME_CREDENTIALS, "bedrock-agentcore", "us-east-1")
string_to_sign: Final = auth.string_to_sign(replayed, auth.canonical_request(replayed))
return authorization.endswith(f"Signature={auth.signature(string_to_sign, replayed)}")
@pytest.mark.parametrize("use_async", [True, False])
@pytest.mark.asyncio
async def test_strands_decider_runtime_arn_sends_a_sigv4_signed_invoke_agent_runtime_call(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
use_async: bool,
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
route: Final = respx_mock.post(_RUNTIME_URL).respond(json=_STRANDS_RESPONSE)
call_args: Final[Mapping[str, object]] = {
"model": "strands_decider/strands-decider-2B-hobson-v19",
"state": "review",
"questions": {"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
"api_base": _RUNTIME_ARN,
"aws_access_key_id": _RUNTIME_CREDENTIALS.access_key,
"aws_secret_access_key": _RUNTIME_CREDENTIALS.secret_key,
}
response: Final = await litellm.asystemone(**call_args) if use_async else litellm.systemone(**call_args)
assert route.call_count == 1
sent: Final = route.calls[0].request
assert _sigv4_verifies(sent)
assert json.loads(sent.content)["model"] == "strands-decider-2B-hobson-v19"
assert sent.headers[_SESSION_HEADER].startswith("litellm-decider-")
assert response.model == _STRANDS_RESPONSE["model"]
@pytest.mark.asyncio
async def test_strands_decider_runtime_arn_router_deployment_answers_openai_format_calls(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
router: Final = litellm.Router(
model_list=[
{
"model_name": "strands",
"litellm_params": {
"model": "strands_decider/strands-decider-2B-hobson-v19",
"api_base": _RUNTIME_ARN,
"aws_access_key_id": _RUNTIME_CREDENTIALS.access_key,
"aws_secret_access_key": _RUNTIME_CREDENTIALS.secret_key,
"aws_region_name": "eu-west-1",
"agentcore_runtime_session_id": "pooled-session-0123456789abcdef0123456",
},
}
]
)
route: Final = respx_mock.post(_RUNTIME_URL).respond(json=_STRANDS_RESPONSE)
response: Final = await router.adecisions(
model="strands",
input="review",
extra_headers={_SESSION_HEADER: "caller-session-0123456789abcdef01234567"},
questions=[
{
"type": "score",
"name": "severity",
"instructions": "How bad?",
"levels": [{"label": "none"}, {"label": "low"}, {"label": "high"}],
}
],
)
sent: Final = route.calls[0].request
assert _sigv4_verifies(sent)
assert sent.headers[_SESSION_HEADER] == "pooled-session-0123456789abcdef0123456"
assert isinstance(response, OpenAIDecisionResponse)
@pytest.mark.asyncio
async def test_strands_decider_runtime_arn_with_api_key_sends_the_bearer_token_unsigned(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
route: Final = respx_mock.post(_RUNTIME_URL).respond(json=_STRANDS_RESPONSE)
await litellm.asystemone(
model="strands_decider/strands-decider-2B-hobson-v19",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
api_base=_RUNTIME_ARN,
api_key="runtime-jwt",
)
sent: Final = route.calls[0].request
assert sent.headers["authorization"] == "Bearer runtime-jwt"
assert "x-amz-date" not in sent.headers
assert _SESSION_HEADER in sent.headers
@pytest.mark.parametrize(
("code", "error_class"),
[
("bad_request", litellm.BadRequestError),
("loading", litellm.ServiceUnavailableError),
("inference_error", litellm.InternalServerError),
],
)
@pytest.mark.asyncio
async def test_strands_decider_runtime_error_envelope_raises_instead_of_answering(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
code: str,
error_class: type[Exception],
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
respx_mock.post(_RUNTIME_URL).respond(json={"error": {"code": code, "message": "at most 16 questions"}})
with pytest.raises(error_class, match="at most 16 questions"):
await litellm.asystemone(
model="strands_decider/strands-decider-2B-hobson-v19",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
api_base=_RUNTIME_ARN,
aws_access_key_id=_RUNTIME_CREDENTIALS.access_key,
aws_secret_access_key=_RUNTIME_CREDENTIALS.secret_key,
)
@pytest.mark.asyncio
async def test_strands_decider_plain_api_base_error_envelope_stays_an_unexpected_response(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False)
respx_mock.post("https://strands.example/v1/systemone").respond(
json={"error": {"code": "bad_request", "message": "at most 16 questions"}}
)
with pytest.raises(litellm.InternalServerError, match="returned an unexpected response"):
await litellm.asystemone(
model="strands_decider/strands-decider-2B-hobson-v19",
state="review",
questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}},
api_base="https://strands.example",
)
_VLLM_RESPONSE: Final[Mapping[str, object]] = {
"id": "systemone-4af3d2c1",
"object": "structured_decision",
"created": 1760012345,
"model": "Qwen/Qwen3-0.6B",
"answers": {
"is_defect": {
"type": "choice",
"choice": "yes",
"confidence": 0.91,
"probabilities": {"yes": 0.91, "no": 0.09},
}
},
"usage": {"input_tokens": 154, "output_tokens": 2},
"diagnostics": {"engine": "vllm", "logprobs_mode": "raw_logprobs"},
}
@pytest.mark.asyncio
async def test_hosted_vllm_requires_api_base_before_http(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
with pytest.raises(litellm.BadRequestError, match="api_base is required"):
await litellm.asystemone(
model="hosted_vllm/Qwen/Qwen3-0.6B",
state="review",
questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}},
)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
async def test_hosted_vllm_without_key_sends_no_authorization_and_preserves_response_extras(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE)
response: Final = await litellm.asystemone(
model="hosted_vllm/Qwen/Qwen3-0.6B",
state="The package arrived broken.",
questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}},
api_base="http://vllm.local:8000",
)
assert route.called
assert "authorization" not in respx_mock.calls[0].request.headers
assert response.model_extra["id"] == _VLLM_RESPONSE["id"]
assert response.model_extra["object"] == _VLLM_RESPONSE["object"]
assert response.model_extra["diagnostics"] == _VLLM_RESPONSE["diagnostics"]
@pytest.mark.asyncio
async def test_hosted_vllm_uses_key_and_base_from_environment(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://vllm.local:8000/v1")
monkeypatch.setenv("HOSTED_VLLM_API_KEY", "vllm-key")
route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE)
await litellm.asystemone(
model="hosted_vllm/Qwen/Qwen3-0.6B",
state="The package arrived broken.",
questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}},
)
assert route.called
assert respx_mock.calls[0].request.headers["authorization"] == "Bearer vllm-key"
@pytest.mark.asyncio
async def test_hosted_vllm_api_base_with_v1_suffix_posts_to_v1_systemone(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE)
await litellm.asystemone(
model="hosted_vllm/Qwen/Qwen3-0.6B",
state="The package arrived broken.",
questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}},
api_base="http://vllm.local:8000/v1",
)
assert route.called
@pytest.mark.asyncio
async def test_hosted_vllm_choice_question_goes_out_as_a_jev_choice_with_the_served_model_name(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE)
await litellm.asystemone(
model="hosted_vllm/Qwen/Qwen3-0.6B",
state="The package arrived broken.",
questions={
"is_defect": {
"type": "choice",
"instructions": "Is this a defect?",
"criteria": {"yes": "defective", "no": "intact"},
}
},
api_base="http://vllm.local:8000",
)
assert route.called
assert json.loads(respx_mock.calls[0].request.content) == {
"model": "Qwen/Qwen3-0.6B",
"state": "The package arrived broken.",
"questions": {
"is_defect": {
"type": "choice",
"instructions": "Is this a defect?",
"criteria": {"yes": "defective", "no": "intact"},
}
},
}
@pytest.mark.asyncio
async def test_databricks_requires_api_base_before_http(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("DATABRICKS_API_BASE", raising=False)
monkeypatch.setenv("DATABRICKS_API_KEY", "dapi-key")
with pytest.raises(litellm.BadRequestError, match="DATABRICKS_API_BASE"):
await litellm.asystemone(
model=f"databricks/{_DATABRICKS_ENDPOINT}",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
@pytest.mark.parametrize(
"endpoint_name", ["serving-endpoints/openjev", "openjev?x=1", "openjev#frag", "..", "open jev"]
)
async def test_databricks_rejects_a_serving_endpoint_name_that_would_rewrite_the_url(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
endpoint_name: str,
) -> None:
monkeypatch.setenv("DATABRICKS_API_BASE", "https://workspace.example/serving-endpoints")
monkeypatch.setenv("DATABRICKS_API_KEY", "dapi-key")
with pytest.raises(litellm.BadRequestError, match="bare endpoint name"):
await litellm.asystemone(
model=f"databricks/{endpoint_name}",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
async def test_databricks_requires_a_key_before_http(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("DATABRICKS_API_BASE", "https://workspace.example/serving-endpoints")
monkeypatch.delenv("DATABRICKS_API_KEY", raising=False)
monkeypatch.delenv("DATABRICKS_TOKEN", raising=False)
with pytest.raises(litellm.AuthenticationError, match="Missing API key for Decisions provider 'databricks'"):
await litellm.asystemone(
model=f"databricks/{_DATABRICKS_ENDPOINT}",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
@pytest.mark.parametrize(
"api_base", ["https://workspace.example/serving-endpoints", "https://workspace.example/serving-endpoints/"]
)
async def test_databricks_posts_the_jev_body_to_the_serving_endpoint_invocations_route(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
api_base: str,
) -> None:
monkeypatch.setenv("DATABRICKS_API_BASE", api_base)
monkeypatch.delenv("DATABRICKS_API_KEY", raising=False)
monkeypatch.setenv("DATABRICKS_TOKEN", "dapi-token")
route: Final = respx_mock.post(
f"https://workspace.example/serving-endpoints/{_DATABRICKS_ENDPOINT}/invocations"
).respond(json=_DATABRICKS_RESPONSE)
questions: Final = {
"is_defect": {"type": "noul", "instructions": "Is this a defect?"},
"sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}},
}
response: Final = await litellm.asystemone(
model=f"databricks/{_DATABRICKS_ENDPOINT}",
state={"ticket": "export hangs"},
questions=questions,
)
assert route.called
sent: Final = respx_mock.calls[0].request
assert sent.headers["authorization"] == "Bearer dapi-token"
assert json.loads(sent.content) == {
"model": _DATABRICKS_ENDPOINT,
"state": {"ticket": "export hangs"},
"questions": questions,
}
assert response.model == "/mosaicml/local_model"
assert isinstance(response.answers["is_defect"], NoulAnswer)
assert isinstance(response.answers["sentiment"], ChoiceAnswer)
assert response.hidden_params["custom_llm_provider"] == "databricks"
assert response.hidden_params["model"] == f"databricks/{_DATABRICKS_ENDPOINT}"
@pytest.mark.asyncio
async def test_databricks_router_deployment_sends_its_own_key_to_its_own_base(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("DATABRICKS_API_BASE", "https://other-workspace.example/serving-endpoints")
monkeypatch.setenv("DATABRICKS_API_KEY", "env-key-stays-home")
router: Final = litellm.Router(
model_list=[
{
"model_name": "decider",
"litellm_params": {
"model": f"databricks/{_DATABRICKS_ENDPOINT}",
"api_base": "https://workspace.example/serving-endpoints",
"api_key": "deployment-key",
},
}
]
)
route: Final = respx_mock.post(
f"https://workspace.example/serving-endpoints/{_DATABRICKS_ENDPOINT}/invocations"
).respond(json=_DATABRICKS_RESPONSE)
response: Final = await router.asystemone(
model="decider",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
)
assert route.called
assert respx_mock.calls[0].request.headers["authorization"] == "Bearer deployment-key"
assert response.answers["is_defect"] == NoulAnswer(type="noul", noul=0.9)
_PROVIDERS_WHOSE_CHAT_MODELS_ALSO_DECIDE: Final = frozenset(
{LlmProviders.OPENAI.value, LlmProviders.OPENROUTER.value, LlmProviders.HOSTED_VLLM.value}
)
_DEDICATED_DECIDER_PROVIDERS: Final[tuple[str, ...]] = tuple(
sorted(
provider.value
for provider in LlmProviders
if provider.value not in _PROVIDERS_WHOSE_CHAT_MODELS_ALSO_DECIDE
and ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None
)
)
@pytest.mark.parametrize("provider", _DEDICATED_DECIDER_PROVIDERS)
def test_every_dedicated_decider_provider_ships_an_evaluation_mode_cost_map_entry(provider: str) -> None:
evaluation_entries: Final = tuple(
name
for name, info in litellm.model_cost.items()
if isinstance(info, Mapping) and info.get("litellm_provider") == provider and info.get("mode") == "evaluation"
)
assert evaluation_entries, f"a {provider} decider would be health-checked as chat without a mode: evaluation entry"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("api_base", "url"),
@ -752,7 +1260,7 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE)
response: Final = await litellm.adecisions(
response: Final = await litellm.asystemone(
model="openai/gpt-6-luna",
state="The package arrived broken.",
questions={
@ -837,7 +1345,7 @@ def test_openai_decisions_use_the_same_settings_as_other_openai_calls(
monkeypatch.setenv(name, value)
route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE)
litellm.decisions(
litellm.systemone(
model="openai/gpt-6-luna",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -900,17 +1408,8 @@ async def test_openai_format_calls_to_a_systemone_provider_get_openai_format_ans
},
"cannot serve this request",
),
(
"openai/gpt-6-luna",
{
"state": "review",
"input": "review",
"questions": [{"type": "predicate", "instructions": "Is this a defect?"}],
},
"not both",
),
),
ids=("image_to_systemone_provider", "state_and_input"),
ids=("image_to_systemone_provider",),
)
async def test_requests_a_provider_cannot_serve_are_rejected_before_http(
respx_mock: respx.MockRouter,
@ -924,6 +1423,175 @@ async def test_requests_a_provider_cannot_serve_are_rejected_before_http(
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
async def test_openrouter_decisions_uses_provider_reported_cost_without_cost_map(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delitem(litellm.model_cost, "openrouter/typesafe/jev-1.13", raising=False)
cost: Final = 1.1802e-5
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(
json={
"model": "typesafe/jev-1.13",
"answers": _RESPONSE["answers"],
"usage": {
"input_tokens": _INPUT_TOKENS,
"output_tokens": _OUTPUT_TOKENS,
"cost": cost,
},
}
)
response: Final = await litellm.asystemone(
model="openrouter/typesafe/jev-1.13",
state={"source": "unit-test"},
questions=_QUESTIONS,
api_key="caller-key",
)
assert route.called
assert get_response_cost_from_hidden_params(response.hidden_params) == cost
_SAFETY_IDENTIFIER: Final = "end-user-7"
_PREDICATE_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = (
{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"},
)
_PREDICATE_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = (
("decisions", MappingProxyType({"input": "review", "questions": _PREDICATE_QUESTIONS})),
(
"systemone",
MappingProxyType(
{"state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}}
),
),
)
_PREDICATE_REQUEST_IDS: Final = ("decisions", "systemone")
@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS)
def test_safety_identifier_is_refused_before_http_when_the_provider_cannot_take_it(
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match=r"safety_identifier.*drop_params") as caught:
getattr(litellm, api)(
model="perplexity/pplx-decider-v1-27b",
safety_identifier=_SAFETY_IDENTIFIER,
api_key="caller-key",
**request_kwargs,
)
assert caught.value.status_code == 400
assert not route.called
@pytest.mark.asyncio
@pytest.mark.parametrize("scope", ("call", "global"))
async def test_safety_identifier_is_dropped_from_the_wire_under_drop_params(
scope: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", scope == "global")
route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
response: Final = await litellm.adecisions(
model="perplexity/pplx-decider-v1-27b",
input="review",
questions=_PREDICATE_QUESTIONS,
safety_identifier=_SAFETY_IDENTIFIER,
api_key="caller-key",
**({"drop_params": True} if scope == "call" else {}),
)
assert route.called
assert json.loads(respx_mock.calls[0].request.content) == {
"model": "pplx-decider-v1-27b",
"state": "review",
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
}
assert isinstance(response, OpenAIDecisionResponse)
assert [answer.model_dump(mode="json") for answer in response.answers] == [
{"type": "predicate", "name": "is_defect", "probability": 0.9}
]
@pytest.mark.asyncio
@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS)
@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params"))
async def test_safety_identifier_reaches_openai_whether_or_not_params_are_dropped(
drop_params: bool,
api: str,
request_kwargs: Mapping[str, object],
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", drop_params)
monkeypatch.setattr(litellm, "api_base", None)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
await getattr(litellm, f"a{api}")(
model="openai/gpt-6-luna",
safety_identifier=_SAFETY_IDENTIFIER,
api_key="caller-key",
**request_kwargs,
)
assert route.called
assert json.loads(respx_mock.calls[0].request.content) == {
"model": "gpt-6-luna",
"input": "review",
"questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
"safety_identifier": _SAFETY_IDENTIFIER,
}
@pytest.mark.asyncio
@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS)
async def test_non_string_safety_identifier_is_rejected_before_http(
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
invalid_safety_identifier: Final[Mapping[str, object]] = MappingProxyType({"safety_identifier": 7})
with pytest.raises(litellm.BadRequestError, match=r"(?s)Invalid Decisions request.*safety_identifier"):
await getattr(litellm, f"a{api}")(
model="openai/gpt-6-luna", api_key="caller-key", **request_kwargs, **invalid_safety_identifier
)
assert not route.called
@pytest.mark.asyncio
@pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS)
async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_params(
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
monkeypatch.setattr(litellm, "api_base", None)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
dropped_safety_identifier: Final[Mapping[str, object]] = MappingProxyType(
{"safety_identifier": 7, "drop_params": True}
)
await getattr(litellm, f"a{api}")(
model="openai/gpt-6-luna", api_key="caller-key", **request_kwargs, **dropped_safety_identifier
)
assert route.called
assert json.loads(respx_mock.calls[0].request.content) == {
"model": "gpt-6-luna",
"input": "review",
"questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"api_base",
@ -949,7 +1617,7 @@ async def test_azure_ai_decision_posts_deployment_to_foundry_systemone_route(
json={**_RESPONSE, "model": "microsoft-decision-1"}
)
response: Final = await litellm.adecisions(
response: Final = await litellm.asystemone(
model="azure_ai/decision-1",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -981,7 +1649,7 @@ async def test_azure_ai_decision_reads_foundry_env_and_keeps_gateway_path_prefix
json=_RESPONSE
)
await litellm.adecisions(
await litellm.asystemone(
model="azure_ai/decision-1",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
@ -1000,7 +1668,7 @@ async def test_azure_ai_decision_requires_api_base_before_http(
monkeypatch.setenv("AZURE_AI_API_KEY", "foundry-key")
with pytest.raises(litellm.BadRequestError, match="Missing AZURE_AI_API_BASE"):
await litellm.adecisions(
await litellm.asystemone(
model="azure_ai/decision-1",
state="review",
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},

75
tests/unit/litellm_core_utils/test_health_check_helpers.py
View file

@ -679,6 +679,81 @@ async def test_ahealth_check_probes_strands_through_decisions_without_mode(
assert "authorization" not in upstream.calls[0].request.headers
@pytest.mark.asyncio
async def test_ahealth_check_probes_hosted_vllm_with_a_choice_question(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(
json={
"model": "Qwen/Qwen3-0.6B",
"answers": {
"reachable": {
"type": "choice",
"choice": "yes",
"confidence": 1.0,
"probabilities": {"yes": 1.0, "no": 0.0},
}
},
"usage": {"input_tokens": 12, "output_tokens": 1},
}
)
result: Final = await ahealth_check(
{
"model": "hosted_vllm/Qwen/Qwen3-0.6B",
"api_base": "http://vllm.local:8000",
},
mode="evaluation",
)
assert "error" not in result, result
assert upstream.called
sent: Final = json.loads(upstream.calls[0].request.content)
assert sent["questions"] == {
"reachable": {
"type": "choice",
"instructions": "Is the service reachable?",
"criteria": {"yes": None, "no": None},
}
}
@pytest.mark.asyncio
async def test_ahealth_check_keeps_the_noul_probe_for_other_decisions_providers(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(
json={
"model": "jev-1.13",
"answers": {"reachable": {"type": "noul", "noul": 1.0}},
"usage": {"input_tokens": 12, "output_tokens": 1},
}
)
result: Final = await ahealth_check(
{
"model": "typesafe/jev-1.13",
"api_key": "sk-test",
},
mode="evaluation",
)
assert "error" not in result, result
assert upstream.called
sent: Final = json.loads(upstream.calls[0].request.content)
assert sent["questions"] == {
"reachable": {"type": "noul", "instructions": "Is the service reachable?"}
}
@pytest.mark.parametrize(
("model", "custom_llm_provider", "expected"),
(

1
tests/unit/llms/azure/test_azure_common_utils.py
View file

@ -471,6 +471,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client():
"llm_passthrough_route",
"asearch",
"adecisions",
"asystemone",
"avector_store_create",
"avector_store_search",
"acreate_skill",

0
tests/unit/llms/databricks/decisions/__init__.py Normal file
View file

56
tests/unit/llms/databricks/decisions/test_transformation.py Normal file
View file

@ -0,0 +1,56 @@
from typing import Final
import pytest
from litellm.llms.databricks.decisions.transformation import DatabricksDecisionsConfig
_BASE: Final = "https://workspace.example/serving-endpoints"
_CONFIG: Final = DatabricksDecisionsConfig()
@pytest.mark.parametrize("api_base", [_BASE, f"{_BASE}/"])
def test_the_complete_url_is_the_serving_endpoint_invocations_route(api_base: str) -> None:
assert _CONFIG.get_complete_url(api_base, "openjev") == f"{_BASE}/openjev/invocations"
@pytest.mark.parametrize(
"name",
["", "serving-endpoints/openjev", "openjev?x=1", "openjev#frag", ".", "..", ".openjev", "open jev", "openjev%2Fx"],
)
def test_a_name_that_would_rewrite_the_url_is_rejected_before_any_request(name: str) -> None:
with pytest.raises(ValueError, match="bare endpoint name"):
_ = _CONFIG.get_complete_url(_BASE, name)
with pytest.raises(ValueError, match="bare endpoint name"):
_ = _CONFIG.canonical_model(name)
def test_connection_prefers_the_configured_values_over_the_environment(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("DATABRICKS_API_BASE", "https://env.example/serving-endpoints")
monkeypatch.setenv("DATABRICKS_API_KEY", "dapi-env")
connection: Final = _CONFIG.connection(f"{_BASE}/", "dapi-configured")
assert (connection.api_base, connection.api_key) == (_BASE, "dapi-configured")
def test_connection_falls_back_to_the_workspace_token_when_the_api_key_variable_is_unset(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("DATABRICKS_API_BASE", _BASE)
monkeypatch.delenv("DATABRICKS_API_KEY", raising=False)
monkeypatch.setenv("DATABRICKS_TOKEN", "dapi-token")
connection: Final = _CONFIG.connection(None, None)
assert (connection.api_base, connection.api_key) == (_BASE, "dapi-token")
def test_connection_never_sends_the_environment_key_to_a_configured_base(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("DATABRICKS_API_BASE", _BASE)
monkeypatch.setenv("DATABRICKS_API_KEY", "dapi-env")
with pytest.raises(ValueError, match="DATABRICKS_API_BASE"):
_ = _CONFIG.connection("https://collector.example/serving-endpoints", None)
def test_classifier_response_accounts_the_serving_endpoint_instead_of_the_container_model() -> None:
body: Final = {"model": "/mosaicml/local_model", "answers": {}, "usage": {"input_tokens": 3, "output_tokens": 0}}
normalized: Final = _CONFIG.classifier_response(body, "openjev")
assert normalized["model"] == "openjev"
assert normalized["usage"] == body["usage"]
assert body["model"] == "/mosaicml/local_model"

17
tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py
View file

@ -96,6 +96,15 @@ def test_openrouter_extra_body_transformation():
]
def test_openrouter_supported_params_include_reasoning_for_unmapped_model():
supported_params = OpenrouterConfig().get_supported_openai_params(
"openrouter/unmapped-model"
)
assert "reasoning_effort" in supported_params
assert "thinking" in supported_params
def test_openrouter_cache_control_flag_removal():
transformed_request = OpenrouterConfig().transform_request(
model="openrouter/deepseek/deepseek-chat",
@ -537,17 +546,15 @@ def test_openrouter_reasoning_models_allow_reasoning_effort_param():
assert supported_params.count("reasoning_effort") == 1
def test_openrouter_non_reasoning_models_do_not_add_reasoning_effort():
"""
Models without reasoning support should not gain reasoning-specific params.
"""
def test_openrouter_non_reasoning_models_support_reasoning_params():
config = OpenrouterConfig()
supported_params = config.get_supported_openai_params(
model="openrouter/anthropic/claude-3-5-haiku"
)
assert "reasoning_effort" not in supported_params
assert "reasoning_effort" in supported_params
assert "thinking" in supported_params
def test_openrouter_reasoning_effort_max_maps_to_xhigh():

73
tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py
View file

@ -9,12 +9,22 @@ reasoning.encrypted_content for multi-turn stateless workflows.
Related issue: https://github.com/BerriAI/litellm/issues/22189
"""
from unittest.mock import MagicMock
import httpx
import pytest
import litellm
from litellm.cost_calculator import get_response_cost_from_hidden_params
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER
from litellm.llms.openrouter.responses.transformation import (
OpenRouterResponsesAPIConfig,
)
from litellm.types.llms.openai import (
ResponseCompletedEvent,
ResponseFailedEvent,
ResponseIncompleteEvent,
)
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
@ -116,3 +126,66 @@ class TestOpenRouterResponsesAPIRegistration:
# The URL should point to OpenRouter's responses endpoint
url = config.get_complete_url(api_base=None, litellm_params={})
assert "/responses" in url
@pytest.mark.parametrize("cost", [1.23e-5, None])
def test_openrouter_responses_returns_provider_reported_cost(cost):
usage: dict[str, object] = {
"input_tokens": 2,
"output_tokens": 3,
"total_tokens": 5,
}
if cost is not None:
usage["cost"] = cost
response = OpenRouterResponsesAPIConfig().transform_response_api_response(
model="openrouter/unmapped-model",
raw_response=httpx.Response(
200,
json={
"id": "resp_test",
"created_at": 1,
"model": "unmapped-model",
"object": "response",
"output": [],
"status": "completed",
"usage": usage,
},
),
logging_obj=MagicMock(),
)
assert get_response_cost_from_hidden_params(response.hidden_params) == cost
@pytest.mark.parametrize(
"event_type",
["response.completed", "response.incomplete", "response.failed"],
)
def test_openrouter_terminal_stream_events_return_provider_reported_cost(event_type):
event = OpenRouterResponsesAPIConfig().transform_streaming_response(
model="openrouter/unmapped-model",
parsed_chunk={
"type": event_type,
"response": {
"id": "resp_test",
"created_at": 1,
"model": "unmapped-model",
"object": "response",
"output": [],
"status": "completed",
"usage": {
"input_tokens": 2,
"output_tokens": 3,
"total_tokens": 5,
"cost": 1.23e-5,
},
},
},
logging_obj=MagicMock(),
)
assert isinstance(
event,
(ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent),
)
assert event.response.hidden_params["additional_headers"][RESPONSE_COST_HEADER] == 1.23e-5

0
tests/unit/llms/strands_decider/__init__.py Normal file
View file

0
tests/unit/llms/strands_decider/decisions/__init__.py Normal file
View file

256
tests/unit/llms/strands_decider/decisions/test_strands_decider_decisions_transformation.py Normal file
View file

@ -0,0 +1,256 @@
import json
from collections.abc import Mapping
from typing import Final
import httpx
import pytest
from botocore.credentials import Credentials
from pydantic import TypeAdapter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import systemone_request_to_ir
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.strands_decider.decisions.transformation import (
AGENTCORE_SESSION_HEADER,
StrandsDeciderDecisionsConfig,
)
from litellm.types.decisions import DecisionsIRResponse, DecisionsRequestBody
from litellm.types.llms.bedrock import AwsAuthParams
_ARN: Final = "arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/qa_decider-AbC123xyz0"
_BODY: Final[Mapping[str, object]] = {
"model": "strands-decider-2B-hobson-v21",
"state": "My order arrived broken.",
"questions": {"is_complaint": {"type": "noul", "instructions": "Is this a complaint?"}},
}
_IR_REQUEST: Final = systemone_request_to_ir(
TypeAdapter(DecisionsRequestBody).validate_python({"state": _BODY["state"], "questions": _BODY["questions"]})
)
class _RecordingAWS(BaseAWSLLM):
def __init__(self) -> None:
super().__init__()
self.resolved: tuple[tuple[AwsAuthParams, str | None], ...] = ()
def resolve_credentials(self, auth_params: AwsAuthParams, aws_region_name: str | None) -> Credentials:
self.resolved = (*self.resolved, (auth_params, aws_region_name))
return Credentials(access_key="AKIDEXAMPLE", secret_key="example-secret")
def _sign(
config: StrandsDeciderDecisionsConfig,
api_base: str,
headers: Mapping[str, str],
api_key: str | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> tuple[Mapping[str, str], bytes | None]:
return config.sign_request(
headers=headers,
url=config.get_complete_url(api_base=api_base, model="strands-decider-2B-hobson-v21"),
api_base=api_base,
body=_BODY,
api_key=api_key,
litellm_params=litellm_params or {},
)
@pytest.mark.parametrize(
("arn", "expected_url"),
[
(
_ARN,
"https://bedrock-agentcore.us-west-2.amazonaws.com/runtimes/"
"arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0/invocations",
),
(
"arn:aws-cn:bedrock-agentcore:cn-north-1:123456789012:runtime/qa_decider-AbC123xyz0",
"https://bedrock-agentcore.cn-north-1.amazonaws.com.cn/runtimes/"
"arn%3Aaws-cn%3Abedrock-agentcore%3Acn-north-1%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0"
"/invocations",
),
(
"arn:aws-us-gov:bedrock-agentcore:us-gov-west-1:123456789012:runtime/qa_decider-AbC123xyz0",
"https://bedrock-agentcore.us-gov-west-1.amazonaws.com/runtimes/"
"arn%3Aaws-us-gov%3Abedrock-agentcore%3Aus-gov-west-1%3A123456789012%3Aruntime%2Fqa_decider-AbC123xyz0"
"/invocations",
),
],
)
def test_runtime_arn_targets_the_regional_invoke_agent_runtime_url(arn: str, expected_url: str) -> None:
assert StrandsDeciderDecisionsConfig().get_complete_url(api_base=arn, model="any") == expected_url
@pytest.mark.parametrize(
"api_base",
[
"https://strands.example",
"arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/qa_decider-AbC123xyz0/runtime-endpoint/DEFAULT",
"arn:aws:bedrock-agentcore:evil.example/x:123456789012:runtime/qa_decider-AbC123xyz0",
"arn:aws:bedrock:us-west-2:123456789012:runtime/qa_decider-AbC123xyz0",
],
)
def test_other_api_bases_keep_the_plain_systemone_request(api_base: str) -> None:
aws: Final = _RecordingAWS()
config: Final = StrandsDeciderDecisionsConfig(aws=aws)
headers: Final = {"Content-Type": "application/json"}
assert config.get_complete_url(api_base=api_base, model="any") == f"{api_base}/v1/systemone"
assert _sign(config, api_base, headers) == (headers, None)
assert aws.resolved == ()
def test_sign_request_signs_the_body_for_the_arn_region_with_deployment_credentials() -> None:
aws: Final = _RecordingAWS()
config: Final = StrandsDeciderDecisionsConfig(aws=aws)
signed_headers, signed_body = _sign(
config,
_ARN,
{"Content-Type": "application/json"},
litellm_params={"aws_profile_name": "qa", "aws_role_name": "decider-role", "aws_region_name": "us-east-1"},
)
assert signed_body == json.dumps(_BODY).encode()
assert len(aws.resolved) == 1
auth_params, region = aws.resolved[0]
assert (auth_params.aws_profile_name, auth_params.aws_role_name, region) == ("qa", "decider-role", "us-west-2")
authorization: Final = signed_headers["Authorization"]
assert authorization.startswith("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/")
assert "/us-west-2/bedrock-agentcore/aws4_request" in authorization
assert AGENTCORE_SESSION_HEADER.lower() in authorization
def test_default_session_id_is_stable_per_runtime_and_within_the_agentcore_length_bounds() -> None:
other_arn: Final = _ARN.replace("AbC123xyz0", "ZyX987cba1")
first, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), _ARN, {})
again, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), _ARN, {})
other, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), other_arn, {})
session_id: Final = first[AGENTCORE_SESSION_HEADER]
assert session_id == again[AGENTCORE_SESSION_HEADER]
assert session_id != other[AGENTCORE_SESSION_HEADER]
assert 33 <= len(session_id) <= 256
_CALLER_RESERVED_HEADERS: Final = {
AGENTCORE_SESSION_HEADER.lower(): "caller-session-0123456789abcdef0123456789",
"X-Amzn-Bedrock-AgentCore-Runtime-User-Id": "someone-else",
"X-Amz-Security-Token": "caller-token",
"Host": "evil.example",
}
@pytest.mark.parametrize("api_key", [None, "runtime-jwt"])
def test_caller_reserved_agentcore_headers_never_reach_the_runtime(api_key: str | None) -> None:
signed_headers, _ = _sign(
StrandsDeciderDecisionsConfig(aws=_RecordingAWS()),
_ARN,
{**_CALLER_RESERVED_HEADERS, "X-Request-Source": "batch"},
api_key=api_key,
)
default_headers, _ = _sign(StrandsDeciderDecisionsConfig(aws=_RecordingAWS()), _ARN, {}, api_key=api_key)
forwarded: Final = {name.lower(): value for name, value in signed_headers.items()}
assert forwarded[AGENTCORE_SESSION_HEADER.lower()] == default_headers[AGENTCORE_SESSION_HEADER]
assert not any(value in forwarded.values() for value in _CALLER_RESERVED_HEADERS.values())
assert "x-amzn-bedrock-agentcore-runtime-user-id" not in forwarded
assert "host" not in forwarded
assert forwarded["x-request-source"] == "batch"
def test_deployment_session_param_picks_the_session_over_the_default_and_any_caller_header() -> None:
replica_session: Final = "strands-decider-replica-a-0000000000"
signed_headers, _ = _sign(
StrandsDeciderDecisionsConfig(aws=_RecordingAWS()),
_ARN,
_CALLER_RESERVED_HEADERS,
litellm_params={"agentcore_runtime_session_id": replica_session},
)
session_headers: Final = [
value for name, value in signed_headers.items() if name.lower() == "x-amzn-bedrock-agentcore-runtime-session-id"
]
assert session_headers == [replica_session]
assert AGENTCORE_SESSION_HEADER.lower() in signed_headers["Authorization"]
def test_api_key_on_a_runtime_arn_sends_the_bearer_token_without_sigv4() -> None:
aws: Final = _RecordingAWS()
signed_headers, signed_body = _sign(
StrandsDeciderDecisionsConfig(aws=aws), _ARN, {"Authorization": "Bearer runtime-jwt"}, api_key="runtime-jwt"
)
assert signed_body is None
assert signed_headers["Authorization"] == "Bearer runtime-jwt"
assert AGENTCORE_SESSION_HEADER in signed_headers
assert aws.resolved == ()
def _response(url: str, payload: object) -> httpx.Response:
return httpx.Response(200, json=payload, request=httpx.Request("POST", url))
def _transform(api_base: str, payload: object) -> DecisionsIRResponse:
config: Final = StrandsDeciderDecisionsConfig()
url: Final = config.get_complete_url(api_base=api_base, model="strands-decider-2B-hobson-v21")
return config.transform_decisions_response(
model="strands-decider-2B-hobson-v21",
custom_llm_provider="strands_decider",
raw_response=_response(url, payload),
request=_IR_REQUEST,
)
@pytest.mark.parametrize(
("code", "status_code"),
[("bad_request", 400), ("invalid_request", 400), ("loading", 503), ("inference_error", 500), ("new_code", 500)],
)
def test_runtime_error_envelope_raises_with_the_mapped_status(code: str, status_code: int) -> None:
with pytest.raises(BaseLLMException) as raised:
_transform(_ARN, {"error": {"code": code, "message": "at most 16 questions"}})
assert raised.value.status_code == status_code
assert code in raised.value.message
assert "at most 16 questions" in raised.value.message
@pytest.mark.parametrize("code", ["bad_request", "loading", "inference_error"])
def test_plain_api_base_error_envelope_keeps_the_unexpected_response_error(code: str) -> None:
with pytest.raises(BaseLLMException) as raised:
_transform("https://strands.example", {"error": {"code": code, "message": "at most 16 questions"}})
assert raised.value.status_code == 500
assert raised.value.message.startswith("Decisions provider 'strands_decider' returned an unexpected response")
@pytest.mark.parametrize(
"url",
[
"https://strands.example/runtimes/arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A123456789012%3Aruntime"
"%2Fqa_decider-AbC123xyz0/invocations",
"https://bedrock-agentcore.us-west-2.amazonaws.com/runtimes/not-an-arn/invocations",
],
)
def test_error_envelope_from_a_url_that_is_not_the_runtime_keeps_the_unexpected_response_error(url: str) -> None:
with pytest.raises(BaseLLMException) as raised:
StrandsDeciderDecisionsConfig().transform_decisions_response(
model="strands-decider-2B-hobson-v21",
custom_llm_provider="strands_decider",
raw_response=_response(url, {"error": {"code": "bad_request", "message": "at most 16 questions"}}),
request=_IR_REQUEST,
)
assert raised.value.status_code == 500
@pytest.mark.parametrize("payload", [["not", "an", "object"], {"error": "unstructured"}])
def test_runtime_reply_that_is_not_an_error_envelope_keeps_the_unexpected_response_error(payload: object) -> None:
with pytest.raises(BaseLLMException) as raised:
_transform(_ARN, payload)
assert raised.value.status_code == 500
assert raised.value.message.startswith("Decisions provider 'strands_decider' returned an unexpected response")

16
tests/unit/proxy/auth/test_auth_utils.py
View file

@ -2854,6 +2854,22 @@ class TestIsRequestBodySafeBlocksBedrockProjectOverride:
)
def test_agentcore_runtime_session_id_in_request_body_is_rejected_without_admin_opt_in():
body: Final = {"model": "strands-decider", "agentcore_runtime_session_id": "caller-session-0123456789abcdef0123"}
with pytest.raises(ValueError, match="agentcore_runtime_session_id"):
is_request_body_safe(request_body=body, general_settings={}, llm_router=None, model="strands-decider")
assert (
is_request_body_safe(
request_body=body,
general_settings={"allow_client_side_credentials": True},
llm_router=None,
model="strands-decider",
)
is True
)
class TestIsRequestBodySafeBlocksClaudePlatformWorkspaceOverride:
@pytest.mark.parametrize(
"alias", ["workspace_id", "aws_workspace_id", "anthropic_workspace_id", "anthropic-workspace-id"]

92
tests/unit/proxy/common_utils/test_custom_openapi_spec.py
View file

@ -4,11 +4,14 @@ Simple unit tests for CustomOpenAPISpec class.
Tests basic functionality of OpenAPI schema generation.
"""
from unittest.mock import Mock, patch
import json
from typing import Annotated, Literal
from unittest.mock import patch
import pytest
from pydantic import BaseModel, ConfigDict, Field
from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec
from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec, inline_request_body
class TestCustomOpenAPISpec:
@ -27,19 +30,13 @@ class TestCustomOpenAPISpec:
},
}
@patch(
"litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema"
)
def test_add_chat_completion_request_schema(
self, mock_add_schema, base_openapi_schema
):
@patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema")
def test_add_chat_completion_request_schema(self, mock_add_schema, base_openapi_schema):
"""Test that chat completion schema is added correctly."""
mock_add_schema.return_value = base_openapi_schema
with patch("litellm.proxy._types.ProxyChatCompletionRequest") as mock_model:
result = CustomOpenAPISpec.add_chat_completion_request_schema(
base_openapi_schema
)
result = CustomOpenAPISpec.add_chat_completion_request_schema(base_openapi_schema)
mock_add_schema.assert_called_once_with(
openapi_schema=base_openapi_schema,
@ -50,9 +47,7 @@ class TestCustomOpenAPISpec:
)
assert result == base_openapi_schema
@patch(
"litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema"
)
@patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema")
def test_add_embedding_request_schema(self, mock_add_schema, base_openapi_schema):
"""Test that embedding schema is added correctly."""
mock_add_schema.return_value = base_openapi_schema
@ -69,19 +64,13 @@ class TestCustomOpenAPISpec:
)
assert result == base_openapi_schema
@patch(
"litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema"
)
def test_add_responses_api_request_schema(
self, mock_add_schema, base_openapi_schema
):
@patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema")
def test_add_responses_api_request_schema(self, mock_add_schema, base_openapi_schema):
"""Test that responses API schema is added correctly."""
mock_add_schema.return_value = base_openapi_schema
with patch("litellm.types.llms.openai.ResponsesAPIRequestParams") as mock_model:
result = CustomOpenAPISpec.add_responses_api_request_schema(
base_openapi_schema
)
result = CustomOpenAPISpec.add_responses_api_request_schema(base_openapi_schema)
mock_add_schema.assert_called_once_with(
openapi_schema=base_openapi_schema,
@ -123,15 +112,11 @@ def test_defs_rewritten_in_add_schema_to_components():
)
assert "$defs" not in openapi_schema
assert (
openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][
"items"
]["anyOf"][0]["$ref"]
openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][0]["$ref"]
== "#/components/schemas/UserMessage"
)
assert (
openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][
"items"
]["anyOf"][1]["$ref"]
openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][1]["$ref"]
== "#/components/schemas/AssistantMessage"
)
@ -188,14 +173,8 @@ def test_rewrite_defs_refs():
rewritten = CustomOpenAPISpec._rewrite_defs_refs(schema=schema, renames={})
assert "$defs" not in rewritten
assert (
rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"]
== "#/components/schemas/UserMessage"
)
assert (
rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"]
== "#/components/schemas/AssistantMessage"
)
assert rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage"
assert rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage"
def test_get_pydantic_schema_generates_schema_for_responses_request_typed_dict():
@ -390,3 +369,42 @@ def test_add_schema_to_components_renames_def_with_different_required_set():
assert schemas["Block"]["required"] == ["type", "x", "y"]
assert schemas["Req_Block"]["required"] == ["keys", "type", "x", "y"]
assert schemas["Req"]["properties"]["b"]["$ref"] == "#/components/schemas/Req_Block"
class _Cat(BaseModel):
kind: Literal["cat"]
lives: int = 9
model_config = ConfigDict(frozen=True)
class _Dog(BaseModel):
kind: Literal["dog"]
good: bool = True
model_config = ConfigDict(frozen=True)
class _Shelter(BaseModel):
pets: dict[str, Annotated[_Cat | _Dog, Field(discriminator="kind")]]
favorite: _Cat | None = None
model_config = ConfigDict(frozen=True)
def test_inline_request_body_expands_every_ref_so_the_operation_stands_alone():
body = inline_request_body(_Shelter, {"pets": {"tom": {"kind": "cat"}}})
media = body["content"]["application/json"]
assert body["required"] is True
assert media["example"] == {"pets": {"tom": {"kind": "cat"}}}
assert "$ref" not in json.dumps(media["schema"])
assert "$defs" not in media["schema"]
pet_schema = media["schema"]["properties"]["pets"]["additionalProperties"]
variants = {variant["properties"]["kind"]["const"]: variant for variant in pet_schema["oneOf"]}
assert variants["cat"]["properties"]["lives"]["default"] == 9
assert variants["dog"]["properties"]["good"]["type"] == "boolean"
assert pet_schema["discriminator"] == {"propertyName": "kind"}
favorite = media["schema"]["properties"]["favorite"]
assert favorite["default"] is None
assert any(option.get("properties", {}).get("lives") for option in favorite["anyOf"])

66
tests/unit/proxy/common_utils/test_registry_read_through.py
View file

@ -203,9 +203,7 @@ def clean_agent_registry():
@pytest.mark.asyncio
async def test_get_agent_with_read_through_recovers_agent_created_on_sibling_replica(
clean_agent_registry, monkeypatch
):
async def test_get_agent_with_read_through_recovers_agent_created_on_sibling_replica(clean_agent_registry, monkeypatch):
from unittest.mock import AsyncMock, MagicMock
import litellm.proxy.proxy_server as proxy_server
@ -601,7 +599,9 @@ async def test_resync_guardrails_syncs_decrypted_litellm_params(monkeypatch):
synced: list[dict] = []
monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
monkeypatch.setattr(proxy_server, "store_model_in_db", True)
monkeypatch.setattr(IN_MEMORY_GUARDRAIL_HANDLER, "sync_guardrail_from_db", lambda guardrail: synced.append(guardrail))
monkeypatch.setattr(
IN_MEMORY_GUARDRAIL_HANDLER, "sync_guardrail_from_db", lambda guardrail: synced.append(guardrail)
)
monkeypatch.setattr(read_through_module, "_initialized_guardrail", lambda guardrail_name: MagicMock())
assert await _resync_guardrails("enc-guardrail") is True
@ -610,15 +610,21 @@ async def test_resync_guardrails_syncs_decrypted_litellm_params(monkeypatch):
@pytest.mark.asyncio
@pytest.mark.parametrize("lookup", ["agent-id", "Agent name"])
async def test_agent_read_through_hydrates_identity_binding(lookup, clean_agent_registry, fresh_agent_read_through, monkeypatch):
async def test_agent_read_through_hydrates_identity_binding(
lookup, clean_agent_registry, fresh_agent_read_through, monkeypatch
):
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
binding = {
"agent_id": "agent-id", "provider": "microsoft_entra", "tenant_id": "tenant", "client_id": "client",
"issuer": "https://login.microsoftonline.com/tenant/v2.0", "revision": "revision",
"agent_id": "agent-id",
"provider": "microsoft_entra",
"tenant_id": "tenant",
"client_id": "client",
"issuer": "https://login.microsoftonline.com/tenant/v2.0",
"revision": "revision",
}
async def load_row(*, where, include):
@ -733,3 +739,49 @@ async def test_agent_read_through_answers_a_loaded_agent_without_reading_the_db(
assert await agent_registry_read_through.attempt("wired-loaded-agent-id") is True
assert await agent_registry_read_through.attempt("wired-loaded-agent") is True
@pytest.mark.asyncio
@pytest.mark.parametrize("router_holds_the_name_after_reconcile", [True, False])
async def test_resync_model_deployments_reconciles_every_db_row_when_the_missed_model_is_an_auto_router(
monkeypatch: pytest.MonkeyPatch, router_holds_the_name_after_reconcile: bool
) -> None:
from unittest.mock import AsyncMock, MagicMock
from litellm.proxy import proxy_server
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-salt-auto-router-read-through")
router_name: Final = "auto-router-created-on-a-sibling-replica"
row: Final = MagicMock()
row.model_name = router_name
row.litellm_params = {
"model": encrypt_value_helper("auto_router/complexity-router"),
"auto_router_config": {"model_tiers": {"SIMPLE": "simple-tier", "COMPLEX": "complex-tier"}},
}
prisma_client: Final = MagicMock()
prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[row])
router: Final = MagicMock()
router.model_names = []
router.has_model_id.return_value = False
reconciled: Final = MagicMock()
async def reconcile_every_row(prisma_client: object, proxy_logging_obj: object) -> None:
reconciled(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
router.model_names = (
[router_name, "simple-tier", "complex-tier"] if router_holds_the_name_after_reconcile else []
)
def reject_single_row_path(db_models: object) -> None:
raise AssertionError(f"an auto router row took the single-row path: {db_models}")
monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
monkeypatch.setattr(proxy_server, "store_model_in_db", True)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", None)
monkeypatch.setattr(proxy_server.proxy_config, "add_deployment", reconcile_every_row)
monkeypatch.setattr(proxy_server.proxy_config, "_add_deployment", reject_single_row_path)
assert await _resync_model_deployments(router_name) is router_holds_the_name_after_reconcile
reconciled.assert_called_once_with(prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj)

171
tests/unit/proxy/decisions_endpoints/test_endpoints.py
View file

@ -12,17 +12,19 @@ import respx
from fastapi import FastAPI
from fastapi.routing import APIRoute
from fastapi.testclient import TestClient
from pydantic import TypeAdapter
from starlette.routing import Match
import litellm
from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features
from litellm.proxy.decisions_endpoints.endpoints import decisions, systemone
from litellm.proxy.decisions_endpoints.endpoints import decisions, router, systemone
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder
from litellm.proxy.proxy_server import (
app,
cleanup_router_config_variables,
initialize,
)
from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody
_INPUT_TOKENS: Final[int] = 367
_OUTPUT_TOKENS: Final[int] = 3
@ -257,6 +259,51 @@ def test_proxy_decisions_dispatches_strands_decider(
assert "authorization" not in upstream.calls[0].request.headers
def test_proxy_systemone_dispatches_databricks_serving_endpoint(
client: TestClient,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("DATABRICKS_API_BASE", raising=False)
monkeypatch.delenv("DATABRICKS_API_KEY", raising=False)
monkeypatch.delenv("DATABRICKS_TOKEN", raising=False)
router: Final = litellm.Router(
model_list=[
{
"model_name": "databricks-decider",
"litellm_params": {
"model": "databricks/databricks-openjev-qwen35-4b",
"api_base": "https://workspace.example/serving-endpoints",
"api_key": "dapi-deployment",
},
}
]
)
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
upstream: Final = respx_mock.post(
"https://workspace.example/serving-endpoints/databricks-openjev-qwen35-4b/invocations"
).respond(json={**_RESPONSE, "model": "/mosaicml/local_model"})
response: Final = client.post(
"/v1/systemone",
json={
"model": "databricks-decider",
"state": {"source": "proxy-test"},
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
},
)
assert response.status_code == 200, response.text
assert response.json()["answers"] == _RESPONSE["answers"]
assert upstream.called
assert json.loads(upstream.calls[0].request.content) == {
"model": "databricks-openjev-qwen35-4b",
"state": {"source": "proxy-test"},
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
}
assert upstream.calls[0].request.headers["authorization"] == "Bearer dapi-deployment"
def test_proxy_decisions_without_model_uses_the_proxy_default_model(
client: TestClient,
monkeypatch: pytest.MonkeyPatch,
@ -530,6 +577,68 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_
assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost)
@pytest.mark.parametrize("safety_identifier", (7, ["end-user-1"]), ids=("numeric", "list"))
@pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params"))
def test_a_non_string_safety_identifier_is_refused_unless_the_deployment_drops_params(
client: TestClient,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
safety_identifier: object,
deployment_drops_params: bool,
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
monkeypatch.setattr(litellm, "api_base", None)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
monkeypatch.setattr(
litellm.proxy.proxy_server,
"llm_router",
litellm.Router(
model_list=[
{
"model_name": "decider",
"litellm_params": {
"model": "openai/gpt-6-luna",
"api_key": "k",
"drop_params": deployment_drops_params,
},
}
]
),
)
upstream: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(
json={
"model": "gpt-6-luna",
"answers": _OPENAI_FORMAT_ANSWERS[:1],
"usage": {
"input_tokens": _INPUT_TOKENS,
"output_tokens": _OUTPUT_TOKENS,
"total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS,
},
}
)
request_body: Final = {
"model": "decider",
"input": "The package arrived with a broken screen.",
"questions": _OPENAI_FORMAT_REQUEST["questions"][:1],
"safety_identifier": safety_identifier,
}
response: Final = client.post("/v1/decisions", json=request_body)
if not deployment_drops_params:
assert response.status_code == 400, response.text
assert "safety_identifier" in response.json()["error"]["message"]
assert not upstream.called
return
assert response.status_code == 200, response.text
assert json.loads(upstream.calls[0].request.content) == {
"model": "gpt-6-luna",
"input": "The package arrived with a broken screen.",
"questions": _OPENAI_FORMAT_REQUEST["questions"][:1],
}
def _decisions_feature() -> LazyFeature:
return next(feature for feature in LAZY_FEATURES if feature.name == "decisions")
@ -581,3 +690,63 @@ def test_with_lazy_routes_disabled_a_config_pass_through_at_v1_decisions_still_w
assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"}
assert _serving_endpoint(bare, "/v1/decisions") is pass_through
assert _serving_endpoint(bare, "/decisions") is decisions
@pytest.fixture(scope="module")
def decisions_openapi() -> dict[str, object]:
app = FastAPI()
app.include_router(router)
return app.openapi()
@pytest.mark.parametrize(
("path", "body_adapter", "response_schema", "required"),
(
("/v1/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}),
("/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}),
(
"/v1/decisions",
TypeAdapter(OpenAIDecisionRequestBody),
"OpenAIDecisionResponse",
{"model", "input", "questions"},
),
(
"/decisions",
TypeAdapter(OpenAIDecisionRequestBody),
"OpenAIDecisionResponse",
{"model", "input", "questions"},
),
),
)
def test_swagger_documents_the_request_format_and_response_of_each_decisions_route(
decisions_openapi: dict[str, object],
path: str,
body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody],
response_schema: str,
required: set[str],
) -> None:
operation = decisions_openapi["paths"][path]["post"]
media = operation["requestBody"]["content"]["application/json"]
assert "Decisions" in operation["summary"] or "System One" in operation["summary"]
assert "docs.litellm.ai/docs/decisions" in operation["description"]
assert required <= set(media["schema"]["required"])
assert required <= set(media["schema"]["properties"])
assert "$ref" not in json.dumps(media["schema"])
assert media["schema"]["properties"]["model"]["description"]
example = {key: value for key, value in media["example"].items() if key != "model"}
assert len(body_adapter.validate_python(example).questions) == 3
response_ref = operation["responses"]["200"]["content"]["application/json"]["schema"]["$ref"]
assert response_ref == f"#/components/schemas/{response_schema}"
assert "answers" in decisions_openapi["components"]["schemas"][response_schema]["properties"]
def test_the_two_decisions_routes_document_different_request_formats(decisions_openapi: dict[str, object]) -> None:
def properties(path: str) -> set[str]:
operation = decisions_openapi["paths"][path]["post"]
return set(operation["requestBody"]["content"]["application/json"]["schema"]["properties"])
assert "state" in properties("/v1/systemone") and "state" not in properties("/v1/decisions")
assert "input" in properties("/v1/decisions") and "input" not in properties("/v1/systemone")

2
tests/unit/proxy/guardrails/guardrail_hooks/test_decision_model.py
View file

@ -414,7 +414,7 @@ def test_initialize_rejects_invalid_streaming_scan_settings(mock_mgr, overrides,
def test_add_guardrail_settings_returns_decisions_providers():
import asyncio
from litellm.decisions.main import supported_decisions_providers
from litellm.decisions.call import supported_decisions_providers
from litellm.proxy.guardrails.guardrail_endpoints import get_guardrail_ui_settings
settings: Final = asyncio.run(get_guardrail_ui_settings())

7
tests/unit/proxy/management_helpers/test_auto_router_permissions.py
View file

@ -146,6 +146,8 @@ def test_tier_config_is_normalized_and_unknown_router_extras_are_rejected() -> N
({"provider": "laya", "model": "english", "api_key": "sk-member"}, "api_key"),
({"provider": "bespoke", "model": "nimble-latest", "api_base": "https://collector.invalid"}, "api_base"),
({"provider": "bespoke", "model": "nimble-latest", "api_key": "sk-member"}, "api_key"),
({"provider": "databricks", "model": "my-openjev", "api_base": "https://collector.invalid"}, "opensource_classifier_config"),
({"provider": "databricks", "model": "my-openjev", "api_key": "dapi-member"}, "api_key"),
],
)
@pytest.mark.parametrize("legacy", [False, True])
@ -164,7 +166,10 @@ def test_members_cannot_move_the_jev_classifier_off_the_proxys_typesafe_account(
assert denied.value.detail == f"Invalid member auto-router configuration at {rejected_at}."
@pytest.mark.parametrize(("provider", "model"), [("typesafe", "jev-preview"), ("laya", "english"), ("bespoke", "nimble-latest")])
@pytest.mark.parametrize(
("provider", "model"),
[("typesafe", "jev-preview"), ("laya", "english"), ("bespoke", "nimble-latest"), ("databricks", "my-openjev")],
)
@pytest.mark.parametrize("legacy", [False, True])
def test_members_can_still_tune_the_jev_classifier(provider: str, model: str, legacy: bool) -> None:
validated: Final = validate_member_auto_router_config(

14
tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py
View file

@ -14,7 +14,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndp
@pytest.fixture(autouse=True)
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
@ -222,14 +222,17 @@ async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost(
await budget_limiter.is_key_within_model_budget(auth, f"{provider}/{requested}")
def test_openrouter_decisions_response_is_priced_from_request_model_registry_row():
def test_openrouter_decisions_response_uses_provider_cost_without_cost_map_row(
monkeypatch: pytest.MonkeyPatch,
):
logging_obj = _logging_obj()
model_cost = litellm.model_cost["openrouter/typesafe/jev-1.13"]
cost: Final = 1.5834e-5
monkeypatch.delitem(litellm.model_cost, "openrouter/typesafe/jev-1.13", raising=False)
response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
httpx_response=_response(),
response_body={
"model": "typesafe/jev-1.13-20260917",
"usage": {"input_tokens": 282, "output_tokens": 20},
"usage": {"input_tokens": 282, "output_tokens": 20, "cost": cost},
},
logging_obj=logging_obj,
url_route="https://openrouter.ai/api/alpha/decisions",
@ -241,10 +244,9 @@ def test_openrouter_decisions_response_is_priced_from_request_model_registry_row
custom_llm_provider="openrouter",
)
expected_cost = 282 * model_cost["input_cost_per_token"] + 20 * model_cost["output_cost_per_token"]
assert response["kwargs"]["model"] == "openrouter/typesafe/jev-1.13-20260917"
assert response["kwargs"]["custom_llm_provider"] == "openrouter"
assert response["kwargs"]["response_cost"] == pytest.approx(expected_cost)
assert response["kwargs"]["response_cost"] == cost
assert response["kwargs"]["combined_usage_object"].prompt_tokens == 282
assert response["kwargs"]["combined_usage_object"].completion_tokens == 20
assert response["kwargs"]["combined_usage_object"].total_tokens == 302

104
tests/unit/router_strategy/complexity_router/test_jev_classifier.py
View file

@ -451,7 +451,14 @@ def test_jev_config_requires_classifier_config() -> None:
)
@pytest.mark.parametrize(
("provider", "model", "canonical_provider"),
[(None, "jev-latest", "jev"), ("typesafe", "jev-latest", "jev"), ("jev", "jev-latest", "jev"), ("laya", "english", "laya"), ("bespoke", "nimble-latest", "bespoke")],
[
(None, "jev-latest", "jev"),
("typesafe", "jev-latest", "jev"),
("jev", "jev-latest", "jev"),
("laya", "english", "laya"),
("bespoke", "nimble-latest", "bespoke"),
("databricks", "databricks-openjev-qwen35-4b", "databricks"),
],
)
def test_classifier_aliases_load_and_serialize_one_canonical_config(
classifier_type: str, config_key: str, provider: str | None, model: str, canonical_provider: str
@ -533,6 +540,85 @@ async def test_oss_routes_with_its_own_credentials_and_accounts_the_checkpoint(
assert recorder.calls[0]["response_cost"] == pytest.approx(0.31)
def test_databricks_requires_the_bare_serving_endpoint_name_as_the_model() -> None:
with pytest.raises(ValueError, match="model is required for provider 'databricks'"):
JevClassifierConfig.model_validate({"provider": "databricks"})
with pytest.raises(ValueError, match="bare endpoint name"):
JevClassifierConfig.model_validate({"provider": "databricks", "model": "serving-endpoints/my-openjev"})
with pytest.raises(ValueError, match="bare endpoint name"):
JevClassifierConfig.model_validate({"provider": "databricks", "model": ".."})
assert JevClassifierConfig.model_validate({"provider": "databricks", "model": "my-openjev"}).model == "my-openjev"
@pytest.mark.asyncio
@pytest.mark.parametrize("configured_key", [False, True])
@pytest.mark.parametrize("legacy", [False, True])
async def test_databricks_routes_through_the_serving_endpoint_and_accounts_the_endpoint_model(
monkeypatch: pytest.MonkeyPatch, configured_key: bool, legacy: bool
) -> None:
endpoint: Final = "databricks-openjev-qwen35-4b"
monkeypatch.setenv("TYPESAFE_API_KEY", "never-send-typesafe-key")
monkeypatch.setenv("DATABRICKS_API_BASE", "https://workspace.test/serving-endpoints")
monkeypatch.delenv("DATABRICKS_API_KEY", raising=False)
monkeypatch.setenv("DATABRICKS_TOKEN", "dapi-env-token")
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setitem(litellm.model_cost, f"databricks/{endpoint}", {"input_cost_per_token": 0.01})
recorder: Final = _UsageRecorder(f"databricks/{endpoint}")
monkeypatch.setattr(litellm, "_async_success_callback", [recorder])
router: Final = ComplexityRouter(
"databricks-route",
litellm.Router(model_list=[]),
{
"classifier_type": "jev" if legacy else "oss_classifier",
"jev_classifier_config" if legacy else "opensource_classifier_config": {
"provider": "databricks",
"model": endpoint,
**({"api_key": "dapi-configured", "api_base": "https://workspace.test/serving-endpoints"} if configured_key else {}),
},
"tiers": {"SIMPLE": "cheap"},
},
derive_savings_baseline=False,
)
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post(f"https://workspace.test/serving-endpoints/{endpoint}/invocations").respond(
200,
json={
"model": "/mosaicml/local_model",
"answers": {"tier": _answer().model_dump()},
"usage": {"input_tokens": 31, "output_tokens": 0},
},
)
outcome: Final = await router.aclassify("choose a tier")
await GLOBAL_LOGGING_WORKER.flush()
assert outcome.cause == "jev_classifier"
assert outcome.jev_verdict is not None
assert (outcome.jev_verdict.provider, outcome.jev_verdict.model) == ("databricks", endpoint)
assert outcome.classifier_cost == pytest.approx(0.31)
sent: Final = route.calls.last.request
assert sent.headers.get("authorization") == ("Bearer dapi-configured" if configured_key else "Bearer dapi-env-token")
assert json.loads(sent.content)["model"] == endpoint
assert len(recorder.calls) == 1
assert recorder.calls[0]["response_cost"] == pytest.approx(0.31)
def test_databricks_without_a_base_or_key_fails_at_client_build(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("DATABRICKS_API_BASE", raising=False)
monkeypatch.delenv("DATABRICKS_API_KEY", raising=False)
monkeypatch.delenv("DATABRICKS_TOKEN", raising=False)
with pytest.raises(ValueError, match="DATABRICKS_API_BASE"):
ComplexityRouter(
"databricks-route",
litellm.Router(model_list=[]),
{
"classifier_type": "oss_classifier",
"opensource_classifier_config": {"provider": "databricks", "model": "databricks-openjev-qwen35-4b"},
"tiers": {"SIMPLE": "cheap"},
},
derive_savings_baseline=False,
)
def test_jev_config_is_rejected_for_other_classifier_types() -> None:
with pytest.raises(ValueError, match="has no effect"):
ComplexityRouterConfig.model_validate(
@ -570,6 +656,22 @@ def test_jev_api_base_without_its_own_key_is_rejected_so_the_environment_key_sta
assert JevClassifierConfig(api_key="sk-own").api_base is None
def test_databricks_api_base_without_its_own_key_is_rejected_so_the_workspace_token_stays_home() -> None:
with pytest.raises(ValueError, match="DATABRICKS_API_KEY or DATABRICKS_TOKEN is only sent to DATABRICKS_API_BASE"):
JevClassifierConfig.model_validate(
{"provider": "databricks", "model": "my-openjev", "api_base": "https://collector.invalid/serving-endpoints"}
)
paired: Final = JevClassifierConfig.model_validate(
{
"provider": "databricks",
"model": "my-openjev",
"api_base": "https://workspace.invalid/serving-endpoints",
"api_key": "dapi-own",
}
)
assert (paired.api_base, paired.api_key) == ("https://workspace.invalid/serving-endpoints", "dapi-own")
@pytest.mark.parametrize(
("probabilities", "confidence"),
[

0
tests/unit/systemone/__init__.py Normal file
View file

138
tests/unit/systemone/test_main.py Normal file
View file

@ -0,0 +1,138 @@
from __future__ import annotations
import asyncio
import json
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import pytest
import respx
from respx.models import Call
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse
_STATE: Final = "Customer asked for a refund"
_SYSTEMONE_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType(
{"is_refund": {"type": "noul", "instructions": "Is this a refund request?"}}
)
_OPENAI_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = (
{"type": "predicate", "name": "is_refund", "instructions": "Is this a refund request?"},
)
_SYSTEMONE_RESPONSE: Final[Mapping[str, object]] = {
"model": "jev-1.13",
"answers": {"is_refund": {"type": "noul", "noul": 0.93}},
"usage": {"input_tokens": 12, "output_tokens": 1},
}
_TYPESAFE_URL: Final = "https://api.typesafe.ai/v1/systemone"
@pytest.fixture(autouse=True)
def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
class _CallTypeRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.call_types: list[str] = [] # mutable-ok: test recorder collects logger callbacks
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
self.call_types.append(kwargs["call_type"])
@pytest.mark.asyncio
@pytest.mark.parametrize("use_async", (True, False), ids=("asystemone", "systemone"))
async def test_systemone_sends_the_systemone_wire_format_and_returns_a_systemone_response(
use_async: bool, respx_mock: respx.MockRouter
) -> None:
route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE)
kwargs: Final[Mapping[str, object]] = MappingProxyType(
{"model": "typesafe/jev-1.13", "state": _STATE, "questions": _SYSTEMONE_QUESTIONS, "api_key": "caller-key"}
)
response: Final = await litellm.asystemone(**kwargs) if use_async else litellm.systemone(**kwargs)
assert route.called
call: Final[Call] = respx_mock.calls.last
assert json.loads(call.request.content) == {
"model": "jev-1.13",
"state": _STATE,
"questions": {"is_refund": {"type": "noul", "instructions": "Is this a refund request?"}},
}
assert isinstance(response, DecisionsResponse)
assert response.answers["is_refund"].model_dump(mode="json") == {"type": "noul", "noul": 0.93}
@pytest.mark.asyncio
async def test_systemone_rejects_openai_input_and_points_at_decisions(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE)
with pytest.raises(litellm.BadRequestError, match=r"litellm\.decisions") as caught:
await litellm.asystemone(
model="typesafe/jev-1.13", input=_STATE, questions=_SYSTEMONE_QUESTIONS, api_key="caller-key"
)
assert caught.value.status_code == 400
assert not route.called
@pytest.mark.asyncio
@pytest.mark.parametrize("use_async", (True, False), ids=("adecisions", "decisions"))
async def test_decisions_with_state_still_sends_the_systemone_format_and_warns(
use_async: bool, respx_mock: respx.MockRouter
) -> None:
route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE)
kwargs: Final[Mapping[str, object]] = MappingProxyType(
{"model": "typesafe/jev-1.13", "state": _STATE, "questions": _SYSTEMONE_QUESTIONS, "api_key": "caller-key"}
)
with pytest.warns(UserWarning, match=r"litellm\.systemone"):
response: Final = await litellm.adecisions(**kwargs) if use_async else litellm.decisions(**kwargs)
assert route.called
call: Final[Call] = respx_mock.calls.last
assert json.loads(call.request.content) == {
"model": "jev-1.13",
"state": _STATE,
"questions": {"is_refund": {"type": "noul", "instructions": "Is this a refund request?"}},
}
assert isinstance(response, DecisionsResponse)
assert response.answers["is_refund"].model_dump(mode="json") == {"type": "noul", "noul": 0.93}
@pytest.mark.asyncio
async def test_systemone_rejects_openai_question_lists_before_http(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE)
with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"):
await litellm.asystemone(
model="typesafe/jev-1.13", state=_STATE, questions=_OPENAI_QUESTIONS, api_key="caller-key"
)
assert not route.called
@pytest.mark.asyncio
async def test_router_asystemone_logs_the_systemone_call_type(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
respx_mock.post(_TYPESAFE_URL).respond(json=_SYSTEMONE_RESPONSE)
recorder: Final = _CallTypeRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
router: Final = litellm.Router(
model_list=[{"model_name": "jev", "litellm_params": {"model": "typesafe/jev-1.13", "api_key": "caller-key"}}]
)
response: Final = await router.asystemone(model="jev", state=_STATE, questions=_SYSTEMONE_QUESTIONS)
await asyncio.sleep(0)
GLOBAL_LOGGING_WORKER.start()
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0)
assert isinstance(response, DecisionsResponse)
assert recorder.call_types == ["asystemone"]
assert not isinstance(response, OpenAIDecisionResponse)

25
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx
View file

@ -22,15 +22,7 @@ vi.mock("./detailNavigation", () => ({
}));
vi.mock("@/components/molecules/DecisionModelsBanner", () => ({
default: ({ onAddModel }: { onAddModel?: () => void }) => (
<section aria-label="Decision models">
{onAddModel && (
<button type="button" onClick={onAddModel}>
Add a decision model
</button>
)}
</section>
),
default: () => <section aria-label="Decision models" />,
}));
vi.mock("@/components/model_info_view", () => ({
default: ({ modelId }: { modelId: string }) => <div data-testid="model-info">model:{modelId}</div>,
@ -107,21 +99,6 @@ describe("ModelsAndEndpointsPage", () => {
expect(screen.queryByTestId("panel-all-models")).not.toBeInTheDocument();
});
it("opens the Add Model tab from the decision models banner", async () => {
const user = userEvent.setup();
renderPage();
await user.click(screen.getByRole("button", { name: "Add a decision model" }));
expect(screen.getByTestId("panel-add")).toBeInTheDocument();
expect(screen.queryByTestId("panel-all-models")).not.toBeInTheDocument();
});
it("does not offer the banner's Add a decision model to a session that cannot add models", () => {
mockUseAuthorized.mockReturnValue(NON_ADMIN);
renderPage();
expect(screen.getByRole("region", { name: "Decision models" })).toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Add a decision model" })).not.toBeInTheDocument();
});
it("does not show a view-only session the decision models banner, whose Playground it cannot open", () => {
mockUseAuthorized.mockReturnValue(VIEW_ONLY_ADMIN);
renderPage();

4
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx
View file

@ -177,9 +177,7 @@ export default function ModelsAndEndpointsPage() {
</div>
</div>
{!isViewOnly && (
<DecisionModelsBanner onAddModel={canCreate && !modelId ? () => setActiveKey("add") : undefined} />
)}
{!isViewOnly && <DecisionModelsBanner />}
{modelId ? (
<ModelInfoView

10
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx
View file

@ -563,14 +563,6 @@ describe("AddModelPanel decision models", () => {
const decisionNotice = () => screen.queryByRole("note", { name: "Decision model notice" });
it("finds a provider by the name of one of its decision models", async () => {
const { user } = await setup();
await user.type(screen.getByRole("combobox", { name: /provider/i }), "jev");
expect(await screen.findByRole("option", { name: /TypeSafe/ })).toBeInTheDocument();
expect(screen.queryByRole("option", { name: /Anthropic/ })).not.toBeInTheDocument();
});
it("points a decision-only provider at the decision routes, the docs, and the playground", async () => {
const { user } = await setup();
await user.type(screen.getByRole("combobox", { name: /provider/i }), "TypeSafe");
@ -582,7 +574,7 @@ describe("AddModelPanel decision models", () => {
"href",
"https://docs.litellm.ai/docs/decisions",
);
expect(within(note).getByRole("link", { name: "test it in the System One playground" })).toHaveAttribute(
expect(within(note).getByRole("link", { name: "test it in the Decisions playground" })).toHaveAttribute(
"href",
"/ui/playground?tab=system-one",
);

43
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/DecisionFormControls.tsx Normal file
View file

@ -0,0 +1,43 @@
import { Button } from "@/components/ui/button";
import { cn } from "@/lib/cva.config";
import { Plus, Trash2 } from "lucide-react";
import { useId, type ReactNode } from "react";
import { IssueList, ValidationStatus } from "./JsonEditor";
import type { PayloadValidation } from "./lib/validatePayload";
export function RemoveButton({ label, onClick }: { label: string; onClick: () => void }) {
return (
<Button variant="ghost" size="icon" aria-label={label} title={label} onClick={onClick}>
<Trash2 />
</Button>
);
}
export function AddButton({ children, onClick }: { children: ReactNode; onClick: () => void }) {
return (
<Button variant="outline" size="sm" className="w-fit" onClick={onClick}>
<Plus />
{children}
</Button>
);
}
export function FormFrame({ validation, children }: { validation: PayloadValidation<unknown>; children: ReactNode }) {
const issuesId = useId();
return (
<div
className={cn(
"flex min-h-96 flex-1 flex-col overflow-hidden rounded-md border bg-background",
!validation.isValid && "border-destructive/60",
)}
>
<div className="flex items-center gap-2 border-b px-3 py-2">
<span className="text-sm font-medium">Request form</span>
<ValidationStatus validation={validation} />
</div>
<div className="grid min-h-80 flex-1 content-start gap-4 overflow-auto p-3">{children}</div>
<IssueList id={issuesId} validation={validation} />
</div>
);
}

132
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/JsonEditor.tsx
View file

@ -1,12 +1,13 @@
import { Badge } from "@/components/ui/badge";
import { cn } from "@/lib/cva.config";
import { CircleAlert, CircleCheck, TriangleAlert } from "lucide-react";
import { useId, useMemo, useRef } from "react";
import { useTheme } from "next-themes";
import { useId, useRef } from "react";
import { createElement, PrismLight as SyntaxHighlighter } from "react-syntax-highlighter";
import type { SyntaxHighlighterProps } from "react-syntax-highlighter";
import json from "react-syntax-highlighter/dist/esm/languages/prism/json";
import { findRootBlocks, ROOT_BLOCK_STYLES, type RootBlock } from "./lib/rootBlocks";
import type { SystemOnePayloadValidation } from "./lib/validatePayload";
import { vs, vscDarkPlus } from "react-syntax-highlighter/dist/esm/styles/prism";
import type { PayloadValidation } from "./lib/validatePayload";
SyntaxHighlighter.registerLanguage("json", json);
@ -14,24 +15,54 @@ const EDITOR_TEXT = "m-0 whitespace-pre-wrap wrap-anywhere py-3 font-mono text-x
const GUTTER_WIDTH = "w-11";
type LineRendererProps = Parameters<NonNullable<SyntaxHighlighterProps["renderer"]>>[0];
const CONTENT_INSET = "pl-14 pr-3";
const CODE_TAG_PROPS = { className: "language-json", style: { whiteSpace: "pre-wrap" } } as const;
type SyntaxTheme = NonNullable<SyntaxHighlighterProps["style"]>;
const THEME_BLOCKS = ['pre[class*="language-"]', 'code[class*="language-"]'] as const;
const TOKEN_COLORS = [
"[&_.token.property]:text-sky-700 dark:[&_.token.property]:text-sky-300",
"[&_.token.string]:text-emerald-700 dark:[&_.token.string]:text-emerald-300",
"[&_.token.number]:text-amber-700 dark:[&_.token.number]:text-amber-300",
"[&_.token.boolean]:text-violet-700 dark:[&_.token.boolean]:text-violet-300",
"[&_.token.null]:text-violet-700 dark:[&_.token.null]:text-violet-300",
"[&_.token.punctuation]:text-muted-foreground [&_.token.operator]:text-muted-foreground",
].join(" ");
function withoutBlockBackground(theme: SyntaxTheme): SyntaxTheme {
const stripped = THEME_BLOCKS.map((block) => {
const { background: _background, backgroundColor: _backgroundColor, ...rest } = theme[block] ?? {};
return [block, rest] as const;
});
return { ...theme, ...Object.fromEntries(stripped) };
}
const VS_CODE_DARK_PLUS = withoutBlockBackground(vscDarkPlus);
const VS_CODE_LIGHT_PLUS = {
...withoutBlockBackground(vs),
property: { color: "#0451a5" },
string: { color: "#a31515" },
number: { color: "#098658" },
boolean: { color: "#0000ff" },
punctuation: { color: "#000000" },
operator: { color: "#000000" },
} as const;
const INHERIT_TEXT = {
fontFamily: "inherit",
fontSize: "inherit",
lineHeight: "inherit",
whiteSpace: "pre-wrap",
overflowWrap: "anywhere",
textShadow: "none",
} as const;
const CODE_TAG_PROPS = { className: "language-json", style: INHERIT_TEXT } as const;
const TRANSPARENT_PRE = {
...INHERIT_TEXT,
background: "transparent",
border: 0,
margin: 0,
padding: 0,
overflow: "visible",
} as const;
interface JsonEditorProps {
value: string;
onChange: (value: string) => void;
validation: SystemOnePayloadValidation;
validation: PayloadValidation<unknown>;
label?: string;
placeholder?: string;
}
export function ValidationStatus({ validation }: { validation: SystemOnePayloadValidation }) {
export function ValidationStatus({ validation }: { validation: PayloadValidation<unknown> }) {
const errorCount = validation.issues.filter((issue) => issue.severity === "error").length;
if (errorCount > 0) {
return (
@ -43,7 +74,7 @@ export function ValidationStatus({ validation }: { validation: SystemOnePayloadV
return <Badge variant="secondary">Valid payload</Badge>;
}
export function IssueList({ id, validation }: { id: string; validation: SystemOnePayloadValidation }) {
export function IssueList({ id, validation }: { id: string; validation: PayloadValidation<unknown> }) {
if (validation.issues.length === 0) {
return (
<p id={id} role="status" className="flex items-center gap-1.5 border-t px-3 py-2 text-xs text-muted-foreground">
@ -69,39 +100,30 @@ export function IssueList({ id, validation }: { id: string; validation: SystemOn
);
}
function renderLines(rootBlocks: readonly RootBlock[]) {
return function LineRows({ rows, stylesheet, useInlineStyles }: LineRendererProps) {
return rows.map((row, line) => {
const lineElement = { node: row, stylesheet, useInlineStyles, key: line };
const block = rootBlocks.find(({ startLine, endLine }) => line >= startLine && line <= endLine);
return (
<div key={line} className="flex">
<span className={cn(GUTTER_WIDTH, "shrink-0 select-none pr-3 text-right text-muted-foreground")}>
{line + 1}
</span>
<span
className={cn(
"relative min-h-5 min-w-0 flex-1 px-3",
block && [
"before:absolute before:inset-y-0 before:left-0 before:w-0.5",
ROOT_BLOCK_STYLES[block.key].band,
line === block.startLine && "rounded-t-sm before:rounded-t-sm",
line === block.endLine && "rounded-b-sm before:rounded-b-sm",
],
)}
>
{createElement(lineElement)}
</span>
</div>
);
});
};
function LineRows({ rows, stylesheet, useInlineStyles }: LineRendererProps) {
return rows.map((row, line) => {
const lineElement = { node: row, stylesheet, useInlineStyles, key: line };
return (
<div key={line} className="flex">
<span className={cn(GUTTER_WIDTH, "shrink-0 select-none pr-3 text-right text-zinc-400 dark:text-zinc-500")}>
{line + 1}
</span>
<span className="min-h-5 min-w-0 flex-1 px-3">{createElement(lineElement)}</span>
</div>
);
});
}
export default function JsonEditor({ value, onChange, validation }: JsonEditorProps) {
export default function JsonEditor({
value,
onChange,
validation,
label = "System One JSON payload",
placeholder = "Paste or write a System One request",
}: JsonEditorProps) {
const issuesId = useId();
const highlightRef = useRef<HTMLDivElement>(null);
const renderer = useMemo(() => renderLines(findRootBlocks(value)), [value]);
const syntaxTheme = useTheme().resolvedTheme === "dark" ? VS_CODE_DARK_PLUS : VS_CODE_LIGHT_PLUS;
const lineCount = value.split("\n").length;
const hasErrors = !validation.isValid;
@ -121,26 +143,28 @@ export default function JsonEditor({ value, onChange, validation }: JsonEditorPr
{lineCount} {lineCount === 1 ? "line" : "lines"}
</span>
</div>
<div className="relative min-h-80 flex-1">
<div aria-hidden="true" className={cn(GUTTER_WIDTH, "absolute inset-y-0 left-0 border-r bg-muted/50")} />
<div className="relative min-h-80 flex-1 bg-background dark:bg-[#1e1e1e]">
<div
ref={highlightRef}
aria-hidden="true"
className={cn("absolute inset-0 overflow-hidden", EDITOR_TEXT, TOKEN_COLORS)}
>
className={cn(
GUTTER_WIDTH,
"absolute inset-y-0 left-0 border-r bg-muted/50 dark:border-zinc-700 dark:bg-zinc-800/60",
)}
/>
<div ref={highlightRef} aria-hidden="true" className={cn("absolute inset-0 overflow-hidden", EDITOR_TEXT)}>
<SyntaxHighlighter
language="json"
style={{}}
useInlineStyles={false}
style={syntaxTheme}
customStyle={TRANSPARENT_PRE}
PreTag="div"
codeTagProps={CODE_TAG_PROPS}
renderer={renderer}
renderer={LineRows}
>
{value}
</SyntaxHighlighter>
</div>
<textarea
aria-label="System One JSON payload"
aria-label={label}
aria-invalid={hasErrors}
aria-describedby={issuesId}
value={value}
@ -153,7 +177,7 @@ export default function JsonEditor({ value, onChange, validation }: JsonEditorPr
spellCheck={false}
autoCapitalize="off"
autoComplete="off"
placeholder="Paste or write a System One request"
placeholder={placeholder}
className={cn(
EDITOR_TEXT,
CONTENT_INSET,

267
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/OpenAIDecisionsForm.tsx Normal file
View file

@ -0,0 +1,267 @@
"use client";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Textarea } from "@/components/ui/textarea";
import { useId, useMemo } from "react";
import { AddButton, FormFrame, RemoveButton } from "./DecisionFormControls";
import { freeName, removeAt, replaceAt } from "./lib/formPayload";
import {
readDecisionsForm,
retypeDecisionQuestion,
type DecisionsFormPayload,
type DecisionsFormQuestion,
type DecisionsQuestionType,
} from "./lib/openAIDecisionsForm";
import type { OpenAIDecisionsRequest } from "./lib/openAIDecisions";
import type { PayloadValidation } from "./lib/validatePayload";
const QUESTION_TYPES: readonly DecisionsQuestionType[] = ["predicate", "choice", "score"];
const TYPE_LABELS = { predicate: "Predicate", choice: "Choice", score: "Score" };
interface QuestionEditorProps<Q extends DecisionsFormQuestion = DecisionsFormQuestion> {
question: Q;
onChange: (question: DecisionsFormQuestion) => void;
}
function Choices({ question, onChange }: QuestionEditorProps<Extract<DecisionsFormQuestion, { type: "choice" }>>) {
const choices = question.choices ?? [];
const write = (next: typeof choices) => onChange({ ...question, choices: next });
return (
<div className="grid gap-2">
<span className="text-sm font-medium">Choices</span>
{choices.map((choice, index) => (
<div key={index} className="grid gap-2 rounded-md border p-2">
<div className="flex items-center gap-2">
<Select
value={typeof choice.value === "string" ? "text" : String(choice.value)}
onValueChange={(type) => {
if (type === "text" || type === "true" || type === "false") {
const value = type === "text" ? String(choice.value) : type === "true";
write(replaceAt(choices, index, { ...choice, value }));
}
}}
>
<SelectTrigger className="w-40" aria-label={`Choice ${index + 1} value type`}>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="text">Text</SelectItem>
<SelectItem value="true">true (boolean)</SelectItem>
<SelectItem value="false">false (boolean)</SelectItem>
</SelectContent>
</Select>
{typeof choice.value === "string" && (
<Input
aria-label={`Choice ${index + 1} value`}
value={choice.value}
onChange={(event) => write(replaceAt(choices, index, { ...choice, value: event.target.value }))}
/>
)}
<RemoveButton label={`Remove choice ${index + 1}`} onClick={() => write(removeAt(choices, index))} />
</div>
<Input
aria-label={`Choice ${index + 1} description`}
value={choice.description ?? ""}
placeholder="Optional description"
onChange={(event) => write(replaceAt(choices, index, { ...choice, description: event.target.value }))}
/>
</div>
))}
<AddButton
onClick={() =>
write([
...choices,
{
value: freeName(
choices.map((c) => String(c.value)),
"option",
),
},
])
}
>
Add choice
</AddButton>
</div>
);
}
function Levels({ question, onChange }: QuestionEditorProps<Extract<DecisionsFormQuestion, { type: "score" }>>) {
const levels = question.levels ?? [];
const write = (next: typeof levels) => onChange({ ...question, levels: next });
return (
<div className="grid gap-2">
<span className="text-sm font-medium">Levels, lowest first</span>
{levels.map((level, index) => (
<div key={index} className="flex items-center gap-2">
<Input
aria-label={`Level ${index + 1} label`}
value={level.label}
placeholder="Label"
onChange={(event) => write(replaceAt(levels, index, { ...level, label: event.target.value }))}
/>
<Input
aria-label={`Level ${index + 1} description`}
value={level.description ?? ""}
placeholder="Optional description"
onChange={(event) => write(replaceAt(levels, index, { ...level, description: event.target.value }))}
/>
<RemoveButton label={`Remove level ${index + 1}`} onClick={() => write(removeAt(levels, index))} />
</div>
))}
<AddButton onClick={() => write([...levels, { label: String(levels.length) }])}>Add level</AddButton>
</div>
);
}
interface QuestionCardProps extends QuestionEditorProps {
index: number;
onRemove: () => void;
}
function QuestionCard({ question, index, onChange, onRemove }: QuestionCardProps) {
const instructionsId = useId();
return (
<fieldset aria-label={`Question ${index + 1}`} className="grid gap-3 rounded-md border p-3">
<div className="flex flex-wrap items-center gap-2">
<Input
aria-label="Question name"
className="min-w-0 flex-1"
value={question.name ?? ""}
placeholder="Optional question name"
onChange={(event) => onChange({ ...question, name: event.target.value })}
/>
<Select
value={question.type}
onValueChange={(type) => {
const next = QUESTION_TYPES.find((value) => value === type);
if (next) onChange(retypeDecisionQuestion(question, next));
}}
>
<SelectTrigger className="w-36" aria-label="Answer type">
<SelectValue>{TYPE_LABELS[question.type]}</SelectValue>
</SelectTrigger>
<SelectContent>
{QUESTION_TYPES.map((type) => (
<SelectItem key={type} value={type}>
{TYPE_LABELS[type]}
</SelectItem>
))}
</SelectContent>
</Select>
<RemoveButton label={`Remove question ${index + 1}`} onClick={onRemove} />
</div>
<div className="grid gap-1.5">
<Label htmlFor={instructionsId}>Instructions</Label>
<Textarea
id={instructionsId}
rows={2}
value={question.instructions ?? ""}
placeholder="What should the model decide?"
onChange={(event) => onChange({ ...question, instructions: event.target.value })}
/>
</div>
{question.type === "choice" && <Choices question={question} onChange={onChange} />}
{question.type === "score" && <Levels question={question} onChange={onChange} />}
</fieldset>
);
}
interface OpenAIDecisionsFormProps {
value: string;
onChange: (value: string) => void;
validation: PayloadValidation<OpenAIDecisionsRequest>;
onOpenJson: () => void;
}
export default function OpenAIDecisionsForm({ value, onChange, validation, onOpenJson }: OpenAIDecisionsFormProps) {
const inputId = useId();
const safetyId = useId();
const form = useMemo(() => readDecisionsForm(value), [value]);
if (form === undefined) {
return (
<FormFrame validation={validation}>
<p className="text-sm text-muted-foreground">
This request has JSON the form can&apos;t show. Edit it in JSON to fix its shape.
</p>
<Button variant="outline" className="w-fit" onClick={onOpenJson}>
Edit in JSON
</Button>
</FormFrame>
);
}
const questions = form.questions ?? [];
const write = (next: DecisionsFormPayload) => onChange(JSON.stringify(next, null, 2));
const writeQuestions = (next: DecisionsFormQuestion[]) => write({ ...form, questions: next });
return (
<FormFrame validation={validation}>
<div className="grid gap-1.5">
{Array.isArray(form.input) ? (
<>
<p className="text-sm text-muted-foreground">
Input contains messages. Edit messages and image parts in JSON; form edits keep them unchanged.
</p>
<Button variant="outline" className="w-fit" onClick={onOpenJson}>
Edit input in JSON
</Button>
</>
) : (
<>
<Label htmlFor={inputId}>Input</Label>
<Textarea
id={inputId}
rows={4}
value={form.input ?? ""}
placeholder="Paste the text the model should decide about"
onChange={(event) => write({ ...form, input: event.target.value })}
/>
</>
)}
</div>
{questions.map((question, index) => (
<QuestionCard
key={index}
index={index}
question={question}
onChange={(next) => writeQuestions(replaceAt(questions, index, next))}
onRemove={() => writeQuestions(removeAt(questions, index))}
/>
))}
<AddButton
onClick={() =>
writeQuestions([
...questions,
{
type: "predicate",
name: freeName(
questions.map((q) => q.name ?? ""),
"question",
),
instructions: "",
},
])
}
>
Add question
</AddButton>
<div className="grid gap-1.5">
<Label htmlFor={safetyId}>Safety identifier</Label>
<Input
id={safetyId}
value={form.safety_identifier ?? ""}
placeholder="Optional"
onChange={(event) => write({ ...form, safety_identifier: event.target.value || undefined })}
/>
</div>
</FormFrame>
);
}

9
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/QuestionBreakdown.tsx
View file

@ -1,7 +1,5 @@
import { Badge } from "@/components/ui/badge";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { cn } from "@/lib/cva.config";
import { ROOT_BLOCK_STYLES } from "./lib/rootBlocks";
import type { PlaygroundQuestion, PlaygroundRequest } from "./lib/schemas";
function formatState(state: unknown): string {
@ -74,10 +72,7 @@ export default function QuestionBreakdown({ payload }: { payload?: PlaygroundReq
<CardDescription>Review the state and criteria that will be sent.</CardDescription>
</CardHeader>
<CardContent className="grid gap-4 wrap-anywhere">
<section
aria-labelledby="system-one-state-heading"
className={cn("grid gap-2 border-l-2 pl-3", ROOT_BLOCK_STYLES.state.accent)}
>
<section aria-labelledby="system-one-state-heading" className="grid gap-2">
<h3 id="system-one-state-heading" className="text-sm font-medium">
State
</h3>
@ -85,7 +80,7 @@ export default function QuestionBreakdown({ payload }: { payload?: PlaygroundReq
{formatState(payload.state)}
</pre>
</section>
<div className={cn("grid gap-3 border-l-2 pl-3", ROOT_BLOCK_STYLES.questions.accent)}>
<div className="grid gap-3">
{Object.entries(payload.questions).map(([id, question]) => (
<section key={id} className="grid gap-3 rounded-md border p-3" aria-labelledby={`question-${id}`}>
<div className="flex flex-wrap items-center justify-between gap-2">

164
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx Normal file
View file

@ -0,0 +1,164 @@
import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { describe, expect, it } from "vitest";
import ResponseView from "./ResponseView";
import type { SystemOneResponse } from "./lib/schemas";
import type { OpenAIDecisionsResponse } from "./lib/openAIDecisions";
const sampleResponse: SystemOneResponse = {
model: "jev-1.13.0",
answers: {
human: { type: "noul", noul: 0.99 },
dept: {
type: "choice",
choice: "technical",
confidence: 0.27,
probabilities: { billing: 0.36, technical: 0.64 },
},
frustration: {
type: "score",
score: 2,
confidence: 0.99,
legend: { "0": "calm", "1": "mild", "2": "angry" },
probabilities: { "0": 0, "1": 0, "2": 1 },
},
},
usage: { input_tokens: 378, output_tokens: 63 },
};
describe("ResponseView integration", () => {
it("shows OpenAI Decisions answer cards in Form view and the complete response in JSON view", async () => {
const response: OpenAIDecisionsResponse = {
model: "jev-latest",
answers: [
{ name: null, type: "predicate", probability: 0 },
{
name: "route",
type: "choice",
choice: false,
confidence: 0.8,
probabilities: [
{ value: false, probability: 0.8 },
{ value: "other", probability: 0.2 },
],
},
{
name: "severity",
type: "score",
score: 0,
confidence: 0,
probabilities: [
{ value: 0, label: "low", probability: 1 },
{ value: 1, label: "high", probability: 0 },
],
},
{ name: "blocked", type: "refusal" },
{ name: "blocked", type: "refusal" },
],
usage: { input_tokens: 3, output_tokens: 4, total_tokens: 7 },
request_id: "keep-extra-fields",
};
const { rerender } = render(<ResponseView response={response} isLoading={false} view="form" />);
expect(screen.getByText("Question 1")).toBeInTheDocument();
expect(screen.getByText("0% yes")).toBeInTheDocument();
expect(screen.getByRole("meter", { name: "false probability" })).toHaveAttribute("aria-valuetext", "80%, selected");
expect(screen.getByRole("meter", { name: "0: low probability" })).toHaveAttribute(
"aria-valuetext",
"100%, selected",
);
expect(screen.getByText("0% confidence")).toBeInTheDocument();
expect(screen.getAllByText("Model refused to answer")).toHaveLength(2);
expect(screen.queryByRole("region", { name: "Decisions response JSON" })).not.toBeInTheDocument();
await userEvent.click(screen.getByRole("button", { name: "Raw response" }));
expect(screen.getByRole("region", { name: "Decisions response JSON" })).toHaveTextContent("keep-extra-fields");
rerender(<ResponseView response={response} isLoading={false} view="json" />);
expect(screen.queryByRole("meter")).not.toBeInTheDocument();
expect(JSON.parse(screen.getByRole("region", { name: "Decisions response JSON" }).textContent ?? "")).toEqual(
response,
);
rerender(<ResponseView response={response} isLoading={false} view="form" />);
expect(screen.getByRole("meter", { name: "false probability" })).toHaveAttribute("aria-valuenow", "80");
});
it("shows System One raw response instead of cards in JSON view", () => {
render(<ResponseView response={sampleResponse} isLoading={false} view="json" />);
expect(screen.queryByRole("meter")).not.toBeInTheDocument();
expect(JSON.parse(screen.getByRole("region", { name: "Decisions response JSON" }).textContent ?? "")).toEqual(
sampleResponse,
);
});
it("renders calibrated probabilities for choice, noul, and score answers", () => {
render(<ResponseView response={sampleResponse} isLoading={false} latencyMs={120} />);
expect(screen.getAllByText("technical")).toHaveLength(2);
const meters = screen.getAllByRole("meter");
expect(meters.map((meter) => meter.getAttribute("aria-label"))).toEqual([
"Yes probability",
"technical probability",
"billing probability",
"0: calm probability",
"1: mild probability",
"2: angry probability",
]);
expect(screen.getByRole("meter", { name: "technical probability" })).toHaveAttribute("aria-valuenow", "64");
expect(screen.getByRole("meter", { name: "billing probability" })).toHaveAttribute("aria-valuenow", "36");
expect(screen.getByText("99% yes")).toBeInTheDocument();
expect(screen.getByRole("meter", { name: "Yes probability" })).toHaveAttribute("aria-valuenow", "99");
expect(screen.getByRole("meter", { name: "2: angry probability" })).toBeInTheDocument();
expect(screen.getByText("378 input / 63 output tokens")).toBeInTheDocument();
});
it("selects the score level nearest a fractional calibrated score", () => {
const fractional: SystemOneResponse = {
answers: {
severity: { type: "score", score: 2.39, probabilities: { "1": 0, "2": 0.62, "3": 0.38 } },
},
};
render(<ResponseView response={fractional} isLoading={false} />);
expect(screen.getByRole("meter", { name: "2 probability" })).toHaveAttribute("aria-valuetext", "62%, selected");
expect(screen.getByRole("meter", { name: "3 probability" })).toHaveAttribute("aria-valuetext", "38%");
expect(screen.getByRole("meter", { name: "1 probability" })).toHaveAttribute("aria-valuetext", "0%");
});
it("rounds the calibrated score pill to two decimals and leaves integers bare", () => {
const precise: SystemOneResponse = {
answers: {
severity: { type: "score", score: 3.9809950180008897, probabilities: { "3": 0.02, "4": 0.98 } },
clarity: { type: "score", score: 2, probabilities: { "2": 1 }, legend: { "2": "Clear enough" } },
},
};
render(<ResponseView response={precise} isLoading={false} />);
expect(screen.getByText("3.98")).toBeInTheDocument();
expect(screen.queryByText("3.9809950180008897")).not.toBeInTheDocument();
expect(screen.getByText("2")).toBeInTheDocument();
});
it("renders structured decision score legends without coercing objects to strings", () => {
const response: SystemOneResponse = {
answers: {
severity: {
type: "score",
score: 0,
probabilities: { "0": 1 },
legend: { "0": { description: "Low severity" } },
},
},
usage: null,
};
render(<ResponseView response={response} isLoading={false} />);
expect(screen.getByRole("meter", { name: '0: {"description":"Low severity"} probability' })).toHaveAttribute(
"aria-valuenow",
"100",
);
expect(screen.queryByText(/\[object Object\]/)).not.toBeInTheDocument();
});
it("shows an inline error message", () => {
render(<ResponseView error="Upstream request failed" isLoading={false} />);
expect(screen.getByRole("alert")).toHaveTextContent("Upstream request failed");
});
});

87
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.test.tsx
View file

@ -1,87 +0,0 @@
import { render, screen } from "@testing-library/react";
import { describe, expect, it } from "vitest";
import ResponseView from "./ResponseView";
import type { SystemOneResponse } from "./lib/schemas";
const sampleResponse: SystemOneResponse = {
model: "jev-1.13.0",
answers: {
human: { type: "noul", noul: 0.99 },
dept: {
type: "choice",
choice: "technical",
confidence: 0.27,
probabilities: { billing: 0.36, technical: 0.64 },
},
frustration: {
type: "score",
score: 2,
confidence: 0.99,
legend: { "0": "calm", "1": "mild", "2": "angry" },
probabilities: { "0": 0, "1": 0, "2": 1 },
},
},
usage: { input_tokens: 378, output_tokens: 63 },
};
describe("ResponseView", () => {
it("renders calibrated probabilities for choice, noul, and score answers", () => {
render(<ResponseView response={sampleResponse} isLoading={false} latencyMs={120} />);
expect(screen.getAllByText("technical")).toHaveLength(2);
const meters = screen.getAllByRole("meter");
expect(meters.map((meter) => meter.getAttribute("aria-label"))).toEqual([
"Yes probability",
"technical probability",
"billing probability",
"0: calm probability",
"1: mild probability",
"2: angry probability",
]);
expect(screen.getByRole("meter", { name: "technical probability" })).toHaveAttribute("aria-valuenow", "64");
expect(screen.getByRole("meter", { name: "billing probability" })).toHaveAttribute("aria-valuenow", "36");
expect(screen.getByText("99% yes")).toBeInTheDocument();
expect(screen.getByRole("meter", { name: "Yes probability" })).toHaveAttribute("aria-valuenow", "99");
expect(screen.getByRole("meter", { name: "2: angry probability" })).toBeInTheDocument();
expect(screen.getByText("378 input / 63 output tokens")).toBeInTheDocument();
});
it("selects the score level nearest a fractional calibrated score", () => {
const fractional: SystemOneResponse = {
answers: {
severity: { type: "score", score: 2.39, probabilities: { "1": 0, "2": 0.62, "3": 0.38 } },
},
};
render(<ResponseView response={fractional} isLoading={false} />);
expect(screen.getByRole("meter", { name: "2 probability" })).toHaveAttribute("aria-valuetext", "62%, selected");
expect(screen.getByRole("meter", { name: "3 probability" })).toHaveAttribute("aria-valuetext", "38%");
expect(screen.getByRole("meter", { name: "1 probability" })).toHaveAttribute("aria-valuetext", "0%");
});
it("renders structured decision score legends without coercing objects to strings", () => {
const response: SystemOneResponse = {
answers: {
severity: {
type: "score",
score: 0,
probabilities: { "0": 1 },
legend: { "0": { description: "Low severity" } },
},
},
usage: null,
};
render(<ResponseView response={response} isLoading={false} />);
expect(screen.getByRole("meter", { name: '0: {"description":"Low severity"} probability' })).toHaveAttribute(
"aria-valuenow",
"100",
);
expect(screen.queryByText(/\[object Object\]/)).not.toBeInTheDocument();
});
it("shows an inline error message", () => {
render(<ResponseView error="Upstream request failed" isLoading={false} />);
expect(screen.getByRole("alert")).toHaveTextContent("Upstream request failed");
});
});

140
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx
View file

@ -6,9 +6,13 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component
import { ChevronDown, LoaderCircle } from "lucide-react";
import { useState } from "react";
import type { SystemOneAnswer, SystemOneResponse } from "./lib/schemas";
import type { OpenAIDecisionsResponse } from "./lib/openAIDecisions";
const SCORE_FORMAT = new Intl.NumberFormat("en-US", { maximumFractionDigits: 2 });
interface ResponseViewProps {
response?: SystemOneResponse;
response?: SystemOneResponse | OpenAIDecisionsResponse;
view?: "form" | "json";
fallbackModel?: string;
latencyMs?: number;
error?: string;
@ -85,7 +89,7 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
<div className="grid gap-4">
<div className="flex flex-wrap items-center gap-2">
<span className="text-xs text-muted-foreground">Score</span>
<Badge>{answer.score}</Badge>
<Badge>{SCORE_FORMAT.format(answer.score)}</Badge>
{answer.confidence !== undefined && (
<Badge variant="outline">{Math.round(answer.confidence * 100)}% confidence</Badge>
)}
@ -111,7 +115,87 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
);
}
export default function ResponseView({ response, fallbackModel, latencyMs, error, isLoading }: ResponseViewProps) {
function OpenAIAnswerDetails({ answer }: { answer: OpenAIDecisionsResponse["answers"][number] }) {
switch (answer.type) {
case "refusal":
return <p className="text-sm text-muted-foreground">Model refused to answer</p>;
case "predicate":
return <AnswerDetails answer={{ type: "noul", noul: answer.probability }} />;
case "choice":
return (
<AnswerDetails
answer={{
type: "choice",
choice: String(answer.choice),
confidence: answer.confidence,
probabilities: Object.fromEntries(
answer.probabilities.map(({ value, probability }) => [String(value), probability]),
),
}}
/>
);
case "score":
return (
<AnswerDetails
answer={{
type: "score",
score: answer.score,
confidence: answer.confidence,
probabilities: Object.fromEntries(
answer.probabilities.map(({ value, probability }) => [String(value), probability]),
),
legend: Object.fromEntries(answer.probabilities.map(({ value, label }) => [String(value), label])),
}}
/>
);
}
}
function RawResponse({ response }: { response: NonNullable<ResponseViewProps["response"]> }) {
return (
<section aria-label="Decisions response JSON">
<pre className="whitespace-pre-wrap wrap-anywhere rounded-md bg-muted p-3 font-mono text-xs">
{JSON.stringify(response, null, 2)}
</pre>
</section>
);
}
function AnswerCards({ answers }: { answers: SystemOneResponse["answers"] | OpenAIDecisionsResponse["answers"] }) {
const cards = Array.isArray(answers)
? answers.map((answer, index) => ({
id: index,
name: answer.name ?? `Question ${index + 1}`,
type: answer.type,
details: <OpenAIAnswerDetails answer={answer} />,
}))
: Object.entries(answers).map(([id, answer]) => ({
id,
name: id,
type: answer.type,
details: <AnswerDetails answer={answer} />,
}));
return cards.map(({ id, name, type, details }) => (
<Card key={id} size="sm">
<CardHeader>
<CardTitle className="font-mono">{name}</CardTitle>
<CardAction>
<Badge variant="secondary">{type}</Badge>
</CardAction>
</CardHeader>
<CardContent>{details}</CardContent>
</Card>
));
}
export default function ResponseView({
response,
view = "form",
fallbackModel,
latencyMs,
error,
isLoading,
}: ResponseViewProps) {
const [showRaw, setShowRaw] = useState(false);
if (isLoading) {
@ -128,7 +212,7 @@ export default function ResponseView({ response, fallbackModel, latencyMs, error
if (error) {
return (
<Alert variant="destructive">
<AlertTitle>System One request failed</AlertTitle>
<AlertTitle>Decisions request failed</AlertTitle>
<AlertDescription>{error}</AlertDescription>
</Alert>
);
@ -139,7 +223,7 @@ export default function ResponseView({ response, fallbackModel, latencyMs, error
<Card>
<CardHeader>
<CardTitle>Calibrated probabilities</CardTitle>
<CardDescription>Send a request to see System One answers and probabilities.</CardDescription>
<CardDescription>Send a request to see decision answers and probabilities.</CardDescription>
</CardHeader>
</Card>
);
@ -152,7 +236,7 @@ export default function ResponseView({ response, fallbackModel, latencyMs, error
<CardHeader>
<div className="flex flex-wrap items-start justify-between gap-3">
<div className="grid gap-1">
<CardTitle>Calibrated probabilities</CardTitle>
<CardTitle>{view === "json" ? "Response JSON" : "Calibrated probabilities"}</CardTitle>
{model && <CardDescription>{model}</CardDescription>}
</div>
<div className="flex flex-wrap items-center gap-2 text-xs">
@ -166,34 +250,22 @@ export default function ResponseView({ response, fallbackModel, latencyMs, error
</div>
</CardHeader>
<CardContent className="grid gap-3">
{Object.entries(response.answers).map(([id, answer]) => (
<Card key={id} size="sm">
<CardHeader>
<CardTitle className="font-mono">{id}</CardTitle>
<CardAction>
<Badge variant="secondary">{answer.type}</Badge>
</CardAction>
</CardHeader>
<CardContent>
<AnswerDetails answer={answer} />
</CardContent>
</Card>
))}
<Collapsible open={showRaw} onOpenChange={setShowRaw}>
<CollapsibleTrigger
render={
<Button variant="ghost" size="sm">
<ChevronDown className="size-4" />
Raw response
</Button>
}
/>
<CollapsibleContent>
<pre className="whitespace-pre-wrap wrap-anywhere rounded-md bg-muted p-3 font-mono text-xs">
{JSON.stringify(response, null, 2)}
</pre>
</CollapsibleContent>
</Collapsible>
{view === "json" ? <RawResponse response={response} /> : <AnswerCards answers={response.answers} />}
{view === "form" && (
<Collapsible open={showRaw} onOpenChange={setShowRaw}>
<CollapsibleTrigger
render={
<Button variant="ghost" size="sm">
<ChevronDown className="size-4" />
Raw response
</Button>
}
/>
<CollapsibleContent>
<RawResponse response={response} />
</CollapsibleContent>
</Collapsible>
)}
</CardContent>
</Card>
);

52
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneForm.tsx
View file

@ -5,10 +5,8 @@ import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Textarea } from "@/components/ui/textarea";
import { cn } from "@/lib/cva.config";
import { Plus, Trash2 } from "lucide-react";
import { useId, useMemo, useState, type ReactNode } from "react";
import { IssueList, ValidationStatus } from "./JsonEditor";
import { useId, useMemo, useState } from "react";
import { AddButton, FormFrame, RemoveButton } from "./DecisionFormControls";
import {
blankQuestion,
freeName,
@ -29,7 +27,12 @@ import {
import type { SystemOnePayloadValidation } from "./lib/validatePayload";
const QUESTION_TYPES: readonly QuestionType[] = ["choice", "noul", "score"];
const TYPE_LABELS: Record<QuestionType, string> = { choice: "Choice", noul: "Yes / no", score: "Score" };
const TYPE_LABELS: Record<QuestionType, string> = { choice: "Choice", noul: "noul", score: "Score" };
const TYPE_HELP: Record<QuestionType, string | undefined> = {
choice: undefined,
noul: "noul is a yes / no question",
score: undefined,
};
const isQuestionType = (value: unknown): value is QuestionType => QUESTION_TYPES.some((type) => type === value);
@ -97,23 +100,6 @@ function KeyInput({ value, taken, onCommit, label, placeholder }: KeyInputProps)
);
}
function RemoveButton({ label, onClick }: { label: string; onClick: () => void }) {
return (
<Button variant="ghost" size="icon" aria-label={label} title={label} onClick={onClick}>
<Trash2 />
</Button>
);
}
function AddButton({ children, onClick }: { children: ReactNode; onClick: () => void }) {
return (
<Button variant="outline" size="sm" className="w-fit" onClick={onClick}>
<Plus />
{children}
</Button>
);
}
function ChoiceOptions({ question, onChange }: QuestionEditorProps<ChoiceQuestion>) {
const labels = Object.keys(question.criteria);
const writeCriteria = (criteria: Record<string, string>) => onChange({ ...question, criteria });
@ -238,7 +224,7 @@ function QuestionCard({ name, question, taken, onChange, onRename, onRemove }: Q
</SelectTrigger>
<SelectContent>
{QUESTION_TYPES.map((type) => (
<SelectItem key={type} value={type}>
<SelectItem key={type} value={type} title={TYPE_HELP[type]}>
{TYPE_LABELS[type]}
</SelectItem>
))}
@ -261,26 +247,6 @@ function QuestionCard({ name, question, taken, onChange, onRename, onRemove }: Q
);
}
function FormFrame({ validation, children }: { validation: SystemOnePayloadValidation; children: ReactNode }) {
const issuesId = useId();
return (
<div
className={cn(
"flex min-h-96 flex-1 flex-col overflow-hidden rounded-md border bg-background",
!validation.isValid && "border-destructive/60",
)}
>
<div className="flex items-center gap-2 border-b px-3 py-2">
<span className="text-sm font-medium">Request form</span>
<ValidationStatus validation={validation} />
</div>
<div className="grid min-h-80 flex-1 content-start gap-4 overflow-auto p-3">{children}</div>
<IssueList id={issuesId} validation={validation} />
</div>
);
}
export default function SystemOneForm({ value, onChange, validation, onOpenJson }: SystemOneFormProps) {
const stateId = useId();
const form = useMemo(() => readForm(value), [value]);

533
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx
View file

@ -11,8 +11,9 @@ import {
import type { ReactElement } from "react";
import userEvent from "@testing-library/user-event";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import SystemOneUI from "./SystemOneUI";
import SystemOneUI, { NOUL_HELP } from "./SystemOneUI";
import type { SystemOneResponse } from "./lib/schemas";
import type { OpenAIDecisionsResponse } from "./lib/openAIDecisions";
interface ModelGroupInfo {
model_group: string;
@ -64,7 +65,19 @@ const openJsonEditor = async () => {
return screen.getByRole("textbox", { name: "System One JSON payload" });
};
const createResponse = (body: SystemOneResponse, status = 200, errorText = "") =>
const selectEndpoint = async (endpoint: "/v1/decisions" | "/v1/systemone") => {
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
await userEvent.keyboard("{ArrowDown}");
await userEvent.click(await screen.findByRole("option", { name: endpoint, exact: true }));
};
const selectPreset = async (label: string) => {
screen.getByRole("combobox", { name: "Example" }).focus();
await userEvent.keyboard("{ArrowDown}");
await userEvent.click(await screen.findByRole("option", { name: label }));
};
const createResponse = (body: SystemOneResponse | OpenAIDecisionsResponse, status = 200, errorText = "") =>
({
ok: status >= 200 && status < 300,
status,
@ -92,28 +105,94 @@ describe("SystemOneUI integration", () => {
document.cookie = "token=; path=/; max-age=0";
});
it("restores the example after the request is edited", async () => {
const user = userEvent.setup();
it("switches between example presets and reads Custom once the request is edited", async () => {
render(<SystemOneUI accessToken="session-key" />);
const editor = await openJsonEditor();
const example = (editor as HTMLTextAreaElement).value;
const resetButton = screen.getByRole("button", { name: "Reset example" });
expect(resetButton).toBeDisabled();
const picker = screen.getByRole("combobox", { name: "Example" });
expect(picker).toHaveTextContent("Triage a bug report");
fireEvent.change(editor, { target: { value: "{}" } });
expect(resetButton).toBeEnabled();
await user.click(resetButton);
expect(picker).toHaveTextContent("Custom");
await selectPreset("Triage a bug report");
expect(editor).toHaveValue(example);
expect(resetButton).toBeDisabled();
expect(picker).toHaveTextContent("Triage a bug report");
await selectPreset("Score a sales lead");
expect(picker).toHaveTextContent("Score a sales lead");
expect(JSON.parse((editor as HTMLTextAreaElement).value)).toMatchObject({ questions: { fit: { type: "score" } } });
});
it("explains the native decision endpoint after opting in without announcing it as an alert", async () => {
it("clears the whole request and lets a preset be picked from nothing", async () => {
const user = userEvent.setup();
render(<SystemOneUI accessToken="session-key" />);
const clear = screen.getByRole("button", { name: "Clear" });
expect(screen.getByRole("textbox", { name: "Input" })).toHaveDisplayValue(/streaming responses/);
await user.click(clear);
expect(screen.getByRole("textbox", { name: "Input" })).toHaveValue("");
expect(screen.queryAllByRole("group", { name: /Question / })).toHaveLength(0);
expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
expect(clear).toBeDisabled();
expect(screen.getByRole("combobox", { name: "Example" })).toHaveTextContent("Custom");
await selectPreset("Route a support ticket");
expect(screen.getByRole("textbox", { name: "Input" })).toHaveDisplayValue(/charged twice/);
expect(screen.getAllByRole("group", { name: /Question / })).toHaveLength(1);
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
});
it("keeps the preset label after picking a model and leaves an omitted model out when clearing", async () => {
mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
render(<SystemOneUI accessToken="session-key" />);
await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest"));
const picker = screen.getByRole("combobox", { name: "Example" });
const editor = await openJsonEditor();
const example = JSON.parse((editor as HTMLTextAreaElement).value) as Record<string, unknown>;
const { model: _model, ...withoutModel } = example;
fireEvent.change(editor, {
target: { value: JSON.stringify({ ...withoutModel, model: "pplx-decider" }, null, 2) },
});
expect(picker).toHaveTextContent("Triage a bug report");
fireEvent.change(editor, { target: { value: JSON.stringify(withoutModel, null, 2) } });
expect(picker).toHaveTextContent("Triage a bug report");
await userEvent.click(screen.getByRole("button", { name: "Clear" }));
expect(JSON.parse((editor as HTMLTextAreaElement).value)).toEqual({ state: "", questions: {} });
await selectPreset("Score a sales lead");
expect(JSON.parse((editor as HTMLTextAreaElement).value)).not.toHaveProperty("model");
});
it("offers the same presets on /v1/decisions in the OpenAI shape and clears to an empty input", async () => {
const user = userEvent.setup();
render(<SystemOneUI accessToken="session-key" />);
await selectEndpoint("/v1/decisions");
await selectPreset("Moderate a forum comment");
await user.click(screen.getByRole("tab", { name: "JSON" }));
const editor = screen.getByRole("textbox", { name: "Decisions JSON payload" }) as HTMLTextAreaElement;
const payload = JSON.parse(editor.value);
expect(payload.input).toMatch(/stream=false/);
expect(payload.questions).toEqual([
{ type: "predicate", name: "breaks_rules", instructions: "Does this comment break the community guidelines?" },
]);
await user.click(screen.getByRole("button", { name: "Clear" }));
expect(JSON.parse(editor.value)).toEqual({ input: "", questions: [] });
expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
});
it("offers both literal proxy endpoints without TypeSafe or product prefixes", async () => {
render(<SystemOneUI accessToken="session-key" />);
expect(screen.getByRole("combobox", { name: "Decision endpoint" })).toHaveTextContent("/v1/systemone");
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
await user.keyboard("{ArrowDown}");
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
await userEvent.keyboard("{ArrowDown}");
expect(await screen.findByRole("option", { name: "/v1/decisions", exact: true })).toBeInTheDocument();
expect(screen.getByRole("option", { name: "/v1/systemone", exact: true })).toBeInTheDocument();
expect(screen.getAllByRole("option")).toHaveLength(2);
expect(screen.queryByText(/TypeSafe/)).not.toBeInTheDocument();
await userEvent.keyboard("{Escape}");
expect(screen.getByRole("note", { name: "Decision endpoint notice" })).toHaveTextContent(
"omit model to use the proxy's configured default.",
@ -134,7 +213,7 @@ describe("SystemOneUI integration", () => {
mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
render(<SystemOneUI accessToken="session-key" />);
expect(await screen.findByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest");
await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest"));
expect(lookupAuthHeaders(mockModelLookup.mock.calls)).toEqual([expect.arrayContaining(["Bearer session-key"])]);
await user.click(screen.getByRole("button", { name: "Send" }));
@ -147,6 +226,297 @@ describe("SystemOneUI integration", () => {
});
});
it("sends each endpoint's real shape and switches existing responses between cards and JSON without resending", async () => {
const user = userEvent.setup();
mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
render(<SystemOneUI accessToken="session-key" />);
await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest"));
const systemOneEditor = await openJsonEditor();
const systemOneDraft = JSON.stringify({
model: "jev-latest",
state: "An outage",
questions: { urgent: { type: "noul", instructions: "Is it urgent?" } },
});
fireEvent.change(systemOneEditor, { target: { value: systemOneDraft } });
await selectEndpoint("/v1/decisions");
const editor = screen.getByRole("textbox", { name: "Decisions JSON payload" });
const nativeExample = (editor as HTMLTextAreaElement).value;
expect(JSON.parse(nativeExample)).toMatchObject({
input: expect.any(String),
questions: expect.arrayContaining([expect.objectContaining({ type: "predicate" })]),
});
expect(screen.getByRole("tab", { name: "Form" })).toBeInTheDocument();
const payload = {
model: "jev-latest",
input: "An outage",
questions: [{ name: "urgent", type: "predicate", instructions: "Is it urgent?" }],
safety_identifier: "qa-session",
};
fireEvent.change(editor, { target: { value: JSON.stringify(payload) } });
const response: OpenAIDecisionsResponse = {
model: "jev-latest",
answers: [{ name: "urgent", type: "predicate", probability: 0.95 }],
usage: { input_tokens: 10, output_tokens: 1, total_tokens: 11 },
};
mockFetch.mockResolvedValueOnce(createResponse(response));
await user.click(screen.getByRole("button", { name: "Send" }));
const result = await screen.findByRole("region", { name: "Decisions response JSON" });
expect(result).toHaveTextContent('"probability": 0.95');
expect(result).toHaveTextContent('"answers": [');
expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/decisions$/);
expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string)).toEqual(payload);
expect(screen.queryByRole("meter")).not.toBeInTheDocument();
await user.click(screen.getByRole("tab", { name: "Form" }));
expect(screen.getByRole("meter", { name: "Yes probability" })).toHaveAttribute("aria-valuenow", "95");
expect(screen.queryByRole("region", { name: "Decisions response JSON" })).not.toBeInTheDocument();
await user.click(screen.getByRole("tab", { name: "JSON" }));
expect(screen.getByRole("region", { name: "Decisions response JSON" })).toHaveTextContent('"probability": 0.95');
expect(mockFetch).toHaveBeenCalledTimes(1);
await selectEndpoint("/v1/systemone");
expect(screen.getByRole("textbox", { name: "System One JSON payload" })).toHaveValue(systemOneDraft);
expect(screen.queryByRole("region", { name: "Decisions response JSON" })).not.toBeInTheDocument();
await user.click(screen.getByRole("button", { name: "Send" }));
expect(await screen.findByRole("region", { name: "Decisions response JSON" })).toHaveTextContent(
'"choice": "technical"',
);
expect(screen.queryByRole("meter")).not.toBeInTheDocument();
await user.click(screen.getByRole("tab", { name: "Form" }));
expect(screen.getByText("Selected choice")).toBeInTheDocument();
expect(screen.getByRole("meter", { name: "technical probability" })).toHaveAttribute("aria-valuenow", "90");
await user.click(screen.getByRole("tab", { name: "JSON" }));
expect(mockFetch).toHaveBeenCalledTimes(2);
expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/systemone$/);
expect(JSON.parse(mockFetch.mock.calls[1]?.[1]?.body as string)).toEqual(JSON.parse(systemOneDraft));
await selectEndpoint("/v1/decisions");
expect(screen.getByRole("textbox", { name: "Decisions JSON payload" })).toHaveValue(JSON.stringify(payload));
await selectPreset("Triage a bug report");
expect(screen.getByRole("textbox", { name: "Decisions JSON payload" })).toHaveValue(nativeExample);
await selectEndpoint("/v1/systemone");
expect(screen.getByRole("textbox", { name: "System One JSON payload" })).toHaveValue(systemOneDraft);
});
it("validates /v1/decisions independently and preserves an invalid draft across switches", async () => {
render(<SystemOneUI accessToken="session-key" />);
await selectEndpoint("/v1/decisions");
await userEvent.click(screen.getByRole("tab", { name: "JSON" }));
const editor = screen.getByRole("textbox", { name: "Decisions JSON payload" });
fireEvent.change(editor, { target: { value: "{" } });
expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
await selectEndpoint("/v1/systemone");
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
await selectEndpoint("/v1/decisions");
expect(screen.getByRole("textbox", { name: "Decisions JSON payload" })).toHaveValue("{");
expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
expect(mockFetch).not.toHaveBeenCalled();
});
it("offers a Decisions form that sends input and typed questions and stays in sync with JSON", async () => {
const user = userEvent.setup();
mockFetch.mockResolvedValueOnce(
createResponse({
model: "jev-latest",
answers: [],
usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
}),
);
render(<SystemOneUI accessToken="session-key" />);
await selectEndpoint("/v1/decisions");
expect(screen.getByRole("tab", { name: "Form" })).toHaveAttribute("aria-selected", "true");
expect(screen.queryByRole("button", { name: "Format JSON" })).not.toBeInTheDocument();
fireEvent.change(screen.getByRole("textbox", { name: "Input" }), { target: { value: "Login is broken" } });
const firstQuestion = within(screen.getByRole("group", { name: "Question 1" }));
fireEvent.change(firstQuestion.getByRole("textbox", { name: "Question name" }), { target: { value: "team" } });
fireEvent.change(firstQuestion.getByRole("textbox", { name: "Instructions" }), {
target: { value: "Which team owns this?" },
});
fireEvent.change(firstQuestion.getByRole("textbox", { name: "Choice 1 value" }), {
target: { value: "infra" },
});
fireEvent.change(screen.getByRole("textbox", { name: "Safety identifier" }), {
target: { value: "qa-session" },
});
await user.click(screen.getByRole("tab", { name: "JSON" }));
const editor = screen.getByRole("textbox", { name: "Decisions JSON payload" });
const payload = JSON.parse((editor as HTMLTextAreaElement).value);
expect(payload).toMatchObject({ input: "Login is broken", safety_identifier: "qa-session" });
const expectedQuestion = {
name: "team",
type: "choice",
instructions: "Which team owns this?",
choices: expect.arrayContaining([expect.objectContaining({ value: "infra" })]),
};
expect(payload.questions[0]).toMatchObject(expectedQuestion);
expect(payload).not.toHaveProperty("state");
fireEvent.change(editor, { target: { value: JSON.stringify({ ...payload, input: "Fixed in JSON" }) } });
await user.click(screen.getByRole("tab", { name: "Form" }));
expect(screen.getByRole("textbox", { name: "Input" })).toHaveValue("Fixed in JSON");
await user.click(screen.getByRole("button", { name: "Send" }));
await waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(1));
expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/decisions$/);
expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string)).toEqual({ ...payload, input: "Fixed in JSON" });
expect(await screen.findByRole("button", { name: "Raw response" })).toBeInTheDocument();
await user.click(screen.getByRole("tab", { name: "JSON" }));
expect(screen.getByRole("region", { name: "Decisions response JSON" })).toBeInTheDocument();
});
it("adds and removes Decisions questions, boolean choices and score levels without stale fields", async () => {
const user = userEvent.setup();
render(<SystemOneUI accessToken="session-key" />);
await selectEndpoint("/v1/decisions");
const question = (index: number) => within(screen.getByRole("group", { name: `Question ${index}` }));
const pick = async (control: HTMLElement, label: string) => {
control.focus();
await user.keyboard("{ArrowDown}");
await user.click(await screen.findByRole("option", { name: label, exact: true }));
};
await user.click(screen.getByRole("button", { name: "Add question" }));
fireEvent.change(question(4).getByRole("textbox", { name: "Instructions" }), {
target: { value: "Should we escalate?" },
});
await pick(question(4).getByRole("combobox", { name: "Answer type" }), "Choice");
await user.click(question(4).getByRole("button", { name: "Remove choice 1" }));
expect(question(4).getByRole("textbox", { name: "Choice 1 value" })).toHaveValue("option_2");
expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
await user.click(question(4).getByRole("button", { name: "Add choice" }));
await pick(question(4).getByRole("combobox", { name: "Choice 1 value type" }), "false (boolean)");
fireEvent.change(question(4).getByRole("textbox", { name: "Choice 1 description" }), {
target: { value: "Not urgent" },
});
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
await user.click(screen.getByRole("tab", { name: "JSON" }));
const choicePayload = JSON.parse(
(screen.getByRole("textbox", { name: "Decisions JSON payload" }) as HTMLTextAreaElement).value,
);
const expectedChoice = {
type: "choice",
name: "question_1",
instructions: "Should we escalate?",
choices: [{ value: false, description: "Not urgent" }, { value: "option_1" }],
};
expect(choicePayload.questions[3]).toEqual(expectedChoice);
await user.click(screen.getByRole("tab", { name: "Form" }));
await pick(question(4).getByRole("combobox", { name: "Answer type" }), "Score");
fireEvent.change(question(4).getByRole("textbox", { name: "Level 1 label" }), { target: { value: "low" } });
fireEvent.change(question(4).getByRole("textbox", { name: "Level 1 description" }), {
target: { value: "No impact" },
});
await user.click(question(4).getByRole("button", { name: "Add level" }));
await user.click(question(4).getByRole("button", { name: "Remove level 2" }));
expect(question(4).getByRole("textbox", { name: "Level 2 label" })).toHaveValue("2");
await user.click(question(2).getByRole("button", { name: "Remove question 2" }));
expect(question(3).getByRole("textbox", { name: "Instructions" })).toHaveValue("Should we escalate?");
await user.click(screen.getByRole("tab", { name: "JSON" }));
const scorePayload = JSON.parse(
(screen.getByRole("textbox", { name: "Decisions JSON payload" }) as HTMLTextAreaElement).value,
);
expect(scorePayload.questions).toHaveLength(3);
const expectedScore = {
type: "score",
name: "question_1",
instructions: "Should we escalate?",
levels: [{ label: "low", description: "No impact" }, { label: "2" }],
};
expect(scorePayload.questions[2]).toEqual(expectedScore);
await user.click(screen.getByRole("tab", { name: "Form" }));
await pick(question(3).getByRole("combobox", { name: "Answer type" }), "Predicate");
await user.click(screen.getByRole("tab", { name: "JSON" }));
const predicatePayload = JSON.parse(
(screen.getByRole("textbox", { name: "Decisions JSON payload" }) as HTMLTextAreaElement).value,
);
expect(predicatePayload.questions[2]).toEqual({
type: "predicate",
name: "question_1",
instructions: "Should we escalate?",
});
});
it("preserves message input and extra fields in the Decisions form, including across routes and reset", async () => {
const user = userEvent.setup();
render(<SystemOneUI accessToken="session-key" />);
await selectEndpoint("/v1/decisions");
await user.click(screen.getByRole("tab", { name: "JSON" }));
const payload = {
model: "jev-latest",
metadata: { source: "playground" },
input: [
{
role: "user",
content: [
{ type: "input_text", text: "Is this urgent?" },
{ type: "input_image", image_url: "https://example.com/image.png" },
],
},
],
questions: [{ type: "predicate", name: null, instructions: "Old instructions" }],
};
fireEvent.change(screen.getByRole("textbox", { name: "Decisions JSON payload" }), {
target: { value: JSON.stringify(payload) },
});
await user.click(screen.getByRole("tab", { name: "Form" }));
expect(screen.getByRole("button", { name: "Edit input in JSON" })).toBeInTheDocument();
fireEvent.change(screen.getByRole("textbox", { name: "Instructions" }), { target: { value: "Is it urgent?" } });
await selectEndpoint("/v1/systemone");
const nativeInput = (screen.getByRole("textbox", { name: "Input" }) as HTMLTextAreaElement).value;
await selectEndpoint("/v1/decisions");
expect(screen.getByRole("textbox", { name: "Instructions" })).toHaveValue("Is it urgent?");
await user.click(screen.getByRole("button", { name: "Edit input in JSON" }));
const next = JSON.parse(
(screen.getByRole("textbox", { name: "Decisions JSON payload" }) as HTMLTextAreaElement).value,
);
expect(next).toEqual({ ...payload, questions: [{ ...payload.questions[0], instructions: "Is it urgent?" }] });
await user.click(screen.getByRole("tab", { name: "Form" }));
await selectPreset("Triage a bug report");
expect(screen.getByRole("textbox", { name: "Input" })).toBeInTheDocument();
expect(screen.getAllByRole("group", { name: /Question \d/ })).toHaveLength(3);
await selectEndpoint("/v1/systemone");
expect(screen.getByRole("textbox", { name: "Input" })).toHaveValue(nativeInput);
});
it("keeps malformed Decisions JSON intact when switching to its form", async () => {
const user = userEvent.setup();
render(<SystemOneUI accessToken="session-key" />);
await selectEndpoint("/v1/decisions");
await user.click(screen.getByRole("tab", { name: "JSON" }));
fireEvent.change(screen.getByRole("textbox", { name: "Decisions JSON payload" }), { target: { value: "{" } });
await user.click(screen.getByRole("tab", { name: "Form" }));
expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
await user.click(screen.getByRole("button", { name: "Edit in JSON" }));
expect(screen.getByRole("textbox", { name: "Decisions JSON payload" })).toHaveValue("{");
});
it.each(["success", "error"])("cancels the previous endpoint request and ignores a late %s", async (outcome) => {
const pending = Promise.withResolvers<Response>();
mockFetch.mockReturnValueOnce(pending.promise);
const queryClient = new QueryClient({ defaultOptions: { mutations: { retry: false } } });
rtlRender(
<QueryClientProvider client={queryClient}>
<SystemOneUI accessToken="session-key" />
</QueryClientProvider>,
);
await userEvent.click(screen.getByRole("button", { name: "Send" }));
await screen.findByRole("button", { name: "Cancel request" });
await selectEndpoint("/v1/decisions");
expect(mockFetch.mock.calls[0]?.[1]?.signal?.aborted).toBe(true);
await act(async () => {
if (outcome === "success") {
pending.resolve(createResponse({ model: "stale-model", answers: {} }));
} else {
pending.reject(new Error("Stale request failed"));
}
await pending.promise.catch(() => undefined);
});
await waitFor(() => expect(queryClient.isMutating()).toBe(0));
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
expect(screen.queryByText("stale-model")).not.toBeInTheDocument();
expect(screen.getByRole("tab", { name: "Form" })).toHaveAttribute("aria-selected", "true");
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
});
it("offers only decision models and writes the picked one into the request", async () => {
const user = userEvent.setup();
mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
@ -165,16 +535,31 @@ describe("SystemOneUI integration", () => {
questions: { has_repro_steps: { type: "noul" } },
});
await user.click(screen.getByRole("button", { name: "Send" }));
await user.click(screen.getByRole("tab", { name: "Form" }));
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string).model).toBe("pplx-decider");
});
it("links to Add Model when the key cannot call any decision model", async () => {
it("routes a configured TypeSafe model through the same proxy endpoint", async () => {
const user = userEvent.setup();
mockModelLookup.mockResolvedValue(
modelGroupInfoResponse([{ model_group: "typesafe/jev-latest", mode: "evaluation" }]),
);
render(<SystemOneUI accessToken="session-key" />);
await waitFor(() =>
expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("typesafe/jev-latest"),
);
await user.click(screen.getByRole("button", { name: "Send" }));
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/systemone$/);
expect(mockFetch.mock.calls[0]?.[0]).not.toMatch(/\/typesafe\//);
expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string).model).toBe("typesafe/jev-latest");
});
it("links to Add Model when the key cannot call any decision model", async () => {
render(<SystemOneUI accessToken="session-key" />);
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
await user.keyboard("{ArrowDown}");
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
expect(await screen.findByRole("link", { name: "Add a decision model" })).toHaveAttribute(
"href",
@ -193,7 +578,7 @@ describe("SystemOneUI integration", () => {
await user.type(screen.getByLabelText("Virtual Key", { exact: true }), "sk-typed-key");
expect(await screen.findByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest");
await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest"));
expect(lookupAuthHeaders(mockModelLookup.mock.calls)).toEqual([expect.arrayContaining(["Bearer sk-typed-key"])]);
});
@ -209,10 +594,10 @@ describe("SystemOneUI integration", () => {
await new Promise((resolve) => setTimeout(resolve, 0));
});
expect(document.cookie).toContain("token=ui-session");
expect(screen.getByRole("combobox", { name: "Decision endpoint" })).toHaveTextContent("/typesafe/v1/systemone");
expect(screen.getByRole("combobox", { name: "Decision endpoint" })).toHaveTextContent("/v1/systemone");
});
it("keeps an edited TypeSafe request in place when decision models load afterwards", async () => {
it("keeps an edited request in place when decision models load afterwards", async () => {
const models = Promise.withResolvers<Response>();
mockModelLookup.mockReturnValue(models.promise);
render(<SystemOneUI accessToken="session-key" />);
@ -226,9 +611,37 @@ describe("SystemOneUI integration", () => {
});
expect(editor).toHaveValue(draft);
expect(screen.queryByRole("combobox", { name: "Decision model" })).not.toBeInTheDocument();
expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("");
});
it.each(["/v1/decisions", "/v1/systemone"] as const)(
"omits model while discovery is pending on %s",
async (endpoint) => {
const models = Promise.withResolvers<Response>();
mockModelLookup.mockReturnValue(models.promise);
mockFetch.mockResolvedValueOnce(
createResponse({
model: "jev-latest",
answers: endpoint === "/v1/decisions" ? [] : {},
usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
}),
);
render(<SystemOneUI accessToken="session-key" />);
if (endpoint === "/v1/decisions") {
await selectEndpoint(endpoint);
}
await userEvent.click(screen.getByRole("button", { name: "Send" }));
expect(await screen.findByText("jev-latest")).toBeInTheDocument();
expect(mockFetch.mock.calls[0]?.[0]).toMatch(new RegExp(`${endpoint}$`));
expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string)).not.toHaveProperty("model");
await act(async () => {
models.resolve(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
await models.promise;
});
expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("");
},
);
it("shows invalid JSON and disables Send", async () => {
render(<SystemOneUI accessToken="session-key" />);
@ -240,7 +653,7 @@ describe("SystemOneUI integration", () => {
expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
});
it("preserves the untouched TypeSafe default and renders calibrated answers", async () => {
it("sends to /v1/systemone even without discovered models and renders calibrated answers", async () => {
const user = userEvent.setup();
sessionStorage.setItem("customProxyBaseUrl", "https://stale.example.com/");
render(<SystemOneUI accessToken="session-key" />);
@ -251,18 +664,19 @@ describe("SystemOneUI integration", () => {
expect(await screen.findAllByText("technical")).toHaveLength(2);
expect(mockFetch).toHaveBeenCalledTimes(1);
const [url, options] = mockFetch.mock.calls[0] ?? [];
expect(url).toBe("https://tenant.example.com/typesafe/v1/systemone");
expect(url).toBe("https://tenant.example.com/v1/systemone");
expect(options?.method).toBe("POST");
expect(options?.headers).toMatchObject({ "Content-Type": "application/json" });
expect(Object.values(options?.headers as Record<string, string>)).toContain("Bearer session-key");
expect(JSON.parse(options?.body as string)).toMatchObject({
model: "jev-latest",
questions: { has_repro_steps: { type: "noul" } },
});
expect(JSON.parse(options?.body as string)).not.toHaveProperty("model");
});
it("uses the requested model when the response omits it", async () => {
const user = userEvent.setup();
mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
mockFetch.mockResolvedValueOnce(
createResponse({
answers: responseBody.answers,
@ -270,13 +684,15 @@ describe("SystemOneUI integration", () => {
);
render(<SystemOneUI accessToken="session-key" />);
await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest"));
await user.click(screen.getByRole("button", { name: "Send" }));
expect(await screen.findByText("jev-latest")).toBeInTheDocument();
expect(screen.getByText("Selected choice")).toBeInTheDocument();
});
it("keeps legacy validation when question values have invalid types", async () => {
it("validates native question values before sending", async () => {
render(<SystemOneUI accessToken="session-key" />);
fireEvent.change(await openJsonEditor(), {
@ -286,25 +702,22 @@ describe("SystemOneUI integration", () => {
questions: {
category: {
type: "choice",
instructions: { text: "Route the message" },
criteria: { support: { text: "Help" } },
instructions: 1,
criteria: { support: true },
},
},
}),
},
});
expect(screen.getByText("Instructions must be a string.")).toBeInTheDocument();
expect(screen.getByText("Choice descriptions must be strings.")).toBeInTheDocument();
expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
await userEvent.click(screen.getByRole("tab", { name: "Form" }));
expect(screen.getByText("Enter a valid request to preview its state and questions.")).toBeInTheDocument();
});
it("preserves each endpoint draft and sends legacy requests to TypeSafe", async () => {
it("sends structured native questions without changing their payload", async () => {
const user = userEvent.setup();
render(<SystemOneUI accessToken="session-key" />);
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
await user.keyboard("{ArrowDown}");
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
const editor = await openJsonEditor();
const draft = JSON.stringify({
model: "my-decider",
@ -315,33 +728,18 @@ describe("SystemOneUI integration", () => {
});
fireEvent.change(editor, { target: { value: draft } });
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
await user.click(screen.getByRole("tab", { name: "Form" }));
expect(screen.getByText(/"text": "Help"/)).toBeInTheDocument();
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
await user.keyboard("{ArrowDown}");
await user.click(await screen.findByRole("option", { name: "TypeSafe · /typesafe/v1/systemone" }));
await user.click(screen.getByRole("button", { name: "Send" }));
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/typesafe\/v1\/systemone$/);
expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string).model).toBe("jev-latest");
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
await user.keyboard("{ArrowDown}");
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
expect(editor).toHaveValue(draft);
expect(screen.queryByText("Selected choice")).not.toBeInTheDocument();
await user.click(screen.getByRole("button", { name: "Send" }));
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/systemone$/);
expect(JSON.parse(mockFetch.mock.calls[1]?.[1]?.body as string)).toEqual(JSON.parse(draft));
expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/systemone$/);
expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string)).toEqual(JSON.parse(draft));
});
it("sends a native request without model so the proxy can select its default", async () => {
const user = userEvent.setup();
render(<SystemOneUI accessToken="session-key" />);
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
await user.keyboard("{ArrowDown}");
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
const payload = { state: "An outage", questions: { urgent: { type: "noul", instructions: "Is this urgent?" } } };
fireEvent.change(await openJsonEditor(), {
target: { value: JSON.stringify(payload) },
@ -355,7 +753,7 @@ describe("SystemOneUI integration", () => {
expect(body).not.toHaveProperty("model");
});
it.each(["success", "error"])("ignores a late %s after switching endpoints", async (outcome) => {
it.each(["success", "error"])("ignores a late %s after editing the request", async (outcome) => {
const user = userEvent.setup();
const pending = Promise.withResolvers<Response>();
mockFetch.mockReturnValueOnce(pending.promise);
@ -367,9 +765,7 @@ describe("SystemOneUI integration", () => {
);
await user.click(screen.getByRole("button", { name: "Send" }));
await screen.findByRole("button", { name: "Cancel request" });
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
await user.keyboard("{ArrowDown}");
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
fireEvent.change(screen.getByRole("textbox", { name: "Input" }), { target: { value: "A new request" } });
expect(mockFetch.mock.calls[0]?.[1]?.signal?.aborted).toBe(true);
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
@ -437,7 +833,7 @@ describe("SystemOneUI integration", () => {
await user.click(screen.getByRole("button", { name: "Send" }));
expect(await screen.findByRole("alert")).toHaveTextContent("System One response has an invalid shape.");
expect(await screen.findByRole("alert")).toHaveTextContent("/v1/systemone response has an invalid shape.");
expect(screen.queryByText("jev-1.13.0")).not.toBeInTheDocument();
});
@ -524,6 +920,22 @@ describe("SystemOneUI integration", () => {
fireEvent.blur(field);
}
it("explains noul on hover in the banner and on the answer type option", async () => {
const user = userEvent.setup();
render(<SystemOneUI accessToken="session-key" />);
await user.hover(screen.getByText("noul", { selector: "span.cursor-help" }));
expect(await screen.findByText(NOUL_HELP)).toBeInTheDocument();
question("has_repro_steps").getByRole("combobox", { name: "Answer type" }).focus();
await user.keyboard("{ArrowDown}");
expect(await screen.findByRole("option", { name: "noul" })).toHaveAttribute(
"title",
"noul is a yes / no question",
);
expect(screen.getByRole("option", { name: "Choice" })).not.toHaveAttribute("title");
});
it("sends a choice and a yes/no question built in the form without opening JSON", async () => {
const user = userEvent.setup();
render(<SystemOneUI accessToken="session-key" />);
@ -545,7 +957,7 @@ describe("SystemOneUI integration", () => {
fill(question("team").getByRole("textbox", { name: "Description of infra" }), "Deploys");
await user.click(screen.getByRole("button", { name: "Add question" }));
await pickAnswerType(user, "question_1", "Yes / no");
await pickAnswerType(user, "question_1", "noul");
rename(question("question_1").getByRole("textbox", { name: "Question name" }), "urgent");
fill(question("urgent").getByRole("textbox", { name: "Instructions" }), "Is this blocking users?");
fill(question("urgent").getByRole("textbox", { name: "Yes means" }), "Nobody can log in");
@ -554,9 +966,8 @@ describe("SystemOneUI integration", () => {
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
expect(screen.queryByRole("textbox", { name: "System One JSON payload" })).not.toBeInTheDocument();
expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/typesafe\/v1\/systemone$/);
expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/systemone$/);
expect(sentBody()).toEqual({
model: "jev-latest",
state: "The login page is blank after the upgrade",
questions: {
team: {
@ -575,7 +986,7 @@ describe("SystemOneUI integration", () => {
const user = userEvent.setup();
mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
render(<SystemOneUI accessToken="session-key" />);
expect(await screen.findByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest");
await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest"));
const level = question("severity").getByRole("textbox", { name: "Level 0" });
fill(level, "Typo");

206
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx
View file

@ -5,21 +5,22 @@ import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { SimpleTooltip } from "@/components/ui/tooltip";
import { SearchSelect } from "@/components/shared/SearchSelect";
import { DECISIONS_DOCS_URL } from "@/lib/decisionModels";
import { uiHref } from "@/utils/uiHref";
import { useMutation } from "@tanstack/react-query";
import { Code, Info, LoaderCircle, RotateCcw, Send } from "lucide-react";
import { Code, Eraser, Info, LoaderCircle, Send } from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import { makeSystemOneRequest } from "../../llm_calls/system_one";
import { PLACEHOLDER_DECISION_MODEL, SYSTEM_ONE_EXAMPLE, decisionsExample } from "./lib/example";
import { makePlaygroundDecisionRequest, type PlaygroundDecisionRequest } from "../../llm_calls/system_one";
import { DECISION_PRESETS, emptyPayload, matchPreset, presetPayload, type DecisionPreset } from "./lib/example";
import { payloadModel, withPayloadModel } from "./lib/payloadModel";
import type { DecisionEndpoint, PlaygroundRequest } from "./lib/schemas";
import JsonEditor from "./JsonEditor";
import QuestionBreakdown from "./QuestionBreakdown";
import ResponseView from "./ResponseView";
import SystemOneForm from "./SystemOneForm";
import { validateSystemOnePayload } from "./lib/validatePayload";
import OpenAIDecisionsForm from "./OpenAIDecisionsForm";
import { validateSystemOnePayload, validateOpenAIDecisionsPayload } from "./lib/validatePayload";
import { useDecisionModels, type ApiKeySource } from "./useDecisionModels";
interface SystemOneUIProps {
@ -30,15 +31,42 @@ interface SystemOneUIProps {
type EditorView = "form" | "json";
interface SystemOneSendVariables {
payload: PlaygroundRequest;
endpoint: DecisionEndpoint;
request: PlaygroundDecisionRequest;
apiKey: string;
signal: AbortSignal;
}
const EXAMPLE_PAYLOAD = JSON.stringify(SYSTEM_ONE_EXAMPLE, null, 2);
export const NOUL_HELP = "noul is a yes / no question. The model returns the probability that the answer is yes.";
const CUSTOM_PRESET = "custom";
const DECISION_MODELS_DISCUSSION_URL = "https://github.com/BerriAI/litellm/discussions/44231";
interface PresetPickerProps {
activePreset: DecisionPreset | undefined;
onPick: (id: string | null) => void;
}
function PresetPicker({ activePreset, onPick }: PresetPickerProps) {
return (
<Select value={activePreset?.id ?? CUSTOM_PRESET} onValueChange={onPick}>
<SelectTrigger className="w-56" aria-label="Example">
<SelectValue>{activePreset?.label ?? "Custom"}</SelectValue>
</SelectTrigger>
<SelectContent>
{activePreset === undefined && (
<SelectItem value={CUSTOM_PRESET} disabled>
Custom
</SelectItem>
)}
{DECISION_PRESETS.map((preset) => (
<SelectItem key={preset.id} value={preset.id}>
{preset.label}
</SelectItem>
))}
</SelectContent>
</Select>
);
}
function getCustomProxyBaseUrl(): string | undefined {
return typeof window === "undefined" ? undefined : window.sessionStorage.getItem("customProxyBaseUrl") || undefined;
}
@ -52,23 +80,27 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
effectiveApiKey,
getCustomProxyBaseUrl(),
);
const [chosenEndpoint, setChosenEndpoint] = useState<DecisionEndpoint | null>(null);
const endpoint: DecisionEndpoint =
chosenEndpoint ?? (decisionModels.length > 0 ? "/v1/systemone" : "/typesafe/v1/systemone");
const [drafts, setDrafts] = useState<Partial<Record<DecisionEndpoint, string>>>({});
const [endpoint, setEndpoint] = useState<PlaygroundDecisionRequest["endpoint"]>("/v1/systemone");
const [drafts, setDrafts] = useState<Partial<Record<PlaygroundDecisionRequest["endpoint"], string>>>({});
const [view, setView] = useState<EditorView>("form");
const examplePayload =
endpoint === "/v1/systemone"
? JSON.stringify(decisionsExample(decisionModels[0] ?? PLACEHOLDER_DECISION_MODEL), null, 2)
: EXAMPLE_PAYLOAD;
const rawPayload = drafts[endpoint] ?? examplePayload;
const rawPayload = drafts[endpoint] ?? presetPayload(DECISION_PRESETS[0], endpoint, decisionModels[0]);
const draftModel = drafts[endpoint] === undefined ? decisionModels[0] : payloadModel(rawPayload);
const activePreset = matchPreset(endpoint, rawPayload);
const clearedPayload = emptyPayload(endpoint, draftModel);
const activeController = useRef<AbortController | null>(null);
const validation = useMemo(() => validateSystemOnePayload(rawPayload, endpoint), [rawPayload, endpoint]);
const validated = useMemo(
() =>
endpoint === "/v1/decisions"
? ({ endpoint, validation: validateOpenAIDecisionsPayload(rawPayload) } as const)
: ({ endpoint, validation: validateSystemOnePayload(rawPayload, endpoint) } as const),
[rawPayload, endpoint],
);
const validation = validated.validation;
const hasSyntaxError = validation.issues.some((issue) => issue.path === "syntax");
const systemOne = useMutation({
mutationFn: ({ payload, apiKey, signal, endpoint }: SystemOneSendVariables) =>
makeSystemOneRequest(payload, apiKey, getCustomProxyBaseUrl(), { signal, endpoint }),
mutationFn: ({ request, apiKey, signal }: SystemOneSendVariables) =>
makePlaygroundDecisionRequest(request, apiKey, getCustomProxyBaseUrl(), { signal }),
});
const isLoading = systemOne.isPending;
const { reset: resetSystemOne } = systemOne;
@ -90,15 +122,15 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
function handlePayloadChange(value: string) {
if (value !== rawPayload) {
clearRequestState();
setChosenEndpoint(endpoint);
setDrafts((current) => ({ ...current, [endpoint]: value }));
}
}
function handleResetExample() {
clearRequestState();
setChosenEndpoint(endpoint);
setDrafts((current) => Object.fromEntries(Object.entries(current).filter(([key]) => key !== endpoint)));
function handlePresetPick(id: string | null) {
const preset = DECISION_PRESETS.find((candidate) => candidate.id === id);
if (preset !== undefined) {
handlePayloadChange(presetPayload(preset, endpoint, draftModel));
}
}
function handleModelPick(model: string | null) {
@ -115,14 +147,18 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
}
function handleSend() {
if (!validation.payload || !effectiveApiKey) {
const request: PlaygroundDecisionRequest | undefined =
validated.endpoint === "/v1/decisions"
? validated.validation.payload && { endpoint: validated.endpoint, payload: validated.validation.payload }
: validated.validation.payload && { endpoint: validated.endpoint, payload: validated.validation.payload };
if (!request || !effectiveApiKey) {
return;
}
clearRequestState();
setChosenEndpoint(endpoint);
setDrafts((current) => ({ ...current, [endpoint]: rawPayload }));
const controller = new AbortController();
activeController.current = controller;
const variables = { payload: validation.payload, apiKey: effectiveApiKey, signal: controller.signal, endpoint };
const variables = { request, apiKey: effectiveApiKey, signal: controller.signal };
systemOne.mutate(variables);
}
@ -134,43 +170,37 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
<Select
value={endpoint}
onValueChange={(value) => {
if (value === "/v1/systemone" || value === "/typesafe/v1/systemone") {
if (value === "/v1/decisions" || value === "/v1/systemone") {
clearRequestState();
setChosenEndpoint(value);
setEndpoint(value);
}
}}
>
<SelectTrigger className="w-80" aria-label="Decision endpoint">
<SelectValue>
{endpoint === "/v1/systemone" ? "System One · /v1/systemone" : "TypeSafe · /typesafe/v1/systemone"}
</SelectValue>
<SelectValue>{endpoint}</SelectValue>
</SelectTrigger>
<SelectContent>
<SelectItem value="/v1/systemone">System One · /v1/systemone</SelectItem>
<SelectItem value="/typesafe/v1/systemone">TypeSafe · /typesafe/v1/systemone</SelectItem>
<SelectItem value="/v1/decisions">/v1/decisions</SelectItem>
<SelectItem value="/v1/systemone">/v1/systemone</SelectItem>
</SelectContent>
</Select>
{endpoint === "/v1/systemone" && (
<>
<span className="ml-2 text-sm font-medium text-muted-foreground">Model</span>
<div className="w-80">
<SearchSelect
aria-label="Decision model"
options={decisionModels.map((model) => ({ label: model, value: model }))}
value={payloadModel(rawPayload) ?? null}
onValueChange={handleModelPick}
placeholder={decisionModels.length > 0 ? "Pick a decision model" : "No decision models yet"}
emptyText="No decision models found"
disabled={hasSyntaxError}
allowClear={false}
/>
</div>
{decisionModelsLoaded && decisionModels.length === 0 && (
<a href={uiHref("models-and-endpoints")} className="text-sm text-primary underline">
Add a decision model
</a>
)}
</>
<span className="ml-2 text-sm font-medium text-muted-foreground">Model</span>
<div className="w-80">
<SearchSelect
aria-label="Decision model"
options={decisionModels.map((model) => ({ label: model, value: model }))}
value={payloadModel(rawPayload) ?? null}
onValueChange={handleModelPick}
placeholder={decisionModels.length > 0 ? "Pick a decision model" : "No decision models yet"}
emptyText="No decision models found"
disabled={hasSyntaxError}
allowClear={false}
/>
</div>
{decisionModelsLoaded && decisionModels.length === 0 && (
<a href={uiHref("models-and-endpoints")} className="text-sm text-primary underline">
Add a decision model
</a>
)}
</div>
<div className="flex flex-wrap items-center gap-3">
@ -207,9 +237,14 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
)}
</div>
<div className="ml-auto flex items-center gap-2">
<Button variant="outline" onClick={handleResetExample} disabled={rawPayload === examplePayload}>
<RotateCcw />
Reset example
<PresetPicker activePreset={activePreset} onPick={handlePresetPick} />
<Button
variant="outline"
onClick={() => handlePayloadChange(clearedPayload)}
disabled={rawPayload === clearedPayload}
>
<Eraser />
Clear
</Button>
{view === "json" && (
<Button variant="outline" onClick={handleFormatJson} disabled={!rawPayload.trim() || hasSyntaxError}>
@ -230,13 +265,20 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
</div>
<Alert role="note" aria-label="Decision endpoint notice">
<Info />
<AlertTitle>
{endpoint === "/v1/systemone" ? "Decision models · System One" : "TypeSafe Jev · System One"}
</AlertTitle>
<AlertTitle>{endpoint}</AlertTitle>
<AlertDescription>
{endpoint === "/v1/systemone"
? "Sends choice, noul, and score questions through /v1/systemone to a decision model on your proxy. Pick one under Model, or omit model to use the proxy's configured default."
: "Sends requests through /typesafe/v1/systemone and requires TYPESAFE_API_KEY on the proxy."}{" "}
{endpoint === "/v1/decisions" ? (
"Uses input and a questions array with predicate, choice, and score questions. Edit the form or JSON and inspect the response JSON in the OpenAI Decisions format."
) : (
<>
Uses state and a questions object with{" "}
<SimpleTooltip content={NOUL_HELP} className="underline decoration-dotted underline-offset-2">
noul
</SimpleTooltip>
, choice, and score questions in the System One format.
</>
)}{" "}
Pick a decision model under Model, or omit model to use the proxy&apos;s configured default.{" "}
<a href={DECISIONS_DOCS_URL} target="_blank" rel="noopener noreferrer" className="underline">
How to call /v1/decisions and /v1/systemone
</a>
@ -249,7 +291,7 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
</section>
<div className="grid gap-4 xl:min-h-0 xl:flex-1 xl:grid-cols-2">
<section className="flex min-h-96 flex-col xl:min-h-0" aria-label="System One request editor">
<section className="flex min-h-96 flex-col xl:min-h-0" aria-label="Decisions request editor">
<Tabs
value={view}
onValueChange={(value) => (value === "form" || value === "json") && setView(value)}
@ -260,27 +302,45 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
<TabsTrigger value="json">JSON</TabsTrigger>
</TabsList>
<TabsContent value="form" className="flex min-h-0 flex-col">
<SystemOneForm
{validated.endpoint === "/v1/decisions" ? (
<OpenAIDecisionsForm
value={rawPayload}
onChange={handlePayloadChange}
validation={validated.validation}
onOpenJson={() => setView("json")}
/>
) : (
<SystemOneForm
value={rawPayload}
onChange={handlePayloadChange}
validation={validated.validation}
onOpenJson={() => setView("json")}
/>
)}
</TabsContent>
<TabsContent value="json" className="flex min-h-0 flex-col">
<JsonEditor
value={rawPayload}
onChange={handlePayloadChange}
validation={validation}
onOpenJson={() => setView("json")}
label={endpoint === "/v1/decisions" ? "Decisions JSON payload" : "System One JSON payload"}
placeholder={`Paste or write a ${endpoint} request`}
/>
</TabsContent>
<TabsContent value="json" className="flex min-h-0 flex-col">
<JsonEditor value={rawPayload} onChange={handlePayloadChange} validation={validation} />
</TabsContent>
</Tabs>
</section>
<section className="grid content-start gap-4 xl:min-h-0 xl:overflow-auto" aria-label="System One results">
<section className="grid content-start gap-4 xl:min-h-0 xl:overflow-auto" aria-label="Decisions results">
<ResponseView
response={systemOne.data?.response}
view={view}
fallbackModel={validation.payload?.model}
latencyMs={systemOne.data?.latencyMs}
error={systemOne.error?.message}
isLoading={isLoading}
/>
<QuestionBreakdown payload={validation.payload} />
{view === "form" && validated.endpoint === "/v1/systemone" && (
<QuestionBreakdown payload={validated.validation.payload} />
)}
</section>
</div>
</div>

53
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/decisions.test.ts
View file

@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest";
import { validateSystemOnePayload } from "./validatePayload";
import { validateSystemOnePayload, validateOpenAIDecisionsPayload } from "./validatePayload";
import { systemOneResponseSchema } from "./schemas";
import { openAIDecisionsExample } from "./example";
const request = {
model: "configured-decision-model",
@ -81,3 +82,53 @@ describe("native decisions validation", () => {
expect(systemOneResponseSchema.parse(response)).toEqual(response);
});
});
describe("/v1/decisions validation", () => {
const example = openAIDecisionsExample("jev-latest");
const validateOpenAI = (value: unknown) => validateOpenAIDecisionsPayload(JSON.stringify(value));
it("keeps input, question arrays, provider extensions and optional model omission intact", () => {
const payload = { input: example.input, questions: example.questions, provider_option: { enabled: true } };
expect(validateOpenAI(payload)).toEqual({ isValid: true, payload, issues: [] });
expect(validateOpenAI(example).isValid).toBe(true);
});
it("accepts multimodal messages and boolean choices without converting them to strings", () => {
const payload = {
input: [
{
role: "user",
content: [
{ type: "input_text", text: "Does this show an error?" },
{ type: "input_image", image_url: "data:image/png;base64,example" },
],
},
],
questions: [{ type: "choice", instructions: "Is this an error?", choices: [{ value: true }, { value: false }] }],
};
expect(validateOpenAI(payload)).toEqual({ isValid: true, payload, issues: [] });
});
it.each([
request,
{ ...example, questions: [] },
{ ...example, questions: request.questions },
{ ...example, input: {} },
{ ...example, questions: [{ type: "noul", instructions: "Test" }] },
{ ...example, questions: [{ type: "predicate" }] },
{ ...example, questions: [{ type: "predicate", instructions: "Test", criteria: {} }] },
{ ...example, questions: [{ type: "choice", instructions: "Test", choices: [{ value: true }] }] },
{
...example,
questions: [{ type: "choice", instructions: "Test", choices: [{ value: true }, { value: "true" }] }],
},
{ ...example, questions: [{ type: "score", instructions: "Test", levels: [{ label: "Low" }] }] },
])("rejects the wrong endpoint's shape and invalid question definitions %#", (payload) => {
expect(validateOpenAI(payload).isValid).toBe(false);
});
it("rejects blank and malformed JSON without throwing", () => {
expect(validateOpenAIDecisionsPayload(" ").issues[0]?.path).toBe("root");
expect(validateOpenAIDecisionsPayload("{").issues[0]?.path).toBe("syntax");
});
});

60
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/example.test.ts Normal file
View file

@ -0,0 +1,60 @@
import { describe, expect, it } from "vitest";
import {
DECISION_PRESETS,
emptyPayload,
openAIDecisionsExample,
presetPayload,
toOpenAIDecisionsRequest,
matchPreset,
} from "./example";
import { validateOpenAIDecisionsPayload, validateSystemOnePayload } from "./validatePayload";
describe("decision presets", () => {
it.each(DECISION_PRESETS.map((preset) => [preset.label, preset] as const))(
"%s is a valid request on both endpoints",
(_label, preset) => {
expect(
validateSystemOnePayload(presetPayload(preset, "/v1/systemone", "jev-latest"), "/v1/systemone"),
).toMatchObject({ isValid: true, issues: [] });
expect(validateOpenAIDecisionsPayload(presetPayload(preset, "/v1/decisions", "jev-latest"))).toMatchObject({
isValid: true,
issues: [],
});
},
);
it("covers every question type across the presets", () => {
const types = new Set(
DECISION_PRESETS.flatMap((preset) => Object.values(preset.request.questions).map((question) => question.type)),
);
expect([...types].sort()).toEqual(["choice", "noul", "score"]);
});
it("maps System One questions onto the OpenAI shape by name", () => {
const request = toOpenAIDecisionsRequest(DECISION_PRESETS[0].request, "jev-latest");
expect(request.questions.map((question) => [question.type, question.name])).toEqual([
["choice", "area"],
["predicate", "has_repro_steps"],
["score", "severity"],
]);
expect(request).toEqual(openAIDecisionsExample("jev-latest"));
});
it("recognises a preset whatever the model key holds or where it sits", () => {
const lead = DECISION_PRESETS.find((preset) => preset.id === "lead-scoring");
const parsed = JSON.parse(presetPayload(lead!, "/v1/systemone", "jev-latest")) as Record<string, unknown>;
const { model: _model, ...rest } = parsed;
expect(matchPreset("/v1/systemone", JSON.stringify({ ...rest, model: "pplx-decider" }))).toBe(lead);
expect(matchPreset("/v1/systemone", JSON.stringify(rest))).toBe(lead);
expect(matchPreset("/v1/decisions", presetPayload(lead!, "/v1/decisions"))).toBe(lead);
expect(matchPreset("/v1/decisions", presetPayload(lead!, "/v1/systemone"))).toBeUndefined();
expect(matchPreset("/v1/systemone", JSON.stringify({ ...rest, state: "edited" }))).toBeUndefined();
expect(matchPreset("/v1/systemone", "{")).toBeUndefined();
});
it("omits the model from payloads when none is known", () => {
expect(JSON.parse(presetPayload(DECISION_PRESETS[1], "/v1/systemone"))).not.toHaveProperty("model");
expect(JSON.parse(emptyPayload("/v1/decisions"))).toEqual({ input: "", questions: [] });
expect(validateSystemOnePayload(emptyPayload("/v1/systemone"), "/v1/systemone").isValid).toBe(false);
});
});

165
ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/example.ts
View file

@ -1,4 +1,39 @@
import type { DecisionRequest, SystemOneRequest } from "./schemas";
import type { OpenAIDecisionsRequest } from "./openAIDecisions";
import { parseJson } from "./validatePayload";
type PresetChoice = {
type: "choice";
instructions: string;
criteria: Record<string, string>;
};
type PresetNoul = {
type: "noul";
instructions: string;
criteria?: { true: string; false: string };
};
type PresetScore = {
type: "score";
instructions: string;
criteria: string[];
};
export type PresetQuestion = PresetChoice | PresetNoul | PresetScore;
export type PresetRequest = {
state: string;
questions: Record<string, PresetQuestion>;
};
type PresetEndpoint = "/v1/decisions" | "/v1/systemone";
export interface DecisionPreset {
id: string;
label: string;
request: PresetRequest;
}
export const SYSTEM_ONE_EXAMPLE = {
model: "jev-latest",
@ -37,6 +72,132 @@ export const SYSTEM_ONE_EXAMPLE = {
},
} satisfies SystemOneRequest;
export const PLACEHOLDER_DECISION_MODEL = "your-decision-model";
const { model: _exampleModel, ...bugTriage }: SystemOneRequest & PresetRequest = SYSTEM_ONE_EXAMPLE;
export const decisionsExample = (model: string): DecisionRequest => ({ ...SYSTEM_ONE_EXAMPLE, model });
const supportRouting = {
state:
"Hi, I was charged twice for my team plan this month and the second charge pushed my card over its limit. I need the duplicate refunded today and want to know why it happened.",
questions: {
department: {
type: "choice",
instructions: "Which team should handle this ticket?",
criteria: {
billing: "Invoices, charges, refunds, and plan changes",
technical: "Bugs, errors, outages, and integration help",
account: "Login, permissions, and profile settings",
sales: "Pricing questions and new purchases",
},
},
},
} satisfies PresetRequest;
const moderation = {
state:
"This guide is garbage and whoever wrote it should be fired. The fix that actually works is setting stream=false, took me two days to figure that out.",
questions: {
breaks_rules: {
type: "noul",
instructions: "Does this comment break the community guidelines?",
criteria: {
true: "Insults, harassment, slurs, or threats aimed at a person",
false: "Blunt or frustrated but still about the content",
},
},
},
} satisfies PresetRequest;
const leadScoring = {
state:
"Inbound form: VP Engineering at a 400 person fintech. Running a self hosted gateway today, evaluating replacements this quarter, budget approved, needs SSO and audit logs.",
questions: {
fit: {
type: "score",
instructions: "How well does this lead fit our ideal customer?",
criteria: [
"No fit, wrong market or no budget",
"Weak fit, early curiosity only",
"Possible fit, needs discovery",
"Strong fit, active evaluation",
"Ideal fit, ready to buy this quarter",
],
},
},
} satisfies PresetRequest;
export const DECISION_PRESETS: readonly DecisionPreset[] = [
{ id: "bug-triage", label: "Triage a bug report", request: bugTriage },
{ id: "support-routing", label: "Route a support ticket", request: supportRouting },
{ id: "moderation", label: "Moderate a forum comment", request: moderation },
{ id: "lead-scoring", label: "Score a sales lead", request: leadScoring },
];
const toOpenAIQuestion = (name: string, question: PresetQuestion): OpenAIDecisionsRequest["questions"][number] => {
switch (question.type) {
case "choice":
return {
type: "choice",
name,
instructions: question.instructions,
choices: Object.entries(question.criteria).map(([value, description]) => ({ value, description })),
};
case "noul":
return { type: "predicate", name, instructions: question.instructions };
case "score":
return {
type: "score",
name,
instructions: question.instructions,
levels: question.criteria.map((description, index) => ({ label: String(index), description })),
};
}
};
export const toSystemOneRequest = (request: PresetRequest, model?: string): DecisionRequest => ({ model, ...request });
export const toOpenAIDecisionsRequest = (request: PresetRequest, model?: string): OpenAIDecisionsRequest => ({
model,
input: request.state,
questions: Object.entries(request.questions).map(([name, question]) => toOpenAIQuestion(name, question)),
});
export const presetPayload = (preset: DecisionPreset, endpoint: PresetEndpoint, model?: string): string =>
JSON.stringify(
endpoint === "/v1/decisions"
? toOpenAIDecisionsRequest(preset.request, model)
: toSystemOneRequest(preset.request, model),
null,
2,
);
const stableJson = (value: unknown): string =>
JSON.stringify(value, (_key, item: unknown) =>
item !== null && typeof item === "object" && !Array.isArray(item)
? Object.fromEntries(Object.entries(item as Record<string, unknown>).sort(([a], [b]) => a.localeCompare(b)))
: item,
);
const withoutModel = (value: unknown): string => {
const { model: _model, ...rest } = (value ?? {}) as Record<string, unknown>;
return stableJson(rest);
};
export const matchPreset = (endpoint: PresetEndpoint, raw: string): DecisionPreset | undefined => {
const json = parseJson(raw);
if (!json.ok) {
return undefined;
}
const current = withoutModel(json.value);
return DECISION_PRESETS.find((preset) => withoutModel(JSON.parse(presetPayload(preset, endpoint))) === current);
};
export const emptyPayload = (endpoint: PresetEndpoint, model?: string): string =>
JSON.stringify(
endpoint === "/v1/decisions" ? { model, input: "", questions: [] } : { model, state: "", questions: {} },
null,
2,
);
export const decisionsExample = (model?: string): DecisionRequest => toSystemOneRequest(bugTriage, model);
export const openAIDecisionsExample = (model?: string): OpenAIDecisionsRequest =>
toOpenAIDecisionsRequest(bugTriage, model);

Some files were not shown because too many files have changed in this diff Show more