mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
refactor(decisions): dispatch /v1/decisions through provider configs and the shared HTTP handler (#45130)
* refactor(decisions): dispatch /v1/decisions through provider configs and the shared HTTP handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): map OpenRouter connection failures to APIConnectionError and drop explanatory docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): keep APIConnectionError for unreachable deployments on every provider `_map_upstream_exception` now maps a status `_handle_error` synthesized for a non-HTTP failure to `APIConnectionError` before `exception_type` runs, so a Perplexity (openai-compatible) deployment that cannot be reached no longer answers `InternalServerError` where main answers `APIConnectionError`. Audit cells: the unreachable-deployment integration cell is parametrized over the five providers, OpenRouter connection failures are pinned on chat, stream, embeddings, decisions and the SDK, a deployment whose provider has no decisions config gets the 400 naming every supported provider, and a unit cell pins the mapper alone. The first docstring sentence of the five translation bases is dropped. --------- Co-authored-by: kerry <kerry@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
e4d9a591dd
commit
0e09e99146
31 changed files with 1034 additions and 294 deletions
|
|
@ -1667,6 +1667,21 @@ if TYPE_CHECKING:
|
|||
from .llms.hosted_vllm.rerank.transformation import (
|
||||
HostedVLLMRerankConfig as HostedVLLMRerankConfig,
|
||||
)
|
||||
from .llms.perplexity.decisions.transformation import (
|
||||
PerplexityDecisionsConfig as PerplexityDecisionsConfig,
|
||||
)
|
||||
from .llms.typesafe.decisions.transformation import (
|
||||
TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
|
||||
)
|
||||
from .llms.openrouter.decisions.transformation import (
|
||||
OpenRouterDecisionsConfig as OpenRouterDecisionsConfig,
|
||||
)
|
||||
from .llms.cloudflare.decisions.transformation import (
|
||||
CloudflareDecisionsConfig as CloudflareDecisionsConfig,
|
||||
)
|
||||
from .llms.strands_decider.decisions.transformation import (
|
||||
StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig,
|
||||
)
|
||||
from .llms.nvidia_nim.rerank.transformation import (
|
||||
NvidiaNimRerankConfig as NvidiaNimRerankConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -156,6 +156,11 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"ScalewayRerankConfig",
|
||||
"DeepinfraRerankConfig",
|
||||
"HostedVLLMRerankConfig",
|
||||
"PerplexityDecisionsConfig",
|
||||
"TypeSafeDecisionsConfig",
|
||||
"OpenRouterDecisionsConfig",
|
||||
"CloudflareDecisionsConfig",
|
||||
"StrandsDeciderDecisionsConfig",
|
||||
"NvidiaNimRerankConfig",
|
||||
"NvidiaNimRankingConfig",
|
||||
"VertexAIRerankConfig",
|
||||
|
|
@ -707,6 +712,14 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
".llms.hosted_vllm.rerank.transformation",
|
||||
"HostedVLLMRerankConfig",
|
||||
),
|
||||
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
|
||||
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
|
||||
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
|
||||
"CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),
|
||||
"StrandsDeciderDecisionsConfig": (
|
||||
".llms.strands_decider.decisions.transformation",
|
||||
"StrandsDeciderDecisionsConfig",
|
||||
),
|
||||
"NvidiaNimRerankConfig": (
|
||||
".llms.nvidia_nim.rerank.transformation",
|
||||
"NvidiaNimRerankConfig",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -8,107 +7,87 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig
|
||||
from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client, get_httpx_client
|
||||
from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT
|
||||
from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT
|
||||
from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT
|
||||
from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_ENDPOINT
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.types.decisions import (
|
||||
DecisionQuestion,
|
||||
DecisionsJSON,
|
||||
DecisionsRequest,
|
||||
DecisionsResponse,
|
||||
)
|
||||
from litellm.utils import client
|
||||
|
||||
DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxyType(
|
||||
{
|
||||
"perplexity": PERPLEXITY_DECISIONS_ENDPOINT,
|
||||
"typesafe": TYPESAFE_DECISIONS_ENDPOINT,
|
||||
"openrouter": OPENROUTER_DECISIONS_ENDPOINT,
|
||||
"cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT,
|
||||
"strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT,
|
||||
}
|
||||
)
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager, client
|
||||
|
||||
_DECISIONS_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequest]] = TypeAdapter(DecisionsRequest)
|
||||
_DECISIONS_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
|
||||
_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse)
|
||||
_HANDLER: Final = BaseLLMHTTPHandler()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, repr=False)
|
||||
class _PreparedDecisionsRequest:
|
||||
config: DecisionsProviderConfig
|
||||
provider: str
|
||||
upstream_model: str
|
||||
url: str
|
||||
class _DecisionsCall:
|
||||
model: str
|
||||
custom_llm_provider: str
|
||||
provider_config: BaseDecisionsConfig
|
||||
request: DecisionsRequest
|
||||
api_base: str
|
||||
api_key: str | None = field(repr=False)
|
||||
headers: Mapping[str, str] = field(repr=False)
|
||||
body: Mapping[str, object] = field(repr=False)
|
||||
logging_obj: LiteLLMLoggingObj | None
|
||||
headers: Mapping[str, str]
|
||||
timeout: float | httpx.Timeout | None
|
||||
|
||||
|
||||
def _resolve_provider_model(model: str, custom_llm_provider: str | None) -> tuple[str, str]:
|
||||
provider: Final = model.partition("/")[0] if custom_llm_provider is None else custom_llm_provider
|
||||
if provider not in DECISIONS_ENDPOINTS:
|
||||
supported: Final = ", ".join(DECISIONS_ENDPOINTS)
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Unknown Decisions provider '{provider}'. Supported providers: {supported}",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
upstream_model: Final = model.removeprefix(f"{provider}/")
|
||||
if not upstream_model:
|
||||
raise litellm.BadRequestError(
|
||||
message="A model name is required for the Decisions API",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
return provider, upstream_model
|
||||
|
||||
|
||||
def _resolve_api_key(
|
||||
*,
|
||||
provider: str,
|
||||
model: str,
|
||||
endpoint: DecisionsProviderConfig,
|
||||
api_key: str | None,
|
||||
) -> str | None:
|
||||
if api_key is not None:
|
||||
return api_key
|
||||
|
||||
server_api_key: Final = next(
|
||||
(key for key in (get_secret_str(name) for name in endpoint.api_key_env) if key),
|
||||
None,
|
||||
def _supported_providers() -> tuple[str, ...]:
|
||||
return tuple(
|
||||
provider.value
|
||||
for provider in LlmProviders
|
||||
if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None
|
||||
)
|
||||
if server_api_key is None:
|
||||
if not endpoint.api_key_required:
|
||||
return None
|
||||
raise litellm.AuthenticationError(
|
||||
message=f"Missing API key for Decisions provider '{provider}'",
|
||||
|
||||
|
||||
def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig:
|
||||
provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None)
|
||||
provider_config: Final = (
|
||||
None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider)
|
||||
)
|
||||
if provider_config is None:
|
||||
supported: Final = ", ".join(_supported_providers())
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
return server_api_key
|
||||
return provider_config
|
||||
|
||||
|
||||
def _prepare_request(
|
||||
def _prepare_call(
|
||||
*,
|
||||
model: str,
|
||||
state: DecisionsJSON,
|
||||
questions: Mapping[str, DecisionQuestion | Mapping[str, object]],
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
timeout: float | httpx.Timeout | None,
|
||||
custom_llm_provider: str | None,
|
||||
extra_headers: Mapping[str, str] | None,
|
||||
) -> _PreparedDecisionsRequest:
|
||||
provider, upstream_model = _resolve_provider_model(model, custom_llm_provider)
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _DecisionsCall:
|
||||
upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
)
|
||||
provider_config: Final = _provider_config(upstream_model, provider)
|
||||
canonical_model: Final = provider_config.canonical_model(upstream_model)
|
||||
if not upstream_model:
|
||||
raise litellm.BadRequestError(
|
||||
message="A model name is required for the Decisions API",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
try:
|
||||
validated_request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python(
|
||||
{"model": model, "state": state, "questions": questions}
|
||||
request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python(
|
||||
{"model": canonical_model, "state": state, "questions": questions}
|
||||
)
|
||||
except ValidationError as error:
|
||||
raise litellm.BadRequestError(
|
||||
|
|
@ -117,109 +96,56 @@ def _prepare_request(
|
|||
llm_provider=provider,
|
||||
) from error
|
||||
|
||||
endpoint: Final = DECISIONS_ENDPOINTS[provider]
|
||||
env_api_base: Final = get_secret_str(endpoint.api_base_env)
|
||||
default_api_base: Final = endpoint.default_api_base()
|
||||
resolved_api_base: Final = api_base or env_api_base or default_api_base
|
||||
resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base)
|
||||
if resolved_api_base is None:
|
||||
raise litellm.BadRequestError(
|
||||
message=endpoint.missing_api_base_message(provider),
|
||||
message=provider_config.missing_api_base_message(provider),
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key)
|
||||
if resolved_api_key is None and provider_config.api_key_required:
|
||||
raise litellm.AuthenticationError(
|
||||
message=f"Missing API key for Decisions provider '{provider}'",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
|
||||
resolved_api_key: Final = _resolve_api_key(
|
||||
provider=provider,
|
||||
model=model,
|
||||
endpoint=endpoint,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
canonical_model: Final = endpoint.canonical_model(upstream_model)
|
||||
outbound_headers: Final = MappingProxyType(
|
||||
{
|
||||
**{
|
||||
name: value
|
||||
for name, value in (extra_headers or {}).items()
|
||||
if name.lower() not in {"authorization", "content-type"}
|
||||
},
|
||||
**({"Authorization": f"Bearer {resolved_api_key}"} if resolved_api_key is not None else {}),
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
)
|
||||
body: Final = MappingProxyType(
|
||||
{
|
||||
"model": endpoint.request_model(canonical_model),
|
||||
"state": validated_request.state,
|
||||
"questions": {
|
||||
name: question.model_dump(mode="json", exclude_none=True)
|
||||
for name, question in validated_request.questions.items()
|
||||
},
|
||||
}
|
||||
)
|
||||
return _PreparedDecisionsRequest(
|
||||
config=endpoint,
|
||||
provider=provider,
|
||||
upstream_model=canonical_model,
|
||||
url=endpoint.endpoint_url(resolved_api_base, canonical_model),
|
||||
api_key=resolved_api_key,
|
||||
headers=outbound_headers,
|
||||
body=body,
|
||||
)
|
||||
|
||||
|
||||
def _log_request(
|
||||
prepared: _PreparedDecisionsRequest,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> LiteLLMLoggingObj | None:
|
||||
logging_obj: Final = kwargs.get("litellm_logging_obj")
|
||||
if not isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
return None
|
||||
logging_obj.update_from_kwargs(
|
||||
kwargs=dict(kwargs),
|
||||
model=prepared.upstream_model,
|
||||
litellm_params={
|
||||
"litellm_call_id": kwargs.get("litellm_call_id"),
|
||||
"api_base": prepared.url,
|
||||
},
|
||||
custom_llm_provider=prepared.provider,
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
logging_obj.update_from_kwargs(
|
||||
kwargs=dict(kwargs),
|
||||
model=canonical_model,
|
||||
litellm_params={
|
||||
"litellm_call_id": kwargs.get("litellm_call_id"),
|
||||
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
|
||||
},
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
return _DecisionsCall(
|
||||
model=canonical_model,
|
||||
custom_llm_provider=provider,
|
||||
provider_config=provider_config,
|
||||
request=request,
|
||||
api_base=resolved_api_base,
|
||||
api_key=resolved_api_key,
|
||||
logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
|
||||
headers=extra_headers or {},
|
||||
timeout=timeout,
|
||||
)
|
||||
request_body: Final = dict(prepared.body)
|
||||
request_headers: Final = dict(prepared.headers)
|
||||
logging_obj.pre_call(
|
||||
input=request_body,
|
||||
api_key=prepared.api_key,
|
||||
model=prepared.upstream_model,
|
||||
additional_args={
|
||||
"api_base": prepared.url,
|
||||
"complete_input_dict": request_body,
|
||||
"headers": request_headers,
|
||||
},
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _parse_response(
|
||||
response: httpx.Response,
|
||||
prepared: _PreparedDecisionsRequest,
|
||||
) -> DecisionsResponse:
|
||||
response.raise_for_status()
|
||||
payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content)
|
||||
result: Final = _DECISIONS_RESPONSE_ADAPTER.validate_python(prepared.config.unwrap_response(payload))
|
||||
result.hidden_params.update(
|
||||
{
|
||||
"model": f"{prepared.provider}/{prepared.upstream_model}",
|
||||
"custom_llm_provider": prepared.provider,
|
||||
"provider_response_model": f"{prepared.provider}/{prepared.upstream_model}",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _map_upstream_exception(error: Exception, prepared: _PreparedDecisionsRequest) -> Exception:
|
||||
def _map_upstream_exception(error: Exception, call: _DecisionsCall) -> Exception:
|
||||
if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
|
||||
provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
|
||||
return litellm.APIConnectionError(
|
||||
message=f"{provider_label} - {error.message}",
|
||||
llm_provider=call.custom_llm_provider,
|
||||
model=f"{call.custom_llm_provider}/{call.model}",
|
||||
)
|
||||
return litellm.exception_type(
|
||||
model=f"{prepared.provider}/{prepared.upstream_model}",
|
||||
custom_llm_provider=prepared.provider,
|
||||
model=f"{call.custom_llm_provider}/{call.model}",
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
original_exception=error,
|
||||
)
|
||||
|
||||
|
|
@ -236,28 +162,31 @@ async def adecisions(
|
|||
extra_headers: Mapping[str, str] | None = None,
|
||||
**kwargs: object,
|
||||
) -> DecisionsResponse:
|
||||
prepared: Final = _prepare_request(
|
||||
call: Final = _prepare_call(
|
||||
model=model,
|
||||
state=state,
|
||||
questions=questions,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
extra_headers=extra_headers,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
logging_obj: Final = _log_request(prepared, kwargs)
|
||||
try:
|
||||
handler: Final = get_async_httpx_client(llm_provider=prepared.provider)
|
||||
response: Final = await handler.post(
|
||||
prepared.url,
|
||||
json=dict(prepared.body),
|
||||
headers=dict(prepared.headers),
|
||||
timeout=timeout,
|
||||
logging_obj=logging_obj,
|
||||
return await _HANDLER.adecisions(
|
||||
model=call.model,
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
logging_obj=call.logging_obj,
|
||||
provider_config=call.provider_config,
|
||||
request=call.request,
|
||||
api_base=call.api_base,
|
||||
api_key=call.api_key,
|
||||
headers=call.headers,
|
||||
timeout=call.timeout,
|
||||
)
|
||||
return _parse_response(response=response, prepared=prepared)
|
||||
except Exception as error:
|
||||
raise _map_upstream_exception(error, prepared) from error
|
||||
raise _map_upstream_exception(error, call) from error
|
||||
|
||||
|
||||
@client
|
||||
|
|
@ -272,28 +201,31 @@ def decisions(
|
|||
extra_headers: Mapping[str, str] | None = None,
|
||||
**kwargs: object,
|
||||
) -> DecisionsResponse:
|
||||
prepared: Final = _prepare_request(
|
||||
call: Final = _prepare_call(
|
||||
model=model,
|
||||
state=state,
|
||||
questions=questions,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
extra_headers=extra_headers,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
logging_obj: Final = _log_request(prepared, kwargs)
|
||||
try:
|
||||
handler: Final = get_httpx_client()
|
||||
response: Final = handler.post(
|
||||
prepared.url,
|
||||
json=dict(prepared.body),
|
||||
headers=dict(prepared.headers),
|
||||
timeout=timeout,
|
||||
logging_obj=logging_obj,
|
||||
return _HANDLER.decisions(
|
||||
model=call.model,
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
logging_obj=call.logging_obj,
|
||||
provider_config=call.provider_config,
|
||||
request=call.request,
|
||||
api_base=call.api_base,
|
||||
api_key=call.api_key,
|
||||
headers=call.headers,
|
||||
timeout=call.timeout,
|
||||
)
|
||||
return _parse_response(response=response, prepared=prepared)
|
||||
except Exception as error:
|
||||
raise _map_upstream_exception(error, prepared) from error
|
||||
raise _map_upstream_exception(error, call) from error
|
||||
|
||||
|
||||
__all__ = ["DECISIONS_ENDPOINTS", "adecisions", "decisions"]
|
||||
__all__ = ["adecisions", "decisions"]
|
||||
|
|
|
|||
|
|
@ -2147,7 +2147,10 @@ def _map_openrouter_exception(
|
|||
exception_provider: str,
|
||||
extra_information: str,
|
||||
) -> None:
|
||||
if hasattr(original_exception, "status_code"):
|
||||
received_status: Final = hasattr(original_exception, "status_code") and not getattr(
|
||||
original_exception, "status_code_is_synthesized", False
|
||||
)
|
||||
if received_status:
|
||||
if original_exception.status_code == 400:
|
||||
raise BadRequestError(
|
||||
message=f"{exception_provider} - {error_str}",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
from .transformation import DecisionsProviderConfig, JevCompatibleDecisionsEndpoint
|
||||
from .transformation import BaseDecisionsConfig
|
||||
|
||||
__all__ = ["DecisionsProviderConfig", "JevCompatibleDecisionsEndpoint"]
|
||||
__all__ = ["BaseDecisionsConfig"]
|
||||
|
|
|
|||
|
|
@ -1,20 +1,40 @@
|
|||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from abc import ABC
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.decisions import DecisionsRequest, DecisionsResponse
|
||||
|
||||
PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
|
||||
_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse)
|
||||
_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class JevCompatibleDecisionsEndpoint:
|
||||
default_api_base_value: str | None
|
||||
path: str
|
||||
api_key_env: tuple[str, ...]
|
||||
api_base_env: str
|
||||
class BaseDecisionsConfig(ABC):
|
||||
path: str = "/v1/systemone"
|
||||
api_key_env: tuple[str, ...] = ()
|
||||
api_base_env: tuple[str, ...] = ()
|
||||
api_key_required: bool = True
|
||||
|
||||
def default_api_base(self) -> str | None:
|
||||
return self.default_api_base_value
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return None
|
||||
|
||||
def missing_api_base_message(self, provider: str) -> str:
|
||||
return f"api_base is required for Decisions provider '{provider}'"
|
||||
def missing_api_base_message(self, custom_llm_provider: str) -> str:
|
||||
return f"api_base is required for Decisions provider '{custom_llm_provider}'"
|
||||
|
||||
def resolve_api_base(self, api_base: str | None) -> str | None:
|
||||
return api_base or self._first_secret(self.api_base_env) or self.get_default_api_base()
|
||||
|
||||
def resolve_api_key(self, api_key: str | None) -> str | None:
|
||||
return api_key or self._first_secret(self.api_key_env)
|
||||
|
||||
@staticmethod
|
||||
def _first_secret(names: tuple[str, ...]) -> str | None:
|
||||
return next((value for value in (get_secret_str(name) for name in names) if value), None)
|
||||
|
||||
def canonical_model(self, model: str) -> str:
|
||||
return model
|
||||
|
|
@ -22,31 +42,66 @@ class JevCompatibleDecisionsEndpoint:
|
|||
def request_model(self, model: str) -> str:
|
||||
return model
|
||||
|
||||
def endpoint_url(self, api_base: str, model: str) -> str:
|
||||
def validate_environment(self, headers: Mapping[str, str], model: str, api_key: str | None) -> dict[str, str]:
|
||||
return {
|
||||
**{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS},
|
||||
**({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}),
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def get_complete_url(self, api_base: str, model: str) -> str:
|
||||
return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}"
|
||||
|
||||
def transform_decisions_request(
|
||||
self,
|
||||
model: str,
|
||||
request: DecisionsRequest,
|
||||
custom_llm_provider: str,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"model": self.request_model(model),
|
||||
"state": request.state,
|
||||
"questions": {
|
||||
name: question.model_dump(mode="json", exclude_none=True)
|
||||
for name, question in request.questions.items()
|
||||
},
|
||||
}
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
return payload
|
||||
|
||||
def transform_decisions_response(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
raw_response: httpx.Response,
|
||||
request: DecisionsRequest,
|
||||
) -> DecisionsResponse:
|
||||
payload: Final[object] = PAYLOAD_ADAPTER.validate_json(raw_response.content)
|
||||
try:
|
||||
response: Final = _RESPONSE_ADAPTER.validate_python(self.unwrap_response(payload))
|
||||
except ValidationError as error:
|
||||
raise BaseLLMException(
|
||||
status_code=500,
|
||||
message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: {error}",
|
||||
) from error
|
||||
self.set_hidden_params(response, model, custom_llm_provider)
|
||||
return response
|
||||
|
||||
class DecisionsProviderConfig(Protocol):
|
||||
@property
|
||||
def api_key_env(self) -> tuple[str, ...]: ...
|
||||
@staticmethod
|
||||
def set_hidden_params(response: DecisionsResponse, model: str, custom_llm_provider: str) -> None:
|
||||
response.set_hidden_params(
|
||||
{
|
||||
"model": f"{custom_llm_provider}/{model}",
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"provider_response_model": f"{custom_llm_provider}/{model}",
|
||||
}
|
||||
)
|
||||
|
||||
@property
|
||||
def api_base_env(self) -> str: ...
|
||||
|
||||
@property
|
||||
def api_key_required(self) -> bool: ...
|
||||
|
||||
def default_api_base(self) -> str | None: ...
|
||||
|
||||
def missing_api_base_message(self, provider: str) -> str: ...
|
||||
|
||||
def canonical_model(self, model: str) -> str: ...
|
||||
|
||||
def request_model(self, model: str) -> str: ...
|
||||
|
||||
def endpoint_url(self, api_base: str, model: str) -> str: ...
|
||||
|
||||
def unwrap_response(self, payload: object) -> object: ...
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, str] | httpx.Headers,
|
||||
) -> BaseLLMException:
|
||||
return BaseLLMException(status_code=status_code, message=error_message, headers=headers)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.secret_managers.main import (
|
||||
get_secret_str,
|
||||
normalize_nonempty_secret_str,
|
||||
|
|
@ -12,19 +12,17 @@ from litellm.secret_managers.main import (
|
|||
_RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CloudflareDecisionsEndpoint:
|
||||
api_key_env: tuple[str, ...] = ("CLOUDFLARE_API_KEY",)
|
||||
api_base_env: str = "CLOUDFLARE_API_BASE"
|
||||
api_key_required: bool = True
|
||||
class CloudflareDecisionsConfig(BaseDecisionsConfig):
|
||||
api_key_env = ("CLOUDFLARE_API_KEY",)
|
||||
api_base_env = ("CLOUDFLARE_API_BASE",)
|
||||
|
||||
def default_api_base(self) -> str | None:
|
||||
def get_default_api_base(self) -> str | None:
|
||||
account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID"))
|
||||
if account_id is None:
|
||||
return None
|
||||
return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run"
|
||||
|
||||
def missing_api_base_message(self, provider: str) -> str:
|
||||
def missing_api_base_message(self, custom_llm_provider: str) -> str:
|
||||
return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly"
|
||||
|
||||
def canonical_model(self, model: str) -> str:
|
||||
|
|
@ -35,7 +33,7 @@ class CloudflareDecisionsEndpoint:
|
|||
def request_model(self, model: str) -> str:
|
||||
return model.rsplit("/", maxsplit=1)[-1]
|
||||
|
||||
def endpoint_url(self, api_base: str, model: str) -> str:
|
||||
def get_complete_url(self, api_base: str, model: str) -> str:
|
||||
normalized_api_base: Final = api_base.rstrip("/")
|
||||
if normalized_api_base.endswith("/ai/v1"):
|
||||
return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}"
|
||||
|
|
@ -44,15 +42,13 @@ class CloudflareDecisionsEndpoint:
|
|||
return f"{normalized_api_base}/ai/run/{model}"
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
if not isinstance(payload, Mapping):
|
||||
try:
|
||||
response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload)
|
||||
except ValidationError:
|
||||
return payload
|
||||
response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload)
|
||||
if "answers" in response_mapping:
|
||||
return payload
|
||||
result: Final = response_mapping.get("result")
|
||||
if isinstance(result, Mapping):
|
||||
return result
|
||||
return payload
|
||||
|
||||
|
||||
CLOUDFLARE_DECISIONS_ENDPOINT: Final[CloudflareDecisionsEndpoint] = CloudflareDecisionsEndpoint()
|
||||
return response_mapping
|
||||
try:
|
||||
return _RESPONSE_MAPPING_ADAPTER.validate_python(response_mapping.get("result"))
|
||||
except ValidationError:
|
||||
return response_mapping
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ from litellm.llms.base_llm.base_model_iterator import (
|
|||
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
|
||||
from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig
|
||||
from litellm.llms.base_llm.files.transformation import (
|
||||
|
|
@ -122,6 +123,7 @@ from litellm.types.containers.main import (
|
|||
ContainerObject,
|
||||
DeleteContainerResult,
|
||||
)
|
||||
from litellm.types.decisions import DecisionsRequest, DecisionsResponse
|
||||
from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
|
||||
|
|
@ -1481,6 +1483,103 @@ class BaseLLMHTTPHandler:
|
|||
request_data=request_data,
|
||||
)
|
||||
|
||||
def _prepare_decisions_request(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
logging_obj: LiteLLMLoggingObj | None,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
request: DecisionsRequest,
|
||||
api_base: str,
|
||||
api_key: str | None,
|
||||
headers: Mapping[str, str],
|
||||
) -> tuple[str, dict[str, str], dict[str, object]]:
|
||||
outbound_headers: Final = provider_config.validate_environment(headers=headers, model=model, api_key=api_key)
|
||||
url: Final = provider_config.get_complete_url(api_base=api_base, model=model)
|
||||
data: Final = provider_config.transform_decisions_request(
|
||||
model=model, request=request, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
if logging_obj is not None:
|
||||
logging_obj.pre_call(
|
||||
input=data,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
additional_args={"api_base": url, "complete_input_dict": data, "headers": outbound_headers},
|
||||
)
|
||||
return url, outbound_headers, data
|
||||
|
||||
def decisions(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
logging_obj: LiteLLMLoggingObj | None,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
request: DecisionsRequest,
|
||||
api_base: str,
|
||||
api_key: str | None,
|
||||
headers: Mapping[str, str],
|
||||
timeout: float | httpx.Timeout | None,
|
||||
client: HTTPHandler | None = None,
|
||||
) -> DecisionsResponse:
|
||||
url, outbound_headers, data = self._prepare_decisions_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
provider_config=provider_config,
|
||||
request=request,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
)
|
||||
sync_httpx_client: Final = client if client is not None else get_httpx_client()
|
||||
try:
|
||||
response: Final = sync_httpx_client.post(
|
||||
url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj
|
||||
)
|
||||
except httpx.HTTPError as e:
|
||||
raise self._handle_error(e=e, provider_config=provider_config)
|
||||
return provider_config.transform_decisions_response(
|
||||
model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request
|
||||
)
|
||||
|
||||
async def adecisions(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
logging_obj: LiteLLMLoggingObj | None,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
request: DecisionsRequest,
|
||||
api_base: str,
|
||||
api_key: str | None,
|
||||
headers: Mapping[str, str],
|
||||
timeout: float | httpx.Timeout | None,
|
||||
client: AsyncHTTPHandler | None = None,
|
||||
) -> DecisionsResponse:
|
||||
url, outbound_headers, data = self._prepare_decisions_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
provider_config=provider_config,
|
||||
request=request,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
)
|
||||
async_httpx_client: Final = (
|
||||
client
|
||||
if client is not None
|
||||
else get_async_httpx_client(llm_provider=litellm.LlmProviders(custom_llm_provider))
|
||||
)
|
||||
try:
|
||||
response: Final = await async_httpx_client.post(
|
||||
url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj
|
||||
)
|
||||
except httpx.HTTPError as e:
|
||||
raise self._handle_error(e=e, provider_config=provider_config)
|
||||
return provider_config.transform_decisions_response(
|
||||
model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request
|
||||
)
|
||||
|
||||
def _prepare_audio_transcription_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -6156,6 +6255,7 @@ class BaseLLMHTTPHandler:
|
|||
e: Exception,
|
||||
provider_config: Union[
|
||||
BaseConfig,
|
||||
BaseDecisionsConfig,
|
||||
BaseRerankConfig,
|
||||
BaseResponsesAPIConfig,
|
||||
BaseImageEditConfig,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from typing import Final
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
|
||||
|
||||
OPENROUTER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
|
||||
default_api_base_value="https://openrouter.ai/api",
|
||||
path="/alpha/decisions",
|
||||
api_key_env=("OPENROUTER_API_KEY",),
|
||||
api_base_env="OPENROUTER_API_BASE",
|
||||
)
|
||||
class OpenRouterDecisionsConfig(BaseDecisionsConfig):
|
||||
path = "/alpha/decisions"
|
||||
api_key_env = ("OPENROUTER_API_KEY",)
|
||||
api_base_env = ("OPENROUTER_API_BASE",)
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://openrouter.ai/api"
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from typing import Final
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
|
||||
|
||||
PERPLEXITY_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
|
||||
default_api_base_value="https://api.perplexity.ai",
|
||||
path="/v1/decisions",
|
||||
api_key_env=("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"),
|
||||
api_base_env="PERPLEXITY_API_BASE",
|
||||
)
|
||||
class PerplexityDecisionsConfig(BaseDecisionsConfig):
|
||||
path = "/v1/decisions"
|
||||
api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY")
|
||||
api_base_env = ("PERPLEXITY_API_BASE",)
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://api.perplexity.ai"
|
||||
|
|
|
|||
|
|
@ -1,11 +1,7 @@
|
|||
from typing import Final
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
|
||||
|
||||
STRANDS_DECIDER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
|
||||
default_api_base_value=None,
|
||||
path="/v1/systemone",
|
||||
api_key_env=("STRANDS_DECIDER_API_KEY",),
|
||||
api_base_env="STRANDS_DECIDER_API_BASE",
|
||||
api_key_required=False,
|
||||
)
|
||||
class StrandsDeciderDecisionsConfig(BaseDecisionsConfig):
|
||||
api_key_env = ("STRANDS_DECIDER_API_KEY",)
|
||||
api_base_env = ("STRANDS_DECIDER_API_BASE",)
|
||||
api_key_required = False
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
from typing import Final
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint
|
||||
|
||||
TYPESAFE_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint(
|
||||
default_api_base_value="https://api.typesafe.ai",
|
||||
path="/v1/systemone",
|
||||
api_key_env=("TYPESAFE_API_KEY",),
|
||||
api_base_env="TYPESAFE_API_BASE",
|
||||
)
|
||||
class TypeSafeDecisionsConfig(BaseDecisionsConfig):
|
||||
api_key_env = ("TYPESAFE_API_KEY",)
|
||||
api_base_env = ("TYPESAFE_API_BASE",)
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://api.typesafe.ai"
|
||||
|
|
|
|||
|
|
@ -125,3 +125,6 @@ class DecisionsResponse(LiteLLMPydanticObjectBase):
|
|||
@property
|
||||
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
|
||||
return self._hidden_params
|
||||
|
||||
def set_hidden_params(self, params: Mapping[str, object]) -> None:
|
||||
self._hidden_params.update(params)
|
||||
|
|
|
|||
|
|
@ -379,6 +379,7 @@ if TYPE_CHECKING:
|
|||
# Type stubs for lazy-loaded config classes and types
|
||||
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
|
||||
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
|
||||
from litellm.llms.base_llm.files.transformation import BaseFilesConfig
|
||||
from litellm.llms.base_llm.google_genai.transformation import (
|
||||
|
|
@ -8961,6 +8962,20 @@ class ProviderConfigManager:
|
|||
return get_dashscope_family_rerank_config(provider.value)
|
||||
return litellm.CohereRerankConfig()
|
||||
|
||||
@staticmethod
|
||||
def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None:
|
||||
if provider == LlmProviders.PERPLEXITY:
|
||||
return litellm.PerplexityDecisionsConfig()
|
||||
if provider == LlmProviders.TYPESAFE:
|
||||
return litellm.TypeSafeDecisionsConfig()
|
||||
if provider == LlmProviders.OPENROUTER:
|
||||
return litellm.OpenRouterDecisionsConfig()
|
||||
if provider == LlmProviders.CLOUDFLARE:
|
||||
return litellm.CloudflareDecisionsConfig()
|
||||
if provider == LlmProviders.STRANDS_DECIDER:
|
||||
return litellm.StrandsDeciderDecisionsConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_provider_anthropic_messages_config(
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ _PASS_THROUGH_MODEL: Final = "gpt-6-luna"
|
|||
_PASS_THROUGH_AUTHORIZATION: Final = "Bearer customer-held-upstream-key"
|
||||
_PASS_THROUGH_NEIGHBOUR: Final = "decisions-beside-a-pass-through"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_QUESTION_MAPPINGS: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 367, "output_tokens": 3}
|
||||
_STATE: Final[dict[str, JsonValue]] = {"ticket": "The export job hangs at 99%", "component": "billing"}
|
||||
_QUESTIONS: Final[dict[str, JsonValue]] = {
|
||||
|
|
@ -32,6 +33,7 @@ _QUESTIONS: Final[dict[str, JsonValue]] = {
|
|||
"severity": {"type": "choice", "criteria": {"low": "cosmetic", "high": "blocks users"}, "weight": 2},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
}
|
||||
_SDK_QUESTIONS: Final = _QUESTION_MAPPINGS.validate_python(_QUESTIONS)
|
||||
_ANSWERS: Final[dict[str, JsonValue]] = {
|
||||
"defect": {"type": "noul", "noul": 0.93},
|
||||
"severity": {"type": "choice", "choice": "high", "confidence": 0.8, "probabilities": {"low": 0.2, "high": 0.8}},
|
||||
|
|
@ -103,6 +105,11 @@ _PROVIDERS: Final = (
|
|||
),
|
||||
)
|
||||
_PERPLEXITY: Final = _PROVIDERS[0]
|
||||
_OPENROUTER: Final = _PROVIDERS[2]
|
||||
_OPENROUTER_CHAT_MODEL: Final = "openrouter/openai/gpt-5-mini"
|
||||
_UNSUPPORTED_PROVIDER_MODEL: Final = "openai/gpt-6-luna"
|
||||
_CONNECTION_ERROR: Final = "litellm.APIConnectionError"
|
||||
_GENERIC_API_ERROR: Final = "litellm.APIError"
|
||||
_INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = (
|
||||
("missing questions", {"state": _STATE}),
|
||||
("missing state", {"questions": _QUESTIONS}),
|
||||
|
|
@ -177,6 +184,12 @@ def _spend_row(call_id: str) -> dict[str, JsonValue]:
|
|||
return rows[0]
|
||||
|
||||
|
||||
def _assert_connection_error(response: httpx.Response) -> None:
|
||||
assert 500 <= response.status_code < 600, response.text
|
||||
assert _CONNECTION_ERROR in response.text, response.text
|
||||
assert _GENERIC_API_ERROR not in response.text, response.text
|
||||
|
||||
|
||||
def _free_closed_port() -> int:
|
||||
with socket.socket() as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
|
|
@ -257,10 +270,10 @@ async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway
|
|||
with gateway.scenario() as scenario:
|
||||
handle: Final = _register(scenario, _answer_body(provider))
|
||||
synchronous: Final = litellm.decisions(
|
||||
model=provider.model, state=_STATE, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY
|
||||
model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY
|
||||
)
|
||||
asynchronous: Final = await litellm.adecisions(
|
||||
model=provider.model, state=_STATE, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY
|
||||
model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY
|
||||
)
|
||||
for response in (synchronous, asynchronous):
|
||||
assert response.model_dump(mode="json") == {
|
||||
|
|
@ -441,16 +454,75 @@ def test_upstream_success_without_answers_is_a_gateway_side_server_error(gateway
|
|||
assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure"
|
||||
|
||||
|
||||
def test_unreachable_upstream_fails_only_its_own_deployment(gateway: Gateway) -> None:
|
||||
@pytest.mark.parametrize("provider", _PROVIDERS, ids=lambda provider: provider.name)
|
||||
def test_unreachable_upstream_fails_only_its_own_deployment(gateway: Gateway, provider: _Provider) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
handle: Final = _register(scenario, _answer_body(_PERPLEXITY))
|
||||
healthy: Final = _deployment(scenario, handle, _PERPLEXITY)
|
||||
handle: Final = _register(scenario, _answer_body(provider))
|
||||
healthy: Final = _deployment(scenario, handle, provider)
|
||||
dead: Final = scenario.model(
|
||||
model=_PERPLEXITY.model, api_base=f"http://127.0.0.1:{_free_closed_port()}", api_key=_API_KEY
|
||||
model=provider.model, api_base=f"http://127.0.0.1:{_free_closed_port()}", api_key=provider.api_key
|
||||
)
|
||||
failed: Final = _decide(gateway, dead, num_retries=0)
|
||||
assert 500 <= failed.status_code < 600, failed.text
|
||||
_assert_connection_error(failed)
|
||||
assert _spend_row(failed.headers["x-litellm-call-id"])["status"] == "failure"
|
||||
served: Final = _decide(gateway, healthy)
|
||||
assert served.status_code == 200, served.text
|
||||
assert len(_upstream_calls(gateway, handle)) == 1
|
||||
|
||||
|
||||
def test_an_unreachable_openrouter_deployment_reports_a_connection_error_on_chat_embeddings_and_decisions(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
dead_api_base: Final = f"http://127.0.0.1:{_free_closed_port()}"
|
||||
chat_model: Final = scenario.model(model=_OPENROUTER_CHAT_MODEL, api_base=dead_api_base, api_key=_API_KEY)
|
||||
decisions_model: Final = scenario.model(model=_OPENROUTER.model, api_base=dead_api_base, api_key=_API_KEY)
|
||||
chat: Final = _chat(gateway, chat_model, num_retries=0)
|
||||
streamed: Final = _chat(gateway, chat_model, num_retries=0, stream=True)
|
||||
embeddings: Final = gateway.request(
|
||||
"POST", "/v1/embeddings", {"model": chat_model, "input": "hi", "num_retries": 0}
|
||||
)
|
||||
decisions: Final = _decide(gateway, decisions_model, num_retries=0)
|
||||
for response in (chat, streamed, embeddings, decisions):
|
||||
_assert_connection_error(response)
|
||||
for response in (chat, decisions):
|
||||
assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure"
|
||||
|
||||
|
||||
async def test_sdk_openrouter_connection_failures_raise_a_connection_error(gateway: Gateway) -> None:
|
||||
dead_api_base: Final = f"http://127.0.0.1:{_free_closed_port()}"
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
litellm.completion(
|
||||
model=_OPENROUTER_CHAT_MODEL,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base=dead_api_base,
|
||||
api_key=_API_KEY,
|
||||
)
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
await litellm.acompletion(
|
||||
model=_OPENROUTER_CHAT_MODEL,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base=dead_api_base,
|
||||
api_key=_API_KEY,
|
||||
)
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
litellm.decisions(
|
||||
model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY
|
||||
)
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
await litellm.adecisions(
|
||||
model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY
|
||||
)
|
||||
|
||||
|
||||
def test_a_deployment_whose_provider_has_no_decisions_support_is_refused_naming_every_supported_provider(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
handle: Final = _register(scenario, _answer_body(_PERPLEXITY))
|
||||
model: Final = scenario.model(model=_UNSUPPORTED_PROVIDER_MODEL, api_base=handle.api_base(), api_key=_API_KEY)
|
||||
response: Final = _decide(gateway, model)
|
||||
assert response.status_code == 400, response.text
|
||||
for provider in _PROVIDERS:
|
||||
assert provider.name in response.text, response.text
|
||||
assert _upstream_calls(gateway, handle) == []
|
||||
|
|
|
|||
|
|
@ -322,6 +322,30 @@ model_list:
|
|||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-bedrock-mantle-key
|
||||
aws_region_name: us-east-1
|
||||
- model_name: perplexity/pplx-decider-v1-27b
|
||||
litellm_params:
|
||||
model: perplexity/pplx-decider-v1-27b
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-perplexity-key
|
||||
- model_name: typesafe/jev-1.13.0
|
||||
litellm_params:
|
||||
model: typesafe/jev-1.13.0
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-typesafe-key
|
||||
- model_name: openrouter/typesafe/jev-1.13
|
||||
litellm_params:
|
||||
model: openrouter/typesafe/jev-1.13
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-openrouter-key
|
||||
- model_name: cloudflare/@cf/cloudflare/clef
|
||||
litellm_params:
|
||||
model: cloudflare/@cf/cloudflare/clef
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-cloudflare-key
|
||||
- model_name: strands_decider/strands-decider-2B-hobson-v19
|
||||
litellm_params:
|
||||
model: strands_decider/strands-decider-2B-hobson-v19
|
||||
api_base: http://127.0.0.1:8191
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
database_url: os.environ/DATABASE_URL
|
||||
|
|
|
|||
0
tests/integration/translation/decisions/__init__.py
Normal file
0
tests/integration/translation/decisions/__init__.py
Normal file
84
tests/integration/translation/decisions/bases/cloudflare.py
Normal file
84
tests/integration/translation/decisions/bases/cloudflare.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
from typing import Final
|
||||
|
||||
from integration.translation.case import TranslationTestCase
|
||||
|
||||
"""Provider request and reply shape from https://developers.cloudflare.com/workers-ai (POST /ai/run/@cf/cloudflare/clef, reply wrapped in result). Mock reply captured live on 2026-10-06.
|
||||
"""
|
||||
CLEF_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/decisions",
|
||||
litellm_request={
|
||||
"model": "cloudflare/@cf/cloudflare/clef",
|
||||
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
||||
"questions": {
|
||||
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"instructions": "How severe is it?",
|
||||
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
||||
},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
},
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/ai/run/@cf/cloudflare/clef",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "clef",
|
||||
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
||||
"questions": {
|
||||
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"instructions": "How severe is it?",
|
||||
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
||||
},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
},
|
||||
},
|
||||
mock_provider_response={
|
||||
"result": {
|
||||
"model": "clef",
|
||||
"answers": {
|
||||
"defect": {"type": "noul", "noul": 0.9345},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"choice": "high",
|
||||
"confidence": 0.8067,
|
||||
"probabilities": {"low": 0.0509, "high": 0.9491},
|
||||
},
|
||||
"confidence": {
|
||||
"type": "score",
|
||||
"score": 0.9036,
|
||||
"confidence": 0.6515,
|
||||
"legend": {"0": "unsure", "1": "sure"},
|
||||
"probabilities": {"0": 0.0964, "1": 0.9036},
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": 290, "output_tokens": 0},
|
||||
},
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"messages": [],
|
||||
},
|
||||
expected_litellm_response={
|
||||
"model": "clef",
|
||||
"answers": {
|
||||
"defect": {"type": "noul", "noul": 0.9345},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"choice": "high",
|
||||
"confidence": 0.8067,
|
||||
"probabilities": {"low": 0.0509, "high": 0.9491},
|
||||
},
|
||||
"confidence": {
|
||||
"type": "score",
|
||||
"score": 0.9036,
|
||||
"confidence": 0.6515,
|
||||
"legend": {"0": "unsure", "1": "sure"},
|
||||
"probabilities": {"0": 0.0964, "1": 0.9036},
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": 290, "output_tokens": 0},
|
||||
},
|
||||
)
|
||||
83
tests/integration/translation/decisions/bases/openrouter.py
Normal file
83
tests/integration/translation/decisions/bases/openrouter.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
from typing import Final
|
||||
|
||||
from integration.translation.case import TranslationTestCase
|
||||
|
||||
"""Provider request and reply shape from https://openrouter.ai/docs (POST /api/alpha/decisions). Mock reply captured live on 2026-10-06.
|
||||
"""
|
||||
TYPESAFE_JEV_1_13_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/decisions",
|
||||
litellm_request={
|
||||
"model": "openrouter/typesafe/jev-1.13",
|
||||
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
||||
"questions": {
|
||||
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"instructions": "How severe is it?",
|
||||
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
||||
},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
},
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/alpha/decisions",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-openrouter-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "typesafe/jev-1.13",
|
||||
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
||||
"questions": {
|
||||
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"instructions": "How severe is it?",
|
||||
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
||||
},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
},
|
||||
},
|
||||
mock_provider_response={
|
||||
"model": "typesafe/jev-1.13-20260917",
|
||||
"answers": {
|
||||
"defect": {"type": "noul", "noul": 0.81},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"choice": "high",
|
||||
"confidence": 0.99,
|
||||
"probabilities": {"low": 0.01, "high": 0.99},
|
||||
},
|
||||
"confidence": {
|
||||
"type": "score",
|
||||
"score": 0.5,
|
||||
"confidence": 0,
|
||||
"legend": {"0": "unsure", "1": "sure"},
|
||||
"probabilities": {"0": 0.5, "1": 0.5},
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05},
|
||||
"id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki",
|
||||
"provider": "TypeSafe",
|
||||
},
|
||||
expected_litellm_response={
|
||||
"model": "typesafe/jev-1.13-20260917",
|
||||
"answers": {
|
||||
"defect": {"type": "noul", "noul": 0.81},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"choice": "high",
|
||||
"confidence": 0.99,
|
||||
"probabilities": {"low": 0.01, "high": 0.99},
|
||||
},
|
||||
"confidence": {
|
||||
"type": "score",
|
||||
"score": 0.5,
|
||||
"confidence": 0,
|
||||
"legend": {"0": "unsure", "1": "sure"},
|
||||
"probabilities": {"0": 0.5, "1": 0.5},
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05},
|
||||
"id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki",
|
||||
"provider": "TypeSafe",
|
||||
},
|
||||
)
|
||||
79
tests/integration/translation/decisions/bases/perplexity.py
Normal file
79
tests/integration/translation/decisions/bases/perplexity.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
from typing import Final
|
||||
|
||||
from integration.translation.case import TranslationTestCase
|
||||
|
||||
"""Provider request and reply shape from https://docs.perplexity.ai (POST /v1/decisions). Mock reply captured live on 2026-10-06.
|
||||
"""
|
||||
PPLX_DECIDER_V1_27B_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/decisions",
|
||||
litellm_request={
|
||||
"model": "perplexity/pplx-decider-v1-27b",
|
||||
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
||||
"questions": {
|
||||
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"instructions": "How severe is it?",
|
||||
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
||||
},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
},
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/v1/decisions",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-perplexity-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
||||
"questions": {
|
||||
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"instructions": "How severe is it?",
|
||||
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
||||
},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
},
|
||||
},
|
||||
mock_provider_response={
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"answers": {
|
||||
"defect": {"type": "noul", "noul": 0.9989100737587077},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"choice": "high",
|
||||
"confidence": 0.9964631215356778,
|
||||
"probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389},
|
||||
},
|
||||
"confidence": {
|
||||
"type": "score",
|
||||
"score": 0.07367392327139817,
|
||||
"confidence": 0.8526521534572037,
|
||||
"legend": {"0": "unsure", "1": "sure"},
|
||||
"probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817},
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": 318, "output_tokens": 3},
|
||||
},
|
||||
expected_litellm_response={
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"answers": {
|
||||
"defect": {"type": "noul", "noul": 0.9989100737587077},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"choice": "high",
|
||||
"confidence": 0.9964631215356778,
|
||||
"probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389},
|
||||
},
|
||||
"confidence": {
|
||||
"type": "score",
|
||||
"score": 0.07367392327139817,
|
||||
"confidence": 0.8526521534572037,
|
||||
"legend": {"0": "unsure", "1": "sure"},
|
||||
"probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817},
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": 318, "output_tokens": 3},
|
||||
},
|
||||
)
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
from typing import Final
|
||||
|
||||
from integration.translation.case import TranslationTestCase
|
||||
|
||||
"""Provider request and reply shape from https://github.com/strands-agents/decider (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-06.
|
||||
"""
|
||||
STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/decisions",
|
||||
litellm_request={
|
||||
"model": "strands_decider/strands-decider-2B-hobson-v19",
|
||||
"state": "Help! My payouts have been failing for 3 days!",
|
||||
"questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}},
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/v1/systemone",
|
||||
expected_provider_headers={"content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "strands-decider-2B-hobson-v19",
|
||||
"state": "Help! My payouts have been failing for 3 days!",
|
||||
"questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}},
|
||||
},
|
||||
mock_provider_response={
|
||||
"model": "strands-decider-2B-hobson-v19",
|
||||
"answers": {"is_urgent": {"type": "noul", "noul": 0.8277}},
|
||||
"usage": {"input_tokens": 86, "output_tokens": 1},
|
||||
"latency_ms": 140.03,
|
||||
},
|
||||
expected_litellm_response={
|
||||
"model": "strands-decider-2B-hobson-v19",
|
||||
"answers": {"is_urgent": {"type": "noul", "noul": 0.8277}},
|
||||
"usage": {"input_tokens": 86, "output_tokens": 1},
|
||||
"latency_ms": 140.03,
|
||||
},
|
||||
)
|
||||
79
tests/integration/translation/decisions/bases/typesafe.py
Normal file
79
tests/integration/translation/decisions/bases/typesafe.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
from typing import Final
|
||||
|
||||
from integration.translation.case import TranslationTestCase
|
||||
|
||||
"""Provider request and reply shape from https://docs.typesafe.ai (POST /v1/systemone). Mock reply captured live on 2026-10-06.
|
||||
"""
|
||||
JEV_1_13_0_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/decisions",
|
||||
litellm_request={
|
||||
"model": "typesafe/jev-1.13.0",
|
||||
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
||||
"questions": {
|
||||
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"instructions": "How severe is it?",
|
||||
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
||||
},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
},
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/v1/systemone",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-typesafe-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "jev-1.13.0",
|
||||
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
||||
"questions": {
|
||||
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"instructions": "How severe is it?",
|
||||
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
||||
},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
},
|
||||
},
|
||||
mock_provider_response={
|
||||
"model": "jev-1.13.0",
|
||||
"answers": {
|
||||
"defect": {"type": "noul", "noul": 0.78},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"choice": "high",
|
||||
"confidence": 0.99,
|
||||
"probabilities": {"low": 0.01, "high": 0.99},
|
||||
},
|
||||
"confidence": {
|
||||
"type": "score",
|
||||
"score": 0.51,
|
||||
"confidence": 0.03,
|
||||
"legend": {"0": "unsure", "1": "sure"},
|
||||
"probabilities": {"0": 0.49, "1": 0.51},
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": 377, "output_tokens": 62},
|
||||
},
|
||||
expected_litellm_response={
|
||||
"model": "jev-1.13.0",
|
||||
"answers": {
|
||||
"defect": {"type": "noul", "noul": 0.78},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"choice": "high",
|
||||
"confidence": 0.99,
|
||||
"probabilities": {"low": 0.01, "high": 0.99},
|
||||
},
|
||||
"confidence": {
|
||||
"type": "score",
|
||||
"score": 0.51,
|
||||
"confidence": 0.03,
|
||||
"legend": {"0": "unsure", "1": "sure"},
|
||||
"probabilities": {"0": 0.49, "1": 0.51},
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": 377, "output_tokens": 62},
|
||||
},
|
||||
)
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [CLEF_TEST_CASE], ids=lambda case: case.id)
|
||||
def test_decisions_basic_cloudflare(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.decisions.bases.openrouter import TYPESAFE_JEV_1_13_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [TYPESAFE_JEV_1_13_TEST_CASE], ids=lambda case: case.id)
|
||||
def test_decisions_basic_openrouter(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.decisions.bases.perplexity import PPLX_DECIDER_V1_27B_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [PPLX_DECIDER_V1_27B_TEST_CASE], ids=lambda case: case.id)
|
||||
def test_decisions_basic_perplexity(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.decisions.bases.strands_decider import STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE], ids=lambda case: case.id)
|
||||
def test_decisions_basic_strands_decider(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE], ids=lambda case: case.id)
|
||||
def test_decisions_basic_typesafe(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
@ -6,6 +6,7 @@ from collections.abc import Mapping
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
|
|
@ -252,10 +253,7 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None:
|
|||
answers={},
|
||||
usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS),
|
||||
)
|
||||
response._hidden_params = {
|
||||
"model": "perplexity/pplx-decider-v1-27b",
|
||||
"custom_llm_provider": "perplexity",
|
||||
}
|
||||
response.set_hidden_params({"model": "perplexity/pplx-decider-v1-27b", "custom_llm_provider": "perplexity"})
|
||||
|
||||
cost: Final = litellm.completion_cost(completion_response=response)
|
||||
perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"]
|
||||
|
|
@ -308,7 +306,7 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
with pytest.raises(litellm.BadRequestError, match="Supported providers"):
|
||||
with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"):
|
||||
await litellm.adecisions(
|
||||
model="unknown/jev-1.13",
|
||||
state="review",
|
||||
|
|
@ -320,17 +318,79 @@ async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRo
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_custom_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
with pytest.raises(litellm.BadRequestError, match="Supported providers"):
|
||||
async def test_provider_without_decisions_support_is_rejected_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
with pytest.raises(litellm.BadRequestError, match=r"Unknown Decisions provider 'anthropic'\. Supported providers"):
|
||||
await litellm.adecisions(
|
||||
model="anthropic/claude-sonnet-4-5",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_custom_provider_falls_back_to_the_model_prefix(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_key="caller-key",
|
||||
custom_llm_provider="",
|
||||
)
|
||||
|
||||
assert route.called
|
||||
assert json.loads(respx_mock.calls[0].request.content)["model"] == "pplx-decider-v1-27b"
|
||||
assert response._hidden_params["custom_llm_provider"] == "perplexity"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upstream_reply_without_answers_is_a_server_error(respx_mock: respx.MockRouter) -> None:
|
||||
respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(
|
||||
json={"model": "pplx-decider-v1-27b", "usage": {"input_tokens": 10, "output_tokens": 0}}
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.InternalServerError, match="unexpected response"):
|
||||
await litellm.adecisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_key="caller-key",
|
||||
custom_llm_provider="",
|
||||
)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_already_resolved(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "jev",
|
||||
"litellm_params": {
|
||||
"model": "openrouter/typesafe/jev-1.13",
|
||||
"api_key": "deployment-key",
|
||||
"api_base": "https://egress.example/openrouter",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
upstream: Final = respx_mock.post("https://egress.example/openrouter/alpha/decisions").respond(json=_RESPONSE)
|
||||
|
||||
await router.adecisions(
|
||||
model="jev",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
)
|
||||
|
||||
assert upstream.called
|
||||
assert respx_mock.calls[0].request.headers["authorization"] == "Bearer deployment-key"
|
||||
assert json.loads(respx_mock.calls[0].request.content)["model"] == "typesafe/jev-1.13"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -361,6 +421,20 @@ def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter
|
|||
)
|
||||
|
||||
|
||||
def test_unreachable_upstream_maps_to_a_connection_error(respx_mock: respx.MockRouter) -> None:
|
||||
respx_mock.post("https://api.perplexity.ai/v1/decisions").mock(
|
||||
side_effect=httpx.ConnectError("Cannot connect to host api.perplexity.ai:443")
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError, match="PerplexityException - Cannot connect to host"):
|
||||
litellm.decisions(
|
||||
model="perplexity/pplx-decider-v1-27b",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_key="caller-key",
|
||||
)
|
||||
|
||||
|
||||
def test_server_key_is_sent_to_an_explicit_api_base(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
|
|
|
|||
|
|
@ -1231,6 +1231,34 @@ def test_branchless_provider_transport_error_maps_to_api_connection_error():
|
|||
)
|
||||
|
||||
|
||||
def test_openrouter_transport_error_maps_to_api_connection_error():
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
original_exception = BaseLLMException(status_code=500, message="[Errno 111] Connection refused")
|
||||
original_exception.status_code_is_synthesized = True
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
exception_type(
|
||||
model="typesafe/jev-1.13",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="openrouter",
|
||||
)
|
||||
|
||||
|
||||
def test_openrouter_upstream_500_still_maps_to_api_error():
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
original_exception = BaseLLMException(status_code=500, message="upstream exploded")
|
||||
|
||||
with pytest.raises(litellm.APIError) as excinfo:
|
||||
exception_type(
|
||||
model="typesafe/jev-1.13",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="openrouter",
|
||||
)
|
||||
assert excinfo.value.status_code == 500
|
||||
|
||||
|
||||
def test_branchless_provider_upstream_500_still_maps_to_internal_server_error():
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue