feat(decisions): add EmpirioLabs AI as a decisions provider (#49977)

* feat(decisions): add EmpirioLabs AI as a decisions provider

empiriolabs/aplomb-1 now works with litellm.decisions and litellm.systemone, the proxy /v1/decisions and /v1/systemone routes, the decision model guardrail and the Admin UI. Requests go to POST /v1/decisions on the EmpirioLabs AI API with the System One body, keyed by EMPIRIOLABS_API_KEY with an optional EMPIRIOLABS_API_BASE

Chat models keep routing through the existing OpenAI-compatible JSON provider, and a test pins that

* ci: run the EmpirioLabs AI decisions tests in the llms unit shard

* fix: align EmpirioLabs registration with CI gates

---------

Co-authored-by: Adam Dalloul <47503782+Adam-Dalloul@users.noreply.github.com>
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-11-04 10:41:16 -08:00 • committed by GitHub
parent 0d70b9b9a1
commit b662f3f7a4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 310 additions and 2 deletions

View file

@ -280,6 +280,7 @@ jobs:
tests/unit/llms/e2b
tests/unit/llms/edenai
tests/unit/llms/elevenlabs
tests/unit/llms/empiriolabs
tests/unit/llms/exa_ai
tests/unit/llms/fal_ai
tests/unit/llms/fastcrw

View file

@ -398,6 +398,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | | | | | | | |
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
| [EmpirioLabs AI (`empiriolabs`)](https://docs.litellm.ai/docs/providers/empiriolabs) | ✅ | | ✅ | | | | | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |

View file

@ -1796,6 +1796,9 @@ if TYPE_CHECKING:
from .llms.bespoke.decisions.transformation import (
BespokeDecisionsConfig as BespokeDecisionsConfig,
)
from .llms.empiriolabs.decisions.transformation import (
EmpirioLabsDecisionsConfig as EmpirioLabsDecisionsConfig,
)
from .llms.eu_jev.decisions.transformation import (
EuJevDecisionsConfig as EuJevDecisionsConfig,
)

View file

@ -176,6 +176,7 @@ LLM_CONFIG_NAMES: Final = (
"OllamaDecisionsConfig",
"LayaDecisionsConfig",
"BespokeDecisionsConfig",
"EmpirioLabsDecisionsConfig",
"EuJevDecisionsConfig",
"OpenAIDecisionsConfig",
"AzureDecisionsConfig",
@ -759,6 +760,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
"EmpirioLabsDecisionsConfig": (".llms.empiriolabs.decisions.transformation", "EmpirioLabsDecisionsConfig"),
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
"AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),

View file

@ -0,0 +1,10 @@
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class EmpirioLabsDecisionsConfig(BaseDecisionsConfig):
path = "/v1/decisions"
api_key_env = ("EMPIRIOLABS_API_KEY",)
api_base_env = ("EMPIRIOLABS_API_BASE",)
def get_default_api_base(self) -> str | None:
return "https://api.empiriolabs.ai"

View file

@ -5277,7 +5277,7 @@
}
},
"empiriolabs": {
"display_name": "EmpirioLabs (`empiriolabs`)",
"display_name": "EmpirioLabs AI (`empiriolabs`)",
"url": "https://docs.litellm.ai/docs/providers/empiriolabs",
"endpoints": {
"chat_completions": false,

View file

@ -1553,6 +1553,34 @@
],
"default_model_placeholder": "gpt-3.5-turbo"
},
{
"provider": "EMPIRIOLABS",
"provider_display_name": "EmpirioLabs AI",
"litellm_provider": "empiriolabs",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://api.empiriolabs.ai/v1",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "empiriolabs/aplomb-1"
},
{
"provider": "EMPOWER",
"provider_display_name": "Empower",

View file

@ -4650,6 +4650,7 @@ class LlmProviders(str, Enum):
STRANDS_DECIDER = "strands_decider"
LAYA = "laya"
BESPOKE = "bespoke"
EMPIRIOLABS = "empiriolabs"
EU_JEV = "eu_jev"
MISTRAL = "mistral"
MILVUS = "milvus"

View file

@ -10296,6 +10296,7 @@ class ProviderConfigManager:
if provider == LlmProviders.BESPOKE:
return litellm.BespokeDecisionsConfig()
if provider in (
LlmProviders.EMPIRIOLABS,
LlmProviders.OPENAI,
LlmProviders.AZURE,
LlmProviders.AZURE_AI,
@ -10303,6 +10304,7 @@ class ProviderConfigManager:
LlmProviders.PARALLEL_AI,
):
configs: Final = {
LlmProviders.EMPIRIOLABS: litellm.EmpirioLabsDecisionsConfig,
LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig,
LlmProviders.AZURE: litellm.AzureDecisionsConfig,
LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig,

View file

@ -5277,7 +5277,7 @@
}
},
"empiriolabs": {
"display_name": "EmpirioLabs (`empiriolabs`)",
"display_name": "EmpirioLabs AI (`empiriolabs`)",
"url": "https://docs.litellm.ai/docs/providers/empiriolabs",
"endpoints": {
"chat_completions": false,

View file

@ -139,6 +139,7 @@ _PROVIDERS: Final = (
False,
"databricks/databricks-openjev-qwen35-4b",
),
_Provider("empiriolabs", "empiriolabs/aplomb-test", "/v1/decisions", "aplomb-test", _API_KEY, False, None),
)
_LOCAL_PROVIDERS: Final = (
pytest.param(

View file

@ -132,6 +132,7 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = (
"https://openrouter.ai/api/alpha/decisions",
"typesafe/jev-1.13",
),
("empiriolabs", "empiriolabs/aplomb-1", "https://api.empiriolabs.ai/v1/decisions", "aplomb-1"),
)
_OPENAI_RESPONSE: Final[Mapping[str, object]] = {
@ -1549,6 +1550,7 @@ _PROVIDERS_WHOSE_CHAT_MODELS_ALSO_DECIDE: Final = frozenset(
LlmProviders.LM_STUDIO.value,
LlmProviders.OLLAMA.value,
LlmProviders.OLLAMA_CHAT.value,
LlmProviders.EMPIRIOLABS.value,
}
)
_DEDICATED_DECIDER_PROVIDERS: Final[tuple[str, ...]] = tuple(
@ -3449,6 +3451,147 @@ async def test_fireworks_ai_bills_cached_input_tokens_at_the_cache_read_rate(
assert recording_logger.standard_logging_object["response_cost"] < 300 * float(prices["input_cost_per_token"])
_EMPIRIOLABS_MODEL: Final = "empiriolabs/aplomb-1"
_EMPIRIOLABS_DEFAULT_URL: Final = "https://api.empiriolabs.ai/v1/decisions"
_EMPIRIOLABS_STATE: Final = "I was charged twice for order 4411 and want a refund today."
_EMPIRIOLABS_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType(
{"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}
)
_EMPIRIOLABS_RESPONSE: Final[Mapping[str, object]] = {
"model": "aplomb-1",
"answers": {"is_refund": {"type": "noul", "noul": 0.97, "confidence": 0.94}},
"usage": {"input_tokens": 24, "output_tokens": 0},
}
@pytest.mark.parametrize(
("api_base", "environment_api_base", "url"),
(
(None, None, _EMPIRIOLABS_DEFAULT_URL),
(None, "https://environment-gateway.example/v1", "https://environment-gateway.example/v1/decisions"),
(
"https://gateway.example/v1/",
"https://environment-gateway.example/v1",
"https://gateway.example/v1/decisions",
),
("https://gateway.example", None, "https://gateway.example/v1/decisions"),
),
ids=("default_base", "environment_base", "explicit_base_over_environment", "explicit_base_without_v1"),
)
def test_empiriolabs_posts_the_systemone_body_to_v1_decisions_on_the_resolved_base(
api_base: str | None,
environment_api_base: str | None,
url: str,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False)
if environment_api_base is not None:
monkeypatch.setenv("EMPIRIOLABS_API_BASE", environment_api_base)
route: Final = respx_mock.post(url).respond(json=_EMPIRIOLABS_RESPONSE)
response: Final = litellm.systemone(
model=_EMPIRIOLABS_MODEL,
state=_EMPIRIOLABS_STATE,
questions=_EMPIRIOLABS_QUESTIONS,
api_base=api_base,
api_key="caller-key",
)
assert route.call_count == 1
assert json.loads(route.calls[0].request.content) == {
"model": "aplomb-1",
"state": _EMPIRIOLABS_STATE,
"questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}},
}
assert response.answers["is_refund"] == NoulAnswer(type="noul", noul=0.97, confidence=0.94)
assert response._hidden_params["custom_llm_provider"] == "empiriolabs"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("api_key", "authorization"),
((None, "Bearer environment-key"), ("caller-key", "Bearer caller-key")),
ids=("environment_key", "caller_key_over_environment"),
)
async def test_empiriolabs_sends_the_caller_key_or_else_the_environment_key(
api_key: str | None,
authorization: str,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False)
monkeypatch.setenv("EMPIRIOLABS_API_KEY", "environment-key")
route: Final = respx_mock.post(_EMPIRIOLABS_DEFAULT_URL).respond(json=_EMPIRIOLABS_RESPONSE)
response: Final = await litellm.adecisions(
model=_EMPIRIOLABS_MODEL,
input=_EMPIRIOLABS_STATE,
questions=[{"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"}],
api_key=api_key,
)
assert route.call_count == 1
assert route.calls[0].request.headers["authorization"] == authorization
assert isinstance(response, OpenAIDecisionResponse)
assert [answer.model_dump(mode="json") for answer in response.answers] == [
{"type": "predicate", "name": "is_refund", "probability": 0.97}
]
def test_empiriolabs_without_a_key_is_refused_before_http(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("EMPIRIOLABS_API_KEY", raising=False)
monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False)
with pytest.raises(litellm.AuthenticationError, match="Missing API key for Decisions provider 'empiriolabs'"):
litellm.systemone(model=_EMPIRIOLABS_MODEL, state=_EMPIRIOLABS_STATE, questions=_EMPIRIOLABS_QUESTIONS)
assert len(respx_mock.calls) == 0
@pytest.mark.parametrize(
("status", "error_type", "code", "exception"),
(
(401, "authentication_error", "invalid_api_key", litellm.AuthenticationError),
(402, "insufficient_credits", "insufficient_credits", litellm.PaymentRequiredError),
(404, "not_found_error", "model_not_found", litellm.NotFoundError),
(422, "invalid_request_error", "invalid_request", litellm.BadRequestError),
(429, "rate_limit_error", "rate_limit_exceeded", litellm.RateLimitError),
(503, "service_unavailable", "model_unavailable", litellm.ServiceUnavailableError),
),
)
def test_empiriolabs_upstream_errors_keep_their_status_and_message(
status: int,
error_type: str,
code: str,
exception: type[Exception],
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False)
message: Final = f"upstream refused the decision with {code}"
route: Final = respx_mock.post(_EMPIRIOLABS_DEFAULT_URL).respond(
status,
json={"error": {"message": message, "type": error_type, "code": code}},
headers={"retry-after": "1"},
)
with pytest.raises(exception, match=message) as caught:
litellm.systemone(
model=_EMPIRIOLABS_MODEL,
state=_EMPIRIOLABS_STATE,
questions=_EMPIRIOLABS_QUESTIONS,
api_key="caller-key",
num_retries=0,
)
assert getattr(caught.value, "status_code") == status
assert route.call_count == 1
_BOX_QUESTION_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = (
(
"decisions",

View file

View file

@ -0,0 +1,40 @@
from collections.abc import Mapping
import pytest
from litellm.llms.empiriolabs.decisions.transformation import EmpirioLabsDecisionsConfig
@pytest.mark.parametrize(
("environment", "connection"),
(
({}, ("https://api.empiriolabs.ai", None)),
(
{"EMPIRIOLABS_API_BASE": "https://gateway.example/v1", "EMPIRIOLABS_API_KEY": "environment-key"},
("https://gateway.example/v1", "environment-key"),
),
),
ids=("public_api_without_a_key", "environment"),
)
def test_connection_comes_from_the_environment_before_the_public_api(
environment: Mapping[str, str],
connection: tuple[str, str | None],
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False)
monkeypatch.delenv("EMPIRIOLABS_API_KEY", raising=False)
for name, value in environment.items():
monkeypatch.setenv(name, value)
assert EmpirioLabsDecisionsConfig().resolve_connection(None, None) == connection
@pytest.mark.parametrize(
"api_base",
("https://api.empiriolabs.ai", "https://api.empiriolabs.ai/v1", "https://api.empiriolabs.ai/v1/"),
)
def test_requests_go_to_v1_decisions_whether_or_not_the_base_ends_in_v1(api_base: str) -> None:
assert (
EmpirioLabsDecisionsConfig().get_complete_url(api_base=api_base, model="aplomb-1")
== "https://api.empiriolabs.ai/v1/decisions"
)

View file

@ -2,13 +2,19 @@
Unit tests for the EmpirioLabs OpenAI-like provider.
"""
import json
import os
import sys
from typing import Final
import pytest
import respx
sys.path.insert(
0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../.."))
)
import litellm
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
@ -61,3 +67,50 @@ def test_empiriolabs_complete_url_appends_endpoint():
stream=False,
)
assert url == f"{EMPIRIOLABS_BASE_URL}/chat/completions"
def test_empiriolabs_chat_models_still_route_through_the_json_provider(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("EMPIRIOLABS_API_KEY", "environment-key")
monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False)
with respx.mock() as upstream:
route: Final = upstream.post(f"{EMPIRIOLABS_BASE_URL}/chat/completions").respond(
200,
json={
"id": "chatcmpl_empiriolabs",
"object": "chat.completion",
"created": 1_789_550_000,
"model": "qwen3-7-plus",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "Hello from EmpirioLabs AI"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 4, "completion_tokens": 5, "total_tokens": 9},
},
)
response: Final = litellm.completion(
model="empiriolabs/qwen3-7-plus",
messages=[{"role": "user", "content": "Say hello"}],
max_completion_tokens=16,
)
request: Final = route.calls.last.request
assert litellm.get_llm_provider("empiriolabs/qwen3-7-plus") == (
"qwen3-7-plus",
"empiriolabs",
"environment-key",
EMPIRIOLABS_BASE_URL,
)
assert route.call_count == 1
assert request.headers["authorization"] == "Bearer environment-key"
assert json.loads(request.content) == {
"model": "qwen3-7-plus",
"messages": [{"role": "user", "content": "Say hello"}],
"max_tokens": 16,
}
assert response.choices[0].message.content == "Hello from EmpirioLabs AI"

View file

@ -529,6 +529,7 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo:
("Nace", "Nace", LlmProviders.NACE),
("PARALLEL_AI", "Parallel AI", LlmProviders.PARALLEL_AI),
("EU_JEV", "eu/jev", LlmProviders.EU_JEV),
("EMPIRIOLABS", "EmpirioLabs AI", LlmProviders.EMPIRIOLABS),
),
)
def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None:

View file

@ -252,6 +252,7 @@ describe("provider_info_helpers", () => {
Providers.DATAROBOT,
Providers.DOCKER_MODEL_RUNNER,
Providers.DOTPROMPT,
Providers.EMPIRIOLABS,
Providers.EMPOWER,
Providers.GALADRIEL,
Providers.GradientAI,
@ -616,6 +617,24 @@ describe("provider_info_helpers", () => {
]);
});
it("should list EmpirioLabs AI chat and decision models for the 'EMPIRIOLABS' provider key", () => {
const modelMap = {
"empiriolabs/aplomb-1": {
litellm_provider: "empiriolabs",
mode: "chat",
supported_endpoints: ["/v1/chat/completions", "/v1/decisions"],
},
"empiriolabs/qwen3-7-plus": { litellm_provider: "empiriolabs", mode: "chat" },
"typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" },
};
expect(getProviderModels("EMPIRIOLABS" as Providers, modelMap)).toEqual([
"empiriolabs/aplomb-1",
"empiriolabs/qwen3-7-plus",
]);
expect(getProviderLogoAndName("empiriolabs").displayName).toBe("EmpirioLabs AI");
expect(getPlaceholder("EMPIRIOLABS")).toBe("empiriolabs/aplomb-1");
});
it("should list only strands_decider models for the 'StrandsDecider' provider key", () => {
const modelMap = {
"strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "decisions" },

View file

@ -118,6 +118,7 @@ export enum Providers {
DOTPROMPT = "Dotprompt",
EDENAI = "Eden AI",
ElevenLabs = "ElevenLabs",
EMPIRIOLABS = "EmpirioLabs AI",
EMPOWER = "Empower",
FalAI = "Fal AI",
FEATHERLESS_AI = "Featherless Ai",
@ -252,6 +253,7 @@ export const provider_map: Record<string, string> = {
DOTPROMPT: "dotprompt",
EDENAI: "edenai",
ElevenLabs: "elevenlabs",
EMPIRIOLABS: "empiriolabs",
EMPOWER: "empower",
FalAI: "fal_ai",
FEATHERLESS_AI: "featherless_ai",
@ -499,6 +501,7 @@ const providerPlaceholderMap: Partial<Record<Providers, string>> = {
[Providers.Cursor]: "cursor/claude-4-sonnet",
[Providers.DeepInfra]: "deepinfra/<any-model-on-deepinfra>",
[Providers.EDENAI]: "edenai/openai/gpt-mini-latest",
[Providers.EMPIRIOLABS]: "empiriolabs/aplomb-1",
[Providers.FalAI]: "fal_ai/fal-ai/flux-pro/v1.1-ultra",
[Providers.Google_AI_Studio]: "gemini-pro",
[Providers.HOSTINGER_ROUTER]: "hostinger_router/claude-sonnet-5",