scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge b31ca149a6 into 72228db5c6

Browse source
This commit is contained in:
etymologiae 2026-11-07 05:46:16 +00:00 • committed by GitHub
commit c1ce583bb2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 631 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/test-unit.yml vendored
View file

@ -268,6 +268,7 @@ jobs:
tests/unit/llms/bigmodel
tests/unit/llms/black_forest_labs
tests/unit/llms/brave
tests/unit/llms/butler
tests/unit/llms/bytez
tests/unit/llms/cerebras
tests/unit/llms/chat

1
README.md
View file

@ -376,6 +376,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Azure AI (`azure_ai`)](https://docs.litellm.ai/docs/providers/azure_ai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [Azure Text (`azure_text`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Baseten (`baseten`)](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | | | | | | | |
| [Butler (`butler`)](https://docs.litellm.ai/docs/providers/butler) | ✅ | ✅ | ✅ | | | | | | | |
| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | |
| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | |
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |

3
litellm/__init__.py
View file

@ -1796,6 +1796,9 @@ if TYPE_CHECKING:
from .llms.bespoke.decisions.transformation import (
BespokeDecisionsConfig as BespokeDecisionsConfig,
)
from .llms.butler.decisions.transformation import (
ButlerDecisionsConfig as ButlerDecisionsConfig,
)
from .llms.empiriolabs.decisions.transformation import (
EmpirioLabsDecisionsConfig as EmpirioLabsDecisionsConfig,
)

2
litellm/_lazy_imports_registry.py
View file

@ -176,6 +176,7 @@ LLM_CONFIG_NAMES: Final = (
"OllamaDecisionsConfig",
"LayaDecisionsConfig",
"BespokeDecisionsConfig",
"ButlerDecisionsConfig",
"EmpirioLabsDecisionsConfig",
"EuJevDecisionsConfig",
"OpenAIDecisionsConfig",
@ -760,6 +761,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
"ButlerDecisionsConfig": (".llms.butler.decisions.transformation", "ButlerDecisionsConfig"),
"EmpirioLabsDecisionsConfig": (".llms.empiriolabs.decisions.transformation", "EmpirioLabsDecisionsConfig"),
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),

62
litellm/llms/butler/decisions/transformation.py Normal file
View file

@ -0,0 +1,62 @@
from collections.abc import Mapping
from dataclasses import replace
from types import MappingProxyType
from typing import Final
from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
from litellm.llms.openai.decisions.transformation import ir_to_openai_request, openai_response_to_ir
from litellm.types.decisions import (
DecisionsIRRequest,
DecisionsIRResponse,
OpenAIDecisionResponse,
UnsupportedDecisionsRequest,
)
class _ButlerUsage(BaseModel):
cost: float | None = None
model_config = ConfigDict(frozen=True)
class _ButlerResponse(BaseModel):
usage: _ButlerUsage
model_config = ConfigDict(frozen=True)
_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse)
_COST_ADAPTER: Final[TypeAdapter[_ButlerResponse]] = TypeAdapter(_ButlerResponse)
class ButlerDecisionsConfig(BaseDecisionsConfig):
"""Butler's /v1/decisions takes the OpenAI Decisions body and reports what it charged in usage.cost"""
path = "/v1/decisions"
api_key_env = ("BUTLER_API_KEY",)
api_base_env = ("BUTLER_API_BASE",)
supports_safety_identifier = True
supports_images = True
inlines_remote_images = True
max_images = 4
def get_default_api_base(self) -> str | None:
return "https://api.butlercompute.com"
def transform_decisions_request(
self, model: str, request: DecisionsIRRequest, custom_llm_provider: str
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
return ir_to_openai_request(self.request_model(model), request)
def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
response: Final = openai_response_to_ir(_RESPONSE_ADAPTER.validate_python(payload), request)
cost: Final = _COST_ADAPTER.validate_python(payload).usage.cost
if response.usage is None or cost is None:
return response
return replace(response, usage=replace(response.usage, extra=MappingProxyType({"cost": cost})))
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
return finite_reported_cost(response.usage.extra.get("cost")) if response.usage is not None else None

6
litellm/llms/openai_like/providers.json
View file

@ -183,6 +183,12 @@
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
},
"butler": {
"base_url": "https://api.butlercompute.com/v1",
"api_key_env": "BUTLER_API_KEY",
"api_base_env": "BUTLER_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
},
"meta": {
"base_url": "https://api.meta.ai/v1",
"api_key_env": "META_API_KEY",

17
litellm/model_prices_and_context_window_backup.json
View file

@ -74235,6 +74235,23 @@
"output_cost_per_token": 0.0,
"source": "https://console.nace.ai/docs/drex/reference/models"
},
"butler/cicero": {
"cache_read_input_token_cost": 5e-09,
"input_cost_per_token": 1e-08,
"litellm_provider": "butler",
"max_input_tokens": 131072,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://butlercompute.com/models/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/decisions",
"/v1/messages",
"/v1/responses",
"/v1/systemone"
],
"supports_vision": true
},
"typesafe/jev-1.13.0": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "typesafe",

26
litellm/provider_endpoints_support_backup.json
View file

@ -672,6 +672,32 @@
"generateContent": true
}
},
"butler": {
"display_name": "Butler (`butler`)",
"url": "https://docs.litellm.ai/docs/providers/butler",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": true,
"a2a": true,
"generateContent": true
}
},
"bytez": {
"display_name": "Bytez (`bytez`)",
"url": "https://docs.litellm.ai/docs/providers/bytez",

28
litellm/proxy/public_endpoints/provider_create_fields.json
View file

@ -850,6 +850,34 @@
],
"default_model_placeholder": "bespoke/nimble"
},
{
"provider": "BUTLER",
"provider_display_name": "Butler",
"litellm_provider": "butler",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://api.butlercompute.com",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "butler/cicero"
},
{
"provider": "BYTEZ",
"provider_display_name": "Bytez",

1
litellm/types/utils.py
View file

@ -4670,6 +4670,7 @@ class LlmProviders(str, Enum):
STRANDS_DECIDER = "strands_decider"
LAYA = "laya"
BESPOKE = "bespoke"
BUTLER = "butler"
EMPIRIOLABS = "empiriolabs"
EU_JEV = "eu_jev"
MISTRAL = "mistral"

2
litellm/utils.py
View file

@ -10345,6 +10345,7 @@ class ProviderConfigManager:
if provider == LlmProviders.BESPOKE:
return litellm.BespokeDecisionsConfig()
if provider in (
LlmProviders.BUTLER,
LlmProviders.EMPIRIOLABS,
LlmProviders.OPENAI,
LlmProviders.AZURE,
@ -10353,6 +10354,7 @@ class ProviderConfigManager:
LlmProviders.PARALLEL_AI,
):
configs: Final = {
LlmProviders.BUTLER: litellm.ButlerDecisionsConfig,
LlmProviders.EMPIRIOLABS: litellm.EmpirioLabsDecisionsConfig,
LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig,
LlmProviders.AZURE: litellm.AzureDecisionsConfig,

17
model_prices_and_context_window.json
View file

@ -74235,6 +74235,23 @@
"output_cost_per_token": 0.0,
"source": "https://console.nace.ai/docs/drex/reference/models"
},
"butler/cicero": {
"cache_read_input_token_cost": 5e-09,
"input_cost_per_token": 1e-08,
"litellm_provider": "butler",
"max_input_tokens": 131072,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://butlercompute.com/models/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/decisions",
"/v1/messages",
"/v1/responses",
"/v1/systemone"
],
"supports_vision": true
},
"typesafe/jev-1.13.0": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "typesafe",

26
provider_endpoints_support.json
View file

@ -672,6 +672,32 @@
"generateContent": true
}
},
"butler": {
"display_name": "Butler (`butler`)",
"url": "https://docs.litellm.ai/docs/providers/butler",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": true,
"a2a": true,
"generateContent": true
}
},
"bytez": {
"display_name": "Bytez (`bytez`)",
"url": "https://docs.litellm.ai/docs/providers/bytez",

255
tests/unit/decisions/test_main.py
View file

@ -3631,6 +3631,261 @@ def test_empiriolabs_upstream_errors_keep_their_status_and_message(
assert route.call_count == 1
_BUTLER_MODEL: Final = "butler/cicero"
_BUTLER_URL: Final = "https://api.butlercompute.com/v1/decisions"
_BUTLER_COST: Final = 1.24e-06
_BUTLER_PREDICATE: Final[tuple[Mapping[str, object], ...]] = (
{"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"},
)
_BUTLER_USAGE: Final[Mapping[str, object]] = {
"input_tokens": 188,
"input_tokens_details": {"cached_tokens": 128, "cache_write_tokens": 0},
"output_tokens": 1,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 189,
}
_BUTLER_RESPONSE: Final[Mapping[str, object]] = {
"id": "req_01",
"model": "cicero",
"answers": [{"type": "predicate", "name": "is_refund", "probability": 0.98}],
"usage": {**_BUTLER_USAGE, "cost": _BUTLER_COST},
}
@pytest.mark.asyncio
async def test_butler_decisions_send_the_openai_body_and_bill_the_reported_cost(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("BUTLER_API_BASE", raising=False)
route: Final = respx_mock.post(_BUTLER_URL).respond(json=_BUTLER_RESPONSE)
recording_logger: Final = _RecordingLogger()
monkeypatch.setattr(litellm, "callbacks", [recording_logger])
response: Final = await litellm.adecisions(
model=_BUTLER_MODEL,
input="I was charged twice for order 4411.",
questions=_BUTLER_PREDICATE,
api_key="caller-key",
safety_identifier="user-4411",
)
await _drain_logging_worker()
assert json.loads(route.calls.last.request.content) == {
"model": "cicero",
"input": "I was charged twice for order 4411.",
"questions": list(_BUTLER_PREDICATE),
"safety_identifier": "user-4411",
}
assert response.usage.input_tokens_details.cached_tokens == 128
assert get_response_cost_from_hidden_params(response.hidden_params) == _BUTLER_COST
assert recording_logger.standard_logging_object is not None
assert recording_logger.standard_logging_object["response_cost"] == pytest.approx(_BUTLER_COST)
def test_butler_without_reported_cost_bills_uncached_and_cached_tokens_at_the_cost_map_rates(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("BUTLER_API_BASE", raising=False)
respx_mock.post(_BUTLER_URL).respond(json={**_BUTLER_RESPONSE, "usage": _BUTLER_USAGE})
row: Final = litellm.model_cost[_BUTLER_MODEL]
response: Final = litellm.decisions(
model=_BUTLER_MODEL,
input="I was charged twice for order 4411.",
questions=_BUTLER_PREDICATE,
api_key="caller-key",
)
expected: Final = (188 - 128) * row["input_cost_per_token"] + 128 * row["cache_read_input_token_cost"]
assert expected > 0
assert response._hidden_params["response_cost"] == pytest.approx(expected)
def test_butler_takes_systemone_calls_in_the_openai_format_and_answers_in_system_one(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("BUTLER_API_BASE", raising=False)
route: Final = respx_mock.post(_BUTLER_URL).respond(
json={
**_BUTLER_RESPONSE,
"answers": [
{"type": "predicate", "name": "is_refund", "probability": 0.98},
{
"type": "choice",
"name": "topic",
"choice": "billing",
"probabilities": [
{"value": "billing", "probability": 0.9},
{"value": "shipping", "probability": 0.1},
],
"confidence": 0.8,
},
],
}
)
response: Final = litellm.systemone(
model=_BUTLER_MODEL,
state="I was charged twice for order 4411.",
questions={
"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"},
"topic": {"type": "choice", "instructions": "Topic?", "criteria": {"billing": None, "shipping": "parcel"}},
},
api_key="caller-key",
)
assert json.loads(route.calls.last.request.content) == {
"model": "cicero",
"input": "I was charged twice for order 4411.",
"questions": [
{"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"},
{
"type": "choice",
"name": "topic",
"instructions": "Topic?",
"choices": [{"value": "billing"}, {"value": "shipping", "description": "parcel"}],
},
],
}
assert response.answers == {
"is_refund": NoulAnswer(type="noul", noul=0.98),
"topic": ChoiceAnswer(
type="choice", choice="billing", confidence=0.8, probabilities={"billing": 0.9, "shipping": 0.1}
),
}
assert get_response_cost_from_hidden_params(response.hidden_params) == _BUTLER_COST
@pytest.mark.asyncio
@pytest.mark.parametrize("remote_image", (False, True), ids=("inline", "remote"))
async def test_butler_receives_images_inline(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
remote_image: bool,
async_only_image_fetch: AsyncOnlyImageFetch,
) -> None:
monkeypatch.delenv("BUTLER_API_BASE", raising=False)
image: Final = "https://images.example/receipt.png" if remote_image else async_only_image_fetch.data_url
route: Final = respx_mock.post(_BUTLER_URL).respond(json=_BUTLER_RESPONSE)
response: Final = await litellm.adecisions(
model=_BUTLER_MODEL,
input=[
{
"role": "user",
"content": [{"type": "input_text", "text": "receipt"}, {"type": "input_image", "image_url": image}],
}
],
questions=_BUTLER_PREDICATE,
api_key="caller-key",
)
assert json.loads(route.calls.last.request.content)["input"] == [
{
"type": "message",
"role": "user",
"content": [
{"type": "input_text", "text": "receipt"},
{"type": "input_image", "image_url": async_only_image_fetch.data_url},
],
}
]
assert response.answers[0].probability == 0.98
def test_more_images_than_butler_takes_are_refused_before_http(respx_mock: respx.MockRouter) -> None:
with pytest.raises(litellm.BadRequestError, match="at most 4 images per request, got 5") as caught:
litellm.systemone(
model=_BUTLER_MODEL,
state="receipts",
images=[_PNG_DATA_URL] * 5,
questions={"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}},
api_key="caller-key",
)
assert caught.value.status_code == 400
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
@pytest.mark.parametrize(
("api_key", "authorization"),
((None, "Bearer environment-key"), ("caller-key", "Bearer caller-key")),
ids=("environment_key", "caller_key_over_environment"),
)
async def test_butler_sends_the_caller_key_or_else_the_environment_key(
api_key: str | None,
authorization: str,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("BUTLER_API_BASE", raising=False)
monkeypatch.setenv("BUTLER_API_KEY", "environment-key")
route: Final = respx_mock.post(_BUTLER_URL).respond(json=_BUTLER_RESPONSE)
await litellm.adecisions(model=_BUTLER_MODEL, input="receipt", questions=_BUTLER_PREDICATE, api_key=api_key)
assert route.call_count == 1
assert route.calls[0].request.headers["authorization"] == authorization
def test_butler_without_a_key_is_refused_before_http(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("BUTLER_API_KEY", raising=False)
monkeypatch.delenv("BUTLER_API_BASE", raising=False)
with pytest.raises(litellm.AuthenticationError, match="Missing API key for Decisions provider 'butler'"):
litellm.decisions(model=_BUTLER_MODEL, input="receipt", questions=_BUTLER_PREDICATE)
assert len(respx_mock.calls) == 0
@pytest.mark.parametrize(
("status", "error_type", "code", "exception"),
(
(400, "invalid_request_error", "invalid_request", litellm.BadRequestError),
(401, "authentication_error", "invalid_api_key", litellm.AuthenticationError),
(404, "invalid_request_error", "model_not_found", litellm.NotFoundError),
(429, "rate_limit_error", "rate_limit_exceeded", litellm.RateLimitError),
(429, "rate_limit_error", "insufficient_quota", litellm.RateLimitError),
(503, "server_error", "no_healthy_workers", litellm.ServiceUnavailableError),
),
)
def test_butler_upstream_errors_keep_their_status_and_message(
status: int,
error_type: str,
code: str,
exception: type[Exception],
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.delenv("BUTLER_API_BASE", raising=False)
message: Final = f"upstream refused the decision with {code}"
route: Final = respx_mock.post(_BUTLER_URL).respond(
status,
json={"error": {"message": message, "type": error_type, "code": code, "param": None}},
headers={"retry-after": "1"},
)
with pytest.raises(exception, match=message) as caught:
litellm.decisions(
model=_BUTLER_MODEL,
input="receipt",
questions=_BUTLER_PREDICATE,
api_key="caller-key",
num_retries=0,
)
assert getattr(caught.value, "status_code") == status
assert route.call_count == 1
_BOX_QUESTION_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = (
(
"decisions",

0
tests/unit/llms/butler/__init__.py Normal file
View file

0
tests/unit/llms/butler/decisions/__init__.py Normal file
View file

40
tests/unit/llms/butler/decisions/test_butler_decisions_transformation.py Normal file
View file

@ -0,0 +1,40 @@
from collections.abc import Mapping
import pytest
from litellm.llms.butler.decisions.transformation import ButlerDecisionsConfig
@pytest.mark.parametrize(
("environment", "connection"),
(
({}, ("https://api.butlercompute.com", None)),
(
{"BUTLER_API_BASE": "https://gateway.example/v1", "BUTLER_API_KEY": "environment-key"},
("https://gateway.example/v1", "environment-key"),
),
),
ids=("public_api_without_a_key", "environment"),
)
def test_connection_comes_from_the_environment_before_the_public_api(
environment: Mapping[str, str],
connection: tuple[str, str | None],
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("BUTLER_API_BASE", raising=False)
monkeypatch.delenv("BUTLER_API_KEY", raising=False)
for name, value in environment.items():
monkeypatch.setenv(name, value)
assert ButlerDecisionsConfig().resolve_connection(None, None) == connection
@pytest.mark.parametrize(
"api_base",
("https://api.butlercompute.com", "https://api.butlercompute.com/v1", "https://api.butlercompute.com/v1/"),
)
def test_requests_go_to_v1_decisions_whether_or_not_the_base_ends_in_v1(api_base: str) -> None:
assert (
ButlerDecisionsConfig().get_complete_url(api_base=api_base, model="cicero")
== "https://api.butlercompute.com/v1/decisions"
)

124
tests/unit/llms/openai_like/test_butler_provider.py Normal file
View file

@ -0,0 +1,124 @@
import json
from typing import Final
import pytest
import respx
import litellm
from litellm.caching.llm_caching_handler import LLMClientCache
_BUTLER_BASE_URL: Final = "https://api.butlercompute.com/v1"
def test_butler_chat_models_route_through_the_json_provider(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("BUTLER_API_KEY", "environment-key")
monkeypatch.delenv("BUTLER_API_BASE", raising=False)
with respx.mock() as upstream:
route: Final = upstream.post(f"{_BUTLER_BASE_URL}/chat/completions").respond(
200,
json={
"id": "chatcmpl-req_01",
"object": "chat.completion",
"created": 1_793_940_103,
"model": "cicero",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "Hello from Butler"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 12, "completion_tokens": 4, "total_tokens": 16},
},
)
response: Final = litellm.completion(
model="butler/cicero",
messages=[{"role": "user", "content": "Say hello"}],
max_completion_tokens=16,
)
request: Final = route.calls.last.request
assert litellm.get_llm_provider("butler/cicero") == ("cicero", "butler", "environment-key", _BUTLER_BASE_URL)
assert route.call_count == 1
assert request.headers["authorization"] == "Bearer environment-key"
assert json.loads(request.content) == {
"model": "cicero",
"messages": [{"role": "user", "content": "Say hello"}],
"max_completion_tokens": 16,
}
assert response.choices[0].message.content == "Hello from Butler"
def test_butler_api_base_from_the_environment_replaces_the_public_api(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("BUTLER_API_BASE", "https://gateway.example/v1")
assert litellm.get_llm_provider("butler/cicero", api_key="caller-key") == (
"cicero",
"butler",
"caller-key",
"https://gateway.example/v1",
)
def test_butler_responses_requests_go_to_butler_natively() -> None:
with respx.mock() as upstream:
route: Final = upstream.post(f"{_BUTLER_BASE_URL}/responses").respond(
200,
json={
"id": "resp_req_01",
"object": "response",
"created_at": 1_793_941_697,
"model": "cicero",
"status": "completed",
"output": [
{
"id": "msg_req_01",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "Hello from Butler", "annotations": []}],
}
],
"usage": {"input_tokens": 12, "output_tokens": 4, "total_tokens": 16},
},
)
response: Final = litellm.responses(model="butler/cicero", input="Say hello", api_key="caller-key")
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert request.headers["authorization"] == "Bearer caller-key"
assert (body["model"], body["input"]) == ("cicero", "Say hello")
assert response.output[0].content[0].text == "Hello from Butler"
@pytest.mark.asyncio
async def test_butler_anthropic_messages_requests_go_to_butler_natively(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
with respx.mock() as upstream:
route: Final = upstream.post(f"{_BUTLER_BASE_URL}/messages").respond(
200,
json={
"id": "msg_req_01",
"type": "message",
"role": "assistant",
"model": "cicero",
"content": [{"type": "text", "text": "Hello from Butler"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 4},
},
)
response: Final = await litellm.anthropic.messages.acreate(
model="butler/cicero",
messages=[{"role": "user", "content": "Say hello"}],
max_tokens=32,
api_key="caller-key",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert request.headers["authorization"] == "Bearer caller-key"
assert (body["model"], body["messages"]) == ("cicero", [{"role": "user", "content": "Say hello"}])
assert response["content"][0]["text"] == "Hello from Butler"

1
tests/unit/proxy/public_endpoints/test_public_endpoints.py
View file

@ -532,6 +532,7 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo:
("PARALLEL_AI", "Parallel AI", LlmProviders.PARALLEL_AI),
("EU_JEV", "eu/jev", LlmProviders.EU_JEV),
("EMPIRIOLABS", "EmpirioLabs AI", LlmProviders.EMPIRIOLABS),
("BUTLER", "Butler", LlmProviders.BUTLER),
),
)
def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None:

BIN
ui/litellm-dashboard/public/assets/logos/butler.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 7.3 KiB

14
ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx
View file

@ -606,6 +606,20 @@ describe("provider_info_helpers", () => {
expect(getPlaceholder("PARALLEL_AI")).toBe("parallel_ai/decision");
});
it("lists Butler decision models and resolves the provider name and model placeholder", () => {
const modelMap = {
"butler/cicero": { litellm_provider: "butler", mode: "evaluation" },
"nace/drex-latest": { litellm_provider: "nace", mode: "evaluation" },
};
expect(getProviderModels("BUTLER", modelMap)).toEqual(["butler/cicero"]);
expect(getProviderLogoAndName("butler")).toEqual({
displayName: "Butler",
logo: providerLogoMap[Providers.BUTLER],
});
expect(providerLogoMap[Providers.BUTLER]).toContain("butler");
expect(getPlaceholder("BUTLER")).toBe("butler/cicero");
});
it("lists Nace decision models and resolves the provider name and model placeholder", () => {
const modelMap = {
"nace/drex-latest": { litellm_provider: "nace", mode: "evaluation" },

5
ui/litellm-dashboard/src/components/provider_info_helpers.tsx
View file

@ -8,6 +8,7 @@ import assemblyaiSmallLogo from "../../public/assets/logos/assemblyai_small.png"
import basetenLogo from "../../public/assets/logos/baseten.svg";
import bedrockLogo from "../../public/assets/logos/bedrock.svg";
import bigmodelLogo from "../../public/assets/logos/bigmodel.png";
import butlerLogo from "../../public/assets/logos/butler.png";
import cerebrasLogo from "../../public/assets/logos/cerebras.svg";
import cloudflareLogo from "../../public/assets/logos/cloudflare.svg";
import cohereLogo from "../../public/assets/logos/cohere.svg";
@ -96,6 +97,7 @@ export enum Providers {
AZURE_TEXT = "Azure Text",
BASETEN = "Baseten",
Bespoke = "Bespoke Nimble",
BUTLER = "Butler",
BYTEZ = "Bytez",
Cerebras = "Cerebras",
CHATGPT = "ChatGPT Subscription",
@ -232,6 +234,7 @@ export const provider_map: Record<string, string> = {
Bedrock: "bedrock",
BedrockMantle: "bedrock_mantle",
Bespoke: "bespoke",
BUTLER: "butler",
BYTEZ: "bytez",
Cerebras: "cerebras",
CHATGPT: "chatgpt",
@ -373,6 +376,7 @@ export const providerLogoMap: Partial<Record<Providers, string>> = {
[Providers.BASETEN]: basetenLogo.src,
[Providers.Bedrock]: bedrockLogo.src,
[Providers.BedrockMantle]: bedrockLogo.src,
[Providers.BUTLER]: butlerLogo.src,
[Providers.SageMaker]: bedrockLogo.src,
[Providers.Cerebras]: cerebrasLogo.src,
[Providers.CHATGPT]: openaiSmallLogo.src,
@ -520,6 +524,7 @@ const providerPlaceholderMap: Partial<Record<Providers, string>> = {
[Providers.Sail]: "sail/openai/gpt-oss-120b",
[Providers.SCX_AI]: "scx-ai/GLM-5.2",
[Providers.Bespoke]: "bespoke/nimble",
[Providers.BUTLER]: "butler/cicero",
[Providers.Laya]: "laya/english",
[Providers.Snowflake]: "snowflake/mistral-7b",
[Providers.StrandsDecider]: "strands_decider/strands-decider-2B-hobson-v19",