From b31ca149a6305e093d9a33737f1847ede5036776 Mon Sep 17 00:00:00 2001 From: etymologiae <342648414+etymologiae@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:45:36 -0600 Subject: [PATCH] feat(decisions): add Butler as a decisions provider --- .github/workflows/test-unit.yml | 1 + README.md | 1 + litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 2 + .../llms/butler/decisions/transformation.py | 62 +++++ litellm/llms/openai_like/providers.json | 6 + ...odel_prices_and_context_window_backup.json | 17 ++ .../provider_endpoints_support_backup.json | 26 ++ .../provider_create_fields.json | 28 ++ litellm/types/utils.py | 1 + litellm/utils.py | 2 + model_prices_and_context_window.json | 17 ++ provider_endpoints_support.json | 26 ++ tests/unit/decisions/test_main.py | 255 ++++++++++++++++++ tests/unit/llms/butler/__init__.py | 0 tests/unit/llms/butler/decisions/__init__.py | 0 .../test_butler_decisions_transformation.py | 40 +++ .../llms/openai_like/test_butler_provider.py | 124 +++++++++ .../public_endpoints/test_public_endpoints.py | 1 + .../public/assets/logos/butler.png | Bin 0 -> 7510 bytes .../components/provider_info_helpers.test.tsx | 14 + .../src/components/provider_info_helpers.tsx | 5 + 22 files changed, 631 insertions(+) create mode 100644 litellm/llms/butler/decisions/transformation.py create mode 100644 tests/unit/llms/butler/__init__.py create mode 100644 tests/unit/llms/butler/decisions/__init__.py create mode 100644 tests/unit/llms/butler/decisions/test_butler_decisions_transformation.py create mode 100644 tests/unit/llms/openai_like/test_butler_provider.py create mode 100644 ui/litellm-dashboard/public/assets/logos/butler.png diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 41198887cab..4d94ace856c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -268,6 +268,7 @@ jobs: tests/unit/llms/bigmodel tests/unit/llms/black_forest_labs tests/unit/llms/brave + tests/unit/llms/butler tests/unit/llms/bytez tests/unit/llms/cerebras tests/unit/llms/chat diff --git a/README.md b/README.md index 31a93bf8c67..dd8bc1265a7 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Azure AI (`azure_ai`)](https://docs.litellm.ai/docs/providers/azure_ai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | [Azure Text (`azure_text`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | | | [Baseten (`baseten`)](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | | | | | | | | +| [Butler (`butler`)](https://docs.litellm.ai/docs/providers/butler) | ✅ | ✅ | ✅ | | | | | | | | | [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | | | [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | | | [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index 8e5dd68a186..69f2af09d4d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1796,6 +1796,9 @@ if TYPE_CHECKING: from .llms.bespoke.decisions.transformation import ( BespokeDecisionsConfig as BespokeDecisionsConfig, ) + from .llms.butler.decisions.transformation import ( + ButlerDecisionsConfig as ButlerDecisionsConfig, + ) from .llms.empiriolabs.decisions.transformation import ( EmpirioLabsDecisionsConfig as EmpirioLabsDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index bf903e23bc0..8c45ed36af4 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -176,6 +176,7 @@ LLM_CONFIG_NAMES: Final = ( "OllamaDecisionsConfig", "LayaDecisionsConfig", "BespokeDecisionsConfig", + "ButlerDecisionsConfig", "EmpirioLabsDecisionsConfig", "EuJevDecisionsConfig", "OpenAIDecisionsConfig", @@ -760,6 +761,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"), "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), + "ButlerDecisionsConfig": (".llms.butler.decisions.transformation", "ButlerDecisionsConfig"), "EmpirioLabsDecisionsConfig": (".llms.empiriolabs.decisions.transformation", "EmpirioLabsDecisionsConfig"), "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), diff --git a/litellm/llms/butler/decisions/transformation.py b/litellm/llms/butler/decisions/transformation.py new file mode 100644 index 00000000000..0d1f86484c5 --- /dev/null +++ b/litellm/llms/butler/decisions/transformation.py @@ -0,0 +1,62 @@ +from collections.abc import Mapping +from dataclasses import replace +from types import MappingProxyType +from typing import Final + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.litellm_core_utils.core_helpers import finite_reported_cost +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.openai.decisions.transformation import ir_to_openai_request, openai_response_to_ir +from litellm.types.decisions import ( + DecisionsIRRequest, + DecisionsIRResponse, + OpenAIDecisionResponse, + UnsupportedDecisionsRequest, +) + + +class _ButlerUsage(BaseModel): + cost: float | None = None + + model_config = ConfigDict(frozen=True) + + +class _ButlerResponse(BaseModel): + usage: _ButlerUsage + + model_config = ConfigDict(frozen=True) + + +_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) +_COST_ADAPTER: Final[TypeAdapter[_ButlerResponse]] = TypeAdapter(_ButlerResponse) + + +class ButlerDecisionsConfig(BaseDecisionsConfig): + """Butler's /v1/decisions takes the OpenAI Decisions body and reports what it charged in usage.cost""" + + path = "/v1/decisions" + api_key_env = ("BUTLER_API_KEY",) + api_base_env = ("BUTLER_API_BASE",) + supports_safety_identifier = True + supports_images = True + inlines_remote_images = True + max_images = 4 + + def get_default_api_base(self) -> str | None: + return "https://api.butlercompute.com" + + def transform_decisions_request( + self, model: str, request: DecisionsIRRequest, custom_llm_provider: str + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + return ir_to_openai_request(self.request_model(model), request) + + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + response: Final = openai_response_to_ir(_RESPONSE_ADAPTER.validate_python(payload), request) + cost: Final = _COST_ADAPTER.validate_python(payload).usage.cost + if response.usage is None or cost is None: + return response + return replace(response, usage=replace(response.usage, extra=MappingProxyType({"cost": cost}))) + + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: + return finite_reported_cost(response.usage.extra.get("cost")) if response.usage is not None else None diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index f937e4d2f75..3d829c4b337 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -183,6 +183,12 @@ }, "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"] }, + "butler": { + "base_url": "https://api.butlercompute.com/v1", + "api_key_env": "BUTLER_API_KEY", + "api_base_env": "BUTLER_API_BASE", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"] + }, "meta": { "base_url": "https://api.meta.ai/v1", "api_key_env": "META_API_KEY", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ad57685da04..3db10299b6c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -74117,6 +74117,23 @@ "output_cost_per_token": 0.0, "source": "https://console.nace.ai/docs/drex/reference/models" }, + "butler/cicero": { + "cache_read_input_token_cost": 5e-09, + "input_cost_per_token": 1e-08, + "litellm_provider": "butler", + "max_input_tokens": 131072, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://butlercompute.com/models/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/decisions", + "/v1/messages", + "/v1/responses", + "/v1/systemone" + ], + "supports_vision": true + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 00fee0a0d5a..12ecd740f16 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -672,6 +672,32 @@ "generateContent": true } }, + "butler": { + "display_name": "Butler (`butler`)", + "url": "https://docs.litellm.ai/docs/providers/butler", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": true, + "a2a": true, + "generateContent": true + } + }, "bytez": { "display_name": "Bytez (`bytez`)", "url": "https://docs.litellm.ai/docs/providers/bytez", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 5ded2fa5f5e..23f3c5a956f 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -850,6 +850,34 @@ ], "default_model_placeholder": "bespoke/nimble" }, + { + "provider": "BUTLER", + "provider_display_name": "Butler", + "litellm_provider": "butler", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.butlercompute.com", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "butler/cicero" + }, { "provider": "BYTEZ", "provider_display_name": "Bytez", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 7c137b72c3e..37d7c38d90a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4657,6 +4657,7 @@ class LlmProviders(str, Enum): STRANDS_DECIDER = "strands_decider" LAYA = "laya" BESPOKE = "bespoke" + BUTLER = "butler" EMPIRIOLABS = "empiriolabs" EU_JEV = "eu_jev" MISTRAL = "mistral" diff --git a/litellm/utils.py b/litellm/utils.py index 53b02ebddac..416b4f0448c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10314,6 +10314,7 @@ class ProviderConfigManager: if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() if provider in ( + LlmProviders.BUTLER, LlmProviders.EMPIRIOLABS, LlmProviders.OPENAI, LlmProviders.AZURE, @@ -10322,6 +10323,7 @@ class ProviderConfigManager: LlmProviders.PARALLEL_AI, ): configs: Final = { + LlmProviders.BUTLER: litellm.ButlerDecisionsConfig, LlmProviders.EMPIRIOLABS: litellm.EmpirioLabsDecisionsConfig, LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig, LlmProviders.AZURE: litellm.AzureDecisionsConfig, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ad57685da04..3db10299b6c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -74117,6 +74117,23 @@ "output_cost_per_token": 0.0, "source": "https://console.nace.ai/docs/drex/reference/models" }, + "butler/cicero": { + "cache_read_input_token_cost": 5e-09, + "input_cost_per_token": 1e-08, + "litellm_provider": "butler", + "max_input_tokens": 131072, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://butlercompute.com/models/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/decisions", + "/v1/messages", + "/v1/responses", + "/v1/systemone" + ], + "supports_vision": true + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 00fee0a0d5a..12ecd740f16 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -672,6 +672,32 @@ "generateContent": true } }, + "butler": { + "display_name": "Butler (`butler`)", + "url": "https://docs.litellm.ai/docs/providers/butler", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": true, + "a2a": true, + "generateContent": true + } + }, "bytez": { "display_name": "Bytez (`bytez`)", "url": "https://docs.litellm.ai/docs/providers/bytez", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 0e2813e0a0d..851190e5bf8 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3631,6 +3631,261 @@ def test_empiriolabs_upstream_errors_keep_their_status_and_message( assert route.call_count == 1 +_BUTLER_MODEL: Final = "butler/cicero" +_BUTLER_URL: Final = "https://api.butlercompute.com/v1/decisions" +_BUTLER_COST: Final = 1.24e-06 +_BUTLER_PREDICATE: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"}, +) +_BUTLER_USAGE: Final[Mapping[str, object]] = { + "input_tokens": 188, + "input_tokens_details": {"cached_tokens": 128, "cache_write_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 189, +} +_BUTLER_RESPONSE: Final[Mapping[str, object]] = { + "id": "req_01", + "model": "cicero", + "answers": [{"type": "predicate", "name": "is_refund", "probability": 0.98}], + "usage": {**_BUTLER_USAGE, "cost": _BUTLER_COST}, +} + + +@pytest.mark.asyncio +async def test_butler_decisions_send_the_openai_body_and_bill_the_reported_cost( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + route: Final = respx_mock.post(_BUTLER_URL).respond(json=_BUTLER_RESPONSE) + recording_logger: Final = _RecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + response: Final = await litellm.adecisions( + model=_BUTLER_MODEL, + input="I was charged twice for order 4411.", + questions=_BUTLER_PREDICATE, + api_key="caller-key", + safety_identifier="user-4411", + ) + await _drain_logging_worker() + + assert json.loads(route.calls.last.request.content) == { + "model": "cicero", + "input": "I was charged twice for order 4411.", + "questions": list(_BUTLER_PREDICATE), + "safety_identifier": "user-4411", + } + + assert response.usage.input_tokens_details.cached_tokens == 128 + assert get_response_cost_from_hidden_params(response.hidden_params) == _BUTLER_COST + assert recording_logger.standard_logging_object is not None + assert recording_logger.standard_logging_object["response_cost"] == pytest.approx(_BUTLER_COST) + + +def test_butler_without_reported_cost_bills_uncached_and_cached_tokens_at_the_cost_map_rates( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + respx_mock.post(_BUTLER_URL).respond(json={**_BUTLER_RESPONSE, "usage": _BUTLER_USAGE}) + row: Final = litellm.model_cost[_BUTLER_MODEL] + + response: Final = litellm.decisions( + model=_BUTLER_MODEL, + input="I was charged twice for order 4411.", + questions=_BUTLER_PREDICATE, + api_key="caller-key", + ) + + expected: Final = (188 - 128) * row["input_cost_per_token"] + 128 * row["cache_read_input_token_cost"] + assert expected > 0 + assert response._hidden_params["response_cost"] == pytest.approx(expected) + + +def test_butler_takes_systemone_calls_in_the_openai_format_and_answers_in_system_one( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + route: Final = respx_mock.post(_BUTLER_URL).respond( + json={ + **_BUTLER_RESPONSE, + "answers": [ + {"type": "predicate", "name": "is_refund", "probability": 0.98}, + { + "type": "choice", + "name": "topic", + "choice": "billing", + "probabilities": [ + {"value": "billing", "probability": 0.9}, + {"value": "shipping", "probability": 0.1}, + ], + "confidence": 0.8, + }, + ], + } + ) + + response: Final = litellm.systemone( + model=_BUTLER_MODEL, + state="I was charged twice for order 4411.", + questions={ + "is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "topic": {"type": "choice", "instructions": "Topic?", "criteria": {"billing": None, "shipping": "parcel"}}, + }, + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "cicero", + "input": "I was charged twice for order 4411.", + "questions": [ + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"}, + { + "type": "choice", + "name": "topic", + "instructions": "Topic?", + "choices": [{"value": "billing"}, {"value": "shipping", "description": "parcel"}], + }, + ], + } + assert response.answers == { + "is_refund": NoulAnswer(type="noul", noul=0.98), + "topic": ChoiceAnswer( + type="choice", choice="billing", confidence=0.8, probabilities={"billing": 0.9, "shipping": 0.1} + ), + } + assert get_response_cost_from_hidden_params(response.hidden_params) == _BUTLER_COST + + +@pytest.mark.asyncio +@pytest.mark.parametrize("remote_image", (False, True), ids=("inline", "remote")) +async def test_butler_receives_images_inline( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + remote_image: bool, + async_only_image_fetch: AsyncOnlyImageFetch, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + image: Final = "https://images.example/receipt.png" if remote_image else async_only_image_fetch.data_url + route: Final = respx_mock.post(_BUTLER_URL).respond(json=_BUTLER_RESPONSE) + + response: Final = await litellm.adecisions( + model=_BUTLER_MODEL, + input=[ + { + "role": "user", + "content": [{"type": "input_text", "text": "receipt"}, {"type": "input_image", "image_url": image}], + } + ], + questions=_BUTLER_PREDICATE, + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content)["input"] == [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "receipt"}, + {"type": "input_image", "image_url": async_only_image_fetch.data_url}, + ], + } + ] + assert response.answers[0].probability == 0.98 + + +def test_more_images_than_butler_takes_are_refused_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="at most 4 images per request, got 5") as caught: + litellm.systemone( + model=_BUTLER_MODEL, + state="receipts", + images=[_PNG_DATA_URL] * 5, + questions={"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + api_key="caller-key", + ) + + assert caught.value.status_code == 400 + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api_key", "authorization"), + ((None, "Bearer environment-key"), ("caller-key", "Bearer caller-key")), + ids=("environment_key", "caller_key_over_environment"), +) +async def test_butler_sends_the_caller_key_or_else_the_environment_key( + api_key: str | None, + authorization: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + monkeypatch.setenv("BUTLER_API_KEY", "environment-key") + route: Final = respx_mock.post(_BUTLER_URL).respond(json=_BUTLER_RESPONSE) + + await litellm.adecisions(model=_BUTLER_MODEL, input="receipt", questions=_BUTLER_PREDICATE, api_key=api_key) + + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == authorization + + +def test_butler_without_a_key_is_refused_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_KEY", raising=False) + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + + with pytest.raises(litellm.AuthenticationError, match="Missing API key for Decisions provider 'butler'"): + litellm.decisions(model=_BUTLER_MODEL, input="receipt", questions=_BUTLER_PREDICATE) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.parametrize( + ("status", "error_type", "code", "exception"), + ( + (400, "invalid_request_error", "invalid_request", litellm.BadRequestError), + (401, "authentication_error", "invalid_api_key", litellm.AuthenticationError), + (404, "invalid_request_error", "model_not_found", litellm.NotFoundError), + (429, "rate_limit_error", "rate_limit_exceeded", litellm.RateLimitError), + (429, "rate_limit_error", "insufficient_quota", litellm.RateLimitError), + (503, "server_error", "no_healthy_workers", litellm.ServiceUnavailableError), + ), +) +def test_butler_upstream_errors_keep_their_status_and_message( + status: int, + error_type: str, + code: str, + exception: type[Exception], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + message: Final = f"upstream refused the decision with {code}" + route: Final = respx_mock.post(_BUTLER_URL).respond( + status, + json={"error": {"message": message, "type": error_type, "code": code, "param": None}}, + headers={"retry-after": "1"}, + ) + + with pytest.raises(exception, match=message) as caught: + litellm.decisions( + model=_BUTLER_MODEL, + input="receipt", + questions=_BUTLER_PREDICATE, + api_key="caller-key", + num_retries=0, + ) + + assert getattr(caught.value, "status_code") == status + assert route.call_count == 1 + + _BOX_QUESTION_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( ( "decisions", diff --git a/tests/unit/llms/butler/__init__.py b/tests/unit/llms/butler/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/butler/decisions/__init__.py b/tests/unit/llms/butler/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/butler/decisions/test_butler_decisions_transformation.py b/tests/unit/llms/butler/decisions/test_butler_decisions_transformation.py new file mode 100644 index 00000000000..603c1fa6dac --- /dev/null +++ b/tests/unit/llms/butler/decisions/test_butler_decisions_transformation.py @@ -0,0 +1,40 @@ +from collections.abc import Mapping + +import pytest + +from litellm.llms.butler.decisions.transformation import ButlerDecisionsConfig + + +@pytest.mark.parametrize( + ("environment", "connection"), + ( + ({}, ("https://api.butlercompute.com", None)), + ( + {"BUTLER_API_BASE": "https://gateway.example/v1", "BUTLER_API_KEY": "environment-key"}, + ("https://gateway.example/v1", "environment-key"), + ), + ), + ids=("public_api_without_a_key", "environment"), +) +def test_connection_comes_from_the_environment_before_the_public_api( + environment: Mapping[str, str], + connection: tuple[str, str | None], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + monkeypatch.delenv("BUTLER_API_KEY", raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + + assert ButlerDecisionsConfig().resolve_connection(None, None) == connection + + +@pytest.mark.parametrize( + "api_base", + ("https://api.butlercompute.com", "https://api.butlercompute.com/v1", "https://api.butlercompute.com/v1/"), +) +def test_requests_go_to_v1_decisions_whether_or_not_the_base_ends_in_v1(api_base: str) -> None: + assert ( + ButlerDecisionsConfig().get_complete_url(api_base=api_base, model="cicero") + == "https://api.butlercompute.com/v1/decisions" + ) diff --git a/tests/unit/llms/openai_like/test_butler_provider.py b/tests/unit/llms/openai_like/test_butler_provider.py new file mode 100644 index 00000000000..669e20d46fa --- /dev/null +++ b/tests/unit/llms/openai_like/test_butler_provider.py @@ -0,0 +1,124 @@ +import json +from typing import Final + +import pytest +import respx + +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache + +_BUTLER_BASE_URL: Final = "https://api.butlercompute.com/v1" + + +def test_butler_chat_models_route_through_the_json_provider(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BUTLER_API_KEY", "environment-key") + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + + with respx.mock() as upstream: + route: Final = upstream.post(f"{_BUTLER_BASE_URL}/chat/completions").respond( + 200, + json={ + "id": "chatcmpl-req_01", + "object": "chat.completion", + "created": 1_793_940_103, + "model": "cicero", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from Butler"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 12, "completion_tokens": 4, "total_tokens": 16}, + }, + ) + response: Final = litellm.completion( + model="butler/cicero", + messages=[{"role": "user", "content": "Say hello"}], + max_completion_tokens=16, + ) + + request: Final = route.calls.last.request + assert litellm.get_llm_provider("butler/cicero") == ("cicero", "butler", "environment-key", _BUTLER_BASE_URL) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer environment-key" + assert json.loads(request.content) == { + "model": "cicero", + "messages": [{"role": "user", "content": "Say hello"}], + "max_completion_tokens": 16, + } + assert response.choices[0].message.content == "Hello from Butler" + + +def test_butler_api_base_from_the_environment_replaces_the_public_api(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BUTLER_API_BASE", "https://gateway.example/v1") + + assert litellm.get_llm_provider("butler/cicero", api_key="caller-key") == ( + "cicero", + "butler", + "caller-key", + "https://gateway.example/v1", + ) + + +def test_butler_responses_requests_go_to_butler_natively() -> None: + with respx.mock() as upstream: + route: Final = upstream.post(f"{_BUTLER_BASE_URL}/responses").respond( + 200, + json={ + "id": "resp_req_01", + "object": "response", + "created_at": 1_793_941_697, + "model": "cicero", + "status": "completed", + "output": [ + { + "id": "msg_req_01", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Hello from Butler", "annotations": []}], + } + ], + "usage": {"input_tokens": 12, "output_tokens": 4, "total_tokens": 16}, + }, + ) + response: Final = litellm.responses(model="butler/cicero", input="Say hello", api_key="caller-key") + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert request.headers["authorization"] == "Bearer caller-key" + assert (body["model"], body["input"]) == ("cicero", "Say hello") + assert response.output[0].content[0].text == "Hello from Butler" + + +@pytest.mark.asyncio +async def test_butler_anthropic_messages_requests_go_to_butler_natively(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + with respx.mock() as upstream: + route: Final = upstream.post(f"{_BUTLER_BASE_URL}/messages").respond( + 200, + json={ + "id": "msg_req_01", + "type": "message", + "role": "assistant", + "model": "cicero", + "content": [{"type": "text", "text": "Hello from Butler"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 4}, + }, + ) + response: Final = await litellm.anthropic.messages.acreate( + model="butler/cicero", + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=32, + api_key="caller-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert request.headers["authorization"] == "Bearer caller-key" + assert (body["model"], body["messages"]) == ("cicero", [{"role": "user", "content": "Say hello"}]) + assert response["content"][0]["text"] == "Hello from Butler" diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index d1afcf80ec9..e198a58b278 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -532,6 +532,7 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: ("PARALLEL_AI", "Parallel AI", LlmProviders.PARALLEL_AI), ("EU_JEV", "eu/jev", LlmProviders.EU_JEV), ("EMPIRIOLABS", "EmpirioLabs AI", LlmProviders.EMPIRIOLABS), + ("BUTLER", "Butler", LlmProviders.BUTLER), ), ) def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None: diff --git a/ui/litellm-dashboard/public/assets/logos/butler.png b/ui/litellm-dashboard/public/assets/logos/butler.png new file mode 100644 index 0000000000000000000000000000000000000000..a67373b656198790f3d1c689c5470c82f4482e23 GIT binary patch literal 7510 zcmdscXH=8f*Do`U0wP9GR6vZ1h*Ctr&wdcTL)OZ(&)MaivwwU4&hvyB=)-uA z3LWL(;NZQdbLSxk$8YcVe_X!<5)dOZnSQ!oXgh9x3svVxub2wpT>2bHYDFNzpQogesKO4j|2jdldG+HP3LVq^Mv-b zdzhFLIvLuo=Prbe{V={7m3>o&N+ixJEt5rj6Bzxe>hh}czU}?B@~g?Kds8zD!>Ox- zv#DjQ`NR=*hY7@a4z6Vh5J%7j^lvBL1RcJ0jQ=!;tPYCf>;I2}cSv97ozeDaX-ipL z)pVo340~OeswY))x+FnVKbnL~rCweX5j2ylnwsm02os~oY`uu*hUmJg=0z@WU2Jp=3T^PEQ36U75|9WmXBEob=Q1bwNA zq-<_@k-;AW$#2Cfluo;~H3qFWrh?7g*0(JyUrtuKSS6)JR@C`DQwX@hWKC>ZECq{z z#c+M2sMya{L2)ToV_u<~!^O>@KNtQe0@2#(PlkC*zD_jyTn%>9_Oc7mTa@{;vTjl% zZLmUy!XB|2X06?>cA+dXH|(j5n_L?D4WE&1t6^da@*7J$F!fgTNq!SdK%~o$5R{F_ zfc1ism(B}!cFBg*XxUn~M%6Ei^cp41v+>CIAwTvyV?uLi>#yN>KwXADds3lpyfdRe~VE%6X{>~Cf^8Hv-}3yUJeX-}1;smv~Ut4~YjU-`|?C4Wrm=<%1BaN-U1 zMpJtI zVguNo)6IDO^clps`dyUhsPFOWpH7ndTe7vNiGLChslo~9LTp6?H)b0hX{D(HFD7d0MZ;>C_D5rMvW_Bo;mupzHs2;Qc4 zxKLm1)6QpCqN@}WnKxnAb=Ph$W$B#8Wj|)HEhi{_D)3R>k1i{4S0FH3lz5pq@xwZI zuh3|<6f3{$c4?GZ9x}dI$1c?v7@#XfS-ZNRRwqS!^EZ3MHn^_lz6q7J{m9dwhHMZr2zIafsQrrm zL-o3)5KBZH;+0#t4$B}>mY3Kf#R#djxPF-OysC7=jPLWp?%LLinPOE~?#%2~s9fUU zxXN+v$a_|BE)oAYDTvc!W8x*zNAR<3TustI8rW*QDm2JWmuaRorDIb1+F51#&!U>1 zx1wh7ge~cY6mpTb4yws#wvn>nZJo}>`PLJ`&W!Yd*y{77Ar#7B;<4EI2W_U$K~RV9RI7y9MhYyLaTEg&XQd zaTefIznmwxipoN|L@rC&62#m?r?(g9nfXSl5A2Hy!VWW3-}K&~ya6vqkQuW;V23KPQRjFw#yKnmi+gRfO%m6A3-3_ypH1mPWb^Y>W+-~S~ekdGP|^lizB zaj>&uxh#G-IqA1?2J$W#%B&1Ii#Anp@zWfxmMC8#Y#VLI%N0qSW=@UxTtb_gd29&I z4-#59%Zb;XwVOK7qz^_$-*^6)?%p0elIlglqo%S0XT$uh=O#9F8B9)Mx1(RTR~luW z>WMthEW=~`NXPy90D%h0f775g)~$AK4{PGiaDf+5I!*Ps`F%SZF-~G z;zX_g?`?WLO$XY*flI2cGJO`4KV=vkKzZN9Z7o}xQc@(VNS+^8;;fmuS=aw;`Na}h z(nmU0PuU~9)HX937z8p~ePhmWuW2ydx8Qh5CyY%O3fp@d|XvnxPSZ0|kFAk-|!- z#F(l4;beSd9vAZ0u1SDN8f7~eH-{DdLDo0kX(PW>qyf;Mk zlr2#2AL->OMqk^d&gb!54J(EFmh_n{_wkiKj+keueqv}!?8MrU;eJw zf3uJ}%m+t#CJwph4WXjRL2baICI(Wj*tCQNAD=)A#(dI`MA7*b4gp!h-&??E&PGjH zKfydrShxAg53eX~0Gh2=yR^1rV#(FJ^`)^1lCijilXs*_n7}h{%R_eDm~kWWgD4IK zGC7<1`@ilGWsr<-KFIgZgYv#obt*S6+XMwS-G8J>(W?a>E}=6^J|f?5JVb_u+&Rae zRpMTJf!98HGJPh=|gLoua0D#Zi zfko?w+imcX75stTH_d#DF(~>IkK)vGq}cB5X*e?E36BAOK^s`xFPoC3I%wp#14Glc zBWHJL!7iVi=6%|l6rFvy3Lnf9)aZ7Tk>3#y9K3IYF;N)qokvC{l*MaQTQ$(PFZvdSI`Bh#LpTG4}qu z0LB|o5Xhe$E~!W`D{`JuC|07oqQzP=T&MlZF^+^YGyyv4EP5Ttb1B1TTz&_w#%kOD z4lf)G9AdXPZ7hP`Jjz<&gMHbjmc(7?_c;QrIABCgLp{wp;50BPHT?GE?H|2E#!dja z8rf%*EXQ$Kjffz-OnJ>F(CFfS$RRQSypBO#E4Zya;qZYZWzc}BVA4rbC=BW}rIAa6 z!l!s9=_Y3S^*U2w{-;&kZ;fUgWH$Y+nd*vU);+9?oaECGG^ zQ0T+!$kMZRW@eR+yB`Wih^_i2`df7A2wOrviQEZc6^%QdMH>hO1IFiA@vxWww6nC&kote;e4R*pum^JZ>U>#ykamX;jYJqPS_(gP zK=|t9c66M;&Ta2<97NwietY|ci|6Ip%TgRR?@1hQHS2%BaD@9`3BCxz=ABr5oAJ^@ z9YZ(weyn+n)a8bh?1vyw-s^V+*uv7ARWC|^id5{jj<=jC$~QmY`i=7*XOYymq(Dg60RK12?k` zt=HD`Dg%dR$E?Kw5b4$y6Bjv?fX*6Rx3Y89qTxq@Th+iffMdbzqpxa^p}Y?EAm0C= zPrljSO!3>fn|mvo>#$UX>le_1sy)R@?!zjt7XAys_j^gDQ{}F|akrox=(^RN0L2`& zx${5aO+e?&7^yfty7f;g<)eE9vD}gCXGDj$2P7-}Fj%YGMM5y;gUHusjJ~T8Tv)8i z+TZ7KG}4W}lRN31E;VlKnEi_xP^M_pMzyep%-G?zt{ZK7RKL$|$K)B(z?QA9gZ#@f zzBE&Jx z2Y|&eD53r21=z&H*+xGq*>aA!u>ELjY{aEzyp-oUQc-{ikp#&7_r>h6#(7BVw_gI} zzjSG&cd{S=BFZ2hzI`4GzY4dvD9C6+j+<%Oy&wO1PErIo)DoNlu9fmoGm@Bw<>muW z4_uX`Fc#ldkEF{G{!5e{A_jji#m76OK_kro*DYB(|M!nWF2kSqjAF<^G?E;E#wIf@ zQ~pL#m!++oU1(NFPITe|aJ{-3tn%CU+2r$ac`M|gHQOt9Q3t}OyoJIDQk$@o6hz9O zMw0TFIi7eVwJ;x#QcR?iF4=7uQq=bnqWN$+{;YZlCxU-a!nq(TlOyP;8DwI$MkD|)Ggo80#KPK z5Ps*)?*Fp!VXp8u_Pbs6Dr*~$*(z1GNInFROhzb=2aj~PlklnhYN5m4g1g{U4 z=l!&r5+5{b5aQ=<=^KU2t_jO{cT+I7$!7Pi1z0pJN{;k09K7QQW4>w1-dA16lTXU; zJUG`kj^+Y@xEfzEQ`%Ln%!JmiFQi-k3bo=4^mM^=3&O?mkxm$~oGp*YXIAP=@!71; zQ`kZA;f1HYe^ffB7}$a-O&M9yx|gMRYAdlLh#(#X#HraQyx8^)fw+SERZvii4K!Az z9W;>MAYM(5eyxXSw5R*f1Ln`LYP{2iXCr$s!~JD!rkCJ@o}v?glN`K$YcU?zEh<&L zZJ-Hkc_h@_pYe|x@X`jES7obG5@8j1Z}y5qkbJ#=T!6hj67hXH$W9TAVU@-e36IyF zKcfk-a)D&i_~qx89(of|yl|RQ4dUAW*oh5I^q8+wvT0dKXEVMGi<){Tiw( z|0w92x>9x=Vee?));R0ivphRbyvkcfoVcVw(NI_)U^&2hNOwz#S%Kyg8iTf;i0N}8 zXkK~n%v4OY_dpc1s&)WHp5Q-ze^>L>o2~+SV>ehA+fc;NH(5zNNrX1>Wk*gL2m&pJVhF zJ^7P9`@gYRTr60uJ^i9mHKLp>$`fN;KQbOjkTZK$! zklAJnBK%R-#d?hpk+MFtr3Z467X?C@IBpDOIp0%O4%P=6z(Cdaq{fB zMM(J8s#b!IeM=67>{8&gnC_m&)@>O6wX)Jfn|OjLn5C)blrE_HtimwT(q$*KpTVX+ z>%l(kNDP?I-H|9%UnOCJSJ8IDyVoQyD?nY0t@$Cli>cM9T0qz=M0WTSaT|E!ZtM?> zGj02D)_3JuUQ&#Nu>g#h?K^p0ff6`ZB{G~blRUjRe*g9Qq { expect(getPlaceholder("PARALLEL_AI")).toBe("parallel_ai/decision"); }); + it("lists Butler decision models and resolves the provider name and model placeholder", () => { + const modelMap = { + "butler/cicero": { litellm_provider: "butler", mode: "evaluation" }, + "nace/drex-latest": { litellm_provider: "nace", mode: "evaluation" }, + }; + expect(getProviderModels("BUTLER", modelMap)).toEqual(["butler/cicero"]); + expect(getProviderLogoAndName("butler")).toEqual({ + displayName: "Butler", + logo: providerLogoMap[Providers.BUTLER], + }); + expect(providerLogoMap[Providers.BUTLER]).toContain("butler"); + expect(getPlaceholder("BUTLER")).toBe("butler/cicero"); + }); + it("lists Nace decision models and resolves the provider name and model placeholder", () => { const modelMap = { "nace/drex-latest": { litellm_provider: "nace", mode: "evaluation" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 167343fc05e..c739fb28614 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -8,6 +8,7 @@ import assemblyaiSmallLogo from "../../public/assets/logos/assemblyai_small.png" import basetenLogo from "../../public/assets/logos/baseten.svg"; import bedrockLogo from "../../public/assets/logos/bedrock.svg"; import bigmodelLogo from "../../public/assets/logos/bigmodel.png"; +import butlerLogo from "../../public/assets/logos/butler.png"; import cerebrasLogo from "../../public/assets/logos/cerebras.svg"; import cloudflareLogo from "../../public/assets/logos/cloudflare.svg"; import cohereLogo from "../../public/assets/logos/cohere.svg"; @@ -95,6 +96,7 @@ export enum Providers { AZURE_TEXT = "Azure Text", BASETEN = "Baseten", Bespoke = "Bespoke Nimble", + BUTLER = "Butler", BYTEZ = "Bytez", Cerebras = "Cerebras", CHATGPT = "ChatGPT Subscription", @@ -230,6 +232,7 @@ export const provider_map: Record = { Bedrock: "bedrock", BedrockMantle: "bedrock_mantle", Bespoke: "bespoke", + BUTLER: "butler", BYTEZ: "bytez", Cerebras: "cerebras", CHATGPT: "chatgpt", @@ -370,6 +373,7 @@ export const providerLogoMap: Partial> = { [Providers.BASETEN]: basetenLogo.src, [Providers.Bedrock]: bedrockLogo.src, [Providers.BedrockMantle]: bedrockLogo.src, + [Providers.BUTLER]: butlerLogo.src, [Providers.SageMaker]: bedrockLogo.src, [Providers.Cerebras]: cerebrasLogo.src, [Providers.CHATGPT]: openaiSmallLogo.src, @@ -515,6 +519,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Sail]: "sail/openai/gpt-oss-120b", [Providers.SCX_AI]: "scx-ai/GLM-5.2", [Providers.Bespoke]: "bespoke/nimble", + [Providers.BUTLER]: "butler/cicero", [Providers.Laya]: "laya/english", [Providers.Snowflake]: "snowflake/mistral-7b", [Providers.StrandsDecider]: "strands_decider/strands-decider-2B-hobson-v19",