diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index fb29a92e306..e5c59946beb 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -268,6 +268,7 @@ jobs: tests/unit/llms/bigmodel tests/unit/llms/black_forest_labs tests/unit/llms/brave + tests/unit/llms/butler tests/unit/llms/bytez tests/unit/llms/cerebras tests/unit/llms/chat diff --git a/README.md b/README.md index 31a93bf8c67..dd8bc1265a7 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Azure AI (`azure_ai`)](https://docs.litellm.ai/docs/providers/azure_ai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | [Azure Text (`azure_text`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | | | [Baseten (`baseten`)](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | | | | | | | | +| [Butler (`butler`)](https://docs.litellm.ai/docs/providers/butler) | ✅ | ✅ | ✅ | | | | | | | | | [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | | | [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | | | [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index 8e5dd68a186..69f2af09d4d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1796,6 +1796,9 @@ if TYPE_CHECKING: from .llms.bespoke.decisions.transformation import ( BespokeDecisionsConfig as BespokeDecisionsConfig, ) + from .llms.butler.decisions.transformation import ( + ButlerDecisionsConfig as ButlerDecisionsConfig, + ) from .llms.empiriolabs.decisions.transformation import ( EmpirioLabsDecisionsConfig as EmpirioLabsDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index bf903e23bc0..8c45ed36af4 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -176,6 +176,7 @@ LLM_CONFIG_NAMES: Final = ( "OllamaDecisionsConfig", "LayaDecisionsConfig", "BespokeDecisionsConfig", + "ButlerDecisionsConfig", "EmpirioLabsDecisionsConfig", "EuJevDecisionsConfig", "OpenAIDecisionsConfig", @@ -760,6 +761,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"), "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), + "ButlerDecisionsConfig": (".llms.butler.decisions.transformation", "ButlerDecisionsConfig"), "EmpirioLabsDecisionsConfig": (".llms.empiriolabs.decisions.transformation", "EmpirioLabsDecisionsConfig"), "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), diff --git a/litellm/llms/butler/decisions/transformation.py b/litellm/llms/butler/decisions/transformation.py new file mode 100644 index 00000000000..0d1f86484c5 --- /dev/null +++ b/litellm/llms/butler/decisions/transformation.py @@ -0,0 +1,62 @@ +from collections.abc import Mapping +from dataclasses import replace +from types import MappingProxyType +from typing import Final + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.litellm_core_utils.core_helpers import finite_reported_cost +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.openai.decisions.transformation import ir_to_openai_request, openai_response_to_ir +from litellm.types.decisions import ( + DecisionsIRRequest, + DecisionsIRResponse, + OpenAIDecisionResponse, + UnsupportedDecisionsRequest, +) + + +class _ButlerUsage(BaseModel): + cost: float | None = None + + model_config = ConfigDict(frozen=True) + + +class _ButlerResponse(BaseModel): + usage: _ButlerUsage + + model_config = ConfigDict(frozen=True) + + +_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) +_COST_ADAPTER: Final[TypeAdapter[_ButlerResponse]] = TypeAdapter(_ButlerResponse) + + +class ButlerDecisionsConfig(BaseDecisionsConfig): + """Butler's /v1/decisions takes the OpenAI Decisions body and reports what it charged in usage.cost""" + + path = "/v1/decisions" + api_key_env = ("BUTLER_API_KEY",) + api_base_env = ("BUTLER_API_BASE",) + supports_safety_identifier = True + supports_images = True + inlines_remote_images = True + max_images = 4 + + def get_default_api_base(self) -> str | None: + return "https://api.butlercompute.com" + + def transform_decisions_request( + self, model: str, request: DecisionsIRRequest, custom_llm_provider: str + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + return ir_to_openai_request(self.request_model(model), request) + + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + response: Final = openai_response_to_ir(_RESPONSE_ADAPTER.validate_python(payload), request) + cost: Final = _COST_ADAPTER.validate_python(payload).usage.cost + if response.usage is None or cost is None: + return response + return replace(response, usage=replace(response.usage, extra=MappingProxyType({"cost": cost}))) + + def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: + return finite_reported_cost(response.usage.extra.get("cost")) if response.usage is not None else None diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index f937e4d2f75..3d829c4b337 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -183,6 +183,12 @@ }, "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"] }, + "butler": { + "base_url": "https://api.butlercompute.com/v1", + "api_key_env": "BUTLER_API_KEY", + "api_base_env": "BUTLER_API_BASE", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"] + }, "meta": { "base_url": "https://api.meta.ai/v1", "api_key_env": "META_API_KEY", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 43c321f16a3..89ae787a61d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -74235,6 +74235,23 @@ "output_cost_per_token": 0.0, "source": "https://console.nace.ai/docs/drex/reference/models" }, + "butler/cicero": { + "cache_read_input_token_cost": 5e-09, + "input_cost_per_token": 1e-08, + "litellm_provider": "butler", + "max_input_tokens": 131072, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://butlercompute.com/models/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/decisions", + "/v1/messages", + "/v1/responses", + "/v1/systemone" + ], + "supports_vision": true + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 00fee0a0d5a..12ecd740f16 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -672,6 +672,32 @@ "generateContent": true } }, + "butler": { + "display_name": "Butler (`butler`)", + "url": "https://docs.litellm.ai/docs/providers/butler", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": true, + "a2a": true, + "generateContent": true + } + }, "bytez": { "display_name": "Bytez (`bytez`)", "url": "https://docs.litellm.ai/docs/providers/bytez", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 5afc3908820..a39ef161d71 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -850,6 +850,34 @@ ], "default_model_placeholder": "bespoke/nimble" }, + { + "provider": "BUTLER", + "provider_display_name": "Butler", + "litellm_provider": "butler", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.butlercompute.com", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "butler/cicero" + }, { "provider": "BYTEZ", "provider_display_name": "Bytez", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3ddeb118c33..244f4bcfc16 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4670,6 +4670,7 @@ class LlmProviders(str, Enum): STRANDS_DECIDER = "strands_decider" LAYA = "laya" BESPOKE = "bespoke" + BUTLER = "butler" EMPIRIOLABS = "empiriolabs" EU_JEV = "eu_jev" MISTRAL = "mistral" diff --git a/litellm/utils.py b/litellm/utils.py index 5ec089a106a..70ec265e219 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10345,6 +10345,7 @@ class ProviderConfigManager: if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() if provider in ( + LlmProviders.BUTLER, LlmProviders.EMPIRIOLABS, LlmProviders.OPENAI, LlmProviders.AZURE, @@ -10353,6 +10354,7 @@ class ProviderConfigManager: LlmProviders.PARALLEL_AI, ): configs: Final = { + LlmProviders.BUTLER: litellm.ButlerDecisionsConfig, LlmProviders.EMPIRIOLABS: litellm.EmpirioLabsDecisionsConfig, LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig, LlmProviders.AZURE: litellm.AzureDecisionsConfig, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 43c321f16a3..89ae787a61d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -74235,6 +74235,23 @@ "output_cost_per_token": 0.0, "source": "https://console.nace.ai/docs/drex/reference/models" }, + "butler/cicero": { + "cache_read_input_token_cost": 5e-09, + "input_cost_per_token": 1e-08, + "litellm_provider": "butler", + "max_input_tokens": 131072, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://butlercompute.com/models/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/decisions", + "/v1/messages", + "/v1/responses", + "/v1/systemone" + ], + "supports_vision": true + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 00fee0a0d5a..12ecd740f16 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -672,6 +672,32 @@ "generateContent": true } }, + "butler": { + "display_name": "Butler (`butler`)", + "url": "https://docs.litellm.ai/docs/providers/butler", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": true, + "a2a": true, + "generateContent": true + } + }, "bytez": { "display_name": "Bytez (`bytez`)", "url": "https://docs.litellm.ai/docs/providers/bytez", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 0e2813e0a0d..851190e5bf8 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3631,6 +3631,261 @@ def test_empiriolabs_upstream_errors_keep_their_status_and_message( assert route.call_count == 1 +_BUTLER_MODEL: Final = "butler/cicero" +_BUTLER_URL: Final = "https://api.butlercompute.com/v1/decisions" +_BUTLER_COST: Final = 1.24e-06 +_BUTLER_PREDICATE: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"}, +) +_BUTLER_USAGE: Final[Mapping[str, object]] = { + "input_tokens": 188, + "input_tokens_details": {"cached_tokens": 128, "cache_write_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 189, +} +_BUTLER_RESPONSE: Final[Mapping[str, object]] = { + "id": "req_01", + "model": "cicero", + "answers": [{"type": "predicate", "name": "is_refund", "probability": 0.98}], + "usage": {**_BUTLER_USAGE, "cost": _BUTLER_COST}, +} + + +@pytest.mark.asyncio +async def test_butler_decisions_send_the_openai_body_and_bill_the_reported_cost( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + route: Final = respx_mock.post(_BUTLER_URL).respond(json=_BUTLER_RESPONSE) + recording_logger: Final = _RecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + response: Final = await litellm.adecisions( + model=_BUTLER_MODEL, + input="I was charged twice for order 4411.", + questions=_BUTLER_PREDICATE, + api_key="caller-key", + safety_identifier="user-4411", + ) + await _drain_logging_worker() + + assert json.loads(route.calls.last.request.content) == { + "model": "cicero", + "input": "I was charged twice for order 4411.", + "questions": list(_BUTLER_PREDICATE), + "safety_identifier": "user-4411", + } + + assert response.usage.input_tokens_details.cached_tokens == 128 + assert get_response_cost_from_hidden_params(response.hidden_params) == _BUTLER_COST + assert recording_logger.standard_logging_object is not None + assert recording_logger.standard_logging_object["response_cost"] == pytest.approx(_BUTLER_COST) + + +def test_butler_without_reported_cost_bills_uncached_and_cached_tokens_at_the_cost_map_rates( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + respx_mock.post(_BUTLER_URL).respond(json={**_BUTLER_RESPONSE, "usage": _BUTLER_USAGE}) + row: Final = litellm.model_cost[_BUTLER_MODEL] + + response: Final = litellm.decisions( + model=_BUTLER_MODEL, + input="I was charged twice for order 4411.", + questions=_BUTLER_PREDICATE, + api_key="caller-key", + ) + + expected: Final = (188 - 128) * row["input_cost_per_token"] + 128 * row["cache_read_input_token_cost"] + assert expected > 0 + assert response._hidden_params["response_cost"] == pytest.approx(expected) + + +def test_butler_takes_systemone_calls_in_the_openai_format_and_answers_in_system_one( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + route: Final = respx_mock.post(_BUTLER_URL).respond( + json={ + **_BUTLER_RESPONSE, + "answers": [ + {"type": "predicate", "name": "is_refund", "probability": 0.98}, + { + "type": "choice", + "name": "topic", + "choice": "billing", + "probabilities": [ + {"value": "billing", "probability": 0.9}, + {"value": "shipping", "probability": 0.1}, + ], + "confidence": 0.8, + }, + ], + } + ) + + response: Final = litellm.systemone( + model=_BUTLER_MODEL, + state="I was charged twice for order 4411.", + questions={ + "is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "topic": {"type": "choice", "instructions": "Topic?", "criteria": {"billing": None, "shipping": "parcel"}}, + }, + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "cicero", + "input": "I was charged twice for order 4411.", + "questions": [ + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"}, + { + "type": "choice", + "name": "topic", + "instructions": "Topic?", + "choices": [{"value": "billing"}, {"value": "shipping", "description": "parcel"}], + }, + ], + } + assert response.answers == { + "is_refund": NoulAnswer(type="noul", noul=0.98), + "topic": ChoiceAnswer( + type="choice", choice="billing", confidence=0.8, probabilities={"billing": 0.9, "shipping": 0.1} + ), + } + assert get_response_cost_from_hidden_params(response.hidden_params) == _BUTLER_COST + + +@pytest.mark.asyncio +@pytest.mark.parametrize("remote_image", (False, True), ids=("inline", "remote")) +async def test_butler_receives_images_inline( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + remote_image: bool, + async_only_image_fetch: AsyncOnlyImageFetch, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + image: Final = "https://images.example/receipt.png" if remote_image else async_only_image_fetch.data_url + route: Final = respx_mock.post(_BUTLER_URL).respond(json=_BUTLER_RESPONSE) + + response: Final = await litellm.adecisions( + model=_BUTLER_MODEL, + input=[ + { + "role": "user", + "content": [{"type": "input_text", "text": "receipt"}, {"type": "input_image", "image_url": image}], + } + ], + questions=_BUTLER_PREDICATE, + api_key="caller-key", + ) + + assert json.loads(route.calls.last.request.content)["input"] == [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "receipt"}, + {"type": "input_image", "image_url": async_only_image_fetch.data_url}, + ], + } + ] + assert response.answers[0].probability == 0.98 + + +def test_more_images_than_butler_takes_are_refused_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="at most 4 images per request, got 5") as caught: + litellm.systemone( + model=_BUTLER_MODEL, + state="receipts", + images=[_PNG_DATA_URL] * 5, + questions={"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + api_key="caller-key", + ) + + assert caught.value.status_code == 400 + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api_key", "authorization"), + ((None, "Bearer environment-key"), ("caller-key", "Bearer caller-key")), + ids=("environment_key", "caller_key_over_environment"), +) +async def test_butler_sends_the_caller_key_or_else_the_environment_key( + api_key: str | None, + authorization: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + monkeypatch.setenv("BUTLER_API_KEY", "environment-key") + route: Final = respx_mock.post(_BUTLER_URL).respond(json=_BUTLER_RESPONSE) + + await litellm.adecisions(model=_BUTLER_MODEL, input="receipt", questions=_BUTLER_PREDICATE, api_key=api_key) + + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == authorization + + +def test_butler_without_a_key_is_refused_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_KEY", raising=False) + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + + with pytest.raises(litellm.AuthenticationError, match="Missing API key for Decisions provider 'butler'"): + litellm.decisions(model=_BUTLER_MODEL, input="receipt", questions=_BUTLER_PREDICATE) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.parametrize( + ("status", "error_type", "code", "exception"), + ( + (400, "invalid_request_error", "invalid_request", litellm.BadRequestError), + (401, "authentication_error", "invalid_api_key", litellm.AuthenticationError), + (404, "invalid_request_error", "model_not_found", litellm.NotFoundError), + (429, "rate_limit_error", "rate_limit_exceeded", litellm.RateLimitError), + (429, "rate_limit_error", "insufficient_quota", litellm.RateLimitError), + (503, "server_error", "no_healthy_workers", litellm.ServiceUnavailableError), + ), +) +def test_butler_upstream_errors_keep_their_status_and_message( + status: int, + error_type: str, + code: str, + exception: type[Exception], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + message: Final = f"upstream refused the decision with {code}" + route: Final = respx_mock.post(_BUTLER_URL).respond( + status, + json={"error": {"message": message, "type": error_type, "code": code, "param": None}}, + headers={"retry-after": "1"}, + ) + + with pytest.raises(exception, match=message) as caught: + litellm.decisions( + model=_BUTLER_MODEL, + input="receipt", + questions=_BUTLER_PREDICATE, + api_key="caller-key", + num_retries=0, + ) + + assert getattr(caught.value, "status_code") == status + assert route.call_count == 1 + + _BOX_QUESTION_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( ( "decisions", diff --git a/tests/unit/llms/butler/__init__.py b/tests/unit/llms/butler/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/butler/decisions/__init__.py b/tests/unit/llms/butler/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/butler/decisions/test_butler_decisions_transformation.py b/tests/unit/llms/butler/decisions/test_butler_decisions_transformation.py new file mode 100644 index 00000000000..603c1fa6dac --- /dev/null +++ b/tests/unit/llms/butler/decisions/test_butler_decisions_transformation.py @@ -0,0 +1,40 @@ +from collections.abc import Mapping + +import pytest + +from litellm.llms.butler.decisions.transformation import ButlerDecisionsConfig + + +@pytest.mark.parametrize( + ("environment", "connection"), + ( + ({}, ("https://api.butlercompute.com", None)), + ( + {"BUTLER_API_BASE": "https://gateway.example/v1", "BUTLER_API_KEY": "environment-key"}, + ("https://gateway.example/v1", "environment-key"), + ), + ), + ids=("public_api_without_a_key", "environment"), +) +def test_connection_comes_from_the_environment_before_the_public_api( + environment: Mapping[str, str], + connection: tuple[str, str | None], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + monkeypatch.delenv("BUTLER_API_KEY", raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + + assert ButlerDecisionsConfig().resolve_connection(None, None) == connection + + +@pytest.mark.parametrize( + "api_base", + ("https://api.butlercompute.com", "https://api.butlercompute.com/v1", "https://api.butlercompute.com/v1/"), +) +def test_requests_go_to_v1_decisions_whether_or_not_the_base_ends_in_v1(api_base: str) -> None: + assert ( + ButlerDecisionsConfig().get_complete_url(api_base=api_base, model="cicero") + == "https://api.butlercompute.com/v1/decisions" + ) diff --git a/tests/unit/llms/openai_like/test_butler_provider.py b/tests/unit/llms/openai_like/test_butler_provider.py new file mode 100644 index 00000000000..669e20d46fa --- /dev/null +++ b/tests/unit/llms/openai_like/test_butler_provider.py @@ -0,0 +1,124 @@ +import json +from typing import Final + +import pytest +import respx + +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache + +_BUTLER_BASE_URL: Final = "https://api.butlercompute.com/v1" + + +def test_butler_chat_models_route_through_the_json_provider(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BUTLER_API_KEY", "environment-key") + monkeypatch.delenv("BUTLER_API_BASE", raising=False) + + with respx.mock() as upstream: + route: Final = upstream.post(f"{_BUTLER_BASE_URL}/chat/completions").respond( + 200, + json={ + "id": "chatcmpl-req_01", + "object": "chat.completion", + "created": 1_793_940_103, + "model": "cicero", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from Butler"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 12, "completion_tokens": 4, "total_tokens": 16}, + }, + ) + response: Final = litellm.completion( + model="butler/cicero", + messages=[{"role": "user", "content": "Say hello"}], + max_completion_tokens=16, + ) + + request: Final = route.calls.last.request + assert litellm.get_llm_provider("butler/cicero") == ("cicero", "butler", "environment-key", _BUTLER_BASE_URL) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer environment-key" + assert json.loads(request.content) == { + "model": "cicero", + "messages": [{"role": "user", "content": "Say hello"}], + "max_completion_tokens": 16, + } + assert response.choices[0].message.content == "Hello from Butler" + + +def test_butler_api_base_from_the_environment_replaces_the_public_api(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BUTLER_API_BASE", "https://gateway.example/v1") + + assert litellm.get_llm_provider("butler/cicero", api_key="caller-key") == ( + "cicero", + "butler", + "caller-key", + "https://gateway.example/v1", + ) + + +def test_butler_responses_requests_go_to_butler_natively() -> None: + with respx.mock() as upstream: + route: Final = upstream.post(f"{_BUTLER_BASE_URL}/responses").respond( + 200, + json={ + "id": "resp_req_01", + "object": "response", + "created_at": 1_793_941_697, + "model": "cicero", + "status": "completed", + "output": [ + { + "id": "msg_req_01", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Hello from Butler", "annotations": []}], + } + ], + "usage": {"input_tokens": 12, "output_tokens": 4, "total_tokens": 16}, + }, + ) + response: Final = litellm.responses(model="butler/cicero", input="Say hello", api_key="caller-key") + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert request.headers["authorization"] == "Bearer caller-key" + assert (body["model"], body["input"]) == ("cicero", "Say hello") + assert response.output[0].content[0].text == "Hello from Butler" + + +@pytest.mark.asyncio +async def test_butler_anthropic_messages_requests_go_to_butler_natively(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + with respx.mock() as upstream: + route: Final = upstream.post(f"{_BUTLER_BASE_URL}/messages").respond( + 200, + json={ + "id": "msg_req_01", + "type": "message", + "role": "assistant", + "model": "cicero", + "content": [{"type": "text", "text": "Hello from Butler"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 4}, + }, + ) + response: Final = await litellm.anthropic.messages.acreate( + model="butler/cicero", + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=32, + api_key="caller-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert request.headers["authorization"] == "Bearer caller-key" + assert (body["model"], body["messages"]) == ("cicero", [{"role": "user", "content": "Say hello"}]) + assert response["content"][0]["text"] == "Hello from Butler" diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 27ba8968200..6554827356e 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -532,6 +532,7 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: ("PARALLEL_AI", "Parallel AI", LlmProviders.PARALLEL_AI), ("EU_JEV", "eu/jev", LlmProviders.EU_JEV), ("EMPIRIOLABS", "EmpirioLabs AI", LlmProviders.EMPIRIOLABS), + ("BUTLER", "Butler", LlmProviders.BUTLER), ), ) def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None: diff --git a/ui/litellm-dashboard/public/assets/logos/butler.png b/ui/litellm-dashboard/public/assets/logos/butler.png new file mode 100644 index 00000000000..a67373b6561 Binary files /dev/null and b/ui/litellm-dashboard/public/assets/logos/butler.png differ diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 6a78fe9441a..11f0922e989 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -606,6 +606,20 @@ describe("provider_info_helpers", () => { expect(getPlaceholder("PARALLEL_AI")).toBe("parallel_ai/decision"); }); + it("lists Butler decision models and resolves the provider name and model placeholder", () => { + const modelMap = { + "butler/cicero": { litellm_provider: "butler", mode: "evaluation" }, + "nace/drex-latest": { litellm_provider: "nace", mode: "evaluation" }, + }; + expect(getProviderModels("BUTLER", modelMap)).toEqual(["butler/cicero"]); + expect(getProviderLogoAndName("butler")).toEqual({ + displayName: "Butler", + logo: providerLogoMap[Providers.BUTLER], + }); + expect(providerLogoMap[Providers.BUTLER]).toContain("butler"); + expect(getPlaceholder("BUTLER")).toBe("butler/cicero"); + }); + it("lists Nace decision models and resolves the provider name and model placeholder", () => { const modelMap = { "nace/drex-latest": { litellm_provider: "nace", mode: "evaluation" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 8efd42f52e5..4fe3d072d49 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -8,6 +8,7 @@ import assemblyaiSmallLogo from "../../public/assets/logos/assemblyai_small.png" import basetenLogo from "../../public/assets/logos/baseten.svg"; import bedrockLogo from "../../public/assets/logos/bedrock.svg"; import bigmodelLogo from "../../public/assets/logos/bigmodel.png"; +import butlerLogo from "../../public/assets/logos/butler.png"; import cerebrasLogo from "../../public/assets/logos/cerebras.svg"; import cloudflareLogo from "../../public/assets/logos/cloudflare.svg"; import cohereLogo from "../../public/assets/logos/cohere.svg"; @@ -96,6 +97,7 @@ export enum Providers { AZURE_TEXT = "Azure Text", BASETEN = "Baseten", Bespoke = "Bespoke Nimble", + BUTLER = "Butler", BYTEZ = "Bytez", Cerebras = "Cerebras", CHATGPT = "ChatGPT Subscription", @@ -232,6 +234,7 @@ export const provider_map: Record = { Bedrock: "bedrock", BedrockMantle: "bedrock_mantle", Bespoke: "bespoke", + BUTLER: "butler", BYTEZ: "bytez", Cerebras: "cerebras", CHATGPT: "chatgpt", @@ -373,6 +376,7 @@ export const providerLogoMap: Partial> = { [Providers.BASETEN]: basetenLogo.src, [Providers.Bedrock]: bedrockLogo.src, [Providers.BedrockMantle]: bedrockLogo.src, + [Providers.BUTLER]: butlerLogo.src, [Providers.SageMaker]: bedrockLogo.src, [Providers.Cerebras]: cerebrasLogo.src, [Providers.CHATGPT]: openaiSmallLogo.src, @@ -520,6 +524,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Sail]: "sail/openai/gpt-oss-120b", [Providers.SCX_AI]: "scx-ai/GLM-5.2", [Providers.Bespoke]: "bespoke/nimble", + [Providers.BUTLER]: "butler/cicero", [Providers.Laya]: "laya/english", [Providers.Snowflake]: "snowflake/mistral-7b", [Providers.StrandsDecider]: "strands_decider/strands-decider-2B-hobson-v19",