fix: address llmman provider review

Signed-off-by: Eric Curtin <eric.curtin@docker.com>

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -658,7 +658,6 @@ def _get_openai_compatible_provider_info(
             dynamic_api_key,
         ) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
     elif custom_llm_provider == "llmman":
-        # llmman is OpenAI compatible.
         (
             api_base,
             dynamic_api_key,

litellm/llms/llmman/chat/transformation.py

@@ -4,40 +4,15 @@ from litellm.secret_managers.main import get_secret_str
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
+DEFAULT_API_BASE: Final = "http://127.0.0.1:17434/v1"
+PLACEHOLDER_API_KEY: Final = "fake-api-key"
+
 
 class LlmmanChatConfig(OpenAIGPTConfig):
-    """Configuration for llmman's OpenAI-compatible chat API.
-
-    llmman is a local model runner that serves OpenAI-, Ollama- and
-    Anthropic-compatible APIs. See https://github.com/llmmanorg/llmman
-    """
-
-    @staticmethod
-    def _resolve_api_key(api_key: str | None = None) -> str:
-        """Resolve the API key, preferring the user-provided value over
-        ``LLMMAN_API_KEY``.
-
-        Returns a placeholder when neither is set: llmman does not require a
-        key, but the underlying OpenAI library expects a non-None value.
-        """
-        return api_key or get_secret_str("LLMMAN_API_KEY") or "fake-api-key"
-
-    @staticmethod
-    def _resolve_api_base(api_base: str | None = None) -> str | None:
-        """Resolve the API base, preferring the user-provided value over
-        ``LLMMAN_API_BASE``, then falling back to the default `llmman serve`
-        address.
-
-        See: https://github.com/llmmanorg/llmman#serve
-        """
-        return (
-            api_base or get_secret_str("LLMMAN_API_BASE") or "http://127.0.0.1:17434/v1"
-        )
-
     def _get_openai_compatible_provider_info(
         self, api_base: str | None, api_key: str | None
     ) -> tuple[str | None, str | None]:
-        api_base = LlmmanChatConfig._resolve_api_base(api_base)
-        dynamic_api_key: Final = LlmmanChatConfig._resolve_api_key(api_key)
-
-        return api_base, dynamic_api_key
+        return (
+            api_base or get_secret_str("LLMMAN_API_BASE") or DEFAULT_API_BASE,
+            api_key or get_secret_str("LLMMAN_API_KEY") or PLACEHOLDER_API_KEY,
+        )

provider_endpoints_support.json

@@ -1474,6 +1474,24 @@
         "interactions": true
       }
     },
+    "llmman": {
+      "display_name": "Llmman (`llmman`)",
+      "url": "https://docs.litellm.ai/docs/providers/llmman",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": true,
+        "responses": true,
+        "embeddings": true,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": true,
+        "interactions": true
+      }
+    },
     "lm_studio": {
       "display_name": "LM Studio (`lm_studio`)",
       "url": "https://docs.litellm.ai/docs/providers/lm_studio",

tests/test_litellm/llms/llmman/chat/test_llmman_chat_transformation.py

@@ -1,179 +1,97 @@
-from typing import Optional
+import json
+from typing import Final
 from unittest.mock import patch
 
+import httpx
 import pytest
+import respx
 
 import litellm
 from litellm.llms.llmman.chat.transformation import LlmmanChatConfig
 
+DEFAULT_API_BASE: Final = "http://127.0.0.1:17434/v1"
+ENV: Final = {
+    "LLMMAN_API_BASE": "https://env-api.example.com",
+    "LLMMAN_API_KEY": "env-key",
+}
+
 
 @pytest.mark.parametrize(
-    "input_api_key, env_api_key, expected_api_key",
+    "api_base, api_key, env, expected_base, expected_key",
     [
-        ("user-provided-key", "secret-key", "user-provided-key"),
-        (None, "secret-key", "secret-key"),
-        (None, None, "fake-api-key"),
-        ("", "secret-key", "secret-key"),  # Empty string should fall back to secret
-        (
-            "",
-            None,
-            "fake-api-key",
-        ),  # Empty string with no secret should use the fake key
+        ("https://user-api.example.com", "user-key", ENV, "https://user-api.example.com", "user-key"),
+        (None, None, ENV, "https://env-api.example.com", "env-key"),
+        (None, None, {}, DEFAULT_API_BASE, "fake-api-key"),
+        ("", "", ENV, "https://env-api.example.com", "env-key"),
+        ("", "", {}, DEFAULT_API_BASE, "fake-api-key"),
+        ("https://user-api.example.com", None, ENV, "https://user-api.example.com", "env-key"),
+        (None, "user-key", ENV, "https://env-api.example.com", "user-key"),
     ],
 )
-def test_resolve_api_key(input_api_key, env_api_key, expected_api_key):
-    env = {}
-    if env_api_key is not None:
-        env["LLMMAN_API_KEY"] = env_api_key
-
+def test_get_openai_compatible_provider_info(api_base, api_key, env, expected_base, expected_key):
     with patch.dict("os.environ", env, clear=True):
-        result = LlmmanChatConfig._resolve_api_key(input_api_key)
-        assert result == expected_api_key
-
-
-@pytest.mark.parametrize(
-    "input_api_base, env_api_base, expected_api_base",
-    [
-        (
-            "https://user-api.example.com",
-            "https://secret-api.example.com",
-            "https://user-api.example.com",
-        ),
-        (
-            None,
-            "https://secret-api.example.com",
-            "https://secret-api.example.com",
-        ),
-        (None, None, "http://127.0.0.1:17434/v1"),
-        (
-            "",
-            "https://secret-api.example.com",
-            "https://secret-api.example.com",
-        ),  # Empty string should fall back
-    ],
-)
-def test_resolve_api_base(
-    input_api_base,
-    env_api_base,
-    expected_api_base,
-):
-    env = {}
-    if env_api_base is not None:
-        env["LLMMAN_API_BASE"] = env_api_base
-
-    with patch.dict("os.environ", env, clear=True):
-        result = LlmmanChatConfig._resolve_api_base(input_api_base)
-        assert result == expected_api_base
-
-
-@pytest.mark.parametrize(
-    "api_base, api_key, env_base, env_key, expected_base, expected_key",
-    [
-        # User-provided values
-        (
-            "https://user-api.example.com",
-            "user-key",
-            "https://secret-api.example.com",
-            "secret-key",
-            "https://user-api.example.com",
-            "user-key",
-        ),
-        # Fallback to env vars
-        (
-            None,
-            None,
-            "https://secret-api.example.com",
-            "secret-key",
-            "https://secret-api.example.com",
-            "secret-key",
-        ),
-        # Nothing provided, use defaults
-        (None, None, None, None, "http://127.0.0.1:17434/v1", "fake-api-key"),
-        # Mixed scenarios
-        (
-            "https://user-api.example.com",
-            None,
-            None,
-            "secret-key",
-            "https://user-api.example.com",
-            "secret-key",
-        ),
-        (
-            None,
-            "user-key",
-            "https://secret-api.example.com",
-            None,
-            "https://secret-api.example.com",
-            "user-key",
-        ),
-    ],
-)
-def test_get_openai_compatible_provider_info(
-    api_base, api_key, env_base, env_key, expected_base, expected_key
-):
-    config = LlmmanChatConfig()
-
-    env = {}
-    if env_base is not None:
-        env["LLMMAN_API_BASE"] = env_base
-    if env_key is not None:
-        env["LLMMAN_API_KEY"] = env_key
-
-    patch_base = patch.object(
-        LlmmanChatConfig,
-        "_resolve_api_base",
-        wraps=LlmmanChatConfig._resolve_api_base,
-    )
-    patch_key = patch.object(
-        LlmmanChatConfig,
-        "_resolve_api_key",
-        wraps=LlmmanChatConfig._resolve_api_key,
-    )
-
-    with (
-        patch.dict("os.environ", env, clear=True),
-        patch_base as mock_base,
-        patch_key as mock_key,
-    ):
-        result_base, result_key = config._get_openai_compatible_provider_info(
-            api_base, api_key
+        assert LlmmanChatConfig()._get_openai_compatible_provider_info(api_base, api_key) == (
+            expected_base,
+            expected_key,
         )
 
-        assert result_base == expected_base
-        assert result_key == expected_key
 
-        mock_base.assert_called_once_with(api_base)
-        mock_key.assert_called_once_with(api_key)
+def test_get_llm_provider_routes_llmman_prefix():
+    with patch.dict("os.environ", {}, clear=True):
+        result = litellm.get_llm_provider("llmman/my-custom-test-model")
+
+    assert result == ("my-custom-test-model", "llmman", "fake-api-key", DEFAULT_API_BASE)
 
 
-def test_completion_with_custom_llmman_model():
-    with patch(
-        "litellm.main.openai_chat_completions.completion"
-    ) as mock_llmman_completion_func:
-        mock_llmman_completion_func.return_value = (
-            {}
-        )  # Return an empty dictionary for the mocked response
+@respx.mock
+def test_completion_hits_default_llmman_endpoint():
+    route = respx.post(f"{DEFAULT_API_BASE}/chat/completions").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "id": "chatcmpl-1",
+                "object": "chat.completion",
+                "created": 1,
+                "model": "my-custom-test-model",
+                "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}],
+                "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+            },
+        )
+    )
 
-        provider = "llmman"
-        model_name = "my-custom-test-model"
-        model = f"{provider}/{model_name}"
-        messages = [{"role": "user", "content": "Hey, how's it going?"}]
-
-        _ = litellm.completion(
-            model=model,
-            messages=messages,
-            max_retries=2,
+    with patch.dict("os.environ", {}, clear=True):
+        response = litellm.completion(
+            model="llmman/my-custom-test-model",
+            messages=[{"role": "user", "content": "Hey, how's it going?"}],
             max_tokens=100,
         )
 
-        mock_llmman_completion_func.assert_called_once()
-        _, call_kwargs = mock_llmman_completion_func.call_args
-        assert call_kwargs.get("custom_llm_provider") == provider
-        assert call_kwargs.get("model") == model_name
-        assert call_kwargs.get("messages") == messages
-        assert call_kwargs.get("api_base") == "http://127.0.0.1:17434/v1"
-        assert call_kwargs.get("api_key") == "fake-api-key"
-        optional_params = call_kwargs.get("optional_params")
-        assert optional_params
-        assert optional_params.get("max_retries") == 2
-        assert optional_params.get("max_tokens") == 100
+    assert response.choices[0].message.content == "hi"
+    request = route.calls.last.request
+    assert request.headers["authorization"] == "Bearer fake-api-key"
+    body = json.loads(request.content)
+    assert body["model"] == "my-custom-test-model"
+    assert body["max_tokens"] == 100
+
+
+@respx.mock
+def test_embedding_hits_default_llmman_endpoint():
+    route = respx.post(f"{DEFAULT_API_BASE}/embeddings").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "object": "list",
+                "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
+                "model": "my-embedding-model",
+                "usage": {"prompt_tokens": 1, "total_tokens": 1},
+            },
+        )
+    )
+
+    with patch.dict("os.environ", {}, clear=True):
+        response = litellm.embedding(model="llmman/my-embedding-model", input=["hello"])
+
+    assert response.data[0]["embedding"] == [0.1, 0.2]
+    request = route.calls.last.request
+    assert request.headers["authorization"] == "Bearer fake-api-key"
+    assert json.loads(request.content)["model"] == "my-embedding-model"