diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 27bc347de75..96465238736 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -658,7 +658,6 @@ def _get_openai_compatible_provider_info( dynamic_api_key, ) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info(api_base, api_key) elif custom_llm_provider == "llmman": - # llmman is OpenAI compatible. ( api_base, dynamic_api_key, diff --git a/litellm/llms/llmman/chat/transformation.py b/litellm/llms/llmman/chat/transformation.py index 0351a44b828..31ce5cc7a2a 100644 --- a/litellm/llms/llmman/chat/transformation.py +++ b/litellm/llms/llmman/chat/transformation.py @@ -4,40 +4,15 @@ from litellm.secret_managers.main import get_secret_str from ...openai.chat.gpt_transformation import OpenAIGPTConfig +DEFAULT_API_BASE: Final = "http://127.0.0.1:17434/v1" +PLACEHOLDER_API_KEY: Final = "fake-api-key" + class LlmmanChatConfig(OpenAIGPTConfig): - """Configuration for llmman's OpenAI-compatible chat API. - - llmman is a local model runner that serves OpenAI-, Ollama- and - Anthropic-compatible APIs. See https://github.com/llmmanorg/llmman - """ - - @staticmethod - def _resolve_api_key(api_key: str | None = None) -> str: - """Resolve the API key, preferring the user-provided value over - ``LLMMAN_API_KEY``. - - Returns a placeholder when neither is set: llmman does not require a - key, but the underlying OpenAI library expects a non-None value. - """ - return api_key or get_secret_str("LLMMAN_API_KEY") or "fake-api-key" - - @staticmethod - def _resolve_api_base(api_base: str | None = None) -> str | None: - """Resolve the API base, preferring the user-provided value over - ``LLMMAN_API_BASE``, then falling back to the default `llmman serve` - address. - - See: https://github.com/llmmanorg/llmman#serve - """ - return ( - api_base or get_secret_str("LLMMAN_API_BASE") or "http://127.0.0.1:17434/v1" - ) - def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: - api_base = LlmmanChatConfig._resolve_api_base(api_base) - dynamic_api_key: Final = LlmmanChatConfig._resolve_api_key(api_key) - - return api_base, dynamic_api_key + return ( + api_base or get_secret_str("LLMMAN_API_BASE") or DEFAULT_API_BASE, + api_key or get_secret_str("LLMMAN_API_KEY") or PLACEHOLDER_API_KEY, + ) diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 7c7d508856f..9a189787635 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1474,6 +1474,24 @@ "interactions": true } }, + "llmman": { + "display_name": "Llmman (`llmman`)", + "url": "https://docs.litellm.ai/docs/providers/llmman", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, "lm_studio": { "display_name": "LM Studio (`lm_studio`)", "url": "https://docs.litellm.ai/docs/providers/lm_studio", diff --git a/tests/test_litellm/llms/llmman/chat/test_llmman_chat_transformation.py b/tests/test_litellm/llms/llmman/chat/test_llmman_chat_transformation.py index c3cc4d6e865..bb707ade4e9 100644 --- a/tests/test_litellm/llms/llmman/chat/test_llmman_chat_transformation.py +++ b/tests/test_litellm/llms/llmman/chat/test_llmman_chat_transformation.py @@ -1,179 +1,97 @@ -from typing import Optional +import json +from typing import Final from unittest.mock import patch +import httpx import pytest +import respx import litellm from litellm.llms.llmman.chat.transformation import LlmmanChatConfig +DEFAULT_API_BASE: Final = "http://127.0.0.1:17434/v1" +ENV: Final = { + "LLMMAN_API_BASE": "https://env-api.example.com", + "LLMMAN_API_KEY": "env-key", +} + @pytest.mark.parametrize( - "input_api_key, env_api_key, expected_api_key", + "api_base, api_key, env, expected_base, expected_key", [ - ("user-provided-key", "secret-key", "user-provided-key"), - (None, "secret-key", "secret-key"), - (None, None, "fake-api-key"), - ("", "secret-key", "secret-key"), # Empty string should fall back to secret - ( - "", - None, - "fake-api-key", - ), # Empty string with no secret should use the fake key + ("https://user-api.example.com", "user-key", ENV, "https://user-api.example.com", "user-key"), + (None, None, ENV, "https://env-api.example.com", "env-key"), + (None, None, {}, DEFAULT_API_BASE, "fake-api-key"), + ("", "", ENV, "https://env-api.example.com", "env-key"), + ("", "", {}, DEFAULT_API_BASE, "fake-api-key"), + ("https://user-api.example.com", None, ENV, "https://user-api.example.com", "env-key"), + (None, "user-key", ENV, "https://env-api.example.com", "user-key"), ], ) -def test_resolve_api_key(input_api_key, env_api_key, expected_api_key): - env = {} - if env_api_key is not None: - env["LLMMAN_API_KEY"] = env_api_key - +def test_get_openai_compatible_provider_info(api_base, api_key, env, expected_base, expected_key): with patch.dict("os.environ", env, clear=True): - result = LlmmanChatConfig._resolve_api_key(input_api_key) - assert result == expected_api_key - - -@pytest.mark.parametrize( - "input_api_base, env_api_base, expected_api_base", - [ - ( - "https://user-api.example.com", - "https://secret-api.example.com", - "https://user-api.example.com", - ), - ( - None, - "https://secret-api.example.com", - "https://secret-api.example.com", - ), - (None, None, "http://127.0.0.1:17434/v1"), - ( - "", - "https://secret-api.example.com", - "https://secret-api.example.com", - ), # Empty string should fall back - ], -) -def test_resolve_api_base( - input_api_base, - env_api_base, - expected_api_base, -): - env = {} - if env_api_base is not None: - env["LLMMAN_API_BASE"] = env_api_base - - with patch.dict("os.environ", env, clear=True): - result = LlmmanChatConfig._resolve_api_base(input_api_base) - assert result == expected_api_base - - -@pytest.mark.parametrize( - "api_base, api_key, env_base, env_key, expected_base, expected_key", - [ - # User-provided values - ( - "https://user-api.example.com", - "user-key", - "https://secret-api.example.com", - "secret-key", - "https://user-api.example.com", - "user-key", - ), - # Fallback to env vars - ( - None, - None, - "https://secret-api.example.com", - "secret-key", - "https://secret-api.example.com", - "secret-key", - ), - # Nothing provided, use defaults - (None, None, None, None, "http://127.0.0.1:17434/v1", "fake-api-key"), - # Mixed scenarios - ( - "https://user-api.example.com", - None, - None, - "secret-key", - "https://user-api.example.com", - "secret-key", - ), - ( - None, - "user-key", - "https://secret-api.example.com", - None, - "https://secret-api.example.com", - "user-key", - ), - ], -) -def test_get_openai_compatible_provider_info( - api_base, api_key, env_base, env_key, expected_base, expected_key -): - config = LlmmanChatConfig() - - env = {} - if env_base is not None: - env["LLMMAN_API_BASE"] = env_base - if env_key is not None: - env["LLMMAN_API_KEY"] = env_key - - patch_base = patch.object( - LlmmanChatConfig, - "_resolve_api_base", - wraps=LlmmanChatConfig._resolve_api_base, - ) - patch_key = patch.object( - LlmmanChatConfig, - "_resolve_api_key", - wraps=LlmmanChatConfig._resolve_api_key, - ) - - with ( - patch.dict("os.environ", env, clear=True), - patch_base as mock_base, - patch_key as mock_key, - ): - result_base, result_key = config._get_openai_compatible_provider_info( - api_base, api_key + assert LlmmanChatConfig()._get_openai_compatible_provider_info(api_base, api_key) == ( + expected_base, + expected_key, ) - assert result_base == expected_base - assert result_key == expected_key - mock_base.assert_called_once_with(api_base) - mock_key.assert_called_once_with(api_key) +def test_get_llm_provider_routes_llmman_prefix(): + with patch.dict("os.environ", {}, clear=True): + result = litellm.get_llm_provider("llmman/my-custom-test-model") + + assert result == ("my-custom-test-model", "llmman", "fake-api-key", DEFAULT_API_BASE) -def test_completion_with_custom_llmman_model(): - with patch( - "litellm.main.openai_chat_completions.completion" - ) as mock_llmman_completion_func: - mock_llmman_completion_func.return_value = ( - {} - ) # Return an empty dictionary for the mocked response +@respx.mock +def test_completion_hits_default_llmman_endpoint(): + route = respx.post(f"{DEFAULT_API_BASE}/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "my-custom-test-model", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) - provider = "llmman" - model_name = "my-custom-test-model" - model = f"{provider}/{model_name}" - messages = [{"role": "user", "content": "Hey, how's it going?"}] - - _ = litellm.completion( - model=model, - messages=messages, - max_retries=2, + with patch.dict("os.environ", {}, clear=True): + response = litellm.completion( + model="llmman/my-custom-test-model", + messages=[{"role": "user", "content": "Hey, how's it going?"}], max_tokens=100, ) - mock_llmman_completion_func.assert_called_once() - _, call_kwargs = mock_llmman_completion_func.call_args - assert call_kwargs.get("custom_llm_provider") == provider - assert call_kwargs.get("model") == model_name - assert call_kwargs.get("messages") == messages - assert call_kwargs.get("api_base") == "http://127.0.0.1:17434/v1" - assert call_kwargs.get("api_key") == "fake-api-key" - optional_params = call_kwargs.get("optional_params") - assert optional_params - assert optional_params.get("max_retries") == 2 - assert optional_params.get("max_tokens") == 100 + assert response.choices[0].message.content == "hi" + request = route.calls.last.request + assert request.headers["authorization"] == "Bearer fake-api-key" + body = json.loads(request.content) + assert body["model"] == "my-custom-test-model" + assert body["max_tokens"] == 100 + + +@respx.mock +def test_embedding_hits_default_llmman_endpoint(): + route = respx.post(f"{DEFAULT_API_BASE}/embeddings").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "my-embedding-model", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + ) + ) + + with patch.dict("os.environ", {}, clear=True): + response = litellm.embedding(model="llmman/my-embedding-model", input=["hello"]) + + assert response.data[0]["embedding"] == [0.1, 0.2] + request = route.calls.last.request + assert request.headers["authorization"] == "Bearer fake-api-key" + assert json.loads(request.content)["model"] == "my-embedding-model"