From 208209cd945c7352596ee293cad9a37571e58e0d Mon Sep 17 00:00:00 2001 From: drvt69talati Date: Thu, 29 Oct 2026 11:49:06 -0400 Subject: [PATCH] feat(providers): add llamay as an OpenAI-compatible local provider llamay/ reaches a llamay server on localhost:11435/v1, or LLAMAY_API_BASE. A local server needs no key, so the JSON provider registry gains an optional default_api_key, sent only when neither the call nor api_key_env supplies one, as LM Studio sends fake-api-key. Co-Authored-By: Claude Opus 5.5 --- litellm/llms/openai_like/dynamic_config.py | 2 +- litellm/llms/openai_like/json_loader.py | 4 ++ litellm/llms/openai_like/providers.json | 13 ++++ .../provider_endpoints_support_backup.json | 26 ++++++++ .../provider_create_fields.json | 28 +++++++++ litellm/types/utils.py | 1 + provider_endpoints_support.json | 26 ++++++++ .../llms/openai_like/test_llamay_provider.py | 62 +++++++++++++++++++ 8 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 tests/unit/llms/openai_like/test_llamay_provider.py diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index d7aafe9f06e..58c8e501eb7 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -77,7 +77,7 @@ def create_config_class(provider: SimpleProviderConfig): resolved_base = provider.base_url # Resolve API key - resolved_key: Final = api_key or get_secret_str(provider.api_key_env) + resolved_key: Final = api_key or get_secret_str(provider.api_key_env) or provider.default_api_key return resolved_base, resolved_key diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index 6e99b9e7fec..ab3e222b033 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -24,6 +24,10 @@ class SimpleProviderConfig: self.base_url = data["base_url"] self.api_key_env: str = data["api_key_env"] self.api_base_env: str | None = data.get("api_base_env") + # A local server that needs no key still has to be sent one: the OpenAI + # client refuses a request without it. Used only when neither the call + # nor api_key_env supplies a key, as LM Studio's "fake-api-key" is. + self.default_api_key: str | None = data.get("default_api_key") self.base_class = data.get("base_class", "openai_gpt") self.param_mappings = data.get("param_mappings", {}) self.constraints = data.get("constraints", {}) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index ad47b334ed4..324b9c7dc90 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -270,6 +270,19 @@ "/v1/responses" ] }, + "llamay": { + "base_url": "http://localhost:11435/v1", + "api_key_env": "LLAMAY_API_KEY", + "api_base_env": "LLAMAY_API_BASE", + "default_api_key": "fake-api-key", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "provider_info": { + "supports_function_calling": true + } + }, "kyma": { "base_url": "https://kymaapi.com/v1", "api_key_env": "KYMA_API_KEY", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 1d487f33835..97672bf0c43 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -59,6 +59,32 @@ "generateContent": true } }, + "llamay": { + "display_name": "llamay (`llamay`)", + "url": "https://docs.litellm.ai/docs/providers/llamay", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, "a2a": { "display_name": "A2A (Agent-to-Agent) (`a2a`)", "url": "https://docs.litellm.ai/docs/providers/a2a", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 04e6c5d8ba9..4c86609c5fa 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3315,6 +3315,34 @@ ], "default_model_placeholder": "refuseless/glm-5.3-abliterated" }, + { + "provider": "LLAMAY", + "provider_display_name": "llamay", + "litellm_provider": "llamay", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "http://localhost:11435/v1", + "tooltip": "Optional. Leave blank for a llamay server on this machine.", + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": "Only needed when the server was started with LLAMAY_API_KEY.", + "required": false, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "llamay/qwen2.5:0.5b" + }, { "provider": "REPLICATE", "provider_display_name": "Replicate", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3bdcba75778..3b2bd8d5f29 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4436,6 +4436,7 @@ GenericBudgetConfigType = dict[str, BudgetConfig] class LlmProviders(str, Enum): REFUSELESS = "refuseless" + LLAMAY = "llamay" OPENAI = "openai" CHATGPT = "chatgpt" OPENAI_LIKE = "openai_like" # embedding only diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 1d487f33835..97672bf0c43 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -59,6 +59,32 @@ "generateContent": true } }, + "llamay": { + "display_name": "llamay (`llamay`)", + "url": "https://docs.litellm.ai/docs/providers/llamay", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, "a2a": { "display_name": "A2A (Agent-to-Agent) (`a2a`)", "url": "https://docs.litellm.ai/docs/providers/a2a", diff --git a/tests/unit/llms/openai_like/test_llamay_provider.py b/tests/unit/llms/openai_like/test_llamay_provider.py new file mode 100644 index 00000000000..06a20b1216b --- /dev/null +++ b/tests/unit/llms/openai_like/test_llamay_provider.py @@ -0,0 +1,62 @@ +import json +from typing import Final + +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + +def test_llamay_needs_no_key_on_this_machine(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LLAMAY_API_KEY", raising=False) + monkeypatch.delenv("LLAMAY_API_BASE", raising=False) + assert get_llm_provider(model="llamay/qwen2.5:0.5b") == ( + "qwen2.5:0.5b", + "llamay", + "fake-api-key", + "http://localhost:11435/v1", + ) + + +def test_llamay_reads_a_key_and_base_from_the_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LLAMAY_API_KEY", "fake-environment-key") + monkeypatch.setenv("LLAMAY_API_BASE", "http://gpu-box:11435/v1") + assert get_llm_provider(model="llamay/hf.co/bartowski/Qwen2.5-Coder-1.5B-Instruct-GGUF:Q4_K_M") == ( + "hf.co/bartowski/Qwen2.5-Coder-1.5B-Instruct-GGUF:Q4_K_M", + "llamay", + "fake-environment-key", + "http://gpu-box:11435/v1", + ) + + +def test_an_explicit_key_and_base_win(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LLAMAY_API_KEY", "fake-environment-key") + assert get_llm_provider( + model="llamay/qwen2.5:0.5b", api_key="fake-explicit-key", api_base="http://other:11435/v1" + ) == ("qwen2.5:0.5b", "llamay", "fake-explicit-key", "http://other:11435/v1") + + +def test_llamay_chat_reaches_the_local_server(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LLAMAY_API_KEY", raising=False) + monkeypatch.delenv("LLAMAY_API_BASE", raising=False) + with respx.mock() as upstream: + route: Final = upstream.post("http://localhost:11435/v1/chat/completions").respond( + 200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "qwen2.5:0.5b", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + }, + ) + response: Final = litellm.completion( + model="llamay/qwen2.5:0.5b", messages=[{"role": "user", "content": "Say hello"}] + ) + body: Final = json.loads(route.calls.last.request.content) + assert body["model"] == "qwen2.5:0.5b" + assert response.choices[0].message.content == "hello"