Merge ef4e641a0a into 7057b2f6c4

litellm/__init__.py

@@ -1978,6 +1978,9 @@ if TYPE_CHECKING:
     from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig
     from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig
     from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig
+    from .llms.api_route.chat.transformation import (
+        APIRouteChatConfig as APIRouteChatConfig,
+    )
     from .llms.volcengine.chat.transformation import (
         VolcEngineChatConfig as VolcEngineChatConfig,
         VolcEngineChatConfig as VolcEngineConfig,

litellm/_lazy_imports_registry.py

@@ -271,6 +271,7 @@ LLM_CONFIG_NAMES: Final = (
     "XAIChatConfig",
     "ZAIChatConfig",
     "AIMLChatConfig",
+    "APIRouteChatConfig",
     "VolcEngineChatConfig",
     "CodestralTextCompletionConfig",
     "InceptionTextCompletionConfig",
@@ -1062,6 +1063,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     "XAIChatConfig": (".llms.xai.chat.transformation", "XAIChatConfig"),
     "ZAIChatConfig": (".llms.zai.chat.transformation", "ZAIChatConfig"),
     "AIMLChatConfig": (".llms.aiml.chat.transformation", "AIMLChatConfig"),
+    "APIRouteChatConfig": (
+        ".llms.api_route.chat.transformation",
+        "APIRouteChatConfig",
+    ),
     "VolcEngineChatConfig": (
         ".llms.volcengine.chat.transformation",
         "VolcEngineChatConfig",

litellm/constants.py

@@ -956,6 +956,7 @@ openai_compatible_providers: Final[list] = [
     "hyperbolic",
     "vercel_ai_gateway",
     "aiml",
+    "api_route",
     "wandb",
     "cometapi",
     "clarifai",

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -849,6 +849,11 @@ def _get_openai_compatible_provider_info(
             api_base,
             dynamic_api_key,
         ) = litellm.AIMLChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
+    elif custom_llm_provider == "api_route":
+        (
+            api_base,
+            dynamic_api_key,
+        ) = litellm.APIRouteChatConfig().get_openai_compatible_provider_info(api_base, api_key)
     elif custom_llm_provider == "wandb":
         api_base = api_base or get_secret("WANDB_API_BASE") or "https://api.inference.wandb.ai/v1"
         dynamic_api_key = api_key or get_secret_str("WANDB_API_KEY")

litellm/llms/api_route/chat/transformation.py

@@ -0,0 +1,17 @@
+from typing import Final
+
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+from litellm.secret_managers.main import get_secret_str
+
+
+class APIRouteChatConfig(OpenAIGPTConfig):
+    @property
+    def custom_llm_provider(self) -> str | None:
+        return "api_route"
+
+    def get_openai_compatible_provider_info(
+        self, api_base: str | None, api_key: str | None
+    ) -> tuple[str | None, str | None]:
+        resolved_api_base: Final = api_base or get_secret_str("API_ROUTE_BASE_URL") or "https://global.api-route.com/v1"
+        resolved_api_key: Final = api_key or get_secret_str("API_ROUTE_API_KEY")
+        return resolved_api_base, resolved_api_key

litellm/provider_endpoints_support_backup.json

@@ -84,6 +84,23 @@
         "interactions": true
       }
     },
+    "api_route": {
+      "display_name": "API Route (`api_route`)",
+      "url": "https://www.api-route.com/docs/overview",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": false,
+        "responses": false,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false
+      }
+    },
     "ai21": {
       "display_name": "AI21 (`ai21`)",
       "url": "https://docs.litellm.ai/docs/providers/ai21",

litellm/types/utils.py

@@ -3984,6 +3984,7 @@ class LlmProviders(str, Enum):
     STABILITY = "stability"
     HEROKU = "heroku"
     AIML = "aiml"
+    API_ROUTE = "api_route"
     COMETAPI = "cometapi"
     OCI = "oci"
     AUTO_ROUTER = "auto_router"

litellm/utils.py

@@ -8167,6 +8167,7 @@ class ProviderConfigManager:
             LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False),
             LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIChatConfig(), False),
             LlmProviders.OPENROUTER: (lambda: litellm.OpenrouterConfig(), False),
+            LlmProviders.API_ROUTE: (lambda: litellm.APIRouteChatConfig(), False),
             LlmProviders.VERCEL_AI_GATEWAY: (
                 lambda: litellm.VercelAIGatewayConfig(),
                 False,

provider_endpoints_support.json

@@ -84,6 +84,23 @@
         "interactions": true
       }
     },
+    "api_route": {
+      "display_name": "API Route (`api_route`)",
+      "url": "https://www.api-route.com/docs/overview",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": false,
+        "responses": false,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false
+      }
+    },
     "ai21": {
       "display_name": "AI21 (`ai21`)",
       "url": "https://docs.litellm.ai/docs/providers/ai21",

tests/test_litellm/llms/api_route/test_api_route.py

@@ -0,0 +1,133 @@
+import json
+
+import litellm
+from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
+from litellm.llms.api_route.chat.transformation import APIRouteChatConfig
+
+
+def test_api_route_provider_routing():
+    assert APIRouteChatConfig().custom_llm_provider == "api_route"
+
+    model, provider, api_key, api_base = get_llm_provider(
+        model="api_route/model-name",
+        custom_llm_provider=None,
+        api_key="test-key",
+        api_base="https://example.com/v1",
+    )
+
+    assert model == "model-name"
+    assert provider == "api_route"
+    assert api_key == "test-key"
+    assert api_base == "https://example.com/v1"
+
+
+def test_api_route_credentials_from_environment(monkeypatch):
+    monkeypatch.setenv("API_ROUTE_API_KEY", "env-key")
+    monkeypatch.setenv("API_ROUTE_BASE_URL", "https://env.example.com/v1")
+
+    _, provider, api_key, api_base = get_llm_provider(
+        model="api_route/model-name",
+        custom_llm_provider=None,
+        api_key=None,
+        api_base=None,
+    )
+
+    assert provider == "api_route"
+    assert api_key == "env-key"
+    assert api_base == "https://env.example.com/v1"
+
+
+def test_api_route_uses_default_base_url(monkeypatch):
+    monkeypatch.delenv("API_ROUTE_BASE_URL", raising=False)
+
+    _, _, api_key, api_base = get_llm_provider(
+        model="api_route/model-name",
+        custom_llm_provider=None,
+        api_key="test-key",
+        api_base=None,
+    )
+
+    assert api_base == "https://global.api-route.com/v1"
+    assert api_key == "test-key"
+
+
+def test_api_route_explicit_api_base_overrides_default(monkeypatch):
+    monkeypatch.delenv("API_ROUTE_BASE_URL", raising=False)
+
+    _, _, _, api_base = get_llm_provider(
+        model="api_route/model-name",
+        custom_llm_provider=None,
+        api_key="test-key",
+        api_base="https://custom.example.com/v1",
+    )
+
+    assert api_base == "https://custom.example.com/v1"
+
+
+def test_api_route_chat_completion_request(monkeypatch, respx_mock):
+    monkeypatch.setenv("API_ROUTE_BASE_URL", "https://env.example.com/v1")
+    route = respx_mock.post("https://env.example.com/v1/chat/completions").respond(
+        json={
+            "id": "chatcmpl-test",
+            "object": "chat.completion",
+            "created": 1,
+            "model": "model-name",
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {"role": "assistant", "content": "Hello!"},
+                    "finish_reason": "stop",
+                }
+            ],
+            "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+        }
+    )
+
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get the weather",
+                "parameters": {"type": "object", "properties": {}},
+            },
+        }
+    ]
+
+    response = litellm.completion(
+        model="api_route/model-name",
+        messages=[{"role": "user", "content": "Hello!"}],
+        api_key="test-key",
+        tools=tools,
+    )
+
+    request = route.calls[0].request
+    body = json.loads(request.content)
+    assert str(request.url) == "https://env.example.com/v1/chat/completions"
+    assert request.headers["Authorization"] == "Bearer test-key"
+    assert body["model"] == "model-name"
+    assert body["messages"] == [{"role": "user", "content": "Hello!"}]
+    assert body["tools"] == tools
+    assert response.choices[0].message.content == "Hello!"
+
+
+def test_api_route_streaming_request(respx_mock):
+    route = respx_mock.post("https://stream.example.com/v1/chat/completions").respond(
+        headers={"content-type": "text/event-stream"},
+        text='data: {"choices":[{"delta":{"content":"Hello"},"index":0}]}\n\ndata: [DONE]\n\n',
+    )
+
+    chunks = list(
+        litellm.completion(
+            model="api_route/model-name",
+            messages=[{"role": "user", "content": "Hello!"}],
+            api_key="stream-key",
+            api_base="https://stream.example.com/v1",
+            stream=True,
+        )
+    )
+
+    body = json.loads(route.calls[0].request.content)
+    assert body["model"] == "model-name"
+    assert body["stream"] is True
+    assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "Hello"