diff --git a/litellm/__init__.py b/litellm/__init__.py index 1dfd146a00e..8a7f1e5dbb5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1978,6 +1978,9 @@ if TYPE_CHECKING: from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig + from .llms.api_route.chat.transformation import ( + APIRouteChatConfig as APIRouteChatConfig, + ) from .llms.volcengine.chat.transformation import ( VolcEngineChatConfig as VolcEngineChatConfig, VolcEngineChatConfig as VolcEngineConfig, diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index dc323c8cc15..c40eab14733 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -271,6 +271,7 @@ LLM_CONFIG_NAMES: Final = ( "XAIChatConfig", "ZAIChatConfig", "AIMLChatConfig", + "APIRouteChatConfig", "VolcEngineChatConfig", "CodestralTextCompletionConfig", "InceptionTextCompletionConfig", @@ -1062,6 +1063,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "XAIChatConfig": (".llms.xai.chat.transformation", "XAIChatConfig"), "ZAIChatConfig": (".llms.zai.chat.transformation", "ZAIChatConfig"), "AIMLChatConfig": (".llms.aiml.chat.transformation", "AIMLChatConfig"), + "APIRouteChatConfig": ( + ".llms.api_route.chat.transformation", + "APIRouteChatConfig", + ), "VolcEngineChatConfig": ( ".llms.volcengine.chat.transformation", "VolcEngineChatConfig", diff --git a/litellm/constants.py b/litellm/constants.py index 6b984c2673c..dffffdf970d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -956,6 +956,7 @@ openai_compatible_providers: Final[list] = [ "hyperbolic", "vercel_ai_gateway", "aiml", + "api_route", "wandb", "cometapi", "clarifai", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index ce51fb19970..eb81eff78fc 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -849,6 +849,11 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.AIMLChatConfig()._get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "api_route": + ( + api_base, + dynamic_api_key, + ) = litellm.APIRouteChatConfig().get_openai_compatible_provider_info(api_base, api_key) elif custom_llm_provider == "wandb": api_base = api_base or get_secret("WANDB_API_BASE") or "https://api.inference.wandb.ai/v1" dynamic_api_key = api_key or get_secret_str("WANDB_API_KEY") diff --git a/litellm/llms/api_route/chat/transformation.py b/litellm/llms/api_route/chat/transformation.py new file mode 100644 index 00000000000..b19a14636cf --- /dev/null +++ b/litellm/llms/api_route/chat/transformation.py @@ -0,0 +1,17 @@ +from typing import Final + +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.secret_managers.main import get_secret_str + + +class APIRouteChatConfig(OpenAIGPTConfig): + @property + def custom_llm_provider(self) -> str | None: + return "api_route" + + def get_openai_compatible_provider_info( + self, api_base: str | None, api_key: str | None + ) -> tuple[str | None, str | None]: + resolved_api_base: Final = api_base or get_secret_str("API_ROUTE_BASE_URL") or "https://global.api-route.com/v1" + resolved_api_key: Final = api_key or get_secret_str("API_ROUTE_API_KEY") + return resolved_api_base, resolved_api_key diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index dbeaccdda2d..4ac5ee6c3a5 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -84,6 +84,23 @@ "interactions": true } }, + "api_route": { + "display_name": "API Route (`api_route`)", + "url": "https://www.api-route.com/docs/overview", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "ai21": { "display_name": "AI21 (`ai21`)", "url": "https://docs.litellm.ai/docs/providers/ai21", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e3ea37dc0c8..12c408a54df 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3984,6 +3984,7 @@ class LlmProviders(str, Enum): STABILITY = "stability" HEROKU = "heroku" AIML = "aiml" + API_ROUTE = "api_route" COMETAPI = "cometapi" OCI = "oci" AUTO_ROUTER = "auto_router" diff --git a/litellm/utils.py b/litellm/utils.py index 1a77655a5a4..c7d24c11f8f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8167,6 +8167,7 @@ class ProviderConfigManager: LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False), LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIChatConfig(), False), LlmProviders.OPENROUTER: (lambda: litellm.OpenrouterConfig(), False), + LlmProviders.API_ROUTE: (lambda: litellm.APIRouteChatConfig(), False), LlmProviders.VERCEL_AI_GATEWAY: ( lambda: litellm.VercelAIGatewayConfig(), False, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index c71f4a82a4a..2580a75467a 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -84,6 +84,23 @@ "interactions": true } }, + "api_route": { + "display_name": "API Route (`api_route`)", + "url": "https://www.api-route.com/docs/overview", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "ai21": { "display_name": "AI21 (`ai21`)", "url": "https://docs.litellm.ai/docs/providers/ai21", diff --git a/tests/test_litellm/llms/api_route/test_api_route.py b/tests/test_litellm/llms/api_route/test_api_route.py new file mode 100644 index 00000000000..218f0590711 --- /dev/null +++ b/tests/test_litellm/llms/api_route/test_api_route.py @@ -0,0 +1,133 @@ +import json + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.api_route.chat.transformation import APIRouteChatConfig + + +def test_api_route_provider_routing(): + assert APIRouteChatConfig().custom_llm_provider == "api_route" + + model, provider, api_key, api_base = get_llm_provider( + model="api_route/model-name", + custom_llm_provider=None, + api_key="test-key", + api_base="https://example.com/v1", + ) + + assert model == "model-name" + assert provider == "api_route" + assert api_key == "test-key" + assert api_base == "https://example.com/v1" + + +def test_api_route_credentials_from_environment(monkeypatch): + monkeypatch.setenv("API_ROUTE_API_KEY", "env-key") + monkeypatch.setenv("API_ROUTE_BASE_URL", "https://env.example.com/v1") + + _, provider, api_key, api_base = get_llm_provider( + model="api_route/model-name", + custom_llm_provider=None, + api_key=None, + api_base=None, + ) + + assert provider == "api_route" + assert api_key == "env-key" + assert api_base == "https://env.example.com/v1" + + +def test_api_route_uses_default_base_url(monkeypatch): + monkeypatch.delenv("API_ROUTE_BASE_URL", raising=False) + + _, _, api_key, api_base = get_llm_provider( + model="api_route/model-name", + custom_llm_provider=None, + api_key="test-key", + api_base=None, + ) + + assert api_base == "https://global.api-route.com/v1" + assert api_key == "test-key" + + +def test_api_route_explicit_api_base_overrides_default(monkeypatch): + monkeypatch.delenv("API_ROUTE_BASE_URL", raising=False) + + _, _, _, api_base = get_llm_provider( + model="api_route/model-name", + custom_llm_provider=None, + api_key="test-key", + api_base="https://custom.example.com/v1", + ) + + assert api_base == "https://custom.example.com/v1" + + +def test_api_route_chat_completion_request(monkeypatch, respx_mock): + monkeypatch.setenv("API_ROUTE_BASE_URL", "https://env.example.com/v1") + route = respx_mock.post("https://env.example.com/v1/chat/completions").respond( + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "model-name", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + + response = litellm.completion( + model="api_route/model-name", + messages=[{"role": "user", "content": "Hello!"}], + api_key="test-key", + tools=tools, + ) + + request = route.calls[0].request + body = json.loads(request.content) + assert str(request.url) == "https://env.example.com/v1/chat/completions" + assert request.headers["Authorization"] == "Bearer test-key" + assert body["model"] == "model-name" + assert body["messages"] == [{"role": "user", "content": "Hello!"}] + assert body["tools"] == tools + assert response.choices[0].message.content == "Hello!" + + +def test_api_route_streaming_request(respx_mock): + route = respx_mock.post("https://stream.example.com/v1/chat/completions").respond( + headers={"content-type": "text/event-stream"}, + text='data: {"choices":[{"delta":{"content":"Hello"},"index":0}]}\n\ndata: [DONE]\n\n', + ) + + chunks = list( + litellm.completion( + model="api_route/model-name", + messages=[{"role": "user", "content": "Hello!"}], + api_key="stream-key", + api_base="https://stream.example.com/v1", + stream=True, + ) + ) + + body = json.loads(route.calls[0].request.content) + assert body["model"] == "model-name" + assert body["stream"] is True + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "Hello"