scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-05 02:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 49

feat: add API Route provider support

Browse source
This commit is contained in:
DennyHo0917 2026-09-06 14:37:39 +08:00
parent 4104868458
commit 98efb067a3
10 changed files with 246 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/__init__.py
View file

@ -1965,6 +1965,9 @@ if TYPE_CHECKING:
from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig
from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig
from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig
from .llms.api_route.chat.transformation import (
APIRouteChatConfig as APIRouteChatConfig,
)
from .llms.volcengine.chat.transformation import (
VolcEngineChatConfig as VolcEngineChatConfig,
VolcEngineChatConfig as VolcEngineConfig,

5
litellm/_lazy_imports_registry.py
View file

@ -270,6 +270,7 @@ LLM_CONFIG_NAMES: Final = (
"XAIChatConfig",
"ZAIChatConfig",
"AIMLChatConfig",
"APIRouteChatConfig",
"VolcEngineChatConfig",
"CodestralTextCompletionConfig",
"InceptionTextCompletionConfig",
@ -1057,6 +1058,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"XAIChatConfig": (".llms.xai.chat.transformation", "XAIChatConfig"),
"ZAIChatConfig": (".llms.zai.chat.transformation", "ZAIChatConfig"),
"AIMLChatConfig": (".llms.aiml.chat.transformation", "AIMLChatConfig"),
"APIRouteChatConfig": (
".llms.api_route.chat.transformation",
"APIRouteChatConfig",
),
"VolcEngineChatConfig": (
".llms.volcengine.chat.transformation",
"VolcEngineChatConfig",

1
litellm/constants.py
View file

@ -936,6 +936,7 @@ openai_compatible_providers: Final[list] = [
"hyperbolic",
"vercel_ai_gateway",
"aiml",
"api_route",
"wandb",
"cometapi",
"clarifai",

5
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -849,6 +849,11 @@ def _get_openai_compatible_provider_info(
api_base,
dynamic_api_key,
) = litellm.AIMLChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "api_route":
(
api_base,
dynamic_api_key,
) = litellm.APIRouteChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "wandb":
api_base = api_base or get_secret("WANDB_API_BASE") or "https://api.inference.wandb.ai/v1"
dynamic_api_key = api_key or get_secret_str("WANDB_API_KEY")

19
litellm/llms/api_route/chat/transformation.py Normal file
View file

@ -0,0 +1,19 @@
from typing import Final
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.secret_managers.main import get_secret_str
class APIRouteChatConfig(OpenAIGPTConfig):
@property
def custom_llm_provider(self) -> str | None:
return "api_route"
def _get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None
) -> tuple[str | None, str | None]:
resolved_api_base: Final = api_base or get_secret_str("API_ROUTE_BASE_URL")
if not resolved_api_base:
raise ValueError("API Route requires API_ROUTE_BASE_URL or api_base parameter.")
resolved_api_key: Final = api_key or get_secret_str("API_ROUTE_API_KEY")
return resolved_api_base, resolved_api_key

17
litellm/provider_endpoints_support_backup.json
View file

@ -84,6 +84,23 @@
"interactions": true
}
},
"api_route": {
"display_name": "API Route (`api_route`)",
"url": "https://www.api-route.com/docs/overview",
"endpoints": {
"chat_completions": true,
"messages": false,
"responses": false,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": false
}
},
"ai21": {
"display_name": "AI21 (`ai21`)",
"url": "https://docs.litellm.ai/docs/providers/ai21",

1
litellm/types/utils.py
View file

@ -3949,6 +3949,7 @@ class LlmProviders(str, Enum):
STABILITY = "stability"
HEROKU = "heroku"
AIML = "aiml"
API_ROUTE = "api_route"
COMETAPI = "cometapi"
OCI = "oci"
AUTO_ROUTER = "auto_router"

1
litellm/utils.py
View file

@ -8072,6 +8072,7 @@ class ProviderConfigManager:
LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False),
LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIChatConfig(), False),
LlmProviders.OPENROUTER: (lambda: litellm.OpenrouterConfig(), False),
LlmProviders.API_ROUTE: (lambda: litellm.APIRouteChatConfig(), False),
LlmProviders.VERCEL_AI_GATEWAY: (
lambda: litellm.VercelAIGatewayConfig(),
False,

17
provider_endpoints_support.json
View file

@ -84,6 +84,23 @@
"interactions": true
}
},
"api_route": {
"display_name": "API Route (`api_route`)",
"url": "https://www.api-route.com/docs/overview",
"endpoints": {
"chat_completions": true,
"messages": false,
"responses": false,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": false
}
},
"ai21": {
"display_name": "AI21 (`ai21`)",
"url": "https://docs.litellm.ai/docs/providers/ai21",

177
tests/test_litellm/llms/api_route/test_api_route.py Normal file
View file

@ -0,0 +1,177 @@
import json
from unittest.mock import patch
import httpx
import pytest
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.api_route.chat.transformation import APIRouteChatConfig
from litellm.llms.openai.openai import OpenAIChatCompletion
def test_api_route_provider_routing():
model, provider, api_key, api_base = get_llm_provider(
model="api_route/model-name",
custom_llm_provider=None,
api_key="test-key",
api_base="https://example.com/v1",
)
assert model == "model-name"
assert provider == "api_route"
assert api_key == "test-key"
assert api_base == "https://example.com/v1"
def test_api_route_credentials_from_environment(monkeypatch):
monkeypatch.setenv("API_ROUTE_API_KEY", "env-key")
monkeypatch.setenv("API_ROUTE_BASE_URL", "https://env.example.com/v1")
_, provider, api_key, api_base = get_llm_provider(
model="api_route/model-name",
custom_llm_provider=None,
api_key=None,
api_base=None,
)
assert provider == "api_route"
assert api_key == "env-key"
assert api_base == "https://env.example.com/v1"
def test_api_route_requires_base_url(monkeypatch):
monkeypatch.delenv("API_ROUTE_BASE_URL", raising=False)
with pytest.raises(
ValueError,
match=r"API Route requires API_ROUTE_BASE_URL or api_base parameter\.",
):
APIRouteChatConfig()._get_openai_compatible_provider_info(
api_base=None,
api_key="test-key",
)
def test_api_route_chat_completion_request(monkeypatch):
captured = {}
monkeypatch.setenv("API_ROUTE_BASE_URL", "https://env.example.com/v1")
def handler(request: httpx.Request) -> httpx.Response:
captured["request"] = request
return httpx.Response(
status_code=200,
json={
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 1,
"model": "model-name",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "Hello!"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
request=request,
)
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather",
"parameters": {"type": "object", "properties": {}},
},
}
]
with httpx.Client(transport=httpx.MockTransport(handler)) as http_client:
with (
patch.object(
OpenAIChatCompletion,
"get_cached_openai_client",
return_value=None,
),
patch.object(
OpenAIChatCompletion,
"set_cached_openai_client",
),
patch.object(
OpenAIChatCompletion,
"_get_sync_http_client",
return_value=http_client,
),
):
response = litellm.completion(
model="api_route/model-name",
messages=[{"role": "user", "content": "Hello!"}],
api_key="test-key",
tools=tools,
)
request = captured["request"]
body = json.loads(request.content)
assert str(request.url) == "https://env.example.com/v1/chat/completions"
assert request.headers["Authorization"] == "Bearer test-key"
assert body["model"] == "model-name"
assert body["messages"] == [{"role": "user", "content": "Hello!"}]
assert body["tools"] == tools
assert response.choices[0].message.content == "Hello!"
def test_api_route_streaming_request():
captured = {}
def handler(request: httpx.Request) -> httpx.Response:
captured["request"] = request
content = """data: {"id":"chatcmpl-stream","object":"chat.completion.chunk","created":1,"model":"model-name","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]}
data: {"id":"chatcmpl-stream","object":"chat.completion.chunk","created":1,"model":"model-name","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
data: [DONE]
"""
return httpx.Response(
status_code=200,
headers={"content-type": "text/event-stream"},
content=content,
request=request,
)
with httpx.Client(transport=httpx.MockTransport(handler)) as http_client:
with (
patch.object(
OpenAIChatCompletion,
"get_cached_openai_client",
return_value=None,
),
patch.object(
OpenAIChatCompletion,
"set_cached_openai_client",
),
patch.object(
OpenAIChatCompletion,
"_get_sync_http_client",
return_value=http_client,
),
):
response = litellm.completion(
model="api_route/model-name",
messages=[{"role": "user", "content": "Hello!"}],
api_key="test-key",
api_base="https://stream.example.com/v1",
stream=True,
)
chunks = list(response)
request = captured["request"]
body = json.loads(request.content)
text = "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices)
assert str(request.url) == "https://stream.example.com/v1/chat/completions"
assert body["model"] == "model-name"
assert body["stream"] is True
assert text == "Hello"