From bde3a39aa9ea3a2be58f8d38d391f26e703233fb Mon Sep 17 00:00:00 2001 From: "jinhao.song" Date: Tue, 23 Jun 2026 21:51:28 +0800 Subject: [PATCH] feat: add OrcaRouter provider (OpenAI-compatible meta-router) OrcaRouter (https://www.orcarouter.ai) is an OpenAI-compatible LLM gateway / adaptive router. This wires it in as a first-class provider so `orcarouter/` resolves automatically (base `https://api.orcarouter.ai/v1`, key `ORCAROUTER_API_KEY`), mirroring the existing `openrouter` / `vercel_ai_gateway` providers. - New `litellm/llms/orcarouter/` chat transformation (`OrcaRouterConfig`, extends `OpenAIGPTConfig`) + exception type. - Registered in `LlmProviders`, `provider_list`, `openai_compatible_providers` / `openai_compatible_endpoints`, `get_llm_provider` (forward + endpoint reverse-lookup), `ProviderConfigManager`, `get_supported_openai_params`, `validate_environment`, and the model/provider maps in `__init__`. - `extra_body` (routing preferences: `models` / `route`) is supported. - Model-id normalization: OrcaRouter routes by namespaced id, so a bare router name (e.g. `auto`) is sent as `orcarouter/auto` while namespaced ids (e.g. `openai/gpt-5`) pass through unchanged. - Unit tests under tests/test_litellm/llms/orcarouter/. Disclosure: I'm an engineer on the OrcaRouter team. --- litellm/__init__.py | 8 ++ litellm/_lazy_imports_registry.py | 5 + litellm/constants.py | 3 + .../get_llm_provider_logic.py | 8 ++ .../get_supported_openai_params.py | 2 + .../llms/orcarouter/chat/transformation.py | 87 ++++++++++++ litellm/llms/orcarouter/common_utils.py | 5 + litellm/types/utils.py | 1 + litellm/utils.py | 6 + provider_endpoints_support.json | 18 +++ .../chat/test_orcarouter_transformation.py | 134 ++++++++++++++++++ 11 files changed, 277 insertions(+) create mode 100644 litellm/llms/orcarouter/chat/transformation.py create mode 100644 litellm/llms/orcarouter/common_utils.py create mode 100644 tests/test_litellm/llms/orcarouter/chat/test_orcarouter_transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 15e95ded906..28a2441b5b1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -635,6 +635,7 @@ recraft_models: Set = set() cometapi_models: Set = set() oci_models: Set = set() vercel_ai_gateway_models: Set = set() +orcarouter_models: Set = set() volcengine_models: Set = set() wandb_models: Set = set(WANDB_MODELS) ovhcloud_models: Set = set() @@ -709,6 +710,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): openrouter_models.add(key) elif value.get("litellm_provider") == "vercel_ai_gateway": vercel_ai_gateway_models.add(key) + elif value.get("litellm_provider") == "orcarouter": + orcarouter_models.add(key) elif value.get("litellm_provider") == "datarobot": datarobot_models.add(key) elif value.get("litellm_provider") == "vertex_ai-text-models": @@ -1036,6 +1039,7 @@ model_list = list( | oci_models | heroku_models | vercel_ai_gateway_models + | orcarouter_models | volcengine_models | wandb_models | ovhcloud_models @@ -1063,6 +1067,7 @@ models_by_provider: dict = { "baseten": baseten_models, "openrouter": openrouter_models, "vercel_ai_gateway": vercel_ai_gateway_models, + "orcarouter": orcarouter_models, "datarobot": datarobot_models, "vertex_ai": vertex_chat_models | vertex_text_models @@ -1975,6 +1980,9 @@ if TYPE_CHECKING: from .llms.vercel_ai_gateway.chat.transformation import ( VercelAIGatewayConfig as VercelAIGatewayConfig, ) + from .llms.orcarouter.chat.transformation import ( + OrcaRouterConfig as OrcaRouterConfig, + ) from .llms.ovhcloud.chat.transformation import ( OVHCloudChatConfig as OVHCloudChatConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 4f131354d2e..020a7a9f7e2 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -316,6 +316,7 @@ LLM_CONFIG_NAMES = ( "InceptionChatConfig", "HyperbolicChatConfig", "VercelAIGatewayConfig", + "OrcaRouterConfig", "OVHCloudChatConfig", "OVHCloudEmbeddingConfig", "CometAPIEmbeddingConfig", @@ -1183,6 +1184,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.vercel_ai_gateway.chat.transformation", "VercelAIGatewayConfig", ), + "OrcaRouterConfig": ( + ".llms.orcarouter.chat.transformation", + "OrcaRouterConfig", + ), "OVHCloudChatConfig": (".llms.ovhcloud.chat.transformation", "OVHCloudChatConfig"), "OVHCloudEmbeddingConfig": ( ".llms.ovhcloud.embedding.transformation", diff --git a/litellm/constants.py b/litellm/constants.py index aeb74a65839..a9111acf4e0 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -540,6 +540,7 @@ LITELLM_CHAT_PROVIDERS = [ "lambda_ai", "inception", "vercel_ai_gateway", + "orcarouter", "wandb", "ovhcloud", "lemonade", @@ -705,6 +706,7 @@ openai_compatible_endpoints: List = [ "https://api.hyperbolic.xyz/v1", "https://ai-gateway.helicone.ai/", "https://ai-gateway.vercel.sh/v1", + "https://api.orcarouter.ai/v1", "https://api.inference.wandb.ai/v1", "https://api.clarifai.com/v2/ext/openai/v1", "https://api.libertai.io/v1", @@ -766,6 +768,7 @@ openai_compatible_providers: List = [ "inception", "hyperbolic", "vercel_ai_gateway", + "orcarouter", "aiml", "wandb", "cometapi", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 122d09c855b..081a1c5676a 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -340,6 +340,9 @@ def get_llm_provider( elif endpoint == "https://ai-gateway.vercel.sh/v1": custom_llm_provider = "vercel_ai_gateway" dynamic_api_key = get_secret_str("VERCEL_AI_GATEWAY_API_KEY") + elif endpoint == "https://api.orcarouter.ai/v1": + custom_llm_provider = "orcarouter" + dynamic_api_key = get_secret_str("ORCAROUTER_API_KEY") elif endpoint == "https://api.inference.wandb.ai/v1": custom_llm_provider = "wandb" dynamic_api_key = get_secret_str("WANDB_API_KEY") @@ -546,6 +549,11 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.PerplexityChatConfig()._get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "orcarouter": + ( + api_base, + dynamic_api_key, + ) = litellm.OrcaRouterConfig()._get_openai_compatible_provider_info(api_base, api_key) elif custom_llm_provider == "aiohttp_openai": return model, "aiohttp_openai", api_key, api_base elif custom_llm_provider == "anyscale": diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index c4ddb4b7ee0..7c57f331bc1 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -137,6 +137,8 @@ def get_supported_openai_params( return litellm.OpenrouterConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "vercel_ai_gateway": return litellm.VercelAIGatewayConfig().get_supported_openai_params(model=model) + elif custom_llm_provider == "orcarouter": + return litellm.OrcaRouterConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "mistral" or custom_llm_provider == "codestral": # mistal and codestral api have the exact same params if request_type == "chat_completion": diff --git a/litellm/llms/orcarouter/chat/transformation.py b/litellm/llms/orcarouter/chat/transformation.py new file mode 100644 index 00000000000..d41cc351c53 --- /dev/null +++ b/litellm/llms/orcarouter/chat/transformation.py @@ -0,0 +1,87 @@ +""" +Support for OpenAI's `/v1/chat/completions` endpoint. + +Calls done in OpenAI/openai.py as OrcaRouter is openai-compatible. + +Docs: https://docs.orcarouter.ai +""" + +from typing import Optional, Union + +import httpx + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig +from ..common_utils import OrcaRouterException + + +class OrcaRouterConfig(OpenAIGPTConfig): + @property + def custom_llm_provider(self) -> Optional[str]: + return "orcarouter" + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> tuple[Optional[str], Optional[str]]: + api_base = api_base or get_secret_str("ORCAROUTER_API_BASE") or "https://api.orcarouter.ai/v1" + dynamic_api_key = api_key or get_secret_str("ORCAROUTER_API_KEY") + return api_base, dynamic_api_key + + def get_supported_openai_params(self, model: str) -> list: + base_params = super().get_supported_openai_params(model) + # OrcaRouter accepts OpenAI's `extra_body` to carry routing preferences + # (e.g. {"models": [...], "route": "fallback"}). See docs.orcarouter.ai. + if "extra_body" not in base_params: + base_params.append("extra_body") + return base_params + + @staticmethod + def _normalize_orcarouter_model(model: str) -> str: + """Map a LiteLLM model name to the id OrcaRouter's API expects. + + OrcaRouter routes by *namespaced* model id. After LiteLLM strips the + ``orcarouter/`` provider prefix, a bare router/model name (no ``/``) + such as ``auto`` would be sent on its own and the backend responds + ``503 No available channel for model auto``. Such bare names must be + re-qualified as ``orcarouter/``. Names that already carry an + upstream namespace (e.g. ``openai/gpt-5``) are what the backend wants + and are sent unchanged. + + Note: this uses a simple "contains ``/``" heuristic. Every model + OrcaRouter serves is namespaced (``/``) and router + names are bare, so the only names without a ``/`` are routers that do + need the prefix. A hypothetical bare model whose own id contained a + ``/`` would be left unchanged; no such id exists in OrcaRouter's + catalog today. + """ + if "/" not in model: + return f"orcarouter/{model}" + return model + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + return super().transform_request( + self._normalize_orcarouter_model(model), + messages, + optional_params, + litellm_params, + headers, + ) + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return OrcaRouterException( + message=error_message, + status_code=status_code, + headers=headers, + ) diff --git a/litellm/llms/orcarouter/common_utils.py b/litellm/llms/orcarouter/common_utils.py new file mode 100644 index 00000000000..6946dd7977a --- /dev/null +++ b/litellm/llms/orcarouter/common_utils.py @@ -0,0 +1,5 @@ +from litellm.llms.base_llm.chat.transformation import BaseLLMException + + +class OrcaRouterException(BaseLLMException): + pass diff --git a/litellm/types/utils.py b/litellm/types/utils.py index dff4e4af89e..4a5c32b53f5 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3342,6 +3342,7 @@ class LlmProviders(str, Enum): OCI = "oci" AUTO_ROUTER = "auto_router" VERCEL_AI_GATEWAY = "vercel_ai_gateway" + ORCAROUTER = "orcarouter" DOTPROMPT = "dotprompt" MANUS = "manus" WANDB = "wandb" diff --git a/litellm/utils.py b/litellm/utils.py index 45ce5332f1d..c5215e062fd 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5875,6 +5875,11 @@ def validate_environment( keys_in_environment = True else: missing_keys.append("OPENROUTER_API_KEY") + elif custom_llm_provider == "orcarouter": + if "ORCAROUTER_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("ORCAROUTER_API_KEY") elif custom_llm_provider == "vercel_ai_gateway": if "VERCEL_AI_GATEWAY_API_KEY" in os.environ: keys_in_environment = True @@ -7606,6 +7611,7 @@ class ProviderConfigManager: lambda: litellm.VercelAIGatewayConfig(), False, ), + LlmProviders.ORCAROUTER: (lambda: litellm.OrcaRouterConfig(), False), LlmProviders.COMETAPI: (lambda: litellm.CometAPIConfig(), False), LlmProviders.DATAROBOT: (lambda: litellm.DataRobotConfig(), False), LlmProviders.GEMINI: (lambda: litellm.GoogleAIStudioGeminiConfig(), False), diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index b137ec59a1f..932f27ac3b8 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2414,6 +2414,24 @@ "interactions": true } }, + "orcarouter": { + "display_name": "OrcaRouter (`orcarouter`)", + "url": "https://docs.litellm.ai/docs/providers/orcarouter", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "vercel_ai_gateway": { "display_name": "Vercel AI Gateway (`vercel_ai_gateway`)", "url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway", diff --git a/tests/test_litellm/llms/orcarouter/chat/test_orcarouter_transformation.py b/tests/test_litellm/llms/orcarouter/chat/test_orcarouter_transformation.py new file mode 100644 index 00000000000..8cf7be94cdd --- /dev/null +++ b/tests/test_litellm/llms/orcarouter/chat/test_orcarouter_transformation.py @@ -0,0 +1,134 @@ +import os +import sys +from unittest.mock import patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +import litellm +from litellm.llms.orcarouter.chat.transformation import OrcaRouterConfig +from litellm.llms.orcarouter.common_utils import OrcaRouterException + + +class TestOrcaRouterProviderRouting: + """get_llm_provider must resolve orcarouter/* to the OrcaRouter endpoint.""" + + def test_router_model_routes_to_orcarouter(self): + model, provider, _, api_base = litellm.get_llm_provider( + model="orcarouter/auto", api_key="sk-orca-test" + ) + assert provider == "orcarouter" + assert api_base == "https://api.orcarouter.ai/v1" + + def test_namespaced_model_routes_to_orcarouter(self): + model, provider, _, api_base = litellm.get_llm_provider( + model="orcarouter/openai/gpt-5", api_key="sk-orca-test" + ) + assert provider == "orcarouter" + assert model == "openai/gpt-5" + assert api_base == "https://api.orcarouter.ai/v1" + + def test_api_key_from_environment(self): + with patch.dict("os.environ", {"ORCAROUTER_API_KEY": "sk-orca-env"}): + _, provider, api_key, _ = litellm.get_llm_provider(model="orcarouter/auto") + assert provider == "orcarouter" + assert api_key == "sk-orca-env" + + def test_validate_environment(self): + with patch.dict("os.environ", {"ORCAROUTER_API_KEY": "sk-orca-env"}): + result = litellm.validate_environment(model="orcarouter/auto") + assert result["keys_in_environment"] is True + assert "ORCAROUTER_API_KEY" not in result["missing_keys"] + + +class TestOrcaRouterModelNormalization: + """OrcaRouter routes by namespaced model id. + + A bare router/model name (no "/") must be re-qualified as + ``orcarouter/`` (otherwise the backend returns 503), while names + that already carry an upstream namespace are sent unchanged. + """ + + @pytest.mark.parametrize( + "model_in,model_out", + [ + ("auto", "orcarouter/auto"), + ("openai/gpt-5", "openai/gpt-5"), + ("anthropic/claude-opus-4.8", "anthropic/claude-opus-4.8"), + ("orcarouter/auto", "orcarouter/auto"), + ], + ) + def test_normalize_orcarouter_model(self, model_in, model_out): + assert OrcaRouterConfig._normalize_orcarouter_model(model_in) == model_out + + def test_transform_request_qualifies_bare_router_name(self): + data = OrcaRouterConfig().transform_request( + model="auto", + messages=[{"role": "user", "content": "Hello, world!"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert data["model"] == "orcarouter/auto" + + def test_transform_request_preserves_namespaced_model(self): + data = OrcaRouterConfig().transform_request( + model="openai/gpt-5", + messages=[{"role": "user", "content": "Hello, world!"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert data["model"] == "openai/gpt-5" + + def test_extra_body_routing_preferences_pass_through(self): + data = OrcaRouterConfig().transform_request( + model="auto", + messages=[{"role": "user", "content": "Hello, world!"}], + optional_params={ + "extra_body": {"models": ["openai/gpt-5"], "route": "fallback"} + }, + litellm_params={}, + headers={}, + ) + assert data["extra_body"]["models"] == ["openai/gpt-5"] + assert data["extra_body"]["route"] == "fallback" + + +class TestOrcaRouterConfig: + def test_get_openai_compatible_provider_info(self): + config = OrcaRouterConfig() + with patch.dict( + "os.environ", + { + "ORCAROUTER_API_BASE": "https://env.orcarouter.ai/v1", + "ORCAROUTER_API_KEY": "sk-orca-env", + }, + ): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://env.orcarouter.ai/v1" + assert api_key == "sk-orca-env" + + def test_get_supported_openai_params_includes_extra_body(self): + supported_params = OrcaRouterConfig().get_supported_openai_params("auto") + assert "extra_body" in supported_params + assert "temperature" in supported_params + assert "stream" in supported_params + + def test_custom_llm_provider(self): + assert OrcaRouterConfig().custom_llm_provider == "orcarouter" + + def test_error_class(self): + config = OrcaRouterConfig() + err = config.get_error_class("boom", 400, {"Content-Type": "application/json"}) + assert isinstance(err, OrcaRouterException) + assert err.status_code == 400 + + def test_exception_inheritance(self): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + exception = OrcaRouterException(message="test", status_code=500, headers={}) + assert isinstance(exception, BaseLLMException)