diff --git a/litellm/__init__.py b/litellm/__init__.py index eebd2dad91e..4ef3003c10a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -658,6 +658,7 @@ dashscope_models: Set = set() moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() +runinfra_models: Set = set() # mutable-ok: provider model registry, appended by add_known_models like every sibling set v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() @@ -913,6 +914,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None: publicai_models.add(key) elif value.get("litellm_provider") == "darkbloom": darkbloom_models.add(key) + elif value.get("litellm_provider") == "runinfra": + runinfra_models.add(key) elif value.get("litellm_provider") == "v0": v0_models.add(key) elif value.get("litellm_provider") == "morph": @@ -1071,6 +1074,7 @@ model_list = list( | moonshot_models | publicai_models | darkbloom_models + | runinfra_models | v0_models | morph_models | lambda_ai_models @@ -1178,6 +1182,7 @@ def _build_models_by_provider() -> dict: "moonshot": moonshot_models, "publicai": publicai_models, "darkbloom": darkbloom_models, + "runinfra": runinfra_models, "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, diff --git a/litellm/constants.py b/litellm/constants.py index 8713bd49f57..eb73920bfca 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -859,6 +859,7 @@ openai_compatible_providers: Final[list] = [ "pinstripes", # Pinstripes - JSON-configured provider "darkbloom", "meta", # Meta Model API (Muse Spark) - JSON-configured provider + "runinfra", "cognition", "scx-ai", ] diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index a458a209ea9..a12e894ff0a 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -189,6 +189,14 @@ }, "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"] }, + "runinfra": { + "base_url": "https://api.runinfra.ai/v1", + "api_key_env": "RUNINFRA_API_KEY", + "api_base_env": "RUNINFRA_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } + }, "scx-ai": { "base_url": "https://api.scx.ai/v1", "api_key_env": "SCX_API_KEY", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ad19818a5ce..8ad0a9b5234 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -50894,6 +50894,126 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://runinfra.ai/inference-api/qwen3-8-2-4t-a95b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/Qwen/Qwen3.8-27B": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://runinfra.ai/inference-api/qwen3-8-27b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "runinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 2.7e-07, + "source": "https://runinfra.ai/inference-api/deepseek-v4-flash", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.9e-06, + "source": "https://runinfra.ai/inference-api/deepseek-v4-pro", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": { + "input_cost_per_token": 5e-08, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://runinfra.ai/inference-api/nemotron-3-5-lightning-30b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/ornith-ai/Ornith-1.5-35B-A3B": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://runinfra.ai/inference-api/ornith-1-5-35b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "xai/grok-4.20-0309-non-reasoning": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 86c14fb4cd8..2b971181a7e 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1869,6 +1869,23 @@ "a2a": false } }, + "runinfra": { + "display_name": "RunInfra (`runinfra`)", + "url": "https://docs.litellm.ai/docs/providers/runinfra", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "predibase": { "display_name": "Predibase (`predibase`)", "url": "https://docs.litellm.ai/docs/providers/predibase", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ef3586f2559..0a6561aeab8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3817,6 +3817,7 @@ class LlmProviders(str, Enum): SCX_AI = "scx-ai" DARKBLOOM = "darkbloom" META = "meta" + RUNINFRA = "runinfra" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ad19818a5ce..8ad0a9b5234 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -50894,6 +50894,126 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://runinfra.ai/inference-api/qwen3-8-2-4t-a95b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/Qwen/Qwen3.8-27B": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://runinfra.ai/inference-api/qwen3-8-27b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "runinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 2.7e-07, + "source": "https://runinfra.ai/inference-api/deepseek-v4-flash", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.9e-06, + "source": "https://runinfra.ai/inference-api/deepseek-v4-pro", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": { + "input_cost_per_token": 5e-08, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://runinfra.ai/inference-api/nemotron-3-5-lightning-30b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/ornith-ai/Ornith-1.5-35B-A3B": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://runinfra.ai/inference-api/ornith-1-5-35b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "xai/grok-4.20-0309-non-reasoning": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 1d8d374c2c4..c6119646495 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2086,6 +2086,23 @@ "a2a": false } }, + "runinfra": { + "display_name": "RunInfra (`runinfra`)", + "url": "https://docs.litellm.ai/docs/providers/runinfra", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "predibase": { "display_name": "Predibase (`predibase`)", "url": "https://docs.litellm.ai/docs/providers/predibase", diff --git a/tests/test_litellm/llms/openai_like/test_json_providers.py b/tests/test_litellm/llms/openai_like/test_json_providers.py index c8743e1809d..1dbdd079ebb 100644 --- a/tests/test_litellm/llms/openai_like/test_json_providers.py +++ b/tests/test_litellm/llms/openai_like/test_json_providers.py @@ -338,6 +338,132 @@ class TestDarkbloom: assert model_cost[model]["output_cost_per_token"] == output_cost +class TestRuninfra: + def test_runinfra_json_config_exists(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + runinfra = JSONProviderRegistry.get("runinfra") + assert runinfra is not None + assert runinfra.base_url == "https://api.runinfra.ai/v1" + assert runinfra.api_key_env == "RUNINFRA_API_KEY" + assert runinfra.api_base_env == "RUNINFRA_API_BASE" + assert runinfra.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_runinfra_provider_resolution(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="runinfra/deepseek-ai/DeepSeek-V4-Flash-0731", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "deepseek-ai/DeepSeek-V4-Flash-0731" + assert provider == "runinfra" + assert api_key is None + assert api_base == "https://api.runinfra.ai/v1" + + def test_runinfra_dynamic_config(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("runinfra") + config_class = create_config_class(provider) + config = config_class() + + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://api.runinfra.ai/v1" + + api_base, api_key = config._get_openai_compatible_provider_info( + "https://custom.runinfra.ai/v1", "test-key" + ) + assert api_base == "https://custom.runinfra.ai/v1" + assert api_key == "test-key" + + def test_runinfra_complete_url_appends_endpoint(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("runinfra") + config_class = create_config_class(provider) + config = config_class() + + url = config.get_complete_url( + api_base="https://api.runinfra.ai/v1", + api_key="test-key", + model="runinfra/deepseek-ai/DeepSeek-V4-Flash-0731", + optional_params={}, + litellm_params={}, + stream=True, + ) + + assert url == "https://api.runinfra.ai/v1/chat/completions" + + def test_runinfra_provider_config_manager(self): + from litellm import LlmProviders + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_chat_config( + model="deepseek-ai/DeepSeek-V4-Flash-0731", provider=LlmProviders.RUNINFRA + ) + + assert config is not None + assert config.custom_llm_provider == "runinfra" + + def test_runinfra_model_cost_map(self): + with open( + os.path.join(workspace_path, "model_prices_and_context_window.json") + ) as f: + model_cost = json.load(f) + + expected_models = { + "runinfra/deepseek-ai/DeepSeek-V4-Flash-0731": (1.3e-07, 2.7e-07, 1e-08), + "runinfra/deepseek-ai/DeepSeek-V4-Pro-0813": (6e-07, 1.9e-06, 3e-08), + "runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": ( + 5e-08, + 1.5e-07, + None, + ), + "runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4": (2e-06, 6e-06, 2e-07), + "runinfra/Qwen/Qwen3.8-27B": (1e-07, 4e-07, 1e-08), + "runinfra/ornith-ai/Ornith-1.5-35B-A3B": (1e-07, 4e-07, 1e-08), + } + for model, (input_cost, output_cost, cache_read_cost) in expected_models.items(): + assert model in model_cost + assert model_cost[model]["litellm_provider"] == "runinfra" + assert model_cost[model]["max_output_tokens"] == 32768 + assert model_cost[model]["supports_function_calling"] is True + assert model_cost[model]["supports_tool_choice"] is True + assert model_cost[model]["supports_reasoning"] is True + assert model_cost[model]["input_cost_per_token"] == input_cost + assert model_cost[model]["output_cost_per_token"] == output_cost + if cache_read_cost is None: + assert "cache_read_input_token_cost" not in model_cost[model] + assert "supports_prompt_caching" not in model_cost[model] + else: + assert model_cost[model]["cache_read_input_token_cost"] == cache_read_cost + assert model_cost[model]["supports_prompt_caching"] is True + + assert model_cost["runinfra/deepseek-ai/DeepSeek-V4-Flash-0731"]["max_input_tokens"] == 1048576 + assert model_cost["runinfra/deepseek-ai/DeepSeek-V4-Pro-0813"]["max_input_tokens"] == 1048576 + assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["max_input_tokens"] == 262144 + assert model_cost["runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4"]["supports_response_schema"] is True + assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["supports_response_schema"] is True + assert model_cost["runinfra/ornith-ai/Ornith-1.5-35B-A3B"]["max_input_tokens"] == 262144 + # Image input is proven on exactly these two and declared on no other, + # so a copied flag on a text-only model reddens here. + for model, expected_vision in ( + ("runinfra/Qwen/Qwen3.8-27B", True), + ("runinfra/ornith-ai/Ornith-1.5-35B-A3B", True), + ("runinfra/deepseek-ai/DeepSeek-V4-Flash-0731", False), + ("runinfra/deepseek-ai/DeepSeek-V4-Pro-0813", False), + ("runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", False), + ("runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4", False), + ): + assert model_cost[model].get("supports_vision", False) is expected_vision + + class TestPublicAIIntegration: """Integration tests for PublicAI provider"""