scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-28 06:23:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 94

Merge a47c0a8383 into f2118fc767

Browse source
This commit is contained in:
Jaber Jaber 2026-10-22 02:16:38 +00:00 • committed by GitHub
commit 77f6ac2ca0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 474 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/__init__.py
View file

@ -680,6 +680,7 @@ moonshot_models: Set = set()
publicai_models: Set = set()
darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
runinfra_models: Set = set() # mutable-ok: provider model registry, appended by add_known_models like every sibling set
v0_models: Set = set()
morph_models: Set = set()
lambda_ai_models: Set = set()
@ -966,6 +967,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
darkbloom_models.add(key)
elif value.get("litellm_provider") == "openinfer":
openinfer_models.add(key)
elif value.get("litellm_provider") == "runinfra":
runinfra_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
@ -1135,6 +1138,7 @@ model_list = list(
| publicai_models
| darkbloom_models
| openinfer_models
| runinfra_models
| v0_models
| morph_models
| lambda_ai_models
@ -1252,6 +1256,7 @@ def _build_models_by_provider() -> dict:
"publicai": publicai_models,
"darkbloom": darkbloom_models,
"openinfer": openinfer_models,
"runinfra": runinfra_models,
"v0": v0_models,
"morph": morph_models,
"lambda_ai": lambda_ai_models,

1
litellm/constants.py
View file

@ -1131,6 +1131,7 @@ openai_compatible_providers: Final[list[str]] = [
"openinfer",
"darkbloom",
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
"runinfra",
"cognition",
"coralbricks",
"cortecs",

8
litellm/llms/openai_like/providers.json
View file

@ -201,6 +201,14 @@
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
},
"runinfra": {
"base_url": "https://api.runinfra.ai/v1",
"api_key_env": "RUNINFRA_API_KEY",
"api_base_env": "RUNINFRA_API_BASE",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
},
"scx-ai": {
"base_url": "https://api.scx.ai/v1",
"api_key_env": "SCX_API_KEY",

144
litellm/model_prices_and_context_window_backup.json
View file

@ -63071,6 +63071,150 @@
"supports_system_messages": true,
"supports_tool_choice": true
},
"runinfra/Qwen/Qwen3.8-27B": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
"litellm_provider": "runinfra",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4e-07,
"source": "https://runinfra.ai/inference-api/qwen3-8-27b",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"runinfra/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "runinfra",
"max_input_tokens": 1048576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 5.8e-07,
"source": "https://runinfra.ai/inference-api/deepseek-v4-1-flash",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"runinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "runinfra",
"max_input_tokens": 1048576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://runinfra.ai/inference-api/deepseek-v4-1-flash-1017",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 5e-08,
"litellm_provider": "runinfra",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1.5e-07,
"source": "https://runinfra.ai/inference-api/nemotron-3-5-lightning-30b",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"runinfra/ornith-ai/Ornith-1.5-35B-A3B": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
"litellm_provider": "runinfra",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4e-07,
"source": "https://runinfra.ai/inference-api/ornith-1-5-35b",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"runinfra/zai-org/GLM-5.3": {
"cache_read_input_token_cost": 1.3e-07,
"input_cost_per_token": 1.3e-06,
"litellm_provider": "runinfra",
"max_input_tokens": 1048576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4.2e-06,
"source": "https://runinfra.ai/inference-api/glm-5-3",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"runinfra/zai-org/GLM-5.3-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.1e-07,
"litellm_provider": "runinfra",
"max_input_tokens": 1048576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4.5e-07,
"source": "https://runinfra.ai/inference-api/glm-5-3-flash",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"xai/grok-4.20-0309-non-reasoning": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,

17
litellm/provider_endpoints_support_backup.json
View file

@ -1961,6 +1961,23 @@
"a2a": false
}
},
"runinfra": {
"display_name": "RunInfra (`runinfra`)",
"url": "https://docs.litellm.ai/docs/providers/runinfra",
"endpoints": {
"chat_completions": true,
"messages": false,
"responses": false,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": false
}
},
"predibase": {
"display_name": "Predibase (`predibase`)",
"url": "https://docs.litellm.ai/docs/providers/predibase",

1
litellm/types/utils.py
View file

@ -4553,6 +4553,7 @@ class LlmProviders(str, Enum):
META = "meta"
SAIL = "sail"
REKA = "reka"
RUNINFRA = "runinfra"
LITELLM_AGENT = "litellm_agent"
CURSOR = "cursor"
BEDROCK_MANTLE = "bedrock_mantle"

144
model_prices_and_context_window.json
View file

@ -63071,6 +63071,150 @@
"supports_system_messages": true,
"supports_tool_choice": true
},
"runinfra/Qwen/Qwen3.8-27B": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
"litellm_provider": "runinfra",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4e-07,
"source": "https://runinfra.ai/inference-api/qwen3-8-27b",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"runinfra/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "runinfra",
"max_input_tokens": 1048576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 5.8e-07,
"source": "https://runinfra.ai/inference-api/deepseek-v4-1-flash",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"runinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "runinfra",
"max_input_tokens": 1048576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://runinfra.ai/inference-api/deepseek-v4-1-flash-1017",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 5e-08,
"litellm_provider": "runinfra",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1.5e-07,
"source": "https://runinfra.ai/inference-api/nemotron-3-5-lightning-30b",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"runinfra/ornith-ai/Ornith-1.5-35B-A3B": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
"litellm_provider": "runinfra",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4e-07,
"source": "https://runinfra.ai/inference-api/ornith-1-5-35b",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"runinfra/zai-org/GLM-5.3": {
"cache_read_input_token_cost": 1.3e-07,
"input_cost_per_token": 1.3e-06,
"litellm_provider": "runinfra",
"max_input_tokens": 1048576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4.2e-06,
"source": "https://runinfra.ai/inference-api/glm-5-3",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"runinfra/zai-org/GLM-5.3-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.1e-07,
"litellm_provider": "runinfra",
"max_input_tokens": 1048576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4.5e-07,
"source": "https://runinfra.ai/inference-api/glm-5-3-flash",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"xai/grok-4.20-0309-non-reasoning": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,

17
provider_endpoints_support.json
View file

@ -2234,6 +2234,23 @@
"a2a": false
}
},
"runinfra": {
"display_name": "RunInfra (`runinfra`)",
"url": "https://docs.litellm.ai/docs/providers/runinfra",
"endpoints": {
"chat_completions": true,
"messages": false,
"responses": false,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": false
}
},
"predibase": {
"display_name": "Predibase (`predibase`)",
"url": "https://docs.litellm.ai/docs/providers/predibase",

137
tests/unit/llms/openai_like/test_json_providers.py
View file

@ -2,6 +2,7 @@
Tests for JSON-based provider configuration system.
"""
import json
import os
import sys
from unittest.mock import patch
@ -309,3 +310,139 @@ class TestDarkbloom:
assert config is not None
assert config.custom_llm_provider == "darkbloom"
class TestRuninfra:
def test_runinfra_json_config_exists(self):
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
runinfra = JSONProviderRegistry.get("runinfra")
assert runinfra is not None
assert runinfra.base_url == "https://api.runinfra.ai/v1"
assert runinfra.api_key_env == "RUNINFRA_API_KEY"
assert runinfra.api_base_env == "RUNINFRA_API_BASE"
assert runinfra.param_mappings.get("max_completion_tokens") == "max_tokens"
def test_runinfra_provider_resolution(self):
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
model, provider, api_key, api_base = get_llm_provider(
model="runinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017",
custom_llm_provider=None,
api_base=None,
api_key=None,
)
assert model == "deepseek-ai/DeepSeek-V4.1-Flash-1017"
assert provider == "runinfra"
assert api_key is None
assert api_base == "https://api.runinfra.ai/v1"
def test_runinfra_dynamic_config(self):
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
provider = JSONProviderRegistry.get("runinfra")
config_class = create_config_class(provider)
config = config_class()
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
assert api_base == "https://api.runinfra.ai/v1"
api_base, api_key = config._get_openai_compatible_provider_info(
"https://custom.runinfra.ai/v1", "test-key"
)
assert api_base == "https://custom.runinfra.ai/v1"
assert api_key == "test-key"
def test_runinfra_complete_url_appends_endpoint(self):
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
provider = JSONProviderRegistry.get("runinfra")
config_class = create_config_class(provider)
config = config_class()
url = config.get_complete_url(
api_base="https://api.runinfra.ai/v1",
api_key="test-key",
model="runinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017",
optional_params={},
litellm_params={},
stream=True,
)
assert url == "https://api.runinfra.ai/v1/chat/completions"
def test_runinfra_provider_config_manager(self):
from litellm import LlmProviders
from litellm.utils import ProviderConfigManager
config = ProviderConfigManager.get_provider_chat_config(
model="deepseek-ai/DeepSeek-V4.1-Flash-1017", provider=LlmProviders.RUNINFRA
)
assert config is not None
assert config.custom_llm_provider == "runinfra"
def test_runinfra_model_cost_map(self):
with open(
os.path.join(workspace_path, "model_prices_and_context_window.json")
) as f:
model_cost = json.load(f)
expected_models = {
"runinfra/deepseek-ai/DeepSeek-V4.1-Flash": (1.4e-07, 5.8e-07, 3e-08),
"runinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017": (1.5e-07, 6e-07, 1e-08),
"runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": (
5e-08,
1.5e-07,
1e-08,
),
"runinfra/Qwen/Qwen3.8-27B": (1e-07, 4e-07, 1e-08),
"runinfra/ornith-ai/Ornith-1.5-35B-A3B": (1e-07, 4e-07, 1e-08),
"runinfra/zai-org/GLM-5.3-Flash": (1.1e-07, 4.5e-07, 3e-08),
"runinfra/zai-org/GLM-5.3": (1.3e-06, 4.2e-06, 1.3e-07),
}
# Every RunInfra entry is pinned here, so a retired model left in the
# cost map, or a new one added without its prices, reddens this test.
assert {
k for k, v in model_cost.items() if v.get("litellm_provider") == "runinfra"
} == set(expected_models)
for model, (input_cost, output_cost, cache_read_cost) in expected_models.items():
assert model in model_cost
assert model_cost[model]["litellm_provider"] == "runinfra"
assert model_cost[model]["max_output_tokens"] == 32768
assert model_cost[model]["supports_function_calling"] is True
assert model_cost[model]["supports_tool_choice"] is True
assert model_cost[model]["supports_reasoning"] is True
assert model_cost[model]["input_cost_per_token"] == input_cost
assert model_cost[model]["output_cost_per_token"] == output_cost
if cache_read_cost is None:
assert "cache_read_input_token_cost" not in model_cost[model]
assert "supports_prompt_caching" not in model_cost[model]
else:
assert model_cost[model]["cache_read_input_token_cost"] == cache_read_cost
assert model_cost[model]["supports_prompt_caching"] is True
assert model_cost["runinfra/deepseek-ai/DeepSeek-V4.1-Flash"]["max_input_tokens"] == 1048576
assert model_cost["runinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017"]["max_input_tokens"] == 1048576
assert model_cost["runinfra/zai-org/GLM-5.3"]["max_input_tokens"] == 1048576
assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["max_input_tokens"] == 262144
assert model_cost["runinfra/zai-org/GLM-5.3"]["supports_response_schema"] is True
assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["supports_response_schema"] is True
assert model_cost["runinfra/ornith-ai/Ornith-1.5-35B-A3B"]["max_input_tokens"] == 262144
# Image input is declared on exactly the four models that RunInfra's
# provider feed lists with an image input modality (Qwen3.8-27B,
# Ornith-1.5-35B-A3B, GLM-5.3-Flash and DeepSeek-V4.1-Flash-1017) and
# on no other, so a copied flag on a text-only model reddens here.
for model, expected_vision in (
("runinfra/Qwen/Qwen3.8-27B", True),
("runinfra/ornith-ai/Ornith-1.5-35B-A3B", True),
("runinfra/zai-org/GLM-5.3-Flash", True),
("runinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017", True),
("runinfra/deepseek-ai/DeepSeek-V4.1-Flash", False),
("runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", False),
("runinfra/zai-org/GLM-5.3", False),
):
assert model_cost[model].get("supports_vision", False) is expected_vision