From 18690ea83586ef5de6df844e5a3f30ad795787e2 Mon Sep 17 00:00:00 2001 From: jaberjaber23 Date: Sun, 16 Aug 2026 03:02:26 +0300 Subject: [PATCH 1/6] feat: add runinfra provider Registers runinfra as a JSON-configured openai_like provider routing to RunInfra's OpenAI compatible API at https://api.runinfra.ai/v1, with cost map entries for the four hosted models and mocked registration tests. --- litellm/__init__.py | 5 + litellm/constants.py | 1 + litellm/llms/openai_like/providers.json | 8 ++ ...odel_prices_and_context_window_backup.json | 75 ++++++++++++ .../provider_endpoints_support_backup.json | 17 +++ litellm/types/utils.py | 1 + model_prices_and_context_window.json | 75 ++++++++++++ provider_endpoints_support.json | 17 +++ .../llms/openai_like/test_json_providers.py | 111 ++++++++++++++++++ 9 files changed, 310 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 8961de940a0..ee6c9f47dd0 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -647,6 +647,7 @@ dashscope_models: Set = set() moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() +runinfra_models: Set = set() v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() @@ -900,6 +901,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None: publicai_models.add(key) elif value.get("litellm_provider") == "darkbloom": darkbloom_models.add(key) + elif value.get("litellm_provider") == "runinfra": + runinfra_models.add(key) elif value.get("litellm_provider") == "v0": v0_models.add(key) elif value.get("litellm_provider") == "morph": @@ -1058,6 +1061,7 @@ model_list = list( | moonshot_models | publicai_models | darkbloom_models + | runinfra_models | v0_models | morph_models | lambda_ai_models @@ -1165,6 +1169,7 @@ def _build_models_by_provider() -> dict: "moonshot": moonshot_models, "publicai": publicai_models, "darkbloom": darkbloom_models, + "runinfra": runinfra_models, "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, diff --git a/litellm/constants.py b/litellm/constants.py index 8f236eba327..1798c0954a1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -823,6 +823,7 @@ openai_compatible_providers: Final[list] = [ "pinstripes", # Pinstripes - JSON-configured provider "darkbloom", "meta", # Meta Model API (Muse Spark) - JSON-configured provider + "runinfra", ] openai_text_completion_compatible_providers: Final[list] = [ # providers that support `/v1/completions` "together_ai", diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 164100d4194..5b9cdc948f7 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -183,5 +183,13 @@ "max_completion_tokens": "max_tokens" }, "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"] + }, + "runinfra": { + "base_url": "https://api.runinfra.ai/v1", + "api_key_env": "RUNINFRA_API_KEY", + "api_base_env": "RUNINFRA_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } } } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e6c6cab0631..e88e3f858ac 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48000,6 +48000,81 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://runinfra.ai/inference-api/qwen3-8-2-4t-a95b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "runinfra/Qwen/Qwen3.8-27B": { + "input_cost_per_token": 1e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://runinfra.ai/inference-api/qwen3-8-27b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 2.7e-07, + "source": "https://runinfra.ai/inference-api/deepseek-v4-flash", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": { + "input_cost_per_token": 5e-08, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://runinfra.ai/inference-api/nemotron-3-5-lightning-30b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "xai/grok-4.20-0309-non-reasoning": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index dd7712aabca..e8d91d0987f 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1852,6 +1852,23 @@ "a2a": false } }, + "runinfra": { + "display_name": "RunInfra (`runinfra`)", + "url": "https://docs.litellm.ai/docs/providers/runinfra", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "predibase": { "display_name": "Predibase (`predibase`)", "url": "https://docs.litellm.ai/docs/providers/predibase", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 272fbabf807..c5962666159 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3727,6 +3727,7 @@ class LlmProviders(str, Enum): PINSTRIPES = "pinstripes" DARKBLOOM = "darkbloom" META = "meta" + RUNINFRA = "runinfra" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e6c6cab0631..e88e3f858ac 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48000,6 +48000,81 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://runinfra.ai/inference-api/qwen3-8-2-4t-a95b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "runinfra/Qwen/Qwen3.8-27B": { + "input_cost_per_token": 1e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://runinfra.ai/inference-api/qwen3-8-27b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 2.7e-07, + "source": "https://runinfra.ai/inference-api/deepseek-v4-flash", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": { + "input_cost_per_token": 5e-08, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://runinfra.ai/inference-api/nemotron-3-5-lightning-30b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "xai/grok-4.20-0309-non-reasoning": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 0712e8e383d..de74719131d 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2069,6 +2069,23 @@ "a2a": false } }, + "runinfra": { + "display_name": "RunInfra (`runinfra`)", + "url": "https://docs.litellm.ai/docs/providers/runinfra", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "predibase": { "display_name": "Predibase (`predibase`)", "url": "https://docs.litellm.ai/docs/providers/predibase", diff --git a/tests/test_litellm/llms/openai_like/test_json_providers.py b/tests/test_litellm/llms/openai_like/test_json_providers.py index c8743e1809d..26cf78629bf 100644 --- a/tests/test_litellm/llms/openai_like/test_json_providers.py +++ b/tests/test_litellm/llms/openai_like/test_json_providers.py @@ -338,6 +338,117 @@ class TestDarkbloom: assert model_cost[model]["output_cost_per_token"] == output_cost +class TestRuninfra: + def test_runinfra_json_config_exists(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + runinfra = JSONProviderRegistry.get("runinfra") + assert runinfra is not None + assert runinfra.base_url == "https://api.runinfra.ai/v1" + assert runinfra.api_key_env == "RUNINFRA_API_KEY" + assert runinfra.api_base_env == "RUNINFRA_API_BASE" + assert runinfra.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_runinfra_provider_resolution(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="runinfra/deepseek-ai/DeepSeek-V4-Flash-0731", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "deepseek-ai/DeepSeek-V4-Flash-0731" + assert provider == "runinfra" + assert api_key is None + assert api_base == "https://api.runinfra.ai/v1" + + def test_runinfra_dynamic_config(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("runinfra") + config_class = create_config_class(provider) + config = config_class() + + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://api.runinfra.ai/v1" + + api_base, api_key = config._get_openai_compatible_provider_info( + "https://custom.runinfra.ai/v1", "test-key" + ) + assert api_base == "https://custom.runinfra.ai/v1" + assert api_key == "test-key" + + def test_runinfra_complete_url_appends_endpoint(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("runinfra") + config_class = create_config_class(provider) + config = config_class() + + url = config.get_complete_url( + api_base="https://api.runinfra.ai/v1", + api_key="test-key", + model="runinfra/deepseek-ai/DeepSeek-V4-Flash-0731", + optional_params={}, + litellm_params={}, + stream=True, + ) + + assert url == "https://api.runinfra.ai/v1/chat/completions" + + def test_runinfra_provider_config_manager(self): + from litellm import LlmProviders + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_chat_config( + model="deepseek-ai/DeepSeek-V4-Flash-0731", provider=LlmProviders.RUNINFRA + ) + + assert config is not None + assert config.custom_llm_provider == "runinfra" + + def test_runinfra_model_cost_map(self): + with open( + os.path.join(workspace_path, "model_prices_and_context_window.json") + ) as f: + model_cost = json.load(f) + + expected_models = { + "runinfra/deepseek-ai/DeepSeek-V4-Flash-0731": (1.3e-07, 2.7e-07, 1e-08), + "runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": ( + 5e-08, + 1.5e-07, + None, + ), + "runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4": (2e-06, 6e-06, 2e-07), + "runinfra/Qwen/Qwen3.8-27B": (1e-07, 4e-07, None), + } + for model, (input_cost, output_cost, cache_read_cost) in expected_models.items(): + assert model in model_cost + assert model_cost[model]["litellm_provider"] == "runinfra" + assert model_cost[model]["max_output_tokens"] == 32768 + assert model_cost[model]["supports_function_calling"] is True + assert model_cost[model]["supports_tool_choice"] is True + assert model_cost[model]["supports_reasoning"] is True + assert model_cost[model]["input_cost_per_token"] == input_cost + assert model_cost[model]["output_cost_per_token"] == output_cost + if cache_read_cost is None: + assert "cache_read_input_token_cost" not in model_cost[model] + assert "supports_prompt_caching" not in model_cost[model] + else: + assert model_cost[model]["cache_read_input_token_cost"] == cache_read_cost + assert model_cost[model]["supports_prompt_caching"] is True + + assert model_cost["runinfra/deepseek-ai/DeepSeek-V4-Flash-0731"]["max_input_tokens"] == 1048576 + assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["max_input_tokens"] == 262144 + assert "supports_response_schema" not in model_cost["runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4"] + assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["supports_response_schema"] is True + + class TestPublicAIIntegration: """Integration tests for PublicAI provider""" From ee51908373db20ff5a69a89fb458e06b7ed6e331 Mon Sep 17 00:00:00 2001 From: jaberjaber23 Date: Sun, 16 Aug 2026 03:22:29 +0300 Subject: [PATCH 2/6] fix: add cache read pricing for runinfra Qwen3.8-27B Verified live on 2026-08-16: two identical 4,217 token prompts through the public endpoint, second response reported 4,160 cached prompt tokens; the model page lists $0.01 per million cached input tokens. --- litellm/model_prices_and_context_window_backup.json | 2 ++ model_prices_and_context_window.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e88e3f858ac..04121620479 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48020,6 +48020,7 @@ "supports_tool_choice": true }, "runinfra/Qwen/Qwen3.8-27B": { + "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, "litellm_provider": "runinfra", "max_input_tokens": 262144, @@ -48033,6 +48034,7 @@ ], "supports_function_calling": true, "supports_native_streaming": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e88e3f858ac..04121620479 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48020,6 +48020,7 @@ "supports_tool_choice": true }, "runinfra/Qwen/Qwen3.8-27B": { + "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, "litellm_provider": "runinfra", "max_input_tokens": 262144, @@ -48033,6 +48034,7 @@ ], "supports_function_calling": true, "supports_native_streaming": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true From 4509ebabb8199543497f7464b73bc21fe065ec46 Mon Sep 17 00:00:00 2001 From: jaberjaber23 Date: Sun, 16 Aug 2026 03:48:33 +0300 Subject: [PATCH 3/6] fix: annotate runinfra model set for the type budget, pin 27B cache pricing in tests The 27B cached input price was verified live after the original test pinned its absence; the test now pins the verified rate. The new provider set line carries a mutable-ok reason like the gate requires. --- litellm/__init__.py | 2 +- tests/test_litellm/llms/openai_like/test_json_providers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index ee6c9f47dd0..be25b918c0a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -647,7 +647,7 @@ dashscope_models: Set = set() moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() -runinfra_models: Set = set() +runinfra_models: Set = set() # mutable-ok: provider model registry, appended by add_known_models like every sibling set v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() diff --git a/tests/test_litellm/llms/openai_like/test_json_providers.py b/tests/test_litellm/llms/openai_like/test_json_providers.py index 26cf78629bf..cc8c570640b 100644 --- a/tests/test_litellm/llms/openai_like/test_json_providers.py +++ b/tests/test_litellm/llms/openai_like/test_json_providers.py @@ -425,7 +425,7 @@ class TestRuninfra: None, ), "runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4": (2e-06, 6e-06, 2e-07), - "runinfra/Qwen/Qwen3.8-27B": (1e-07, 4e-07, None), + "runinfra/Qwen/Qwen3.8-27B": (1e-07, 4e-07, 1e-08), } for model, (input_cost, output_cost, cache_read_cost) in expected_models.items(): assert model in model_cost From 323c90e91f0fea0d14e4aba4c8c4996894ca5a7e Mon Sep 17 00:00:00 2001 From: jaberjaber23 Date: Mon, 17 Aug 2026 12:29:50 +0300 Subject: [PATCH 4/6] fix: Qwen3.8 2.4T now supports response schema on runinfra JSON mode went live on the host on 2026-08-17, verified on the public endpoint: 3 of 3 json_object responses parsed, 3 of 3 strict json_schema responses validated, warm constrained requests at 0.35x baseline latency. The test that pinned the deliberate absence now pins the verified presence. --- litellm/model_prices_and_context_window_backup.json | 1 + model_prices_and_context_window.json | 1 + tests/test_litellm/llms/openai_like/test_json_providers.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 04121620479..db9e3e4c711 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48017,6 +48017,7 @@ "supports_native_streaming": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true }, "runinfra/Qwen/Qwen3.8-27B": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 04121620479..db9e3e4c711 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48017,6 +48017,7 @@ "supports_native_streaming": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true }, "runinfra/Qwen/Qwen3.8-27B": { diff --git a/tests/test_litellm/llms/openai_like/test_json_providers.py b/tests/test_litellm/llms/openai_like/test_json_providers.py index cc8c570640b..7c9b2f1a98b 100644 --- a/tests/test_litellm/llms/openai_like/test_json_providers.py +++ b/tests/test_litellm/llms/openai_like/test_json_providers.py @@ -445,7 +445,7 @@ class TestRuninfra: assert model_cost["runinfra/deepseek-ai/DeepSeek-V4-Flash-0731"]["max_input_tokens"] == 1048576 assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["max_input_tokens"] == 262144 - assert "supports_response_schema" not in model_cost["runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4"] + assert model_cost["runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4"]["supports_response_schema"] is True assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["supports_response_schema"] is True From 79e2537cbb811773d73b17b6a89963ac5a99754b Mon Sep 17 00:00:00 2001 From: jaberjaber23 Date: Tue, 18 Aug 2026 19:00:12 +0300 Subject: [PATCH 5/6] feat: add DeepSeek V4 Pro to the runinfra provider Fifth hosted model, USD per token from the published per-1M rates (0.60 input, 1.90 output, 0.03 cached input read), 1,048,576 context, 32,768 output cap. Verified live on the public endpoint on 2026-08-18 before this commit: json_object and strict json_schema 3 of 3 each, one tool call round-tripped, a prefix-cache hit on a repeated identical prompt, reasoning content present on the wire when enabled (reasoning is toggleable on this host), and the output cap refusing over-cap requests with 400. --- ...odel_prices_and_context_window_backup.json | 20 +++++++++++++++++++ model_prices_and_context_window.json | 20 +++++++++++++++++++ .../llms/openai_like/test_json_providers.py | 2 ++ 3 files changed, 42 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index db9e3e4c711..787c6b66367 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48060,6 +48060,26 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "runinfra/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.9e-06, + "source": "https://runinfra.ai/inference-api/deepseek-v4-pro", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": { "input_cost_per_token": 5e-08, "litellm_provider": "runinfra", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index db9e3e4c711..787c6b66367 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48060,6 +48060,26 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "runinfra/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.9e-06, + "source": "https://runinfra.ai/inference-api/deepseek-v4-pro", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": { "input_cost_per_token": 5e-08, "litellm_provider": "runinfra", diff --git a/tests/test_litellm/llms/openai_like/test_json_providers.py b/tests/test_litellm/llms/openai_like/test_json_providers.py index 7c9b2f1a98b..bf27307d8fc 100644 --- a/tests/test_litellm/llms/openai_like/test_json_providers.py +++ b/tests/test_litellm/llms/openai_like/test_json_providers.py @@ -419,6 +419,7 @@ class TestRuninfra: expected_models = { "runinfra/deepseek-ai/DeepSeek-V4-Flash-0731": (1.3e-07, 2.7e-07, 1e-08), + "runinfra/deepseek-ai/DeepSeek-V4-Pro-0813": (6e-07, 1.9e-06, 3e-08), "runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": ( 5e-08, 1.5e-07, @@ -444,6 +445,7 @@ class TestRuninfra: assert model_cost[model]["supports_prompt_caching"] is True assert model_cost["runinfra/deepseek-ai/DeepSeek-V4-Flash-0731"]["max_input_tokens"] == 1048576 + assert model_cost["runinfra/deepseek-ai/DeepSeek-V4-Pro-0813"]["max_input_tokens"] == 1048576 assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["max_input_tokens"] == 262144 assert model_cost["runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4"]["supports_response_schema"] is True assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["supports_response_schema"] is True From 404609ccc41d306bc220034d098c9d9a1943bcab Mon Sep 17 00:00:00 2001 From: jaberjaber23 Date: Sun, 23 Aug 2026 04:42:54 +0300 Subject: [PATCH 6/6] feat(runinfra): add Ornith 1.5 35B and declare image input on the two vision models Ornith 1.5 35B A3B went public on RunInfra after this PR was opened, so the cost map advertised five of our six conversational models and LiteLLM users could not route to the sixth at all. Prices are read from the production row and match the published rates: $0.10/M input, $0.40/M output, $0.01/M cached. Also declares supports_vision on the two models that accept image input, Qwen3.8 27B and Ornith. Both accept image_url and input_image content parts on the live gateway; the other four are text-only and stay unflagged. Without it a user filtering LiteLLM for vision-capable providers sees neither. Every value traces to a primary source: the production model row for prices and capabilities, and the public model page named in each entry's `source`. No capability is copied from a sibling entry. The per-model test table gains Ornith, plus a vision table that asserts the flag on exactly the two proven models and its ABSENCE on the other four, so a flag copied onto a text-only model fails rather than passing quietly. Verified by flipping Ornith's flag to false and watching test_runinfra_model_cost_map fail, then restoring it. Both price JSONs stay byte-identical, confirmed on the staged content. tests/test_litellm/llms/openai_like/test_json_providers.py: 24 passed, 4 skipped. Co-Authored-By: Claude Opus 5 (1M context) --- ...odel_prices_and_context_window_backup.json | 24 ++++++++++++++++++- model_prices_and_context_window.json | 24 ++++++++++++++++++- .../llms/openai_like/test_json_providers.py | 13 ++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2b2c5483d45..8f182b515e6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -49944,7 +49944,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "runinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 1e-08, @@ -50004,6 +50005,27 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "runinfra/ornith-ai/Ornith-1.5-35B-A3B": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://runinfra.ai/inference-api/ornith-1-5-35b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "xai/grok-4.20-0309-non-reasoning": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2b2c5483d45..8f182b515e6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -49944,7 +49944,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "runinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 1e-08, @@ -50004,6 +50005,27 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "runinfra/ornith-ai/Ornith-1.5-35B-A3B": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "runinfra", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://runinfra.ai/inference-api/ornith-1-5-35b", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "xai/grok-4.20-0309-non-reasoning": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, diff --git a/tests/test_litellm/llms/openai_like/test_json_providers.py b/tests/test_litellm/llms/openai_like/test_json_providers.py index bf27307d8fc..1dbdd079ebb 100644 --- a/tests/test_litellm/llms/openai_like/test_json_providers.py +++ b/tests/test_litellm/llms/openai_like/test_json_providers.py @@ -427,6 +427,7 @@ class TestRuninfra: ), "runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4": (2e-06, 6e-06, 2e-07), "runinfra/Qwen/Qwen3.8-27B": (1e-07, 4e-07, 1e-08), + "runinfra/ornith-ai/Ornith-1.5-35B-A3B": (1e-07, 4e-07, 1e-08), } for model, (input_cost, output_cost, cache_read_cost) in expected_models.items(): assert model in model_cost @@ -449,6 +450,18 @@ class TestRuninfra: assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["max_input_tokens"] == 262144 assert model_cost["runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4"]["supports_response_schema"] is True assert model_cost["runinfra/Qwen/Qwen3.8-27B"]["supports_response_schema"] is True + assert model_cost["runinfra/ornith-ai/Ornith-1.5-35B-A3B"]["max_input_tokens"] == 262144 + # Image input is proven on exactly these two and declared on no other, + # so a copied flag on a text-only model reddens here. + for model, expected_vision in ( + ("runinfra/Qwen/Qwen3.8-27B", True), + ("runinfra/ornith-ai/Ornith-1.5-35B-A3B", True), + ("runinfra/deepseek-ai/DeepSeek-V4-Flash-0731", False), + ("runinfra/deepseek-ai/DeepSeek-V4-Pro-0813", False), + ("runinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", False), + ("runinfra/Inferact/Qwen3.8-2.4T-A95B-NVFP4", False), + ): + assert model_cost[model].get("supports_vision", False) is expected_vision class TestPublicAIIntegration: