From bf40f9be110b552b676718936dd4ba11af4d5151 Mon Sep 17 00:00:00 2001 From: katestover Date: Wed, 1 Apr 2026 12:32:09 +0100 Subject: [PATCH] Add IO Intelligence provider support Adds IO Intelligence (io.net) as a new OpenAI-compatible LLM provider with support for chat completions and embeddings. - Chat completion support with streaming, function calling, and tool use - Embedding support via BAAI/bge-multilingual-gemma2 model - 17 models including Llama, DeepSeek, Qwen, Mistral, and GPT-OSS variants - Model pricing configuration in model_prices_and_context_window.json - LiteLLM dashboard UI integration - Unit tests for provider config, routing, and mock API calls API Base: https://api.intelligence.io.solutions/api/v1 Auth: IO_INTELLIGENCE_API_KEY environment variable Co-Authored-By: Claude Opus 4.6 --- litellm/__init__.py | 4 + litellm/constants.py | 3 + .../get_llm_provider_logic.py | 10 + litellm/llms/iointelligence/__init__.py | 0 litellm/llms/iointelligence/chat.py | 89 +++++++ litellm/llms/iointelligence/common_utils.py | 6 + .../llms/iointelligence/embedding/__init__.py | 0 .../embedding/transformation.py | 131 ++++++++++ litellm/main.py | 23 ++ litellm/types/utils.py | 1 + litellm/utils.py | 3 + model_prices_and_context_window.json | 204 ++++++++++++++++ .../llms/iointelligence/__init__.py | 0 .../iointelligence/test_iointelligence.py | 223 ++++++++++++++++++ .../src/components/provider_info_helpers.tsx | 3 + 15 files changed, 700 insertions(+) create mode 100644 litellm/llms/iointelligence/__init__.py create mode 100644 litellm/llms/iointelligence/chat.py create mode 100644 litellm/llms/iointelligence/common_utils.py create mode 100644 litellm/llms/iointelligence/embedding/__init__.py create mode 100644 litellm/llms/iointelligence/embedding/transformation.py create mode 100644 tests/test_litellm/llms/iointelligence/__init__.py create mode 100644 tests/test_litellm/llms/iointelligence/test_iointelligence.py diff --git a/litellm/__init__.py b/litellm/__init__.py index e45d926e8db..b50375e7fd9 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1757,6 +1757,10 @@ if TYPE_CHECKING: from .llms.sambanova.embedding.transformation import ( SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig, ) + from .llms.iointelligence.chat import IOIntelligenceConfig as IOIntelligenceConfig + from .llms.iointelligence.embedding.transformation import ( + IOIntelligenceEmbeddingConfig as IOIntelligenceEmbeddingConfig, + ) from .llms.fireworks_ai.chat.transformation import ( FireworksAIConfig as FireworksAIConfig, ) diff --git a/litellm/constants.py b/litellm/constants.py index 252068bd7b0..a9a0968fb02 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -533,6 +533,7 @@ LITELLM_CHAT_PROVIDERS = [ "text-completion-codestral", "deepseek", "sambanova", + "io_intelligence", "maritalk", "cloudflare", "fireworks_ai", @@ -706,6 +707,7 @@ openai_compatible_endpoints: List = [ "app.empower.dev/api/v1", "https://api.friendli.ai/serverless/v1", "api.sambanova.ai/v1", + "api.intelligence.io.solutions/api/v1", "api.x.ai/v1", "ollama.com", "api.galadriel.ai/v1", @@ -739,6 +741,7 @@ openai_compatible_providers: List = [ "cerebras", "baseten", "sambanova", + "io_intelligence", "ai21_chat", "ai21", "volcengine", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 0a2f07bcb25..585c9c010a4 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -238,6 +238,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.sambanova.ai/v1": custom_llm_provider = "sambanova" dynamic_api_key = get_secret_str("SAMBANOVA_API_KEY") + elif endpoint == "https://api.intelligence.io.solutions/api/v1": + custom_llm_provider = "io_intelligence" + dynamic_api_key = get_secret_str("IO_INTELLIGENCE_API_KEY") elif endpoint == "https://api.ai21.com/studio/v1": custom_llm_provider = "ai21_chat" dynamic_api_key = get_secret_str("AI21_API_KEY") @@ -605,6 +608,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or "https://api.sambanova.ai/v1" ) # type: ignore dynamic_api_key = api_key or get_secret_str("SAMBANOVA_API_KEY") + elif custom_llm_provider == "io_intelligence": + api_base = ( + api_base + or get_secret("IO_INTELLIGENCE_API_BASE") + or "https://api.intelligence.io.solutions/api/v1" + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("IO_INTELLIGENCE_API_KEY") elif custom_llm_provider == "meta_llama": api_base = ( api_base diff --git a/litellm/llms/iointelligence/__init__.py b/litellm/llms/iointelligence/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/iointelligence/chat.py b/litellm/llms/iointelligence/chat.py new file mode 100644 index 00000000000..57392e0dd52 --- /dev/null +++ b/litellm/llms/iointelligence/chat.py @@ -0,0 +1,89 @@ +""" +IO Intelligence Chat Completions API + +This is OpenAI compatible - no translation needed / occurs. +""" + +from typing import Optional, Union + +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + +class IOIntelligenceConfig(OpenAIGPTConfig): + """ + Reference: https://io.net/docs/guides/intelligence/io-intelligence-apis + + IO Intelligence provides an OpenAI-compatible API at + https://api.intelligence.io.solutions/api/v1 + + Below are the parameters: + """ + + max_tokens: Optional[int] = None + temperature: Optional[int] = None + top_p: Optional[int] = None + stop: Optional[Union[str, list]] = None + stream: Optional[bool] = None + stream_options: Optional[dict] = None + tool_choice: Optional[str] = None + response_format: Optional[dict] = None + tools: Optional[list] = None + frequency_penalty: Optional[float] = None + presence_penalty: Optional[float] = None + + def __init__( + self, + max_tokens: Optional[int] = None, + response_format: Optional[dict] = None, + stop: Optional[str] = None, + stream: Optional[bool] = None, + stream_options: Optional[dict] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + tool_choice: Optional[str] = None, + tools: Optional[list] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return super().get_config() + + def get_supported_openai_params(self, model: str) -> list: + """ + Get the supported OpenAI params for the given model + """ + return [ + "max_completion_tokens", + "max_tokens", + "response_format", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "frequency_penalty", + "presence_penalty", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_openai_params = self.get_supported_openai_params(model=model) + for param, value in non_default_params.items(): + if param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_openai_params: + optional_params[param] = value + return optional_params diff --git a/litellm/llms/iointelligence/common_utils.py b/litellm/llms/iointelligence/common_utils.py new file mode 100644 index 00000000000..75a8ba4c208 --- /dev/null +++ b/litellm/llms/iointelligence/common_utils.py @@ -0,0 +1,6 @@ +from litellm.llms.base_llm.chat.transformation import BaseLLMException + + +class IOIntelligenceError(BaseLLMException): + def __init__(self, status_code, message, headers): + super().__init__(status_code=status_code, message=message, headers=headers) diff --git a/litellm/llms/iointelligence/embedding/__init__.py b/litellm/llms/iointelligence/embedding/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/iointelligence/embedding/transformation.py b/litellm/llms/iointelligence/embedding/transformation.py new file mode 100644 index 00000000000..159b9d125d6 --- /dev/null +++ b/litellm/llms/iointelligence/embedding/transformation.py @@ -0,0 +1,131 @@ +""" +IO Intelligence Embedding API + +This is OpenAI compatible - no transformation is applied. +""" + +from typing import List, Optional, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues +from litellm.types.utils import EmbeddingResponse, Usage + +from ..common_utils import IOIntelligenceError + + +class IOIntelligenceEmbeddingConfig(BaseEmbeddingConfig): + def __init__(self) -> None: + pass + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + if api_base is None: + raise ValueError("api_base is required for IO Intelligence embeddings") + api_base = api_base.rstrip("/") + if not api_base.endswith("/embeddings"): + api_base = f"{api_base}/embeddings" + return api_base + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + if api_key is None: + api_key = get_secret_str("IO_INTELLIGENCE_API_KEY") + + default_headers = { + "Authorization": f"Bearer {api_key}", + "accept": "application/json", + "Content-Type": "application/json", + } + + if "Authorization" in headers: + default_headers["Authorization"] = headers["Authorization"] + + return {**default_headers, **headers} + + def get_supported_openai_params(self, model: str): + return ["dimensions", "encoding_format"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ): + supported_openai_params = self.get_supported_openai_params(model) + for param, value in non_default_params.items(): + if param in supported_openai_params: + optional_params[param] = value + return optional_params + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + ) -> dict: + return { + "input": input, + "model": model, + **optional_params, + } + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, + ) -> EmbeddingResponse: + try: + raw_response_json = raw_response.json() + except Exception: + raise IOIntelligenceError( + message=raw_response.text, + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + model_response.model = raw_response_json.get("model") + model_response.data = raw_response_json.get("data") + model_response.object = raw_response_json.get("object") + + usage = Usage( + prompt_tokens=raw_response_json.get("usage", {}).get("prompt_tokens", 0), + total_tokens=raw_response_json.get("usage", {}).get("total_tokens", 0), + ) + + model_response.usage = usage + return model_response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return IOIntelligenceError( + message=error_message, status_code=status_code, headers=headers + ) diff --git a/litellm/main.py b/litellm/main.py index eace9c630ba..e15456e73ca 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2569,6 +2569,7 @@ def completion( # type: ignore # noqa: PLR0915 or custom_llm_provider == "cerebras" or custom_llm_provider == "baseten" or custom_llm_provider == "sambanova" + or custom_llm_provider == "io_intelligence" or custom_llm_provider == "volcengine" or custom_llm_provider == "anyscale" or custom_llm_provider == "openai" @@ -5448,6 +5449,28 @@ def embedding( # noqa: PLR0915 aembedding=aembedding, litellm_params={}, ) + elif custom_llm_provider == "io_intelligence": + api_key = api_key or litellm.api_key or get_secret_str("IO_INTELLIGENCE_API_KEY") + api_base = ( + api_base + or litellm.api_base + or get_secret_str("IO_INTELLIGENCE_API_BASE") + or "https://api.intelligence.io.solutions/api/v1" + ) + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params={}, + ) elif custom_llm_provider == "voyage": response = base_llm_http_handler.embedding( model=model, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 82557513a8a..e1a37520108 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3224,6 +3224,7 @@ class LlmProviders(str, Enum): LAMBDA_AI = "lambda_ai" DEEPSEEK = "deepseek" SAMBANOVA = "sambanova" + IO_INTELLIGENCE = "io_intelligence" MARITALK = "maritalk" VOYAGE = "voyage" CLOUDFLARE = "cloudflare" diff --git a/litellm/utils.py b/litellm/utils.py index e2dac1c9f62..f6988c2ae12 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8095,6 +8095,7 @@ class ProviderConfigManager: False, ), LlmProviders.SAMBANOVA: (lambda: litellm.SambanovaConfig(), False), + LlmProviders.IO_INTELLIGENCE: (lambda: litellm.IOIntelligenceConfig(), False), LlmProviders.MARITALK: (lambda: litellm.MaritalkConfig(), False), LlmProviders.VLLM: (lambda: litellm.VLLMConfig(), False), LlmProviders.OLLAMA: (lambda: litellm.OllamaConfig(), False), @@ -8263,6 +8264,8 @@ class ProviderConfigManager: return litellm.InfinityEmbeddingConfig() elif litellm.LlmProviders.SAMBANOVA == provider: return litellm.SambaNovaEmbeddingConfig() + elif litellm.LlmProviders.IO_INTELLIGENCE == provider: + return litellm.IOIntelligenceEmbeddingConfig() elif ( litellm.LlmProviders.COHERE == provider or litellm.LlmProviders.COHERE_CHAT == provider diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d4f986edd9b..f1e1884e494 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27040,6 +27040,210 @@ "supports_reasoning": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, + "io_intelligence/meta-llama/Llama-3.2-90B-Vision-Instruct": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_vision": true, + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/openai/gpt-oss-120b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/Qwen/Qwen2.5-VL-32B-Instruct": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_vision": true, + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/deepseek-ai/DeepSeek-R1-0528": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/meta-llama/Llama-3.3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/mistralai/Mistral-Large-Instruct-2411": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/Qwen3-Next-80B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/Qwen3-235B-A22B-Thinking": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/Qwen3-Coder-480B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/Llama-4-Maverick-17B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/K2-Think": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_reasoning": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/Apertus-70B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/Mistral-Nemo": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/GPT-OSS-20B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/Devstral-Small": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/Magistral-Small": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "io_intelligence", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, + "io_intelligence/BAAI/bge-multilingual-gemma2": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0.0, + "litellm_provider": "io_intelligence", + "mode": "embedding", + "source": "https://io.net/docs/guides/payment/io-intelligence-payments" + }, "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 18000, diff --git a/tests/test_litellm/llms/iointelligence/__init__.py b/tests/test_litellm/llms/iointelligence/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/iointelligence/test_iointelligence.py b/tests/test_litellm/llms/iointelligence/test_iointelligence.py new file mode 100644 index 00000000000..5bb53055379 --- /dev/null +++ b/tests/test_litellm/llms/iointelligence/test_iointelligence.py @@ -0,0 +1,223 @@ +import json +import os +from unittest.mock import patch + +import pytest + +from litellm import completion, embedding +from litellm.llms.custom_httpx.http_handler import HTTPHandler + + +@patch.dict(os.environ, {}, clear=True) +def test_completion_io_intelligence(): + """Ensure that the completion function works with IO Intelligence API.""" + messages = [{"role": "user", "content": "What's the weather like in San Francisco?"}] + try: + client = HTTPHandler() + with patch.object(client, "post") as mock_post: + completion( + model="io_intelligence/meta-llama/Llama-3.3-70B-Instruct", + messages=messages, + client=client, + max_tokens=5, + api_key="fake-api-key", + ) + + mock_post.assert_called_once() + mock_kwargs = mock_post.call_args.kwargs + assert "api.intelligence.io.solutions/api/v1" in mock_kwargs["url"] + assert mock_kwargs["headers"]["Authorization"] == "Bearer fake-api-key" + json_data = json.loads(mock_kwargs["data"]) + assert json_data["max_tokens"] == 5 + assert json_data["model"] == "meta-llama/Llama-3.3-70B-Instruct" + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + +@patch.dict(os.environ, {}, clear=True) +def test_completion_io_intelligence_with_custom_api_base(): + """Ensure custom api_base is respected.""" + messages = [{"role": "user", "content": "Hello"}] + try: + client = HTTPHandler() + with patch.object(client, "post") as mock_post: + completion( + model="io_intelligence/deepseek-ai/DeepSeek-R1-0528", + messages=messages, + client=client, + max_tokens=10, + api_key="fake-api-key", + api_base="https://custom.api.base/v1", + ) + + mock_post.assert_called_once() + mock_kwargs = mock_post.call_args.kwargs + assert "custom.api.base" in mock_kwargs["url"] + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + +@patch.dict(os.environ, {}, clear=True) +def test_completion_io_intelligence_streaming(): + """Ensure streaming parameter is passed correctly.""" + messages = [{"role": "user", "content": "Hello"}] + try: + client = HTTPHandler() + with patch.object(client, "post") as mock_post: + completion( + model="io_intelligence/meta-llama/Llama-3.3-70B-Instruct", + messages=messages, + client=client, + max_tokens=5, + api_key="fake-api-key", + stream=True, + ) + + mock_post.assert_called_once() + mock_kwargs = mock_post.call_args.kwargs + json_data = json.loads(mock_kwargs["data"]) + assert json_data["stream"] is True + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + +@patch.dict( + os.environ, + {"IO_INTELLIGENCE_API_KEY": "test-env-key"}, + clear=True, +) +def test_completion_io_intelligence_env_key(): + """Ensure API key is picked up from environment variable.""" + messages = [{"role": "user", "content": "Hello"}] + try: + client = HTTPHandler() + with patch.object(client, "post") as mock_post: + completion( + model="io_intelligence/meta-llama/Llama-3.3-70B-Instruct", + messages=messages, + client=client, + max_tokens=5, + ) + + mock_post.assert_called_once() + mock_kwargs = mock_post.call_args.kwargs + assert mock_kwargs["headers"]["Authorization"] == "Bearer test-env-key" + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + +def test_completion_io_intelligence_with_tools(): + """Ensure function calling params are passed correctly.""" + messages = [{"role": "user", "content": "What's the weather?"}] + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather info", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"], + }, + }, + } + ] + try: + client = HTTPHandler() + with patch.object(client, "post") as mock_post: + completion( + model="io_intelligence/meta-llama/Llama-3.3-70B-Instruct", + messages=messages, + client=client, + max_tokens=5, + api_key="fake-api-key", + tools=tools, + tool_choice="auto", + ) + + mock_post.assert_called_once() + mock_kwargs = mock_post.call_args.kwargs + json_data = json.loads(mock_kwargs["data"]) + assert "tools" in json_data + assert json_data["tool_choice"] == "auto" + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + +def test_io_intelligence_provider_in_model_prices(): + """Ensure IO Intelligence models are registered in model prices.""" + from litellm import model_cost + + io_models = [k for k in model_cost if k.startswith("io_intelligence/")] + assert len(io_models) > 0, "No IO Intelligence models found in model_cost" + + # Check a specific model + assert "io_intelligence/meta-llama/Llama-3.3-70B-Instruct" in model_cost + model_info = model_cost["io_intelligence/meta-llama/Llama-3.3-70B-Instruct"] + assert model_info["litellm_provider"] == "io_intelligence" + assert model_info["mode"] == "chat" + assert model_info["input_cost_per_token"] > 0 + + +def test_io_intelligence_embedding_model_in_prices(): + """Ensure IO Intelligence embedding model is registered.""" + from litellm import model_cost + + assert "io_intelligence/BAAI/bge-multilingual-gemma2" in model_cost + model_info = model_cost["io_intelligence/BAAI/bge-multilingual-gemma2"] + assert model_info["litellm_provider"] == "io_intelligence" + assert model_info["mode"] == "embedding" + + +def test_io_intelligence_in_openai_compatible_providers(): + """Ensure io_intelligence is in the openai_compatible_providers list.""" + from litellm.constants import openai_compatible_providers + + assert "io_intelligence" in openai_compatible_providers + + +def test_io_intelligence_config_supported_params(): + """Test that IOIntelligenceConfig returns expected supported params.""" + from litellm.llms.iointelligence.chat import IOIntelligenceConfig + + config = IOIntelligenceConfig() + params = config.get_supported_openai_params( + model="meta-llama/Llama-3.3-70B-Instruct" + ) + assert "max_tokens" in params + assert "temperature" in params + assert "tools" in params + assert "stream" in params + + +def test_io_intelligence_config_map_params(): + """Test that IOIntelligenceConfig maps max_completion_tokens to max_tokens.""" + from litellm.llms.iointelligence.chat import IOIntelligenceConfig + + config = IOIntelligenceConfig() + result = config.map_openai_params( + non_default_params={"max_completion_tokens": 100}, + optional_params={}, + model="meta-llama/Llama-3.3-70B-Instruct", + drop_params=False, + ) + assert result["max_tokens"] == 100 + + +def test_completion_io_intelligence_live(): + """Live integration test - only runs if API key is set.""" + if os.environ.get("IO_INTELLIGENCE_API_KEY") is None: + pytest.skip("IO_INTELLIGENCE_API_KEY not set") + + messages = [{"role": "user", "content": "Say hello in one word."}] + response = completion( + model="io_intelligence/meta-llama/Llama-3.3-70B-Instruct", + messages=messages, + max_tokens=10, + ) + assert response["object"] == "chat.completion" + assert len(response["choices"]) == 1 + assert len(response["choices"][0]["message"]["content"]) > 0 diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index e833d0eb4fb..39c7cf8ed43 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -85,6 +85,7 @@ export enum Providers { RunwayML = "RunwayML", SAGEMAKER_LEGACY = "Sagemaker", Sambanova = "Sambanova", + IOIntelligence = "IO Intelligence", SAP = "SAP Generative AI Hub", Snowflake = "Snowflake", TEXT_COMPLETION_CODESTRAL = "Text-Completion-Codestral", @@ -192,6 +193,7 @@ export const provider_map: Record = { SAGEMAKER_LEGACY: "sagemaker", SageMaker: "sagemaker_chat", Sambanova: "sambanova", + IOIntelligence: "io_intelligence", SAP: "sap", Snowflake: "snowflake", TEXT_COMPLETION_CODESTRAL: "text-completion-codestral", @@ -282,6 +284,7 @@ export const providerLogoMap: Record = { [Providers.RunwayML]: `${asset_logos_folder}runwayml.png`, [Providers.SAGEMAKER_LEGACY]: `${asset_logos_folder}bedrock.svg`, [Providers.Sambanova]: `${asset_logos_folder}sambanova.svg`, + [Providers.IOIntelligence]: `${asset_logos_folder}io_intelligence.png`, [Providers.SAP]: `${asset_logos_folder}sap.png`, [Providers.Snowflake]: `${asset_logos_folder}snowflake.svg`, [Providers.TEXT_COMPLETION_CODESTRAL]: `${asset_logos_folder}mistral.svg`,