diff --git a/.env.example b/.env.example index 24c2b608414..7602611a6bc 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,9 @@ ANTHROPIC_API_KEY = "" INFISICAL_TOKEN = "" # Novita AI NOVITA_API_KEY = "" +# HPC-AI (OpenAI-compatible inference) +HPC_AI_API_KEY = "" +HPC_AI_API_BASE = "" # INFINITY INFINITY_API_KEY = "" diff --git a/docs/my-website/docs/providers/hpc_ai.md b/docs/my-website/docs/providers/hpc_ai.md new file mode 100644 index 00000000000..4be00e9a7f4 --- /dev/null +++ b/docs/my-website/docs/providers/hpc_ai.md @@ -0,0 +1,86 @@ +# HPC-AI + +[HPC-AI](https://api.hpc-ai.com) provides an OpenAI-compatible inference API at `https://api.hpc-ai.com/inference/v1`. + +:::tip + +Use the `hpc_ai/` prefix with the upstream model id (for example `hpc_ai/minimax/minimax-m2.5`). LiteLLM strips the prefix and forwards the remainder as the OpenAI `model` field. + +::: + +## API Key + +```python +import os + +os.environ["HPC_AI_API_KEY"] = "your-api-key" +``` + +Optional: override the base URL (defaults to `https://api.hpc-ai.com/inference/v1`). + +```python +os.environ["HPC_AI_API_BASE"] = "https://api.hpc-ai.com/inference/v1" +``` + +If you use another env name such as `HPC_AI_BASE_URL`, map it to `api_base` in your LiteLLM call or proxy `litellm_params`; LiteLLM reads `HPC_AI_API_BASE` by default. + +## Sample Usage: Chat completion + +```python +from litellm import completion +import os + +os.environ["HPC_AI_API_KEY"] = "your-api-key" +response = completion( + model="hpc_ai/minimax/minimax-m2.5", + messages=[{"role": "user", "content": "Hello!"}], + max_tokens=256, +) +print(response) +``` + +## Sample Usage: Streaming + +```python +from litellm import completion +import os + +os.environ["HPC_AI_API_KEY"] = "your-api-key" +response = completion( + model="hpc_ai/moonshotai/kimi-k2.5", + messages=[{"role": "user", "content": "Hello!"}], + stream=True, +) + +for chunk in response: + print(chunk) +``` + +## Usage with LiteLLM Proxy Server + +1. Add a model to your `config.yaml`: + +```yaml +model_list: + - model_name: hpc-ai-minimax + litellm_params: + model: hpc_ai/minimax/minimax-m2.5 + api_key: os.environ/HPC_AI_API_KEY +``` + +2. Start the proxy: + +```bash +litellm --config /path/to/config.yaml +``` + +3. Send requests to the proxy using your alias (`hpc-ai-minimax` in the example above). + +## Supported models (examples) + +| LiteLLM model id | Notes | +| ---------------- | ----- | +| `hpc_ai/minimax/minimax-m2.5` | MiniMax M2.5 | +| `hpc_ai/moonshotai/kimi-k2.5` | Kimi K2.5 | + +Pricing in `model_prices_and_context_window.json` may use placeholder token costs; set real rates when your billing API is available. diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 6446e227d99..3c51f9ecd84 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -943,6 +943,7 @@ const sidebars = { "providers/moonshot", "providers/morph", "providers/nebius", + "providers/hpc_ai", "providers/nlp_cloud", "providers/nano-gpt", "providers/novita", diff --git a/litellm/__init__.py b/litellm/__init__.py index e45d926e8db..e5e5653730b 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -254,6 +254,7 @@ novita_api_key: Optional[str] = None snowflake_key: Optional[str] = None gradient_ai_api_key: Optional[str] = None nebius_key: Optional[str] = None +hpc_ai_key: Optional[str] = None wandb_key: Optional[str] = None heroku_key: Optional[str] = None cometapi_key: Optional[str] = None @@ -569,6 +570,7 @@ llama_models: Set = set() nscale_models: Set = set() nebius_models: Set = set() nebius_embedding_models: Set = set() +hpc_ai_models: Set = set() aiml_models: Set = set() deepgram_models: Set = set() elevenlabs_models: Set = set() @@ -795,6 +797,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): nebius_models.add(key) elif value.get("litellm_provider") == "nebius-embedding-models": nebius_embedding_models.add(key) + elif value.get("litellm_provider") == "hpc_ai": + hpc_ai_models.add(key) elif value.get("litellm_provider") == "aiml": aiml_models.add(key) elif value.get("litellm_provider") == "assemblyai": @@ -947,6 +951,7 @@ model_list = list( | sambanova_models | azure_text_models | novita_models + | hpc_ai_models | assemblyai_models | jina_ai_models | snowflake_models @@ -1043,6 +1048,7 @@ models_by_provider: dict = { "sambanova": sambanova_models | sambanova_embedding_models, "novita": novita_models, "nebius": nebius_models | nebius_embedding_models, + "hpc_ai": hpc_ai_models, "aiml": aiml_models, "assemblyai": assemblyai_models, "jina_ai": jina_ai_models, @@ -1826,6 +1832,7 @@ if TYPE_CHECKING: GigaChatEmbeddingConfig as GigaChatEmbeddingConfig, ) from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig + from .llms.hpc_ai.chat.transformation import HpcAiConfig as HpcAiConfig from .llms.wandb.chat.transformation import WandbConfig as WandbConfig from .llms.dashscope.chat.transformation import ( DashScopeChatConfig as DashScopeChatConfig, diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 9164a3c8ae4..b23e53f2e1b 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -295,6 +295,7 @@ LLM_CONFIG_NAMES = ( "ManusResponsesAPIConfig", "GithubCopilotEmbeddingConfig", "NebiusConfig", + "HpcAiConfig", "WandbConfig", "GigaChatConfig", "GigaChatEmbeddingConfig", @@ -1115,6 +1116,7 @@ _LLM_CONFIGS_IMPORT_MAP = { "ChatGPTResponsesAPIConfig", ), "NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"), + "HpcAiConfig": (".llms.hpc_ai.chat.transformation", "HpcAiConfig"), "WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"), "GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"), "GigaChatEmbeddingConfig": ( diff --git a/litellm/constants.py b/litellm/constants.py index 423f01afac1..22fb420873d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -558,6 +558,7 @@ LITELLM_CHAT_PROVIDERS = [ "featherless_ai", "nscale", "nebius", + "hpc_ai", "dashscope", "moonshot", "publicai", @@ -713,6 +714,7 @@ openai_compatible_endpoints: List = [ "api.featherless.ai/v1", "inference.api.nscale.com/v1", "api.studio.nebius.ai/v1", + "api.hpc-ai.com/inference/v1", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", "https://api.moonshot.ai/v1", "https://api.publicai.co/v1", @@ -773,6 +775,7 @@ openai_compatible_providers: List = [ "featherless_ai", "nscale", "nebius", + "hpc_ai", "dashscope", "moonshot", "v0", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 36218417377..08439bc3090 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -324,6 +324,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.inference.wandb.ai/v1": custom_llm_provider = "wandb" dynamic_api_key = get_secret_str("WANDB_API_KEY") + elif endpoint == "api.hpc-ai.com/inference/v1": + custom_llm_provider = "hpc_ai" + dynamic_api_key = get_secret_str("HPC_AI_API_KEY") if api_base is not None and not isinstance(api_base, str): raise Exception( @@ -619,6 +622,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or "https://api.studio.nebius.ai/v1" ) # type: ignore dynamic_api_key = api_key or get_secret_str("NEBIUS_API_KEY") + elif custom_llm_provider == "hpc_ai": + ( + api_base, + dynamic_api_key, + ) = litellm.HpcAiConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "ollama": api_base = ( api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434" diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index b72d7abeae0..e62530454f8 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -170,6 +170,9 @@ def get_supported_openai_params( # noqa: PLR0915 elif custom_llm_provider == "nebius": if request_type == "chat_completion": return litellm.NebiusConfig().get_supported_openai_params(model=model) + elif custom_llm_provider == "hpc_ai": + if request_type == "chat_completion": + return litellm.HpcAiConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "wandb": if request_type == "chat_completion": return litellm.WandbConfig().get_supported_openai_params(model=model) diff --git a/litellm/llms/hpc_ai/chat/transformation.py b/litellm/llms/hpc_ai/chat/transformation.py new file mode 100644 index 00000000000..4c0df6bc372 --- /dev/null +++ b/litellm/llms/hpc_ai/chat/transformation.py @@ -0,0 +1,39 @@ +""" +HPC-AI Chat Completions API — OpenAI-compatible endpoint. + +Reference: https://api.hpc-ai.com/inference/v1 +""" + +from typing import Optional, Tuple + +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.secret_managers.main import get_secret_str + + +class HpcAiConfig(OpenAIGPTConfig): + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """Map max_completion_tokens to max_tokens for OpenAI-compatible API.""" + supported_openai_params = self.get_supported_openai_params(model=model) + for param, value in non_default_params.items(): + if param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_openai_params: + optional_params[param] = value + return optional_params + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + api_base = ( + api_base + or get_secret_str("HPC_AI_API_BASE") + or "https://api.hpc-ai.com/inference/v1" + ) + dynamic_api_key = api_key or get_secret_str("HPC_AI_API_KEY") + return api_base, dynamic_api_key diff --git a/litellm/proxy/_experimental/out/assets/logos/hpc_ai.svg b/litellm/proxy/_experimental/out/assets/logos/hpc_ai.svg new file mode 100644 index 00000000000..52dad8269ec --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/hpc_ai.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index dda8e49d4c8..2212333a30b 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1792,6 +1792,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "HPC_AI", + "provider_display_name": "HPC-AI", + "litellm_provider": "hpc_ai", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "hpc_ai/minimax/minimax-m2.5" + }, { "provider": "NLP_CLOUD", "provider_display_name": "Nlp Cloud", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index bd673da8bed..1794da11cac 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3227,6 +3227,7 @@ class LlmProviders(str, Enum): LM_STUDIO = "lm_studio" GALADRIEL = "galadriel" NEBIUS = "nebius" + HPC_AI = "hpc_ai" INFINITY = "infinity" DEEPGRAM = "deepgram" ELEVENLABS = "elevenlabs" diff --git a/litellm/utils.py b/litellm/utils.py index 088ee07d630..1ff4bf1d73f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4638,6 +4638,17 @@ def get_optional_params( # noqa: PLR0915 else False ), ) + elif custom_llm_provider == "hpc_ai": + optional_params = litellm.HpcAiConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=( + drop_params + if drop_params is not None and isinstance(drop_params, bool) + else False + ), + ) elif custom_llm_provider == "azure": if litellm.AzureOpenAIO1Config().is_o_series_model(model=model): optional_params = litellm.AzureOpenAIO1Config().map_openai_params( @@ -5131,6 +5142,9 @@ def get_api_key(llm_provider: str, dynamic_api_key: Optional[str]): # nebius elif llm_provider == "nebius": api_key = api_key or litellm.nebius_key or get_secret("NEBIUS_API_KEY") + # hpc_ai + elif llm_provider == "hpc_ai": + api_key = api_key or litellm.hpc_ai_key or get_secret("HPC_AI_API_KEY") # wandb elif llm_provider == "wandb": api_key = api_key or litellm.wandb_key or get_secret("WANDB_API_KEY") @@ -6436,6 +6450,11 @@ def validate_environment( # noqa: PLR0915 keys_in_environment = True else: missing_keys.append("NEBIUS_API_KEY") + elif custom_llm_provider == "hpc_ai": + if "HPC_AI_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("HPC_AI_API_KEY") elif custom_llm_provider == "wandb": if "WANDB_API_KEY" in os.environ: keys_in_environment = True @@ -6558,6 +6577,11 @@ def validate_environment( # noqa: PLR0915 keys_in_environment = True else: missing_keys.append("NEBIUS_API_KEY") + elif model in litellm.hpc_ai_models: + if "HPC_AI_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("HPC_AI_API_KEY") elif model in litellm.wandb_models: if "WANDB_API_KEY" in os.environ: keys_in_environment = True @@ -8082,6 +8106,7 @@ class ProviderConfigManager: LlmProviders.FEATHERLESS_AI: (lambda: litellm.FeatherlessAIConfig(), False), LlmProviders.NOVITA: (lambda: litellm.NovitaConfig(), False), LlmProviders.NEBIUS: (lambda: litellm.NebiusConfig(), False), + LlmProviders.HPC_AI: (lambda: litellm.HpcAiConfig(), False), LlmProviders.WANDB: (lambda: litellm.WandbConfig(), False), LlmProviders.DASHSCOPE: (lambda: litellm.DashScopeChatConfig(), False), LlmProviders.MOONSHOT: (lambda: litellm.MoonshotChatConfig(), False), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c53ee943c58..1758ea01481 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20188,6 +20188,28 @@ "/v1/images/generations" ] }, + "hpc_ai/minimax/minimax-m2.5": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "hpc_ai", + "mode": "chat", + "supports_function_calling": true, + "source": "https://api.hpc-ai.com/inference/v1" + }, + "hpc_ai/moonshotai/kimi-k2.5": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "hpc_ai", + "mode": "chat", + "supports_function_calling": true, + "source": "https://api.hpc-ai.com/inference/v1" + }, "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": { "input_cost_per_token": 1.2e-07, "litellm_provider": "hyperbolic", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 2f3302bb574..b84468b09e2 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1503,6 +1503,24 @@ "interactions": true } }, + "hpc_ai": { + "display_name": "HPC-AI (`hpc_ai`)", + "url": "https://docs.litellm.ai/docs/providers/hpc_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, "nlp_cloud": { "display_name": "NLP Cloud (`nlp_cloud`)", "url": "https://docs.litellm.ai/docs/providers/nlp_cloud", diff --git a/tests/test_litellm/llms/hpc_ai/test_hpc_ai_chat_transformation.py b/tests/test_litellm/llms/hpc_ai/test_hpc_ai_chat_transformation.py new file mode 100644 index 00000000000..5226d1b1717 --- /dev/null +++ b/tests/test_litellm/llms/hpc_ai/test_hpc_ai_chat_transformation.py @@ -0,0 +1,75 @@ +""" +Unit tests for HPC-AI OpenAI-compatible configuration. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../../..")) + +import pytest + +import litellm +from litellm import completion +from litellm.llms.hpc_ai.chat.transformation import HpcAiConfig + + +class TestHpcAiConfig: + def test_validate_environment_sets_auth_header(self): + config = HpcAiConfig() + headers = {} + api_key = "fake-hpc-ai-key" + result = config.validate_environment( + headers=headers, + model="hpc_ai/minimax/minimax-m2.5", + messages=[{"role": "user", "content": "Hey"}], + optional_params={}, + litellm_params={}, + api_key=api_key, + api_base=None, + ) + assert result["Authorization"] == f"Bearer {api_key}" + assert result["Content-Type"] == "application/json" + + @pytest.mark.respx() + def test_hpc_ai_completion_mock(self, respx_mock): + litellm.disable_aiohttp_transport = True + + api_key = "fake-hpc-ai-key" + api_base = "https://api.hpc-ai.com/inference/v1" + model = "hpc_ai/minimax/minimax-m2.5" + model_name = "minimax/minimax-m2.5" + + respx_mock.post(f"{api_base}/chat/completions").respond( + json={ + "id": "chatcmpl-hpc-1", + "object": "chat.completion", + "created": 1677652288, + "model": model_name, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello from HPC-AI.", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 8, + "total_tokens": 13, + }, + }, + status_code=200, + ) + + response = completion( + model=model, + messages=[{"role": "user", "content": "Hello"}], + api_key=api_key, + api_base=api_base, + ) + + assert response.choices[0].message.content == "Hello from HPC-AI." diff --git a/ui/litellm-dashboard/public/assets/logos/hpc_ai.svg b/ui/litellm-dashboard/public/assets/logos/hpc_ai.svg new file mode 100644 index 00000000000..52dad8269ec --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/hpc_ai.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index a8021f94d84..ae89863827e 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -144,6 +144,10 @@ describe("provider_info_helpers", () => { expect(getPlaceholder(Providers.DeepInfra)).toBe("deepinfra/"); }); + it("should return hpc_ai placeholder for HPC_AI provider", () => { + expect(getPlaceholder(Providers.HPC_AI)).toBe("hpc_ai/minimax/minimax-m2.5"); + }); + it("should return fal_ai placeholder for FalAI provider", () => { expect(getPlaceholder(Providers.FalAI)).toBe("fal_ai/fal-ai/flux-pro/v1.1-ultra"); }); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index e833d0eb4fb..7e03ce7133c 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -45,6 +45,7 @@ export enum Providers { GradientAI = "GradientAI", Groq = "Groq", HEROKU = "Heroku", + HPC_AI = "HPC-AI", Hosted_Vllm = "vllm", HUGGINGFACE = "Huggingface", HYPERBOLIC = "Hyperbolic", @@ -151,6 +152,7 @@ export const provider_map: Record = { GradientAI: "gradient_ai", Groq: "groq", HEROKU: "heroku", + HPC_AI: "hpc_ai", Hosted_Vllm: "hosted_vllm", HUGGINGFACE: "huggingface", HYPERBOLIC: "hyperbolic", @@ -251,6 +253,7 @@ export const providerLogoMap: Record = { [Providers.Google_AI_Studio]: `${asset_logos_folder}google.svg`, [Providers.GradientAI]: `${asset_logos_folder}gradientai.svg`, [Providers.Groq]: `${asset_logos_folder}groq.svg`, + [Providers.HPC_AI]: `${asset_logos_folder}hpc_ai.svg`, [Providers.Hosted_Vllm]: `${asset_logos_folder}vllm.png`, [Providers.HUGGINGFACE]: `${asset_logos_folder}huggingface.svg`, [Providers.HYPERBOLIC]: `${asset_logos_folder}hyperbolic.svg`, @@ -358,6 +361,8 @@ export const getPlaceholder = (selectedProvider: string): string => { return "volcengine/"; } else if (selectedProvider == Providers.DeepInfra) { return "deepinfra/"; + } else if (selectedProvider === Providers.HPC_AI) { + return "hpc_ai/minimax/minimax-m2.5"; } else if (selectedProvider == Providers.FalAI) { return "fal_ai/fal-ai/flux-pro/v1.1-ultra"; } else if (selectedProvider == Providers.RunwayML) {