diff --git a/docs/my-website/docs/providers/crusoe.md b/docs/my-website/docs/providers/crusoe.md new file mode 100644 index 00000000000..aa737cbdcd8 --- /dev/null +++ b/docs/my-website/docs/providers/crusoe.md @@ -0,0 +1,196 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Crusoe + +## Overview + +| Property | Details | +|-------|-------| +| Description | Crusoe Cloud provides GPU-accelerated inference for open-source large language models, optimized for performance and cost efficiency. | +| Provider Route on LiteLLM | `crusoe/` | +| Link to Provider Doc | [Crusoe Managed Inference Documentation ↗](https://docs.crusoecloud.com/managed-inference/overview/index.html) | +| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+
+ +**We support ALL Crusoe models, just set `crusoe/` as a prefix when sending completion requests** + +## Available Models + +| Model | Description | Context Window | +|-------|-------------|----------------| +| `crusoe/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 reasoning model (May 2025) | 163,840 tokens | +| `crusoe/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 chat model (March 2025) | 163,840 tokens | +| `crusoe/google/gemma-3-12b-it` | Google Gemma 3 12B instruction-tuned | 131,072 tokens | +| `crusoe/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B instruction-tuned | 131,072 tokens | +| `crusoe/moonshotai/Kimi-K2-Thinking` | Kimi K2 extended thinking model | 262,144 tokens | +| `crusoe/openai/gpt-oss-120b` | OpenAI 120B open-source model | 131,072 tokens | +| `crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507` | Qwen3 235B MoE instruction-tuned | 262,144 tokens | + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key +``` + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Crusoe Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Crusoe call +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Crusoe Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key + +messages = [{"content": "Write a short story about AI", "role": "user"}] + +# Crusoe call with streaming +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +### Function Calling + +```python showLineNumbers title="Crusoe Function Calling" +import os +import litellm +from litellm import completion + +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key + +tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + } + }, + "required": ["location"] + } + } +}] + +messages = [{"role": "user", "content": "What's the weather in Boston?"}] + +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=messages, + tools=tools, + tool_choice="auto" +) + +print(response) +``` + +## Usage - LiteLLM Proxy Server + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: llama-3.3-70b + litellm_params: + model: crusoe/meta-llama/Llama-3.3-70B-Instruct + api_key: os.environ/CRUSOE_API_KEY + - model_name: deepseek-r1 + litellm_params: + model: crusoe/deepseek-ai/DeepSeek-R1-0528 + api_key: os.environ/CRUSOE_API_KEY + - model_name: deepseek-v3 + litellm_params: + model: crusoe/deepseek-ai/DeepSeek-V3-0324 + api_key: os.environ/CRUSOE_API_KEY + - model_name: qwen3-235b + litellm_params: + model: crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507 + api_key: os.environ/CRUSOE_API_KEY + - model_name: kimi-k2 + litellm_params: + model: crusoe/moonshotai/Kimi-K2-Thinking + api_key: os.environ/CRUSOE_API_KEY +``` + +## Custom API Base + +**Option 1: Environment variable** + +```python showLineNumbers title="Custom API Base via env var" +import os +from litellm import completion + +os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1" +os.environ["CRUSOE_API_KEY"] = "" # your API key + +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=[{"content": "Hello!", "role": "user"}], +) +``` + +**Option 2: Pass directly** + +```python showLineNumbers title="Custom API Base via parameter" +from litellm import completion + +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=[{"content": "Hello!", "role": "user"}], + api_base="https://custom.crusoecloud.com/v1", + api_key="your-api-key", +) +``` + +## Supported OpenAI Parameters + +- `temperature` +- `max_tokens` +- `max_completion_tokens` +- `top_p` +- `frequency_penalty` +- `presence_penalty` +- `stop` +- `n` +- `stream` +- `tools` +- `tool_choice` +- `response_format` +- `seed` +- `user` +- `logit_bias` +- `logprobs` +- `top_logprobs` diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 275c352b39e..8c2f5fa4ffb 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -101,5 +101,13 @@ "param_mappings": { "max_completion_tokens": "max_tokens" } + }, + "crusoe": { + "base_url": "https://managed-inference-api-proxy.crusoecloud.com/v1", + "api_key_env": "CRUSOE_API_KEY", + "api_base_env": "CRUSOE_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } } } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c53ee943c58..8c180094371 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -20575,6 +20575,98 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "crusoe/deepseek-ai/DeepSeek-R1-0528": { + "input_cost_per_token": 3e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 7e-06, + "supports_function_calling": false, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "crusoe/deepseek-ai/DeepSeek-V3-0324": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/google/gemma-3-12b-it": { + "input_cost_per_token": 1e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "crusoe/meta-llama/Llama-3.3-70B-Instruct": { + "input_cost_per_token": 2e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/moonshotai/Kimi-K2-Thinking": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": false, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "crusoe/openai/gpt-oss-120b": { + "input_cost_per_token": 8e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507": { + "input_cost_per_token": 3e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "lambda_ai/deepseek-llama3.3-70b": { "input_cost_per_token": 2e-07, "litellm_provider": "lambda_ai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c53ee943c58..8c180094371 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20575,6 +20575,98 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "crusoe/deepseek-ai/DeepSeek-R1-0528": { + "input_cost_per_token": 3e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 7e-06, + "supports_function_calling": false, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "crusoe/deepseek-ai/DeepSeek-V3-0324": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/google/gemma-3-12b-it": { + "input_cost_per_token": 1e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "crusoe/meta-llama/Llama-3.3-70B-Instruct": { + "input_cost_per_token": 2e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/moonshotai/Kimi-K2-Thinking": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": false, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "crusoe/openai/gpt-oss-120b": { + "input_cost_per_token": 8e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507": { + "input_cost_per_token": 3e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "lambda_ai/deepseek-llama3.3-70b": { "input_cost_per_token": 2e-07, "litellm_provider": "lambda_ai", diff --git a/tests/llm_translation/test_crusoe.py b/tests/llm_translation/test_crusoe.py new file mode 100644 index 00000000000..56aa4e4cd42 --- /dev/null +++ b/tests/llm_translation/test_crusoe.py @@ -0,0 +1,108 @@ +""" +Tests for Crusoe provider integration +""" +import os +from unittest import mock + +import litellm + +CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1" + + +def test_crusoe_json_registry(): + """Test CrusoeChatConfig is loaded from JSON provider registry""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("crusoe") + config = JSONProviderRegistry.get("crusoe") + assert config is not None + assert config.base_url == CRUSOE_API_BASE + assert config.api_key_env == "CRUSOE_API_KEY" + assert config.api_base_env == "CRUSOE_API_BASE" + + +def test_crusoe_get_openai_compatible_provider_info(): + """Test Crusoe provider info retrieval""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + + # Test with default values (no env vars set) + with mock.patch.dict(os.environ, {}, clear=True): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == CRUSOE_API_BASE + assert api_key is None + + # Test with environment variables + with mock.patch.dict( + os.environ, + { + "CRUSOE_API_KEY": "test-key", + "CRUSOE_API_BASE": "https://custom.crusoecloud.com/v1", + }, + ): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://custom.crusoecloud.com/v1" + assert api_key == "test-key" + + # Test with explicit parameters (should override env vars) + with mock.patch.dict( + os.environ, + { + "CRUSOE_API_KEY": "env-key", + "CRUSOE_API_BASE": "https://env.crusoecloud.com/v1", + }, + ): + api_base, api_key = config._get_openai_compatible_provider_info( + "https://param.crusoecloud.com/v1", "param-key" + ) + assert api_base == "https://param.crusoecloud.com/v1" + assert api_key == "param-key" + + +def test_get_llm_provider_crusoe(): + """Test that get_llm_provider correctly identifies Crusoe""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + # Test with crusoe/model-name format + model, provider, api_key, api_base = get_llm_provider( + "crusoe/meta-llama/Llama-3.3-70B-Instruct" + ) + assert model == "meta-llama/Llama-3.3-70B-Instruct" + assert provider == "crusoe" + + +def test_crusoe_models_configuration(): + """Test that Crusoe models are configured correctly""" + from litellm import get_model_info + + original_model_cost = litellm.model_cost + original_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + try: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + crusoe_models = [ + "crusoe/meta-llama/Llama-3.3-70B-Instruct", + "crusoe/deepseek-ai/DeepSeek-R1-0528", + "crusoe/deepseek-ai/DeepSeek-V3-0324", + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", + "crusoe/moonshotai/Kimi-K2-Thinking", + "crusoe/openai/gpt-oss-120b", + "crusoe/google/gemma-3-12b-it", + ] + + for model in crusoe_models: + model_info = get_model_info(model) + assert model_info is not None, f"Model info not found for {model}" + assert model_info.get("litellm_provider") == "crusoe", ( + f"{model} should have crusoe as provider" + ) + assert model_info.get("mode") == "chat", f"{model} should be in chat mode" + finally: + litellm.model_cost = original_model_cost + if original_env is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = original_env diff --git a/tests/test_litellm/llms/crusoe/__init__.py b/tests/test_litellm/llms/crusoe/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/crusoe/test_crusoe.py b/tests/test_litellm/llms/crusoe/test_crusoe.py new file mode 100644 index 00000000000..0a05126919a --- /dev/null +++ b/tests/test_litellm/llms/crusoe/test_crusoe.py @@ -0,0 +1,135 @@ +import os +from unittest.mock import patch + +CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1" + + +def test_crusoe_json_registry(): + """Test Crusoe is registered in the JSON provider registry""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("crusoe") + config = JSONProviderRegistry.get("crusoe") + assert config is not None + assert config.base_url == CRUSOE_API_BASE + assert config.api_key_env == "CRUSOE_API_KEY" + assert config.api_base_env == "CRUSOE_API_BASE" + + +def test_crusoe_dynamic_config_defaults(): + """Test dynamic config returns correct default API base""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + + with patch.dict(os.environ, {}, clear=True): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + + assert api_base == CRUSOE_API_BASE + assert api_key is None + + +def test_crusoe_dynamic_config_env_vars(): + """Test dynamic config reads CRUSOE_API_KEY and CRUSOE_API_BASE from env""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + + with patch.dict( + os.environ, + {"CRUSOE_API_KEY": "test-key", "CRUSOE_API_BASE": "https://custom.crusoe.com/v1"}, + ): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + + assert api_base == "https://custom.crusoe.com/v1" + assert api_key == "test-key" + + +def test_crusoe_dynamic_config_explicit_params(): + """Test explicit params override env vars""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + + with patch.dict(os.environ, {"CRUSOE_API_KEY": "env-key"}): + api_base, api_key = config._get_openai_compatible_provider_info( + "https://override.crusoe.com/v1", "override-key" + ) + + assert api_base == "https://override.crusoe.com/v1" + assert api_key == "override-key" + + +def test_crusoe_supported_params(): + """Test dynamic config returns standard OpenAI params""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + params = config.get_supported_openai_params(model="meta-llama/Llama-3.3-70B-Instruct") + + assert isinstance(params, list) + assert len(params) > 0 + assert "temperature" in params + assert "max_tokens" in params + assert "stream" in params + + +def test_crusoe_param_mapping_max_completion_tokens(): + """Test max_completion_tokens is mapped to max_tokens for Crusoe""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + optional_params = config.map_openai_params( + non_default_params={"max_completion_tokens": 1024}, + optional_params={}, + model="meta-llama/Llama-3.3-70B-Instruct", + drop_params=False, + ) + + assert "max_tokens" in optional_params, "max_completion_tokens should be mapped to max_tokens" + assert optional_params["max_tokens"] == 1024 + assert "max_completion_tokens" not in optional_params + + +def test_crusoe_provider_detection_by_prefix(): + """Test crusoe/model prefix is correctly routed""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, _, _ = get_llm_provider("crusoe/meta-llama/Llama-3.3-70B-Instruct") + assert provider == "crusoe" + assert model == "meta-llama/Llama-3.3-70B-Instruct" + + +def test_crusoe_model_list_populated(): + """Test Crusoe models are present in model_prices_and_context_window.json""" + import litellm + + original_model_cost = litellm.model_cost + original_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + try: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + expected = [ + "crusoe/meta-llama/Llama-3.3-70B-Instruct", + "crusoe/deepseek-ai/DeepSeek-R1-0528", + "crusoe/deepseek-ai/DeepSeek-V3-0324", + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", + "crusoe/moonshotai/Kimi-K2-Thinking", + "crusoe/openai/gpt-oss-120b", + "crusoe/google/gemma-3-12b-it", + ] + for model in expected: + assert model in litellm.model_cost, f"{model} not found in model_cost" + assert litellm.model_cost[model].get("litellm_provider") == "crusoe" + finally: + litellm.model_cost = original_model_cost + if original_env is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = original_env