Merge pull request #23195 from acheamponge/add-crusoe-provider

Add Crusoe Cloud as a new LiteLLM provider
This commit is contained in:
Krish Dholakia 2026-03-23 22:47:41 -07:00 committed by GitHub
commit 49c8fa0e3c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 631 additions and 0 deletions

View file

@ -0,0 +1,196 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Crusoe
## Overview
| Property | Details |
|-------|-------|
| Description | Crusoe Cloud provides GPU-accelerated inference for open-source large language models, optimized for performance and cost efficiency. |
| Provider Route on LiteLLM | `crusoe/` |
| Link to Provider Doc | [Crusoe Managed Inference Documentation ↗](https://docs.crusoecloud.com/managed-inference/overview/index.html) |
| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
<br />
**We support ALL Crusoe models, just set `crusoe/` as a prefix when sending completion requests**
## Available Models
| Model | Description | Context Window |
|-------|-------------|----------------|
| `crusoe/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 reasoning model (May 2025) | 163,840 tokens |
| `crusoe/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 chat model (March 2025) | 163,840 tokens |
| `crusoe/google/gemma-3-12b-it` | Google Gemma 3 12B instruction-tuned | 131,072 tokens |
| `crusoe/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B instruction-tuned | 131,072 tokens |
| `crusoe/moonshotai/Kimi-K2-Thinking` | Kimi K2 extended thinking model | 262,144 tokens |
| `crusoe/openai/gpt-oss-120b` | OpenAI 120B open-source model | 131,072 tokens |
| `crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507` | Qwen3 235B MoE instruction-tuned | 262,144 tokens |
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
```
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Crusoe Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Crusoe call
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Crusoe Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
messages = [{"content": "Write a short story about AI", "role": "user"}]
# Crusoe call with streaming
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
### Function Calling
```python showLineNumbers title="Crusoe Function Calling"
import os
import litellm
from litellm import completion
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
tools = [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA"
}
},
"required": ["location"]
}
}
}]
messages = [{"role": "user", "content": "What's the weather in Boston?"}]
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=messages,
tools=tools,
tool_choice="auto"
)
print(response)
```
## Usage - LiteLLM Proxy Server
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: llama-3.3-70b
litellm_params:
model: crusoe/meta-llama/Llama-3.3-70B-Instruct
api_key: os.environ/CRUSOE_API_KEY
- model_name: deepseek-r1
litellm_params:
model: crusoe/deepseek-ai/DeepSeek-R1-0528
api_key: os.environ/CRUSOE_API_KEY
- model_name: deepseek-v3
litellm_params:
model: crusoe/deepseek-ai/DeepSeek-V3-0324
api_key: os.environ/CRUSOE_API_KEY
- model_name: qwen3-235b
litellm_params:
model: crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507
api_key: os.environ/CRUSOE_API_KEY
- model_name: kimi-k2
litellm_params:
model: crusoe/moonshotai/Kimi-K2-Thinking
api_key: os.environ/CRUSOE_API_KEY
```
## Custom API Base
**Option 1: Environment variable**
```python showLineNumbers title="Custom API Base via env var"
import os
from litellm import completion
os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1"
os.environ["CRUSOE_API_KEY"] = "" # your API key
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=[{"content": "Hello!", "role": "user"}],
)
```
**Option 2: Pass directly**
```python showLineNumbers title="Custom API Base via parameter"
from litellm import completion
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=[{"content": "Hello!", "role": "user"}],
api_base="https://custom.crusoecloud.com/v1",
api_key="your-api-key",
)
```
## Supported OpenAI Parameters
- `temperature`
- `max_tokens`
- `max_completion_tokens`
- `top_p`
- `frequency_penalty`
- `presence_penalty`
- `stop`
- `n`
- `stream`
- `tools`
- `tool_choice`
- `response_format`
- `seed`
- `user`
- `logit_bias`
- `logprobs`
- `top_logprobs`

View file

@ -101,5 +101,13 @@
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
},
"crusoe": {
"base_url": "https://managed-inference-api-proxy.crusoecloud.com/v1",
"api_key_env": "CRUSOE_API_KEY",
"api_base_env": "CRUSOE_API_BASE",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
}
}

View file

@ -20575,6 +20575,98 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"crusoe/deepseek-ai/DeepSeek-R1-0528": {
"input_cost_per_token": 3e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 7e-06,
"supports_function_calling": false,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": false
},
"crusoe/deepseek-ai/DeepSeek-V3-0324": {
"input_cost_per_token": 1.5e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"crusoe/google/gemma-3-12b-it": {
"input_cost_per_token": 1e-07,
"litellm_provider": "crusoe",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"crusoe/meta-llama/Llama-3.3-70B-Instruct": {
"input_cost_per_token": 2e-07,
"litellm_provider": "crusoe",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"crusoe/moonshotai/Kimi-K2-Thinking": {
"input_cost_per_token": 2.5e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"supports_function_calling": false,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": false
},
"crusoe/openai/gpt-oss-120b": {
"input_cost_per_token": 8e-07,
"litellm_provider": "crusoe",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"input_cost_per_token": 3e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"lambda_ai/deepseek-llama3.3-70b": {
"input_cost_per_token": 2e-07,
"litellm_provider": "lambda_ai",

View file

@ -20575,6 +20575,98 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"crusoe/deepseek-ai/DeepSeek-R1-0528": {
"input_cost_per_token": 3e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 7e-06,
"supports_function_calling": false,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": false
},
"crusoe/deepseek-ai/DeepSeek-V3-0324": {
"input_cost_per_token": 1.5e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"crusoe/google/gemma-3-12b-it": {
"input_cost_per_token": 1e-07,
"litellm_provider": "crusoe",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"crusoe/meta-llama/Llama-3.3-70B-Instruct": {
"input_cost_per_token": 2e-07,
"litellm_provider": "crusoe",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"crusoe/moonshotai/Kimi-K2-Thinking": {
"input_cost_per_token": 2.5e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"supports_function_calling": false,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": false
},
"crusoe/openai/gpt-oss-120b": {
"input_cost_per_token": 8e-07,
"litellm_provider": "crusoe",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"input_cost_per_token": 3e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"lambda_ai/deepseek-llama3.3-70b": {
"input_cost_per_token": 2e-07,
"litellm_provider": "lambda_ai",

View file

@ -0,0 +1,108 @@
"""
Tests for Crusoe provider integration
"""
import os
from unittest import mock
import litellm
CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1"
def test_crusoe_json_registry():
"""Test CrusoeChatConfig is loaded from JSON provider registry"""
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
assert JSONProviderRegistry.exists("crusoe")
config = JSONProviderRegistry.get("crusoe")
assert config is not None
assert config.base_url == CRUSOE_API_BASE
assert config.api_key_env == "CRUSOE_API_KEY"
assert config.api_base_env == "CRUSOE_API_BASE"
def test_crusoe_get_openai_compatible_provider_info():
"""Test Crusoe provider info retrieval"""
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
config = create_config_class(JSONProviderRegistry.get("crusoe"))()
# Test with default values (no env vars set)
with mock.patch.dict(os.environ, {}, clear=True):
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
assert api_base == CRUSOE_API_BASE
assert api_key is None
# Test with environment variables
with mock.patch.dict(
os.environ,
{
"CRUSOE_API_KEY": "test-key",
"CRUSOE_API_BASE": "https://custom.crusoecloud.com/v1",
},
):
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
assert api_base == "https://custom.crusoecloud.com/v1"
assert api_key == "test-key"
# Test with explicit parameters (should override env vars)
with mock.patch.dict(
os.environ,
{
"CRUSOE_API_KEY": "env-key",
"CRUSOE_API_BASE": "https://env.crusoecloud.com/v1",
},
):
api_base, api_key = config._get_openai_compatible_provider_info(
"https://param.crusoecloud.com/v1", "param-key"
)
assert api_base == "https://param.crusoecloud.com/v1"
assert api_key == "param-key"
def test_get_llm_provider_crusoe():
"""Test that get_llm_provider correctly identifies Crusoe"""
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
# Test with crusoe/model-name format
model, provider, api_key, api_base = get_llm_provider(
"crusoe/meta-llama/Llama-3.3-70B-Instruct"
)
assert model == "meta-llama/Llama-3.3-70B-Instruct"
assert provider == "crusoe"
def test_crusoe_models_configuration():
"""Test that Crusoe models are configured correctly"""
from litellm import get_model_info
original_model_cost = litellm.model_cost
original_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
try:
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
crusoe_models = [
"crusoe/meta-llama/Llama-3.3-70B-Instruct",
"crusoe/deepseek-ai/DeepSeek-R1-0528",
"crusoe/deepseek-ai/DeepSeek-V3-0324",
"crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507",
"crusoe/moonshotai/Kimi-K2-Thinking",
"crusoe/openai/gpt-oss-120b",
"crusoe/google/gemma-3-12b-it",
]
for model in crusoe_models:
model_info = get_model_info(model)
assert model_info is not None, f"Model info not found for {model}"
assert model_info.get("litellm_provider") == "crusoe", (
f"{model} should have crusoe as provider"
)
assert model_info.get("mode") == "chat", f"{model} should be in chat mode"
finally:
litellm.model_cost = original_model_cost
if original_env is None:
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
else:
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = original_env

View file

@ -0,0 +1,135 @@
import os
from unittest.mock import patch
CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1"
def test_crusoe_json_registry():
"""Test Crusoe is registered in the JSON provider registry"""
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
assert JSONProviderRegistry.exists("crusoe")
config = JSONProviderRegistry.get("crusoe")
assert config is not None
assert config.base_url == CRUSOE_API_BASE
assert config.api_key_env == "CRUSOE_API_KEY"
assert config.api_base_env == "CRUSOE_API_BASE"
def test_crusoe_dynamic_config_defaults():
"""Test dynamic config returns correct default API base"""
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
config = create_config_class(JSONProviderRegistry.get("crusoe"))()
with patch.dict(os.environ, {}, clear=True):
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
assert api_base == CRUSOE_API_BASE
assert api_key is None
def test_crusoe_dynamic_config_env_vars():
"""Test dynamic config reads CRUSOE_API_KEY and CRUSOE_API_BASE from env"""
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
config = create_config_class(JSONProviderRegistry.get("crusoe"))()
with patch.dict(
os.environ,
{"CRUSOE_API_KEY": "test-key", "CRUSOE_API_BASE": "https://custom.crusoe.com/v1"},
):
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
assert api_base == "https://custom.crusoe.com/v1"
assert api_key == "test-key"
def test_crusoe_dynamic_config_explicit_params():
"""Test explicit params override env vars"""
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
config = create_config_class(JSONProviderRegistry.get("crusoe"))()
with patch.dict(os.environ, {"CRUSOE_API_KEY": "env-key"}):
api_base, api_key = config._get_openai_compatible_provider_info(
"https://override.crusoe.com/v1", "override-key"
)
assert api_base == "https://override.crusoe.com/v1"
assert api_key == "override-key"
def test_crusoe_supported_params():
"""Test dynamic config returns standard OpenAI params"""
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
config = create_config_class(JSONProviderRegistry.get("crusoe"))()
params = config.get_supported_openai_params(model="meta-llama/Llama-3.3-70B-Instruct")
assert isinstance(params, list)
assert len(params) > 0
assert "temperature" in params
assert "max_tokens" in params
assert "stream" in params
def test_crusoe_param_mapping_max_completion_tokens():
"""Test max_completion_tokens is mapped to max_tokens for Crusoe"""
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
config = create_config_class(JSONProviderRegistry.get("crusoe"))()
optional_params = config.map_openai_params(
non_default_params={"max_completion_tokens": 1024},
optional_params={},
model="meta-llama/Llama-3.3-70B-Instruct",
drop_params=False,
)
assert "max_tokens" in optional_params, "max_completion_tokens should be mapped to max_tokens"
assert optional_params["max_tokens"] == 1024
assert "max_completion_tokens" not in optional_params
def test_crusoe_provider_detection_by_prefix():
"""Test crusoe/model prefix is correctly routed"""
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
model, provider, _, _ = get_llm_provider("crusoe/meta-llama/Llama-3.3-70B-Instruct")
assert provider == "crusoe"
assert model == "meta-llama/Llama-3.3-70B-Instruct"
def test_crusoe_model_list_populated():
"""Test Crusoe models are present in model_prices_and_context_window.json"""
import litellm
original_model_cost = litellm.model_cost
original_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
try:
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
expected = [
"crusoe/meta-llama/Llama-3.3-70B-Instruct",
"crusoe/deepseek-ai/DeepSeek-R1-0528",
"crusoe/deepseek-ai/DeepSeek-V3-0324",
"crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507",
"crusoe/moonshotai/Kimi-K2-Thinking",
"crusoe/openai/gpt-oss-120b",
"crusoe/google/gemma-3-12b-it",
]
for model in expected:
assert model in litellm.model_cost, f"{model} not found in model_cost"
assert litellm.model_cost[model].get("litellm_provider") == "crusoe"
finally:
litellm.model_cost = original_model_cost
if original_env is None:
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
else:
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = original_env