mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 36145c4c3b into e16d79d635
This commit is contained in:
commit
419f15e56a
16 changed files with 634 additions and 0 deletions
|
|
@ -427,6 +427,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
|
|||
| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Xinference (`xinference`)](https://docs.litellm.ai/docs/providers/xinference) | | | | ✅ | | | | | | |
|
||||
| [ZeroGPU (`zerogpu`)](https://docs.litellm.ai/docs/providers/zerogpu) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
||||
[**Read the Docs**](https://docs.litellm.ai/docs/)
|
||||
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
|
|||
"meta",
|
||||
"cognition",
|
||||
"scx-ai",
|
||||
"zerogpu",
|
||||
];
|
||||
|
||||
impl ExceptionFamily {
|
||||
|
|
|
|||
|
|
@ -680,6 +680,7 @@ moonshot_models: Set = set()
|
|||
publicai_models: Set = set()
|
||||
darkbloom_models: Set = set()
|
||||
openinfer_models: Final[Set[str]] = set()
|
||||
zerogpu_models: Final[Set[str]] = set()
|
||||
v0_models: Set = set()
|
||||
morph_models: Set = set()
|
||||
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
|
||||
|
|
@ -967,6 +968,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
darkbloom_models.add(key)
|
||||
elif value.get("litellm_provider") == "openinfer":
|
||||
openinfer_models.add(key)
|
||||
elif value.get("litellm_provider") == "zerogpu":
|
||||
zerogpu_models.add(key)
|
||||
elif value.get("litellm_provider") == "v0":
|
||||
v0_models.add(key)
|
||||
elif value.get("litellm_provider") == "morph":
|
||||
|
|
@ -1138,6 +1141,7 @@ model_list = list(
|
|||
| publicai_models
|
||||
| darkbloom_models
|
||||
| openinfer_models
|
||||
| zerogpu_models
|
||||
| v0_models
|
||||
| morph_models
|
||||
| opencode_go_models
|
||||
|
|
@ -1256,6 +1260,7 @@ def _build_models_by_provider() -> dict:
|
|||
"publicai": publicai_models,
|
||||
"darkbloom": darkbloom_models,
|
||||
"openinfer": openinfer_models,
|
||||
"zerogpu": zerogpu_models,
|
||||
"v0": v0_models,
|
||||
"morph": morph_models,
|
||||
"opencode_go": opencode_go_models,
|
||||
|
|
|
|||
|
|
@ -1025,6 +1025,7 @@ openai_compatible_endpoints: Final[list[str]] = [
|
|||
"api.x.ai/v1",
|
||||
"ollama.com",
|
||||
"api.galadriel.ai/v1",
|
||||
"https://api.zerogpu.ai/v1",
|
||||
"api.llama.com/compat/v1/",
|
||||
"api.featherless.ai/v1",
|
||||
"inference.api.nscale.com/v1",
|
||||
|
|
@ -1094,6 +1095,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"llamafile",
|
||||
"lm_studio",
|
||||
"galadriel",
|
||||
"zerogpu",
|
||||
"github_copilot", # GitHub Copilot Chat API
|
||||
"chatgpt", # ChatGPT subscription API
|
||||
"novita",
|
||||
|
|
|
|||
|
|
@ -58,6 +58,12 @@
|
|||
"max_completion_tokens": "max_tokens"
|
||||
}
|
||||
},
|
||||
"zerogpu": {
|
||||
"base_url": "https://api.zerogpu.ai/v1",
|
||||
"api_key_env": "ZEROGPU_API_KEY",
|
||||
"api_base_env": "ZEROGPU_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
|
||||
},
|
||||
"chutes": {
|
||||
"base_url": "https://llm.chutes.ai/v1/",
|
||||
"api_key_env": "CHUTES_API_KEY",
|
||||
|
|
|
|||
|
|
@ -65151,6 +65151,139 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/deepseek-v4.1-flash": {
|
||||
"cache_read_input_token_cost": 3e-09,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/glm-5.3-flash": {
|
||||
"cache_read_input_token_cost": 1.5e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.5e-07,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/gpt-4.1-mini": {
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.6e-06,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/gpt-5.4-nano": {
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.25e-06,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/gpt-5.6-luna": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/gpt-oss-120b": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/LFM2.5-1.2B-Instruct": {
|
||||
"input_cost_per_token": 2e-08,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-08,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog"
|
||||
},
|
||||
"zerogpu/llama-3.1-8b-instruct-fast": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.8e-07,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/qwen3-30b-a3b-fp8": {
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 16384,
|
||||
"max_tokens": 16384,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.5e-07,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"together_ai/Qwen/Qwen3.8-Flash": {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
|
|
|
|||
|
|
@ -2529,6 +2529,23 @@
|
|||
"interactions": true
|
||||
}
|
||||
},
|
||||
"zerogpu": {
|
||||
"display_name": "ZeroGPU (`zerogpu`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/zerogpu",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"messages": true,
|
||||
"responses": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"moderations": false,
|
||||
"batches": false,
|
||||
"rerank": false,
|
||||
"a2a": false
|
||||
}
|
||||
},
|
||||
"ragflow": {
|
||||
"display_name": "RAGFlow (`ragflow`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/ragflow",
|
||||
|
|
|
|||
|
|
@ -4117,6 +4117,34 @@
|
|||
],
|
||||
"default_model_placeholder": "gpt-3.5-turbo"
|
||||
},
|
||||
{
|
||||
"provider": "ZeroGPU",
|
||||
"provider_display_name": "ZeroGPU",
|
||||
"litellm_provider": "zerogpu",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://api.zerogpu.ai/v1",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "zerogpu/gpt-oss-120b"
|
||||
},
|
||||
{
|
||||
"provider": "CURSOR",
|
||||
"provider_display_name": "Cursor",
|
||||
|
|
|
|||
|
|
@ -4585,6 +4585,7 @@ class LlmProviders(str, Enum):
|
|||
PRISM = "prism"
|
||||
OPENINFER = "openinfer"
|
||||
INFERENCEAPIS = "inferenceapis"
|
||||
ZEROGPU = "zerogpu"
|
||||
DARKBLOOM = "darkbloom"
|
||||
META = "meta"
|
||||
SAIL = "sail"
|
||||
|
|
|
|||
|
|
@ -65151,6 +65151,139 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/deepseek-v4.1-flash": {
|
||||
"cache_read_input_token_cost": 3e-09,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/glm-5.3-flash": {
|
||||
"cache_read_input_token_cost": 1.5e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.5e-07,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/gpt-4.1-mini": {
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 1047576,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.6e-06,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/gpt-5.4-nano": {
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.25e-06,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/gpt-5.6-luna": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/gpt-oss-120b": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/LFM2.5-1.2B-Instruct": {
|
||||
"input_cost_per_token": 2e-08,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-08,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog"
|
||||
},
|
||||
"zerogpu/llama-3.1-8b-instruct-fast": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.8e-07,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zerogpu/qwen3-30b-a3b-fp8": {
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "zerogpu",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 16384,
|
||||
"max_tokens": 16384,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.5e-07,
|
||||
"source": "https://docs.zerogpu.ai/docs/model-catalog",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"together_ai/Qwen/Qwen3.8-Flash": {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
|
|
|
|||
|
|
@ -2900,6 +2900,23 @@
|
|||
"interactions": true
|
||||
}
|
||||
},
|
||||
"zerogpu": {
|
||||
"display_name": "ZeroGPU (`zerogpu`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/zerogpu",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"messages": true,
|
||||
"responses": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"moderations": false,
|
||||
"batches": false,
|
||||
"rerank": false,
|
||||
"a2a": false
|
||||
}
|
||||
},
|
||||
"ragflow": {
|
||||
"display_name": "RAGFlow (`ragflow`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/ragflow",
|
||||
|
|
|
|||
241
tests/unit/llms/openai_like/test_zerogpu_provider.py
Normal file
241
tests/unit/llms/openai_like/test_zerogpu_provider.py
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from openai import OpenAI
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.proxy.auth.model_checks import get_provider_models
|
||||
|
||||
ZEROGPU_API_BASE: Final = "https://api.zerogpu.ai/v1"
|
||||
|
||||
ZEROGPU_MODELS: Final = (
|
||||
"deepseek-v4.1-flash",
|
||||
"glm-5.3-flash",
|
||||
"gpt-4.1-mini",
|
||||
"gpt-5.4-nano",
|
||||
"gpt-5.6-luna",
|
||||
"gpt-oss-120b",
|
||||
"LFM2.5-1.2B-Instruct",
|
||||
"llama-3.1-8b-instruct-fast",
|
||||
"qwen3-30b-a3b-fp8",
|
||||
)
|
||||
|
||||
|
||||
WEATHER_TOOL: Final = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class _RecordingZeroGPU:
|
||||
def __init__(self, usage: Mapping[str, object]) -> None:
|
||||
self.usage: Final = usage
|
||||
self.requests: tuple[httpx.Request, ...] = ()
|
||||
|
||||
def respond(self, request: httpx.Request) -> httpx.Response:
|
||||
self.requests = (*self.requests, request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-zerogpu",
|
||||
"object": "chat.completion",
|
||||
"created": 1234567890,
|
||||
"model": json.loads(request.content)["model"],
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
"usage": dict(self.usage),
|
||||
},
|
||||
)
|
||||
|
||||
def client(self) -> OpenAI:
|
||||
return OpenAI(
|
||||
api_key="zerogpu-test-key",
|
||||
base_url=ZEROGPU_API_BASE,
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(self.respond)),
|
||||
)
|
||||
|
||||
|
||||
def test_prefixed_model_reaches_zerogpu_chat_completions_without_the_prefix():
|
||||
zerogpu: Final = _RecordingZeroGPU({"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4})
|
||||
|
||||
with zerogpu.client() as client:
|
||||
litellm.completion(
|
||||
model="zerogpu/gpt-oss-120b",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert len(zerogpu.requests) == 1
|
||||
assert str(zerogpu.requests[0].url) == f"{ZEROGPU_API_BASE}/chat/completions"
|
||||
assert json.loads(zerogpu.requests[0].content) == {
|
||||
"model": "gpt-oss-120b",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
}
|
||||
|
||||
|
||||
def test_responses_request_reaches_zerogpu_responses_natively():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post(f"{ZEROGPU_API_BASE}/responses").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "resp_zerogpu",
|
||||
"object": "response",
|
||||
"created_at": 1_792_000_000,
|
||||
"model": "gpt-oss-120b",
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_zerogpu",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "hi", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4},
|
||||
},
|
||||
)
|
||||
response: Final = litellm.responses(model="zerogpu/gpt-oss-120b", input="hello", api_key="zerogpu-test-key")
|
||||
|
||||
body: Final = json.loads(route.calls.last.request.content)
|
||||
assert body["model"] == "gpt-oss-120b"
|
||||
assert body["input"] == "hello"
|
||||
assert response.output[0].content[0].text == "hi"
|
||||
|
||||
|
||||
def test_anthropic_messages_request_is_bridged_to_zerogpu_chat_completions():
|
||||
zerogpu: Final = _RecordingZeroGPU({"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4})
|
||||
|
||||
with zerogpu.client() as client:
|
||||
response: Final = litellm.anthropic.messages.create(
|
||||
model="zerogpu/gpt-oss-120b",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
max_tokens=10,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert [str(request.url) for request in zerogpu.requests] == [f"{ZEROGPU_API_BASE}/chat/completions"]
|
||||
assert json.loads(zerogpu.requests[0].content) == {
|
||||
"model": "gpt-oss-120b",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_tokens": 10,
|
||||
}
|
||||
assert response["content"] == [{"type": "text", "text": "hi"}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ZEROGPU_MODELS)
|
||||
def test_zerogpu_completion_is_charged_at_the_cost_map_rates(model: str):
|
||||
usage: Final = {
|
||||
"prompt_tokens": 1000,
|
||||
"completion_tokens": 200,
|
||||
"total_tokens": 1200,
|
||||
"prompt_tokens_details": {"cached_tokens": 400},
|
||||
}
|
||||
zerogpu: Final = _RecordingZeroGPU(usage)
|
||||
|
||||
with zerogpu.client() as client:
|
||||
response: Final = litellm.completion(
|
||||
model=f"zerogpu/{model}",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
client=client,
|
||||
)
|
||||
|
||||
rates: Final = litellm.model_cost[f"zerogpu/{model}"]
|
||||
cached_rate: Final = rates.get("cache_read_input_token_cost", rates["input_cost_per_token"])
|
||||
expected_cost: Final = (
|
||||
600 * rates["input_cost_per_token"] + 400 * cached_rate + 200 * rates["output_cost_per_token"]
|
||||
)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(expected_cost)
|
||||
assert expected_cost > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ZEROGPU_MODELS)
|
||||
def test_trim_messages_fits_zerogpu_history_into_the_model_context_window(model: str):
|
||||
budget: Final = 20
|
||||
trim_ratio: Final = budget / litellm.model_cost[f"zerogpu/{model}"]["max_input_tokens"]
|
||||
messages: Final = [{"role": "user", "content": "lorem ipsum " * 300}]
|
||||
|
||||
trimmed: Final = litellm.utils.trim_messages(messages, model=f"zerogpu/{model}", trim_ratio=trim_ratio)
|
||||
|
||||
trimmed_tokens: Final = litellm.token_counter(model=f"zerogpu/{model}", messages=trimmed)
|
||||
assert trimmed_tokens <= budget, trimmed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "forwards_tools", "forwards_reasoning_effort"),
|
||||
[("gpt-5.6-luna", True, True), ("gpt-oss-120b", True, True), ("LFM2.5-1.2B-Instruct", False, False)],
|
||||
)
|
||||
def test_tools_and_reasoning_effort_reach_zerogpu_only_for_models_that_support_them(
|
||||
model: str, forwards_tools: bool, forwards_reasoning_effort: bool
|
||||
):
|
||||
zerogpu: Final = _RecordingZeroGPU({"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4})
|
||||
|
||||
with zerogpu.client() as client:
|
||||
litellm.completion(
|
||||
model=f"zerogpu/{model}",
|
||||
messages=[{"role": "user", "content": "weather in Paris?"}],
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="auto",
|
||||
reasoning_effort="high",
|
||||
drop_params=True,
|
||||
client=client,
|
||||
)
|
||||
|
||||
tools: Final = {"tools": [WEATHER_TOOL], "tool_choice": "auto"} if forwards_tools else {}
|
||||
reasoning: Final = {"reasoning_effort": "high"} if forwards_reasoning_effort else {}
|
||||
assert json.loads(zerogpu.requests[0].content) == {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "weather in Paris?"}],
|
||||
**tools,
|
||||
**reasoning,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ZEROGPU_MODELS)
|
||||
def test_model_info_reports_the_reasoning_support_litellm_routes_with(model: str):
|
||||
model_info: Final = litellm.get_model_info(f"zerogpu/{model}")
|
||||
|
||||
assert bool(model_info.get("supports_reasoning")) == litellm.utils.supports_reasoning(f"zerogpu/{model}")
|
||||
|
||||
|
||||
def test_zerogpu_key_and_base_come_from_zerogpu_env_vars(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("ZEROGPU_API_KEY", "zerogpu-env-key")
|
||||
monkeypatch.delenv("ZEROGPU_API_BASE", raising=False)
|
||||
|
||||
assert get_llm_provider(model="zerogpu/gpt-oss-120b") == (
|
||||
"gpt-oss-120b",
|
||||
"zerogpu",
|
||||
"zerogpu-env-key",
|
||||
ZEROGPU_API_BASE,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("ZEROGPU_API_BASE", "https://zerogpu.internal.example/v1")
|
||||
|
||||
assert get_llm_provider(model="zerogpu/gpt-oss-120b")[3] == "https://zerogpu.internal.example/v1"
|
||||
|
||||
|
||||
def test_zerogpu_api_base_alone_selects_the_zerogpu_provider(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("ZEROGPU_API_KEY", "zerogpu-env-key")
|
||||
|
||||
assert get_llm_provider(model="gpt-oss-120b", api_base=ZEROGPU_API_BASE) == (
|
||||
"gpt-oss-120b",
|
||||
"zerogpu",
|
||||
"zerogpu-env-key",
|
||||
ZEROGPU_API_BASE,
|
||||
)
|
||||
|
||||
|
||||
def test_zerogpu_wildcard_lists_every_zerogpu_catalog_model():
|
||||
catalog_models: Final = sorted(
|
||||
key for key, row in litellm.model_cost.items() if row.get("litellm_provider") == "zerogpu"
|
||||
)
|
||||
|
||||
assert catalog_models
|
||||
assert sorted(get_provider_models("zerogpu") or []) == catalog_models
|
||||
|
|
@ -363,6 +363,26 @@ def test_cognition_provider_fields():
|
|||
assert fields_by_key["api_base"]["required"] is False
|
||||
|
||||
|
||||
def test_zerogpu_add_model_form_defaults_resolve_to_a_priced_zerogpu_model():
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
app_instance: Final = FastAPI()
|
||||
app_instance.include_router(router)
|
||||
providers: Final = TestClient(app_instance).get("/public/providers/fields").json()
|
||||
|
||||
zerogpu: Final = next(p for p in providers if p["litellm_provider"] == "zerogpu")
|
||||
fields_by_key: Final = {f["key"]: f for f in zerogpu["credential_fields"]}
|
||||
placeholder: Final = zerogpu["default_model_placeholder"]
|
||||
_, provider, _, api_base = get_llm_provider(model=placeholder)
|
||||
|
||||
assert provider == "zerogpu"
|
||||
assert api_base == fields_by_key["api_base"]["placeholder"]
|
||||
assert litellm.model_cost[placeholder]["litellm_provider"] == "zerogpu"
|
||||
assert fields_by_key["api_key"]["required"] is True
|
||||
assert fields_by_key["api_key"]["field_type"] == "password"
|
||||
|
||||
|
||||
def test_qwen_mainland_provider_fields_carry_the_qianwen_brand():
|
||||
app_instance = FastAPI()
|
||||
app_instance.include_router(router)
|
||||
|
|
|
|||
|
|
@ -115,6 +115,27 @@ def test_github_copilot_backup_rows_match_the_main_catalog() -> None:
|
|||
assert _github_copilot_catalog_rows(PRICES_PATH) == _github_copilot_catalog_rows(BACKUP_PRICES_PATH)
|
||||
|
||||
|
||||
class _TokenLimits(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
|
||||
max_input_tokens: int
|
||||
max_output_tokens: int
|
||||
max_tokens: int
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
|
||||
def test_zerogpu_rows_cap_max_tokens_at_an_output_limit_that_fits_the_context_window(path: Path) -> None:
|
||||
prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes())
|
||||
rows: Final = MappingProxyType(
|
||||
{key: _TokenLimits.model_validate(value) for key, value in prices.items() if key.startswith("zerogpu/")}
|
||||
)
|
||||
assert rows, f"No ZeroGPU rows found in {path}"
|
||||
assert {key: row.max_tokens for key, row in rows.items()} == {
|
||||
key: row.max_output_tokens for key, row in rows.items()
|
||||
}
|
||||
assert tuple(key for key, row in rows.items() if row.max_output_tokens > row.max_input_tokens) == ()
|
||||
|
||||
|
||||
def test_github_copilot_rows_resolve_through_get_model_info() -> None:
|
||||
rows: Final[Mapping[str, _CopilotEndpointRow]] = _copilot_endpoint_rows(PRICES_PATH)
|
||||
assert {
|
||||
|
|
|
|||
|
|
@ -224,6 +224,7 @@ describe("provider_info_helpers", () => {
|
|||
Providers.Sail,
|
||||
Providers.WANDB,
|
||||
Providers.ZAI,
|
||||
Providers.ZeroGPU,
|
||||
];
|
||||
const logolessProviders = Object.values(Providers).filter((provider) => !providerLogoMap[provider]);
|
||||
expect([...logolessProviders].sort()).toEqual([...knownLogolessProviders].sort());
|
||||
|
|
@ -357,6 +358,10 @@ describe("provider_info_helpers", () => {
|
|||
expect(getPlaceholder(Providers.Cognition)).toBe("cognition/swe-1.7");
|
||||
});
|
||||
|
||||
it("should return zerogpu/gpt-oss-120b placeholder for ZeroGPU provider", () => {
|
||||
expect(getPlaceholder("ZeroGPU")).toBe("zerogpu/gpt-oss-120b");
|
||||
});
|
||||
|
||||
it("should return a chatgpt/ placeholder for the CHATGPT dropdown key", () => {
|
||||
expect(getPlaceholder("CHATGPT")).toBe("chatgpt/gpt-5.4");
|
||||
});
|
||||
|
|
|
|||
|
|
@ -197,6 +197,7 @@ export enum Providers {
|
|||
xAI = "xAI",
|
||||
XINFERENCE = "Xinference",
|
||||
ZAI = "Z.AI (Zhipu AI)",
|
||||
ZeroGPU = "ZeroGPU",
|
||||
}
|
||||
|
||||
export const provider_map: Record<string, string> = {
|
||||
|
|
@ -325,6 +326,7 @@ export const provider_map: Record<string, string> = {
|
|||
xAI: "xai",
|
||||
XINFERENCE: "xinference",
|
||||
ZAI: "zai",
|
||||
ZeroGPU: "zerogpu",
|
||||
};
|
||||
|
||||
const standaloneSubproviderSlugs = new Set<string>(["bedrock_mantle"]);
|
||||
|
|
@ -497,6 +499,7 @@ const providerPlaceholderMap: Partial<Record<Providers, string>> = {
|
|||
[Providers.Voyage]: "voyage/",
|
||||
[Providers.WATSONX]: "watsonx/ibm/granite-3-3-8b-instruct",
|
||||
[Providers.ZAI]: "zai/glm-4.5",
|
||||
[Providers.ZeroGPU]: "zerogpu/gpt-oss-120b",
|
||||
};
|
||||
|
||||
export const getPlaceholder = (selectedProvider: string): string => {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue