From 08f67f1e60e98e3e3d804b20376b989f475991d7 Mon Sep 17 00:00:00 2001 From: Amaan Date: Fri, 25 Sep 2026 19:58:51 +0530 Subject: [PATCH 01/18] feat(providers): add ZeroGPU as an OpenAI-compatible provider Registers zerogpu through the JSON provider registry so zerogpu/ routes to https://api.zerogpu.ai/v1/chat/completions with ZEROGPU_API_KEY and an optional ZEROGPU_API_BASE. Adds LlmProviders.ZEROGPU, the constants.py provider and endpoint lists, cost map entries for the 10 ZeroGPU text generation models, the endpoint matrix, the Add Model form fields, the README row and the dashboard provider entry --- README.md | 1 + .../src/exception_mapping_utils/original.rs | 1 + litellm/constants.py | 2 + litellm/llms/openai_like/providers.json | 6 + ...odel_prices_and_context_window_backup.json | 80 +++++++++++++ .../provider_endpoints_support_backup.json | 17 +++ .../provider_create_fields.json | 28 +++++ litellm/types/utils.py | 1 + model_prices_and_context_window.json | 80 +++++++++++++ provider_endpoints_support.json | 17 +++ .../llms/openai_like/test_zerogpu_provider.py | 110 ++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 20 ++++ .../components/provider_info_helpers.test.tsx | 5 + .../src/components/provider_info_helpers.tsx | 3 + 14 files changed, 371 insertions(+) create mode 100644 tests/test_litellm/llms/openai_like/test_zerogpu_provider.py diff --git a/README.md b/README.md index e927c80b8b4..0bbef0c0919 100644 --- a/README.md +++ b/README.md @@ -378,6 +378,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse | [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | | | [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | | | [Xinference (`xinference`)](https://docs.litellm.ai/docs/providers/xinference) | | | | ✅ | | | | | | | +| [ZeroGPU (`zerogpu`)](https://docs.litellm.ai/docs/providers/zerogpu) | ✅ | ✅ | ✅ | | | | | | | | [**Read the Docs**](https://docs.litellm.ai/docs/) diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs index 82392cbd7ee..b360acdc90e 100644 --- a/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs @@ -98,6 +98,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[ "meta", "cognition", "scx-ai", + "zerogpu", ]; impl ExceptionFamily { diff --git a/litellm/constants.py b/litellm/constants.py index e7ba1f6b07f..8a26fdce041 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -947,6 +947,7 @@ openai_compatible_endpoints: Final[list] = [ "https://api.meta.ai/v1", "https://api.cognition.ai/v1", "https://api.scx.ai/v1", + "https://api.zerogpu.ai/v1", "https://gigachat.devices.sberbank.ru/api/v1", ] @@ -1020,6 +1021,7 @@ openai_compatible_providers: Final[list] = [ "meta", # Meta Model API (Muse Spark) - JSON-configured provider "cognition", "scx-ai", + "zerogpu", ] OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers)) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index fe10293c420..dcbc34519fa 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -200,5 +200,11 @@ "temperature_max": 1.99 }, "supported_endpoints": ["/v1/chat/completions"] + }, + "zerogpu": { + "base_url": "https://api.zerogpu.ai/v1", + "api_key_env": "ZEROGPU_API_KEY", + "api_base_env": "ZEROGPU_API_BASE", + "supported_endpoints": ["/v1/chat/completions"] } } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5a207dc4c02..a0c919bafa8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63602,6 +63602,86 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "zerogpu/deepseek-v4.1-flash": { + "input_cost_per_token": 3e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/glm-5.3-flash": { + "input_cost_per_token": 1e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.5e-07, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/gpt-4.1-mini": { + "input_cost_per_token": 4e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 1047576, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/gpt-5.4-nano": { + "input_cost_per_token": 2e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 400000, + "mode": "chat", + "output_cost_per_token": 1.25e-06, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/gpt-5.6-luna": { + "input_cost_per_token": 2e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 272000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/gpt-oss-120b": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/LFM2.5-1.2B-Instruct": { + "input_cost_per_token": 2e-08, + "litellm_provider": "zerogpu", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-08, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/LFM2.5-1.2B-Thinking": { + "input_cost_per_token": 2e-08, + "litellm_provider": "zerogpu", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-08, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/llama-3.1-8b-instruct-fast": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/qwen3-30b-a3b-fp8": { + "input_cost_per_token": 1e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4.5e-07, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, "together_ai/Qwen/Qwen3.8-Flash": { "input_cost_per_token": 9e-08, "litellm_provider": "together_ai", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 1fcb7600a5e..4b21d11e8cf 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2405,6 +2405,23 @@ "interactions": true } }, + "zerogpu": { + "display_name": "ZeroGPU (`zerogpu`)", + "url": "https://docs.litellm.ai/docs/providers/zerogpu", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "ragflow": { "display_name": "RAGFlow (`ragflow`)", "url": "https://docs.litellm.ai/docs/providers/ragflow", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 87d38606aba..52fa136fc8f 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3573,6 +3573,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "ZeroGPU", + "provider_display_name": "ZeroGPU", + "litellm_provider": "zerogpu", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.zerogpu.ai/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "zerogpu/gpt-oss-120b" + }, { "provider": "CURSOR", "provider_display_name": "Cursor", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3e306b48887..f23dfb61c6e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4116,6 +4116,7 @@ class LlmProviders(str, Enum): PINSTRIPES = "pinstripes" COGNITION = "cognition" SCX_AI = "scx-ai" + ZEROGPU = "zerogpu" DARKBLOOM = "darkbloom" META = "meta" LITELLM_AGENT = "litellm_agent" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5a207dc4c02..a0c919bafa8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63602,6 +63602,86 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "zerogpu/deepseek-v4.1-flash": { + "input_cost_per_token": 3e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/glm-5.3-flash": { + "input_cost_per_token": 1e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.5e-07, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/gpt-4.1-mini": { + "input_cost_per_token": 4e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 1047576, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/gpt-5.4-nano": { + "input_cost_per_token": 2e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 400000, + "mode": "chat", + "output_cost_per_token": 1.25e-06, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/gpt-5.6-luna": { + "input_cost_per_token": 2e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 272000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/gpt-oss-120b": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/LFM2.5-1.2B-Instruct": { + "input_cost_per_token": 2e-08, + "litellm_provider": "zerogpu", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-08, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/LFM2.5-1.2B-Thinking": { + "input_cost_per_token": 2e-08, + "litellm_provider": "zerogpu", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-08, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/llama-3.1-8b-instruct-fast": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, + "zerogpu/qwen3-30b-a3b-fp8": { + "input_cost_per_token": 1e-07, + "litellm_provider": "zerogpu", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4.5e-07, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, "together_ai/Qwen/Qwen3.8-Flash": { "input_cost_per_token": 9e-08, "litellm_provider": "together_ai", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e6cb0592a15..c95a6857fea 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2734,6 +2734,23 @@ "interactions": true } }, + "zerogpu": { + "display_name": "ZeroGPU (`zerogpu`)", + "url": "https://docs.litellm.ai/docs/providers/zerogpu", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "ragflow": { "display_name": "RAGFlow (`ragflow`)", "url": "https://docs.litellm.ai/docs/providers/ragflow", diff --git a/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py b/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py new file mode 100644 index 00000000000..aebe4e8bc27 --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py @@ -0,0 +1,110 @@ +import json +from typing import Final + +import httpx +import pytest +from openai import OpenAI + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + +ZEROGPU_API_BASE: Final = "https://api.zerogpu.ai/v1" + +ZEROGPU_MODELS: Final = ( + "deepseek-v4.1-flash", + "glm-5.3-flash", + "gpt-4.1-mini", + "gpt-5.4-nano", + "gpt-5.6-luna", + "gpt-oss-120b", + "LFM2.5-1.2B-Instruct", + "LFM2.5-1.2B-Thinking", + "llama-3.1-8b-instruct-fast", + "qwen3-30b-a3b-fp8", +) + + +def _zerogpu_client(requests: list[httpx.Request], usage: dict[str, int]) -> OpenAI: + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-zerogpu", + "object": "chat.completion", + "created": 1234567890, + "model": json.loads(request.content)["model"], + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": usage, + }, + ) + + return OpenAI( + api_key="zerogpu-test-key", + base_url=ZEROGPU_API_BASE, + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + + +def test_prefixed_model_reaches_zerogpu_chat_completions_without_the_prefix(): + requests: Final[list[httpx.Request]] = [] + client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) + + with client: + litellm.completion( + model="zerogpu/gpt-oss-120b", + messages=[{"role": "user", "content": "hello"}], + client=client, + ) + + assert len(requests) == 1 + assert str(requests[0].url) == f"{ZEROGPU_API_BASE}/chat/completions" + assert json.loads(requests[0].content) == { + "model": "gpt-oss-120b", + "messages": [{"role": "user", "content": "hello"}], + } + + +@pytest.mark.parametrize("model", ZEROGPU_MODELS) +def test_zerogpu_completion_is_charged_at_the_cost_map_rates(model: str): + requests: Final[list[httpx.Request]] = [] + client: Final = _zerogpu_client(requests, {"prompt_tokens": 1000, "completion_tokens": 200, "total_tokens": 1200}) + + with client: + response: Final = litellm.completion( + model=f"zerogpu/{model}", + messages=[{"role": "user", "content": "hello"}], + client=client, + ) + + rates: Final = litellm.model_cost[f"zerogpu/{model}"] + expected_cost: Final = 1000 * rates["input_cost_per_token"] + 200 * rates["output_cost_per_token"] + assert response._hidden_params["response_cost"] == pytest.approx(expected_cost) + assert expected_cost > 0 + + +def test_zerogpu_key_and_base_come_from_zerogpu_env_vars(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("ZEROGPU_API_KEY", "zerogpu-env-key") + monkeypatch.delenv("ZEROGPU_API_BASE", raising=False) + + assert get_llm_provider(model="zerogpu/gpt-oss-120b") == ( + "gpt-oss-120b", + "zerogpu", + "zerogpu-env-key", + ZEROGPU_API_BASE, + ) + + monkeypatch.setenv("ZEROGPU_API_BASE", "https://zerogpu.internal.example/v1") + + assert get_llm_provider(model="zerogpu/gpt-oss-120b")[3] == "https://zerogpu.internal.example/v1" + + +def test_zerogpu_api_base_alone_selects_the_zerogpu_provider(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("ZEROGPU_API_KEY", "zerogpu-env-key") + + assert get_llm_provider(model="gpt-oss-120b", api_base=ZEROGPU_API_BASE) == ( + "gpt-oss-120b", + "zerogpu", + "zerogpu-env-key", + ZEROGPU_API_BASE, + ) diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 0dec44af402..0782848fd38 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -340,6 +340,26 @@ def test_cognition_provider_fields(): assert fields_by_key["api_base"]["required"] is False +def test_zerogpu_add_model_form_defaults_resolve_to_a_priced_zerogpu_model(): + import litellm + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + app_instance: Final = FastAPI() + app_instance.include_router(router) + providers: Final = TestClient(app_instance).get("/public/providers/fields").json() + + zerogpu: Final = next(p for p in providers if p["litellm_provider"] == "zerogpu") + fields_by_key: Final = {f["key"]: f for f in zerogpu["credential_fields"]} + placeholder: Final = zerogpu["default_model_placeholder"] + _, provider, _, api_base = get_llm_provider(model=placeholder) + + assert provider == "zerogpu" + assert api_base == fields_by_key["api_base"]["placeholder"] + assert litellm.model_cost[placeholder]["litellm_provider"] == "zerogpu" + assert fields_by_key["api_key"]["required"] is True + assert fields_by_key["api_key"]["field_type"] == "password" + + def test_qwen_mainland_provider_fields_carry_the_qianwen_brand(): app_instance = FastAPI() app_instance.include_router(router) diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index d6b8d00a023..f82464e0393 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -196,6 +196,7 @@ describe("provider_info_helpers", () => { Providers.PREDIBASE, Providers.WANDB, Providers.ZAI, + Providers.ZeroGPU, ]; const logolessProviders = Object.values(Providers).filter((provider) => !providerLogoMap[provider]); expect([...logolessProviders].sort()).toEqual([...knownLogolessProviders].sort()); @@ -304,6 +305,10 @@ describe("provider_info_helpers", () => { expect(getPlaceholder(Providers.Cognition)).toBe("cognition/swe-1.7"); }); + it("should return zerogpu/gpt-oss-120b placeholder for ZeroGPU provider", () => { + expect(getPlaceholder("ZeroGPU")).toBe("zerogpu/gpt-oss-120b"); + }); + it("should return a chatgpt/ placeholder for the CHATGPT dropdown key", () => { expect(getPlaceholder("CHATGPT")).toBe("chatgpt/gpt-5.4"); }); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index eab55503da8..5a6e94ace81 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -182,6 +182,7 @@ export enum Providers { xAI = "xAI", XINFERENCE = "Xinference", ZAI = "Z.AI (Zhipu AI)", + ZeroGPU = "ZeroGPU", } export const provider_map: Record = { @@ -300,6 +301,7 @@ export const provider_map: Record = { xAI: "xai", XINFERENCE: "xinference", ZAI: "zai", + ZeroGPU: "zerogpu", }; const standaloneSubproviderSlugs = new Set(["bedrock_mantle"]); @@ -455,6 +457,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Voyage]: "voyage/", [Providers.WATSONX]: "watsonx/ibm/granite-3-3-8b-instruct", [Providers.ZAI]: "zai/glm-4.5", + [Providers.ZeroGPU]: "zerogpu/gpt-oss-120b", }; export const getPlaceholder = (selectedProvider: string): string => { From e8f1ce3908fe6aa2c6075d0ac70f7e1678cc6977 Mon Sep 17 00:00:00 2001 From: Amaan Date: Fri, 25 Sep 2026 22:49:51 +0530 Subject: [PATCH 02/18] feat(zerogpu): add capability flags verified against the live API Live calls to https://api.zerogpu.ai/v1/chat/completions confirmed tools, tool_choice and json_schema response_format on 8 of the 10 catalog text models, cached token reporting on deepseek-v4.1-flash, gpt-4.1-mini, gpt-5.4-nano and gpt-5.6-luna, and reasoning_effort on gpt-5.6-luna. LFM2.5-1.2B-Instruct answers stream requests with plain JSON instead of SSE and LFM2.5-1.2B-Thinking returns 500 on every request, so both stay out of the cost map until the API serves them correctly --- ...odel_prices_and_context_window_backup.json | 68 ++++++++++++------- model_prices_and_context_window.json | 68 ++++++++++++------- .../llms/openai_like/test_zerogpu_provider.py | 57 ++++++++++++++-- 3 files changed, 140 insertions(+), 53 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a0c919bafa8..75000a9cc90 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63603,76 +63603,93 @@ "supports_tool_choice": true }, "zerogpu/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 3e-09, "input_cost_per_token": 3e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/glm-5.3-flash": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.5e-07, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/gpt-4.1-mini": { + "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1047576, "mode": "chat", "output_cost_per_token": 1.6e-06, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/gpt-5.4-nano": { + "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 400000, "mode": "chat", "output_cost_per_token": 1.25e-06, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/gpt-5.6-luna": { + "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 272000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/gpt-oss-120b": { + "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://docs.zerogpu.ai/docs/model-catalog" - }, - "zerogpu/LFM2.5-1.2B-Instruct": { - "input_cost_per_token": 2e-08, - "litellm_provider": "zerogpu", - "max_input_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 5e-08, - "source": "https://docs.zerogpu.ai/docs/model-catalog" - }, - "zerogpu/LFM2.5-1.2B-Thinking": { - "input_cost_per_token": 2e-08, - "litellm_provider": "zerogpu", - "max_input_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 5e-08, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/llama-3.1-8b-instruct-fast": { + "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.8e-07, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/qwen3-30b-a3b-fp8": { "input_cost_per_token": 1e-07, @@ -63680,7 +63697,10 @@ "max_input_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.5e-07, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "together_ai/Qwen/Qwen3.8-Flash": { "input_cost_per_token": 9e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a0c919bafa8..75000a9cc90 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63603,76 +63603,93 @@ "supports_tool_choice": true }, "zerogpu/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 3e-09, "input_cost_per_token": 3e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/glm-5.3-flash": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.5e-07, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/gpt-4.1-mini": { + "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1047576, "mode": "chat", "output_cost_per_token": 1.6e-06, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/gpt-5.4-nano": { + "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 400000, "mode": "chat", "output_cost_per_token": 1.25e-06, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/gpt-5.6-luna": { + "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 272000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/gpt-oss-120b": { + "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://docs.zerogpu.ai/docs/model-catalog" - }, - "zerogpu/LFM2.5-1.2B-Instruct": { - "input_cost_per_token": 2e-08, - "litellm_provider": "zerogpu", - "max_input_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 5e-08, - "source": "https://docs.zerogpu.ai/docs/model-catalog" - }, - "zerogpu/LFM2.5-1.2B-Thinking": { - "input_cost_per_token": 2e-08, - "litellm_provider": "zerogpu", - "max_input_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 5e-08, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/llama-3.1-8b-instruct-fast": { + "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.8e-07, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "zerogpu/qwen3-30b-a3b-fp8": { "input_cost_per_token": 1e-07, @@ -63680,7 +63697,10 @@ "max_input_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.5e-07, - "source": "https://docs.zerogpu.ai/docs/model-catalog" + "source": "https://docs.zerogpu.ai/docs/model-catalog", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "together_ai/Qwen/Qwen3.8-Flash": { "input_cost_per_token": 9e-08, diff --git a/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py b/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py index aebe4e8bc27..4ae42b7084e 100644 --- a/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py +++ b/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py @@ -17,14 +17,21 @@ ZEROGPU_MODELS: Final = ( "gpt-5.4-nano", "gpt-5.6-luna", "gpt-oss-120b", - "LFM2.5-1.2B-Instruct", - "LFM2.5-1.2B-Thinking", "llama-3.1-8b-instruct-fast", "qwen3-30b-a3b-fp8", ) -def _zerogpu_client(requests: list[httpx.Request], usage: dict[str, int]) -> OpenAI: +WEATHER_TOOL: Final = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, +} + + +def _zerogpu_client(requests: list[httpx.Request], usage: dict[str, object]) -> OpenAI: def respond(request: httpx.Request) -> httpx.Response: requests.append(request) return httpx.Response( @@ -68,7 +75,13 @@ def test_prefixed_model_reaches_zerogpu_chat_completions_without_the_prefix(): @pytest.mark.parametrize("model", ZEROGPU_MODELS) def test_zerogpu_completion_is_charged_at_the_cost_map_rates(model: str): requests: Final[list[httpx.Request]] = [] - client: Final = _zerogpu_client(requests, {"prompt_tokens": 1000, "completion_tokens": 200, "total_tokens": 1200}) + usage: Final = { + "prompt_tokens": 1000, + "completion_tokens": 200, + "total_tokens": 1200, + "prompt_tokens_details": {"cached_tokens": 400}, + } + client: Final = _zerogpu_client(requests, usage) with client: response: Final = litellm.completion( @@ -78,11 +91,45 @@ def test_zerogpu_completion_is_charged_at_the_cost_map_rates(model: str): ) rates: Final = litellm.model_cost[f"zerogpu/{model}"] - expected_cost: Final = 1000 * rates["input_cost_per_token"] + 200 * rates["output_cost_per_token"] + cached_rate: Final = rates.get("cache_read_input_token_cost", rates["input_cost_per_token"]) + expected_cost: Final = ( + 600 * rates["input_cost_per_token"] + 400 * cached_rate + 200 * rates["output_cost_per_token"] + ) assert response._hidden_params["response_cost"] == pytest.approx(expected_cost) assert expected_cost > 0 +@pytest.mark.parametrize( + ("model", "forwards_reasoning_effort"), + [("gpt-5.6-luna", True), ("gpt-oss-120b", False)], +) +def test_tools_reach_zerogpu_and_reasoning_effort_only_for_reasoning_models( + model: str, forwards_reasoning_effort: bool +): + requests: Final[list[httpx.Request]] = [] + client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) + + with client: + litellm.completion( + model=f"zerogpu/{model}", + messages=[{"role": "user", "content": "weather in Paris?"}], + tools=[WEATHER_TOOL], + tool_choice="auto", + reasoning_effort="high", + drop_params=True, + client=client, + ) + + reasoning: Final = {"reasoning_effort": "high"} if forwards_reasoning_effort else {} + assert json.loads(requests[0].content) == { + "model": model, + "messages": [{"role": "user", "content": "weather in Paris?"}], + "tools": [WEATHER_TOOL], + "tool_choice": "auto", + **reasoning, + } + + def test_zerogpu_key_and_base_come_from_zerogpu_env_vars(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("ZEROGPU_API_KEY", "zerogpu-env-key") monkeypatch.delenv("ZEROGPU_API_BASE", raising=False) From 8581195a7bd9dcb32519b7e9f8973217e757d46a Mon Sep 17 00:00:00 2001 From: Amaan Date: Wed, 30 Sep 2026 21:22:19 +0530 Subject: [PATCH 03/18] feat(zerogpu): price LFM2.5-1.2B-Instruct Adds zerogpu/LFM2.5-1.2B-Instruct to the cost map with the catalog prices and context window. It gets no capability flags because the live API ignores tools, response_format and max_completion_tokens for this model, so LiteLLM drops tool params for it instead of sending them. LFM2.5-1.2B-Thinking stays out of the cost map --- litellm/model_prices_and_context_window_backup.json | 8 ++++++++ model_prices_and_context_window.json | 8 ++++++++ .../llms/openai_like/test_zerogpu_provider.py | 13 +++++++------ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 75000a9cc90..26820b4b629 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63679,6 +63679,14 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "zerogpu/LFM2.5-1.2B-Instruct": { + "input_cost_per_token": 2e-08, + "litellm_provider": "zerogpu", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-08, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, "zerogpu/llama-3.1-8b-instruct-fast": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1.5e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 75000a9cc90..26820b4b629 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63679,6 +63679,14 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "zerogpu/LFM2.5-1.2B-Instruct": { + "input_cost_per_token": 2e-08, + "litellm_provider": "zerogpu", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-08, + "source": "https://docs.zerogpu.ai/docs/model-catalog" + }, "zerogpu/llama-3.1-8b-instruct-fast": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1.5e-07, diff --git a/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py b/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py index 4ae42b7084e..c1994499908 100644 --- a/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py +++ b/tests/test_litellm/llms/openai_like/test_zerogpu_provider.py @@ -17,6 +17,7 @@ ZEROGPU_MODELS: Final = ( "gpt-5.4-nano", "gpt-5.6-luna", "gpt-oss-120b", + "LFM2.5-1.2B-Instruct", "llama-3.1-8b-instruct-fast", "qwen3-30b-a3b-fp8", ) @@ -100,11 +101,11 @@ def test_zerogpu_completion_is_charged_at_the_cost_map_rates(model: str): @pytest.mark.parametrize( - ("model", "forwards_reasoning_effort"), - [("gpt-5.6-luna", True), ("gpt-oss-120b", False)], + ("model", "forwards_tools", "forwards_reasoning_effort"), + [("gpt-5.6-luna", True, True), ("gpt-oss-120b", True, False), ("LFM2.5-1.2B-Instruct", False, False)], ) -def test_tools_reach_zerogpu_and_reasoning_effort_only_for_reasoning_models( - model: str, forwards_reasoning_effort: bool +def test_tools_and_reasoning_effort_reach_zerogpu_only_for_models_that_support_them( + model: str, forwards_tools: bool, forwards_reasoning_effort: bool ): requests: Final[list[httpx.Request]] = [] client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) @@ -120,12 +121,12 @@ def test_tools_reach_zerogpu_and_reasoning_effort_only_for_reasoning_models( client=client, ) + tools: Final = {"tools": [WEATHER_TOOL], "tool_choice": "auto"} if forwards_tools else {} reasoning: Final = {"reasoning_effort": "high"} if forwards_reasoning_effort else {} assert json.loads(requests[0].content) == { "model": model, "messages": [{"role": "user", "content": "weather in Paris?"}], - "tools": [WEATHER_TOOL], - "tool_choice": "auto", + **tools, **reasoning, } From 249b4212a59f934b9e868689bcd5f5f80291ba62 Mon Sep 17 00:00:00 2001 From: Amaan Date: Mon, 5 Oct 2026 18:47:41 +0530 Subject: [PATCH 04/18] fix(zerogpu): reject transcription requests before calling the upstream API ZeroGPU only serves chat completions, so drop it from the OpenAI transcription transport set. A zerogpu transcription request now fails locally instead of hitting an /audio/transcriptions endpoint that does not exist --- litellm/constants.py | 2 +- tests/unit/llms/openai_like/test_zerogpu_provider.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/litellm/constants.py b/litellm/constants.py index 2790fdef995..3a35b80ba8a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1029,7 +1029,7 @@ openai_compatible_providers: Final[list] = [ "zerogpu", ] -OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers)) +OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai", *openai_compatible_providers} - {"zerogpu"}) openai_text_completion_compatible_providers: Final[list] = [ # providers that support `/v1/completions` "together_ai", diff --git a/tests/unit/llms/openai_like/test_zerogpu_provider.py b/tests/unit/llms/openai_like/test_zerogpu_provider.py index c1994499908..ef402dc7da6 100644 --- a/tests/unit/llms/openai_like/test_zerogpu_provider.py +++ b/tests/unit/llms/openai_like/test_zerogpu_provider.py @@ -73,6 +73,16 @@ def test_prefixed_model_reaches_zerogpu_chat_completions_without_the_prefix(): } +def test_transcription_is_rejected_without_calling_zerogpu(): + requests: Final[list[httpx.Request]] = [] + client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) + + with client, pytest.raises(ValueError, match="Unmapped provider"): + litellm.transcription(model="zerogpu/whisper-1", file=("audio.wav", b"RIFF", "audio/wav"), client=client) + + assert requests == [] + + @pytest.mark.parametrize("model", ZEROGPU_MODELS) def test_zerogpu_completion_is_charged_at_the_cost_map_rates(model: str): requests: Final[list[httpx.Request]] = [] From 6819c566589fa54cb7952354f7d5bc87f9228a57 Mon Sep 17 00:00:00 2001 From: Amaan Date: Mon, 5 Oct 2026 18:47:43 +0530 Subject: [PATCH 05/18] test(zerogpu): cover the responses and messages bridges Both endpoints reach ZeroGPU through chat completions, so assert the request each one sends upstream and the reply it returns --- .../llms/openai_like/test_zerogpu_provider.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/unit/llms/openai_like/test_zerogpu_provider.py b/tests/unit/llms/openai_like/test_zerogpu_provider.py index ef402dc7da6..a9e2ff9f344 100644 --- a/tests/unit/llms/openai_like/test_zerogpu_provider.py +++ b/tests/unit/llms/openai_like/test_zerogpu_provider.py @@ -73,6 +73,42 @@ def test_prefixed_model_reaches_zerogpu_chat_completions_without_the_prefix(): } +def test_responses_request_is_bridged_to_zerogpu_chat_completions(): + requests: Final[list[httpx.Request]] = [] + client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) + + with client: + response: Final = litellm.responses(model="zerogpu/gpt-oss-120b", input="hello", client=client) + + assert [str(request.url) for request in requests] == [f"{ZEROGPU_API_BASE}/chat/completions"] + assert json.loads(requests[0].content) == { + "model": "gpt-oss-120b", + "messages": [{"role": "user", "content": "hello"}], + } + assert response.output[0].content[0].text == "hi" + + +def test_anthropic_messages_request_is_bridged_to_zerogpu_chat_completions(): + requests: Final[list[httpx.Request]] = [] + client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) + + with client: + response: Final = litellm.anthropic.messages.create( + model="zerogpu/gpt-oss-120b", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + client=client, + ) + + assert [str(request.url) for request in requests] == [f"{ZEROGPU_API_BASE}/chat/completions"] + assert json.loads(requests[0].content) == { + "model": "gpt-oss-120b", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + } + assert response["content"] == [{"type": "text", "text": "hi"}] + + def test_transcription_is_rejected_without_calling_zerogpu(): requests: Final[list[httpx.Request]] = [] client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) From 480ecd8f09b4a52d8d61fbb922f09865eb6413e6 Mon Sep 17 00:00:00 2001 From: Amaan Date: Thu, 8 Oct 2026 18:54:33 +0530 Subject: [PATCH 06/18] refactor(zerogpu): move zerogpu out of the provider list tails New upstream providers append to the end of openai_compatible_endpoints, openai_compatible_providers and providers.json, so an entry at the tail conflicts with every one of them. Placing zerogpu next to long-stable entries keeps those merges clean. No behavior change --- litellm/constants.py | 4 ++-- litellm/llms/openai_like/providers.json | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 54139a10c9f..9e7571bbef5 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -940,6 +940,7 @@ openai_compatible_endpoints: Final[list] = [ "api.x.ai/v1", "ollama.com", "api.galadriel.ai/v1", + "https://api.zerogpu.ai/v1", "api.llama.com/compat/v1/", "api.featherless.ai/v1", "inference.api.nscale.com/v1", @@ -974,7 +975,6 @@ openai_compatible_endpoints: Final[list] = [ "https://api.scx.ai/v1", "https://api.prisminference.com/v1", "https://api.reka.ai/v1", - "https://api.zerogpu.ai/v1", "https://gigachat.devices.sberbank.ru/api/v1", ] @@ -1008,6 +1008,7 @@ openai_compatible_providers: Final[list] = [ "llamafile", "lm_studio", "galadriel", + "zerogpu", "github_copilot", # GitHub Copilot Chat API "chatgpt", # ChatGPT subscription API "novita", @@ -1052,7 +1053,6 @@ openai_compatible_providers: Final[list] = [ "prism", "sail", "reka", - "zerogpu", ] OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai", *openai_compatible_providers} - {"zerogpu"}) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 9095e610ca7..53f114ca19d 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -58,6 +58,12 @@ "max_completion_tokens": "max_tokens" } }, + "zerogpu": { + "base_url": "https://api.zerogpu.ai/v1", + "api_key_env": "ZEROGPU_API_KEY", + "api_base_env": "ZEROGPU_API_BASE", + "supported_endpoints": ["/v1/chat/completions"] + }, "chutes": { "base_url": "https://llm.chutes.ai/v1/", "api_key_env": "CHUTES_API_KEY", @@ -223,11 +229,5 @@ "base_url": "https://api.reka.ai/v1", "api_key_env": "REKA_API_KEY", "api_base_env": "REKA_API_BASE" - }, - "zerogpu": { - "base_url": "https://api.zerogpu.ai/v1", - "api_key_env": "ZEROGPU_API_KEY", - "api_base_env": "ZEROGPU_API_BASE", - "supported_endpoints": ["/v1/chat/completions"] } } From dd6c1c7930ed45f149778b5d3abf04b4c012bd16 Mon Sep 17 00:00:00 2001 From: Amaan Date: Fri, 9 Oct 2026 18:30:51 +0530 Subject: [PATCH 07/18] fix(zerogpu): price cached input for qwen3-30b-a3b-fp8 The ZeroGPU model catalog (https://docs.zerogpu.ai/docs/model-catalog, checked 2026-10-09) lists cached input for qwen3-30b-a3b-fp8 at $0.05 per 1M tokens. Without cache_read_input_token_cost, cached tokens were billed at the full $0.10 input rate --- litellm/model_prices_and_context_window_backup.json | 1 + model_prices_and_context_window.json | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4b636a952bd..9bdf638d28a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -65321,6 +65321,7 @@ "supports_tool_choice": true }, "zerogpu/qwen3-30b-a3b-fp8": { + "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 32768, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4b636a952bd..9bdf638d28a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -65321,6 +65321,7 @@ "supports_tool_choice": true }, "zerogpu/qwen3-30b-a3b-fp8": { + "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 32768, From d6ce2e33a7dbf70eb2ae9ecdf17c0c70315c52fe Mon Sep 17 00:00:00 2001 From: Amaan Date: Fri, 9 Oct 2026 18:58:14 +0530 Subject: [PATCH 08/18] revert(zerogpu): leave the audio transcription provider set unchanged Per review, keep OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS as upstream defines it. This PR only covers chat completions, and ZeroGPU now serves /audio/transcriptions anyway, so the local refusal and its test are removed --- litellm/constants.py | 2 +- tests/unit/llms/openai_like/test_zerogpu_provider.py | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 9e7571bbef5..d8807a4e615 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1055,7 +1055,7 @@ openai_compatible_providers: Final[list] = [ "reka", ] -OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai", *openai_compatible_providers} - {"zerogpu"}) +OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers)) openai_text_completion_compatible_providers: Final[list] = [ # providers that support `/v1/completions` "together_ai", diff --git a/tests/unit/llms/openai_like/test_zerogpu_provider.py b/tests/unit/llms/openai_like/test_zerogpu_provider.py index a9e2ff9f344..6beafc57b23 100644 --- a/tests/unit/llms/openai_like/test_zerogpu_provider.py +++ b/tests/unit/llms/openai_like/test_zerogpu_provider.py @@ -109,16 +109,6 @@ def test_anthropic_messages_request_is_bridged_to_zerogpu_chat_completions(): assert response["content"] == [{"type": "text", "text": "hi"}] -def test_transcription_is_rejected_without_calling_zerogpu(): - requests: Final[list[httpx.Request]] = [] - client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) - - with client, pytest.raises(ValueError, match="Unmapped provider"): - litellm.transcription(model="zerogpu/whisper-1", file=("audio.wav", b"RIFF", "audio/wav"), client=client) - - assert requests == [] - - @pytest.mark.parametrize("model", ZEROGPU_MODELS) def test_zerogpu_completion_is_charged_at_the_cost_map_rates(model: str): requests: Final[list[httpx.Request]] = [] From a48ea92db7e65838fd7ab6c89b200e50e02222c9 Mon Sep 17 00:00:00 2001 From: Amaan Date: Sat, 10 Oct 2026 23:41:05 +0530 Subject: [PATCH 09/18] fix(zerogpu): add max_tokens so trim_messages can size ZeroGPU history trim_messages reads max_tokens as the fallback for max_input_tokens and evaluates it eagerly, so without it the lookup raised, got swallowed, and the full history came back untrimmed. max_tokens matches each model's maxTokens in ZeroGPU's models API (https://api-dashboard.zerogpu.ai/api/models, checked 2026-10-10) --- litellm/model_prices_and_context_window_backup.json | 9 +++++++++ model_prices_and_context_window.json | 9 +++++++++ tests/unit/llms/openai_like/test_zerogpu_provider.py | 12 ++++++++++++ 3 files changed, 30 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9bdf638d28a..c0898bbdd88 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -65228,6 +65228,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, + "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65241,6 +65242,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, + "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65253,6 +65255,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1047576, + "max_tokens": 1047576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65266,6 +65269,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 400000, + "max_tokens": 400000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65279,6 +65283,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 272000, + "max_tokens": 272000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65293,6 +65298,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65304,6 +65310,7 @@ "input_cost_per_token": 2e-08, "litellm_provider": "zerogpu", "max_input_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 5e-08, "source": "https://docs.zerogpu.ai/docs/model-catalog" @@ -65313,6 +65320,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65325,6 +65333,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9bdf638d28a..c0898bbdd88 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -65228,6 +65228,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, + "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65241,6 +65242,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, + "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65253,6 +65255,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1047576, + "max_tokens": 1047576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65266,6 +65269,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 400000, + "max_tokens": 400000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65279,6 +65283,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 272000, + "max_tokens": 272000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65293,6 +65298,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65304,6 +65310,7 @@ "input_cost_per_token": 2e-08, "litellm_provider": "zerogpu", "max_input_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 5e-08, "source": "https://docs.zerogpu.ai/docs/model-catalog" @@ -65313,6 +65320,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -65325,6 +65333,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", diff --git a/tests/unit/llms/openai_like/test_zerogpu_provider.py b/tests/unit/llms/openai_like/test_zerogpu_provider.py index 6beafc57b23..3754a400c69 100644 --- a/tests/unit/llms/openai_like/test_zerogpu_provider.py +++ b/tests/unit/llms/openai_like/test_zerogpu_provider.py @@ -136,6 +136,18 @@ def test_zerogpu_completion_is_charged_at_the_cost_map_rates(model: str): assert expected_cost > 0 +@pytest.mark.parametrize("model", ZEROGPU_MODELS) +def test_trim_messages_fits_zerogpu_history_into_the_model_context_window(model: str): + budget: Final = 20 + trim_ratio: Final = budget / litellm.model_cost[f"zerogpu/{model}"]["max_input_tokens"] + messages: Final = [{"role": "user", "content": "lorem ipsum " * 300}] + + trimmed: Final = litellm.utils.trim_messages(messages, model=f"zerogpu/{model}", trim_ratio=trim_ratio) + + trimmed_tokens: Final = litellm.token_counter(model=f"zerogpu/{model}", messages=trimmed) + assert trimmed_tokens <= budget, trimmed + + @pytest.mark.parametrize( ("model", "forwards_tools", "forwards_reasoning_effort"), [("gpt-5.6-luna", True, True), ("gpt-oss-120b", True, False), ("LFM2.5-1.2B-Instruct", False, False)], From bb9bf1a88e401a0627864ac6ade39bc42a11e106 Mon Sep 17 00:00:00 2001 From: Amaan Date: Mon, 12 Oct 2026 11:14:19 +0530 Subject: [PATCH 10/18] test(zerogpu): record mocked requests on a recorder object The mock handler appended to a list the caller passed in, which AGENTS.md forbids. A recorder now owns the captured requests as a tuple --- .../llms/openai_like/test_zerogpu_provider.py | 65 ++++++++++--------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/tests/unit/llms/openai_like/test_zerogpu_provider.py b/tests/unit/llms/openai_like/test_zerogpu_provider.py index 3754a400c69..614418d486d 100644 --- a/tests/unit/llms/openai_like/test_zerogpu_provider.py +++ b/tests/unit/llms/openai_like/test_zerogpu_provider.py @@ -1,4 +1,5 @@ import json +from collections.abc import Mapping from typing import Final import httpx @@ -32,9 +33,13 @@ WEATHER_TOOL: Final = { } -def _zerogpu_client(requests: list[httpx.Request], usage: dict[str, object]) -> OpenAI: - def respond(request: httpx.Request) -> httpx.Response: - requests.append(request) +class _RecordingZeroGPU: + def __init__(self, usage: Mapping[str, object]) -> None: + self.usage: Final = usage + self.requests: tuple[httpx.Request, ...] = () + + def respond(self, request: httpx.Request) -> httpx.Response: + self.requests = (*self.requests, request) return httpx.Response( 200, json={ @@ -43,45 +48,44 @@ def _zerogpu_client(requests: list[httpx.Request], usage: dict[str, object]) -> "created": 1234567890, "model": json.loads(request.content)["model"], "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], - "usage": usage, + "usage": dict(self.usage), }, ) - return OpenAI( - api_key="zerogpu-test-key", - base_url=ZEROGPU_API_BASE, - http_client=httpx.Client(transport=httpx.MockTransport(respond)), - ) + def client(self) -> OpenAI: + return OpenAI( + api_key="zerogpu-test-key", + base_url=ZEROGPU_API_BASE, + http_client=httpx.Client(transport=httpx.MockTransport(self.respond)), + ) def test_prefixed_model_reaches_zerogpu_chat_completions_without_the_prefix(): - requests: Final[list[httpx.Request]] = [] - client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) + zerogpu: Final = _RecordingZeroGPU({"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) - with client: + with zerogpu.client() as client: litellm.completion( model="zerogpu/gpt-oss-120b", messages=[{"role": "user", "content": "hello"}], client=client, ) - assert len(requests) == 1 - assert str(requests[0].url) == f"{ZEROGPU_API_BASE}/chat/completions" - assert json.loads(requests[0].content) == { + assert len(zerogpu.requests) == 1 + assert str(zerogpu.requests[0].url) == f"{ZEROGPU_API_BASE}/chat/completions" + assert json.loads(zerogpu.requests[0].content) == { "model": "gpt-oss-120b", "messages": [{"role": "user", "content": "hello"}], } def test_responses_request_is_bridged_to_zerogpu_chat_completions(): - requests: Final[list[httpx.Request]] = [] - client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) + zerogpu: Final = _RecordingZeroGPU({"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) - with client: + with zerogpu.client() as client: response: Final = litellm.responses(model="zerogpu/gpt-oss-120b", input="hello", client=client) - assert [str(request.url) for request in requests] == [f"{ZEROGPU_API_BASE}/chat/completions"] - assert json.loads(requests[0].content) == { + assert [str(request.url) for request in zerogpu.requests] == [f"{ZEROGPU_API_BASE}/chat/completions"] + assert json.loads(zerogpu.requests[0].content) == { "model": "gpt-oss-120b", "messages": [{"role": "user", "content": "hello"}], } @@ -89,10 +93,9 @@ def test_responses_request_is_bridged_to_zerogpu_chat_completions(): def test_anthropic_messages_request_is_bridged_to_zerogpu_chat_completions(): - requests: Final[list[httpx.Request]] = [] - client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) + zerogpu: Final = _RecordingZeroGPU({"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) - with client: + with zerogpu.client() as client: response: Final = litellm.anthropic.messages.create( model="zerogpu/gpt-oss-120b", messages=[{"role": "user", "content": "hello"}], @@ -100,8 +103,8 @@ def test_anthropic_messages_request_is_bridged_to_zerogpu_chat_completions(): client=client, ) - assert [str(request.url) for request in requests] == [f"{ZEROGPU_API_BASE}/chat/completions"] - assert json.loads(requests[0].content) == { + assert [str(request.url) for request in zerogpu.requests] == [f"{ZEROGPU_API_BASE}/chat/completions"] + assert json.loads(zerogpu.requests[0].content) == { "model": "gpt-oss-120b", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10, @@ -111,16 +114,15 @@ def test_anthropic_messages_request_is_bridged_to_zerogpu_chat_completions(): @pytest.mark.parametrize("model", ZEROGPU_MODELS) def test_zerogpu_completion_is_charged_at_the_cost_map_rates(model: str): - requests: Final[list[httpx.Request]] = [] usage: Final = { "prompt_tokens": 1000, "completion_tokens": 200, "total_tokens": 1200, "prompt_tokens_details": {"cached_tokens": 400}, } - client: Final = _zerogpu_client(requests, usage) + zerogpu: Final = _RecordingZeroGPU(usage) - with client: + with zerogpu.client() as client: response: Final = litellm.completion( model=f"zerogpu/{model}", messages=[{"role": "user", "content": "hello"}], @@ -155,10 +157,9 @@ def test_trim_messages_fits_zerogpu_history_into_the_model_context_window(model: def test_tools_and_reasoning_effort_reach_zerogpu_only_for_models_that_support_them( model: str, forwards_tools: bool, forwards_reasoning_effort: bool ): - requests: Final[list[httpx.Request]] = [] - client: Final = _zerogpu_client(requests, {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) + zerogpu: Final = _RecordingZeroGPU({"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) - with client: + with zerogpu.client() as client: litellm.completion( model=f"zerogpu/{model}", messages=[{"role": "user", "content": "weather in Paris?"}], @@ -171,7 +172,7 @@ def test_tools_and_reasoning_effort_reach_zerogpu_only_for_models_that_support_t tools: Final = {"tools": [WEATHER_TOOL], "tool_choice": "auto"} if forwards_tools else {} reasoning: Final = {"reasoning_effort": "high"} if forwards_reasoning_effort else {} - assert json.loads(requests[0].content) == { + assert json.loads(zerogpu.requests[0].content) == { "model": model, "messages": [{"role": "user", "content": "weather in Paris?"}], **tools, From 1308baa019080c3e4211e2c5142ac8a76722839d Mon Sep 17 00:00:00 2001 From: Amaan Date: Tue, 20 Oct 2026 19:28:31 +0530 Subject: [PATCH 11/18] fix(zerogpu): cap max_tokens at each model's output limit Each cap is the max_output_tokens ZeroGPU stores for the model in its production models table (read 2026-10-20), the same column its /v1/models serves as max_length. That gives 65536 for deepseek-v4.1-flash, 32768 for gpt-4.1-mini and gpt-oss-120b, 128000 for gpt-5.4-nano and gpt-5.6-luna, 8192 for LFM2.5-1.2B-Instruct and llama-3.1-8b-instruct-fast, and 16384 for qwen3-30b-a3b-fp8. glm-5.3-flash is stored as 32786, which reads as a typo for 32768, so that row uses 32768 trim_messages sizes history from max_input_tokens, so lowering max_tokens leaves it unaffected --- ...odel_prices_and_context_window_backup.json | 27 ++++++++++++------- model_prices_and_context_window.json | 27 ++++++++++++------- .../test_model_prices_and_context_window.py | 21 +++++++++++++++ 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 931d9fefa62..53bf67eb1ce 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -67100,7 +67100,8 @@ "input_cost_per_token": 3e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67114,7 +67115,8 @@ "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67127,7 +67129,8 @@ "input_cost_per_token": 4e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1047576, - "max_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67141,7 +67144,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 400000, - "max_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67155,7 +67159,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 272000, - "max_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67170,7 +67175,8 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67182,7 +67188,8 @@ "input_cost_per_token": 2e-08, "litellm_provider": "zerogpu", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-08, "source": "https://docs.zerogpu.ai/docs/model-catalog" @@ -67192,7 +67199,8 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67205,7 +67213,8 @@ "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 931d9fefa62..53bf67eb1ce 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -67100,7 +67100,8 @@ "input_cost_per_token": 3e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67114,7 +67115,8 @@ "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67127,7 +67129,8 @@ "input_cost_per_token": 4e-07, "litellm_provider": "zerogpu", "max_input_tokens": 1047576, - "max_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67141,7 +67144,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 400000, - "max_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67155,7 +67159,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", "max_input_tokens": 272000, - "max_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67170,7 +67175,8 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67182,7 +67188,8 @@ "input_cost_per_token": 2e-08, "litellm_provider": "zerogpu", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-08, "source": "https://docs.zerogpu.ai/docs/model-catalog" @@ -67192,7 +67199,8 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", @@ -67205,7 +67213,8 @@ "input_cost_per_token": 1e-07, "litellm_provider": "zerogpu", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 395fe8c9750..8835689df0b 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -94,6 +94,27 @@ def test_github_copilot_backup_rows_match_the_main_catalog() -> None: assert _github_copilot_catalog_rows(PRICES_PATH) == _github_copilot_catalog_rows(BACKUP_PRICES_PATH) +class _TokenLimits(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + max_input_tokens: int + max_output_tokens: int + max_tokens: int + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_zerogpu_rows_cap_max_tokens_at_an_output_limit_that_fits_the_context_window(path: Path) -> None: + prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) + rows: Final = MappingProxyType( + {key: _TokenLimits.model_validate(value) for key, value in prices.items() if key.startswith("zerogpu/")} + ) + assert rows, f"No ZeroGPU rows found in {path}" + assert {key: row.max_tokens for key, row in rows.items()} == { + key: row.max_output_tokens for key, row in rows.items() + } + assert tuple(key for key, row in rows.items() if row.max_output_tokens > row.max_input_tokens) == () + + def test_github_copilot_rows_resolve_through_get_model_info() -> None: rows: Final[Mapping[str, _CopilotEndpointRow]] = _copilot_endpoint_rows(PRICES_PATH) assert { From 0080e0ce51addda91aeda673e34b0341bbf40d66 Mon Sep 17 00:00:00 2001 From: Amaan Date: Tue, 20 Oct 2026 19:35:54 +0530 Subject: [PATCH 12/18] fix(zerogpu): flag prompt caching on glm-5.3-flash Repeating the same ~4.3K-token prompt against ZeroGPU on 2026-10-20 returned cached_tokens of 4320 on some glm-5.3-flash calls. gpt-oss-120b and llama-3.1-8b-instruct-fast returned 0 on all nine calls, and qwen3-30b-a3b-fp8 never returned cached_tokens, so they stay unflagged. Their cached prices stay as ZeroGPU publishes them, so any cache hit is still billed at the right rate --- litellm/model_prices_and_context_window_backup.json | 1 + model_prices_and_context_window.json | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 53bf67eb1ce..62235df4cb9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -67121,6 +67121,7 @@ "output_cost_per_token": 3.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 53bf67eb1ce..62235df4cb9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -67121,6 +67121,7 @@ "output_cost_per_token": 3.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, From 1648ac0ea81ff42ab670f431f0c363dd9cafc476 Mon Sep 17 00:00:00 2001 From: Amaan Date: Wed, 21 Oct 2026 17:04:31 +0530 Subject: [PATCH 13/18] fix(zerogpu): set gpt-5.4-nano context window to 272000 ZeroGPU accepts a 271001-token gpt-5.4-nano prompt and rejects 273000 tokens as over the context window (checked 2026-10-21), so max_input_tokens is 272000 rather than the 400000 its catalog lists --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 62235df4cb9..d07e8b3fa5f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -67144,7 +67144,7 @@ "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 62235df4cb9..d07e8b3fa5f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -67144,7 +67144,7 @@ "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "zerogpu", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", From d359e1086efca759193ac3a2e501b287bca96c4e Mon Sep 17 00:00:00 2001 From: Amaan Date: Wed, 21 Oct 2026 17:07:26 +0530 Subject: [PATCH 14/18] fix(zerogpu): list zerogpu models for zerogpu/* wildcards zerogpu had no models_by_provider entry, so get_provider_models("zerogpu") returned None and a zerogpu/* wildcard listed no models. zerogpu_models is now filled from the cost map rows like openinfer_models and registered in models_by_provider --- litellm/__init__.py | 5 +++++ tests/unit/llms/openai_like/test_zerogpu_provider.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 0fe3d091c2a..610e7dadf95 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -679,6 +679,7 @@ moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() openinfer_models: Final[Set[str]] = set() +zerogpu_models: Final[Set[str]] = set() v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() @@ -965,6 +966,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje darkbloom_models.add(key) elif value.get("litellm_provider") == "openinfer": openinfer_models.add(key) + elif value.get("litellm_provider") == "zerogpu": + zerogpu_models.add(key) elif value.get("litellm_provider") == "v0": v0_models.add(key) elif value.get("litellm_provider") == "morph": @@ -1134,6 +1137,7 @@ model_list = list( | publicai_models | darkbloom_models | openinfer_models + | zerogpu_models | v0_models | morph_models | lambda_ai_models @@ -1251,6 +1255,7 @@ def _build_models_by_provider() -> dict: "publicai": publicai_models, "darkbloom": darkbloom_models, "openinfer": openinfer_models, + "zerogpu": zerogpu_models, "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, diff --git a/tests/unit/llms/openai_like/test_zerogpu_provider.py b/tests/unit/llms/openai_like/test_zerogpu_provider.py index 614418d486d..7164ae1b454 100644 --- a/tests/unit/llms/openai_like/test_zerogpu_provider.py +++ b/tests/unit/llms/openai_like/test_zerogpu_provider.py @@ -8,6 +8,7 @@ from openai import OpenAI import litellm from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.proxy.auth.model_checks import get_provider_models ZEROGPU_API_BASE: Final = "https://api.zerogpu.ai/v1" @@ -205,3 +206,12 @@ def test_zerogpu_api_base_alone_selects_the_zerogpu_provider(monkeypatch: pytest "zerogpu-env-key", ZEROGPU_API_BASE, ) + + +def test_zerogpu_wildcard_lists_every_zerogpu_catalog_model(): + catalog_models: Final = sorted( + key for key, row in litellm.model_cost.items() if row.get("litellm_provider") == "zerogpu" + ) + + assert catalog_models + assert sorted(get_provider_models("zerogpu") or []) == catalog_models From 10392d2f38f4c223afeef79ccb1118926343ed15 Mon Sep 17 00:00:00 2001 From: Amaan Date: Wed, 21 Oct 2026 17:10:36 +0530 Subject: [PATCH 15/18] fix(zerogpu): flag gpt-5.4-nano as a reasoning model ZeroGPU honors reasoning_effort on gpt-5.4-nano (checked 2026-10-21). Without it the model spends 0 reasoning tokens, and with low or high it spends 32 or more. LiteLLM only treated it as a reasoning model by falling back to OpenAI's bare gpt-5.4-nano row, so get_model_info and /model/info reported supports_reasoning as null. The row now carries the flag itself --- litellm/model_prices_and_context_window_backup.json | 1 + model_prices_and_context_window.json | 1 + tests/unit/llms/openai_like/test_zerogpu_provider.py | 7 +++++++ 3 files changed, 9 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d07e8b3fa5f..c086a137ab9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -67152,6 +67152,7 @@ "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d07e8b3fa5f..c086a137ab9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -67152,6 +67152,7 @@ "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/tests/unit/llms/openai_like/test_zerogpu_provider.py b/tests/unit/llms/openai_like/test_zerogpu_provider.py index 7164ae1b454..6d566a305c7 100644 --- a/tests/unit/llms/openai_like/test_zerogpu_provider.py +++ b/tests/unit/llms/openai_like/test_zerogpu_provider.py @@ -181,6 +181,13 @@ def test_tools_and_reasoning_effort_reach_zerogpu_only_for_models_that_support_t } +@pytest.mark.parametrize("model", ZEROGPU_MODELS) +def test_model_info_reports_the_reasoning_support_litellm_routes_with(model: str): + model_info: Final = litellm.get_model_info(f"zerogpu/{model}") + + assert bool(model_info.get("supports_reasoning")) == litellm.utils.supports_reasoning(f"zerogpu/{model}") + + def test_zerogpu_key_and_base_come_from_zerogpu_env_vars(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("ZEROGPU_API_KEY", "zerogpu-env-key") monkeypatch.delenv("ZEROGPU_API_BASE", raising=False) From b12f3bb0ad901c0700d653c57f5f9018fa750b01 Mon Sep 17 00:00:00 2001 From: Amaan Date: Wed, 21 Oct 2026 17:20:02 +0530 Subject: [PATCH 16/18] fix(zerogpu): flag deepseek, glm, gpt-oss and qwen3 as reasoning models ZeroGPU's /v1/models marks these four as reasoning models and accepts reasoning_effort on all of them (checked 2026-10-21), but LiteLLM rejected the param with UnsupportedParamsError because their rows had no supports_reasoning flag. glm reports reasoning_tokens under reasoning_effort, and gpt-oss and qwen3 spend hidden reasoning tokens on every reply. The gpt-oss test case now expects reasoning_effort to reach ZeroGPU --- litellm/model_prices_and_context_window_backup.json | 4 ++++ model_prices_and_context_window.json | 4 ++++ tests/unit/llms/openai_like/test_zerogpu_provider.py | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c086a137ab9..8d2c984efb1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -67107,6 +67107,7 @@ "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -67122,6 +67123,7 @@ "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -67183,6 +67185,7 @@ "output_cost_per_token": 6e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -67221,6 +67224,7 @@ "output_cost_per_token": 4.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c086a137ab9..8d2c984efb1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -67107,6 +67107,7 @@ "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -67122,6 +67123,7 @@ "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -67183,6 +67185,7 @@ "output_cost_per_token": 6e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -67221,6 +67224,7 @@ "output_cost_per_token": 4.5e-07, "source": "https://docs.zerogpu.ai/docs/model-catalog", "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/tests/unit/llms/openai_like/test_zerogpu_provider.py b/tests/unit/llms/openai_like/test_zerogpu_provider.py index 6d566a305c7..25a5dbbcd9a 100644 --- a/tests/unit/llms/openai_like/test_zerogpu_provider.py +++ b/tests/unit/llms/openai_like/test_zerogpu_provider.py @@ -153,7 +153,7 @@ def test_trim_messages_fits_zerogpu_history_into_the_model_context_window(model: @pytest.mark.parametrize( ("model", "forwards_tools", "forwards_reasoning_effort"), - [("gpt-5.6-luna", True, True), ("gpt-oss-120b", True, False), ("LFM2.5-1.2B-Instruct", False, False)], + [("gpt-5.6-luna", True, True), ("gpt-oss-120b", True, True), ("LFM2.5-1.2B-Instruct", False, False)], ) def test_tools_and_reasoning_effort_reach_zerogpu_only_for_models_that_support_them( model: str, forwards_tools: bool, forwards_reasoning_effort: bool From 6605e31c4499c3f43b01c5bb34f1d8d7a614a8e2 Mon Sep 17 00:00:00 2001 From: Amaan Date: Wed, 21 Oct 2026 17:23:09 +0530 Subject: [PATCH 17/18] fix(zerogpu): set gpt-oss-120b context window to 128000 ZeroGPU routes gpt-oss-120b across several upstream backends, and at least one of them returns 413 past a 128000-token window, so max_input_tokens uses that as the safe floor. The backends reached on 2026-10-21 accepted up to about 131K prompt tokens, so this only gives up a little headroom on the larger ones --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8d2c984efb1..801ccf27514 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -67178,7 +67178,7 @@ "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", - "max_input_tokens": 131072, + "max_input_tokens": 128000, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8d2c984efb1..801ccf27514 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -67178,7 +67178,7 @@ "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "zerogpu", - "max_input_tokens": 131072, + "max_input_tokens": 128000, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", From c3e965c674715f7c23b9d70b426cd43cab6f39a2 Mon Sep 17 00:00:00 2001 From: Amaan Date: Thu, 22 Oct 2026 00:18:04 +0530 Subject: [PATCH 18/18] feat(zerogpu): call ZeroGPU's Responses API natively ZeroGPU serves /v1/responses for all nine catalog models, so LiteLLM no longer bridges Responses requests through chat completions. Native calls return ZeroGPU's reasoning output items for glm, gpt-oss and qwen3, which the bridge dropped. Checked live on 2026-10-22 through litellm.responses for every model, non-streaming and streaming, plus a gpt-oss tool call. LFM2.5-1.2B-Instruct streams correctly now that ZeroGPU's text_generation lane emits Responses SSE --- litellm/llms/openai_like/providers.json | 2 +- .../llms/openai_like/test_zerogpu_provider.py | 37 ++++++++++++++----- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 441d094bc05..be6c7aa7ed4 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -62,7 +62,7 @@ "base_url": "https://api.zerogpu.ai/v1", "api_key_env": "ZEROGPU_API_KEY", "api_base_env": "ZEROGPU_API_BASE", - "supported_endpoints": ["/v1/chat/completions"] + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] }, "chutes": { "base_url": "https://llm.chutes.ai/v1/", diff --git a/tests/unit/llms/openai_like/test_zerogpu_provider.py b/tests/unit/llms/openai_like/test_zerogpu_provider.py index 25a5dbbcd9a..991796ae268 100644 --- a/tests/unit/llms/openai_like/test_zerogpu_provider.py +++ b/tests/unit/llms/openai_like/test_zerogpu_provider.py @@ -4,6 +4,7 @@ from typing import Final import httpx import pytest +import respx from openai import OpenAI import litellm @@ -79,17 +80,33 @@ def test_prefixed_model_reaches_zerogpu_chat_completions_without_the_prefix(): } -def test_responses_request_is_bridged_to_zerogpu_chat_completions(): - zerogpu: Final = _RecordingZeroGPU({"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}) +def test_responses_request_reaches_zerogpu_responses_natively(): + with respx.mock() as upstream: + route: Final = upstream.post(f"{ZEROGPU_API_BASE}/responses").respond( + 200, + json={ + "id": "resp_zerogpu", + "object": "response", + "created_at": 1_792_000_000, + "model": "gpt-oss-120b", + "status": "completed", + "output": [ + { + "id": "msg_zerogpu", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4}, + }, + ) + response: Final = litellm.responses(model="zerogpu/gpt-oss-120b", input="hello", api_key="zerogpu-test-key") - with zerogpu.client() as client: - response: Final = litellm.responses(model="zerogpu/gpt-oss-120b", input="hello", client=client) - - assert [str(request.url) for request in zerogpu.requests] == [f"{ZEROGPU_API_BASE}/chat/completions"] - assert json.loads(zerogpu.requests[0].content) == { - "model": "gpt-oss-120b", - "messages": [{"role": "user", "content": "hello"}], - } + body: Final = json.loads(route.calls.last.request.content) + assert body["model"] == "gpt-oss-120b" + assert body["input"] == "hello" assert response.output[0].content[0].text == "hi"