From e6b8aabbfb0227a9b2ad099d6ddc42d261ff2930 Mon Sep 17 00:00:00 2001 From: stimQQ Date: Thu, 8 Oct 2026 01:02:25 +0900 Subject: [PATCH] fix(apimodels): document supported endpoints and cover request routing in tests - provider_endpoints_support.json + litellm/provider_endpoints_support_backup.json: add the apimodels entry (chat_completions, messages, responses; everything else false), same shape as reka/cortecs, so check_provider_folders_documented passes. - tests/unit/llms/openai_like/test_apimodels_provider.py: rewrite in the style of test_reka_provider.py. respx asserts that litellm.completion sends apimodels/ to https://api.apimodels.app/v1/chat/completions with the APIMODELS_API_KEY bearer token, that an explicit api_base is honoured, that litellm.responses hits the native /v1/responses, and that Anthropic /v1/messages calls are bridged to chat completions. Drops the structure-only config assertions. Co-Authored-By: Claude Opus 5.5 --- .../provider_endpoints_support_backup.json | 18 + provider_endpoints_support.json | 18 + .../openai_like/test_apimodels_provider.py | 310 ++++++++++++------ 3 files changed, 249 insertions(+), 97 deletions(-) diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 0e325bb61fe..75fb81b3bca 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -193,6 +193,24 @@ "a2a": false } }, + "apimodels": { + "display_name": "apimodels.app (`apimodels`)", + "url": "https://docs.litellm.ai/docs/providers/apimodels", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "assemblyai": { "display_name": "AssemblyAI (`assemblyai`)", "url": "https://docs.litellm.ai/docs/pass_through/assembly_ai", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 00904219b81..886dd3c4733 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -194,6 +194,24 @@ "a2a": false } }, + "apimodels": { + "display_name": "apimodels.app (`apimodels`)", + "url": "https://docs.litellm.ai/docs/providers/apimodels", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "aihubmix": { "display_name": "AIHubMix (`aihubmix`)", "url": "https://docs.litellm.ai/docs/providers/aihubmix", diff --git a/tests/unit/llms/openai_like/test_apimodels_provider.py b/tests/unit/llms/openai_like/test_apimodels_provider.py index fd2985ea70e..eb8b28690a6 100644 --- a/tests/unit/llms/openai_like/test_apimodels_provider.py +++ b/tests/unit/llms/openai_like/test_apimodels_provider.py @@ -1,128 +1,244 @@ -""" -Tests for the APIMODELS provider configuration. +import json +from pathlib import Path +from typing import Final -APIMODELS (https://apimodels.app) is an OpenAI-compatible gateway, so it is -registered through the JSON provider system rather than a bespoke Python module. -These tests are pure configuration assertions and mocks — no network calls. -""" +import pytest +import respx import litellm +from litellm.caching.llm_caching_handler import LLMClientCache -class TestAPIModelsProviderConfig: - def test_apimodels_in_provider_list(self): - from litellm import LlmProviders +def _chat_completion(model: str) -> dict: + return { + "id": "chatcmpl_apimodels", + "object": "chat.completion", + "created": 1_790_000_000, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from APIMODELS"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, + } - assert hasattr(LlmProviders, "APIMODELS") - assert LlmProviders.APIMODELS.value == "apimodels" - assert "apimodels" in litellm.provider_list - def test_apimodels_json_config_exists(self): - from litellm.llms.openai_like.json_loader import JSONProviderRegistry +def test_apimodels_is_a_registered_provider(): + assert litellm.LlmProviders.APIMODELS.value == "apimodels" + assert "apimodels" in litellm.provider_list + assert "apimodels" in litellm.constants.openai_compatible_providers - assert JSONProviderRegistry.exists("apimodels") - provider = JSONProviderRegistry.get("apimodels") - assert provider is not None - assert provider.base_url == "https://api.apimodels.app/v1" - assert provider.api_key_env == "APIMODELS_API_KEY" +def test_apimodels_provider_resolution(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - def test_apimodels_in_openai_compatible_providers(self): - from litellm.constants import openai_compatible_providers + monkeypatch.setenv("APIMODELS_API_KEY", "apimodels-test-key") + monkeypatch.delenv("APIMODELS_API_BASE", raising=False) - assert "apimodels" in openai_compatible_providers + model, provider, api_key, api_base = get_llm_provider( + model="apimodels/gpt-5.6-luna", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) - def test_apimodels_supports_responses_api(self): - from litellm.llms.openai_like.json_loader import JSONProviderRegistry + assert model == "gpt-5.6-luna" + assert provider == "apimodels" + assert api_key == "apimodels-test-key" + assert api_base == "https://api.apimodels.app/v1" - assert JSONProviderRegistry.supports_responses_api("apimodels") - def test_apimodels_provider_resolution(self): - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +def test_apimodels_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - model, provider, api_key, api_base = get_llm_provider( + monkeypatch.setenv("APIMODELS_API_KEY", "apimodels-env-key") + monkeypatch.setenv("APIMODELS_API_BASE", "https://apimodels.env.example/v1") + + _, provider, api_key, api_base = get_llm_provider( + model="apimodels/gpt-5.6-luna", + custom_llm_provider=None, + api_base="https://apimodels.internal.example/v1", + api_key="apimodels-explicit-key", + ) + + assert provider == "apimodels" + assert api_key == "apimodels-explicit-key" + assert api_base == "https://apimodels.internal.example/v1" + + +def test_apimodels_api_base_env_overrides_default(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("APIMODELS_API_KEY", "apimodels-env-key") + monkeypatch.setenv("APIMODELS_API_BASE", "https://apimodels.env.example/v1") + + _, provider, _, api_base = get_llm_provider( + model="apimodels/gpt-5.6-luna", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert provider == "apimodels" + assert api_base == "https://apimodels.env.example/v1" + + +def test_apimodels_api_base_autodetects_provider(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("APIMODELS_API_KEY", "apimodels-env-key") + + model, provider, api_key, api_base = get_llm_provider( + model="gpt-5.6-luna", + custom_llm_provider=None, + api_base="https://api.apimodels.app/v1", + api_key=None, + ) + + assert model == "gpt-5.6-luna" + assert provider == "apimodels" + assert api_key == "apimodels-env-key" + assert api_base == "https://api.apimodels.app/v1" + + +def test_apimodels_is_available_in_add_model_form(): + fields_path = Path(litellm.__file__).parent / "proxy" / "public_endpoints" / "provider_create_fields.json" + providers = json.loads(fields_path.read_text()) + apimodels = next(provider for provider in providers if provider["litellm_provider"] == "apimodels") + + assert apimodels["provider"] == "APIMODELS" + assert apimodels["provider_display_name"] == "APIMODELS" + assert apimodels["default_model_placeholder"] == "apimodels/gpt-5.6-luna" + assert {field["key"]: field["required"] for field in apimodels["credential_fields"]} == { + "api_base": False, + "api_key": True, + } + + +def test_apimodels_supported_endpoints(): + expected: Final = { + "chat_completions": True, + "messages": True, + "responses": True, + "embeddings": False, + "image_generations": False, + "audio_transcriptions": False, + "audio_speech": False, + "moderations": False, + "batches": False, + "rerank": False, + "a2a": False, + "interactions": False, + } + backup_path = Path(litellm.__file__).parent / "provider_endpoints_support_backup.json" + root_path = Path(litellm.__file__).parent.parent / "provider_endpoints_support.json" + + assert json.loads(backup_path.read_text())["providers"]["apimodels"]["endpoints"] == expected + assert json.loads(root_path.read_text())["providers"]["apimodels"]["endpoints"] == expected + + +def test_apimodels_chat_completion_request_uses_env_api_key(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("APIMODELS_API_KEY", "apimodels-env-key") + monkeypatch.delenv("APIMODELS_API_BASE", raising=False) + with respx.mock() as upstream: + route: Final = upstream.post("https://api.apimodels.app/v1/chat/completions").respond( + 200, json=_chat_completion("gpt-5.6-luna") + ) + response: Final = litellm.completion( model="apimodels/gpt-5.6-luna", - custom_llm_provider=None, - api_base=None, - api_key=None, + messages=[{"role": "user", "content": "Say hello"}], ) - assert model == "gpt-5.6-luna" - assert provider == "apimodels" - assert api_base == "https://api.apimodels.app/v1" + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert str(request.url) == "https://api.apimodels.app/v1/chat/completions" + assert request.headers["authorization"] == "Bearer apimodels-env-key" + assert body["model"] == "gpt-5.6-luna" + assert body["messages"] == [{"role": "user", "content": "Say hello"}] + assert response.choices[0].message.content == "Hello from APIMODELS" - def test_apimodels_api_base_override(self): - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - model, provider, api_key, api_base = get_llm_provider( +def test_apimodels_chat_completion_request_honours_api_base_override(): + with respx.mock() as upstream: + route: Final = upstream.post("https://apimodels.internal.example/v1/chat/completions").respond( + 200, json=_chat_completion("gpt-5.6-luna") + ) + response: Final = litellm.completion( model="apimodels/gpt-5.6-luna", - custom_llm_provider=None, - api_base="https://custom.apimodels.app/v1", - api_key="sk-test", + messages=[{"role": "user", "content": "Say hello"}], + api_base="https://apimodels.internal.example/v1", + api_key="apimodels-explicit-key", ) - assert provider == "apimodels" - assert api_base == "https://custom.apimodels.app/v1" - assert api_key == "sk-test" + request: Final = route.calls.last.request + assert route.call_count == 1 + assert str(request.url) == "https://apimodels.internal.example/v1/chat/completions" + assert request.headers["authorization"] == "Bearer apimodels-explicit-key" + assert response.choices[0].message.content == "Hello from APIMODELS" - def test_apimodels_url_autodetection(self): - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - model, provider, api_key, api_base = get_llm_provider( - model="gpt-5.6-luna", - custom_llm_provider=None, - api_base="https://api.apimodels.app/v1", - api_key=None, +def test_apimodels_responses_request(): + with respx.mock() as upstream: + route: Final = upstream.post("https://api.apimodels.app/v1/responses").respond( + 200, + json={ + "id": "resp_apimodels", + "object": "response", + "created_at": 1_790_000_000, + "model": "gpt-5.6-luna", + "status": "completed", + "output": [ + { + "id": "msg_apimodels", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Hello from APIMODELS", "annotations": []}], + } + ], + "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7}, + }, + ) + response: Final = litellm.responses( + model="apimodels/gpt-5.6-luna", + input="Say hello", + api_key="apimodels-test-key", ) - assert provider == "apimodels" - assert api_base == "https://api.apimodels.app/v1" + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert str(request.url) == "https://api.apimodels.app/v1/responses" + assert request.headers["authorization"] == "Bearer apimodels-test-key" + assert body["model"] == "gpt-5.6-luna" + assert body["input"] == "Say hello" + assert response.output[0].content[0].text == "Hello from APIMODELS" - def test_apimodels_router_config(self): - from litellm import Router - router = Router( - model_list=[ - { - "model_name": "apimodels-chat", - "litellm_params": { - "model": "apimodels/gpt-5.6-luna", - "api_key": "test-key", - }, - } - ] +@pytest.mark.asyncio +async def test_apimodels_anthropic_messages_request_is_bridged_to_chat_completions(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + with respx.mock() as upstream: + route: Final = upstream.post("https://api.apimodels.app/v1/chat/completions").respond( + 200, json=_chat_completion("gpt-5.6-luna") + ) + response: Final = await litellm.anthropic.messages.acreate( + model="apimodels/gpt-5.6-luna", + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=32, + api_key="apimodels-test-key", ) - assert len(router.model_list) == 1 - assert router.model_list[0]["model_name"] == "apimodels-chat" - - -class TestAPIModelsDashboardRegistration: - @staticmethod - def _provider_create_fields(): - import json - from pathlib import Path - - path = ( - Path(litellm.__file__).parent - / "proxy" - / "public_endpoints" - / "provider_create_fields.json" - ) - with open(path) as f: - return json.load(f) - - def test_apimodels_is_selectable_in_the_add_model_form(self): - entries = [ - e for e in self._provider_create_fields() if e["litellm_provider"] == "apimodels" - ] - assert len(entries) == 1, "apimodels must appear exactly once in provider_create_fields.json" - - entry = entries[0] - assert entry["provider"] == "APIMODELS" - assert entry["default_model_placeholder"].startswith("apimodels/") - - fields = {f["key"]: f for f in entry["credential_fields"]} - assert fields["api_key"]["required"] is True - assert fields["api_key"]["field_type"] == "password" - assert fields["api_base"]["required"] is False + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer apimodels-test-key" + assert body["model"] == "gpt-5.6-luna" + assert body["messages"] == [{"role": "user", "content": "Say hello"}] + assert body["max_tokens"] == 32 + assert response["content"][0]["text"] == "Hello from APIMODELS"