scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-23 05:34:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 24

feat(openinfer): add OpenInfer as an OpenAI-compatible provider (#41034)

Browse source 
* feat(openinfer): add OpenInfer as OpenAI-compatible provider

Register OpenInfer in the JSON provider registry with catalog models,
proxy create fields, and unit coverage so requests route via openinfer/

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(openinfer): assert catalog invariants instead of vendor rates

Pinning per-token prices made CI fail when the vendor list changed. Keep
the backup agreement and positivity checks our code owns.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(openinfer): add openinfer to the Rust OpenAI-compatible provider mirror

* fix(openinfer): align catalog with OpenInfer's live /v1/models

Replace the four catalog rows with the two models OpenInfer serves and
publishes a price for (@oi/GLM-5.3-Flash, @oi/Qwen3.5-27B), using its
listed per-token rates and context windows, and price-check them in tests

* fix(openinfer): use OpenInfer's native Responses and Messages APIs, allow tools on unpriced models, price Qwen3.5-9B

Declare /v1/responses and /v1/messages for openinfer, since OpenInfer serves both natively. Let JSON providers declare provider-wide capabilities in providers.json, read through get_provider_info, so tool calls reach OpenInfer for models without a cost map row. Send max_completion_tokens as is, set output caps to each model's context window, mark the three rows as reasoning models, add Qwen3.5-9B at its listed rate, and wire openinfer into models_by_provider so the openinfer/* wildcard lists its models.

* fix(openinfer): let a model's own capability row win over provider defaults in get_model_info

Provider defaults now only fill capability fields the cost-map row leaves unset, so a row or deployment that turns function calling off stays off. Also marks openinfer_models as Final.

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Aayush Jain 2026-10-18 10:22:19 +11:00 • committed by GitHub
parent 8b442f00fc
commit e2295b596f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 420 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs
View file

@ -94,6 +94,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
"docker_model_runner",
"ragflow",
"pinstripes",
"openinfer",
"darkbloom",
"meta",
"cognition",

5
litellm/__init__.py
View file

@ -676,6 +676,7 @@ qwen_ai_platform_models: Set = set()
moonshot_models: Set = set()
publicai_models: Set = set()
darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
lambda_ai_models: Set = set()
@ -945,6 +946,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
publicai_models.add(key)
elif value.get("litellm_provider") == "darkbloom":
darkbloom_models.add(key)
elif value.get("litellm_provider") == "openinfer":
openinfer_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
@ -1113,6 +1116,7 @@ model_list = list(
| moonshot_models
| publicai_models
| darkbloom_models
| openinfer_models
| v0_models
| morph_models
| lambda_ai_models
@ -1229,6 +1233,7 @@ def _build_models_by_provider() -> dict:
"moonshot": moonshot_models,
"publicai": publicai_models,
"darkbloom": darkbloom_models,
"openinfer": openinfer_models,
"v0": v0_models,
"morph": morph_models,
"lambda_ai": lambda_ai_models,

2
litellm/constants.py
View file

@ -1030,6 +1030,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"https://api.prisminference.com/v1",
"https://api.reka.ai/v1",
"https://gigachat.devices.sberbank.ru/api/v1",
"https://api.openinfer.ai/v1",
]
@ -1098,6 +1099,7 @@ openai_compatible_providers: Final[list] = [
"docker_model_runner",
"ragflow",
"pinstripes", # Pinstripes - JSON-configured provider
"openinfer",
"darkbloom",
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
"cognition",

6
litellm/llms/openai_like/README.md
View file

@ -63,6 +63,12 @@ That's it! The provider will be automatically loaded and available.
// Optional: Special handling flags
"special_handling": {
"convert_content_list_to_string": true
},
// Optional: Capabilities every model of this provider has, used when a
// model has no cost map row or its row leaves the field unset
"provider_info": {
"supports_function_calling": true
}
}
}

8
litellm/llms/openai_like/json_loader.py
View file

@ -6,7 +6,12 @@ import json
from pathlib import Path
from typing import Final
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.types.utils import ProviderSpecificModelInfo
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
class SimpleProviderConfig:
@ -22,6 +27,9 @@ class SimpleProviderConfig:
self.constraints = data.get("constraints", {})
self.special_handling = data.get("special_handling", {})
self.supported_endpoints = data.get("supported_endpoints", [])
self.provider_info: ProviderSpecificModelInfo = (
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
)
class JSONProviderRegistry:

10
litellm/llms/openai_like/providers.json
View file

@ -230,6 +230,16 @@
"api_key_env": "REKA_API_KEY",
"api_base_env": "REKA_API_BASE"
},
"openinfer": {
"base_url": "https://api.openinfer.ai/v1",
"api_key_env": "OPENINFER_API_KEY",
"api_base_env": "OPENINFER_API_BASE",
"base_class": "openai_gpt",
"provider_info": {
"supports_function_calling": true
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
},
"inferenceapis": {
"base_url": "https://api.inferenceapis.com/v1",
"api_key_env": "INFERENCEAPIS_API_KEY",

36
litellm/model_prices_and_context_window_backup.json
View file

@ -75526,6 +75526,42 @@
"supports_reasoning": true,
"supports_vision": true
},
"openinfer/@oi/GLM-5.3-Flash": {
"max_tokens": 393216,
"max_input_tokens": 393216,
"max_output_tokens": 393216,
"input_cost_per_token": 1.95e-07,
"output_cost_per_token": 1.56e-06,
"litellm_provider": "openinfer",
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://api.openinfer.ai/v1/models"
},
"openinfer/@oi/Qwen3.5-27B": {
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 32768,
"input_cost_per_token": 1.95e-07,
"output_cost_per_token": 1.56e-06,
"litellm_provider": "openinfer",
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://api.openinfer.ai/v1/models"
},
"openinfer/@oi/Qwen3.5-9B": {
"max_tokens": 100000,
"max_input_tokens": 100000,
"max_output_tokens": 100000,
"input_cost_per_token": 5e-06,
"output_cost_per_token": 1.5e-07,
"litellm_provider": "openinfer",
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://api.openinfer.ai/v1/models"
},
"xai/grok-voice-3.0": {
"input_cost_per_second": 0.001333333333333,
"input_cost_per_text_message": 0.004,

17
litellm/provider_endpoints_support_backup.json
View file

@ -2731,6 +2731,23 @@
"messages": true,
"responses": true
}
},
"openinfer": {
"display_name": "OpenInfer (`openinfer`)",
"url": "https://docs.litellm.ai/docs/providers/openinfer",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": false
}
}
},
"endpoints": {

28
litellm/proxy/public_endpoints/provider_create_fields.json
View file

@ -2735,6 +2735,34 @@
],
"default_model_placeholder": "gpt-3.5-turbo"
},
{
"provider": "OPENINFER",
"provider_display_name": "OpenInfer",
"litellm_provider": "openinfer",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://api.openinfer.ai/v1",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "openinfer/@oi/Qwen3.5-27B"
},
{
"provider": "OPENAI_LIKE",
"provider_display_name": "Openai Like",

1
litellm/types/utils.py
View file

@ -4389,6 +4389,7 @@ class LlmProviders(str, Enum):
CORTECS = "cortecs"
SCX_AI = "scx-ai"
PRISM = "prism"
OPENINFER = "openinfer"
INFERENCEAPIS = "inferenceapis"
DARKBLOOM = "darkbloom"
META = "meta"

10
litellm/utils.py
View file

@ -6265,11 +6265,13 @@ def get_provider_info(model: str, custom_llm_provider: str | None) -> ProviderSp
model=model, provider=LlmProviders(custom_llm_provider)
)
model_info: ProviderSpecificModelInfo | None = None
if provider_config:
model_info = provider_config.get_provider_info(model=model)
return provider_config.get_provider_info(model=model)
return model_info
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
json_provider: Final = JSONProviderRegistry.get(custom_llm_provider) if custom_llm_provider else None
return json_provider.provider_info if json_provider is not None and json_provider.provider_info else None
def _is_potential_model_name_in_model_cost(
@ -6726,7 +6728,7 @@ def _build_model_info(
provider_info: Final = get_provider_info(model=model, custom_llm_provider=custom_llm_provider)
if provider_info:
for key, value in provider_info.items():
if value is not None:
if value is not None and _model_info.get(key) is None:
_model_info[key] = value
# if verbose_logger.isEnabledFor(logging.DEBUG):

36
model_prices_and_context_window.json
View file

@ -75526,6 +75526,42 @@
"supports_reasoning": true,
"supports_vision": true
},
"openinfer/@oi/GLM-5.3-Flash": {
"max_tokens": 393216,
"max_input_tokens": 393216,
"max_output_tokens": 393216,
"input_cost_per_token": 1.95e-07,
"output_cost_per_token": 1.56e-06,
"litellm_provider": "openinfer",
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://api.openinfer.ai/v1/models"
},
"openinfer/@oi/Qwen3.5-27B": {
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 32768,
"input_cost_per_token": 1.95e-07,
"output_cost_per_token": 1.56e-06,
"litellm_provider": "openinfer",
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://api.openinfer.ai/v1/models"
},
"openinfer/@oi/Qwen3.5-9B": {
"max_tokens": 100000,
"max_input_tokens": 100000,
"max_output_tokens": 100000,
"input_cost_per_token": 5e-06,
"output_cost_per_token": 1.5e-07,
"litellm_provider": "openinfer",
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://api.openinfer.ai/v1/models"
},
"xai/grok-voice-3.0": {
"input_cost_per_second": 0.001333333333333,
"input_cost_per_text_message": 0.004,

17
provider_endpoints_support.json
View file

@ -3254,6 +3254,23 @@
"rerank": false,
"a2a": false
}
},
"openinfer": {
"display_name": "OpenInfer (`openinfer`)",
"url": "https://docs.litellm.ai/docs/providers/openinfer",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": false
}
}
},
"endpoints": {

247
tests/unit/llms/openai_like/test_openinfer_provider.py Normal file
View file

@ -0,0 +1,247 @@
import json
from pathlib import Path
from typing import Final
import pytest
import respx
import litellm
from litellm.caching.llm_caching_handler import LLMClientCache
from litellm.proxy.auth.model_checks import get_provider_models
_REPO_ROOT: Final = Path(__file__).resolve().parents[4]
_PRIMARY_PRICES: Final = json.loads((_REPO_ROOT / "model_prices_and_context_window.json").read_text())
_BACKUP_PRICES: Final = json.loads(
(_REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text()
)
_CATALOG_MODELS: Final = tuple(
key for key, row in _PRIMARY_PRICES.items() if isinstance(row, dict) and row.get("litellm_provider") == "openinfer"
)
_REASONING_MODELS: Final = tuple(key for key in _CATALOG_MODELS if _PRIMARY_PRICES[key].get("supports_reasoning"))
_CHAT_URL: Final = "https://api.openinfer.ai/v1/chat/completions"
_WEATHER_TOOL: Final = {
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
},
}
_CHAT_RESPONSE: Final = {
"id": "chatcmpl_openinfer",
"object": "chat.completion",
"created": 1_792_000_000,
"model": "@oi/Qwen3.5-27B",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello from OpenInfer"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
}
def _sent_chat_request(model: str, **params: object) -> tuple[str, str, dict[str, object]]:
with respx.mock() as upstream:
route: Final = upstream.post(_CHAT_URL).respond(200, json=_CHAT_RESPONSE)
litellm.completion(
model=f"openinfer/{model}",
messages=[{"role": "user", "content": "Say hello"}],
api_key="sk-test",
**params,
)
request: Final = route.calls.last.request
return str(request.url), request.headers["authorization"], json.loads(request.content)
def test_provider_prefixed_model_routes_to_openinfer():
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
model, provider, api_key, api_base = get_llm_provider(
model="openinfer/@oi/Qwen3.5-27B",
custom_llm_provider=None,
api_base=None,
api_key="sk-test",
)
assert model == "@oi/Qwen3.5-27B"
assert provider == "openinfer"
assert api_key == "sk-test"
assert api_base == "https://api.openinfer.ai/v1"
def test_api_key_and_base_resolved_from_env(monkeypatch: pytest.MonkeyPatch):
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
monkeypatch.setenv("OPENINFER_API_KEY", "sk-env-key")
monkeypatch.setenv("OPENINFER_API_BASE", "https://proxy.internal/v1")
_, provider, api_key, api_base = get_llm_provider(
model="openinfer/@oi/Qwen3.5-9B",
custom_llm_provider=None,
api_base=None,
api_key=None,
)
assert provider == "openinfer"
assert api_key == "sk-env-key"
assert api_base == "https://proxy.internal/v1"
def test_url_autodetection_from_api_base(monkeypatch: pytest.MonkeyPatch):
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
monkeypatch.setenv("OPENINFER_API_KEY", "sk-env-key")
_, provider, api_key, _ = get_llm_provider(
model="@oi/Qwen3.5-27B",
custom_llm_provider=None,
api_base="https://api.openinfer.ai/v1",
api_key=None,
)
assert provider == "openinfer"
assert api_key == "sk-env-key"
def test_chat_request_reaches_openinfer_with_the_bare_model_id():
url, authorization, body = _sent_chat_request("@oi/Qwen3.5-27B")
assert url == _CHAT_URL
assert authorization == "Bearer sk-test"
assert body["model"] == "@oi/Qwen3.5-27B"
def test_tools_reach_openinfer_for_a_model_without_a_cost_map_row():
_, _, body = _sent_chat_request("@oi/model-without-a-catalog-row", tools=[_WEATHER_TOOL])
assert body["tools"] == [_WEATHER_TOOL]
def _register_openinfer_row(model: str, **capabilities: bool) -> None:
litellm.register_model(
{f"openinfer/{model}": {"litellm_provider": "openinfer", "mode": "chat", **capabilities}},
persist_across_reloads=False,
add_to_provider_model_sets=False,
)
@pytest.mark.usefixtures("local_model_cost_map")
def test_model_info_keeps_a_row_that_turns_tools_off():
model: Final = "@oi/row-without-tools"
_register_openinfer_row(model, supports_function_calling=False)
assert litellm.get_model_info(model, custom_llm_provider="openinfer")["supports_function_calling"] is False
assert litellm.supports_function_calling(model, custom_llm_provider="openinfer") is False
@pytest.mark.usefixtures("local_model_cost_map")
def test_model_info_fills_tool_support_from_the_openinfer_default():
model: Final = "@oi/row-silent-on-tools"
_register_openinfer_row(model)
assert litellm.get_model_info(model, custom_llm_provider="openinfer")["supports_function_calling"] is True
assert litellm.supports_function_calling(model, custom_llm_provider="openinfer") is True
def test_max_tokens_and_max_completion_tokens_both_reach_openinfer():
_, _, body = _sent_chat_request("@oi/Qwen3.5-27B", max_tokens=64, max_completion_tokens=128)
assert body["max_tokens"] == 64
assert body["max_completion_tokens"] == 128
@pytest.mark.parametrize("model", _REASONING_MODELS)
def test_reasoning_effort_reaches_openinfer_for_reasoning_rows(model: str):
_, _, body = _sent_chat_request(model.removeprefix("openinfer/"), reasoning_effort="low")
assert body["reasoning_effort"] == "low"
def test_responses_request_goes_to_openinfer_natively():
with respx.mock() as upstream:
route: Final = upstream.post("https://api.openinfer.ai/v1/responses").respond(
200,
json={
"id": "resp_openinfer",
"object": "response",
"created_at": 1_792_000_000,
"model": "@oi/Qwen3.5-27B",
"status": "completed",
"output": [
{
"id": "msg_openinfer",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "Hello from OpenInfer", "annotations": []}],
}
],
"usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7},
},
)
response: Final = litellm.responses(model="openinfer/@oi/Qwen3.5-27B", input="Say hello", api_key="sk-test")
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert request.headers["authorization"] == "Bearer sk-test"
assert body["model"] == "@oi/Qwen3.5-27B"
assert body["input"] == "Say hello"
assert response.output[0].content[0].text == "Hello from OpenInfer"
@pytest.mark.asyncio
async def test_anthropic_messages_request_goes_to_openinfer_natively(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
with respx.mock() as upstream:
route: Final = upstream.post("https://api.openinfer.ai/v1/messages").respond(
200,
json={
"id": "msg_openinfer",
"type": "message",
"role": "assistant",
"model": "@oi/Qwen3.5-27B",
"content": [{"type": "text", "text": "Hello from OpenInfer"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 4, "output_tokens": 3},
},
)
response: Final = await litellm.anthropic.messages.acreate(
model="openinfer/@oi/Qwen3.5-27B",
messages=[{"role": "user", "content": "Say hello"}],
max_tokens=32,
api_key="sk-test",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert request.headers["authorization"] == "Bearer sk-test"
assert body["model"] == "@oi/Qwen3.5-27B"
assert body["messages"] == [{"role": "user", "content": "Say hello"}]
assert response["content"][0]["text"] == "Hello from OpenInfer"
def test_wildcard_lists_every_catalog_model():
assert sorted(get_provider_models("openinfer") or []) == sorted(_CATALOG_MODELS)
@pytest.mark.parametrize("model", _CATALOG_MODELS)
def test_catalog_row_matches_its_backup_and_fits_its_context(model: str):
row: Final = _PRIMARY_PRICES[model]
assert row == _BACKUP_PRICES[model]
assert row["mode"] == "chat"
assert row["max_tokens"] == row["max_output_tokens"]
assert 0 < row["max_output_tokens"] <= row["max_input_tokens"]
assert row["input_cost_per_token"] > 0
assert row["output_cost_per_token"] > 0
@pytest.mark.parametrize("model", _CATALOG_MODELS)
def test_chat_response_is_priced_at_its_catalog_row(model: str):
row: Final = litellm.model_cost[model]
response: Final = litellm.ModelResponse(
model=model.removeprefix("openinfer/"),
usage=litellm.Usage(prompt_tokens=1000, completion_tokens=200, total_tokens=1200),
)
cost: Final = litellm.completion_cost(completion_response=response, model=model)
assert cost == pytest.approx(1000 * row["input_cost_per_token"] + 200 * row["output_cost_per_token"])