diff --git a/.github/scripts/auto_update_price_and_context_window_file.py b/.github/scripts/auto_update_price_and_context_window_file.py index 596f9a42760..df1ac738681 100644 --- a/.github/scripts/auto_update_price_and_context_window_file.py +++ b/.github/scripts/auto_update_price_and_context_window_file.py @@ -27,35 +27,13 @@ FRIENDLI_API_URL = "https://api.friendli.ai/serverless/v1/models" FRIENDLI_PROVIDER = "friendliai" INHERITABLE_BASE_KEYS = ( - "supports_reasoning", - "supports_function_calling", - "supports_parallel_function_calling", - "supports_response_schema", - "supports_system_messages", - "supports_tool_choice", - "supports_vision", "supports_pdf_input", - "supports_prompt_caching", "supports_assistant_prefill", - "supports_low_reasoning_effort", - "supports_minimal_reasoning_effort", - "supports_max_reasoning_effort", - "supports_xhigh_reasoning_effort", - "supports_none_reasoning_effort", "supports_adaptive_thinking", "supports_output_config", - "supports_native_structured_output", ) -EFFORT_FLAG_MAP = { - "none": "supports_none_reasoning_effort", - "minimal": "supports_minimal_reasoning_effort", - "low": "supports_low_reasoning_effort", - "medium": "supports_low_reasoning_effort", - "high": "supports_max_reasoning_effort", - "xhigh": "supports_xhigh_reasoning_effort", - "max": "supports_max_reasoning_effort", -} +REASONING_EFFORT_LEVEL_ORDER = ("none", "minimal", "low", "medium", "high", "xhigh", "max") def _find_base_model_entry(base_model: str, local_data: dict) -> str | None: @@ -72,15 +50,14 @@ def _find_base_model_entry(base_model: str, local_data: dict) -> str | None: return None -def _effort_flags(reasoning_options: list) -> dict: - flags: dict[str, bool] = {flag: False for flag in EFFORT_FLAG_MAP.values()} - for opt in reasoning_options or []: - if opt.get("type") == "effort": - for val in opt.get("values", []): - flag = EFFORT_FLAG_MAP.get(val) - if flag: - flags[flag] = True - return flags +def _reasoning_effort_levels(reasoning_options: list) -> list: + offered = { + val + for opt in reasoning_options or [] + if opt.get("type") == "effort" + for val in opt.get("values", []) + } + return [level for level in REASONING_EFFORT_LEVEL_ORDER if level in offered] def _pricing(pricing: dict) -> dict: @@ -97,15 +74,19 @@ def _pricing(pricing: dict) -> dict: def _modality_flags(input_mods: list) -> dict: - has_image = "image" in (input_mods or []) + mods = input_mods or [] + has_image = "image" in mods return { "supports_vision": has_image, "supports_image_input": has_image, + "supports_video_input": "video" in mods, } def transform_friendli_data(data: list, local_data: dict) -> dict: transformed: dict[str, dict] = {} + if not data: + return transformed for model in data: model_id = model["id"] base_model = model.get("base_model") or "" @@ -123,17 +104,21 @@ def transform_friendli_data(data: list, local_data: dict) -> dict: ctx = model.get("context_length") if ctx is not None: entry["max_input_tokens"] = int(ctx) - entry["max_tokens"] = int(ctx) max_out = model.get("max_completion_tokens") if max_out is not None: entry["max_output_tokens"] = int(max_out) + entry["max_tokens"] = int(max_out) - entry.update(_pricing(model.get("pricing", {}))) + pricing = _pricing(model.get("pricing", {})) + entry.update(pricing) + entry["supports_prompt_caching"] = "cache_read_input_token_cost" in pricing reasoning = model.get("reasoning") is True entry["supports_reasoning"] = reasoning if reasoning: - entry.update(_effort_flags(model.get("reasoning_options", []))) + entry["reasoning_effort_levels"] = _reasoning_effort_levels( + model.get("reasoning_options", []) + ) func = model.get("functionality", {}) entry["supports_function_calling"] = func.get("tool_call") is True diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5c32759f41c..5aa1bd53605 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19564,145 +19564,49 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "friendliai/LGAI-EXAONE/K-EXAONE-236B-A23B": { + "friendliai/zai-org/GLM-5.3-Flash": { "litellm_provider": "friendliai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 8e-07, - "cache_read_input_token_cost": 1e-07, - "supports_reasoning": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "mode": "chat", - "comment": "Open multilingual MoE model for reasoning, agentic tool use, and long-context work with strong Korean capabilities", - "deprecation_date": "2026-08-20", - "source": "https://api.friendli.ai/serverless/v1/models", - "supports_vision": false, - "supports_image_input": false - }, - "friendliai/MiniMaxAI/MiniMax-M2.5": { - "litellm_provider": "friendliai", - "max_input_tokens": 196608, - "max_tokens": 196608, - "max_output_tokens": 196608, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "cache_read_input_token_cost": 6e-08, - "supports_reasoning": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "mode": "chat", - "comment": "Prior MiniMax coding model for agent workflows, office edits, and automation", - "source": "https://api.friendli.ai/serverless/v1/models", - "supports_vision": false, - "supports_image_input": false - }, - "friendliai/deepseek-ai/DeepSeek-V3.2": { - "litellm_provider": "friendliai", - "supports_reasoning": true, - "supports_function_calling": true, - "supports_tool_choice": true, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 3e-08, "supports_prompt_caching": true, - "supports_assistant_prefill": true, - "max_input_tokens": 163840, - "max_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.5e-06, - "cache_read_input_token_cost": 2.5e-07, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_system_messages": true, - "mode": "chat", - "comment": "DeepSeek chat model for instruction following, coding, and analysis", - "source": "https://api.friendli.ai/serverless/v1/models", - "supports_vision": false, - "supports_image_input": false - }, - "friendliai/zai-org/GLM-5.1": { - "litellm_provider": "friendliai", "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "supports_function_calling": true, - "supports_tool_choice": true, - "supports_prompt_caching": true, - "max_input_tokens": 202752, - "max_tokens": 202752, - "max_output_tokens": 202752, - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 2.6e-07, "supports_parallel_function_calling": true, "supports_response_schema": true, "supports_native_structured_output": true, "supports_system_messages": true, - "mode": "chat", - "comment": "Strong GLM coding model for agentic engineering, terminals, and repository generation", - "source": "https://api.friendli.ai/serverless/v1/models", - "supports_vision": false, - "supports_image_input": false - }, - "friendliai/google/gemma-4-31B-it": { - "litellm_provider": "friendliai", + "supports_tool_choice": true, "supports_vision": true, - "max_input_tokens": 8192, - "max_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 4e-07, - "supports_reasoning": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_system_messages": true, - "supports_tool_choice": true, "supports_image_input": true, + "supports_video_input": true, "mode": "chat", - "comment": "Largest Gemma 4 instruction model for open, self-hosted chat and reasoning", + "comment": "Native multimodal GLM model for efficient coding and long-horizon agent tasks", "source": "https://api.friendli.ai/serverless/v1/models" }, - "friendliai/zai-org/GLM-5.2": { + "friendliai/zai-org/GLM-5.3": { "litellm_provider": "friendliai", - "supports_reasoning": true, - "supports_function_calling": true, "max_input_tokens": 1048576, - "max_tokens": 1048576, "max_output_tokens": 1048576, - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 2.6e-07, - "supports_max_reasoning_effort": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "mode": "chat", - "comment": "Open flagship GLM for long-horizon coding agents and million-token context work", - "source": "https://api.friendli.ai/serverless/v1/models", - "supports_vision": false, - "supports_image_input": false - }, - "friendliai/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B": { - "litellm_provider": "friendliai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.4e-06, - "cache_read_input_token_cost": 1.2e-07, + "max_tokens": 1048576, + "input_cost_per_token": 1.26e-06, + "output_cost_per_token": 3.96e-06, + "cache_read_input_token_cost": 2.34e-07, + "supports_prompt_caching": true, "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -19711,8 +19615,156 @@ "supports_tool_choice": true, "supports_vision": false, "supports_image_input": false, + "supports_video_input": false, + "mode": "chat", + "comment": "Flagship GLM model for long-horizon coding, agents, and complex project delivery", + "source": "https://api.friendli.ai/serverless/v1/models" + }, + "friendliai/google/gemma-4-31B-it": { + "litellm_provider": "friendliai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 4e-07, + "supports_prompt_caching": false, + "supports_reasoning": true, + "reasoning_effort_levels": [], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_image_input": true, + "supports_video_input": false, + "mode": "chat", + "comment": "Largest Gemma 4 instruction model for open, self-hosted chat and reasoning", + "source": "https://api.friendli.ai/serverless/v1/models" + }, + "friendliai/zai-org/GLM-5.2": { + "litellm_provider": "friendliai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 2.6e-07, + "supports_prompt_caching": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "high", + "max" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false, + "supports_image_input": false, + "supports_video_input": false, + "mode": "chat", + "comment": "Open flagship GLM for long-horizon coding agents and million-token context work", + "source": "https://api.friendli.ai/serverless/v1/models" + }, + "friendliai/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B": { + "litellm_provider": "friendliai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 1.2e-07, + "supports_prompt_caching": true, + "supports_reasoning": true, + "reasoning_effort_levels": [], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false, + "supports_image_input": false, + "supports_video_input": false, "mode": "chat", "comment": "Frontier-scale multilingual language model developed by LG AI Research", + "deprecation_date": "2026-09-06", + "source": "https://api.friendli.ai/serverless/v1/models" + }, + "friendliai/deepseek-ai/DeepSeek-V3.2": { + "litellm_provider": "friendliai", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 2.5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true, + "reasoning_effort_levels": [], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false, + "supports_image_input": false, + "supports_video_input": false, + "mode": "chat", + "comment": "DeepSeek chat model for instruction following, coding, and analysis", + "source": "https://api.friendli.ai/serverless/v1/models" + }, + "friendliai/MiniMaxAI/MiniMax-M2.5": { + "litellm_provider": "friendliai", + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "max_tokens": 196608, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 6e-08, + "supports_prompt_caching": true, + "supports_reasoning": true, + "reasoning_effort_levels": [], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false, + "supports_image_input": false, + "supports_video_input": false, + "mode": "chat", + "comment": "Prior MiniMax coding model for agent workflows, office edits, and automation", + "source": "https://api.friendli.ai/serverless/v1/models" + }, + "friendliai/zai-org/GLM-5.1": { + "litellm_provider": "friendliai", + "max_input_tokens": 202752, + "max_output_tokens": 202752, + "max_tokens": 202752, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 2.6e-07, + "supports_prompt_caching": true, + "supports_reasoning": true, + "reasoning_effort_levels": [], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false, + "supports_image_input": false, + "supports_video_input": false, + "mode": "chat", + "comment": "Strong GLM coding model for agentic engineering, terminals, and repository generation", "source": "https://api.friendli.ai/serverless/v1/models" }, "ft:babbage-002": { diff --git a/tests/test_litellm/test_auto_update_price_and_context_window_file.py b/tests/test_litellm/test_auto_update_price_and_context_window_file.py new file mode 100644 index 00000000000..7dbd24a1b81 --- /dev/null +++ b/tests/test_litellm/test_auto_update_price_and_context_window_file.py @@ -0,0 +1,136 @@ +"""Unit tests for the Friendli transform in +`.github/scripts/auto_update_price_and_context_window_file.py`.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[2] + / ".github" + / "scripts" + / "auto_update_price_and_context_window_file.py" +) + + +@pytest.fixture(scope="module") +def sync_module(): + spec = importlib.util.spec_from_file_location( + "auto_update_price_and_context_window_file", SCRIPT_PATH + ) + assert spec and spec.loader, f"Could not load spec for {SCRIPT_PATH}" + module = importlib.util.module_from_spec(spec) + sys.modules["auto_update_price_and_context_window_file"] = module + spec.loader.exec_module(module) + return module + + +def _reasoning_model(**overrides: object) -> dict: + model = { + "id": "zai-org/GLM-Test", + "base_model": "zhipuai/glm-test", + "context_length": 1048576, + "max_completion_tokens": 131072, + "pricing": {"input": "0.00000015", "output": "0.0000005", "input_cache_read": "0.00000003"}, + "reasoning": True, + "reasoning_options": [{"type": "effort", "values": ["max", "high", "low"]}], + "functionality": { + "tool_call": True, + "parallel_tool_call": True, + "structured_output": True, + "system_messages": True, + "tool_choice": True, + }, + "input_modalities": ["text", "image", "video"], + "mode": "chat", + } + model.update(overrides) + return model + + +def test_transform_emits_declared_effort_levels_in_canonical_order(sync_module): + entry = sync_module.transform_friendli_data([_reasoning_model()], {})[ + "friendliai/zai-org/GLM-Test" + ] + assert entry["supports_reasoning"] is True + assert entry["reasoning_effort_levels"] == ["low", "high", "max"] + assert not any(k.endswith("_reasoning_effort") for k in entry) + + +def test_transform_reasoning_model_without_effort_options_declares_empty_levels(sync_module): + model = _reasoning_model(reasoning_options=[{"type": "budget_tokens", "values": []}]) + entry = sync_module.transform_friendli_data([model], {})["friendliai/zai-org/GLM-Test"] + assert entry["reasoning_effort_levels"] == [] + + +def test_transform_non_reasoning_model_declares_no_levels(sync_module): + model = _reasoning_model(reasoning=False, reasoning_options=[]) + entry = sync_module.transform_friendli_data([model], {})["friendliai/zai-org/GLM-Test"] + assert entry["supports_reasoning"] is False + assert "reasoning_effort_levels" not in entry + + +def test_transform_max_tokens_mirrors_output_cap_not_context(sync_module): + entry = sync_module.transform_friendli_data([_reasoning_model()], {})[ + "friendliai/zai-org/GLM-Test" + ] + assert entry["max_input_tokens"] == 1048576 + assert entry["max_output_tokens"] == 131072 + assert entry["max_tokens"] == entry["max_output_tokens"] + + +def test_transform_prompt_caching_follows_cache_pricing(sync_module): + cached = sync_module.transform_friendli_data([_reasoning_model()], {})[ + "friendliai/zai-org/GLM-Test" + ] + assert cached["supports_prompt_caching"] is True + assert cached["cache_read_input_token_cost"] == 3e-08 + + uncached_model = _reasoning_model(pricing={"input": "0.00000014", "output": "0.0000004"}) + uncached = sync_module.transform_friendli_data([uncached_model], {})[ + "friendliai/zai-org/GLM-Test" + ] + assert uncached["supports_prompt_caching"] is False + assert "cache_read_input_token_cost" not in uncached + + +def test_transform_modalities_set_vision_image_and_video_flags(sync_module): + entry = sync_module.transform_friendli_data([_reasoning_model()], {})[ + "friendliai/zai-org/GLM-Test" + ] + assert entry["supports_vision"] is True + assert entry["supports_image_input"] is True + assert entry["supports_video_input"] is True + + text_only = _reasoning_model(input_modalities=["text"]) + entry_text = sync_module.transform_friendli_data([text_only], {})[ + "friendliai/zai-org/GLM-Test" + ] + assert entry_text["supports_vision"] is False + assert entry_text["supports_image_input"] is False + assert entry_text["supports_video_input"] is False + + +def test_transform_survives_failed_fetch(sync_module): + assert sync_module.transform_friendli_data(None, {}) == {} + assert sync_module.transform_friendli_data([], {}) == {} + + +def test_transform_inherits_allowlisted_keys_from_base_model_entry(sync_module): + local = { + "zhipuai/glm-test": { + "supports_pdf_input": True, + "supports_assistant_prefill": True, + "input_cost_per_token": 9e-06, + } + } + entry = sync_module.transform_friendli_data([_reasoning_model()], local)[ + "friendliai/zai-org/GLM-Test" + ] + assert entry["supports_pdf_input"] is True + assert entry["supports_assistant_prefill"] is True + assert entry["input_cost_per_token"] == 1.5e-07