mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
Merge pull request #38263 from BerriAI/litellm_together_reasoning_effort
feat(together_ai): map reasoning_effort per model class
This commit is contained in:
Mateo Wang
GitHub
commit
7083c47998
3 changed files with 204 additions and 4 deletions
|
|
@ -4,7 +4,8 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
|
|||
Docs: https://docs.together.ai/docs/chat-overview
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Container, Coroutine
|
||||
from collections.abc import Callable, Container, Coroutine, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
Final,
|
||||
Literal,
|
||||
|
|
@ -12,11 +13,13 @@ from typing import (
|
|||
overload,
|
||||
)
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.utils import supports_function_calling, supports_response_schema
|
||||
from litellm.utils import supports_function_calling, supports_reasoning, supports_response_schema
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
|
|
@ -38,6 +41,34 @@ def _registry_verdict(model: str, flag: str, check: Callable[[str], bool]) -> bo
|
|||
return None
|
||||
|
||||
|
||||
ADJUSTABLE_EFFORT_REASONING_MODELS: Final = frozenset(
|
||||
{
|
||||
"openai/gpt-oss-120b",
|
||||
"openai/gpt-oss-20b",
|
||||
}
|
||||
)
|
||||
HYBRID_REASONING_MODELS: Final = frozenset(
|
||||
{
|
||||
"MiniMaxAI/MiniMax-M3",
|
||||
"Qwen/Qwen3.5-9B",
|
||||
"Qwen/Qwen3.6-Plus",
|
||||
"deepseek-ai/DeepSeek-V4-Pro",
|
||||
"moonshotai/Kimi-K3",
|
||||
"nvidia/nemotron-3-ultra-550b-a55b",
|
||||
"zai-org/GLM-5.2",
|
||||
}
|
||||
)
|
||||
HIGH_MAX_EFFORT_MODEL_PREFIX: Final = "deepseek-ai/DeepSeek-V4-Pro"
|
||||
EFFORT_TRANSLATION: Final = MappingProxyType({"minimal": "low", "xhigh": "high", "max": "high"})
|
||||
HIGH_MAX_EFFORT_TRANSLATION: Final = MappingProxyType(
|
||||
{"minimal": "high", "low": "high", "medium": "high", "xhigh": "max"}
|
||||
)
|
||||
|
||||
|
||||
class TogetherReasoningToggle(TypedDict):
|
||||
enabled: ReadOnly[bool]
|
||||
|
||||
|
||||
def _function_calling_verdict(model: str) -> bool | None:
|
||||
return _registry_verdict(
|
||||
model,
|
||||
|
|
@ -83,6 +114,36 @@ def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params:
|
|||
)
|
||||
|
||||
|
||||
def _supports_together_reasoning(model: str) -> bool:
|
||||
if model in ADJUSTABLE_EFFORT_REASONING_MODELS or model in HYBRID_REASONING_MODELS:
|
||||
return True
|
||||
if model.startswith(HIGH_MAX_EFFORT_MODEL_PREFIX):
|
||||
return True
|
||||
return supports_reasoning(model, custom_llm_provider="together_ai")
|
||||
|
||||
|
||||
def _adjustable_effort(effort: str, model: str) -> str:
|
||||
if effort == "none":
|
||||
verbose_logger.debug(
|
||||
"together_ai model %s cannot disable reasoning; mapping reasoning_effort=none to low", model
|
||||
)
|
||||
return "low"
|
||||
return EFFORT_TRANSLATION.get(effort, effort)
|
||||
|
||||
|
||||
def _reasoning_effort_payload(effort: str, model: str) -> Mapping[str, object]:
|
||||
if effort == "default":
|
||||
return MappingProxyType({})
|
||||
if model in ADJUSTABLE_EFFORT_REASONING_MODELS:
|
||||
return MappingProxyType({"reasoning_effort": _adjustable_effort(effort, model)})
|
||||
if effort == "none":
|
||||
disable_reasoning: Final[TogetherReasoningToggle] = {"enabled": False}
|
||||
return MappingProxyType({"reasoning": disable_reasoning})
|
||||
if model.startswith(HIGH_MAX_EFFORT_MODEL_PREFIX):
|
||||
return MappingProxyType({"reasoning_effort": HIGH_MAX_EFFORT_TRANSLATION.get(effort, effort)})
|
||||
return MappingProxyType({"reasoning_effort": EFFORT_TRANSLATION.get(effort, effort)})
|
||||
|
||||
|
||||
def _drop_response_format(passed_params: Container[str], model: str, drop_params: bool) -> bool:
|
||||
if "response_format" not in passed_params:
|
||||
return False
|
||||
|
|
@ -153,6 +214,15 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
|
|||
return super()._transform_messages(stripped, model, is_async=True)
|
||||
return super()._transform_messages(stripped, model, is_async=False)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: inherited contract
|
||||
supported_params: Final = super().get_supported_openai_params(model)
|
||||
if not _supports_together_reasoning(model):
|
||||
return supported_params
|
||||
return [ # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
|
||||
*supported_params,
|
||||
"reasoning_effort",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
|
|
@ -165,4 +235,10 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
|
|||
mapped_openai_params.pop(param)
|
||||
if _drop_response_format(mapped_openai_params, model, drop_params):
|
||||
mapped_openai_params.pop("response_format")
|
||||
effort: Final = mapped_openai_params.get("reasoning_effort")
|
||||
if not isinstance(effort, str):
|
||||
return mapped_openai_params
|
||||
mapped_openai_params.pop("reasoning_effort")
|
||||
for key, value in _reasoning_effort_payload(effort, model).items():
|
||||
mapped_openai_params.setdefault(key, value)
|
||||
return mapped_openai_params
|
||||
|
|
|
|||
|
|
@ -3025,8 +3025,7 @@ def register_model(
|
|||
and value.get("cache_read_input_token_cost") is None
|
||||
and value.get("tiered_pricing") is None
|
||||
and (
|
||||
value.get("input_cost_per_token") is not None
|
||||
or value.get("output_cost_per_token") is not None
|
||||
value.get("input_cost_per_token") is not None or value.get("output_cost_per_token") is not None
|
||||
)
|
||||
):
|
||||
verbose_logger.warning(
|
||||
|
|
|
|||
|
|
@ -18,8 +18,14 @@ from litellm.types.utils import LlmProviders, ModelResponse
|
|||
|
||||
TOOL_CALLING_MODEL = "openai/gpt-oss-20b"
|
||||
REASONING_MODEL = "deepseek-ai/DeepSeek-V3.1"
|
||||
PLAIN_MODEL = "Qwen/Qwen3-235B-A22B-fp8-tput"
|
||||
UNMAPPED_MODEL = "example-org/brand-new-model"
|
||||
NO_TOOLS_MODEL = "example-org/no-tools-model"
|
||||
ADJUSTABLE_REASONING_MODEL = "openai/gpt-oss-120b"
|
||||
HYBRID_REASONING_MODEL = "Qwen/Qwen3.5-9B"
|
||||
HIGH_MAX_REASONING_MODEL = "deepseek-ai/DeepSeek-V4-Pro"
|
||||
REGISTRY_FLAGGED_REASONING_MODEL = "zai-org/GLM-4.6"
|
||||
NON_REASONING_MODEL = "meta-llama/Llama-3.3-70B-Instruct-Turbo"
|
||||
NO_SCHEMA_MODEL = "example-org/no-schema-model"
|
||||
|
||||
TOOL_PARAMS = ("tools", "tool_choice", "function_call")
|
||||
|
|
@ -39,6 +45,15 @@ JSON_SCHEMA_RESPONSE_FORMAT = {
|
|||
REGEX_RESPONSE_FORMAT = {"type": "regex", "pattern": "(positive|neutral|negative)"}
|
||||
|
||||
|
||||
def _map_reasoning_effort(model: str, effort: str) -> dict:
|
||||
return TogetherAIChatConfig().map_openai_params(
|
||||
non_default_params={"reasoning_effort": effort},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def force_local_model_cost(monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
|
@ -191,6 +206,116 @@ def test_map_openai_params_schema_model_passes_response_format_through(response_
|
|||
assert mapped["response_format"] == response_format
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[ADJUSTABLE_REASONING_MODEL, HYBRID_REASONING_MODEL, HIGH_MAX_REASONING_MODEL, REGISTRY_FLAGGED_REASONING_MODEL],
|
||||
)
|
||||
def test_supported_params_includes_reasoning_effort_for_reasoning_models(model):
|
||||
supported = TogetherAIChatConfig().get_supported_openai_params(model=model)
|
||||
|
||||
assert "reasoning_effort" in supported
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [NON_REASONING_MODEL, PLAIN_MODEL])
|
||||
def test_supported_params_excludes_reasoning_effort_for_non_reasoning_models(model):
|
||||
supported = TogetherAIChatConfig().get_supported_openai_params(model=model)
|
||||
|
||||
assert "reasoning_effort" not in supported
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"effort, expected",
|
||||
[("low", "low"), ("medium", "medium"), ("high", "high"), ("minimal", "low"), ("xhigh", "high"), ("max", "high")],
|
||||
)
|
||||
def test_adjustable_model_translates_reasoning_effort(effort, expected):
|
||||
mapped = _map_reasoning_effort(ADJUSTABLE_REASONING_MODEL, effort)
|
||||
|
||||
assert mapped["reasoning_effort"] == expected
|
||||
assert "reasoning" not in mapped
|
||||
|
||||
|
||||
def test_adjustable_model_cannot_disable_reasoning_so_none_becomes_low():
|
||||
mapped = _map_reasoning_effort(ADJUSTABLE_REASONING_MODEL, "none")
|
||||
|
||||
assert mapped["reasoning_effort"] == "low"
|
||||
assert "reasoning" not in mapped
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"effort, expected",
|
||||
[("low", "low"), ("medium", "medium"), ("high", "high"), ("minimal", "low"), ("xhigh", "high"), ("max", "high")],
|
||||
)
|
||||
def test_hybrid_model_translates_reasoning_effort(effort, expected):
|
||||
mapped = _map_reasoning_effort(HYBRID_REASONING_MODEL, effort)
|
||||
|
||||
assert mapped["reasoning_effort"] == expected
|
||||
assert "reasoning" not in mapped
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [HYBRID_REASONING_MODEL, HIGH_MAX_REASONING_MODEL, REGISTRY_FLAGGED_REASONING_MODEL])
|
||||
def test_reasoning_effort_none_becomes_reasoning_toggle(model):
|
||||
mapped = _map_reasoning_effort(model, "none")
|
||||
|
||||
assert mapped["reasoning"] == {"enabled": False}
|
||||
assert "reasoning_effort" not in mapped
|
||||
|
||||
|
||||
def test_reasoning_effort_none_does_not_clobber_user_reasoning():
|
||||
mapped = TogetherAIChatConfig().map_openai_params(
|
||||
non_default_params={"reasoning_effort": "none"},
|
||||
optional_params={"reasoning": {"enabled": True}},
|
||||
model=HYBRID_REASONING_MODEL,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert mapped["reasoning"] == {"enabled": True}
|
||||
assert "reasoning_effort" not in mapped
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"effort, expected",
|
||||
[("minimal", "high"), ("low", "high"), ("medium", "high"), ("high", "high"), ("xhigh", "max"), ("max", "max")],
|
||||
)
|
||||
def test_deepseek_v4_pro_remaps_to_high_max(effort, expected):
|
||||
mapped = _map_reasoning_effort(HIGH_MAX_REASONING_MODEL, effort)
|
||||
|
||||
assert mapped["reasoning_effort"] == expected
|
||||
|
||||
|
||||
def test_deepseek_v4_pro_dated_variant_remaps_via_prefix():
|
||||
mapped = _map_reasoning_effort(f"{HIGH_MAX_REASONING_MODEL}-0813", "low")
|
||||
|
||||
assert mapped["reasoning_effort"] == "high"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [ADJUSTABLE_REASONING_MODEL, HYBRID_REASONING_MODEL, HIGH_MAX_REASONING_MODEL])
|
||||
def test_reasoning_effort_default_is_dropped(model):
|
||||
mapped = _map_reasoning_effort(model, "default")
|
||||
|
||||
assert "reasoning_effort" not in mapped
|
||||
assert "reasoning" not in mapped
|
||||
|
||||
|
||||
def test_get_optional_params_translates_reasoning_effort_for_together():
|
||||
optional_params = litellm.get_optional_params(
|
||||
model=ADJUSTABLE_REASONING_MODEL,
|
||||
custom_llm_provider="together_ai",
|
||||
reasoning_effort="max",
|
||||
)
|
||||
|
||||
assert optional_params["reasoning_effort"] == "high"
|
||||
|
||||
|
||||
def test_get_optional_params_rejects_reasoning_effort_for_non_reasoning_together_model():
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
litellm.get_optional_params(
|
||||
model=NON_REASONING_MODEL,
|
||||
custom_llm_provider="together_ai",
|
||||
reasoning_effort="low",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", [False, True])
|
||||
def test_map_openai_params_unmapped_model_passes_response_format_through(drop_params, together_warning_log):
|
||||
mapped = TogetherAIChatConfig().map_openai_params(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue