From 30ff3723b2cc59b16c00644891ac309a06d6da11 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 27 Aug 2026 15:38:01 -0700 Subject: [PATCH] feat(model_prices): let a map entry declare its exact reasoning_effort levels (#38481) Kimi K3 accepts exactly low, high and max, defaults to max, and always thinks. The map could not say that: medium and high have no supports_*_reasoning_effort flag because every other reasoning model takes them, so the ten kimi-k3 entries carried supports_reasoning alone and resolved to unknown. The dashboard then fell back to a capability-blind level list that deliberately omits max, which is why a kimi-k3 tier cannot be set to max thinking today. Add reasoning_effort_levels, an array key in the shape the map already uses for supported_endpoints and supported_modalities. Where present it is read first and wins whole; every other entry keeps answering through the per-level flags, unchanged. It is deliberately a different name from the computed ModelGroupInfo.supported_reasoning_efforts, which stays derived from a group's deployments and is never seeded from one deployment's model_info. The levels are per entry rather than per model, because the deployments differ: Moonshot, Together, Fireworks and Azure Foundry all forward the level unchanged and get the model's own low/high/max, while Perplexity documents a six-value enum it maps down internally and gets that. The /v1/messages degradation chain consults the same declaration, so the level the map advertises is the level that path forwards. --- ci_cd/generate_model_prices_schema.py | 5 + .../experimental_pass_through/utils.py | 29 ++++ ...odel_prices_and_context_window_backup.json | 53 ++++++ .../reasoning_effort_capability.py | 32 +++- litellm/types/utils.py | 1 + litellm/utils.py | 1 + model_prices_and_context_window.json | 53 ++++++ model_prices_and_context_window.schema.json | 16 ++ .../test_reasoning_effort_fields.py | 90 ++++++++++ .../test_reasoning_effort_capability.py | 156 ++++++++++++++++++ tests/test_litellm/test_utils.py | 4 + 11 files changed, 435 insertions(+), 5 deletions(-) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 276e5da5a23..5e1c4b0dcd9 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -73,6 +73,11 @@ ARRAY_KEYS: dict[str, JsonSchema] = { "description": "Output modalities the model can produce.", "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]}, }, + "reasoning_effort_levels": { + "type": "array", + "description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.", + "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]}, + }, "supported_regions": { "type": "array", "description": "Cloud regions the model is available in ('global' or region ids).", diff --git a/litellm/llms/anthropic/experimental_pass_through/utils.py b/litellm/llms/anthropic/experimental_pass_through/utils.py index 29661572b73..8441dde23d6 100644 --- a/litellm/llms/anthropic/experimental_pass_through/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/utils.py @@ -1,4 +1,6 @@ import os +from collections.abc import Mapping +from types import MappingProxyType from typing import Final import litellm @@ -23,6 +25,29 @@ def is_reasoning_auto_summary_enabled() -> bool: return litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true" +_DECLARED_DEGRADATION_CHAINS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( + {"max": ("max", "xhigh", "high"), "xhigh": ("xhigh", "high"), "minimal": ("minimal", "low")} +) + + +def _effort_from_declaration(model_info: ModelInfo, effort: str) -> str | None: + """A declared level set is the WHOLE answer for this gate, so a level it omits degrades even + where a per-level flag would have allowed it. Honoring both would let /model_group/info and + this path disagree about the same entry. None means the entry declares nothing, and the flag + chain below decides as before. + + A declaration that omits every level in a chain still lands on that chain's terminal, which can + itself be undeclared. Picking a nearer declared level instead would need a strength ordering, + and the advertisement order is presentation only by design, so the terminal stays the answer.""" + from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts + + declared: Final = declared_reasoning_efforts(model_info) + if declared is None: + return None + chain: Final = _DECLARED_DEGRADATION_CHAINS[effort] + return next((level for level in chain if level in declared), chain[-1]) + + def normalize_reasoning_effort_value( effort: str, model: str, @@ -48,6 +73,10 @@ def normalize_reasoning_effort_value( except Exception: model_info = None + declared_effort: Final = _effort_from_declaration(model_info, effort) if model_info is not None else None + if declared_effort is not None: + return declared_effort + if effort == "max": if model_info and model_info.get("supports_max_reasoning_effort"): return "max" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 095f49cb991..77572f69b8b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9315,6 +9315,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k3-through-fireworks-ai-on-microsoft-foundry/4540187", "supported_modalities": [ "text", @@ -31897,6 +31902,11 @@ "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://platform.kimi.ai/docs/pricing/chat-k3", "supports_function_calling": true, "supports_reasoning": true, @@ -36505,6 +36515,14 @@ "litellm_provider": "perplexity", "mode": "responses", "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ], "source": "https://docs.perplexity.ai/docs/agent-api/models", "supports_web_search": true, "supports_reasoning": true, @@ -38986,6 +39004,11 @@ "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -51774,6 +51797,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -51838,6 +51866,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -51854,6 +51887,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -51870,6 +51908,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -52042,6 +52085,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -52058,6 +52106,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 3e4478e5e21..08feb96e36a 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -1,10 +1,13 @@ """Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts. -The model map's supports_*_reasoning_effort flags are the only signal, and each level's polarity -mirrors how a request path reads that same flag. medium and high are unconditional for a reasoning -model. minimal and low are opt-out: openai/chat/gpt_5_transformation.py refuses them only when the -map says false. xhigh and max are opt-in. none is opt-out everywhere except the azure gpt-5 family, -whose config raises UnsupportedParamsError without an explicit true. +An entry that states its levels outright in reasoning_effort_levels is read first and wins +whole, for a model whose set the per-level flags cannot express: Kimi K3 takes low, high and max, +and no flag can drop medium because medium has none. Every other entry answers through the +supports_*_reasoning_effort flags below, whose polarity mirrors how a request path reads that same +flag. medium and high are unconditional for a reasoning model. minimal and low are opt-out: +openai/chat/gpt_5_transformation.py refuses them only when the map says false. xhigh and max are +opt-in. none is opt-out everywhere except the azure gpt-5 family, whose config raises +UnsupportedParamsError without an explicit true. xhigh is gated on the request path by the openai and azure gpt-5 configs. max is not gated there at all: every entry carrying supports_max_reasoning_effort is Claude-family, and @@ -41,6 +44,7 @@ _EFFORT_FLAGS: Final = ( ("xhigh", "supports_xhigh_reasoning_effort"), ("max", "supports_max_reasoning_effort"), ) +_DECLARED_EFFORTS_KEY: Final = "reasoning_effort_levels" _OPT_OUT_EFFORTS: Final = ("minimal", "low") _OPT_IN_EFFORTS: Final = ("xhigh", "max") _UNCONDITIONAL_EFFORTS: Final = frozenset(("medium", "high")) @@ -69,6 +73,20 @@ def _declared_effort_flags(model_info: Mapping[str, object]) -> Mapping[str, obj ) +def declared_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, ...] | None: + """The entry's own answer, read through the same bare twin as the flags so both spellings of one + model agree. Present-and-a-list IS the answer, so a declared [] correctly empties the group and + an unknown level is dropped rather than raised: the bundled map is enum-validated by + validate-model-prices-json, but an operator can put this key on a config.yaml model_info block + where that schema never runs, and one mistyped level must not fail every sibling on the proxy.""" + own: Final = model_info.get(_DECLARED_EFFORTS_KEY) + raw: Final = own if own is not None else _bare_model_entry(model_info).get(_DECLARED_EFFORTS_KEY) + if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)): + return None + declared: Final = frozenset(effort for effort in raw if isinstance(effort, str)) + return tuple(effort for effort in REASONING_EFFORT_ADVERTISEMENT_ORDER if effort in declared) + + def _supports_none_reasoning_effort(model_info: Mapping[str, object], flag: object) -> bool: """Opt-in only where a request path refuses the level. AzureOpenAIGPT5Config raises UnsupportedParamsError on reasoning_effort='none' without an explicit true, and it is selected @@ -119,6 +137,10 @@ def resolve_supported_reasoning_efforts( if supports_reasoning is not True: return () if supports_reasoning is False or deployment_is_mapped else None + declared: Final = declared_reasoning_efforts(model_info) + if declared is not None: + return declared + flags: Final = _declared_effort_flags(model_info) if all(value is None for value in flags.values()): return None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 58103b84749..95429e899c9 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -164,6 +164,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_low_reasoning_effort: bool | None supports_xhigh_reasoning_effort: bool | None supports_max_reasoning_effort: bool | None + reasoning_effort_levels: ReadOnly[Sequence[str] | None] supports_output_config: bool | None supports_image_size: bool | None bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None diff --git a/litellm/utils.py b/litellm/utils.py index b75d0161cb4..520c40f67c0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5889,6 +5889,7 @@ def _get_model_info_helper( supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None), supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None), supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None), + reasoning_effort_levels=_model_info.get("reasoning_effort_levels", None), bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None), bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None), supports_computer_use=_model_info.get("supports_computer_use", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 095f49cb991..77572f69b8b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9315,6 +9315,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k3-through-fireworks-ai-on-microsoft-foundry/4540187", "supported_modalities": [ "text", @@ -31897,6 +31902,11 @@ "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://platform.kimi.ai/docs/pricing/chat-k3", "supports_function_calling": true, "supports_reasoning": true, @@ -36505,6 +36515,14 @@ "litellm_provider": "perplexity", "mode": "responses", "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ], "source": "https://docs.perplexity.ai/docs/agent-api/models", "supports_web_search": true, "supports_reasoning": true, @@ -38986,6 +39004,11 @@ "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -51774,6 +51797,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -51838,6 +51866,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -51854,6 +51887,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -51870,6 +51908,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -52042,6 +52085,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -52058,6 +52106,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index f68644705b6..6e837354c60 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -532,6 +532,22 @@ "type": "object", "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema)." }, + "reasoning_effort_levels": { + "type": "array", + "description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.", + "items": { + "type": "string", + "enum": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ] + } + }, "regional_endpoint_uplift_multiplier": { "type": "number", "minimum": 1, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py index 08fef8c6a24..8de40bbaf6f 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py @@ -14,6 +14,8 @@ from unittest.mock import patch import pytest +import litellm + from litellm.llms.anthropic.experimental_pass_through.utils import ( normalize_reasoning_effort_value, ) @@ -291,3 +293,91 @@ class TestAdapterAdaptiveThinking: ) assert result is not None assert result["effort"] == "medium" + + +class TestDeclaredEffortsAnswerTheDegradationGate: + """Without this the chain reads only the per-level booleans, so a kimi-k3 request asking for + max silently arrives as high.""" + + @pytest.mark.parametrize( + "model, provider", + [("kimi-k3", "moonshot"), ("kimi-k3", "fireworks_ai"), ("kimi-k3-us", "fireworks_ai")], + ) + def test_a_declared_level_survives_instead_of_degrading(self, local_model_cost_map, model, provider): + assert normalize_reasoning_effort_value("max", model, provider) == "max" + + def test_a_level_the_entry_does_not_declare_still_degrades(self, local_model_cost_map): + """xhigh is not on kimi-k3's declaration, so it must keep degrading rather than be waved + past by the mere presence of one.""" + assert normalize_reasoning_effort_value("xhigh", "kimi-k3", "moonshot") == "high" + assert normalize_reasoning_effort_value("minimal", "kimi-k3", "moonshot") == "low" + + def test_the_wider_perplexity_entry_keeps_the_levels_it_declares(self, local_model_cost_map): + assert normalize_reasoning_effort_value("xhigh", "perplexity/kimi-k3", "perplexity") == "xhigh" + assert normalize_reasoning_effort_value("minimal", "perplexity/kimi-k3", "perplexity") == "minimal" + + @pytest.mark.parametrize( + "model, provider, effort, expected", + [ + ("claude-opus-4-7", "anthropic", "max", "max"), + ("claude-sonnet-4-6", "anthropic", "minimal", "low"), + ("gpt-5-mini", "azure", "max", "high"), + ], + ) + def test_an_entry_on_the_per_level_flags_is_untouched( + self, local_model_cost_map, model, provider, effort, expected + ): + """The negative class that bounds this change to entries carrying the key.""" + assert normalize_reasoning_effort_value(effort, model, provider) == expected + + +class TestDeclarationBeatsThePerLevelFlags: + """An entry can carry both shapes. The declaration wins whole, or /model_group/info and this + path would disagree about the same deployment. Driven through the public entry point over a + seeded map entry rather than a patched get_model_info, so it pins behaviour and not wiring.""" + + MODEL = "declared-and-flagged" + + @pytest.fixture + def seeded(self, local_model_cost_map, monkeypatch): + def _seed(**entry): + monkeypatch.setitem( + litellm.model_cost, + self.MODEL, + {"litellm_provider": "openai", "mode": "chat", "supports_reasoning": True, **entry}, + ) + litellm.get_model_info.cache_clear() + + return _seed + + @pytest.mark.parametrize("effort, expected", [("max", "max"), ("xhigh", "high"), ("minimal", "low")]) + def test_a_flag_cannot_re_add_a_level_the_declaration_omits(self, seeded, effort, expected): + seeded( + reasoning_effort_levels=["low", "high", "max"], + supports_xhigh_reasoning_effort=True, + supports_minimal_reasoning_effort=True, + supports_max_reasoning_effort=False, + ) + + assert normalize_reasoning_effort_value(effort, self.MODEL, "openai") == expected + + def test_a_flag_cannot_keep_max_when_the_declaration_drops_it(self, seeded): + seeded( + reasoning_effort_levels=["low", "high"], + supports_max_reasoning_effort=True, + supports_xhigh_reasoning_effort=True, + ) + + assert normalize_reasoning_effort_value("max", self.MODEL, "openai") == "high" + + def test_a_false_flag_cannot_remove_a_level_the_declaration_names(self, seeded): + seeded(reasoning_effort_levels=["high", "xhigh"], supports_xhigh_reasoning_effort=False) + + assert normalize_reasoning_effort_value("xhigh", self.MODEL, "openai") == "xhigh" + assert normalize_reasoning_effort_value("max", self.MODEL, "openai") == "xhigh" + + def test_a_chain_the_declaration_omits_entirely_lands_on_its_terminal(self, seeded): + """Documented residual: no strength ordering exists to pick a nearer declared level.""" + seeded(reasoning_effort_levels=["high", "xhigh"]) + + assert normalize_reasoning_effort_value("minimal", self.MODEL, "openai") == "low" diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py index 6cd5b956b05..ff5660288f5 100644 --- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py +++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py @@ -1,5 +1,6 @@ import pytest +import litellm from litellm.router_utils.reasoning_effort_capability import ( deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, @@ -196,3 +197,158 @@ class TestIntersectSupportedReasoningEfforts: def test_disjoint_sets_intersect_to_empty(self): assert intersect_supported_reasoning_efforts(["max"], ["minimal"]) == () + + +class TestDeclaredEffortList: + """reasoning_effort_levels is what the catalog DECLARES per deployment; + ModelGroupInfo.supported_reasoning_efforts is what a group COMPUTED. test_router.py pins that + the computed one is never seeded from model_info, so the two names must stay apart.""" + + def test_a_declared_list_answers_where_no_flag_could(self): + """No flag can drop medium, so before this key the entry could only stay silent or + over-advertise a level the model does not document.""" + resolved = resolve_supported_reasoning_efforts( + {"supports_reasoning": True, "reasoning_effort_levels": ["low", "high", "max"]}, + deployment_is_mapped=True, + ) + assert resolved == ("low", "high", "max") + + def test_a_declared_list_wins_whole_over_the_flags(self): + resolved = resolve_supported_reasoning_efforts( + { + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "high", "max"], + "supports_none_reasoning_effort": True, + "supports_minimal_reasoning_effort": True, + "supports_xhigh_reasoning_effort": True, + "supports_max_reasoning_effort": False, + }, + deployment_is_mapped=True, + ) + assert resolved == ("low", "high", "max") + + def test_a_declaration_is_reordered_into_the_advertisement_order(self): + resolved = resolve_supported_reasoning_efforts( + {"supports_reasoning": True, "reasoning_effort_levels": ["max", "low", "high"]}, + deployment_is_mapped=True, + ) + assert resolved == ("low", "high", "max") + + def test_a_declared_empty_list_empties_the_group(self): + assert ( + resolve_supported_reasoning_efforts( + {"supports_reasoning": True, "reasoning_effort_levels": []}, + deployment_is_mapped=True, + ) + == () + ) + + @pytest.mark.parametrize("declared", [["low", "bogus"], ["bogus"], ["low", 7, None]]) + def test_an_unknown_level_is_dropped_rather_than_raised(self, declared): + """A config.yaml model_info block bypasses the map's enum schema, and one mistyped level + must not fail every sibling on the proxy.""" + resolved = resolve_supported_reasoning_efforts( + {"supports_reasoning": True, "reasoning_effort_levels": declared}, + deployment_is_mapped=True, + ) + assert resolved == tuple(effort for effort in ("low",) if effort in declared) + + @pytest.mark.parametrize("malformed", ["low,high,max", {"low": True}, 3, True]) + def test_a_malformed_declaration_falls_through_to_the_flags(self, malformed): + resolved = resolve_supported_reasoning_efforts( + { + "supports_reasoning": True, + "reasoning_effort_levels": malformed, + "supports_max_reasoning_effort": True, + }, + deployment_is_mapped=True, + ) + assert resolved == ("none", "minimal", "low", "medium", "high", "max") + + def test_a_model_the_map_calls_non_reasoning_ignores_its_declaration(self): + assert ( + resolve_supported_reasoning_efforts( + {"supports_reasoning": False, "reasoning_effort_levels": ["low", "high", "max"]}, + deployment_is_mapped=True, + ) + == () + ) + + def test_a_declaration_is_read_through_the_bare_twin(self, monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "some-declared-reasoner", + {"supports_reasoning": True, "reasoning_effort_levels": ["low", "max"]}, + ) + resolved = resolve_supported_reasoning_efforts( + { + "supports_reasoning": True, + "litellm_provider": "openai", + "key": "openai/some-declared-reasoner", + }, + deployment_is_mapped=True, + ) + assert resolved == ("low", "max") + + +KIMI_K3_PASSTHROUGH_KEYS = ( + "azure_ai/FW-Kimi-K3", + "moonshot/kimi-k3", + "together_ai/moonshotai/Kimi-K3", + "fireworks_ai/kimi-k3", + "fireworks_ai/kimi-k3-fast", + "fireworks_ai/kimi-k3-us", + "fireworks_ai/accounts/fireworks/models/kimi-k3", + "fireworks_ai/accounts/fireworks/routers/kimi-k3-fast", + "fireworks_ai/accounts/fireworks/routers/kimi-k3-us", +) +KIMI_K3_PERPLEXITY_KEY = "perplexity/perplexity/kimi-k3" + + +class TestKimiK3AdvertisesItsDocumentedLevels: + @pytest.mark.parametrize("model_key", KIMI_K3_PASSTHROUGH_KEYS) + def test_a_passthrough_entry_advertises_the_models_own_levels(self, local_model_cost_map, model_key): + """platform.kimi.ai documents exactly low, high and max, and these providers forward the + level unchanged. Undeclared, each entry resolves to unknown and the dashboard falls back to + a capability-blind list that omits max.""" + entry = dict(litellm.model_cost[model_key], key=model_key) + + assert resolve_supported_reasoning_efforts(entry, deployment_is_mapped=True) == ("low", "high", "max") + + def test_the_perplexity_entry_advertises_the_wider_set_it_maps_down(self, local_model_cost_map): + """Perplexity's Agent API takes a six-value enum and maps it down internally, so this + deployment is legitimately wider than a passthrough. One blanket list could not say both.""" + entry = dict(litellm.model_cost[KIMI_K3_PERPLEXITY_KEY], key=KIMI_K3_PERPLEXITY_KEY) + + assert resolve_supported_reasoning_efforts(entry, deployment_is_mapped=True) == ( + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + ) + + @pytest.mark.parametrize("model, provider", [("kimi-k3", "moonshot"), ("kimi-k3", "fireworks_ai")]) + def test_the_declaration_survives_model_info_hydration(self, local_model_cost_map, model, provider): + """The hydration line is the load-bearing seam: without it the key the map carries never + reaches the resolver and reads as absent everywhere downstream.""" + from litellm.utils import _get_model_info_helper + + model_info = dict(_get_model_info_helper(model=model, custom_llm_provider=provider)) + + assert model_info["reasoning_effort_levels"] == ["low", "high", "max"] + assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ("low", "high", "max") + + def test_a_kimi_k3_deployment_now_narrows_a_mixed_group(self, local_model_cost_map): + """kimi used to contribute unknown, which never narrows, so the group advertised whatever + its other deployments agreed on.""" + kimi = resolve_supported_reasoning_efforts( + dict(litellm.model_cost["fireworks_ai/kimi-k3"], key="fireworks_ai/kimi-k3"), + deployment_is_mapped=True, + ) + + assert intersect_supported_reasoning_efforts(("none", "minimal", "low", "medium", "high", "xhigh"), kimi) == ( + "low", + "high", + ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 20e67b902b8..e9b2851f717 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1014,6 +1014,10 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_none_reasoning_effort": {"type": "boolean"}, "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_max_reasoning_effort": {"type": "boolean"}, + "reasoning_effort_levels": { + "type": "array", + "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]}, + }, "supports_adaptive_thinking": {"type": "boolean"}, "supports_legacy_thinking": {"type": "boolean"}, "thinking_always_on": {"type": "boolean"},