From 4ede2c012f15bad9df94ed08a0688c8a744c38b3 Mon Sep 17 00:00:00 2001 From: jesus-berri Date: Mon, 2 Nov 2026 13:53:58 -0800 Subject: [PATCH] feat(model_info): add supports_continuation capability flag for Gemini long decode continuation (#49500) * feat(model_info): add supports_continuation capability flag for Gemini long decode continuation * fix(router): keep a deployment's supports_continuation override on that deployment A supports_continuation value set in one deployment's model_info was written onto the shared provider/model cost map key, so every other deployment of that model reported it too. List it with the other per-deployment capability flags, and cite Google's long decode page next to the cost map assertion * fix(cost-map): flag vertex_ai/gemini-4-argon-cyber-permissive with supports_continuation Google's Vertex Gemini 4 Argon guide gives gemini-4-argon-cyber-permissive the same long decode limit as gemini-4-argon, and LiteLLM already continues its long requests, so the cost map now says so in both copies. * fix(router): count only a boolean true supports_continuation when building a model group --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 1 + litellm/_lazy_imports_registry.py | 2 + ...odel_prices_and_context_window_backup.json | 3 ++ litellm/router_python_core.py | 3 ++ litellm/types/router.py | 1 + litellm/types/utils.py | 2 + litellm/utils.py | 10 +++++ model_prices_and_context_window.json | 3 ++ .../test_model_prices_and_context_window.py | 20 +++++++++ .../test_router/test_router_helper_utils.py | 45 +++++++++++++++++++ .../unit/test_router_model_cost_isolation.py | 5 ++- tests/unit/test_utils.py | 29 ++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 +++ 13 files changed, 128 insertions(+), 1 deletion(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 0e5101b49c3..baee4e9f7ba 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2374,6 +2374,7 @@ if TYPE_CHECKING: supports_web_search: Callable[..., bool] supports_tool_search: Callable[..., bool] supports_url_context: Callable[..., bool] + supports_continuation: Callable[..., bool] supports_response_schema: Callable[..., bool] supports_parallel_function_calling: Callable[..., bool] supports_vision: Callable[..., bool] diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index de290070ed3..c9949613303 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -33,6 +33,7 @@ UTILS_NAMES: Final = ( "supports_web_search", "supports_tool_search", "supports_url_context", + "supports_continuation", "supports_response_schema", "supports_parallel_function_calling", "supports_vision", @@ -488,6 +489,7 @@ _UTILS_IMPORT_MAP: Final = { "supports_web_search": (".utils", "supports_web_search"), "supports_tool_search": (".utils", "supports_tool_search"), "supports_url_context": (".utils", "supports_url_context"), + "supports_continuation": (".utils", "supports_continuation"), "supports_response_schema": (".utils", "supports_response_schema"), "supports_parallel_function_calling": ( ".utils", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e6f65d6b416..8bfb8949890 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76813,6 +76813,7 @@ "supports_audio_input": true, "supports_audio_output": false, "supports_computer_use": true, + "supports_continuation": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, @@ -76872,6 +76873,7 @@ ], "supports_audio_input": true, "supports_computer_use": true, + "supports_continuation": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, @@ -76929,6 +76931,7 @@ "text" ], "supports_audio_input": true, + "supports_continuation": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 98c9e641278..2235abc9897 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -13247,6 +13247,7 @@ class PythonCore(DeprecatedRouterMembers): "providers": [llm_provider], "description": model_group_description, **model_info, + "supports_continuation": model_info.get("supports_continuation") is True, "supports_fast_mode": True, "supported_reasoning_efforts": None, "supported_endpoints": None, @@ -13314,6 +13315,8 @@ class PythonCore(DeprecatedRouterMembers): and model_info["supports_url_context"] is True ): model_group_info.supports_url_context = True + if model_info.get("supports_continuation") is True: + model_group_info.supports_continuation = True if model_info.get("supports_reasoning", None) is not None and model_info["supports_reasoning"] is True: model_group_info.supports_reasoning = True diff --git a/litellm/types/router.py b/litellm/types/router.py index 41a986837da..4e3e4d92b3c 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1061,6 +1061,7 @@ class ModelGroupInfo(LiteLLMBaseModel): supports_vision: bool = Field(default=False) supports_web_search: bool = Field(default=False) supports_url_context: bool = Field(default=False) + supports_continuation: bool = Field(default=False) supports_reasoning: bool = Field(default=False) supports_function_calling: bool = Field(default=False) supports_fast_mode: bool = Field(default=False) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index bf69445fdd1..eeb0956a06b 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -227,6 +227,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_tool_search: bool | None supports_mid_conversation_system: bool | None supports_url_context: bool | None + supports_continuation: ReadOnly[bool | None] supports_none_reasoning_effort: bool | None supports_minimal_reasoning_effort: bool | None supports_low_reasoning_effort: bool | None @@ -4234,6 +4235,7 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( "supports_assistant_prefill", "supports_bedrock_native_structured_output", "supports_pdf_input", + "supports_continuation", "reasoning_modes", "reasoning_contexts", } diff --git a/litellm/utils.py b/litellm/utils.py index f03b1eb7ad9..55423a42260 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3187,6 +3187,14 @@ def supports_url_context(model: str, custom_llm_provider: str | None = None) -> ) +def supports_continuation(model: str, custom_llm_provider: str | None = None) -> bool: + return supports_factory( + model=model, + custom_llm_provider=custom_llm_provider, + key="supports_continuation", + ) + + def supports_native_streaming(model: str, custom_llm_provider: str | None) -> bool: """ Check if the given model supports native streaming and return a boolean value. @@ -7377,6 +7385,7 @@ def get_model_info_helper( ), supports_web_search=_model_info.get("supports_web_search", None), supports_url_context=_model_info.get("supports_url_context", None), + supports_continuation=_model_info.get("supports_continuation", None), supports_reasoning=_model_info.get("supports_reasoning", None), supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None), @@ -7536,6 +7545,7 @@ def get_model_info( supports_pdf_input: Optional[bool] supports_web_search: Optional[bool] supports_url_context: Optional[bool] + supports_continuation: Optional[bool] supports_reasoning: Optional[bool] Raises: Exception: If the model is not mapped yet. diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e6f65d6b416..8bfb8949890 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76813,6 +76813,7 @@ "supports_audio_input": true, "supports_audio_output": false, "supports_computer_use": true, + "supports_continuation": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, @@ -76872,6 +76873,7 @@ ], "supports_audio_input": true, "supports_computer_use": true, + "supports_continuation": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, @@ -76929,6 +76931,7 @@ "text" ], "supports_audio_input": true, + "supports_continuation": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index e4a2beb3a86..4e8ede5c446 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -845,6 +845,26 @@ def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]] ) +def _supports_continuation_rows(path: Path) -> frozenset[str]: + prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) + return frozenset( + key for key, row in prices.items() if isinstance(row, dict) and row.get("supports_continuation") is True + ) + + +def test_gemini_4_argon_google_rows_carry_supports_continuation_in_both_cost_map_copies() -> None: + main_rows: Final = _supports_continuation_rows(PRICES_PATH) + # long decode models: https://ai.google.dev/gemini-api/docs/generate-content/long-decode and + # https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/guides/gemini-4-argon (2026-11-02) + long_decode_rows: Final = { + "gemini/gemini-4-argon", + "vertex_ai/gemini-4-argon", + "vertex_ai/gemini-4-argon-cyber-permissive", + } + assert long_decode_rows <= main_rows, sorted(main_rows) + assert _supports_continuation_rows(BACKUP_PRICES_PATH) == main_rows + + @pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) def test_search_free_requests_rows_are_grounding_rows_with_one_allowance_per_group(path: Path) -> None: rows = _search_free_requests_rows(path) diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 4964dea498f..b46a57ad5f1 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -1497,6 +1497,51 @@ def test_set_model_group_info(model_list, user_facing_model_group_name): assert resp.model_group == user_facing_model_group_name +def test_set_model_group_info_reports_supports_continuation_when_any_deployment_has_it(): + router: Final = Router( + model_list=[ + { + "model_name": "long-decode", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "fake"}, + }, + { + "model_name": "long-decode", + "litellm_params": {"model": "openai/gpt-4.1-mini", "api_key": "fake"}, + "model_info": {"supports_continuation": True}, + }, + { + "model_name": "long-decode", + "litellm_params": {"model": "openai/gpt-4.1-nano", "api_key": "fake"}, + }, + { + "model_name": "plain", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "fake"}, + }, + ] + ) + long_decode: Final = router._set_model_group_info( + model_group="long-decode", user_facing_model_group_name="long-decode" + ) + plain: Final = router._set_model_group_info(model_group="plain", user_facing_model_group_name="plain") + assert long_decode is not None and long_decode.supports_continuation is True + assert plain is not None and plain.supports_continuation is False + + +@pytest.mark.parametrize("value", ["maybe", "", ["x"], "true", 1]) +def test_set_model_group_info_counts_only_a_boolean_true_supports_continuation(value: object) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "long-decode", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "fake"}, + "model_info": {"supports_continuation": value}, + }, + ] + ) + group: Final = router._set_model_group_info(model_group="long-decode", user_facing_model_group_name="long-decode") + assert group is not None and group.supports_continuation is False + + @pytest.mark.asyncio async def test_set_response_headers(model_list): """Test if the 'set_response_headers' function is working correctly""" diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 447a4dfb81d..96519a0acbe 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -585,7 +585,10 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): ) -@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params"]) +@pytest.mark.parametrize( + "flag", + ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params", "supports_continuation"], +) def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: """A deployment's capability override must not land on the shared ``{provider}/{model}`` key, or every sibling deployment of that model would inherit it.""" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 61825e463f8..a9b56568afb 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -476,6 +476,34 @@ def test_supports_video_input_reads_its_own_cost_map_flag( assert litellm.get_model_info(model)["supports_video_input"] is video_input +@pytest.mark.parametrize("continuation", [True, False]) +def test_supports_continuation_reads_its_own_cost_map_flag( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, continuation: bool +) -> None: + model: Final = "gemini/continuation-flag-model" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "gemini", + "mode": "chat", + "supports_continuation": continuation, + "supports_url_context": not continuation, + }, + ) + assert litellm.supports_continuation(model) is continuation + assert litellm.get_model_info(model).get("supports_continuation") is continuation + + +def test_supports_continuation_is_unset_when_the_cost_map_omits_it( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + model: Final = "gemini/no-continuation-flag-model" + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "gemini", "mode": "chat"}) + assert litellm.supports_continuation(model) is False + assert litellm.get_model_info(model).get("supports_continuation") is None + + def test_get_optional_params_image_gen(): from litellm.llms.azure.image_generation import AzureGPTImageGenerationConfig @@ -1192,6 +1220,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_bedrock_runtime_chat_completions_inline_reasoning": {"type": "boolean"}, "supports_bedrock_converse_tool_result_images": {"type": "boolean"}, "supports_url_context": {"type": "boolean"}, + "supports_continuation": {"type": "boolean"}, "supports_multimodal": {"type": "boolean"}, "uses_embed_content": {"type": "boolean"}, "supports_reasoning": {"type": "boolean"}, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 96d4c0e1bc4..3aa7c888e98 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -38948,6 +38948,11 @@ export interface components { * @default false */ supports_background_chat: boolean; + /** + * Supports Continuation + * @default false + */ + supports_continuation: boolean; /** * Supports Fast Mode * @default false