diff --git a/litellm/router.py b/litellm/router.py index 26ef92edc57..6d8d4dfd032 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2242,6 +2242,7 @@ class Router: messages=messages, generated_content=e.generated_content, model_group=model_group, + fallbacks=fallbacks, ) ) self._update_kwargs_before_fallbacks( @@ -2801,6 +2802,7 @@ class Router: messages=messages, generated_content=e.generated_content, model_group=model_group, + fallbacks=fallbacks, ) ) router_self._update_kwargs_before_fallbacks( diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 3bc609a7333..09377f40782 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -43,6 +43,7 @@ def build_mid_stream_continuation_messages( messages: List[Any], generated_content: str, model_group: Optional[str], + fallbacks: Optional[List[Any]] = None, ) -> List[Any]: """Build the message list a mid-stream fallback uses to resume an interrupted stream. @@ -56,8 +57,24 @@ def build_mid_stream_continuation_messages( a trailing USER message instead — the continuation pattern Anthropic's migration guide documents: https://platform.claude.com/docs/en/about-claude/models/migration-guide + + The same continuation messages are sent to every deployment the fallback + chain tries, so the safe pattern engages when the primary model OR any + configured fallback target for this model group is explicitly marked as + not supporting prefill. """ - if _prefill_explicitly_unsupported(model_group): + candidate_models: List[Optional[str]] = [model_group] + if fallbacks is not None and model_group is not None: + try: + fallback_model_group, _ = get_fallback_model_group( + fallbacks=fallbacks, model_group=model_group + ) + if fallback_model_group: + candidate_models.extend(fallback_model_group) + except Exception: + pass + + if any(_prefill_explicitly_unsupported(m) for m in candidate_models): return messages + [ { "role": "user", diff --git a/litellm/utils.py b/litellm/utils.py index a1fb87e75ef..916260cab5a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2448,30 +2448,6 @@ def supports_url_context(model: str, custom_llm_provider: Optional[str] = None) ) -def supports_assistant_prefill( - model: str, custom_llm_provider: Optional[str] = None -) -> bool: - """ - Check if the given model supports assistant message prefill and return a boolean value. - - Anthropic removed assistant prefill starting with Claude Sonnet 4.6 / Opus 4.6 — - a prefilled assistant message returns a 400 error on those models. - https://platform.claude.com/docs/en/about-claude/models/migration-guide - - Parameters: - model (str): The model name to be checked. - custom_llm_provider (Optional[str]): The provider to be checked. - - Returns: - bool: True if the model supports assistant prefill, False otherwise (including unknown models). - """ - return _supports_factory( - model=model, - custom_llm_provider=custom_llm_provider, - key="supports_assistant_prefill", - ) - - def supports_native_streaming(model: str, custom_llm_provider: Optional[str]) -> bool: """ Check if the given model supports native streaming and return a boolean value. diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index e65de229a5d..d83e8c0b551 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -93,3 +93,29 @@ def test_other_models_keep_legacy_prefill_resume(model): """Anything not explicitly marked supports_assistant_prefill=false keeps the pre-existing prefill-resume behavior (back-compat).""" _assert_legacy_prefill(_build(model)) + + +def test_prefill_rejecting_fallback_target_gets_user_continuation(): + """The same continuation messages go to every fallback target — a primary + that supports prefill must still use the user continuation when any + configured fallback target rejects it (e.g. claude-3-5 → claude-sonnet-4-6).""" + result = build_mid_stream_continuation_messages( + messages=MESSAGES, + generated_content=PARTIAL, + model_group="claude-3-5-sonnet-20241022", + fallbacks=[{"claude-3-5-sonnet-20241022": ["claude-sonnet-4-6"]}], + ) + assert len(result) == 2 + assert result[1]["role"] == "user" + assert PARTIAL in result[1]["content"] + + +def test_unrelated_fallback_groups_do_not_affect_prefill(): + """Fallback config for OTHER model groups must not flip this group's behavior.""" + result = build_mid_stream_continuation_messages( + messages=MESSAGES, + generated_content=PARTIAL, + model_group="gpt-4", + fallbacks=[{"some-other-model": ["claude-sonnet-4-6"]}], + ) + _assert_legacy_prefill(result) diff --git a/tests/test_litellm/test_claude_sonnet_4_6_config.py b/tests/test_litellm/test_claude_sonnet_4_6_config.py index 27023d4ee6d..7c89cdfaf99 100644 --- a/tests/test_litellm/test_claude_sonnet_4_6_config.py +++ b/tests/test_litellm/test_claude_sonnet_4_6_config.py @@ -48,7 +48,9 @@ def test_bedrock_sonnet_4_6_region_prefixes(): assert model_info.get("supports_prompt_caching") is True assert model_info.get("supports_response_schema") is True assert model_info.get("supports_pdf_input") is True - assert model_info.get("supports_assistant_prefill") is True + # Assistant prefill was removed in Sonnet 4.6 — a prefilled assistant + # message returns a 400 on every platform (Anthropic migration guide). + assert model_info.get("supports_assistant_prefill") is False assert model_info.get("supports_reasoning") is True