From 5d4848b3f9b43709335311875d194fb7f2ed0537 Mon Sep 17 00:00:00 2001 From: Chengxuan Wang Date: Wed, 17 Jun 2026 20:57:28 -0700 Subject: [PATCH] address review: flat-string fallback prefill check + snowflake registry flag Greptile findings: 1. build_mid_stream_continuation_messages now scans every entry of a flat string-format fallback list directly. get_fallback_model_group pops a single string mid-iteration, so a prefill-rejecting model at a non-first position was never capability-checked; since the continuation is built once and reused across all fallback hops, the chain still 400'd when it reached that model. Dict/standard format keeps using get_fallback_model_group (already returns the full group). 2. snowflake/claude-sonnet-4-6 gets supports_assistant_prefill:false in both the root map and the bundled backup (it existed only in root before; the backup is the offline/test source), completing the 4-6 sweep. Tests: +snowflake registry param, +flat-string non-first-position -> user continuation, +flat-string all-supporting -> legacy prefill. --- ...odel_prices_and_context_window_backup.json | 16 +++++++++ .../router_utils/fallback_event_handlers.py | 34 ++++++++++++------- model_prices_and_context_window.json | 1 + .../test_fallback_event_handlers.py | 31 +++++++++++++++++ 4 files changed, 70 insertions(+), 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7a800bf5693..a551cab1dd5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31306,6 +31306,22 @@ "mode": "chat", "supports_computer_use": true }, + "snowflake/claude-sonnet-4-6": { + "cache_read_input_token_cost": 0.0000003, + "input_cost_per_token": 0.000003, + "litellm_provider": "snowflake", + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 0.000015, + "supports_assistant_prefill": false, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true + }, "snowflake/deepseek-r1": { "litellm_provider": "snowflake", "max_input_tokens": 32768, diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index b6e59861590..2461f907bef 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -65,18 +65,28 @@ def build_mid_stream_continuation_messages( """ candidate_models: list[Optional[str]] = [model_group] if fallbacks is not None and model_group is not None: - try: - # Shallow copy: get_fallback_model_group POPS a matching entry from - # flat string-format fallback lists — mutating the live list here - # would silently drop one fallback target before the actual - # fallback execution runs. - fallback_model_group, _ = get_fallback_model_group( - fallbacks=list(fallbacks), model_group=model_group - ) - if fallback_model_group: - candidate_models.extend(fallback_model_group) - except Exception: - pass + if fallbacks and all(isinstance(f, str) for f in fallbacks): + # Flat string-format lists (["model-a", "model-b", ...]) are tried in + # order for every model_group. get_fallback_model_group surfaces only + # ONE entry from them — it pops a single string mid-iteration and + # leaves the rest hidden — so a prefill-rejecting model at a non-first + # position would slip through, and the once-built continuation that is + # reused across every hop would 400 when the chain reaches it. Check + # every entry directly instead. + candidate_models.extend(fallbacks) + else: + try: + # list() copy: get_fallback_model_group POPS string entries from a + # mixed fallbacks list while iterating — mutating the live list + # here would silently drop a fallback target before the actual + # fallback execution runs. + fallback_model_group, _ = get_fallback_model_group( + fallbacks=list(fallbacks), model_group=model_group + ) + if fallback_model_group: + candidate_models.extend(fallback_model_group) + except Exception: + pass if any(_prefill_explicitly_unsupported(m) for m in candidate_models): return messages + [ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 19115b9c13a..02846ad7bf9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -43019,6 +43019,7 @@ "cache_read_input_token_cost": 0.0000003, "litellm_provider": "snowflake", "mode": "chat", + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_vision": true, "supports_prompt_caching": true, diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index d7e5e19edda..4d4ca491742 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -32,6 +32,7 @@ def local_model_cost_map(monkeypatch): litellm.model_cost = original_model_cost litellm.get_model_info.cache_clear() + MESSAGES = [{"role": "user", "content": "Plan my trip to Tokyo"}] PARTIAL = "Here are the best flight options I found so" @@ -66,6 +67,7 @@ def _assert_legacy_prefill(result): "vertex_ai/claude-sonnet-4-6", "claude-opus-4-6", "openrouter/anthropic/claude-sonnet-4.6", # dot-variant registry key + "snowflake/claude-sonnet-4-6", # snowflake provider entry ], ) def test_prefill_rejecting_models_get_user_continuation(model): @@ -143,3 +145,32 @@ def test_fallbacks_list_is_not_mutated_by_capability_check(): fallbacks=dict_format_fallbacks, ) assert dict_format_fallbacks == [{"gpt-4": ["claude-sonnet-4-6"]}] + + +def test_flat_string_fallback_prefill_rejecter_at_non_first_position(): + """Flat string-format lists are tried in order for every model group, but + get_fallback_model_group surfaces only one entry. A prefill-rejecting model + anywhere in the list — not just position 0 — must flip the reused + continuation to the user-message form.""" + result = build_mid_stream_continuation_messages( + messages=MESSAGES, + generated_content=PARTIAL, + model_group="gpt-4", + fallbacks=["gpt-4o", "claude-sonnet-4-6", "gpt-3.5-turbo"], + ) + assert len(result) == 2 + assert result[1]["role"] == "user" + assert PARTIAL in result[1]["content"] + assert all(m.get("prefix") is not True for m in result) + + +def test_flat_string_fallback_all_prefill_supporting_keeps_legacy(): + """A flat string-format list with no prefill-rejecting model keeps the + legacy prefill resume.""" + result = build_mid_stream_continuation_messages( + messages=MESSAGES, + generated_content=PARTIAL, + model_group="gpt-4", + fallbacks=["gpt-4o", "gpt-3.5-turbo"], + ) + _assert_legacy_prefill(result)