diff --git a/litellm/router.py b/litellm/router.py index bb8485e0ef5..0b75a422a45 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -478,6 +478,14 @@ _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) _CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) +def _extend_trailing_text_block(blocks: Sequence[Mapping[str, object]], text: str) -> Sequence[Mapping[str, object]]: + """Continue the last text block in place so the prefill reads exactly as the text the caller already received.""" + last: Final = blocks[-1] if blocks else None + if last is not None and last.get("type") == "text" and isinstance(last_text := last.get("text"), str): + return [*blocks[:-1], {**last, "text": f"{last_text}{text}"}] + return [*blocks, {"type": "text", "text": text}] + + def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else () @@ -3260,7 +3268,7 @@ class Router: blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(content) except ValidationError: return None - extended: Final = [*blocks, {"type": "text", "text": generated_content}] + extended: Final = _extend_trailing_text_block(blocks, generated_content) return [*messages[:-1], {**last, "content": extended, "prefix": True}] @staticmethod diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3d592737e1e..86778b54158 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2902,17 +2902,19 @@ def test_build_completion_continuation_input_folds_into_trailing_plain_assistant def test_build_completion_continuation_input_keeps_structured_assistant_content(): - """Content blocks on a trailing assistant turn stay blocks: the partial lands as one more text - block instead of a stringified list, and a content shape that cannot be extended declines.""" + """Content blocks on a trailing assistant turn stay blocks: the partial continues the last text + block in place (a block boundary would let a provider drop the space between "Sure, " and "here"), + lands as a new text block only after a non-text block, and a content shape that cannot be extended declines.""" + from litellm.router import _extend_trailing_text_block + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": [{"type": "text", "text": "Sure, "}]}] built = litellm.Router._build_completion_continuation_input(messages, "here it is") assert built is not None assert [m["role"] for m in built] == ["user", "assistant"] - assert built[-1] == { - "role": "assistant", - "content": [{"type": "text", "text": "Sure, "}, {"type": "text", "text": "here it is"}], - "prefix": True, - } + assert built[-1] == {"role": "assistant", "content": [{"type": "text", "text": "Sure, here it is"}], "prefix": True} + image_block = {"type": "image_url", "image_url": {"url": "https://example.test/a.png"}} + assert _extend_trailing_text_block([image_block], "here it is") == [image_block, {"type": "text", "text": "here it is"}] + assert _extend_trailing_text_block([], "here it is") == [{"type": "text", "text": "here it is"}] assert litellm.Router._build_completion_continuation_input([{"role": "assistant", "content": 42}], "x") is None