fix(streaming): drop token ids from streamed chunks while post_call_rules run

vLLM holds text back behind a possible stop string but sends its token ids at once, so an SDK stream sent ids
before litellm.post_call_rules checked their text. While those rules are set, streamed choices drop token_ids
and chunks that carry only ids are skipped, as the proxy already does ahead of a per-chunk check
This commit is contained in:
mateo-berri 2026-11-02 12:00:27 -08:00
parent 90865c1f2b
commit 41a51d0b9e
2 changed files with 56 additions and 12 deletions

View file

@ -42,7 +42,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_items
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject, drop_token_ids
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.base import LiteLLMBaseModel
@ -195,6 +195,14 @@ def _choice_fields_chunk(
return model_response
def _without_token_ids_under_post_call_rules(chunk: ModelResponseStream | None) -> ModelResponseStream | None:
if chunk is None or not litellm.post_call_rules:
return chunk
for choice in chunk.choices:
drop_token_ids(choice)
return chunk
def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool:
return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields)
@ -1248,8 +1256,10 @@ class CustomStreamWrapper:
)
is_chunk_non_empty: Final = self.is_chunk_non_empty(completion_obj, model_response, response_obj)
upstream_chunk_choice_fields: Final[Mapping[str, object]] = response_obj.get(
"chunk_choice_fields", _NO_UPSTREAM_CHOICE_FIELDS
upstream_chunk_choice_fields: Final[Mapping[str, object]] = (
_NO_UPSTREAM_CHOICE_FIELDS
if litellm.post_call_rules
else response_obj.get("chunk_choice_fields", _NO_UPSTREAM_CHOICE_FIELDS)
)
chunk_choice_fields: Final = self._take_carried_chunk_choice_fields(response_obj, upstream_chunk_choice_fields)
sibling_finish_reason: Final[str | None] = response_obj.get("sibling_finish_reason")
@ -1810,7 +1820,7 @@ class CustomStreamWrapper:
completion_obj=completion_obj,
)
if isinstance(dispatch_result, _ProviderChunkEarlyReturn):
return dispatch_result.value
return _without_token_ids_under_post_call_rules(dispatch_result.value)
response_obj = dispatch_result.response_obj
model_response.model = self._served_openrouter_model() or self.model
@ -1900,7 +1910,7 @@ class CustomStreamWrapper:
response_obj=response_obj,
plain_delta_created=plain_delta_created,
)
return result
return _without_token_ids_under_post_call_rules(result)
except StopIteration:
raise StopIteration

View file

@ -6986,9 +6986,9 @@ def _joined_token_ids(choices: Sequence[Mapping[str, object]]) -> tuple[int, ...
return tuple(chain.from_iterable(_TOKEN_IDS.validate_python(choice.get("token_ids")) or () for choice in choices))
def _hosted_vllm_stream_choices(
def _hosted_vllm_stream(
logging_obj: Logging, upstream_choices: Sequence[Mapping[str, object]]
) -> tuple[dict[str, object], ...]:
) -> CustomStreamWrapper:
sse_lines: Final = (
*(
"data: " + json.dumps({"id": "chatcmpl-1", "object": "chat.completion.chunk", "model": "m", "choices": [c]})
@ -6999,14 +6999,17 @@ def _hosted_vllm_stream_choices(
completion_stream: Final = HostedVLLMChatConfig().get_model_response_iterator(
streaming_response=iter(sse_lines), sync_stream=True, json_mode=False
)
return tuple(
chunk.choices[0].model_dump()
for chunk in CustomStreamWrapper(
completion_stream=completion_stream, model="m", custom_llm_provider="hosted_vllm", logging_obj=logging_obj
)
return CustomStreamWrapper(
completion_stream=completion_stream, model="m", custom_llm_provider="hosted_vllm", logging_obj=logging_obj
)
def _hosted_vllm_stream_choices(
logging_obj: Logging, upstream_choices: Sequence[Mapping[str, object]]
) -> tuple[dict[str, object], ...]:
return tuple(chunk.choices[0].model_dump() for chunk in _hosted_vllm_stream(logging_obj, upstream_choices))
@pytest.mark.parametrize("stop_reason", [",5", 151645])
@pytest.mark.parametrize("final_content", [",3", ""], ids=["content-then-finish", "finish-only"])
def test_hosted_vllm_stream_keeps_stop_reason_and_every_token_id(
@ -7074,6 +7077,37 @@ def test_hosted_vllm_stream_keeps_held_back_token_ids_on_their_own_choice(loggin
]
def _rejects_a_two(text: str) -> bool:
return "2" not in text
def _choices_sent_before_rule_failure(stream: CustomStreamWrapper) -> Iterator[dict[str, object]]:
try:
yield from (chunk.choices[0].model_dump() for chunk in stream)
except MidStreamFallbackError:
return
def test_hosted_vllm_stream_sends_no_token_ids_ahead_of_a_failing_post_call_rule(
logging_obj: Logging, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "post_call_rules", [_rejects_a_two])
stream: Final = _hosted_vllm_stream(
logging_obj,
(
{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None},
{"index": 0, "delta": {"content": ""}, "finish_reason": None, "token_ids": [16]},
{"index": 0, "delta": {"content": "1"}, "finish_reason": None, "token_ids": [11, 17]},
{"index": 0, "delta": {"content": ",2"}, "finish_reason": "stop", "stop_reason": None, "token_ids": [20]},
),
)
assert [
(_DELTA.validate_python(choice["delta"]).get("content"), choice.get("token_ids"))
for choice in _choices_sent_before_rule_failure(stream)
] == [("", None), ("1", None)]
def _held_back_stop_string_stream(token_ids: Mapping[str, None]) -> tuple[Mapping[str, object], ...]:
return (
{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None},