mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(streaming): drop token ids from streamed chunks while post_call_rules run
vLLM holds text back behind a possible stop string but sends its token ids at once, so an SDK stream sent ids before litellm.post_call_rules checked their text. While those rules are set, streamed choices drop token_ids and chunks that carry only ids are skipped, as the proxy already does ahead of a per-chunk check
This commit is contained in:
parent
90865c1f2b
commit
41a51d0b9e
2 changed files with 56 additions and 12 deletions
|
|
@ -42,7 +42,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_items
|
|||
from litellm.litellm_core_utils.model_response_utils import (
|
||||
is_model_response_stream_empty,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
|
||||
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject, drop_token_ids
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -195,6 +195,14 @@ def _choice_fields_chunk(
|
|||
return model_response
|
||||
|
||||
|
||||
def _without_token_ids_under_post_call_rules(chunk: ModelResponseStream | None) -> ModelResponseStream | None:
|
||||
if chunk is None or not litellm.post_call_rules:
|
||||
return chunk
|
||||
for choice in chunk.choices:
|
||||
drop_token_ids(choice)
|
||||
return chunk
|
||||
|
||||
|
||||
def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool:
|
||||
return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields)
|
||||
|
||||
|
|
@ -1248,8 +1256,10 @@ class CustomStreamWrapper:
|
|||
)
|
||||
|
||||
is_chunk_non_empty: Final = self.is_chunk_non_empty(completion_obj, model_response, response_obj)
|
||||
upstream_chunk_choice_fields: Final[Mapping[str, object]] = response_obj.get(
|
||||
"chunk_choice_fields", _NO_UPSTREAM_CHOICE_FIELDS
|
||||
upstream_chunk_choice_fields: Final[Mapping[str, object]] = (
|
||||
_NO_UPSTREAM_CHOICE_FIELDS
|
||||
if litellm.post_call_rules
|
||||
else response_obj.get("chunk_choice_fields", _NO_UPSTREAM_CHOICE_FIELDS)
|
||||
)
|
||||
chunk_choice_fields: Final = self._take_carried_chunk_choice_fields(response_obj, upstream_chunk_choice_fields)
|
||||
sibling_finish_reason: Final[str | None] = response_obj.get("sibling_finish_reason")
|
||||
|
|
@ -1810,7 +1820,7 @@ class CustomStreamWrapper:
|
|||
completion_obj=completion_obj,
|
||||
)
|
||||
if isinstance(dispatch_result, _ProviderChunkEarlyReturn):
|
||||
return dispatch_result.value
|
||||
return _without_token_ids_under_post_call_rules(dispatch_result.value)
|
||||
response_obj = dispatch_result.response_obj
|
||||
|
||||
model_response.model = self._served_openrouter_model() or self.model
|
||||
|
|
@ -1900,7 +1910,7 @@ class CustomStreamWrapper:
|
|||
response_obj=response_obj,
|
||||
plain_delta_created=plain_delta_created,
|
||||
)
|
||||
return result
|
||||
return _without_token_ids_under_post_call_rules(result)
|
||||
|
||||
except StopIteration:
|
||||
raise StopIteration
|
||||
|
|
|
|||
|
|
@ -6986,9 +6986,9 @@ def _joined_token_ids(choices: Sequence[Mapping[str, object]]) -> tuple[int, ...
|
|||
return tuple(chain.from_iterable(_TOKEN_IDS.validate_python(choice.get("token_ids")) or () for choice in choices))
|
||||
|
||||
|
||||
def _hosted_vllm_stream_choices(
|
||||
def _hosted_vllm_stream(
|
||||
logging_obj: Logging, upstream_choices: Sequence[Mapping[str, object]]
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
) -> CustomStreamWrapper:
|
||||
sse_lines: Final = (
|
||||
*(
|
||||
"data: " + json.dumps({"id": "chatcmpl-1", "object": "chat.completion.chunk", "model": "m", "choices": [c]})
|
||||
|
|
@ -6999,14 +6999,17 @@ def _hosted_vllm_stream_choices(
|
|||
completion_stream: Final = HostedVLLMChatConfig().get_model_response_iterator(
|
||||
streaming_response=iter(sse_lines), sync_stream=True, json_mode=False
|
||||
)
|
||||
return tuple(
|
||||
chunk.choices[0].model_dump()
|
||||
for chunk in CustomStreamWrapper(
|
||||
completion_stream=completion_stream, model="m", custom_llm_provider="hosted_vllm", logging_obj=logging_obj
|
||||
)
|
||||
return CustomStreamWrapper(
|
||||
completion_stream=completion_stream, model="m", custom_llm_provider="hosted_vllm", logging_obj=logging_obj
|
||||
)
|
||||
|
||||
|
||||
def _hosted_vllm_stream_choices(
|
||||
logging_obj: Logging, upstream_choices: Sequence[Mapping[str, object]]
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
return tuple(chunk.choices[0].model_dump() for chunk in _hosted_vllm_stream(logging_obj, upstream_choices))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stop_reason", [",5", 151645])
|
||||
@pytest.mark.parametrize("final_content", [",3", ""], ids=["content-then-finish", "finish-only"])
|
||||
def test_hosted_vllm_stream_keeps_stop_reason_and_every_token_id(
|
||||
|
|
@ -7074,6 +7077,37 @@ def test_hosted_vllm_stream_keeps_held_back_token_ids_on_their_own_choice(loggin
|
|||
]
|
||||
|
||||
|
||||
def _rejects_a_two(text: str) -> bool:
|
||||
return "2" not in text
|
||||
|
||||
|
||||
def _choices_sent_before_rule_failure(stream: CustomStreamWrapper) -> Iterator[dict[str, object]]:
|
||||
try:
|
||||
yield from (chunk.choices[0].model_dump() for chunk in stream)
|
||||
except MidStreamFallbackError:
|
||||
return
|
||||
|
||||
|
||||
def test_hosted_vllm_stream_sends_no_token_ids_ahead_of_a_failing_post_call_rule(
|
||||
logging_obj: Logging, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "post_call_rules", [_rejects_a_two])
|
||||
stream: Final = _hosted_vllm_stream(
|
||||
logging_obj,
|
||||
(
|
||||
{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None},
|
||||
{"index": 0, "delta": {"content": ""}, "finish_reason": None, "token_ids": [16]},
|
||||
{"index": 0, "delta": {"content": "1"}, "finish_reason": None, "token_ids": [11, 17]},
|
||||
{"index": 0, "delta": {"content": ",2"}, "finish_reason": "stop", "stop_reason": None, "token_ids": [20]},
|
||||
),
|
||||
)
|
||||
|
||||
assert [
|
||||
(_DELTA.validate_python(choice["delta"]).get("content"), choice.get("token_ids"))
|
||||
for choice in _choices_sent_before_rule_failure(stream)
|
||||
] == [("", None), ("1", None)]
|
||||
|
||||
|
||||
def _held_back_stop_string_stream(token_ids: Mapping[str, None]) -> tuple[Mapping[str, object], ...]:
|
||||
return (
|
||||
{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue