diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 0c9702ff884..a9ef06c3b4f 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -42,7 +42,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_items from litellm.litellm_core_utils.model_response_utils import ( is_model_response_stream_empty, ) -from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject +from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject, drop_token_ids from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer from litellm.types.llms.base import LiteLLMBaseModel @@ -195,6 +195,14 @@ def _choice_fields_chunk( return model_response +def _without_token_ids_under_post_call_rules(chunk: ModelResponseStream | None) -> ModelResponseStream | None: + if chunk is None or not litellm.post_call_rules: + return chunk + for choice in chunk.choices: + drop_token_ids(choice) + return chunk + + def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool: return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields) @@ -1248,8 +1256,10 @@ class CustomStreamWrapper: ) is_chunk_non_empty: Final = self.is_chunk_non_empty(completion_obj, model_response, response_obj) - upstream_chunk_choice_fields: Final[Mapping[str, object]] = response_obj.get( - "chunk_choice_fields", _NO_UPSTREAM_CHOICE_FIELDS + upstream_chunk_choice_fields: Final[Mapping[str, object]] = ( + _NO_UPSTREAM_CHOICE_FIELDS + if litellm.post_call_rules + else response_obj.get("chunk_choice_fields", _NO_UPSTREAM_CHOICE_FIELDS) ) chunk_choice_fields: Final = self._take_carried_chunk_choice_fields(response_obj, upstream_chunk_choice_fields) sibling_finish_reason: Final[str | None] = response_obj.get("sibling_finish_reason") @@ -1810,7 +1820,7 @@ class CustomStreamWrapper: completion_obj=completion_obj, ) if isinstance(dispatch_result, _ProviderChunkEarlyReturn): - return dispatch_result.value + return _without_token_ids_under_post_call_rules(dispatch_result.value) response_obj = dispatch_result.response_obj model_response.model = self._served_openrouter_model() or self.model @@ -1900,7 +1910,7 @@ class CustomStreamWrapper: response_obj=response_obj, plain_delta_created=plain_delta_created, ) - return result + return _without_token_ids_under_post_call_rules(result) except StopIteration: raise StopIteration diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index b4ebdd4266e..cf8de9c40f5 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -6986,9 +6986,9 @@ def _joined_token_ids(choices: Sequence[Mapping[str, object]]) -> tuple[int, ... return tuple(chain.from_iterable(_TOKEN_IDS.validate_python(choice.get("token_ids")) or () for choice in choices)) -def _hosted_vllm_stream_choices( +def _hosted_vllm_stream( logging_obj: Logging, upstream_choices: Sequence[Mapping[str, object]] -) -> tuple[dict[str, object], ...]: +) -> CustomStreamWrapper: sse_lines: Final = ( *( "data: " + json.dumps({"id": "chatcmpl-1", "object": "chat.completion.chunk", "model": "m", "choices": [c]}) @@ -6999,14 +6999,17 @@ def _hosted_vllm_stream_choices( completion_stream: Final = HostedVLLMChatConfig().get_model_response_iterator( streaming_response=iter(sse_lines), sync_stream=True, json_mode=False ) - return tuple( - chunk.choices[0].model_dump() - for chunk in CustomStreamWrapper( - completion_stream=completion_stream, model="m", custom_llm_provider="hosted_vllm", logging_obj=logging_obj - ) + return CustomStreamWrapper( + completion_stream=completion_stream, model="m", custom_llm_provider="hosted_vllm", logging_obj=logging_obj ) +def _hosted_vllm_stream_choices( + logging_obj: Logging, upstream_choices: Sequence[Mapping[str, object]] +) -> tuple[dict[str, object], ...]: + return tuple(chunk.choices[0].model_dump() for chunk in _hosted_vllm_stream(logging_obj, upstream_choices)) + + @pytest.mark.parametrize("stop_reason", [",5", 151645]) @pytest.mark.parametrize("final_content", [",3", ""], ids=["content-then-finish", "finish-only"]) def test_hosted_vllm_stream_keeps_stop_reason_and_every_token_id( @@ -7074,6 +7077,37 @@ def test_hosted_vllm_stream_keeps_held_back_token_ids_on_their_own_choice(loggin ] +def _rejects_a_two(text: str) -> bool: + return "2" not in text + + +def _choices_sent_before_rule_failure(stream: CustomStreamWrapper) -> Iterator[dict[str, object]]: + try: + yield from (chunk.choices[0].model_dump() for chunk in stream) + except MidStreamFallbackError: + return + + +def test_hosted_vllm_stream_sends_no_token_ids_ahead_of_a_failing_post_call_rule( + logging_obj: Logging, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "post_call_rules", [_rejects_a_two]) + stream: Final = _hosted_vllm_stream( + logging_obj, + ( + {"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}, + {"index": 0, "delta": {"content": ""}, "finish_reason": None, "token_ids": [16]}, + {"index": 0, "delta": {"content": "1"}, "finish_reason": None, "token_ids": [11, 17]}, + {"index": 0, "delta": {"content": ",2"}, "finish_reason": "stop", "stop_reason": None, "token_ids": [20]}, + ), + ) + + assert [ + (_DELTA.validate_python(choice["delta"]).get("content"), choice.get("token_ids")) + for choice in _choices_sent_before_rule_failure(stream) + ] == [("", None), ("1", None)] + + def _held_back_stop_string_stream(token_ids: Mapping[str, None]) -> tuple[Mapping[str, object], ...]: return ( {"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None},