scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

fix(guardrails): default optional streaming knobs to None

Browse source 
Non-None Pydantic defaults on GenericGuardrailAPIOptionalParams made
_get_config_value treat unset nested fields as explicit values, which
shadowed top-level litellm_params streaming flags whenever any other
optional_params key was present. Real defaults stay in the constructor.
This commit is contained in:
Marton Schneider 2026-06-21 15:30:40 +02:00
parent 069c79d21d
commit 3a20cd7479
2 changed files with 79 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py
View file

@ -40,21 +40,25 @@ class GenericGuardrailAPIOptionalParams(BaseModel):
)
streaming_end_of_stream_only: Optional[bool] = Field(
default=False,
default=None,
description=(
"If False (default), the guardrail runs on sampled chunks during the stream "
"at the cadence set by streaming_sampling_rate, and an in-flight BLOCKED "
"stops further chunks from streaming. If True, the guardrail runs once at "
"end of stream over the assembled response; lower cost and latency, but "
"flagged content has already streamed to the client before the terminal block."
"If False (default when unset), the guardrail runs on sampled chunks during "
"the stream at the cadence set by streaming_sampling_rate, and an in-flight "
"BLOCKED stops further chunks from streaming. If True, the guardrail runs "
"once at end of stream over the assembled response; lower cost and latency, "
"but flagged content has already streamed to the client before the terminal "
"block. Defaults are applied in GenericGuardrailAPI.__init__ when None so "
"unset optional_params does not shadow top-level litellm_params."
),
)
streaming_sampling_rate: Optional[int] = Field(
default=5,
default=None,
description=(
"When streaming_end_of_stream_only is False, the guardrail runs every Nth "
"streamed chunk. Ignored when streaming_end_of_stream_only is True."
"streamed chunk. Ignored when streaming_end_of_stream_only is True. "
"Defaults to 5 in GenericGuardrailAPI.__init__ when None so unset "
"optional_params does not shadow top-level litellm_params."
),
)

67
tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py
View file

@ -1124,6 +1124,73 @@ class TestGenericGuardrailAPIStreamingConfig:
assert guardrail.streaming_end_of_stream_only is False
assert guardrail.streaming_sampling_rate == 3
def test_initialize_guardrail_optional_params_defaults_do_not_shadow_top_level(
self,
):
"""Top-level streaming knobs win when optional_params only carries siblings."""
from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
initialize_guardrail,
)
from litellm.types.guardrails import LitellmParams
from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
GenericGuardrailAPIOptionalParams,
)
litellm_params = LitellmParams(
guardrail="generic_guardrail_api",
mode="post_call",
api_base="https://api.test.guardrail.com",
default_on=False,
)
litellm_params.streaming_end_of_stream_only = True # type: ignore[attr-defined]
litellm_params.streaming_sampling_rate = 2 # type: ignore[attr-defined]
# Sibling optional_params only; streaming fields stay at Pydantic default None.
litellm_params.optional_params = GenericGuardrailAPIOptionalParams( # type: ignore[attr-defined]
additional_provider_specific_params={"tenant": "acme"},
)
guardrail_config = {"guardrail_name": "test-generic-streaming-mixed"}
with patch(
"litellm.logging_callback_manager.add_litellm_callback"
):
guardrail = initialize_guardrail(litellm_params, guardrail_config)
assert guardrail.streaming_end_of_stream_only is True
assert guardrail.streaming_sampling_rate == 2
def test_initialize_guardrail_explicit_optional_params_streaming_wins(self):
from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
initialize_guardrail,
)
from litellm.types.guardrails import LitellmParams
from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
GenericGuardrailAPIOptionalParams,
)
litellm_params = LitellmParams(
guardrail="generic_guardrail_api",
mode="post_call",
api_base="https://api.test.guardrail.com",
default_on=False,
)
litellm_params.streaming_end_of_stream_only = False # type: ignore[attr-defined]
litellm_params.streaming_sampling_rate = 9 # type: ignore[attr-defined]
litellm_params.optional_params = GenericGuardrailAPIOptionalParams( # type: ignore[attr-defined]
streaming_end_of_stream_only=True,
streaming_sampling_rate=1,
)
guardrail_config = {"guardrail_name": "test-generic-streaming-nested-wins"}
with patch(
"litellm.logging_callback_manager.add_litellm_callback"
):
guardrail = initialize_guardrail(litellm_params, guardrail_config)
assert guardrail.streaming_end_of_stream_only is True
assert guardrail.streaming_sampling_rate == 1
class TestGenericGuardrailAPIStreamingViaUnified:
"""Streaming output checks routed through UnifiedLLMGuardrails."""