mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-11 03:38:38 +00:00
fix(guardrails): keep the response verdict for explicit logging_only_scope=both on input-scan failure
Greptile P1: an explicitly configured both-direction observer asked for a verdict on each direction, so a failed request scan must not silently drop the response verdict. The implicit default (logging_only_scope None) keeps the abort semantics of a logging_only hook whose scan raised, which is the base behavior the earlier fix restored. Also drops two test comments that restated their assertions (Greptile P2).
This commit is contained in:
parent
7d293a6e20
commit
5830494d43
3 changed files with 32 additions and 5 deletions
|
|
@ -1038,7 +1038,18 @@ class CustomGuardrail(CustomLogger):
|
|||
"metadata": scratch_metadata,
|
||||
}
|
||||
if self.logging_only_scope != "output":
|
||||
await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self)
|
||||
if self.logging_only_scope == "both":
|
||||
# An explicitly configured "both" observer asked for a verdict on
|
||||
# each direction, so a failed request scan must not silently drop
|
||||
# the response verdict. The implicit default (logging_only_scope
|
||||
# None) keeps the abort semantics of a logging_only hook whose
|
||||
# scan raised.
|
||||
try:
|
||||
await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self)
|
||||
except Exception as e: # noqa: BLE001 # one direction's scan failure must not drop the other direction's verdict
|
||||
verbose_logger.warning("Guardrail %s: logging_only scan raised: %s", self.guardrail_name, e)
|
||||
else:
|
||||
await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self)
|
||||
if response is None or self.logging_only_scope == "input":
|
||||
return
|
||||
await output_translation.process_output_response(
|
||||
|
|
|
|||
|
|
@ -2654,9 +2654,6 @@ class TestLoggingOnlyApplyGuardrail:
|
|||
|
||||
out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value)
|
||||
|
||||
# A raising input scan aborts the hook: the response scan must not run,
|
||||
# and exactly one intervened verdict is recorded (base semantics for a
|
||||
# guardrail that never selected a logging_only_scope).
|
||||
assert guardrail.calls == [("request", ["flagged content"])]
|
||||
entries = out_kwargs["standard_logging_object"]["guardrail_information"]
|
||||
assert [e["guardrail_status"] for e in entries] == ["guardrail_intervened"]
|
||||
|
|
@ -2680,6 +2677,26 @@ class TestLoggingOnlyApplyGuardrail:
|
|||
entries = out_kwargs["standard_logging_object"]["guardrail_information"]
|
||||
assert [e["guardrail_status"] for e in entries] == ["guardrail_failed_to_respond"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_scan_error_does_not_drop_the_response_scan_for_explicit_both_scope(self):
|
||||
class _FailingBothObserver(_ApplyOnlyObserver):
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
|
||||
self.calls.append((input_type, list(inputs.get("texts") or [])))
|
||||
if input_type == "request":
|
||||
raise RuntimeError("guardrail service unavailable")
|
||||
return GenericGuardrailAPIInputs(texts=[])
|
||||
|
||||
guardrail = _FailingBothObserver()
|
||||
guardrail.logging_only_scope = "both"
|
||||
kwargs, response = _logged_call([{"role": "user", "content": "hello there"}])
|
||||
|
||||
out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value)
|
||||
|
||||
assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])]
|
||||
entries = out_kwargs["standard_logging_object"]["guardrail_information"]
|
||||
assert [e["guardrail_status"] for e in entries] == ["guardrail_failed_to_respond", "success"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_type_without_translation_is_skipped(self):
|
||||
guardrail = _ApplyOnlyObserver()
|
||||
|
|
|
|||
|
|
@ -428,7 +428,6 @@ def test_sync_guardrail_from_db_reject_flag_keeps_callback_order_on_noop_update(
|
|||
handler.initialize_guardrail(guardrail=_mode_following_db_row("123", "pre_call"), source="db")
|
||||
original = handler.guardrail_id_to_custom_guardrail["123"]
|
||||
assert original is not None
|
||||
# Park another callback after the guardrail so a re-append would be visible.
|
||||
litellm.callbacks.append(sentinel)
|
||||
index_before = litellm.callbacks.index(original)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue