mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(guardrails): mask the supabase failure log and redact when a sync mask returns no pair
This commit is contained in:
parent
0232e11705
commit
83e74b9ec8
2 changed files with 66 additions and 4 deletions
|
|
@ -4745,13 +4745,15 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if not _masks_synchronously(guardrail):
|
||||
return None
|
||||
try:
|
||||
hooked: Final = guardrail.logging_hook(
|
||||
kwargs=_failure_log_hook_kwargs(details), result=None, call_type=self._logging_hook_call_type
|
||||
hooked_details: Final = _METADATA_MAPPING.validate_python(
|
||||
guardrail.logging_hook(
|
||||
kwargs=_failure_log_hook_kwargs(details), result=None, call_type=self._logging_hook_call_type
|
||||
)[0]
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # a failing mask redacts the log instead of stopping failure logging
|
||||
self._report_failure_log_mask_error(guardrail, error)
|
||||
return None
|
||||
return hooked[0]
|
||||
return hooked_details
|
||||
|
||||
def _failure_handler_helper_fn(self, exception, traceback_exception, start_time=None, end_time=None):
|
||||
if start_time is None:
|
||||
|
|
@ -4941,7 +4943,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
print_verbose(f"supabaseClient: {supabaseClient}")
|
||||
supabaseClient.log_event(
|
||||
model=self.model if hasattr(self, "model") else "",
|
||||
messages=self.messages,
|
||||
messages=(
|
||||
self.messages
|
||||
if self._failure_logged_messages is self._unmasked_messages
|
||||
else self._failure_logged_messages
|
||||
),
|
||||
end_user=self._model_call_details_view.get("user", "default"),
|
||||
response_obj=result,
|
||||
start_time=start_time,
|
||||
|
|
|
|||
|
|
@ -8884,6 +8884,13 @@ class _HangingSyncMaskingGuardrail(_NameMaskingGuardrail):
|
|||
return self._mask(kwargs), result
|
||||
|
||||
|
||||
class _UnpairedSyncMaskingGuardrail(_NameMaskingGuardrail):
|
||||
def logging_hook(
|
||||
self, kwargs: dict[str, object], result: object, call_type: str
|
||||
) -> tuple[dict[str, object], object]:
|
||||
return cast(tuple[dict[str, object], object], None)
|
||||
|
||||
|
||||
class _AsyncOnlyNameMaskingGuardrail(CustomGuardrail):
|
||||
masks_logged_request: ClassVar[bool] = True
|
||||
|
||||
|
|
@ -9196,6 +9203,55 @@ def test_sync_failure_is_logged_redacted_when_the_masking_guardrail_has_only_an_
|
|||
assert recorder.logged == [(_REDACTED_LOG, _REDACTED_LOG)]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_global_callbacks")
|
||||
def test_sync_failure_is_logged_redacted_when_the_masking_hook_returns_no_pair() -> None:
|
||||
recorder: Final = _FailureLogRecorder()
|
||||
logging_obj: Final = _sync_failure_logging("unpaired-mask-failure", _UnpairedSyncMaskingGuardrail(), recorder)
|
||||
|
||||
logging_obj.failure_handler(litellm.ServiceUnavailableError("503", "openai", "gpt-4o-mini"), "tb")
|
||||
|
||||
assert recorder.logged == [(_REDACTED_LOG, _REDACTED_LOG)]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_global_callbacks")
|
||||
def test_supabase_failure_log_gets_the_masked_prompt(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
supabase_messages: Final[list[object]] = []
|
||||
|
||||
class SupabaseRecorder:
|
||||
def log_event(self, **call: object) -> None:
|
||||
supabase_messages.append(call["messages"])
|
||||
|
||||
monkeypatch.setattr(litellm, "failure_callback", ["supabase"])
|
||||
monkeypatch.setattr(litellm_logging, "supabaseClient", SupabaseRecorder())
|
||||
logging_obj: Final = _sync_failure_logging("supabase-failure", _NameMaskingGuardrail(), _FailureLogRecorder())
|
||||
|
||||
logging_obj.failure_handler(litellm.ServiceUnavailableError("503", "openai", "gpt-4o-mini"), "tb")
|
||||
|
||||
assert supabase_messages == [_MASKED_LOG]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_global_callbacks")
|
||||
def test_supabase_failure_log_without_a_masking_guardrail_gets_the_request_messages(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
supabase_messages: Final[list[object]] = []
|
||||
|
||||
class SupabaseRecorder:
|
||||
def log_event(self, **call: object) -> None:
|
||||
supabase_messages.append(call["messages"])
|
||||
|
||||
monkeypatch.setattr(litellm, "failure_callback", ["supabase"])
|
||||
monkeypatch.setattr(litellm_logging, "supabaseClient", SupabaseRecorder())
|
||||
logging_obj: Final = _sync_failure_logging(
|
||||
"supabase-unmasked-failure", _NameMaskingGuardrail(event_hook=GuardrailEventHooks.pre_call), _FailureLogRecorder()
|
||||
)
|
||||
logging_obj.messages = [{"role": "user", "content": "rewritten by prompt management"}]
|
||||
|
||||
logging_obj.failure_handler(litellm.ServiceUnavailableError("503", "openai", "gpt-4o-mini"), "tb")
|
||||
|
||||
assert supabase_messages == [logging_obj.messages]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_global_callbacks")
|
||||
def test_sync_failure_mask_still_queued_at_its_time_bound_never_runs() -> None:
|
||||
guardrail: Final = _NameMaskingGuardrail()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue