mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(logging): hand sync failure callbacks the masked log when the failure dispatch is cancelled mid-mask
This commit is contained in:
parent
a247d9e553
commit
b0f9a0fa6d
2 changed files with 76 additions and 5 deletions
|
|
@ -726,6 +726,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
_failure_log_masked: bool = False
|
||||
_unmasked_messages: object = None
|
||||
_failure_logged_messages: object = None
|
||||
_failure_log_mask: "asyncio.Task[None] | None" = None
|
||||
|
||||
@property
|
||||
def _defer_async_logging(self) -> bool:
|
||||
|
|
@ -2767,8 +2768,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
bypasses the sync-SDK-only shortcut (e.g. ``async for`` on a stream from
|
||||
``completion()``); legacy string callbacks still run via
|
||||
``executor.submit(failure_handler)`` when configured, and still get submitted
|
||||
when the awaiting task is cancelled (e.g. the event loop shuts down right after
|
||||
the request failed).
|
||||
once the failure log mask finishes when the awaiting task is cancelled (e.g. the
|
||||
event loop shuts down right after the request failed).
|
||||
"""
|
||||
litellm_params: Final = self.model_call_details.get("litellm_params", {}) or {}
|
||||
sync_sdk: Final = self.is_sync_litellm_request(litellm_params)
|
||||
|
|
@ -2783,7 +2784,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
await self.async_failure_handler(exception, traceback_exception)
|
||||
finally:
|
||||
if self._should_run_sync_failure_callbacks_for_async_calls():
|
||||
executor.submit(self.failure_handler, exception, traceback_exception, for_async_request=True)
|
||||
self._once_failure_log_is_masked(
|
||||
lambda: executor.submit(
|
||||
self.failure_handler, exception, traceback_exception, for_async_request=True
|
||||
)
|
||||
)
|
||||
|
||||
def should_run_logging(
|
||||
self,
|
||||
|
|
@ -4478,16 +4483,31 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if not maskers:
|
||||
return
|
||||
mask: Final = asyncio.create_task(self._mask_failure_log_request_with(maskers))
|
||||
self._failure_log_mask = mask
|
||||
try:
|
||||
await asyncio.shield(mask)
|
||||
except asyncio.CancelledError:
|
||||
_keep_alive_until_done(mask)
|
||||
if on_cancelled is not None:
|
||||
mask.add_done_callback(lambda _: on_cancelled())
|
||||
self._once_failure_log_is_masked(on_cancelled)
|
||||
if on_cancelled_async is not None:
|
||||
mask.add_done_callback(lambda _: _keep_alive_until_done(asyncio.ensure_future(on_cancelled_async())))
|
||||
self._once_failure_log_is_masked(
|
||||
lambda: _keep_alive_until_done(asyncio.ensure_future(on_cancelled_async()))
|
||||
)
|
||||
raise
|
||||
|
||||
def _once_failure_log_is_masked(self, run: Callable[[], object]) -> None:
|
||||
mask: Final = self._failure_log_mask
|
||||
if mask is not None and not mask.done():
|
||||
mask.add_done_callback(lambda finished: self._run_after_failure_log_mask(finished, run))
|
||||
return
|
||||
self._run_after_failure_log_mask(mask, run)
|
||||
|
||||
def _run_after_failure_log_mask(self, mask: "asyncio.Future[None] | None", run: Callable[[], object]) -> None:
|
||||
if mask is not None and mask.cancelled():
|
||||
self._redact_failure_log_request()
|
||||
_ = run()
|
||||
|
||||
def mask_failure_log_request(self, mask_executor: concurrent.futures.Executor = failure_log_mask_executor) -> None:
|
||||
if is_event_loop_running() or not self._claim_failure_log_mask():
|
||||
return
|
||||
|
|
|
|||
|
|
@ -8938,6 +8938,57 @@ async def test_failure_log_still_goes_out_masked_when_the_caller_is_cancelled_mi
|
|||
assert recorder.logged == [(_MASKED_LOG, _MASKED_LOG)]
|
||||
|
||||
|
||||
def _async_failure_logging(call_id: str, guardrail: CustomGuardrail) -> LitellmLogging:
|
||||
logging_obj: Final = LitellmLogging(
|
||||
model="gpt-4o-mini",
|
||||
messages=copy.deepcopy(_RAW_LOG),
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
start_time=datetime.datetime(2026, 1, 1),
|
||||
litellm_call_id=call_id,
|
||||
function_id=call_id,
|
||||
dynamic_async_failure_callbacks=[guardrail],
|
||||
)
|
||||
logging_obj.update_environment_variables(litellm_params={"acompletion": True, "metadata": {}}, optional_params={})
|
||||
return logging_obj
|
||||
|
||||
|
||||
async def _cancel_failure_dispatch_mid_mask(
|
||||
logging_obj: LitellmLogging, guardrail: _BlockingNameMaskingGuardrail
|
||||
) -> None:
|
||||
dispatch: Final = asyncio.create_task(logging_obj.dispatch_failure_handlers(_bad_key_error(), "tb"))
|
||||
await asyncio.wait_for(guardrail.scanning.wait(), timeout=10)
|
||||
dispatch.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await dispatch
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_failure_callbacks_get_the_masked_prompt_when_the_failure_dispatch_is_cancelled_mid_mask(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
guardrail: Final = _BlockingNameMaskingGuardrail()
|
||||
_, sync_logged = _register_failure_logging(monkeypatch, guardrail)
|
||||
|
||||
await _cancel_failure_dispatch_mid_mask(_async_failure_logging("cancelled-dispatch", guardrail), guardrail)
|
||||
guardrail.release.set()
|
||||
await asyncio.wait_for(_until(lambda: bool(sync_logged)), timeout=10)
|
||||
|
||||
assert sync_logged == [_MASKED_LOG]
|
||||
|
||||
|
||||
def test_sync_failure_callbacks_get_a_redacted_prompt_when_the_loop_shuts_down_mid_mask(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
guardrail: Final = _BlockingNameMaskingGuardrail()
|
||||
_, sync_logged = _register_failure_logging(monkeypatch, guardrail)
|
||||
|
||||
asyncio.run(_cancel_failure_dispatch_mid_mask(_async_failure_logging("shut-down-dispatch", guardrail), guardrail))
|
||||
asyncio.run(asyncio.wait_for(_until(lambda: bool(sync_logged)), timeout=10))
|
||||
|
||||
assert sync_logged == [_REDACTED_LOG]
|
||||
|
||||
|
||||
def _sync_failure_logging(call_id: str, guardrail: CustomGuardrail, recorder: _FailureLogRecorder) -> LitellmLogging:
|
||||
logging_obj: Final = LitellmLogging(
|
||||
model="gpt-4o-mini",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue