mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-13 03:59:12 +00:00
fix(ci): wait for SIGSTOP to take effect before handing back a suspended proxy process (#46088)
* fix(ci): wait for SIGSTOP to take effect before handing back a suspended proxy process Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ci): resume a process whose SIGSTOP never lands before failing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo <mateo@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
56a1e94a31
commit
82749836df
1 changed files with 15 additions and 3 deletions
|
|
@ -129,6 +129,18 @@ def _is_stopped(process: psutil.Process) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _suspend(process: psutil.Process) -> None:
|
||||
process.suspend()
|
||||
deadline: Final = time.monotonic() + STRICT_HEALTHCHECK_SECONDS
|
||||
while not _is_stopped(process) and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
if _is_stopped(process):
|
||||
return
|
||||
with suppress(psutil.NoSuchProcess):
|
||||
process.resume()
|
||||
raise AssertionError(f"Process {process.pid} did not stop within {STRICT_HEALTHCHECK_SECONDS}s of SIGSTOP")
|
||||
|
||||
|
||||
def _is_reaped(process: psutil.Process) -> bool:
|
||||
try:
|
||||
process.status()
|
||||
|
|
@ -152,7 +164,7 @@ class WorkerKill:
|
|||
fired: threading.Event
|
||||
|
||||
def freeze(self) -> psutil.Process:
|
||||
self.victim.suspend()
|
||||
_suspend(self.victim)
|
||||
return self.victim
|
||||
|
||||
def __call__(self) -> None:
|
||||
|
|
@ -235,7 +247,7 @@ class OwnedProxy:
|
|||
)
|
||||
assert pid == self.process.pid or pid in self.worker_pids(), f"{pid} is not part of this proxy"
|
||||
target: Final = psutil.Process(pid)
|
||||
target.suspend()
|
||||
_suspend(target)
|
||||
try:
|
||||
yield target
|
||||
finally:
|
||||
|
|
@ -307,7 +319,7 @@ def kill_stray_process(pid: int) -> None:
|
|||
@contextmanager
|
||||
def paused(process: subprocess.Popen[bytes] | subprocess.Popen[str]) -> Iterator[None]:
|
||||
target: Final = psutil.Process(process.pid)
|
||||
target.suspend()
|
||||
_suspend(target)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue