diff --git a/litellm/proxy/prometheus_cleanup.py b/litellm/proxy/prometheus_cleanup.py index 34c48de74fa..4dec57fa806 100644 --- a/litellm/proxy/prometheus_cleanup.py +++ b/litellm/proxy/prometheus_cleanup.py @@ -1,11 +1,8 @@ """ Prometheus multiprocess directory cleanup utilities. -When running with multiple workers and PROMETHEUS_MULTIPROC_DIR set, -each worker creates memory-mapped .db files (e.g., counter_1234.db). -When workers die or restart, gauge_live* files for dead PIDs must be -cleaned up via mark_process_dead(). Counter and histogram files are -kept since they contain cumulative data needed for correct aggregation. +mark_process_dead() only removes gauge_live* files — counter/histogram +files are kept for correct aggregation and wiped in bulk on startup. """ from __future__ import annotations @@ -28,39 +25,18 @@ def _get_multiproc_dir() -> Optional[str]: def _is_pid_alive(pid: int) -> bool: - """ - Check if a process with the given PID is alive. - - Uses os.kill(pid, 0) which doesn't send a signal but checks existence. - - ProcessLookupError: process does not exist (dead) - - PermissionError: process exists but we can't signal it (alive, conservative) - - OSError: other error, treat as alive (conservative) - """ + """Check if a process is alive using signal 0 (conservative: unknown = alive).""" try: os.kill(pid, 0) return True except ProcessLookupError: return False - except PermissionError: - # Process exists but we don't have permission to signal it - return True - except OSError: - # Conservative: treat unknown errors as alive + except (PermissionError, OSError): return True def _extract_pids_from_dir(directory: str) -> Set[int]: - """ - Scan .db filenames in a directory and extract PIDs. - - Prometheus client creates files like: - - counter_1234.db - - histogram_1234.db - - gauge_livesum_1234.db - - gauge_liveall_1234.db - - Returns a set of integer PIDs found. - """ + """Scan .db filenames in a directory and extract PIDs (e.g. counter_1234.db -> 1234).""" pids: Set[int] = set() try: for filename in os.listdir(directory): @@ -75,15 +51,7 @@ def _extract_pids_from_dir(directory: str) -> Set[int]: def wipe_directory(directory: str) -> None: - """ - Delete all .db files in the prometheus multiproc directory. - - Called once in the master process before workers fork. Per the - prometheus_client docs: "This directory must be wiped between - process runs (before startup is recommended)." - - Any .db files present at this point are stale from a previous run. - """ + """Delete all .db files in the directory. Called once before workers fork.""" files = glob.glob(os.path.join(directory, "*.db")) for filepath in files: try: @@ -99,14 +67,7 @@ def wipe_directory(directory: str) -> None: def cleanup_own_pid_files() -> None: - """ - Mark the current process as dead for prometheus multiproc cleanup. - - Called during per-worker shutdown. Uses mark_process_dead() which - only removes gauge_live* files — counter and histogram files are - preserved since they contain cumulative data needed for correct - aggregation until the directory is wiped on next startup. - """ + """Mark the current process as dead via mark_process_dead() (worker shutdown).""" directory = _get_multiproc_dir() if not directory or not os.path.isdir(directory): return @@ -126,18 +87,7 @@ def cleanup_own_pid_files() -> None: def mark_dead_pids(skip_pid: Optional[int] = None) -> None: - """ - Scan the prometheus multiproc directory and call mark_process_dead() - for PIDs that no longer exist. - - Uses prometheus_client.multiprocess.mark_process_dead() which only - removes gauge_live* files — counter and histogram files are preserved - since they contain cumulative data needed for correct aggregation. - - Args: - skip_pid: PID to skip (typically os.getpid()). If None, skips - the current process's PID. - """ + """Scan for dead PIDs and call mark_process_dead() for each. Skips current process by default.""" directory = _get_multiproc_dir() if not directory or not os.path.isdir(directory): return