From 48b95aa0c733c2ecc97624851c50ce94bb292a9d Mon Sep 17 00:00:00 2001 From: xuan <118178092+yuee3@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:45:04 +0800 Subject: [PATCH 1/4] fix(prometheus): give each proxy boot its own auto-created PROMETHEUS_MULTIPROC_DIR --- litellm/proxy/proxy_cli.py | 14 +++--- tests/unit/proxy/test_prometheus_cleanup.py | 47 +++++++++++++++++++++ 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index c7371f294f0..2e677857573 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -724,12 +724,16 @@ class ProxyInitializationHelpers: wipe_directory(configured_dir) return None - multiproc_dir: Final = configured_dir or os.path.join(tempfile.gettempdir(), "litellm_prometheus_multiproc") - os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir + if configured_dir: + # An operator-set directory is theirs to manage; keep the historical wipe-on-boot. + os.makedirs(configured_dir, exist_ok=True) + wipe_directory(configured_dir) + multiproc_dir, action = configured_dir, "Using existing" + else: + # Unique per boot: a fixed shared name lets a second proxy on this host wipe the first one's counters. + multiproc_dir, action = tempfile.mkdtemp(prefix="litellm_prometheus_multiproc_"), "Auto-created" - os.makedirs(multiproc_dir, exist_ok=True) - wipe_directory(multiproc_dir) - action: Final = "Using existing" if configured_dir else "Auto-created" + os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir print(f"LiteLLM: {action} PROMETHEUS_MULTIPROC_DIR={multiproc_dir}") return multiproc_dir diff --git a/tests/unit/proxy/test_prometheus_cleanup.py b/tests/unit/proxy/test_prometheus_cleanup.py index c774d376d33..95903540c47 100644 --- a/tests/unit/proxy/test_prometheus_cleanup.py +++ b/tests/unit/proxy/test_prometheus_cleanup.py @@ -6,8 +6,10 @@ ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir. from __future__ import annotations import os +import shutil import subprocess import sys +import tempfile from pathlib import Path from typing import Final from unittest.mock import patch @@ -193,6 +195,8 @@ class TestMaybeSetupPrometheusMultiprocDir: result_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR") assert result_dir is not None assert os.path.isdir(result_dir) + legacy_shared = os.path.join(tempfile.gettempdir(), "litellm_prometheus_multiproc") + assert result_dir != legacy_shared # Cleanup os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) @@ -257,3 +261,46 @@ class TestMaybeSetupPrometheusMultiprocDir: assert os.environ["PROMETHEUS_MULTIPROC_DIR"] == str(tmp_path) assert not admitted.exists() assert not samples.exists() + + def test_auto_created_dir_is_unique_per_boot_and_keeps_sibling_counters(self): + """A fixed default dir made the second proxy boot wipe the first proxy's live counters (issue #42566).""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + os.environ.pop("prometheus_multiproc_dir", None) + litellm_settings = {"callbacks": ["prometheus"]} + + first = ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=4, litellm_settings=litellm_settings + ) + assert first is not None and os.path.isdir(first) + + stale = os.path.join(first, "counter_proxy_one.db") + with open(stale, "w") as f: + f.write("live") + + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) # simulate a second, independent boot + second = ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=4, litellm_settings=litellm_settings + ) + assert second is not None and os.path.isdir(second) + + assert first != second + assert os.path.exists(stale) + + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + shutil.rmtree(first, ignore_errors=True) + shutil.rmtree(second, ignore_errors=True) + + def test_configured_dir_is_still_wiped_on_boot(self, tmp_path): + """A directory the operator set keeps its historical wipe-on-boot lifecycle.""" + stale = tmp_path / "counter_stale.db" + stale.write_text("old") + + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + result_dir = ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=4, + litellm_settings={"callbacks": ["prometheus"]}, + ) + + assert result_dir == str(tmp_path) + assert not stale.exists() From 6f356cd920dee3f0c36bfa61cfa6c215662b5203 Mon Sep 17 00:00:00 2001 From: xuan <118178092+yuee3@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:45:12 +0800 Subject: [PATCH 2/4] fix: clean up auto-created prometheus multiproc dirs Auto-created mkdtemp dirs were never removed. Register an atexit hook that rmtree's the dir only from the creating PID so forked workers sharing it do not delete samples their siblings are still writing. Close the remaining test leaks: test_handles_string_callbacks and test_separate_metrics_port_forces_dir_for_single_worker now rmtree the dirs they create. --- litellm/proxy/proxy_cli.py | 13 +++++++++++++ tests/unit/proxy/test_prometheus_cleanup.py | 6 +++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 2e677857573..242b8ba0c68 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -712,6 +712,7 @@ class ProxyInitializationHelpers: with prometheus configured as a callback in config.yaml, or the separate metrics server (always, since callbacks may also be enabled from the DB after startup). """ + import shutil import tempfile from litellm.proxy.prometheus_cleanup import wipe_directory @@ -731,8 +732,20 @@ class ProxyInitializationHelpers: multiproc_dir, action = configured_dir, "Using existing" else: # Unique per boot: a fixed shared name lets a second proxy on this host wipe the first one's counters. + import atexit + multiproc_dir, action = tempfile.mkdtemp(prefix="litellm_prometheus_multiproc_"), "Auto-created" + # Only the creating process removes the dir on exit. Forked workers share it and + # must not delete samples their siblings are still writing. + creator_pid = os.getpid() + + def _cleanup_auto_created_dir() -> None: + if os.getpid() == creator_pid: + shutil.rmtree(multiproc_dir, ignore_errors=True) + + atexit.register(_cleanup_auto_created_dir) + os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir print(f"LiteLLM: {action} PROMETHEUS_MULTIPROC_DIR={multiproc_dir}") return multiproc_dir diff --git a/tests/unit/proxy/test_prometheus_cleanup.py b/tests/unit/proxy/test_prometheus_cleanup.py index 95903540c47..65163e7eb65 100644 --- a/tests/unit/proxy/test_prometheus_cleanup.py +++ b/tests/unit/proxy/test_prometheus_cleanup.py @@ -146,13 +146,15 @@ class TestMaybeSetupPrometheusMultiprocDir: os.environ.pop("prometheus_multiproc_dir", None) # Should not raise TypeError - ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + result_dir = ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( num_workers=4, litellm_settings=litellm_settings, ) # Cleanup os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + if result_dir is not None: + shutil.rmtree(result_dir, ignore_errors=True) @pytest.mark.parametrize( "num_workers, litellm_settings", @@ -200,6 +202,7 @@ class TestMaybeSetupPrometheusMultiprocDir: # Cleanup os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + shutil.rmtree(result_dir, ignore_errors=True) @pytest.mark.parametrize( "litellm_settings", @@ -227,6 +230,7 @@ class TestMaybeSetupPrometheusMultiprocDir: assert os.path.isdir(result_dir) os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + shutil.rmtree(result_dir, ignore_errors=True) def test_lowercase_env_var_is_reused_and_exported_uppercase(self, tmp_path): """prometheus_client honours both spellings; the metrics server only reads the uppercase one.""" From 327c956c46ff0197c53d64cf31c06c3f1a44cad4 Mon Sep 17 00:00:00 2001 From: xuan <118178092+yuee3@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:20:40 +0800 Subject: [PATCH 3/4] fix: Final-annotate the per-boot dir pick and test the exit hook main's type-discipline gate counts assignments without a Final declaration against the merge-base ceiling, and unpacking targets cannot carry Final: the second `multiproc_dir, action = ...` pick re-binds both names (LIT010). Declare both as Final and defer the assignment to the branch that picks the value. The creator-PID exit hook becomes a staticmethod so the creator-only-removes branch is unit-covered, which also lifts patch coverage back above target. --- litellm/proxy/proxy_cli.py | 25 ++++++++++++--------- tests/unit/proxy/test_prometheus_cleanup.py | 9 +++++++- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 242b8ba0c68..a029289c256 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -4,6 +4,7 @@ import json import os import random import re +import shutil import subprocess import sys import urllib.parse as urlparse @@ -712,7 +713,6 @@ class ProxyInitializationHelpers: with prometheus configured as a callback in config.yaml, or the separate metrics server (always, since callbacks may also be enabled from the DB after startup). """ - import shutil import tempfile from litellm.proxy.prometheus_cleanup import wipe_directory @@ -725,26 +725,24 @@ class ProxyInitializationHelpers: wipe_directory(configured_dir) return None + multiproc_dir: Final[str] + action: Final[str] if configured_dir: # An operator-set directory is theirs to manage; keep the historical wipe-on-boot. os.makedirs(configured_dir, exist_ok=True) wipe_directory(configured_dir) - multiproc_dir, action = configured_dir, "Using existing" + multiproc_dir = configured_dir + action = "Using existing" else: # Unique per boot: a fixed shared name lets a second proxy on this host wipe the first one's counters. import atexit - multiproc_dir, action = tempfile.mkdtemp(prefix="litellm_prometheus_multiproc_"), "Auto-created" + multiproc_dir = tempfile.mkdtemp(prefix="litellm_prometheus_multiproc_") + action = "Auto-created" # Only the creating process removes the dir on exit. Forked workers share it and # must not delete samples their siblings are still writing. - creator_pid = os.getpid() - - def _cleanup_auto_created_dir() -> None: - if os.getpid() == creator_pid: - shutil.rmtree(multiproc_dir, ignore_errors=True) - - atexit.register(_cleanup_auto_created_dir) + atexit.register(ProxyInitializationHelpers._rmtree_auto_created_dir, multiproc_dir, os.getpid()) os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir print(f"LiteLLM: {action} PROMETHEUS_MULTIPROC_DIR={multiproc_dir}") @@ -752,6 +750,13 @@ class ProxyInitializationHelpers: _maybe_setup_prometheus_multiproc_dir = maybe_setup_prometheus_multiproc_dir + @staticmethod + def _rmtree_auto_created_dir(multiproc_dir: str, creator_pid: int) -> None: + """atexit hook for a boot-auto-created PROMETHEUS_MULTIPROC_DIR: only the creating process + removes the dir, so forked workers sharing it never delete samples their siblings still write.""" + if os.getpid() == creator_pid: + shutil.rmtree(multiproc_dir, ignore_errors=True) + @click.command() @click.argument("cli_args", nargs=-1) diff --git a/tests/unit/proxy/test_prometheus_cleanup.py b/tests/unit/proxy/test_prometheus_cleanup.py index 65163e7eb65..2e1ce52b96c 100644 --- a/tests/unit/proxy/test_prometheus_cleanup.py +++ b/tests/unit/proxy/test_prometheus_cleanup.py @@ -292,8 +292,15 @@ class TestMaybeSetupPrometheusMultiprocDir: assert os.path.exists(stale) os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + + # The exit hook removes the dir only from the creating process: a forked worker + # (different PID) sharing the dir must leave its siblings' samples alone. + ProxyInitializationHelpers._rmtree_auto_created_dir(first, 999999) + assert os.path.exists(first) + ProxyInitializationHelpers._rmtree_auto_created_dir(second, os.getpid()) + assert not os.path.exists(second) + shutil.rmtree(first, ignore_errors=True) - shutil.rmtree(second, ignore_errors=True) def test_configured_dir_is_still_wiped_on_boot(self, tmp_path): """A directory the operator set keeps its historical wipe-on-boot lifecycle.""" From 7a8b25771ac2390cf5168bd09f4cf9a12543c21c Mon Sep 17 00:00:00 2001 From: xuan <118178092+yuee3@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:05:20 +0800 Subject: [PATCH 4/4] fix: pick the multiproc dir in a helper so the names stay single-assignment basedpyright rejects the bare `Final[str]` declarations being assigned in both branches of the if/else ("cannot be re-assigned"), and the gate counts those two reportGeneralTypeIssues against the merge-base ceiling. Move the pick into `_pick_multiproc_dir`, which returns the (dir, verb) pair, so the caller binds both names exactly once via unpacking. --- litellm/proxy/proxy_cli.py | 46 +++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index a029289c256..25b1f0e4a96 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -713,8 +713,6 @@ class ProxyInitializationHelpers: with prometheus configured as a callback in config.yaml, or the separate metrics server (always, since callbacks may also be enabled from the DB after startup). """ - import tempfile - from litellm.proxy.prometheus_cleanup import wipe_directory configured_dir: Final = os.environ.get("PROMETHEUS_MULTIPROC_DIR") or os.environ.get("prometheus_multiproc_dir") @@ -725,24 +723,7 @@ class ProxyInitializationHelpers: wipe_directory(configured_dir) return None - multiproc_dir: Final[str] - action: Final[str] - if configured_dir: - # An operator-set directory is theirs to manage; keep the historical wipe-on-boot. - os.makedirs(configured_dir, exist_ok=True) - wipe_directory(configured_dir) - multiproc_dir = configured_dir - action = "Using existing" - else: - # Unique per boot: a fixed shared name lets a second proxy on this host wipe the first one's counters. - import atexit - - multiproc_dir = tempfile.mkdtemp(prefix="litellm_prometheus_multiproc_") - action = "Auto-created" - - # Only the creating process removes the dir on exit. Forked workers share it and - # must not delete samples their siblings are still writing. - atexit.register(ProxyInitializationHelpers._rmtree_auto_created_dir, multiproc_dir, os.getpid()) + multiproc_dir, action = ProxyInitializationHelpers._pick_multiproc_dir(configured_dir) os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir print(f"LiteLLM: {action} PROMETHEUS_MULTIPROC_DIR={multiproc_dir}") @@ -757,6 +738,31 @@ class ProxyInitializationHelpers: if os.getpid() == creator_pid: shutil.rmtree(multiproc_dir, ignore_errors=True) + @staticmethod + def _pick_multiproc_dir(configured_dir: str | None) -> tuple[str, str]: + """Pick the PROMETHEUS_MULTIPROC_DIR to use, with the boot-message verb for it. + + An operator-set directory keeps its historical wipe-on-boot lifecycle. An auto-created one + is unique per boot — a fixed shared name lets a second proxy on this host wipe the first + one's counters — and registers the creator-only atexit cleanup. + """ + import atexit + import tempfile + + from litellm.proxy.prometheus_cleanup import wipe_directory + + if configured_dir: + os.makedirs(configured_dir, exist_ok=True) + wipe_directory(configured_dir) + return configured_dir, "Using existing" + + picked: Final = tempfile.mkdtemp(prefix="litellm_prometheus_multiproc_") + + # Only the creating process removes the dir on exit. Forked workers share it and + # must not delete samples their siblings are still writing. + atexit.register(ProxyInitializationHelpers._rmtree_auto_created_dir, picked, os.getpid()) + return picked, "Auto-created" + @click.command() @click.argument("cli_args", nargs=-1)