mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 2471071b57 into f96200bcea
This commit is contained in:
commit
2c7a699632
2 changed files with 94 additions and 8 deletions
|
|
@ -4,6 +4,7 @@ import json
|
|||
import os
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.parse as urlparse
|
||||
|
|
@ -744,8 +745,6 @@ class ProxyInitializationHelpers:
|
|||
with prometheus configured as a callback in config.yaml, or the separate metrics server (always, since
|
||||
callbacks may also be enabled from the DB after startup).
|
||||
"""
|
||||
import tempfile
|
||||
|
||||
from litellm.proxy.prometheus_cleanup import wipe_directory
|
||||
|
||||
configured_dir: Final = os.environ.get("PROMETHEUS_MULTIPROC_DIR") or os.environ.get("prometheus_multiproc_dir")
|
||||
|
|
@ -756,17 +755,46 @@ class ProxyInitializationHelpers:
|
|||
wipe_directory(configured_dir)
|
||||
return None
|
||||
|
||||
multiproc_dir: Final = configured_dir or os.path.join(tempfile.gettempdir(), "litellm_prometheus_multiproc")
|
||||
os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir
|
||||
multiproc_dir, action = ProxyInitializationHelpers._pick_multiproc_dir(configured_dir)
|
||||
|
||||
os.makedirs(multiproc_dir, exist_ok=True)
|
||||
wipe_directory(multiproc_dir)
|
||||
action: Final = "Using existing" if configured_dir else "Auto-created"
|
||||
os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir
|
||||
print(f"LiteLLM: {action} PROMETHEUS_MULTIPROC_DIR={multiproc_dir}")
|
||||
return multiproc_dir
|
||||
|
||||
_maybe_setup_prometheus_multiproc_dir = maybe_setup_prometheus_multiproc_dir
|
||||
|
||||
@staticmethod
|
||||
def _rmtree_auto_created_dir(multiproc_dir: str, creator_pid: int) -> None:
|
||||
"""atexit hook for a boot-auto-created PROMETHEUS_MULTIPROC_DIR: only the creating process
|
||||
removes the dir, so forked workers sharing it never delete samples their siblings still write."""
|
||||
if os.getpid() == creator_pid:
|
||||
shutil.rmtree(multiproc_dir, ignore_errors=True)
|
||||
|
||||
@staticmethod
|
||||
def _pick_multiproc_dir(configured_dir: str | None) -> tuple[str, str]:
|
||||
"""Pick the PROMETHEUS_MULTIPROC_DIR to use, with the boot-message verb for it.
|
||||
|
||||
An operator-set directory keeps its historical wipe-on-boot lifecycle. An auto-created one
|
||||
is unique per boot — a fixed shared name lets a second proxy on this host wipe the first
|
||||
one's counters — and registers the creator-only atexit cleanup.
|
||||
"""
|
||||
import atexit
|
||||
import tempfile
|
||||
|
||||
from litellm.proxy.prometheus_cleanup import wipe_directory
|
||||
|
||||
if configured_dir:
|
||||
os.makedirs(configured_dir, exist_ok=True)
|
||||
wipe_directory(configured_dir)
|
||||
return configured_dir, "Using existing"
|
||||
|
||||
picked: Final = tempfile.mkdtemp(prefix="litellm_prometheus_multiproc_")
|
||||
|
||||
# Only the creating process removes the dir on exit. Forked workers share it and
|
||||
# must not delete samples their siblings are still writing.
|
||||
atexit.register(ProxyInitializationHelpers._rmtree_auto_created_dir, picked, os.getpid())
|
||||
return picked, "Auto-created"
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("cli_args", nargs=-1)
|
||||
|
|
|
|||
|
|
@ -6,8 +6,10 @@ ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir.
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
|
@ -144,13 +146,15 @@ class TestMaybeSetupPrometheusMultiprocDir:
|
|||
os.environ.pop("prometheus_multiproc_dir", None)
|
||||
|
||||
# Should not raise TypeError
|
||||
ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir(
|
||||
result_dir = ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir(
|
||||
num_workers=4,
|
||||
litellm_settings=litellm_settings,
|
||||
)
|
||||
|
||||
# Cleanup
|
||||
os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None)
|
||||
if result_dir is not None:
|
||||
shutil.rmtree(result_dir, ignore_errors=True)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_workers, litellm_settings",
|
||||
|
|
@ -193,9 +197,12 @@ class TestMaybeSetupPrometheusMultiprocDir:
|
|||
result_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
|
||||
assert result_dir is not None
|
||||
assert os.path.isdir(result_dir)
|
||||
legacy_shared = os.path.join(tempfile.gettempdir(), "litellm_prometheus_multiproc")
|
||||
assert result_dir != legacy_shared
|
||||
|
||||
# Cleanup
|
||||
os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None)
|
||||
shutil.rmtree(result_dir, ignore_errors=True)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"litellm_settings",
|
||||
|
|
@ -223,6 +230,7 @@ class TestMaybeSetupPrometheusMultiprocDir:
|
|||
assert os.path.isdir(result_dir)
|
||||
|
||||
os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None)
|
||||
shutil.rmtree(result_dir, ignore_errors=True)
|
||||
|
||||
def test_lowercase_env_var_is_reused_and_exported_uppercase(self, tmp_path):
|
||||
"""prometheus_client honours both spellings; the metrics server only reads the uppercase one."""
|
||||
|
|
@ -257,3 +265,53 @@ class TestMaybeSetupPrometheusMultiprocDir:
|
|||
assert os.environ["PROMETHEUS_MULTIPROC_DIR"] == str(tmp_path)
|
||||
assert not admitted.exists()
|
||||
assert not samples.exists()
|
||||
|
||||
def test_auto_created_dir_is_unique_per_boot_and_keeps_sibling_counters(self):
|
||||
"""A fixed default dir made the second proxy boot wipe the first proxy's live counters (issue #42566)."""
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None)
|
||||
os.environ.pop("prometheus_multiproc_dir", None)
|
||||
litellm_settings = {"callbacks": ["prometheus"]}
|
||||
|
||||
first = ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir(
|
||||
num_workers=4, litellm_settings=litellm_settings
|
||||
)
|
||||
assert first is not None and os.path.isdir(first)
|
||||
|
||||
stale = os.path.join(first, "counter_proxy_one.db")
|
||||
with open(stale, "w") as f:
|
||||
f.write("live")
|
||||
|
||||
os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) # simulate a second, independent boot
|
||||
second = ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir(
|
||||
num_workers=4, litellm_settings=litellm_settings
|
||||
)
|
||||
assert second is not None and os.path.isdir(second)
|
||||
|
||||
assert first != second
|
||||
assert os.path.exists(stale)
|
||||
|
||||
os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None)
|
||||
|
||||
# The exit hook removes the dir only from the creating process: a forked worker
|
||||
# (different PID) sharing the dir must leave its siblings' samples alone.
|
||||
ProxyInitializationHelpers._rmtree_auto_created_dir(first, 999999)
|
||||
assert os.path.exists(first)
|
||||
ProxyInitializationHelpers._rmtree_auto_created_dir(second, os.getpid())
|
||||
assert not os.path.exists(second)
|
||||
|
||||
shutil.rmtree(first, ignore_errors=True)
|
||||
|
||||
def test_configured_dir_is_still_wiped_on_boot(self, tmp_path):
|
||||
"""A directory the operator set keeps its historical wipe-on-boot lifecycle."""
|
||||
stale = tmp_path / "counter_stale.db"
|
||||
stale.write_text("old")
|
||||
|
||||
with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}):
|
||||
result_dir = ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir(
|
||||
num_workers=4,
|
||||
litellm_settings={"callbacks": ["prometheus"]},
|
||||
)
|
||||
|
||||
assert result_dir == str(tmp_path)
|
||||
assert not stale.exists()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue