From ecd25257485efe250fdb357ff960d9f14122406e Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Fri, 13 Mar 2026 22:56:36 +0000
Subject: [PATCH] fix: add periodic malloc_trim to return freed memory to OS

Python's pymalloc allocator retains freed arena memory by default. After
traffic spikes, RSS stays at peak indefinitely even though all objects
have been garbage-collected.

Add _periodic_memory_cleanup() that runs gc.collect() + malloc_trim(0)
every 60s (configurable via LITELLM_MEMORY_CLEANUP_INTERVAL). This forces
glibc to return unused heap pages to the kernel, allowing RSS to shrink
during idle periods.

Registered as an APScheduler interval job in proxy_server.py.

Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
---
 litellm/proxy/common_utils/memory_utils.py | 64 ++++++++++++++++++++++
 litellm/proxy/proxy_server.py              | 22 ++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 litellm/proxy/common_utils/memory_utils.py

diff --git a/litellm/proxy/common_utils/memory_utils.py b/litellm/proxy/common_utils/memory_utils.py
new file mode 100644
index 00000000000..1aa08a92bf1
--- /dev/null
+++ b/litellm/proxy/common_utils/memory_utils.py
@@ -0,0 +1,64 @@
+"""
+Periodic memory cleanup utilities for the LiteLLM proxy.
+
+Python's memory allocator (pymalloc) does not return freed memory arenas back
+to the OS by default. After handling a burst of traffic, process RSS stays
+elevated even though the objects have been garbage-collected. This module
+provides a lightweight scheduled job that:
+
+1. Runs a full GC collection cycle
+2. Calls ``malloc_trim(0)`` on Linux (glibc) to return unused heap pages to
+   the OS
+
+This is especially important for long-running proxy deployments where memory
+grows during load tests / peak traffic and never shrinks back.
+
+Environment variables:
+    LITELLM_MEMORY_CLEANUP_INTERVAL  – seconds between cleanup runs (default 60)
+"""
+
+import ctypes
+import gc
+import sys
+
+from litellm._logging import verbose_proxy_logger
+
+_libc = None
+_malloc_trim_available = False
+
+if sys.platform == "linux":
+    try:
+        _libc = ctypes.CDLL("libc.so.6")
+        _malloc_trim_available = hasattr(_libc, "malloc_trim")
+    except OSError:
+        pass
+
+
+def _periodic_memory_cleanup() -> None:
+    """Run a full GC cycle and release freed heap memory back to the OS.
+
+    On Linux with glibc, ``malloc_trim(0)`` forces the allocator to return
+    unused pages to the kernel. Without this, RSS never shrinks even after
+    all Python objects are freed.
+
+    Safe to call frequently (every 60s by default). ``gc.collect()`` is ~1ms
+    when there is little garbage, and ``malloc_trim`` is a no-op when there
+    are no reclaimable pages.
+    """
+    collected = gc.collect()
+    trimmed = False
+
+    if _malloc_trim_available and _libc is not None:
+        try:
+            _libc.malloc_trim(0)
+            trimmed = True
+        except Exception as exc:
+            verbose_proxy_logger.debug(
+                "malloc_trim failed (non-critical): %s", exc
+            )
+
+    verbose_proxy_logger.debug(
+        "Periodic memory cleanup: gc collected %d objects, malloc_trim=%s",
+        collected,
+        trimmed,
+    )
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 2a9be0a67c9..c4fbf7804a1 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -6135,6 +6135,28 @@ class ProxyStartupEvent:
                 )
                 pass
 
+        # Periodic memory cleanup: gc.collect() + malloc_trim(0) to return freed
+        # heap pages to the OS.  Without this, RSS never shrinks after traffic spikes.
+        from litellm.proxy.common_utils.memory_utils import _periodic_memory_cleanup
+
+        memory_cleanup_interval = int(
+            os.getenv("LITELLM_MEMORY_CLEANUP_INTERVAL", "60")
+        )
+        scheduler.add_job(
+            _periodic_memory_cleanup,
+            "interval",
+            seconds=memory_cleanup_interval,
+            id="memory_cleanup_job",
+            replace_existing=True,
+            misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
+            coalesce=APSCHEDULER_COALESCE,
+            max_instances=APSCHEDULER_MAX_INSTANCES,
+        )
+        verbose_proxy_logger.info(
+            "Periodic memory cleanup job scheduled every %ds",
+            memory_cleanup_interval,
+        )
+
         # MEMORY LEAK FIX: Start scheduler with paused=False to avoid backlog processing
         # Do NOT reset job times to "now" as this can trigger the memory leak
         # The misfire_grace_time and coalesce settings will handle any missed runs properly