From 7463f0e7230f9926045af087b2af2080a095969d Mon Sep 17 00:00:00 2001 From: Apeksha_2022 <70309060+Apeksha9389@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:40:15 +0530 Subject: [PATCH 1/8] feat(proxy): add LITELLM_GC_FREEZE_AFTER_STARTUP setting --- litellm/constants.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/constants.py b/litellm/constants.py index 274854e1c40..d18cbddccdf 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1663,6 +1663,9 @@ INVALID_VIRTUAL_KEY_ERROR_MARKER: Final = "_litellm_invalid_virtual_key_error" # Python garbage collection threshold configuration # Format: "gen0,gen1,gen2" e.g., "1000,50,50" PYTHON_GC_THRESHOLD: Final = os.getenv("PYTHON_GC_THRESHOLD") +# Move objects created during proxy startup into the permanent GC generation so +# full collections stop walking them (shorter stop-the-world pauses) +LITELLM_GC_FREEZE_AFTER_STARTUP: Final = os.getenv("LITELLM_GC_FREEZE_AFTER_STARTUP", "false").lower() == "true" # pass through route constansts BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES: Final = [ From d9e3ae8100d77fbf24f5cc9259d2e94befea377e Mon Sep 17 00:00:00 2001 From: Apeksha_2022 <70309060+Apeksha9389@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:40:58 +0530 Subject: [PATCH 2/8] feat(proxy): add freeze_gc_after_startup helper --- litellm/proxy/common_utils/debug_utils.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index 9c2cc768ae8..a6001932984 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -15,7 +15,7 @@ from typing_extensions import ReadOnly from litellm import get_secret_str from litellm._logging import verbose_proxy_logger -from litellm.constants import PYTHON_GC_THRESHOLD +from litellm.constants import LITELLM_GC_FREEZE_AFTER_STARTUP, PYTHON_GC_THRESHOLD from litellm.litellm_core_utils.bug_report import EnvironmentReport from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -60,6 +60,21 @@ def configure_gc_thresholds(): configure_gc_thresholds() +def freeze_gc_after_startup(enabled: bool = LITELLM_GC_FREEZE_AFTER_STARTUP) -> None: + """Freeze the objects that exist once startup is done (LITELLM_GC_FREEZE_AFTER_STARTUP). + + Full collections are stop-the-world and walk every tracked object; the startup heap + (modules, typing/Pydantic metadata, cost map, config) never becomes garbage, so moving + it to the permanent generation shortens those pauses. Startup garbage is collected first + so it is not frozen with the rest. + """ + if not enabled: + return + gc.collect() + gc.freeze() + verbose_proxy_logger.info("GC: froze %s objects after startup", gc.get_freeze_count()) + + @router.get( "/debug/asyncio-tasks", dependencies=[Depends(user_api_key_auth)], From f7e3daeecbdd4a0934bcf1ea0b5376f311b02bac Mon Sep 17 00:00:00 2001 From: Apeksha_2022 <70309060+Apeksha9389@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:41:38 +0530 Subject: [PATCH 3/8] feat(proxy): freeze GC before serving when enabled --- litellm/proxy/proxy_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 03ab1752a92..5a8f3753733 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -431,7 +431,7 @@ from litellm.proxy.common_utils.codex_model_catalog import codex_model_list_body from litellm.proxy.common_utils.config_includes import resolve_include_file_path, resolve_includes from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber from litellm.proxy.common_utils.credential_hydration import decrypted_or_stored -from litellm.proxy.common_utils.debug_utils import init_verbose_loggers +from litellm.proxy.common_utils.debug_utils import freeze_gc_after_startup, init_verbose_loggers from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router from litellm.proxy.common_utils.discoverable_model_filter import ( discoverable_rows, @@ -1737,6 +1737,7 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState async with AsyncExitStack() as admin_mcp_stack: try: await admin_mcp_stack.enter_async_context(admin_mcp_lifespan(app)) + freeze_gc_after_startup() yield state finally: if model_info_scheduler is not None and model_info_scheduler.running: From 127b8b9e2c2d110af69a9147f24e6c31c8dee5ae Mon Sep 17 00:00:00 2001 From: Apeksha_2022 <70309060+Apeksha9389@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:42:15 +0530 Subject: [PATCH 4/8] test(proxy): cover freeze_gc_after_startup --- .../proxy/common_utils/test_debug_utils.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/proxy/common_utils/test_debug_utils.py b/tests/unit/proxy/common_utils/test_debug_utils.py index 456be4e8c00..30550cb141a 100644 --- a/tests/unit/proxy/common_utils/test_debug_utils.py +++ b/tests/unit/proxy/common_utils/test_debug_utils.py @@ -1,3 +1,4 @@ +import gc import json import os import socket @@ -17,6 +18,7 @@ from litellm.proxy.common_utils.debug_utils import ( PSUTIL_MISSING_ERROR, _ProcFilesystemProcess, _summary_process_memory, + freeze_gc_after_startup, get_memory_summary, ) from litellm.proxy.common_utils.debug_utils import router as debug_router @@ -198,3 +200,27 @@ def test_memory_routes_count_prompt_caching_pins( assert response.status_code == 200 assert pin_count(response.json()) == expected + + +@pytest.fixture +def unfrozen_gc() -> Iterator[None]: + yield + gc.unfreeze() + + +def test_freeze_gc_after_startup_freezes_existing_objects(unfrozen_gc: None) -> None: + gc.unfreeze() + startup_object = {"loaded": "at startup"} + + freeze_gc_after_startup(enabled=True) + + assert gc.get_freeze_count() > 0 + assert not any(obj is startup_object for obj in gc.get_objects()) + + +def test_freeze_gc_after_startup_is_off_by_default(unfrozen_gc: None) -> None: + gc.unfreeze() + + freeze_gc_after_startup() + + assert gc.get_freeze_count() == 0 From bda4f80c9b1ff3ab1c537581334cf2c3b2c63d7d Mon Sep 17 00:00:00 2001 From: Apeksha_2022 <70309060+Apeksha9389@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:43:00 +0530 Subject: [PATCH 5/8] test(proxy): lifespan freezes GC before serving --- .../unit/proxy/proxy_server/test_lifecycle.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/unit/proxy/proxy_server/test_lifecycle.py b/tests/unit/proxy/proxy_server/test_lifecycle.py index 69f9c72c9c8..7dae370606b 100644 --- a/tests/unit/proxy/proxy_server/test_lifecycle.py +++ b/tests/unit/proxy/proxy_server/test_lifecycle.py @@ -900,6 +900,26 @@ async def test_proxy_startup_event_honors_disable_model_info_refresh( ) +@pytest.mark.asyncio +async def test_proxy_startup_event_freezes_gc_before_serving() -> None: + """The opt-in gc.freeze (#46517) runs once startup is done and before the first request.""" + scheduler = AsyncIOScheduler() + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} | { + "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true" + } + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.object(ps, "scheduler", scheduler), + patch.object(ps, "freeze_gc_after_startup") as freeze_gc, + ): + try: + async with proxy_startup_event(app=None): + freeze_gc.assert_called_once_with() + finally: + if scheduler.running: + scheduler.shutdown(wait=False) + + def test_otel_global_provider_published_after_callback_init(): """The OTel V2 global-provider publish must run after callback initialization in ``proxy_startup_event``. From f8b423166dc3913c72104e5756a08f32b2a3416d Mon Sep 17 00:00:00 2001 From: Apeksha_2022 <70309060+Apeksha9389@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:59:52 +0530 Subject: [PATCH 6/8] refactor(proxy): read the gc-freeze setting at call time --- litellm/proxy/common_utils/debug_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index a6001932984..23cbabb3caf 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -60,7 +60,7 @@ def configure_gc_thresholds(): configure_gc_thresholds() -def freeze_gc_after_startup(enabled: bool = LITELLM_GC_FREEZE_AFTER_STARTUP) -> None: +def freeze_gc_after_startup() -> None: """Freeze the objects that exist once startup is done (LITELLM_GC_FREEZE_AFTER_STARTUP). Full collections are stop-the-world and walk every tracked object; the startup heap @@ -68,7 +68,7 @@ def freeze_gc_after_startup(enabled: bool = LITELLM_GC_FREEZE_AFTER_STARTUP) -> it to the permanent generation shortens those pauses. Startup garbage is collected first so it is not frozen with the rest. """ - if not enabled: + if not LITELLM_GC_FREEZE_AFTER_STARTUP: return gc.collect() gc.freeze() From 1560f0d666e9737b346a0568c05cf5354e8fc1a6 Mon Sep 17 00:00:00 2001 From: Apeksha_2022 <70309060+Apeksha9389@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:00:57 +0530 Subject: [PATCH 7/8] test(proxy): toggle the gc-freeze setting via monkeypatch --- tests/unit/proxy/common_utils/test_debug_utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/unit/proxy/common_utils/test_debug_utils.py b/tests/unit/proxy/common_utils/test_debug_utils.py index 30550cb141a..d635091a6ff 100644 --- a/tests/unit/proxy/common_utils/test_debug_utils.py +++ b/tests/unit/proxy/common_utils/test_debug_utils.py @@ -14,11 +14,11 @@ from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.bug_report_config import build_proxy_bug_report +from litellm.proxy.common_utils import debug_utils from litellm.proxy.common_utils.debug_utils import ( PSUTIL_MISSING_ERROR, _ProcFilesystemProcess, _summary_process_memory, - freeze_gc_after_startup, get_memory_summary, ) from litellm.proxy.common_utils.debug_utils import router as debug_router @@ -204,23 +204,24 @@ def test_memory_routes_count_prompt_caching_pins( @pytest.fixture def unfrozen_gc() -> Iterator[None]: + gc.unfreeze() yield gc.unfreeze() -def test_freeze_gc_after_startup_freezes_existing_objects(unfrozen_gc: None) -> None: - gc.unfreeze() +def test_freeze_gc_after_startup_freezes_existing_objects(unfrozen_gc: None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(debug_utils, "LITELLM_GC_FREEZE_AFTER_STARTUP", True) startup_object = {"loaded": "at startup"} - freeze_gc_after_startup(enabled=True) + debug_utils.freeze_gc_after_startup() assert gc.get_freeze_count() > 0 assert not any(obj is startup_object for obj in gc.get_objects()) -def test_freeze_gc_after_startup_is_off_by_default(unfrozen_gc: None) -> None: - gc.unfreeze() +def test_freeze_gc_after_startup_does_nothing_when_disabled(unfrozen_gc: None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(debug_utils, "LITELLM_GC_FREEZE_AFTER_STARTUP", False) - freeze_gc_after_startup() + debug_utils.freeze_gc_after_startup() assert gc.get_freeze_count() == 0 From 03008a5a734d1d1238a1d9081298a0e434354332 Mon Sep 17 00:00:00 2001 From: Apeksha_2022 <70309060+Apeksha9389@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:01:40 +0530 Subject: [PATCH 8/8] test(proxy): assert the heap is frozen while serving (TQ002) --- tests/unit/proxy/proxy_server/test_lifecycle.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/unit/proxy/proxy_server/test_lifecycle.py b/tests/unit/proxy/proxy_server/test_lifecycle.py index 7dae370606b..73f4358d437 100644 --- a/tests/unit/proxy/proxy_server/test_lifecycle.py +++ b/tests/unit/proxy/proxy_server/test_lifecycle.py @@ -17,6 +17,7 @@ Pins covered: from __future__ import annotations +import gc import inspect import json import logging @@ -31,6 +32,7 @@ from pydantic import BaseModel from typing_extensions import TypedDict import litellm.proxy.proxy_server as ps +from litellm.proxy.common_utils import debug_utils from litellm.proxy.proxy_server import ( ProxyStartupEvent, initialize_shared_aiohttp_session, @@ -902,23 +904,28 @@ async def test_proxy_startup_event_honors_disable_model_info_refresh( @pytest.mark.asyncio async def test_proxy_startup_event_freezes_gc_before_serving() -> None: - """The opt-in gc.freeze (#46517) runs once startup is done and before the first request.""" + """With LITELLM_GC_FREEZE_AFTER_STARTUP (#46517) the startup heap is already frozen + when the proxy starts serving.""" scheduler = AsyncIOScheduler() clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} | { "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true" } + gc.unfreeze() with ( patch.dict(os.environ, clean_env, clear=True), patch.object(ps, "scheduler", scheduler), - patch.object(ps, "freeze_gc_after_startup") as freeze_gc, + patch.object(debug_utils, "LITELLM_GC_FREEZE_AFTER_STARTUP", True), ): try: async with proxy_startup_event(app=None): - freeze_gc.assert_called_once_with() + frozen_while_serving = gc.get_freeze_count() finally: + gc.unfreeze() if scheduler.running: scheduler.shutdown(wait=False) + assert frozen_while_serving > 0 + def test_otel_global_provider_published_after_callback_init(): """The OTel V2 global-provider publish must run after callback