Health check app on separate port (#12718)

* add separate health app

* add new docs

* refactor

* fix colons

* Update config_settings.md

* refactor

* docs

* add unit test

* added supervisord

* remove app

* add supervisor conf

* Add markdown

* add video to md

* remove test

* docs build failure

* add to all docker files, change prod.md and add tests

* change dockerfiles

* remove extra file

* remove extra file

* remove extra file

* change apt->apk

* remove rdb file

* add fixed file

Dockerfile

@@ -72,6 +72,9 @@ RUN chmod +x docker/prod_entrypoint.sh
 
 EXPOSE 4000/tcp
 
+RUN apk add --no-cache supervisor
+COPY docker/supervisord.conf /etc/supervisord.conf
+
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 
 # Append "--detailed_debug" to the end of CMD to view detailed debug logs

docker/Dockerfile.database

@@ -71,8 +71,12 @@ RUN chmod +x docker/entrypoint.sh
 RUN chmod +x docker/prod_entrypoint.sh
 EXPOSE 4000/tcp
 
+RUN apk add --no-cache supervisor
+COPY docker/supervisord.conf /etc/supervisord.conf
+
 # # Set your entrypoint and command
 
+
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 
 # Append "--detailed_debug" to the end of CMD to view detailed debug logs 

docker/Dockerfile.non_root

@@ -41,6 +41,8 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
+# Ensure we are root for package installation
+USER root
 # Update dependencies and clean up - handles debian security issue
 RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
 
@@ -87,6 +89,11 @@ RUN prisma generate
 
 EXPOSE 4000/tcp
 
+USER root
+RUN apt-get update && apt-get install -y supervisor && rm -rf /var/lib/apt/lists/*
+USER nobody
+COPY docker/supervisord.conf /etc/supervisord.conf
+
 # # Set your entrypoint and command
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 

docker/prod_entrypoint.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 
+if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
+    exec supervisord -c /etc/supervisord.conf
+fi
+
 if [ "$USE_DDTRACE" = "true" ]; then
     export DD_TRACE_OPENAI_ENABLED="False"
     exec ddtrace-run litellm "$@"

docker/supervisord.conf

@@ -0,0 +1,42 @@
+[supervisord]
+nodaemon=true
+loglevel=info
+
+[group:litellm]
+programs=main,health
+
+[program:main]
+command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000; fi'
+autostart=true
+autorestart=true
+startretries=3
+priority=1
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+environment=PYTHONUNBUFFERED=true
+
+[program:health]
+command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
+autostart=true
+autorestart=true
+startretries=3
+priority=2
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+environment=PYTHONUNBUFFERED=true
+
+[eventlistener:process_monitor]
+command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
+events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
+autostart=true
+autorestart=true

docs/my-website/docs/proxy/prod.md

@@ -233,6 +233,10 @@ To fix this, just set `LITELLM_MIGRATION_DIR="/path/to/writeable/directory"` in
 LiteLLM will use this directory to write migration files.
 
 ## 10. Use a Separate Health Check App
+:::info
+The Separate Health Check App only runs when running via the the LiteLLM Docker Image and using Docker and setting the SEPARATE_HEALTH_APP env var to "1"
+:::
+
 Using a separate health check app ensures that your liveness and readiness probes remain responsive even when the main application is under heavy load. 
 
 **Why is this important?**
@@ -240,7 +244,8 @@ Using a separate health check app ensures that your liveness and readiness probe
 - If your health endpoints share the same process as your main app, high traffic or resource exhaustion can cause health checks to hang or fail.
 - When Kubernetes liveness probes hang or time out, it may incorrectly assume your pod is unhealthy and restart it—even if the main app is just busy, not dead.
 - By running health endpoints on a separate lightweight FastAPI app (with its own port), you guarantee that health checks remain fast and reliable, preventing unnecessary pod restarts during traffic spikes or heavy workloads.
-- Since the proxy and health app are running in the same pod, if health check app fails, it signifies that the pod is unhealthy and needs to restart/have action taken upon.
+- The way it works is, if either of the health or main proxy app dies due to whatever reason, it will kill the pod and which would be marked as unhealthy prompting the orchestrator to restart the pod
+- Since the proxy and health app are running in the same pod, if the pod dies the health check probe fails, it signifies that the pod is unhealthy and needs to restart/have action taken upon.
 
 **How to enable:**
 
@@ -250,6 +255,14 @@ SEPARATE_HEALTH_APP="1" # Default "0"
 SEPARATE_HEALTH_PORT="8001" # Default "4001", Works only if `SEPARATE_HEALTH_APP` is "1"
 ```
 
+<video controls width="100%" style={{ borderRadius: '8px', marginBottom: '1em' }}>
+  <source src="https://cdn.loom.com/sessions/thumbnails/b08be303331246b88fdc053940d03281-1718990992822.mp4" type="video/mp4" />
+  Your browser does not support the video tag.
+</video>
+
+Or [watch on Loom](https://www.loom.com/share/b08be303331246b88fdc053940d03281?sid=a145ec66-d55f-41f7-aade-a9f41fbe752d).
+
+
 ## Extras
 ### Expected Performance in Production
 

litellm/proxy/health_endpoints/health_app_factory.py

@@ -0,0 +1,7 @@
+from fastapi import FastAPI
+from litellm.proxy.health_endpoints._health_endpoints import router as health_router
+
+def build_health_app():
+    health_app = FastAPI(title="LiteLLM Health Endpoints")
+    health_app.include_router(health_router)
+    return health_app 

litellm/proxy/proxy_cli.py

@@ -301,24 +301,6 @@ class ProxyInitializationHelpers:
             return None  # Let uvicorn choose the default loop on Windows
         return "uvloop"
 
-
-def run_separate_health_app():
-    separate_health = os.getenv("SEPARATE_HEALTH_APP", "0")
-    if separate_health == "1":
-        from fastapi import FastAPI
-        from litellm.proxy.health_endpoints._health_endpoints import router as health_router
-        health_app = FastAPI(title="LiteLLM Health Endpoints")
-        health_app.include_router(health_router)
-        health_port = int(os.getenv("SEPARATE_HEALTH_PORT", 4001))
-        print(f"\033[1;32mLiteLLM Health Endpoints: Running on http://0.0.0.0:{health_port}\033[0m")
-        import threading
-        def run_health_app():
-            import uvicorn
-            uvicorn.run(health_app, host="0.0.0.0", port=health_port, log_level="info")
-        t = threading.Thread(target=run_health_app, daemon=True)
-        t.start()
-
-
 @click.command()
 @click.option(
     "--host", default="0.0.0.0", help="Host for the server to listen on.", envvar="HOST"
@@ -809,7 +791,9 @@ def run_server(  # noqa: PLR0915
         from litellm.proxy.proxy_server import app  # noqa
         
         # --- SEPARATE HEALTH APP LOGIC ---
-        run_separate_health_app()
+        # To run the health app separately, use:
+        #   uvicorn litellm.proxy.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=4001
+        # This is compatible with the SEPARATE_HEALTH_APP Docker/supervisord pattern.
         # --- END SEPARATE HEALTH APP LOGIC ---
         
         # Skip server startup if requested (after all setup is done)

tests/test_litellm/proxy/test_proxy_cli.py

@@ -3,14 +3,15 @@ import sys
 from unittest.mock import MagicMock, patch
 
 import pytest
+import fastapi
 
 sys.path.insert(
     0, os.path.abspath("../../..")
 )  # Adds the parent directory to the system-path
 
 from litellm.proxy.proxy_cli import ProxyInitializationHelpers
+from litellm.proxy.health_endpoints.health_app_factory import build_health_app
 import builtins
-from litellm.proxy.proxy_cli import run_separate_health_app
 import types
 
 
@@ -313,47 +314,55 @@ class TestProxyInitializationHelpers:
             assert call_args[1]["timeout_keep_alive"] == 30
 
 
-def test_run_separate_health_app_runs_only_when_flag(monkeypatch):
-    # Patch print to track output
-    printed = []
-    monkeypatch.setattr(builtins, "print", lambda *args, **kwargs: printed.append(args))
-    # Patch threading.Thread to track if it's started
-    started = {}
-    class DummyThread:
-        def __init__(self, target, daemon):
-            started['called'] = True
-        def start(self):
-            started['started'] = True
-    monkeypatch.setattr("threading.Thread", DummyThread)
-    # Patch uvicorn.run to prevent actual server start
-    monkeypatch.setattr("uvicorn.run", lambda *a, **k: None)
-    # Patch FastAPI import by patching sys.modules['fastapi']
-    class DummyFastAPI:
-        def __init__(self, *a, **k):
-            pass
-        def include_router(self, router):
-            pass
-    dummy_fastapi = types.ModuleType("fastapi")
-    dummy_fastapi.FastAPI = DummyFastAPI
-    monkeypatch.setitem(sys.modules, "fastapi", dummy_fastapi)
-    # Patch the health_endpoints module in sys.modules
-    dummy_health_endpoints = types.ModuleType("litellm.proxy.health_endpoints._health_endpoints")
-    dummy_health_endpoints.router = object()
-    monkeypatch.setitem(sys.modules, "litellm.proxy.health_endpoints._health_endpoints", dummy_health_endpoints)
-
-    # Should NOT run if flag is not set
-    monkeypatch.setenv("SEPARATE_HEALTH_APP", "0")
-    started.clear()
-    printed.clear()
-    run_separate_health_app()
-    assert not started, "Should not start thread if flag is not '1'"
-    assert not printed, "Should not print if flag is not '1'"
-
-    # Should run if flag is set
-    monkeypatch.setenv("SEPARATE_HEALTH_APP", "1")
-    started.clear()
-    printed.clear()
-    run_separate_health_app()
-    assert started.get('called'), "Should create thread if flag is '1'"
-    assert started.get('started') is None or started.get('started'), "Should start thread if flag is '1'"
-    assert any("LiteLLM Health Endpoints" in str(args) for args in printed), "Should print health endpoint message if flag is '1'"
+class TestHealthAppFactory:
+    """Test cases for the health app factory module"""
+    
+    def test_build_health_app(self):
+        """Test that build_health_app creates a FastAPI app with the correct title and includes the health router"""
+        # Execute
+        health_app = build_health_app()
+        
+        # Assert
+        assert health_app.title == "LiteLLM Health Endpoints"
+        assert isinstance(health_app, fastapi.FastAPI)
+        
+        # Verify that the app has the expected health endpoints by checking route paths
+        # When a router is included, its routes are flattened into the main app's routes
+        route_paths = []
+        for route in health_app.routes:
+            if hasattr(route, 'path'):
+                route_paths.append(route.path)
+        
+        # Check for some expected health endpoints
+        expected_paths = [
+            "/test",
+            "/health/services", 
+            "/health",
+            "/health/history",
+            "/health/latest",
+            "/settings",
+            "/active/callbacks",
+            "/health/readiness",
+            "/health/liveliness",
+            "/health/liveness",
+            "/health/test_connection"
+        ]
+        
+        # At least some of the expected health endpoints should be present
+        found_paths = [path for path in expected_paths if path in route_paths]
+        assert len(found_paths) > 0, f"Expected to find health endpoints, but found: {route_paths}"
+        
+        # Verify that the app has routes (indicating the router was included)
+        assert len(health_app.routes) > 0, "Health app should have routes from the included router"
+        
+    def test_build_health_app_returns_different_instances(self):
+        """Test that build_health_app returns different FastAPI instances on each call"""
+        # Execute
+        health_app_1 = build_health_app()
+        health_app_2 = build_health_app()
+        
+        # Assert
+        assert health_app_1 is not health_app_2
+        assert health_app_1.title == health_app_2.title
+        assert isinstance(health_app_1, fastapi.FastAPI)
+        assert isinstance(health_app_2, fastapi.FastAPI)