From a112ec5b022049e2247e1ea74d1ad1ab7bc90486 Mon Sep 17 00:00:00 2001 From: "Jugal D. Bhatt" <55304795+jugaldb@users.noreply.github.com> Date: Fri, 18 Jul 2025 23:47:15 +0530 Subject: [PATCH] Health check app on separate port (#12718) * add separate health app * add new docs * refactor * fix colons * Update config_settings.md * refactor * docs * add unit test * added supervisord * remove app * add supervisor conf * Add markdown * add video to md * remove test * docs build failure * add to all docker files, change prod.md and add tests * change dockerfiles * remove extra file * remove extra file * remove extra file * change apt->apk * remove rdb file * add fixed file --- Dockerfile | 3 + docker/Dockerfile.database | 4 + docker/Dockerfile.non_root | 7 ++ docker/prod_entrypoint.sh | 4 + docker/supervisord.conf | 42 ++++++++ docs/my-website/docs/proxy/prod.md | 15 ++- .../health_endpoints/health_app_factory.py | 7 ++ litellm/proxy/proxy_cli.py | 22 +---- tests/test_litellm/proxy/test_proxy_cli.py | 99 ++++++++++--------- 9 files changed, 138 insertions(+), 65 deletions(-) create mode 100644 docker/supervisord.conf create mode 100644 litellm/proxy/health_endpoints/health_app_factory.py diff --git a/Dockerfile b/Dockerfile index b972aab0961..e88c9d99165 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,6 +72,9 @@ RUN chmod +x docker/prod_entrypoint.sh EXPOSE 4000/tcp +RUN apk add --no-cache supervisor +COPY docker/supervisord.conf /etc/supervisord.conf + ENTRYPOINT ["docker/prod_entrypoint.sh"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index da0326fd2cd..3b8566c31ba 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -71,8 +71,12 @@ RUN chmod +x docker/entrypoint.sh RUN chmod +x docker/prod_entrypoint.sh EXPOSE 4000/tcp +RUN apk add --no-cache supervisor +COPY docker/supervisord.conf /etc/supervisord.conf + # # Set your entrypoint and command + ENTRYPOINT ["docker/prod_entrypoint.sh"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 079778cafb8..05d38ef27f7 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -41,6 +41,8 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt # Runtime stage FROM $LITELLM_RUNTIME_IMAGE AS runtime +# Ensure we are root for package installation +USER root # Update dependencies and clean up - handles debian security issue RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* @@ -87,6 +89,11 @@ RUN prisma generate EXPOSE 4000/tcp +USER root +RUN apt-get update && apt-get install -y supervisor && rm -rf /var/lib/apt/lists/* +USER nobody +COPY docker/supervisord.conf /etc/supervisord.conf + # # Set your entrypoint and command ENTRYPOINT ["docker/prod_entrypoint.sh"] diff --git a/docker/prod_entrypoint.sh b/docker/prod_entrypoint.sh index ea94c343801..5a0dd8660ec 100644 --- a/docker/prod_entrypoint.sh +++ b/docker/prod_entrypoint.sh @@ -1,5 +1,9 @@ #!/bin/sh +if [ "$SEPARATE_HEALTH_APP" = "1" ]; then + exec supervisord -c /etc/supervisord.conf +fi + if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False" exec ddtrace-run litellm "$@" diff --git a/docker/supervisord.conf b/docker/supervisord.conf new file mode 100644 index 00000000000..193709e7ee9 --- /dev/null +++ b/docker/supervisord.conf @@ -0,0 +1,42 @@ +[supervisord] +nodaemon=true +loglevel=info + +[group:litellm] +programs=main,health + +[program:main] +command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000; fi' +autostart=true +autorestart=true +startretries=3 +priority=1 +exitcodes=0 +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stderr +stdout_logfile_maxbytes = 0 +stderr_logfile_maxbytes = 0 +environment=PYTHONUNBUFFERED=true + +[program:health] +command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0' +autostart=true +autorestart=true +startretries=3 +priority=2 +exitcodes=0 +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stderr +stdout_logfile_maxbytes = 0 +stderr_logfile_maxbytes = 0 +environment=PYTHONUNBUFFERED=true + +[eventlistener:process_monitor] +command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]" +events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL +autostart=true +autorestart=true \ No newline at end of file diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index ba99f412199..c443e42f78c 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -233,6 +233,10 @@ To fix this, just set `LITELLM_MIGRATION_DIR="/path/to/writeable/directory"` in LiteLLM will use this directory to write migration files. ## 10. Use a Separate Health Check App +:::info +The Separate Health Check App only runs when running via the the LiteLLM Docker Image and using Docker and setting the SEPARATE_HEALTH_APP env var to "1" +::: + Using a separate health check app ensures that your liveness and readiness probes remain responsive even when the main application is under heavy load. **Why is this important?** @@ -240,7 +244,8 @@ Using a separate health check app ensures that your liveness and readiness probe - If your health endpoints share the same process as your main app, high traffic or resource exhaustion can cause health checks to hang or fail. - When Kubernetes liveness probes hang or time out, it may incorrectly assume your pod is unhealthy and restart it—even if the main app is just busy, not dead. - By running health endpoints on a separate lightweight FastAPI app (with its own port), you guarantee that health checks remain fast and reliable, preventing unnecessary pod restarts during traffic spikes or heavy workloads. -- Since the proxy and health app are running in the same pod, if health check app fails, it signifies that the pod is unhealthy and needs to restart/have action taken upon. +- The way it works is, if either of the health or main proxy app dies due to whatever reason, it will kill the pod and which would be marked as unhealthy prompting the orchestrator to restart the pod +- Since the proxy and health app are running in the same pod, if the pod dies the health check probe fails, it signifies that the pod is unhealthy and needs to restart/have action taken upon. **How to enable:** @@ -250,6 +255,14 @@ SEPARATE_HEALTH_APP="1" # Default "0" SEPARATE_HEALTH_PORT="8001" # Default "4001", Works only if `SEPARATE_HEALTH_APP` is "1" ``` + + +Or [watch on Loom](https://www.loom.com/share/b08be303331246b88fdc053940d03281?sid=a145ec66-d55f-41f7-aade-a9f41fbe752d). + + ## Extras ### Expected Performance in Production diff --git a/litellm/proxy/health_endpoints/health_app_factory.py b/litellm/proxy/health_endpoints/health_app_factory.py new file mode 100644 index 00000000000..7737969318e --- /dev/null +++ b/litellm/proxy/health_endpoints/health_app_factory.py @@ -0,0 +1,7 @@ +from fastapi import FastAPI +from litellm.proxy.health_endpoints._health_endpoints import router as health_router + +def build_health_app(): + health_app = FastAPI(title="LiteLLM Health Endpoints") + health_app.include_router(health_router) + return health_app \ No newline at end of file diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 0be75a8b99b..729db54b1b0 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -301,24 +301,6 @@ class ProxyInitializationHelpers: return None # Let uvicorn choose the default loop on Windows return "uvloop" - -def run_separate_health_app(): - separate_health = os.getenv("SEPARATE_HEALTH_APP", "0") - if separate_health == "1": - from fastapi import FastAPI - from litellm.proxy.health_endpoints._health_endpoints import router as health_router - health_app = FastAPI(title="LiteLLM Health Endpoints") - health_app.include_router(health_router) - health_port = int(os.getenv("SEPARATE_HEALTH_PORT", 4001)) - print(f"\033[1;32mLiteLLM Health Endpoints: Running on http://0.0.0.0:{health_port}\033[0m") - import threading - def run_health_app(): - import uvicorn - uvicorn.run(health_app, host="0.0.0.0", port=health_port, log_level="info") - t = threading.Thread(target=run_health_app, daemon=True) - t.start() - - @click.command() @click.option( "--host", default="0.0.0.0", help="Host for the server to listen on.", envvar="HOST" @@ -809,7 +791,9 @@ def run_server( # noqa: PLR0915 from litellm.proxy.proxy_server import app # noqa # --- SEPARATE HEALTH APP LOGIC --- - run_separate_health_app() + # To run the health app separately, use: + # uvicorn litellm.proxy.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=4001 + # This is compatible with the SEPARATE_HEALTH_APP Docker/supervisord pattern. # --- END SEPARATE HEALTH APP LOGIC --- # Skip server startup if requested (after all setup is done) diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index b78c08d5a9c..22d0f56dbe7 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -3,14 +3,15 @@ import sys from unittest.mock import MagicMock, patch import pytest +import fastapi sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system-path from litellm.proxy.proxy_cli import ProxyInitializationHelpers +from litellm.proxy.health_endpoints.health_app_factory import build_health_app import builtins -from litellm.proxy.proxy_cli import run_separate_health_app import types @@ -313,47 +314,55 @@ class TestProxyInitializationHelpers: assert call_args[1]["timeout_keep_alive"] == 30 -def test_run_separate_health_app_runs_only_when_flag(monkeypatch): - # Patch print to track output - printed = [] - monkeypatch.setattr(builtins, "print", lambda *args, **kwargs: printed.append(args)) - # Patch threading.Thread to track if it's started - started = {} - class DummyThread: - def __init__(self, target, daemon): - started['called'] = True - def start(self): - started['started'] = True - monkeypatch.setattr("threading.Thread", DummyThread) - # Patch uvicorn.run to prevent actual server start - monkeypatch.setattr("uvicorn.run", lambda *a, **k: None) - # Patch FastAPI import by patching sys.modules['fastapi'] - class DummyFastAPI: - def __init__(self, *a, **k): - pass - def include_router(self, router): - pass - dummy_fastapi = types.ModuleType("fastapi") - dummy_fastapi.FastAPI = DummyFastAPI - monkeypatch.setitem(sys.modules, "fastapi", dummy_fastapi) - # Patch the health_endpoints module in sys.modules - dummy_health_endpoints = types.ModuleType("litellm.proxy.health_endpoints._health_endpoints") - dummy_health_endpoints.router = object() - monkeypatch.setitem(sys.modules, "litellm.proxy.health_endpoints._health_endpoints", dummy_health_endpoints) - - # Should NOT run if flag is not set - monkeypatch.setenv("SEPARATE_HEALTH_APP", "0") - started.clear() - printed.clear() - run_separate_health_app() - assert not started, "Should not start thread if flag is not '1'" - assert not printed, "Should not print if flag is not '1'" - - # Should run if flag is set - monkeypatch.setenv("SEPARATE_HEALTH_APP", "1") - started.clear() - printed.clear() - run_separate_health_app() - assert started.get('called'), "Should create thread if flag is '1'" - assert started.get('started') is None or started.get('started'), "Should start thread if flag is '1'" - assert any("LiteLLM Health Endpoints" in str(args) for args in printed), "Should print health endpoint message if flag is '1'" +class TestHealthAppFactory: + """Test cases for the health app factory module""" + + def test_build_health_app(self): + """Test that build_health_app creates a FastAPI app with the correct title and includes the health router""" + # Execute + health_app = build_health_app() + + # Assert + assert health_app.title == "LiteLLM Health Endpoints" + assert isinstance(health_app, fastapi.FastAPI) + + # Verify that the app has the expected health endpoints by checking route paths + # When a router is included, its routes are flattened into the main app's routes + route_paths = [] + for route in health_app.routes: + if hasattr(route, 'path'): + route_paths.append(route.path) + + # Check for some expected health endpoints + expected_paths = [ + "/test", + "/health/services", + "/health", + "/health/history", + "/health/latest", + "/settings", + "/active/callbacks", + "/health/readiness", + "/health/liveliness", + "/health/liveness", + "/health/test_connection" + ] + + # At least some of the expected health endpoints should be present + found_paths = [path for path in expected_paths if path in route_paths] + assert len(found_paths) > 0, f"Expected to find health endpoints, but found: {route_paths}" + + # Verify that the app has routes (indicating the router was included) + assert len(health_app.routes) > 0, "Health app should have routes from the included router" + + def test_build_health_app_returns_different_instances(self): + """Test that build_health_app returns different FastAPI instances on each call""" + # Execute + health_app_1 = build_health_app() + health_app_2 = build_health_app() + + # Assert + assert health_app_1 is not health_app_2 + assert health_app_1.title == health_app_2.title + assert isinstance(health_app_1, fastapi.FastAPI) + assert isinstance(health_app_2, fastapi.FastAPI)