diff --git a/docs/my-website/docs/proxy/pyroscope_profiling.md b/docs/my-website/docs/proxy/pyroscope_profiling.md new file mode 100644 index 00000000000..fa3db3a8782 --- /dev/null +++ b/docs/my-website/docs/proxy/pyroscope_profiling.md @@ -0,0 +1,43 @@ +# Grafana Pyroscope CPU profiling + +LiteLLM proxy can send continuous CPU profiles to [Grafana Pyroscope](https://grafana.com/docs/pyroscope/latest/) when enabled via environment variables. This is optional and off by default. + +## Quick start + +1. **Install the optional dependency** (required only when enabling Pyroscope): + + ```bash + pip install pyroscope-io + ``` + + Or install the proxy extra: + + ```bash + pip install "litellm[proxy]" + ``` + +2. **Set environment variables** before starting the proxy: + + | Variable | Required | Description | + |----------|----------|-------------| + | `LITELLM_ENABLE_PYROSCOPE` | Yes (to enable) | Set to `true` to enable Pyroscope profiling. | + | `PYROSCOPE_APP_NAME` | Yes (when enabled) | Application name shown in the Pyroscope UI. | + | `PYROSCOPE_SERVER_ADDRESS` | Yes (when enabled) | Pyroscope server URL (e.g. `http://localhost:4040`). | + | `PYROSCOPE_SAMPLE_RATE` | No | Sample rate (integer). If unset, the pyroscope-io library default is used. | + +3. **Start the proxy**; profiling will begin automatically when the proxy starts. + + ```bash + export LITELLM_ENABLE_PYROSCOPE=true + export PYROSCOPE_APP_NAME=litellm-proxy + export PYROSCOPE_SERVER_ADDRESS=http://localhost:4040 + litellm --config config.yaml + ``` + +4. **View profiles** in the Pyroscope (or Grafana) UI and select your `PYROSCOPE_APP_NAME`. + +## Notes + +- **Optional dependency**: `pyroscope-io` is an optional dependency. If it is not installed and `LITELLM_ENABLE_PYROSCOPE=true`, the proxy will log a warning and continue without profiling. +- **Platform support**: The `pyroscope-io` package uses a native extension and is not available on all platforms (e.g. Windows is excluded by the package). +- **Other settings**: See [Configuration settings](/proxy/config_settings) for all proxy environment variables. diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 20381f8c46a..06402d8ffcd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5559,9 +5559,9 @@ class ProxyStartupEvent: Optional continuous profiling via Grafana Pyroscope. Off by default. Enable with LITELLM_ENABLE_PYROSCOPE=true. - Requires: pip install pyroscope-io - Sends profiles to PYROSCOPE_SERVER_ADDRESS (default http://localhost:4040). - Optional: PYROSCOPE_SAMPLE_RATE (float, no default) to set the sample rate. + Requires: pip install pyroscope-io (optional dependency). + When enabled, PYROSCOPE_SERVER_ADDRESS and PYROSCOPE_APP_NAME are required (no defaults). + Optional: PYROSCOPE_SAMPLE_RATE (parsed as integer) to set the sample rate. """ if not get_secret_bool("LITELLM_ENABLE_PYROSCOPE", False): verbose_proxy_logger.info( diff --git a/pyproject.toml b/pyproject.toml index 0d3f4e5af2d..5a30915d638 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ polars = {version = "^1.31.0", optional = true, python = ">=3.10"} semantic-router = {version = ">=0.1.12", optional = true, python = ">=3.9,<3.14"} mlflow = {version = ">3.1.4", optional = true, python = ">=3.10"} soundfile = {version = "^0.12.1", optional = true} -pyroscope-io = {version = "^0.8", markers = "sys_platform != 'win32'"} +pyroscope-io = {version = "^0.8", optional = true, markers = "sys_platform != 'win32'"} # grpcio constraints: # - 1.62.3+ required by grpcio-status # - 1.68.0-1.68.1 has reconnect bug (https://github.com/grpc/grpc/issues/38290) diff --git a/tests/test_litellm/proxy/test_pyroscope.py b/tests/test_litellm/proxy/test_pyroscope.py new file mode 100644 index 00000000000..6bfdf81ec1a --- /dev/null +++ b/tests/test_litellm/proxy/test_pyroscope.py @@ -0,0 +1,138 @@ +"""Unit tests for ProxyStartupEvent._init_pyroscope (Grafana Pyroscope profiling).""" + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.proxy.proxy_server import ProxyStartupEvent + + +def _mock_pyroscope_module(): + """Return a mock module so 'import pyroscope' succeeds in _init_pyroscope.""" + m = MagicMock() + m.configure = MagicMock() + return m + + +def test_init_pyroscope_returns_cleanly_when_disabled(): + """When LITELLM_ENABLE_PYROSCOPE is false, _init_pyroscope returns without error.""" + with patch( + "litellm.proxy.proxy_server.get_secret_bool", + return_value=False, + ): + ProxyStartupEvent._init_pyroscope() + + +def test_init_pyroscope_raises_when_enabled_but_missing_app_name(): + """When LITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_APP_NAME is not set, raises ValueError.""" + mock_pyroscope = _mock_pyroscope_module() + with patch( + "litellm.proxy.proxy_server.get_secret_bool", + return_value=True, + ), patch.dict( + sys.modules, + {"pyroscope": mock_pyroscope}, + ), patch.dict( + os.environ, + { + "PYROSCOPE_APP_NAME": "", + "PYROSCOPE_SERVER_ADDRESS": "http://localhost:4040", + }, + clear=False, + ): + with pytest.raises(ValueError, match="PYROSCOPE_APP_NAME"): + ProxyStartupEvent._init_pyroscope() + + +def test_init_pyroscope_raises_when_enabled_but_missing_server_address(): + """When LITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_SERVER_ADDRESS is not set, raises ValueError.""" + mock_pyroscope = _mock_pyroscope_module() + with patch( + "litellm.proxy.proxy_server.get_secret_bool", + return_value=True, + ), patch.dict( + sys.modules, + {"pyroscope": mock_pyroscope}, + ), patch.dict( + os.environ, + { + "PYROSCOPE_APP_NAME": "myapp", + "PYROSCOPE_SERVER_ADDRESS": "", + }, + clear=False, + ): + with pytest.raises(ValueError, match="PYROSCOPE_SERVER_ADDRESS"): + ProxyStartupEvent._init_pyroscope() + + +def test_init_pyroscope_raises_when_sample_rate_invalid(): + """When PYROSCOPE_SAMPLE_RATE is not a number, raises ValueError.""" + mock_pyroscope = _mock_pyroscope_module() + with patch( + "litellm.proxy.proxy_server.get_secret_bool", + return_value=True, + ), patch.dict( + sys.modules, + {"pyroscope": mock_pyroscope}, + ), patch.dict( + os.environ, + { + "PYROSCOPE_APP_NAME": "myapp", + "PYROSCOPE_SERVER_ADDRESS": "http://localhost:4040", + "PYROSCOPE_SAMPLE_RATE": "not-a-number", + }, + clear=False, + ): + with pytest.raises(ValueError, match="PYROSCOPE_SAMPLE_RATE"): + ProxyStartupEvent._init_pyroscope() + + +def test_init_pyroscope_accepts_integer_sample_rate(): + """When enabled with valid config and integer sample rate, configures pyroscope.""" + mock_pyroscope = _mock_pyroscope_module() + with patch( + "litellm.proxy.proxy_server.get_secret_bool", + return_value=True, + ), patch.dict( + sys.modules, + {"pyroscope": mock_pyroscope}, + ), patch.dict( + os.environ, + { + "PYROSCOPE_APP_NAME": "myapp", + "PYROSCOPE_SERVER_ADDRESS": "http://localhost:4040", + "PYROSCOPE_SAMPLE_RATE": "100", + }, + clear=False, + ): + ProxyStartupEvent._init_pyroscope() + mock_pyroscope.configure.assert_called_once() + call_kw = mock_pyroscope.configure.call_args[1] + assert call_kw["app_name"] == "myapp" + assert call_kw["server_address"] == "http://localhost:4040" + assert call_kw["sample_rate"] == 100 + + +def test_init_pyroscope_accepts_float_sample_rate_parsed_as_int(): + """PYROSCOPE_SAMPLE_RATE can be a float string; it is parsed as integer.""" + mock_pyroscope = _mock_pyroscope_module() + with patch( + "litellm.proxy.proxy_server.get_secret_bool", + return_value=True, + ), patch.dict( + sys.modules, + {"pyroscope": mock_pyroscope}, + ), patch.dict( + os.environ, + { + "PYROSCOPE_APP_NAME": "myapp", + "PYROSCOPE_SERVER_ADDRESS": "http://localhost:4040", + "PYROSCOPE_SAMPLE_RATE": "100.7", + }, + clear=False, + ): + ProxyStartupEvent._init_pyroscope() + call_kw = mock_pyroscope.configure.call_args[1] + assert call_kw["sample_rate"] == 100