mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
Address Greptile PR feedback: Pyroscope optional, docs, tests, docstring
- pyproject.toml: mark pyroscope-io as optional=true (proxy extra only) - Add docs/my-website/docs/proxy/pyroscope_profiling.md (fix broken sidebar link) - Add tests/test_litellm/proxy/test_pyroscope.py for _init_pyroscope() - proxy_server: fix _init_pyroscope docstring (required server/app name, sample rate as int)
This commit is contained in:
Alexsander Hamir
parent
8087cab8aa
commit
dfa0293340
4 changed files with 185 additions and 4 deletions
43
docs/my-website/docs/proxy/pyroscope_profiling.md
Normal file
43
docs/my-website/docs/proxy/pyroscope_profiling.md
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
# Grafana Pyroscope CPU profiling
|
||||
|
||||
LiteLLM proxy can send continuous CPU profiles to [Grafana Pyroscope](https://grafana.com/docs/pyroscope/latest/) when enabled via environment variables. This is optional and off by default.
|
||||
|
||||
## Quick start
|
||||
|
||||
1. **Install the optional dependency** (required only when enabling Pyroscope):
|
||||
|
||||
```bash
|
||||
pip install pyroscope-io
|
||||
```
|
||||
|
||||
Or install the proxy extra:
|
||||
|
||||
```bash
|
||||
pip install "litellm[proxy]"
|
||||
```
|
||||
|
||||
2. **Set environment variables** before starting the proxy:
|
||||
|
||||
| Variable | Required | Description |
|
||||
|----------|----------|-------------|
|
||||
| `LITELLM_ENABLE_PYROSCOPE` | Yes (to enable) | Set to `true` to enable Pyroscope profiling. |
|
||||
| `PYROSCOPE_APP_NAME` | Yes (when enabled) | Application name shown in the Pyroscope UI. |
|
||||
| `PYROSCOPE_SERVER_ADDRESS` | Yes (when enabled) | Pyroscope server URL (e.g. `http://localhost:4040`). |
|
||||
| `PYROSCOPE_SAMPLE_RATE` | No | Sample rate (integer). If unset, the pyroscope-io library default is used. |
|
||||
|
||||
3. **Start the proxy**; profiling will begin automatically when the proxy starts.
|
||||
|
||||
```bash
|
||||
export LITELLM_ENABLE_PYROSCOPE=true
|
||||
export PYROSCOPE_APP_NAME=litellm-proxy
|
||||
export PYROSCOPE_SERVER_ADDRESS=http://localhost:4040
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
4. **View profiles** in the Pyroscope (or Grafana) UI and select your `PYROSCOPE_APP_NAME`.
|
||||
|
||||
## Notes
|
||||
|
||||
- **Optional dependency**: `pyroscope-io` is an optional dependency. If it is not installed and `LITELLM_ENABLE_PYROSCOPE=true`, the proxy will log a warning and continue without profiling.
|
||||
- **Platform support**: The `pyroscope-io` package uses a native extension and is not available on all platforms (e.g. Windows is excluded by the package).
|
||||
- **Other settings**: See [Configuration settings](/proxy/config_settings) for all proxy environment variables.
|
||||
|
|
@ -5559,9 +5559,9 @@ class ProxyStartupEvent:
|
|||
Optional continuous profiling via Grafana Pyroscope.
|
||||
|
||||
Off by default. Enable with LITELLM_ENABLE_PYROSCOPE=true.
|
||||
Requires: pip install pyroscope-io
|
||||
Sends profiles to PYROSCOPE_SERVER_ADDRESS (default http://localhost:4040).
|
||||
Optional: PYROSCOPE_SAMPLE_RATE (float, no default) to set the sample rate.
|
||||
Requires: pip install pyroscope-io (optional dependency).
|
||||
When enabled, PYROSCOPE_SERVER_ADDRESS and PYROSCOPE_APP_NAME are required (no defaults).
|
||||
Optional: PYROSCOPE_SAMPLE_RATE (parsed as integer) to set the sample rate.
|
||||
"""
|
||||
if not get_secret_bool("LITELLM_ENABLE_PYROSCOPE", False):
|
||||
verbose_proxy_logger.info(
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ polars = {version = "^1.31.0", optional = true, python = ">=3.10"}
|
|||
semantic-router = {version = ">=0.1.12", optional = true, python = ">=3.9,<3.14"}
|
||||
mlflow = {version = ">3.1.4", optional = true, python = ">=3.10"}
|
||||
soundfile = {version = "^0.12.1", optional = true}
|
||||
pyroscope-io = {version = "^0.8", markers = "sys_platform != 'win32'"}
|
||||
pyroscope-io = {version = "^0.8", optional = true, markers = "sys_platform != 'win32'"}
|
||||
# grpcio constraints:
|
||||
# - 1.62.3+ required by grpcio-status
|
||||
# - 1.68.0-1.68.1 has reconnect bug (https://github.com/grpc/grpc/issues/38290)
|
||||
|
|
|
|||
138
tests/test_litellm/proxy/test_pyroscope.py
Normal file
138
tests/test_litellm/proxy/test_pyroscope.py
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
"""Unit tests for ProxyStartupEvent._init_pyroscope (Grafana Pyroscope profiling)."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.proxy_server import ProxyStartupEvent
|
||||
|
||||
|
||||
def _mock_pyroscope_module():
|
||||
"""Return a mock module so 'import pyroscope' succeeds in _init_pyroscope."""
|
||||
m = MagicMock()
|
||||
m.configure = MagicMock()
|
||||
return m
|
||||
|
||||
|
||||
def test_init_pyroscope_returns_cleanly_when_disabled():
|
||||
"""When LITELLM_ENABLE_PYROSCOPE is false, _init_pyroscope returns without error."""
|
||||
with patch(
|
||||
"litellm.proxy.proxy_server.get_secret_bool",
|
||||
return_value=False,
|
||||
):
|
||||
ProxyStartupEvent._init_pyroscope()
|
||||
|
||||
|
||||
def test_init_pyroscope_raises_when_enabled_but_missing_app_name():
|
||||
"""When LITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_APP_NAME is not set, raises ValueError."""
|
||||
mock_pyroscope = _mock_pyroscope_module()
|
||||
with patch(
|
||||
"litellm.proxy.proxy_server.get_secret_bool",
|
||||
return_value=True,
|
||||
), patch.dict(
|
||||
sys.modules,
|
||||
{"pyroscope": mock_pyroscope},
|
||||
), patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"PYROSCOPE_APP_NAME": "",
|
||||
"PYROSCOPE_SERVER_ADDRESS": "http://localhost:4040",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
with pytest.raises(ValueError, match="PYROSCOPE_APP_NAME"):
|
||||
ProxyStartupEvent._init_pyroscope()
|
||||
|
||||
|
||||
def test_init_pyroscope_raises_when_enabled_but_missing_server_address():
|
||||
"""When LITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_SERVER_ADDRESS is not set, raises ValueError."""
|
||||
mock_pyroscope = _mock_pyroscope_module()
|
||||
with patch(
|
||||
"litellm.proxy.proxy_server.get_secret_bool",
|
||||
return_value=True,
|
||||
), patch.dict(
|
||||
sys.modules,
|
||||
{"pyroscope": mock_pyroscope},
|
||||
), patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"PYROSCOPE_APP_NAME": "myapp",
|
||||
"PYROSCOPE_SERVER_ADDRESS": "",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
with pytest.raises(ValueError, match="PYROSCOPE_SERVER_ADDRESS"):
|
||||
ProxyStartupEvent._init_pyroscope()
|
||||
|
||||
|
||||
def test_init_pyroscope_raises_when_sample_rate_invalid():
|
||||
"""When PYROSCOPE_SAMPLE_RATE is not a number, raises ValueError."""
|
||||
mock_pyroscope = _mock_pyroscope_module()
|
||||
with patch(
|
||||
"litellm.proxy.proxy_server.get_secret_bool",
|
||||
return_value=True,
|
||||
), patch.dict(
|
||||
sys.modules,
|
||||
{"pyroscope": mock_pyroscope},
|
||||
), patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"PYROSCOPE_APP_NAME": "myapp",
|
||||
"PYROSCOPE_SERVER_ADDRESS": "http://localhost:4040",
|
||||
"PYROSCOPE_SAMPLE_RATE": "not-a-number",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
with pytest.raises(ValueError, match="PYROSCOPE_SAMPLE_RATE"):
|
||||
ProxyStartupEvent._init_pyroscope()
|
||||
|
||||
|
||||
def test_init_pyroscope_accepts_integer_sample_rate():
|
||||
"""When enabled with valid config and integer sample rate, configures pyroscope."""
|
||||
mock_pyroscope = _mock_pyroscope_module()
|
||||
with patch(
|
||||
"litellm.proxy.proxy_server.get_secret_bool",
|
||||
return_value=True,
|
||||
), patch.dict(
|
||||
sys.modules,
|
||||
{"pyroscope": mock_pyroscope},
|
||||
), patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"PYROSCOPE_APP_NAME": "myapp",
|
||||
"PYROSCOPE_SERVER_ADDRESS": "http://localhost:4040",
|
||||
"PYROSCOPE_SAMPLE_RATE": "100",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
ProxyStartupEvent._init_pyroscope()
|
||||
mock_pyroscope.configure.assert_called_once()
|
||||
call_kw = mock_pyroscope.configure.call_args[1]
|
||||
assert call_kw["app_name"] == "myapp"
|
||||
assert call_kw["server_address"] == "http://localhost:4040"
|
||||
assert call_kw["sample_rate"] == 100
|
||||
|
||||
|
||||
def test_init_pyroscope_accepts_float_sample_rate_parsed_as_int():
|
||||
"""PYROSCOPE_SAMPLE_RATE can be a float string; it is parsed as integer."""
|
||||
mock_pyroscope = _mock_pyroscope_module()
|
||||
with patch(
|
||||
"litellm.proxy.proxy_server.get_secret_bool",
|
||||
return_value=True,
|
||||
), patch.dict(
|
||||
sys.modules,
|
||||
{"pyroscope": mock_pyroscope},
|
||||
), patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"PYROSCOPE_APP_NAME": "myapp",
|
||||
"PYROSCOPE_SERVER_ADDRESS": "http://localhost:4040",
|
||||
"PYROSCOPE_SAMPLE_RATE": "100.7",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
ProxyStartupEvent._init_pyroscope()
|
||||
call_kw = mock_pyroscope.configure.call_args[1]
|
||||
assert call_kw["sample_rate"] == 100
|
||||
Loading…
Add table
Reference in a new issue