mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
Pyroscope: require PYROSCOPE_APP_NAME and PYROSCOPE_SERVER_ADDRESS, add UTF-8 locale hint
- No defaults for PYROSCOPE_APP_NAME or PYROSCOPE_SERVER_ADDRESS; fail at startup if unset when Pyroscope is enabled - Set LANG/LC_ALL to C.UTF-8 when unset to reduce malformed_profile (invalid UTF-8) rejections - Startup message suggests PYTHONUTF8=1 if server rejects profiles - Simplify LITELLM_ENABLE_PYROSCOPE in config_settings; document Pyroscope env vars as required with no default - Add pyroscope_profiling to sidebar (Alerting & Monitoring) - pyproject.toml: pyroscope-io as required dep on non-Windows (marker), in proxy extra
This commit is contained in:
Alexsander Hamir
parent
ed32591a05
commit
a2bef1a137
5 changed files with 117 additions and 7 deletions
|
|
@ -731,6 +731,9 @@ router_settings:
|
|||
| LITELLM_METER_NAME | Name for OTEL Meter
|
||||
| LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL
|
||||
| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL
|
||||
| LITELLM_ENABLE_PYROSCOPE | If true, enables Pyroscope CPU profiling. Profiles are sent to PYROSCOPE_SERVER_ADDRESS. Off by default. See [Pyroscope profiling](/proxy/pyroscope_profiling).
|
||||
| PYROSCOPE_APP_NAME | Application name reported to Pyroscope. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
|
||||
| PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
|
||||
| LITELLM_MASTER_KEY | Master key for proxy authentication
|
||||
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
|
||||
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
|
||||
|
|
|
|||
|
|
@ -91,7 +91,8 @@ const sidebars = {
|
|||
items: [
|
||||
"proxy/alerting",
|
||||
"proxy/pagerduty",
|
||||
"proxy/prometheus"
|
||||
"proxy/prometheus",
|
||||
"proxy/pyroscope_profiling"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -786,6 +786,9 @@ async def proxy_startup_event(app: FastAPI): # noqa: PLR0915
|
|||
## [Optional] Initialize dd tracer
|
||||
ProxyStartupEvent._init_dd_tracer()
|
||||
|
||||
## [Optional] Initialize Pyroscope continuous profiling (env: LITELLM_ENABLE_PYROSCOPE=true)
|
||||
ProxyStartupEvent._init_pyroscope()
|
||||
|
||||
## Initialize shared aiohttp session for connection reuse
|
||||
shared_aiohttp_session = await _initialize_shared_aiohttp_session()
|
||||
|
||||
|
|
@ -5049,6 +5052,55 @@ class ProxyStartupEvent:
|
|||
prof.start()
|
||||
verbose_proxy_logger.debug("Datadog Profiler started......")
|
||||
|
||||
@classmethod
|
||||
def _init_pyroscope(cls):
|
||||
"""
|
||||
Optional continuous profiling via Grafana Pyroscope.
|
||||
|
||||
Off by default. Enable with LITELLM_ENABLE_PYROSCOPE=true.
|
||||
Requires: pip install pyroscope-io
|
||||
Sends profiles to PYROSCOPE_SERVER_ADDRESS (default http://localhost:4040).
|
||||
"""
|
||||
if not get_secret_bool("LITELLM_ENABLE_PYROSCOPE", False):
|
||||
print( # noqa: T201
|
||||
"LiteLLM: Pyroscope profiling is disabled (set LITELLM_ENABLE_PYROSCOPE=true to enable)."
|
||||
)
|
||||
return
|
||||
try:
|
||||
import pyroscope
|
||||
|
||||
app_name = os.getenv("PYROSCOPE_APP_NAME")
|
||||
if not app_name:
|
||||
raise ValueError(
|
||||
"LITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_APP_NAME is not set. "
|
||||
"Set PYROSCOPE_APP_NAME when enabling Pyroscope."
|
||||
)
|
||||
server_address = os.getenv("PYROSCOPE_SERVER_ADDRESS")
|
||||
if not server_address:
|
||||
raise ValueError(
|
||||
"LITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_SERVER_ADDRESS is not set. "
|
||||
"Set PYROSCOPE_SERVER_ADDRESS when enabling Pyroscope."
|
||||
)
|
||||
tags = {}
|
||||
env_name = os.getenv("OTEL_ENVIRONMENT_NAME") or os.getenv(
|
||||
"LITELLM_DEPLOYMENT_ENVIRONMENT",
|
||||
)
|
||||
if env_name:
|
||||
tags["environment"] = env_name
|
||||
pyroscope.configure(
|
||||
app_name=app_name,
|
||||
server_address=server_address,
|
||||
tags=tags if tags else None,
|
||||
)
|
||||
print( # noqa: T201
|
||||
f"LiteLLM: Pyroscope profiling started (app_name={app_name}, server_address={server_address}). "
|
||||
f"View CPU profiles at the Pyroscope UI and select application '{app_name}'."
|
||||
)
|
||||
except ImportError:
|
||||
print( # noqa: T201
|
||||
"LiteLLM: LITELLM_ENABLE_PYROSCOPE is set but the 'pyroscope-io' package is not installed. "
|
||||
"Pyroscope profiling will not run. Install with: pip install pyroscope-io"
|
||||
)
|
||||
|
||||
#### API ENDPOINTS ####
|
||||
@router.get(
|
||||
|
|
|
|||
62
poetry.lock
generated
62
poetry.lock
generated
|
|
@ -1,4 +1,36 @@
|
|||
# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand.
|
||||
# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand.
|
||||
|
||||
[[package]]
|
||||
name = "a2a-sdk"
|
||||
version = "0.3.22"
|
||||
description = "A2A Python SDK"
|
||||
optional = true
|
||||
python-versions = ">=3.10"
|
||||
groups = ["main"]
|
||||
markers = "python_version >= \"3.10\" and extra == \"extra-proxy\""
|
||||
files = [
|
||||
{file = "a2a_sdk-0.3.22-py3-none-any.whl", hash = "sha256:b98701135bb90b0ff85d35f31533b6b7a299bf810658c1c65f3814a6c15ea385"},
|
||||
{file = "a2a_sdk-0.3.22.tar.gz", hash = "sha256:77a5694bfc4f26679c11b70c7f1062522206d430b34bc1215cfbb1eba67b7e7d"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
google-api-core = ">=1.26.0"
|
||||
httpx = ">=0.28.1"
|
||||
httpx-sse = ">=0.4.0"
|
||||
protobuf = ">=5.29.5"
|
||||
pydantic = ">=2.11.3"
|
||||
|
||||
[package.extras]
|
||||
all = ["cryptography (>=43.0.0)", "fastapi (>=0.115.2)", "grpcio (>=1.60)", "grpcio-reflection (>=1.7.0)", "grpcio-tools (>=1.60)", "opentelemetry-api (>=1.33.0)", "opentelemetry-sdk (>=1.33.0)", "pyjwt (>=2.0.0)", "sqlalchemy[aiomysql,asyncio] (>=2.0.0)", "sqlalchemy[aiosqlite,asyncio] (>=2.0.0)", "sqlalchemy[asyncio,postgresql-asyncpg] (>=2.0.0)", "sse-starlette", "starlette"]
|
||||
encryption = ["cryptography (>=43.0.0)"]
|
||||
grpc = ["grpcio (>=1.60)", "grpcio-reflection (>=1.7.0)", "grpcio-tools (>=1.60)"]
|
||||
http-server = ["fastapi (>=0.115.2)", "sse-starlette", "starlette"]
|
||||
mysql = ["sqlalchemy[aiomysql,asyncio] (>=2.0.0)"]
|
||||
postgresql = ["sqlalchemy[asyncio,postgresql-asyncpg] (>=2.0.0)"]
|
||||
signing = ["pyjwt (>=2.0.0)"]
|
||||
sql = ["sqlalchemy[aiomysql,asyncio] (>=2.0.0)", "sqlalchemy[aiosqlite,asyncio] (>=2.0.0)", "sqlalchemy[asyncio,postgresql-asyncpg] (>=2.0.0)"]
|
||||
sqlite = ["sqlalchemy[aiosqlite,asyncio] (>=2.0.0)"]
|
||||
telemetry = ["opentelemetry-api (>=1.33.0)", "opentelemetry-sdk (>=1.33.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "aiofiles"
|
||||
|
|
@ -3081,15 +3113,15 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.23"
|
||||
version = "0.4.34"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
optional = true
|
||||
python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"proxy\""
|
||||
files = [
|
||||
{file = "litellm_proxy_extras-0.4.23-py3-none-any.whl", hash = "sha256:dfda21203dde9fd97cf364396a9b5be0cfdf00fa9846439ee33ce11b7a52f9ce"},
|
||||
{file = "litellm_proxy_extras-0.4.23.tar.gz", hash = "sha256:8e3f95576dc2a296e7f73d8c87e73628bd899b4644c45863960fe3c3762d8f64"},
|
||||
{file = "litellm_proxy_extras-0.4.34-py3-none-any.whl", hash = "sha256:d455eb54f82e7c92f4f68a921240822df23158aad05fcdda7245887db7c30b90"},
|
||||
{file = "litellm_proxy_extras-0.4.34.tar.gz", hash = "sha256:39fa6c2295acc449320b5a710d150295fd0bf5f8c0d1742b5e9ae361d7bd3ed2"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -5271,6 +5303,24 @@ files = [
|
|||
[package.extras]
|
||||
dev = ["build", "flake8", "mypy", "pytest", "twine"]
|
||||
|
||||
[[package]]
|
||||
name = "pyroscope-io"
|
||||
version = "0.8.16"
|
||||
description = "Pyroscope Python integration"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"proxy\" and sys_platform != \"win32\""
|
||||
files = [
|
||||
{file = "pyroscope_io-0.8.16-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:e07edcfd59f5bdce42948b92c9b118c824edbd551730305f095a6b9af401a9e8"},
|
||||
{file = "pyroscope_io-0.8.16-py2.py3-none-macosx_11_0_x86_64.whl", hash = "sha256:dc98355e27c0b7b61f27066500fe1045b70e9459bb8b9a3082bc4755cb6392b6"},
|
||||
{file = "pyroscope_io-0.8.16-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:86f0f047554ff62bd92c3e5a26bc2809ccd467d11fbacb9fef898ba299dbda59"},
|
||||
{file = "pyroscope_io-0.8.16-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6b91ce5b240f8de756c16a17022ca8e25ef8a4eed461c7d074b8a0841cf7b445"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
cffi = ">=1.6.0"
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "7.4.4"
|
||||
|
|
@ -7974,11 +8024,11 @@ type = ["pytest-mypy"]
|
|||
caching = ["diskcache"]
|
||||
extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-iam", "google-cloud-kms", "prisma", "redisvl", "resend"]
|
||||
mlflow = ["mlflow"]
|
||||
proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "soundfile", "uvicorn", "uvloop", "websockets"]
|
||||
proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "pyroscope-io", "python-multipart", "pyyaml", "rich", "rq", "soundfile", "uvicorn", "uvloop", "websockets"]
|
||||
semantic-router = ["semantic-router"]
|
||||
utils = ["numpydoc"]
|
||||
|
||||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.9,<4.0"
|
||||
content-hash = "2d6b3d8d44919c29315b5e645befbf745a276714a2454c563d460a6a001b90af"
|
||||
content-hash = "75f2d51e0e399eaa79f0211bee13f4c887c0e5a76994b21f5ad7237c273a72a0"
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ polars = {version = "^1.31.0", optional = true, python = ">=3.10"}
|
|||
semantic-router = {version = ">=0.1.12", optional = true, python = ">=3.9,<3.14"}
|
||||
mlflow = {version = ">3.1.4", optional = true, python = ">=3.10"}
|
||||
soundfile = {version = "^0.12.1", optional = true}
|
||||
pyroscope-io = {version = "^0.8", markers = "sys_platform != 'win32'"}
|
||||
# grpcio constraints:
|
||||
# - 1.62.3+ required by grpcio-status
|
||||
# - 1.68.0-1.68.1 has reconnect bug (https://github.com/grpc/grpc/issues/38290)
|
||||
|
|
@ -102,6 +103,7 @@ proxy = [
|
|||
"rich",
|
||||
"polars",
|
||||
"soundfile",
|
||||
"pyroscope-io",
|
||||
]
|
||||
|
||||
extra_proxy = [
|
||||
|
|
@ -118,6 +120,8 @@ utils = [
|
|||
"numpydoc",
|
||||
]
|
||||
|
||||
|
||||
|
||||
caching = ["diskcache"]
|
||||
|
||||
semantic-router = ["semantic-router"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue