From a2bef1a1370983e5594990fab00f3a5ffa7a8302 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Thu, 12 Feb 2026 12:25:44 -0800 Subject: [PATCH] Pyroscope: require PYROSCOPE_APP_NAME and PYROSCOPE_SERVER_ADDRESS, add UTF-8 locale hint - No defaults for PYROSCOPE_APP_NAME or PYROSCOPE_SERVER_ADDRESS; fail at startup if unset when Pyroscope is enabled - Set LANG/LC_ALL to C.UTF-8 when unset to reduce malformed_profile (invalid UTF-8) rejections - Startup message suggests PYTHONUTF8=1 if server rejects profiles - Simplify LITELLM_ENABLE_PYROSCOPE in config_settings; document Pyroscope env vars as required with no default - Add pyroscope_profiling to sidebar (Alerting & Monitoring) - pyproject.toml: pyroscope-io as required dep on non-Windows (marker), in proxy extra --- docs/my-website/docs/proxy/config_settings.md | 3 + docs/my-website/sidebars.js | 3 +- litellm/proxy/proxy_server.py | 52 ++++++++++++++++ poetry.lock | 62 +++++++++++++++++-- pyproject.toml | 4 ++ 5 files changed, 117 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 53d9c775972..c2aed65c84f 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -731,6 +731,9 @@ router_settings: | LITELLM_METER_NAME | Name for OTEL Meter | LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL | LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL +| LITELLM_ENABLE_PYROSCOPE | If true, enables Pyroscope CPU profiling. Profiles are sent to PYROSCOPE_SERVER_ADDRESS. Off by default. See [Pyroscope profiling](/proxy/pyroscope_profiling). +| PYROSCOPE_APP_NAME | Application name reported to Pyroscope. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. +| PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | LITELLM_MASTER_KEY | Master key for proxy authentication | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 102e3dfe1c5..204e8049047 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -91,7 +91,8 @@ const sidebars = { items: [ "proxy/alerting", "proxy/pagerduty", - "proxy/prometheus" + "proxy/prometheus", + "proxy/pyroscope_profiling" ] }, { diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f4e68c481a0..30d3d078c23 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -786,6 +786,9 @@ async def proxy_startup_event(app: FastAPI): # noqa: PLR0915 ## [Optional] Initialize dd tracer ProxyStartupEvent._init_dd_tracer() + ## [Optional] Initialize Pyroscope continuous profiling (env: LITELLM_ENABLE_PYROSCOPE=true) + ProxyStartupEvent._init_pyroscope() + ## Initialize shared aiohttp session for connection reuse shared_aiohttp_session = await _initialize_shared_aiohttp_session() @@ -5049,6 +5052,55 @@ class ProxyStartupEvent: prof.start() verbose_proxy_logger.debug("Datadog Profiler started......") + @classmethod + def _init_pyroscope(cls): + """ + Optional continuous profiling via Grafana Pyroscope. + + Off by default. Enable with LITELLM_ENABLE_PYROSCOPE=true. + Requires: pip install pyroscope-io + Sends profiles to PYROSCOPE_SERVER_ADDRESS (default http://localhost:4040). + """ + if not get_secret_bool("LITELLM_ENABLE_PYROSCOPE", False): + print( # noqa: T201 + "LiteLLM: Pyroscope profiling is disabled (set LITELLM_ENABLE_PYROSCOPE=true to enable)." + ) + return + try: + import pyroscope + + app_name = os.getenv("PYROSCOPE_APP_NAME") + if not app_name: + raise ValueError( + "LITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_APP_NAME is not set. " + "Set PYROSCOPE_APP_NAME when enabling Pyroscope." + ) + server_address = os.getenv("PYROSCOPE_SERVER_ADDRESS") + if not server_address: + raise ValueError( + "LITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_SERVER_ADDRESS is not set. " + "Set PYROSCOPE_SERVER_ADDRESS when enabling Pyroscope." + ) + tags = {} + env_name = os.getenv("OTEL_ENVIRONMENT_NAME") or os.getenv( + "LITELLM_DEPLOYMENT_ENVIRONMENT", + ) + if env_name: + tags["environment"] = env_name + pyroscope.configure( + app_name=app_name, + server_address=server_address, + tags=tags if tags else None, + ) + print( # noqa: T201 + f"LiteLLM: Pyroscope profiling started (app_name={app_name}, server_address={server_address}). " + f"View CPU profiles at the Pyroscope UI and select application '{app_name}'." + ) + except ImportError: + print( # noqa: T201 + "LiteLLM: LITELLM_ENABLE_PYROSCOPE is set but the 'pyroscope-io' package is not installed. " + "Pyroscope profiling will not run. Install with: pip install pyroscope-io" + ) #### API ENDPOINTS #### @router.get( diff --git a/poetry.lock b/poetry.lock index b76f9c30f00..6d98b45978d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,36 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. + +[[package]] +name = "a2a-sdk" +version = "0.3.22" +description = "A2A Python SDK" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"extra-proxy\"" +files = [ + {file = "a2a_sdk-0.3.22-py3-none-any.whl", hash = "sha256:b98701135bb90b0ff85d35f31533b6b7a299bf810658c1c65f3814a6c15ea385"}, + {file = "a2a_sdk-0.3.22.tar.gz", hash = "sha256:77a5694bfc4f26679c11b70c7f1062522206d430b34bc1215cfbb1eba67b7e7d"}, +] + +[package.dependencies] +google-api-core = ">=1.26.0" +httpx = ">=0.28.1" +httpx-sse = ">=0.4.0" +protobuf = ">=5.29.5" +pydantic = ">=2.11.3" + +[package.extras] +all = ["cryptography (>=43.0.0)", "fastapi (>=0.115.2)", "grpcio (>=1.60)", "grpcio-reflection (>=1.7.0)", "grpcio-tools (>=1.60)", "opentelemetry-api (>=1.33.0)", "opentelemetry-sdk (>=1.33.0)", "pyjwt (>=2.0.0)", "sqlalchemy[aiomysql,asyncio] (>=2.0.0)", "sqlalchemy[aiosqlite,asyncio] (>=2.0.0)", "sqlalchemy[asyncio,postgresql-asyncpg] (>=2.0.0)", "sse-starlette", "starlette"] +encryption = ["cryptography (>=43.0.0)"] +grpc = ["grpcio (>=1.60)", "grpcio-reflection (>=1.7.0)", "grpcio-tools (>=1.60)"] +http-server = ["fastapi (>=0.115.2)", "sse-starlette", "starlette"] +mysql = ["sqlalchemy[aiomysql,asyncio] (>=2.0.0)"] +postgresql = ["sqlalchemy[asyncio,postgresql-asyncpg] (>=2.0.0)"] +signing = ["pyjwt (>=2.0.0)"] +sql = ["sqlalchemy[aiomysql,asyncio] (>=2.0.0)", "sqlalchemy[aiosqlite,asyncio] (>=2.0.0)", "sqlalchemy[asyncio,postgresql-asyncpg] (>=2.0.0)"] +sqlite = ["sqlalchemy[aiosqlite,asyncio] (>=2.0.0)"] +telemetry = ["opentelemetry-api (>=1.33.0)", "opentelemetry-sdk (>=1.33.0)"] [[package]] name = "aiofiles" @@ -3081,15 +3113,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.23" +version = "0.4.34" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.23-py3-none-any.whl", hash = "sha256:dfda21203dde9fd97cf364396a9b5be0cfdf00fa9846439ee33ce11b7a52f9ce"}, - {file = "litellm_proxy_extras-0.4.23.tar.gz", hash = "sha256:8e3f95576dc2a296e7f73d8c87e73628bd899b4644c45863960fe3c3762d8f64"}, + {file = "litellm_proxy_extras-0.4.34-py3-none-any.whl", hash = "sha256:d455eb54f82e7c92f4f68a921240822df23158aad05fcdda7245887db7c30b90"}, + {file = "litellm_proxy_extras-0.4.34.tar.gz", hash = "sha256:39fa6c2295acc449320b5a710d150295fd0bf5f8c0d1742b5e9ae361d7bd3ed2"}, ] [[package]] @@ -5271,6 +5303,24 @@ files = [ [package.extras] dev = ["build", "flake8", "mypy", "pytest", "twine"] +[[package]] +name = "pyroscope-io" +version = "0.8.16" +description = "Pyroscope Python integration" +optional = false +python-versions = "*" +groups = ["main"] +markers = "extra == \"proxy\" and sys_platform != \"win32\"" +files = [ + {file = "pyroscope_io-0.8.16-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:e07edcfd59f5bdce42948b92c9b118c824edbd551730305f095a6b9af401a9e8"}, + {file = "pyroscope_io-0.8.16-py2.py3-none-macosx_11_0_x86_64.whl", hash = "sha256:dc98355e27c0b7b61f27066500fe1045b70e9459bb8b9a3082bc4755cb6392b6"}, + {file = "pyroscope_io-0.8.16-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:86f0f047554ff62bd92c3e5a26bc2809ccd467d11fbacb9fef898ba299dbda59"}, + {file = "pyroscope_io-0.8.16-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6b91ce5b240f8de756c16a17022ca8e25ef8a4eed461c7d074b8a0841cf7b445"}, +] + +[package.dependencies] +cffi = ">=1.6.0" + [[package]] name = "pytest" version = "7.4.4" @@ -7974,11 +8024,11 @@ type = ["pytest-mypy"] caching = ["diskcache"] extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-iam", "google-cloud-kms", "prisma", "redisvl", "resend"] mlflow = ["mlflow"] -proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "soundfile", "uvicorn", "uvloop", "websockets"] +proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "pyroscope-io", "python-multipart", "pyyaml", "rich", "rq", "soundfile", "uvicorn", "uvloop", "websockets"] semantic-router = ["semantic-router"] utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "2d6b3d8d44919c29315b5e645befbf745a276714a2454c563d460a6a001b90af" +content-hash = "75f2d51e0e399eaa79f0211bee13f4c887c0e5a76994b21f5ad7237c273a72a0" diff --git a/pyproject.toml b/pyproject.toml index d6242b27786..a7982142dfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ polars = {version = "^1.31.0", optional = true, python = ">=3.10"} semantic-router = {version = ">=0.1.12", optional = true, python = ">=3.9,<3.14"} mlflow = {version = ">3.1.4", optional = true, python = ">=3.10"} soundfile = {version = "^0.12.1", optional = true} +pyroscope-io = {version = "^0.8", markers = "sys_platform != 'win32'"} # grpcio constraints: # - 1.62.3+ required by grpcio-status # - 1.68.0-1.68.1 has reconnect bug (https://github.com/grpc/grpc/issues/38290) @@ -102,6 +103,7 @@ proxy = [ "rich", "polars", "soundfile", + "pyroscope-io", ] extra_proxy = [ @@ -118,6 +120,8 @@ utils = [ "numpydoc", ] + + caching = ["diskcache"] semantic-router = ["semantic-router"]