diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 38ad9bdd0ee..cbd56d7c091 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -772,6 +772,9 @@ router_settings: | LITELLM_METER_NAME | Name for OTEL Meter | LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL | LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL +| LITELLM_ENABLE_PYROSCOPE | If true, enables Pyroscope CPU profiling. Profiles are sent to PYROSCOPE_SERVER_ADDRESS. Off by default. See [Pyroscope profiling](/proxy/pyroscope_profiling). +| PYROSCOPE_APP_NAME | Application name reported to Pyroscope. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. +| PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | LITELLM_MASTER_KEY | Master key for proxy authentication | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 579b5699101..e5c9c88d310 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -106,7 +106,8 @@ const sidebars = { items: [ "proxy/alerting", "proxy/pagerduty", - "proxy/prometheus" + "proxy/prometheus", + "proxy/pyroscope_profiling" ] }, { diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 03151ef96f3..0bb8eb985c2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -861,6 +861,9 @@ async def proxy_startup_event(app: FastAPI): # noqa: PLR0915 ## [Optional] Initialize dd tracer ProxyStartupEvent._init_dd_tracer() + ## [Optional] Initialize Pyroscope continuous profiling (env: LITELLM_ENABLE_PYROSCOPE=true) + ProxyStartupEvent._init_pyroscope() + ## Initialize shared aiohttp session for connection reuse shared_aiohttp_session = await _initialize_shared_aiohttp_session() @@ -5550,6 +5553,55 @@ class ProxyStartupEvent: prof.start() verbose_proxy_logger.debug("Datadog Profiler started......") + @classmethod + def _init_pyroscope(cls): + """ + Optional continuous profiling via Grafana Pyroscope. + + Off by default. Enable with LITELLM_ENABLE_PYROSCOPE=true. + Requires: pip install pyroscope-io + Sends profiles to PYROSCOPE_SERVER_ADDRESS (default http://localhost:4040). + """ + if not get_secret_bool("LITELLM_ENABLE_PYROSCOPE", False): + print( # noqa: T201 + "LiteLLM: Pyroscope profiling is disabled (set LITELLM_ENABLE_PYROSCOPE=true to enable)." + ) + return + try: + import pyroscope + + app_name = os.getenv("PYROSCOPE_APP_NAME") + if not app_name: + raise ValueError( + "LITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_APP_NAME is not set. " + "Set PYROSCOPE_APP_NAME when enabling Pyroscope." + ) + server_address = os.getenv("PYROSCOPE_SERVER_ADDRESS") + if not server_address: + raise ValueError( + "LITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_SERVER_ADDRESS is not set. " + "Set PYROSCOPE_SERVER_ADDRESS when enabling Pyroscope." + ) + tags = {} + env_name = os.getenv("OTEL_ENVIRONMENT_NAME") or os.getenv( + "LITELLM_DEPLOYMENT_ENVIRONMENT", + ) + if env_name: + tags["environment"] = env_name + pyroscope.configure( + app_name=app_name, + server_address=server_address, + tags=tags if tags else None, + ) + print( # noqa: T201 + f"LiteLLM: Pyroscope profiling started (app_name={app_name}, server_address={server_address}). " + f"View CPU profiles at the Pyroscope UI and select application '{app_name}'." + ) + except ImportError: + print( # noqa: T201 + "LiteLLM: LITELLM_ENABLE_PYROSCOPE is set but the 'pyroscope-io' package is not installed. " + "Pyroscope profiling will not run. Install with: pip install pyroscope-io" + ) #### API ENDPOINTS #### @router.get( diff --git a/poetry.lock b/poetry.lock index 533b48b793c..2a4301d3c90 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "a2a-sdk" @@ -3459,15 +3459,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.33" +version = "0.4.34" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.33-py3-none-any.whl", hash = "sha256:bebea1b091490df19cfa773bd311f08254dee5bb53f92d282b7a5bdfba936334"}, - {file = "litellm_proxy_extras-0.4.33.tar.gz", hash = "sha256:133dc5476b540d99e75d4baef622267e7344ced97737c174679baff429e7f212"}, + {file = "litellm_proxy_extras-0.4.34-py3-none-any.whl", hash = "sha256:d455eb54f82e7c92f4f68a921240822df23158aad05fcdda7245887db7c30b90"}, + {file = "litellm_proxy_extras-0.4.34.tar.gz", hash = "sha256:39fa6c2295acc449320b5a710d150295fd0bf5f8c0d1742b5e9ae361d7bd3ed2"}, ] [[package]] @@ -5659,6 +5659,24 @@ files = [ [package.extras] dev = ["build", "flake8", "mypy", "pytest", "twine"] +[[package]] +name = "pyroscope-io" +version = "0.8.16" +description = "Pyroscope Python integration" +optional = false +python-versions = "*" +groups = ["main"] +markers = "extra == \"proxy\" and sys_platform != \"win32\"" +files = [ + {file = "pyroscope_io-0.8.16-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:e07edcfd59f5bdce42948b92c9b118c824edbd551730305f095a6b9af401a9e8"}, + {file = "pyroscope_io-0.8.16-py2.py3-none-macosx_11_0_x86_64.whl", hash = "sha256:dc98355e27c0b7b61f27066500fe1045b70e9459bb8b9a3082bc4755cb6392b6"}, + {file = "pyroscope_io-0.8.16-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:86f0f047554ff62bd92c3e5a26bc2809ccd467d11fbacb9fef898ba299dbda59"}, + {file = "pyroscope_io-0.8.16-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6b91ce5b240f8de756c16a17022ca8e25ef8a4eed461c7d074b8a0841cf7b445"}, +] + +[package.dependencies] +cffi = ">=1.6.0" + [[package]] name = "pytest" version = "7.4.4" @@ -8516,11 +8534,11 @@ extra-proxy = ["a2a-sdk", "azure-identity", "azure-keyvault-secrets", "google-cl google = ["google-cloud-aiplatform"] grpc = ["grpcio", "grpcio"] mlflow = ["mlflow"] -proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "soundfile", "uvicorn", "uvloop", "websockets"] +proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "pyroscope-io", "python-multipart", "pyyaml", "rich", "rq", "soundfile", "uvicorn", "uvloop", "websockets"] semantic-router = ["semantic-router"] utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "4d88cb21bfd0bcc87b8894e2915d297c2d0b15f164a468e712872b1f1b87db43" +content-hash = "75f2d51e0e399eaa79f0211bee13f4c887c0e5a76994b21f5ad7237c273a72a0" diff --git a/pyproject.toml b/pyproject.toml index acb8bc2ada3..0d3f4e5af2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ polars = {version = "^1.31.0", optional = true, python = ">=3.10"} semantic-router = {version = ">=0.1.12", optional = true, python = ">=3.9,<3.14"} mlflow = {version = ">3.1.4", optional = true, python = ">=3.10"} soundfile = {version = "^0.12.1", optional = true} +pyroscope-io = {version = "^0.8", markers = "sys_platform != 'win32'"} # grpcio constraints: # - 1.62.3+ required by grpcio-status # - 1.68.0-1.68.1 has reconnect bug (https://github.com/grpc/grpc/issues/38290) @@ -104,6 +105,7 @@ proxy = [ "rich", "polars", "soundfile", + "pyroscope-io", ] extra_proxy = [ @@ -121,6 +123,8 @@ utils = [ "numpydoc", ] + + caching = ["diskcache"] semantic-router = ["semantic-router"]