diff --git a/docker/component_entrypoint.sh b/docker/component_entrypoint.sh index 413957b9929..173afafe1ad 100755 --- a/docker/component_entrypoint.sh +++ b/docker/component_entrypoint.sh @@ -1,5 +1,11 @@ #!/bin/sh +# stale samples from a previous container incarnation would be summed into the aggregate +if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then + mkdir -p "$PROMETHEUS_MULTIPROC_DIR" + rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db +fi + case "$USE_DDTRACE" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED="False" diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml index 52ffd117535..f7c918a6827 100644 --- a/helm/litellm-helm/templates/deployment.yaml +++ b/helm/litellm-helm/templates/deployment.yaml @@ -152,6 +152,13 @@ spec: {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsEnv" . | nindent 12 }} {{- end }} + {{- if .Values.metricsServer.enabled }} + {{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }} + {{- fail "metricsServer.port must differ from service.port" }} + {{- end }} + - name: PROMETHEUS_METRICS_PORT + value: {{ .Values.metricsServer.port | quote }} + {{- end }} {{- if .Values.migrationJob.enabled }} # Schema updates are owned by the dedicated migrations Job; skip # the proxy's startup `prisma db push` so N replicas don't race @@ -189,6 +196,11 @@ spec: - name: http containerPort: {{ .Values.service.port }} protocol: TCP + {{- if .Values.metricsServer.enabled }} + - name: metrics + containerPort: {{ .Values.metricsServer.port }} + protocol: TCP + {{- end }} livenessProbe: httpGet: path: {{ .Values.livenessProbe.path | quote }} diff --git a/helm/litellm-helm/templates/service-metrics.yaml b/helm/litellm-helm/templates/service-metrics.yaml new file mode 100644 index 00000000000..1d23fe39606 --- /dev/null +++ b/helm/litellm-helm/templates/service-metrics.yaml @@ -0,0 +1,17 @@ +{{- if .Values.metricsServer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "litellm.fullname" . }}-metrics + labels: + {{- include "litellm.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.metricsServer.port }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "litellm.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/litellm-helm/templates/servicemonitor.yaml b/helm/litellm-helm/templates/servicemonitor.yaml index 743098deb3f..68083d0da61 100644 --- a/helm/litellm-helm/templates/servicemonitor.yaml +++ b/helm/litellm-helm/templates/servicemonitor.yaml @@ -26,7 +26,7 @@ spec: {{- toYaml .namespaceSelector.matchNames | nindent 4 }} {{- end }} endpoints: - - port: http + - port: {{ ternary "metrics" "http" $.Values.metricsServer.enabled }} path: /metrics/ interval: {{ .interval }} scrapeTimeout: {{ .scrapeTimeout }} diff --git a/helm/litellm-helm/tests/metrics_server_tests.yaml b/helm/litellm-helm/tests/metrics_server_tests.yaml new file mode 100644 index 00000000000..085d69ac640 --- /dev/null +++ b/helm/litellm-helm/tests/metrics_server_tests.yaml @@ -0,0 +1,106 @@ +suite: separate metrics server +templates: + - configmap-litellm.yaml + - deployment.yaml + - service.yaml + - service-metrics.yaml + - servicemonitor.yaml +tests: + - it: should not expose a metrics port or PROMETHEUS_METRICS_PORT by default + asserts: + - notContains: + path: spec.template.spec.containers[0].ports + content: + name: metrics + any: true + template: deployment.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: + name: PROMETHEUS_METRICS_PORT + any: true + template: deployment.yaml + - lengthEqual: + path: spec.ports + count: 1 + template: service.yaml + - hasDocuments: + count: 0 + template: service-metrics.yaml + + - it: should scrape the proxy port when the metrics server is disabled + template: servicemonitor.yaml + set: + serviceMonitor.enabled: true + asserts: + - equal: + path: spec.endpoints[0].port + value: http + + - it: should wire the separate metrics server through container, a ClusterIP metrics service and servicemonitor + set: + metricsServer.enabled: true + metricsServer.port: 4101 + serviceMonitor.enabled: true + service.type: LoadBalancer + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROMETHEUS_METRICS_PORT + value: "4101" + template: deployment.yaml + - contains: + path: spec.template.spec.containers[0].ports + content: + name: metrics + containerPort: 4101 + protocol: TCP + template: deployment.yaml + - lengthEqual: + path: spec.ports + count: 1 + template: service.yaml + - equal: + path: spec.type + value: LoadBalancer + template: service.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-metrics + template: service-metrics.yaml + - equal: + path: spec.type + value: ClusterIP + template: service-metrics.yaml + - equal: + path: spec.ports + value: + - port: 4101 + targetPort: metrics + protocol: TCP + name: metrics + template: service-metrics.yaml + - equal: + path: spec.selector + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME + template: service-metrics.yaml + - equal: + path: spec.endpoints[0].port + value: metrics + template: servicemonitor.yaml + - equal: + path: spec.endpoints[0].path + value: /metrics/ + template: servicemonitor.yaml + + - it: should reject a metrics port equal to the proxy port + template: deployment.yaml + set: + metricsServer.enabled: true + metricsServer.port: 4000 + asserts: + - failedTemplate: + errorMessage: metricsServer.port must differ from service.port diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 637be2322e3..8dc7b967e11 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -180,6 +180,16 @@ proxy_config: general_settings: master_key: os.environ/PROXY_MASTER_KEY +# Serve Prometheus /metrics from a separate process (PROMETHEUS_METRICS_PORT) +# so a scrape never runs on an inference worker. Adds a `metrics` port to the +# container and a dedicated ClusterIP `-metrics` Service, and the +# ServiceMonitor scrapes it instead of the proxy port. The separate port has +# no virtual-key auth: keep it off public ingress. Needs the proxy image +# v1.101.0 or newer. +metricsServer: + enabled: false + port: 4001 + resources: {} # Unset by default so the chart installs on small clusters such as Minikube, and so an diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index be6b9093f53..c459512c7b9 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -441,3 +441,5 @@ ImplementationSpecific {{- .pathType -}} {{- end -}} {{- end -}} + +{{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 5030ba2c9dc..9cb6b07e77b 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -64,14 +64,25 @@ spec: {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsEnv" . | nindent 12 }} {{- end }} + {{- if .Values.gateway.metricsServer.enabled }} + {{- if eq (int .Values.gateway.metricsServer.port) 4000 }} + {{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }} + {{- end }} + - name: PROMETHEUS_MULTIPROC_DIR + value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + {{- end }} {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }} volumeMounts: {{- if .Values.gateway.config.create }} - name: gateway-config mountPath: /app/config/config.yaml subPath: config.yaml {{- end }} + {{- if .Values.gateway.metricsServer.enabled }} + - name: prometheus-multiproc + mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + {{- end }} {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} {{- end }} @@ -97,16 +108,54 @@ spec: {{- end }} resources: {{- toYaml .Values.gateway.resources | nindent 12 }} + {{- if .Values.gateway.metricsServer.enabled }} + - name: metrics + image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + command: + - python + - -m + - litellm.proxy.prometheus_metrics_server + - --port + - {{ .Values.gateway.metricsServer.port | quote }} + env: + - name: PROMETHEUS_MULTIPROC_DIR + value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + ports: + - name: metrics + containerPort: {{ .Values.gateway.metricsServer.port }} + protocol: TCP + volumeMounts: + - name: prometheus-multiproc + mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + readinessProbe: + tcpSocket: { port: metrics } + periodSeconds: 10 + livenessProbe: + tcpSocket: { port: metrics } + periodSeconds: 15 + failureThreshold: 6 + resources: + {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }} + {{- end }} {{- with .Values.gateway.extraContainers }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled }} + {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }} volumes: {{- if .Values.gateway.config.create }} - name: gateway-config configMap: name: {{ include "litellm.gateway.fullname" . }}-config {{- end }} + {{- if .Values.gateway.metricsServer.enabled }} + - name: prometheus-multiproc + emptyDir: {} + {{- end }} {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} {{- end }} diff --git a/helm/litellm/templates/gateway/service-metrics.yaml b/helm/litellm/templates/gateway/service-metrics.yaml new file mode 100644 index 00000000000..ad9bc05a9fd --- /dev/null +++ b/helm/litellm/templates/gateway/service-metrics.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.gateway.enabled .Values.gateway.metricsServer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "litellm.gateway.fullname" . }}-metrics + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: gateway +spec: + type: ClusterIP + ports: + - port: {{ .Values.gateway.metricsServer.port }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "litellm.gateway.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/litellm/tests/metrics_server_tests.yaml b/helm/litellm/tests/metrics_server_tests.yaml new file mode 100644 index 00000000000..0e7d9d9e9ee --- /dev/null +++ b/helm/litellm/tests/metrics_server_tests.yaml @@ -0,0 +1,148 @@ +suite: test gateway metrics sidecar +templates: + - gateway/configmap.yaml + - gateway/deployment.yaml + - gateway/service.yaml + - gateway/service-metrics.yaml +values: + - ./values/required.yaml +tests: + - it: adds no sidecar, volume, env or service port when the metrics server is off + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: + name: PROMETHEUS_MULTIPROC_DIR + any: true + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.volumes + content: + name: prometheus-multiproc + any: true + template: gateway/deployment.yaml + - lengthEqual: + path: spec.ports + count: 1 + template: gateway/service.yaml + - hasDocuments: + count: 0 + template: gateway/service-metrics.yaml + + - it: runs the metrics server as a sidecar over a shared multiproc dir and exposes it on a ClusterIP metrics service + set: + gateway.metricsServer.enabled: true + gateway.metricsServer.port: 4101 + gateway.service.type: LoadBalancer + gateway.image.tag: v1.101.0 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROMETHEUS_MULTIPROC_DIR + value: /tmp/litellm_prometheus_multiproc + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: prometheus-multiproc + mountPath: /tmp/litellm_prometheus_multiproc + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].name + value: metrics + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].image + value: ghcr.io/berriai/litellm-gateway:v1.101.0 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].command + value: + - python + - -m + - litellm.proxy.prometheus_metrics_server + - --port + - "4101" + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].env + value: + - name: PROMETHEUS_MULTIPROC_DIR + value: /tmp/litellm_prometheus_multiproc + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].ports + value: + - name: metrics + containerPort: 4101 + protocol: TCP + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].volumeMounts + value: + - name: prometheus-multiproc + mountPath: /tmp/litellm_prometheus_multiproc + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].readinessProbe.tcpSocket.port + value: metrics + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].livenessProbe.tcpSocket.port + value: metrics + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].resources.requests.cpu + value: 50m + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.volumes + content: + name: prometheus-multiproc + emptyDir: {} + template: gateway/deployment.yaml + - lengthEqual: + path: spec.ports + count: 1 + template: gateway/service.yaml + - equal: + path: spec.type + value: LoadBalancer + template: gateway/service.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-gateway-metrics + template: gateway/service-metrics.yaml + - equal: + path: spec.type + value: ClusterIP + template: gateway/service-metrics.yaml + - equal: + path: spec.ports + value: + - port: 4101 + targetPort: metrics + protocol: TCP + name: metrics + template: gateway/service-metrics.yaml + - equal: + path: spec.selector + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: gateway + template: gateway/service-metrics.yaml + + - it: rejects a metrics port equal to the gateway port + template: gateway/deployment.yaml + set: + gateway.metricsServer.enabled: true + gateway.metricsServer.port: 4000 + asserts: + - failedTemplate: + errorMessage: gateway.metricsServer.port must differ from the gateway port 4000 diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 6c9fb9440c7..b5d535c992d 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -268,6 +268,22 @@ gateway: config: create: true proxy_config: {} + # Serve Prometheus /metrics from a `metrics` sidecar container (same image, + # `python -m litellm.proxy.prometheus_metrics_server`) that aggregates the + # workers' PROMETHEUS_MULTIPROC_DIR samples over a shared emptyDir, so a + # scrape never runs on an inference worker. Adds a `metrics` port to the pod + # and a dedicated ClusterIP `-metrics` Service; point your scrape + # config at it. The port has no virtual-key auth: keep it off public ingress. + # Needs the gateway image v1.101.0 or newer. + metricsServer: + enabled: false + port: 4001 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi image: repository: ghcr.io/berriai/litellm-gateway tag: "" # defaults to .Chart.AppVersion diff --git a/litellm/proxy/prometheus_cleanup.py b/litellm/proxy/prometheus_cleanup.py index d9827723887..c65aaeedfaa 100644 --- a/litellm/proxy/prometheus_cleanup.py +++ b/litellm/proxy/prometheus_cleanup.py @@ -8,10 +8,13 @@ from __future__ import annotations import glob import os +import re from typing import Final from litellm._logging import verbose_proxy_logger +_LIVE_GAUGE_PID: Final = re.compile(r"gauge_live[a-z]*_(\d+)\.db$") + def wipe_directory(directory: str) -> None: """Delete all .db files in the directory. Called once before workers fork.""" @@ -38,3 +41,35 @@ def mark_worker_exit(worker_pid: int) -> None: verbose_proxy_logger.info("Prometheus cleanup: marked worker %s as dead", worker_pid) except Exception as e: verbose_proxy_logger.warning("Failed to mark prometheus worker %s as dead: %s", worker_pid, e) + + +def _is_running(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def mark_dead_workers(directory: str) -> tuple[int, ...]: + """Drop the live-gauge files of workers that no longer exist and return their pids. + + Uvicorn's multi-worker supervisor has no exit hook, so a replacement worker calls this at startup; without it + a crashed worker's in-flight gauges stay in the aggregate forever. + """ + owners: Final = frozenset( + int(match.group(1)) + for match in map(_LIVE_GAUGE_PID.search, glob.glob(os.path.join(directory, "gauge_live*_*.db"))) + if match is not None + ) + dead: Final = tuple(sorted(pid for pid in owners if pid != os.getpid() and not _is_running(pid))) + if not dead: + return dead + from prometheus_client import multiprocess + + for pid in dead: + multiprocess.mark_process_dead(pid, path=directory) + verbose_proxy_logger.info("Prometheus cleanup: marked dead workers %s in %s", dead, directory) + return dead diff --git a/litellm/proxy/prometheus_metrics_server.py b/litellm/proxy/prometheus_metrics_server.py index 4a9651d62e1..01479f9dd41 100644 --- a/litellm/proxy/prometheus_metrics_server.py +++ b/litellm/proxy/prometheus_metrics_server.py @@ -30,6 +30,7 @@ from litellm.integrations.prometheus_metrics_endpoint import make_metrics_asgi_a from litellm.llms.custom_httpx.http_handler import HTTPHandler METRICS_PATH: Final = "/metrics" +HEALTH_PATH: Final = "/health" PID_HEADER: Final = "x-litellm-metrics-pid" _PARENT_POLL_INTERVAL_SECONDS: Final = 1.0 _STARTUP_TIMEOUT_SECONDS: Final = 30.0 @@ -77,6 +78,10 @@ def build_metrics_app(multiproc_dir: str) -> FastAPI: app: Final = FastAPI(title="LiteLLM Prometheus metrics", docs_url=None, redoc_url=None, openapi_url=None) app.mount(METRICS_PATH, _add_pid_header(make_metrics_asgi_app(registry))) + @app.get(HEALTH_PATH) + def health() -> dict[str, str]: + return {"status": "healthy", "multiproc_dir": multiproc_dir} + return app diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1d6baf5fded..38c4c2e96c9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -631,6 +631,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( router as pass_through_router, ) +from litellm.proxy.prometheus_cleanup import mark_dead_workers, mark_worker_exit from litellm.proxy.public_endpoints import router as public_endpoints_router from litellm.proxy.public_endpoints.public_v1 import router as public_v1_router from litellm.proxy.rag_endpoints.endpoints import router as rag_router @@ -1059,6 +1060,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: init_verbose_loggers() + prometheus_multiproc_dir: Final = os.environ.get("PROMETHEUS_MULTIPROC_DIR") + if prometheus_multiproc_dir: + mark_dead_workers(prometheus_multiproc_dir) + ## RUN WORKER STARTUP HOOKS (e.g., gflags initialization) ## _startup_hooks_env: Final = os.environ.get("LITELLM_WORKER_STARTUP_HOOKS", "") if _startup_hooks_env: @@ -1365,6 +1370,9 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: await proxy_shutdown_event(worker_heartbeat=worker_heartbeat) + if prometheus_multiproc_dir: + mark_worker_exit(os.getpid()) + def _generate_stable_operation_id(route: "APIRoute") -> str: operation_id = re.sub(r"\W", "_", f"{route.name}{route.path_format}") diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 389027bf5ca..923a4ca2da8 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -242,6 +242,22 @@ this with `litellm_license`. To tune the export cadence, set `LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS` through `gateway_extra_env` / `backend_extra_env` +### Prometheus metrics sidecar + +`gateway_metrics_port` adds a `metrics` sidecar +(`python -m litellm.proxy.prometheus_metrics_server`) to the gateway task that +aggregates the workers' samples over a shared task volume, so a scrape never +runs on an inference worker. The ALB never routes to that port and the tasks +security group only opens it to `gateway_metrics_scrape_cidrs`. Needs +`gateway_image` v1.101.0 or newer. See +[Prometheus metrics](https://docs.litellm.ai/docs/proxy/prometheus) for the +metrics themselves. + +```hcl +gateway_metrics_port = 4001 +gateway_metrics_scrape_cidrs = ["10.0.0.0/16"] +``` + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index 2fb1ca08205..aa2c3d558e2 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -212,6 +212,45 @@ locals { # pull the config from S3 first, so the command goes through `sh -c`; # otherwise we keep the image's ENTRYPOINT and only override `command`. gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}" + + metrics_enabled = var.gateway_metrics_port != null + metrics_multiproc_dir = "/tmp/litellm_prometheus_multiproc" + metrics_volume = "prometheus-multiproc" + metrics_env = local.metrics_enabled ? [{ name = "PROMETHEUS_MULTIPROC_DIR", value = local.metrics_multiproc_dir }] : [] + metrics_mount_points = local.metrics_enabled ? [{ sourceVolume = local.metrics_volume, containerPath = local.metrics_multiproc_dir }] : [] + metrics_health_cmd = "import socket; socket.create_connection(('127.0.0.1', ${coalesce(var.gateway_metrics_port, 0)}), timeout=2).close()" + + gateway_metrics_container = local.metrics_enabled ? [ + { + name = "metrics" + image = var.gateway_image + essential = false + entryPoint = ["python", "-m", "litellm.proxy.prometheus_metrics_server"] + command = ["--port", tostring(var.gateway_metrics_port)] + + portMappings = [{ containerPort = var.gateway_metrics_port, protocol = "tcp" }] + environment = local.metrics_env + mountPoints = local.metrics_mount_points + + healthCheck = { + command = ["CMD", "python", "-c", local.metrics_health_cmd] + interval = 30 + timeout = 5 + retries = 3 + startPeriod = 30 + } + + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.gateway.name + awslogs-region = var.region + awslogs-stream-prefix = "metrics" + } + } + } + ] : [] + backend_uvicorn_args = "--host 0.0.0.0 --port 4001" gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args};; *) exec uvicorn gateway.main:app ${local.gateway_uvicorn_args};; esac" @@ -269,7 +308,7 @@ resource "aws_ecs_task_definition" "gateway" { execution_role_arn = aws_iam_role.task_execution.arn task_role_arn = aws_iam_role.task.arn - container_definitions = jsonencode([ + container_definitions = jsonencode(concat([ merge( { name = "gateway" @@ -283,8 +322,10 @@ resource "aws_ecs_task_definition" "gateway" { local.billing_metrics_env, local.gateway_extra_env_list, local.proxy_config_env, + local.metrics_env, ) - secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list) + secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list) + mountPoints = local.metrics_mount_points # Container-level healthCheck intentionally omitted — the wolfi # runtime image doesn't ship curl/wget. The ALB target group polls @@ -301,7 +342,14 @@ resource "aws_ecs_task_definition" "gateway" { }, local.gateway_proxy_overrides, ) - ]) + ], local.gateway_metrics_container)) + + dynamic "volume" { + for_each = local.metrics_enabled ? [1] : [] + content { + name = local.metrics_volume + } + } tags = local.tags } diff --git a/terraform/litellm/aws/examples/default/main.tf b/terraform/litellm/aws/examples/default/main.tf index 2eeaf6adb50..115003f7ddd 100644 --- a/terraform/litellm/aws/examples/default/main.tf +++ b/terraform/litellm/aws/examples/default/main.tf @@ -48,4 +48,7 @@ module "litellm" { backend_extra_env = var.backend_extra_env gateway_extra_secrets = var.gateway_extra_secrets backend_extra_secrets = var.backend_extra_secrets + + gateway_metrics_port = var.gateway_metrics_port + gateway_metrics_scrape_cidrs = var.gateway_metrics_scrape_cidrs } diff --git a/terraform/litellm/aws/examples/default/terraform.tfvars.example b/terraform/litellm/aws/examples/default/terraform.tfvars.example index 59301ea6aa5..880ecf56555 100644 --- a/terraform/litellm/aws/examples/default/terraform.tfvars.example +++ b/terraform/litellm/aws/examples/default/terraform.tfvars.example @@ -102,6 +102,13 @@ env = "stage" # } # } +# ---------- Prometheus metrics sidecar ---------- +# Serve /metrics from a sidecar in the gateway task instead of the inference +# workers. The port is not behind the ALB and has no auth: open it only to +# your Prometheus subnets. +# gateway_metrics_port = 4001 +# gateway_metrics_scrape_cidrs = ["10.0.0.0/16"] + # ---------- Extra env / secrets ---------- # Plain-text env vars (non-sensitive). Land directly in the ECS task def. # gateway_extra_env = { diff --git a/terraform/litellm/aws/examples/default/variables.tf b/terraform/litellm/aws/examples/default/variables.tf index d8ab56b13af..f8140266fca 100644 --- a/terraform/litellm/aws/examples/default/variables.tf +++ b/terraform/litellm/aws/examples/default/variables.tf @@ -158,3 +158,15 @@ variable "backend_extra_secrets" { type = map(string) default = {} } + +variable "gateway_metrics_port" { + description = "Port for the Prometheus metrics sidecar in the gateway task. Null keeps /metrics on the gateway port only." + type = number + default = null +} + +variable "gateway_metrics_scrape_cidrs" { + description = "CIDRs allowed to scrape gateway_metrics_port." + type = list(string) + default = [] +} diff --git a/terraform/litellm/aws/network.tf b/terraform/litellm/aws/network.tf index 4563eefbba5..c54949b4a59 100644 --- a/terraform/litellm/aws/network.tf +++ b/terraform/litellm/aws/network.tf @@ -156,6 +156,17 @@ resource "aws_security_group" "tasks" { security_groups = [aws_security_group.alb.id] } + dynamic "ingress" { + for_each = local.metrics_enabled && length(var.gateway_metrics_scrape_cidrs) > 0 ? [1] : [] + content { + description = "Prometheus scrapers to the gateway metrics sidecar" + from_port = var.gateway_metrics_port + to_port = var.gateway_metrics_port + protocol = "tcp" + cidr_blocks = var.gateway_metrics_scrape_cidrs + } + } + egress { description = "All egress (LLM providers, RDS, Redis)" from_port = 0 diff --git a/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl b/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl new file mode 100644 index 00000000000..de839d7e5be --- /dev/null +++ b/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl @@ -0,0 +1,108 @@ +# Plan-only coverage for the Prometheus metrics sidecar wiring. Offline via +# mock_provider, same as byo_infrastructure.tftest.hcl. + +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} +mock_provider "random" {} + +variables { + region = "us-east-1" + tenant = "acme" + env = "test" + allow_plaintext_alb = true + azs = ["us-east-1a", "us-east-1b"] +} + +run "defaults_change_nothing" { + command = plan + + assert { + condition = alltrue([ + length(local.gateway_metrics_container) == 0, + length(local.metrics_env) == 0, + length(local.metrics_mount_points) == 0, + length([for r in aws_security_group.tasks.ingress : r if r.description == "Prometheus scrapers to the gateway metrics sidecar"]) == 0, + ]) + error_message = "The metrics sidecar, its env, its volume, and its security-group rule must all be absent by default." + } +} + +run "metrics_port_adds_a_sidecar_volume_and_scrape_rule" { + command = plan + + variables { + gateway_metrics_port = 9464 + gateway_metrics_scrape_cidrs = ["10.20.0.0/16"] + } + + assert { + condition = length(local.metrics_env) == 1 && local.metrics_env[0].name == "PROMETHEUS_MULTIPROC_DIR" && local.metrics_env[0].value == "/tmp/litellm_prometheus_multiproc" + error_message = "The gateway workers must write multiprocess samples to the shared dir." + } + + assert { + condition = length(local.metrics_mount_points) == 1 && local.metrics_mount_points[0].sourceVolume == "prometheus-multiproc" && local.metrics_mount_points[0].containerPath == "/tmp/litellm_prometheus_multiproc" + error_message = "Gateway and sidecar must mount the same task volume at the multiproc dir." + } + + assert { + condition = alltrue([ + length(local.gateway_metrics_container) == 1, + local.gateway_metrics_container[0].name == "metrics", + local.gateway_metrics_container[0].essential == false, + join(" ", local.gateway_metrics_container[0].entryPoint) == "python -m litellm.proxy.prometheus_metrics_server", + join(" ", local.gateway_metrics_container[0].command) == "--port 9464", + one(local.gateway_metrics_container[0].portMappings).containerPort == 9464, + one(local.gateway_metrics_container[0].environment).value == "/tmp/litellm_prometheus_multiproc", + one(local.gateway_metrics_container[0].mountPoints).sourceVolume == "prometheus-multiproc", + strcontains(local.gateway_metrics_container[0].healthCheck.command[3], "9464"), + ]) + error_message = "The metrics sidecar must run prometheus_metrics_server on the configured port, share the multiproc volume, and health-check that port." + } + + assert { + condition = length(aws_ecs_task_definition.gateway.volume) == 1 && one(aws_ecs_task_definition.gateway.volume).name == "prometheus-multiproc" + error_message = "The gateway task must declare the multiproc volume." + } + + assert { + condition = length([ + for r in aws_security_group.tasks.ingress : r + if r.from_port == 9464 && r.to_port == 9464 && r.protocol == "tcp" && r.cidr_blocks == tolist(["10.20.0.0/16"]) + ]) == 1 + error_message = "The scrape CIDRs must be allowed to reach the metrics port on the tasks security group." + } + + assert { + condition = aws_lb_target_group.gateway.port == 4000 && one(aws_ecs_service.gateway.load_balancer).container_port == 4000 + error_message = "The ALB must keep targeting the gateway port only; the metrics port is never load balanced." + } +} + +run "metrics_port_without_scrape_cidrs_opens_nothing" { + command = plan + + variables { + gateway_metrics_port = 9464 + } + + assert { + condition = length(local.gateway_metrics_container) == 1 && length([for r in aws_security_group.tasks.ingress : r if r.from_port == 9464]) == 0 + error_message = "Without scrape CIDRs the sidecar runs but the metrics port stays closed to everything but the ALB group." + } +} + +run "metrics_port_may_not_reuse_the_gateway_port" { + command = plan + + variables { + gateway_metrics_port = 4000 + } + + expect_failures = [var.gateway_metrics_port] +} diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index 522138953d6..667f6db63c9 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -549,6 +549,44 @@ variable "proxy_config" { default = {} } +# ---------- Prometheus metrics sidecar ---------- + +variable "gateway_metrics_port" { + description = <<-EOT + Serve Prometheus /metrics from a `metrics` sidecar container in the + gateway task on this port (1-65535, not 4000), so a scrape never runs on + an inference worker. The sidecar runs the gateway image with + `python -m litellm.proxy.prometheus_metrics_server` and aggregates the + workers' PROMETHEUS_MULTIPROC_DIR samples over a task volume. Null (the + default) leaves /metrics on the gateway port only. The sidecar port has + no virtual-key auth and is not routed through the ALB; open it to your + scrapers with gateway_metrics_scrape_cidrs. Needs gateway_image v1.101.0 + or newer. + EOT + type = number + default = null + + validation { + condition = var.gateway_metrics_port == null || (var.gateway_metrics_port >= 1 && var.gateway_metrics_port <= 65535 && var.gateway_metrics_port != 4000) + error_message = "gateway_metrics_port must be between 1 and 65535 and must not be 4000 (the gateway port)." + } +} + +variable "gateway_metrics_scrape_cidrs" { + description = <<-EOT + CIDR blocks allowed to reach gateway_metrics_port on the gateway tasks + (your Prometheus or collector subnets). Empty by default, so only the + ALB can reach the tasks. Ignored when gateway_metrics_port is null. + EOT + type = list(string) + default = [] + + validation { + condition = alltrue([for c in var.gateway_metrics_scrape_cidrs : can(cidrnetmask(c))]) + error_message = "gateway_metrics_scrape_cidrs must contain valid IPv4 CIDR blocks." + } +} + variable "log_retention_days" { description = "CloudWatch log retention for the three services." type = number diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index 7db85dc6943..dafe686c4f2 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -22,6 +22,7 @@ import inspect import json import logging import os +import subprocess from collections.abc import Awaitable, Callable from typing import List, Optional, Union from unittest.mock import AsyncMock, MagicMock, patch @@ -749,6 +750,30 @@ async def test_proxy_startup_event_invalid_missing_app_arg_raises(): pass +@pytest.mark.asyncio +async def test_proxy_startup_event_prunes_dead_workers_live_gauges(tmp_path): + """With PROMETHEUS_MULTIPROC_DIR set, a booting worker drops the live-gauge files of pids that no longer + exist, so a crashed worker's in-flight samples leave the aggregate as soon as its replacement starts.""" + exited = subprocess.Popen(["true"]) + assert exited.wait(timeout=30) == 0 + stale = tmp_path / f"gauge_livesum_{exited.pid}.db" + stale.touch() + counter = tmp_path / f"counter_{exited.pid}.db" + counter.touch() + + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} + clean_env["PROMETHEUS_MULTIPROC_DIR"] = str(tmp_path) + with patch.dict(os.environ, clean_env, clear=True): + try: + async with proxy_startup_event(app=None): + pass + except Exception: + pass + + assert not stale.exists() + assert counter.exists() + + def test_otel_global_provider_published_after_callback_init(): """The OTel V2 global-provider publish must run after callback initialization in ``proxy_startup_event``. diff --git a/tests/test_litellm/proxy/test_prometheus_cleanup.py b/tests/test_litellm/proxy/test_prometheus_cleanup.py index 93b9b694c2c..6a1b95c51ff 100644 --- a/tests/test_litellm/proxy/test_prometheus_cleanup.py +++ b/tests/test_litellm/proxy/test_prometheus_cleanup.py @@ -6,13 +6,77 @@ ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir. from __future__ import annotations import os +import subprocess +import sys +from pathlib import Path +from typing import Final from unittest.mock import patch import pytest +from prometheus_client import CollectorRegistry, multiprocess -from litellm.proxy.prometheus_cleanup import mark_worker_exit, wipe_directory +from litellm.proxy.prometheus_cleanup import mark_dead_workers, mark_worker_exit, wipe_directory from litellm.proxy.proxy_cli import ProxyInitializationHelpers +_WORKER: Final = """ +import sys, time +from prometheus_client import Gauge +Gauge("litellm_in_flight", "", multiprocess_mode="livesum").set(float(sys.argv[1])) +print("ready", flush=True) +if sys.argv[2] == "stay": + time.sleep(120) +""" + + +def _spawn_worker(directory: Path, in_flight: str, lifetime: str) -> subprocess.Popen[str]: + env = {**os.environ, "PROMETHEUS_MULTIPROC_DIR": str(directory)} + worker = subprocess.Popen( + [sys.executable, "-c", _WORKER, in_flight, lifetime], env=env, stdout=subprocess.PIPE, text=True + ) + assert worker.stdout is not None and worker.stdout.readline() == "ready\n" + return worker + + +def _livesum(directory: Path) -> float: + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry, path=str(directory)) + value = registry.get_sample_value("litellm_in_flight") + return 0.0 if value is None else value + + +class TestMarkDeadWorkers: + def test_drops_live_gauges_of_exited_workers_and_keeps_running_ones(self, tmp_path: Path) -> None: + """A worker that died mid-request leaves its livesum file behind; the replacement worker's startup prune + must remove exactly that file so the aggregate stops counting requests nobody is serving.""" + dead = _spawn_worker(tmp_path, "3", "exit") + assert dead.wait(timeout=30) == 0 + alive = _spawn_worker(tmp_path, "2", "stay") + try: + assert (tmp_path / f"gauge_livesum_{dead.pid}.db").exists() + assert _livesum(tmp_path) == 5.0 + + assert mark_dead_workers(str(tmp_path)) == (dead.pid,) + + assert not (tmp_path / f"gauge_livesum_{dead.pid}.db").exists() + assert (tmp_path / f"gauge_livesum_{alive.pid}.db").exists() + assert _livesum(tmp_path) == 2.0 + assert mark_dead_workers(str(tmp_path)) == () + finally: + alive.kill() + alive.wait(timeout=30) + + def test_leaves_counters_of_exited_workers_alone(self, tmp_path: Path) -> None: + (tmp_path / "counter_424242.db").touch() + (tmp_path / "histogram_424242.db").touch() + assert mark_dead_workers(str(tmp_path)) == () + assert sorted(p.name for p in tmp_path.glob("*.db")) == ["counter_424242.db", "histogram_424242.db"] + + def test_keeps_live_gauges_of_workers_it_may_not_signal(self, tmp_path: Path) -> None: + """Signal 0 to pid 1 raises PermissionError for an unprivileged proxy; that pid is alive, not dead.""" + (tmp_path / "gauge_livesum_1.db").touch() + assert mark_dead_workers(str(tmp_path)) == () + assert (tmp_path / "gauge_livesum_1.db").exists() + class TestWipeDirectory: def test_deletes_all_db_files(self, tmp_path): diff --git a/tests/test_litellm/proxy/test_prometheus_metrics_server.py b/tests/test_litellm/proxy/test_prometheus_metrics_server.py index fc1fa381fa4..e2f461e61a6 100644 --- a/tests/test_litellm/proxy/test_prometheus_metrics_server.py +++ b/tests/test_litellm/proxy/test_prometheus_metrics_server.py @@ -1,5 +1,5 @@ -"""The separate metrics server must aggregate PROMETHEUS_MULTIPROC_DIR, expose only /metrics, and follow its -parent's lifetime. +"""The separate metrics server must aggregate PROMETHEUS_MULTIPROC_DIR, expose /metrics plus a probe-friendly +/health, and follow its parent's lifetime. Everything here runs on loopback against a child of this test process; no LLM keys or external network. """ @@ -91,7 +91,10 @@ def test_metrics_app_aggregates_multiproc_dir_and_reports_pid(tmp_path: Path, mo assert metrics.headers[PID_HEADER] == str(os.getpid()) assert 'litellm_requests_metric_total{model="gpt-5"} 5.0' in metrics.text - assert client.get("/health").status_code == 404 + health: Final = client.get("/health") + assert health.status_code == 200 + assert health.json() == {"status": "healthy", "multiproc_dir": str(tmp_path)} + assert client.get("/docs").status_code == 404 empty: Final = TestClient(build_metrics_app(str(other_dir))).get("/metrics") assert empty.status_code == 200 diff --git a/tests/test_litellm/test_component_entrypoint.py b/tests/test_litellm/test_component_entrypoint.py index b0969e2c694..09837d2b233 100644 --- a/tests/test_litellm/test_component_entrypoint.py +++ b/tests/test_litellm/test_component_entrypoint.py @@ -223,6 +223,59 @@ def test_gating_matches_the_monolithic_entrypoint_and_get_secret_bool( assert monolith[1] == ("args=litellm --port 4000" if traced else "args=--port 4000") +def test_wipes_the_prometheus_multiproc_dir_before_uvicorn_forks(tmp_path: Path) -> None: + """A restarted container inherits the emptyDir of its predecessor, whose worker pids it may reuse, so the + stale .db files must be gone before any worker opens the one carrying its own pid.""" + multiproc_dir = tmp_path / "multiproc" + multiproc_dir.mkdir() + (multiproc_dir / "gauge_livesum_7.db").write_bytes(b"stale") + (multiproc_dir / "counter_7.db").write_bytes(b"stale") + (multiproc_dir / "keep.txt").write_text("not a sample") + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_stubs(bin_dir, ("uvicorn",)) + record = tmp_path / "record.txt" + env = { + **os.environ, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "RECORD": str(record), + "PROMETHEUS_MULTIPROC_DIR": str(multiproc_dir), + } + env.pop("USE_DDTRACE", None) + result = subprocess.run( + ["sh", str(COMPONENT_ENTRYPOINT), "uvicorn", "gateway.main:app"], + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + assert sorted(p.name for p in multiproc_dir.iterdir()) == ["keep.txt"] + assert record.read_text().splitlines()[0] == "exec=uvicorn" + + +def test_creates_a_missing_prometheus_multiproc_dir(tmp_path: Path) -> None: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_stubs(bin_dir, ("uvicorn",)) + missing = tmp_path / "multiproc" + env = { + **os.environ, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "RECORD": str(tmp_path / "record.txt"), + "PROMETHEUS_MULTIPROC_DIR": str(missing), + } + env.pop("USE_DDTRACE", None) + result = subprocess.run( + ["sh", str(COMPONENT_ENTRYPOINT), "uvicorn", "gateway.main:app"], env=env, capture_output=True, text=True + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + assert missing.is_dir() + + def _copied_script(dockerfile: Path, image_path: str) -> Path: """Resolve the repo file a Dockerfile `COPY`s to `image_path`, so tests run what the image ships.""" matches = _COPY_RE.findall(dockerfile.read_text())