mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
feat(deploy): metrics sidecar and separate metrics port in Helm and Terraform (#40163)
* feat(deploy): expose SSE keepalive, pre-call checks and a metrics sidecar in Helm and Terraform Typed reliability values on both Helm charts and the AWS/GCP Terraform modules, a dedicated ClusterIP Service for the separate Prometheus port, a /health route on the metrics server and dead-worker pruning so the aggregate does not keep stale multiprocess samples. Resolves LIT-7142 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(deploy): drop reliability config from Helm and Terraform, keep only the metrics sidecar Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): cover startup pruning of dead workers' live gauges and unsignalable pids Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin <yassin@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
4a3a78c256
commit
9e18526887
26 changed files with 830 additions and 10 deletions
|
|
@ -1,5 +1,11 @@
|
|||
#!/bin/sh
|
||||
|
||||
# stale samples from a previous container incarnation would be summed into the aggregate
|
||||
if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then
|
||||
mkdir -p "$PROMETHEUS_MULTIPROC_DIR"
|
||||
rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db
|
||||
fi
|
||||
|
||||
case "$USE_DDTRACE" in
|
||||
[Tt][Rr][Uu][Ee])
|
||||
export DD_TRACE_OPENAI_ENABLED="False"
|
||||
|
|
|
|||
|
|
@ -152,6 +152,13 @@ spec:
|
|||
{{- if .Values.billingMetrics.enabled }}
|
||||
{{- include "litellm.billingMetricsEnv" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.metricsServer.enabled }}
|
||||
{{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }}
|
||||
{{- fail "metricsServer.port must differ from service.port" }}
|
||||
{{- end }}
|
||||
- name: PROMETHEUS_METRICS_PORT
|
||||
value: {{ .Values.metricsServer.port | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.migrationJob.enabled }}
|
||||
# Schema updates are owned by the dedicated migrations Job; skip
|
||||
# the proxy's startup `prisma db push` so N replicas don't race
|
||||
|
|
@ -189,6 +196,11 @@ spec:
|
|||
- name: http
|
||||
containerPort: {{ .Values.service.port }}
|
||||
protocol: TCP
|
||||
{{- if .Values.metricsServer.enabled }}
|
||||
- name: metrics
|
||||
containerPort: {{ .Values.metricsServer.port }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: {{ .Values.livenessProbe.path | quote }}
|
||||
|
|
|
|||
17
helm/litellm-helm/templates/service-metrics.yaml
Normal file
17
helm/litellm-helm/templates/service-metrics.yaml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
{{- if .Values.metricsServer.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "litellm.fullname" . }}-metrics
|
||||
labels:
|
||||
{{- include "litellm.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: {{ .Values.metricsServer.port }}
|
||||
targetPort: metrics
|
||||
protocol: TCP
|
||||
name: metrics
|
||||
selector:
|
||||
{{- include "litellm.selectorLabels" . | nindent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -26,7 +26,7 @@ spec:
|
|||
{{- toYaml .namespaceSelector.matchNames | nindent 4 }}
|
||||
{{- end }}
|
||||
endpoints:
|
||||
- port: http
|
||||
- port: {{ ternary "metrics" "http" $.Values.metricsServer.enabled }}
|
||||
path: /metrics/
|
||||
interval: {{ .interval }}
|
||||
scrapeTimeout: {{ .scrapeTimeout }}
|
||||
|
|
|
|||
106
helm/litellm-helm/tests/metrics_server_tests.yaml
Normal file
106
helm/litellm-helm/tests/metrics_server_tests.yaml
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
suite: separate metrics server
|
||||
templates:
|
||||
- configmap-litellm.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- service-metrics.yaml
|
||||
- servicemonitor.yaml
|
||||
tests:
|
||||
- it: should not expose a metrics port or PROMETHEUS_METRICS_PORT by default
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].ports
|
||||
content:
|
||||
name: metrics
|
||||
any: true
|
||||
template: deployment.yaml
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: PROMETHEUS_METRICS_PORT
|
||||
any: true
|
||||
template: deployment.yaml
|
||||
- lengthEqual:
|
||||
path: spec.ports
|
||||
count: 1
|
||||
template: service.yaml
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
template: service-metrics.yaml
|
||||
|
||||
- it: should scrape the proxy port when the metrics server is disabled
|
||||
template: servicemonitor.yaml
|
||||
set:
|
||||
serviceMonitor.enabled: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.endpoints[0].port
|
||||
value: http
|
||||
|
||||
- it: should wire the separate metrics server through container, a ClusterIP metrics service and servicemonitor
|
||||
set:
|
||||
metricsServer.enabled: true
|
||||
metricsServer.port: 4101
|
||||
serviceMonitor.enabled: true
|
||||
service.type: LoadBalancer
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: PROMETHEUS_METRICS_PORT
|
||||
value: "4101"
|
||||
template: deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].ports
|
||||
content:
|
||||
name: metrics
|
||||
containerPort: 4101
|
||||
protocol: TCP
|
||||
template: deployment.yaml
|
||||
- lengthEqual:
|
||||
path: spec.ports
|
||||
count: 1
|
||||
template: service.yaml
|
||||
- equal:
|
||||
path: spec.type
|
||||
value: LoadBalancer
|
||||
template: service.yaml
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: RELEASE-NAME-litellm-metrics
|
||||
template: service-metrics.yaml
|
||||
- equal:
|
||||
path: spec.type
|
||||
value: ClusterIP
|
||||
template: service-metrics.yaml
|
||||
- equal:
|
||||
path: spec.ports
|
||||
value:
|
||||
- port: 4101
|
||||
targetPort: metrics
|
||||
protocol: TCP
|
||||
name: metrics
|
||||
template: service-metrics.yaml
|
||||
- equal:
|
||||
path: spec.selector
|
||||
value:
|
||||
app.kubernetes.io/name: litellm
|
||||
app.kubernetes.io/instance: RELEASE-NAME
|
||||
template: service-metrics.yaml
|
||||
- equal:
|
||||
path: spec.endpoints[0].port
|
||||
value: metrics
|
||||
template: servicemonitor.yaml
|
||||
- equal:
|
||||
path: spec.endpoints[0].path
|
||||
value: /metrics/
|
||||
template: servicemonitor.yaml
|
||||
|
||||
- it: should reject a metrics port equal to the proxy port
|
||||
template: deployment.yaml
|
||||
set:
|
||||
metricsServer.enabled: true
|
||||
metricsServer.port: 4000
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: metricsServer.port must differ from service.port
|
||||
|
|
@ -180,6 +180,16 @@ proxy_config:
|
|||
general_settings:
|
||||
master_key: os.environ/PROXY_MASTER_KEY
|
||||
|
||||
# Serve Prometheus /metrics from a separate process (PROMETHEUS_METRICS_PORT)
|
||||
# so a scrape never runs on an inference worker. Adds a `metrics` port to the
|
||||
# container and a dedicated ClusterIP `<release>-metrics` Service, and the
|
||||
# ServiceMonitor scrapes it instead of the proxy port. The separate port has
|
||||
# no virtual-key auth: keep it off public ingress. Needs the proxy image
|
||||
# v1.101.0 or newer.
|
||||
metricsServer:
|
||||
enabled: false
|
||||
port: 4001
|
||||
|
||||
resources:
|
||||
{}
|
||||
# Unset by default so the chart installs on small clusters such as Minikube, and so an
|
||||
|
|
|
|||
|
|
@ -441,3 +441,5 @@ ImplementationSpecific
|
|||
{{- .pathType -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}}
|
||||
|
|
|
|||
|
|
@ -64,14 +64,25 @@ spec:
|
|||
{{- if .Values.billingMetrics.enabled }}
|
||||
{{- include "litellm.billingMetricsEnv" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.gateway.metricsServer.enabled }}
|
||||
{{- if eq (int .Values.gateway.metricsServer.port) 4000 }}
|
||||
{{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }}
|
||||
{{- end }}
|
||||
- name: PROMETHEUS_MULTIPROC_DIR
|
||||
value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
|
||||
{{- end }}
|
||||
{{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
|
||||
{{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled }}
|
||||
{{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }}
|
||||
volumeMounts:
|
||||
{{- if .Values.gateway.config.create }}
|
||||
- name: gateway-config
|
||||
mountPath: /app/config/config.yaml
|
||||
subPath: config.yaml
|
||||
{{- end }}
|
||||
{{- if .Values.gateway.metricsServer.enabled }}
|
||||
- name: prometheus-multiproc
|
||||
mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.billingMetrics.enabled }}
|
||||
{{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
|
||||
{{- end }}
|
||||
|
|
@ -97,16 +108,54 @@ spec:
|
|||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.gateway.resources | nindent 12 }}
|
||||
{{- if .Values.gateway.metricsServer.enabled }}
|
||||
- name: metrics
|
||||
image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
|
||||
{{- with .Values.gateway.securityContext }}
|
||||
securityContext:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- litellm.proxy.prometheus_metrics_server
|
||||
- --port
|
||||
- {{ .Values.gateway.metricsServer.port | quote }}
|
||||
env:
|
||||
- name: PROMETHEUS_MULTIPROC_DIR
|
||||
value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: {{ .Values.gateway.metricsServer.port }}
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: prometheus-multiproc
|
||||
mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
|
||||
readinessProbe:
|
||||
tcpSocket: { port: metrics }
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
tcpSocket: { port: metrics }
|
||||
periodSeconds: 15
|
||||
failureThreshold: 6
|
||||
resources:
|
||||
{{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.gateway.extraContainers }}
|
||||
{{- tpl (toYaml .) $ | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled }}
|
||||
{{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }}
|
||||
volumes:
|
||||
{{- if .Values.gateway.config.create }}
|
||||
- name: gateway-config
|
||||
configMap:
|
||||
name: {{ include "litellm.gateway.fullname" . }}-config
|
||||
{{- end }}
|
||||
{{- if .Values.gateway.metricsServer.enabled }}
|
||||
- name: prometheus-multiproc
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if .Values.billingMetrics.enabled }}
|
||||
{{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
|
||||
{{- end }}
|
||||
|
|
|
|||
18
helm/litellm/templates/gateway/service-metrics.yaml
Normal file
18
helm/litellm/templates/gateway/service-metrics.yaml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{{- if and .Values.gateway.enabled .Values.gateway.metricsServer.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "litellm.gateway.fullname" . }}-metrics
|
||||
labels:
|
||||
{{- include "litellm.commonLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: gateway
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: {{ .Values.gateway.metricsServer.port }}
|
||||
targetPort: metrics
|
||||
protocol: TCP
|
||||
name: metrics
|
||||
selector:
|
||||
{{- include "litellm.gateway.selectorLabels" . | nindent 4 }}
|
||||
{{- end }}
|
||||
148
helm/litellm/tests/metrics_server_tests.yaml
Normal file
148
helm/litellm/tests/metrics_server_tests.yaml
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
suite: test gateway metrics sidecar
|
||||
templates:
|
||||
- gateway/configmap.yaml
|
||||
- gateway/deployment.yaml
|
||||
- gateway/service.yaml
|
||||
- gateway/service-metrics.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
tests:
|
||||
- it: adds no sidecar, volume, env or service port when the metrics server is off
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.template.spec.containers
|
||||
count: 1
|
||||
template: gateway/deployment.yaml
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: PROMETHEUS_MULTIPROC_DIR
|
||||
any: true
|
||||
template: gateway/deployment.yaml
|
||||
- notContains:
|
||||
path: spec.template.spec.volumes
|
||||
content:
|
||||
name: prometheus-multiproc
|
||||
any: true
|
||||
template: gateway/deployment.yaml
|
||||
- lengthEqual:
|
||||
path: spec.ports
|
||||
count: 1
|
||||
template: gateway/service.yaml
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
template: gateway/service-metrics.yaml
|
||||
|
||||
- it: runs the metrics server as a sidecar over a shared multiproc dir and exposes it on a ClusterIP metrics service
|
||||
set:
|
||||
gateway.metricsServer.enabled: true
|
||||
gateway.metricsServer.port: 4101
|
||||
gateway.service.type: LoadBalancer
|
||||
gateway.image.tag: v1.101.0
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: PROMETHEUS_MULTIPROC_DIR
|
||||
value: /tmp/litellm_prometheus_multiproc
|
||||
template: gateway/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
content:
|
||||
name: prometheus-multiproc
|
||||
mountPath: /tmp/litellm_prometheus_multiproc
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[1].name
|
||||
value: metrics
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[1].image
|
||||
value: ghcr.io/berriai/litellm-gateway:v1.101.0
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[1].command
|
||||
value:
|
||||
- python
|
||||
- -m
|
||||
- litellm.proxy.prometheus_metrics_server
|
||||
- --port
|
||||
- "4101"
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[1].env
|
||||
value:
|
||||
- name: PROMETHEUS_MULTIPROC_DIR
|
||||
value: /tmp/litellm_prometheus_multiproc
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[1].ports
|
||||
value:
|
||||
- name: metrics
|
||||
containerPort: 4101
|
||||
protocol: TCP
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[1].volumeMounts
|
||||
value:
|
||||
- name: prometheus-multiproc
|
||||
mountPath: /tmp/litellm_prometheus_multiproc
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[1].readinessProbe.tcpSocket.port
|
||||
value: metrics
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[1].livenessProbe.tcpSocket.port
|
||||
value: metrics
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[1].resources.requests.cpu
|
||||
value: 50m
|
||||
template: gateway/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.volumes
|
||||
content:
|
||||
name: prometheus-multiproc
|
||||
emptyDir: {}
|
||||
template: gateway/deployment.yaml
|
||||
- lengthEqual:
|
||||
path: spec.ports
|
||||
count: 1
|
||||
template: gateway/service.yaml
|
||||
- equal:
|
||||
path: spec.type
|
||||
value: LoadBalancer
|
||||
template: gateway/service.yaml
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: RELEASE-NAME-litellm-gateway-metrics
|
||||
template: gateway/service-metrics.yaml
|
||||
- equal:
|
||||
path: spec.type
|
||||
value: ClusterIP
|
||||
template: gateway/service-metrics.yaml
|
||||
- equal:
|
||||
path: spec.ports
|
||||
value:
|
||||
- port: 4101
|
||||
targetPort: metrics
|
||||
protocol: TCP
|
||||
name: metrics
|
||||
template: gateway/service-metrics.yaml
|
||||
- equal:
|
||||
path: spec.selector
|
||||
value:
|
||||
app.kubernetes.io/name: litellm
|
||||
app.kubernetes.io/instance: RELEASE-NAME
|
||||
app.kubernetes.io/component: gateway
|
||||
template: gateway/service-metrics.yaml
|
||||
|
||||
- it: rejects a metrics port equal to the gateway port
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
gateway.metricsServer.enabled: true
|
||||
gateway.metricsServer.port: 4000
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: gateway.metricsServer.port must differ from the gateway port 4000
|
||||
|
|
@ -268,6 +268,22 @@ gateway:
|
|||
config:
|
||||
create: true
|
||||
proxy_config: {}
|
||||
# Serve Prometheus /metrics from a `metrics` sidecar container (same image,
|
||||
# `python -m litellm.proxy.prometheus_metrics_server`) that aggregates the
|
||||
# workers' PROMETHEUS_MULTIPROC_DIR samples over a shared emptyDir, so a
|
||||
# scrape never runs on an inference worker. Adds a `metrics` port to the pod
|
||||
# and a dedicated ClusterIP `<gateway>-metrics` Service; point your scrape
|
||||
# config at it. The port has no virtual-key auth: keep it off public ingress.
|
||||
# Needs the gateway image v1.101.0 or newer.
|
||||
metricsServer:
|
||||
enabled: false
|
||||
port: 4001
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
image:
|
||||
repository: ghcr.io/berriai/litellm-gateway
|
||||
tag: "" # defaults to .Chart.AppVersion
|
||||
|
|
|
|||
|
|
@ -8,10 +8,13 @@ from __future__ import annotations
|
|||
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
_LIVE_GAUGE_PID: Final = re.compile(r"gauge_live[a-z]*_(\d+)\.db$")
|
||||
|
||||
|
||||
def wipe_directory(directory: str) -> None:
|
||||
"""Delete all .db files in the directory. Called once before workers fork."""
|
||||
|
|
@ -38,3 +41,35 @@ def mark_worker_exit(worker_pid: int) -> None:
|
|||
verbose_proxy_logger.info("Prometheus cleanup: marked worker %s as dead", worker_pid)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.warning("Failed to mark prometheus worker %s as dead: %s", worker_pid, e)
|
||||
|
||||
|
||||
def _is_running(pid: int) -> bool:
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def mark_dead_workers(directory: str) -> tuple[int, ...]:
|
||||
"""Drop the live-gauge files of workers that no longer exist and return their pids.
|
||||
|
||||
Uvicorn's multi-worker supervisor has no exit hook, so a replacement worker calls this at startup; without it
|
||||
a crashed worker's in-flight gauges stay in the aggregate forever.
|
||||
"""
|
||||
owners: Final = frozenset(
|
||||
int(match.group(1))
|
||||
for match in map(_LIVE_GAUGE_PID.search, glob.glob(os.path.join(directory, "gauge_live*_*.db")))
|
||||
if match is not None
|
||||
)
|
||||
dead: Final = tuple(sorted(pid for pid in owners if pid != os.getpid() and not _is_running(pid)))
|
||||
if not dead:
|
||||
return dead
|
||||
from prometheus_client import multiprocess
|
||||
|
||||
for pid in dead:
|
||||
multiprocess.mark_process_dead(pid, path=directory)
|
||||
verbose_proxy_logger.info("Prometheus cleanup: marked dead workers %s in %s", dead, directory)
|
||||
return dead
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.integrations.prometheus_metrics_endpoint import make_metrics_asgi_a
|
|||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
METRICS_PATH: Final = "/metrics"
|
||||
HEALTH_PATH: Final = "/health"
|
||||
PID_HEADER: Final = "x-litellm-metrics-pid"
|
||||
_PARENT_POLL_INTERVAL_SECONDS: Final = 1.0
|
||||
_STARTUP_TIMEOUT_SECONDS: Final = 30.0
|
||||
|
|
@ -77,6 +78,10 @@ def build_metrics_app(multiproc_dir: str) -> FastAPI:
|
|||
app: Final = FastAPI(title="LiteLLM Prometheus metrics", docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.mount(METRICS_PATH, _add_pid_header(make_metrics_asgi_app(registry)))
|
||||
|
||||
@app.get(HEALTH_PATH)
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "healthy", "multiproc_dir": multiproc_dir}
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -631,6 +631,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
|||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
||||
router as pass_through_router,
|
||||
)
|
||||
from litellm.proxy.prometheus_cleanup import mark_dead_workers, mark_worker_exit
|
||||
from litellm.proxy.public_endpoints import router as public_endpoints_router
|
||||
from litellm.proxy.public_endpoints.public_v1 import router as public_v1_router
|
||||
from litellm.proxy.rag_endpoints.endpoints import router as rag_router
|
||||
|
|
@ -1059,6 +1060,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
|
|||
|
||||
init_verbose_loggers()
|
||||
|
||||
prometheus_multiproc_dir: Final = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
|
||||
if prometheus_multiproc_dir:
|
||||
mark_dead_workers(prometheus_multiproc_dir)
|
||||
|
||||
## RUN WORKER STARTUP HOOKS (e.g., gflags initialization) ##
|
||||
_startup_hooks_env: Final = os.environ.get("LITELLM_WORKER_STARTUP_HOOKS", "")
|
||||
if _startup_hooks_env:
|
||||
|
|
@ -1365,6 +1370,9 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
|
|||
|
||||
await proxy_shutdown_event(worker_heartbeat=worker_heartbeat)
|
||||
|
||||
if prometheus_multiproc_dir:
|
||||
mark_worker_exit(os.getpid())
|
||||
|
||||
|
||||
def _generate_stable_operation_id(route: "APIRoute") -> str:
|
||||
operation_id = re.sub(r"\W", "_", f"{route.name}{route.path_format}")
|
||||
|
|
|
|||
|
|
@ -242,6 +242,22 @@ this with `litellm_license`. To tune the export cadence, set
|
|||
`LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS` through `gateway_extra_env` /
|
||||
`backend_extra_env`
|
||||
|
||||
### Prometheus metrics sidecar
|
||||
|
||||
`gateway_metrics_port` adds a `metrics` sidecar
|
||||
(`python -m litellm.proxy.prometheus_metrics_server`) to the gateway task that
|
||||
aggregates the workers' samples over a shared task volume, so a scrape never
|
||||
runs on an inference worker. The ALB never routes to that port and the tasks
|
||||
security group only opens it to `gateway_metrics_scrape_cidrs`. Needs
|
||||
`gateway_image` v1.101.0 or newer. See
|
||||
[Prometheus metrics](https://docs.litellm.ai/docs/proxy/prometheus) for the
|
||||
metrics themselves.
|
||||
|
||||
```hcl
|
||||
gateway_metrics_port = 4001
|
||||
gateway_metrics_scrape_cidrs = ["10.0.0.0/16"]
|
||||
```
|
||||
|
||||
## Tenant deployment
|
||||
|
||||
Every resource the stack creates is named `${tenant}-litellm-${env}` (or
|
||||
|
|
|
|||
|
|
@ -212,6 +212,45 @@ locals {
|
|||
# pull the config from S3 first, so the command goes through `sh -c`;
|
||||
# otherwise we keep the image's ENTRYPOINT and only override `command`.
|
||||
gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}"
|
||||
|
||||
metrics_enabled = var.gateway_metrics_port != null
|
||||
metrics_multiproc_dir = "/tmp/litellm_prometheus_multiproc"
|
||||
metrics_volume = "prometheus-multiproc"
|
||||
metrics_env = local.metrics_enabled ? [{ name = "PROMETHEUS_MULTIPROC_DIR", value = local.metrics_multiproc_dir }] : []
|
||||
metrics_mount_points = local.metrics_enabled ? [{ sourceVolume = local.metrics_volume, containerPath = local.metrics_multiproc_dir }] : []
|
||||
metrics_health_cmd = "import socket; socket.create_connection(('127.0.0.1', ${coalesce(var.gateway_metrics_port, 0)}), timeout=2).close()"
|
||||
|
||||
gateway_metrics_container = local.metrics_enabled ? [
|
||||
{
|
||||
name = "metrics"
|
||||
image = var.gateway_image
|
||||
essential = false
|
||||
entryPoint = ["python", "-m", "litellm.proxy.prometheus_metrics_server"]
|
||||
command = ["--port", tostring(var.gateway_metrics_port)]
|
||||
|
||||
portMappings = [{ containerPort = var.gateway_metrics_port, protocol = "tcp" }]
|
||||
environment = local.metrics_env
|
||||
mountPoints = local.metrics_mount_points
|
||||
|
||||
healthCheck = {
|
||||
command = ["CMD", "python", "-c", local.metrics_health_cmd]
|
||||
interval = 30
|
||||
timeout = 5
|
||||
retries = 3
|
||||
startPeriod = 30
|
||||
}
|
||||
|
||||
logConfiguration = {
|
||||
logDriver = "awslogs"
|
||||
options = {
|
||||
awslogs-group = aws_cloudwatch_log_group.gateway.name
|
||||
awslogs-region = var.region
|
||||
awslogs-stream-prefix = "metrics"
|
||||
}
|
||||
}
|
||||
}
|
||||
] : []
|
||||
|
||||
backend_uvicorn_args = "--host 0.0.0.0 --port 4001"
|
||||
|
||||
gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args};; *) exec uvicorn gateway.main:app ${local.gateway_uvicorn_args};; esac"
|
||||
|
|
@ -269,7 +308,7 @@ resource "aws_ecs_task_definition" "gateway" {
|
|||
execution_role_arn = aws_iam_role.task_execution.arn
|
||||
task_role_arn = aws_iam_role.task.arn
|
||||
|
||||
container_definitions = jsonencode([
|
||||
container_definitions = jsonencode(concat([
|
||||
merge(
|
||||
{
|
||||
name = "gateway"
|
||||
|
|
@ -283,8 +322,10 @@ resource "aws_ecs_task_definition" "gateway" {
|
|||
local.billing_metrics_env,
|
||||
local.gateway_extra_env_list,
|
||||
local.proxy_config_env,
|
||||
local.metrics_env,
|
||||
)
|
||||
secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list)
|
||||
secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list)
|
||||
mountPoints = local.metrics_mount_points
|
||||
|
||||
# Container-level healthCheck intentionally omitted — the wolfi
|
||||
# runtime image doesn't ship curl/wget. The ALB target group polls
|
||||
|
|
@ -301,7 +342,14 @@ resource "aws_ecs_task_definition" "gateway" {
|
|||
},
|
||||
local.gateway_proxy_overrides,
|
||||
)
|
||||
])
|
||||
], local.gateway_metrics_container))
|
||||
|
||||
dynamic "volume" {
|
||||
for_each = local.metrics_enabled ? [1] : []
|
||||
content {
|
||||
name = local.metrics_volume
|
||||
}
|
||||
}
|
||||
|
||||
tags = local.tags
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,4 +48,7 @@ module "litellm" {
|
|||
backend_extra_env = var.backend_extra_env
|
||||
gateway_extra_secrets = var.gateway_extra_secrets
|
||||
backend_extra_secrets = var.backend_extra_secrets
|
||||
|
||||
gateway_metrics_port = var.gateway_metrics_port
|
||||
gateway_metrics_scrape_cidrs = var.gateway_metrics_scrape_cidrs
|
||||
}
|
||||
|
|
|
|||
|
|
@ -102,6 +102,13 @@ env = "stage"
|
|||
# }
|
||||
# }
|
||||
|
||||
# ---------- Prometheus metrics sidecar ----------
|
||||
# Serve /metrics from a sidecar in the gateway task instead of the inference
|
||||
# workers. The port is not behind the ALB and has no auth: open it only to
|
||||
# your Prometheus subnets.
|
||||
# gateway_metrics_port = 4001
|
||||
# gateway_metrics_scrape_cidrs = ["10.0.0.0/16"]
|
||||
|
||||
# ---------- Extra env / secrets ----------
|
||||
# Plain-text env vars (non-sensitive). Land directly in the ECS task def.
|
||||
# gateway_extra_env = {
|
||||
|
|
|
|||
|
|
@ -158,3 +158,15 @@ variable "backend_extra_secrets" {
|
|||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "gateway_metrics_port" {
|
||||
description = "Port for the Prometheus metrics sidecar in the gateway task. Null keeps /metrics on the gateway port only."
|
||||
type = number
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "gateway_metrics_scrape_cidrs" {
|
||||
description = "CIDRs allowed to scrape gateway_metrics_port."
|
||||
type = list(string)
|
||||
default = []
|
||||
}
|
||||
|
|
|
|||
|
|
@ -156,6 +156,17 @@ resource "aws_security_group" "tasks" {
|
|||
security_groups = [aws_security_group.alb.id]
|
||||
}
|
||||
|
||||
dynamic "ingress" {
|
||||
for_each = local.metrics_enabled && length(var.gateway_metrics_scrape_cidrs) > 0 ? [1] : []
|
||||
content {
|
||||
description = "Prometheus scrapers to the gateway metrics sidecar"
|
||||
from_port = var.gateway_metrics_port
|
||||
to_port = var.gateway_metrics_port
|
||||
protocol = "tcp"
|
||||
cidr_blocks = var.gateway_metrics_scrape_cidrs
|
||||
}
|
||||
}
|
||||
|
||||
egress {
|
||||
description = "All egress (LLM providers, RDS, Redis)"
|
||||
from_port = 0
|
||||
|
|
|
|||
108
terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl
Normal file
108
terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
# Plan-only coverage for the Prometheus metrics sidecar wiring. Offline via
|
||||
# mock_provider, same as byo_infrastructure.tftest.hcl.
|
||||
|
||||
mock_provider "aws" {
|
||||
mock_data "aws_iam_policy_document" {
|
||||
defaults = {
|
||||
json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"
|
||||
}
|
||||
}
|
||||
}
|
||||
mock_provider "random" {}
|
||||
|
||||
variables {
|
||||
region = "us-east-1"
|
||||
tenant = "acme"
|
||||
env = "test"
|
||||
allow_plaintext_alb = true
|
||||
azs = ["us-east-1a", "us-east-1b"]
|
||||
}
|
||||
|
||||
run "defaults_change_nothing" {
|
||||
command = plan
|
||||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
length(local.gateway_metrics_container) == 0,
|
||||
length(local.metrics_env) == 0,
|
||||
length(local.metrics_mount_points) == 0,
|
||||
length([for r in aws_security_group.tasks.ingress : r if r.description == "Prometheus scrapers to the gateway metrics sidecar"]) == 0,
|
||||
])
|
||||
error_message = "The metrics sidecar, its env, its volume, and its security-group rule must all be absent by default."
|
||||
}
|
||||
}
|
||||
|
||||
run "metrics_port_adds_a_sidecar_volume_and_scrape_rule" {
|
||||
command = plan
|
||||
|
||||
variables {
|
||||
gateway_metrics_port = 9464
|
||||
gateway_metrics_scrape_cidrs = ["10.20.0.0/16"]
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = length(local.metrics_env) == 1 && local.metrics_env[0].name == "PROMETHEUS_MULTIPROC_DIR" && local.metrics_env[0].value == "/tmp/litellm_prometheus_multiproc"
|
||||
error_message = "The gateway workers must write multiprocess samples to the shared dir."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = length(local.metrics_mount_points) == 1 && local.metrics_mount_points[0].sourceVolume == "prometheus-multiproc" && local.metrics_mount_points[0].containerPath == "/tmp/litellm_prometheus_multiproc"
|
||||
error_message = "Gateway and sidecar must mount the same task volume at the multiproc dir."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
length(local.gateway_metrics_container) == 1,
|
||||
local.gateway_metrics_container[0].name == "metrics",
|
||||
local.gateway_metrics_container[0].essential == false,
|
||||
join(" ", local.gateway_metrics_container[0].entryPoint) == "python -m litellm.proxy.prometheus_metrics_server",
|
||||
join(" ", local.gateway_metrics_container[0].command) == "--port 9464",
|
||||
one(local.gateway_metrics_container[0].portMappings).containerPort == 9464,
|
||||
one(local.gateway_metrics_container[0].environment).value == "/tmp/litellm_prometheus_multiproc",
|
||||
one(local.gateway_metrics_container[0].mountPoints).sourceVolume == "prometheus-multiproc",
|
||||
strcontains(local.gateway_metrics_container[0].healthCheck.command[3], "9464"),
|
||||
])
|
||||
error_message = "The metrics sidecar must run prometheus_metrics_server on the configured port, share the multiproc volume, and health-check that port."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = length(aws_ecs_task_definition.gateway.volume) == 1 && one(aws_ecs_task_definition.gateway.volume).name == "prometheus-multiproc"
|
||||
error_message = "The gateway task must declare the multiproc volume."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = length([
|
||||
for r in aws_security_group.tasks.ingress : r
|
||||
if r.from_port == 9464 && r.to_port == 9464 && r.protocol == "tcp" && r.cidr_blocks == tolist(["10.20.0.0/16"])
|
||||
]) == 1
|
||||
error_message = "The scrape CIDRs must be allowed to reach the metrics port on the tasks security group."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = aws_lb_target_group.gateway.port == 4000 && one(aws_ecs_service.gateway.load_balancer).container_port == 4000
|
||||
error_message = "The ALB must keep targeting the gateway port only; the metrics port is never load balanced."
|
||||
}
|
||||
}
|
||||
|
||||
run "metrics_port_without_scrape_cidrs_opens_nothing" {
|
||||
command = plan
|
||||
|
||||
variables {
|
||||
gateway_metrics_port = 9464
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = length(local.gateway_metrics_container) == 1 && length([for r in aws_security_group.tasks.ingress : r if r.from_port == 9464]) == 0
|
||||
error_message = "Without scrape CIDRs the sidecar runs but the metrics port stays closed to everything but the ALB group."
|
||||
}
|
||||
}
|
||||
|
||||
run "metrics_port_may_not_reuse_the_gateway_port" {
|
||||
command = plan
|
||||
|
||||
variables {
|
||||
gateway_metrics_port = 4000
|
||||
}
|
||||
|
||||
expect_failures = [var.gateway_metrics_port]
|
||||
}
|
||||
|
|
@ -549,6 +549,44 @@ variable "proxy_config" {
|
|||
default = {}
|
||||
}
|
||||
|
||||
# ---------- Prometheus metrics sidecar ----------
|
||||
|
||||
variable "gateway_metrics_port" {
|
||||
description = <<-EOT
|
||||
Serve Prometheus /metrics from a `metrics` sidecar container in the
|
||||
gateway task on this port (1-65535, not 4000), so a scrape never runs on
|
||||
an inference worker. The sidecar runs the gateway image with
|
||||
`python -m litellm.proxy.prometheus_metrics_server` and aggregates the
|
||||
workers' PROMETHEUS_MULTIPROC_DIR samples over a task volume. Null (the
|
||||
default) leaves /metrics on the gateway port only. The sidecar port has
|
||||
no virtual-key auth and is not routed through the ALB; open it to your
|
||||
scrapers with gateway_metrics_scrape_cidrs. Needs gateway_image v1.101.0
|
||||
or newer.
|
||||
EOT
|
||||
type = number
|
||||
default = null
|
||||
|
||||
validation {
|
||||
condition = var.gateway_metrics_port == null || (var.gateway_metrics_port >= 1 && var.gateway_metrics_port <= 65535 && var.gateway_metrics_port != 4000)
|
||||
error_message = "gateway_metrics_port must be between 1 and 65535 and must not be 4000 (the gateway port)."
|
||||
}
|
||||
}
|
||||
|
||||
variable "gateway_metrics_scrape_cidrs" {
|
||||
description = <<-EOT
|
||||
CIDR blocks allowed to reach gateway_metrics_port on the gateway tasks
|
||||
(your Prometheus or collector subnets). Empty by default, so only the
|
||||
ALB can reach the tasks. Ignored when gateway_metrics_port is null.
|
||||
EOT
|
||||
type = list(string)
|
||||
default = []
|
||||
|
||||
validation {
|
||||
condition = alltrue([for c in var.gateway_metrics_scrape_cidrs : can(cidrnetmask(c))])
|
||||
error_message = "gateway_metrics_scrape_cidrs must contain valid IPv4 CIDR blocks."
|
||||
}
|
||||
}
|
||||
|
||||
variable "log_retention_days" {
|
||||
description = "CloudWatch log retention for the three services."
|
||||
type = number
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import inspect
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import List, Optional, Union
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -749,6 +750,30 @@ async def test_proxy_startup_event_invalid_missing_app_arg_raises():
|
|||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_startup_event_prunes_dead_workers_live_gauges(tmp_path):
|
||||
"""With PROMETHEUS_MULTIPROC_DIR set, a booting worker drops the live-gauge files of pids that no longer
|
||||
exist, so a crashed worker's in-flight samples leave the aggregate as soon as its replacement starts."""
|
||||
exited = subprocess.Popen(["true"])
|
||||
assert exited.wait(timeout=30) == 0
|
||||
stale = tmp_path / f"gauge_livesum_{exited.pid}.db"
|
||||
stale.touch()
|
||||
counter = tmp_path / f"counter_{exited.pid}.db"
|
||||
counter.touch()
|
||||
|
||||
clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")}
|
||||
clean_env["PROMETHEUS_MULTIPROC_DIR"] = str(tmp_path)
|
||||
with patch.dict(os.environ, clean_env, clear=True):
|
||||
try:
|
||||
async with proxy_startup_event(app=None):
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
assert not stale.exists()
|
||||
assert counter.exists()
|
||||
|
||||
|
||||
def test_otel_global_provider_published_after_callback_init():
|
||||
"""The OTel V2 global-provider publish must run after callback
|
||||
initialization in ``proxy_startup_event``.
|
||||
|
|
|
|||
|
|
@ -6,13 +6,77 @@ ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir.
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from prometheus_client import CollectorRegistry, multiprocess
|
||||
|
||||
from litellm.proxy.prometheus_cleanup import mark_worker_exit, wipe_directory
|
||||
from litellm.proxy.prometheus_cleanup import mark_dead_workers, mark_worker_exit, wipe_directory
|
||||
from litellm.proxy.proxy_cli import ProxyInitializationHelpers
|
||||
|
||||
_WORKER: Final = """
|
||||
import sys, time
|
||||
from prometheus_client import Gauge
|
||||
Gauge("litellm_in_flight", "", multiprocess_mode="livesum").set(float(sys.argv[1]))
|
||||
print("ready", flush=True)
|
||||
if sys.argv[2] == "stay":
|
||||
time.sleep(120)
|
||||
"""
|
||||
|
||||
|
||||
def _spawn_worker(directory: Path, in_flight: str, lifetime: str) -> subprocess.Popen[str]:
|
||||
env = {**os.environ, "PROMETHEUS_MULTIPROC_DIR": str(directory)}
|
||||
worker = subprocess.Popen(
|
||||
[sys.executable, "-c", _WORKER, in_flight, lifetime], env=env, stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
assert worker.stdout is not None and worker.stdout.readline() == "ready\n"
|
||||
return worker
|
||||
|
||||
|
||||
def _livesum(directory: Path) -> float:
|
||||
registry = CollectorRegistry()
|
||||
multiprocess.MultiProcessCollector(registry, path=str(directory))
|
||||
value = registry.get_sample_value("litellm_in_flight")
|
||||
return 0.0 if value is None else value
|
||||
|
||||
|
||||
class TestMarkDeadWorkers:
|
||||
def test_drops_live_gauges_of_exited_workers_and_keeps_running_ones(self, tmp_path: Path) -> None:
|
||||
"""A worker that died mid-request leaves its livesum file behind; the replacement worker's startup prune
|
||||
must remove exactly that file so the aggregate stops counting requests nobody is serving."""
|
||||
dead = _spawn_worker(tmp_path, "3", "exit")
|
||||
assert dead.wait(timeout=30) == 0
|
||||
alive = _spawn_worker(tmp_path, "2", "stay")
|
||||
try:
|
||||
assert (tmp_path / f"gauge_livesum_{dead.pid}.db").exists()
|
||||
assert _livesum(tmp_path) == 5.0
|
||||
|
||||
assert mark_dead_workers(str(tmp_path)) == (dead.pid,)
|
||||
|
||||
assert not (tmp_path / f"gauge_livesum_{dead.pid}.db").exists()
|
||||
assert (tmp_path / f"gauge_livesum_{alive.pid}.db").exists()
|
||||
assert _livesum(tmp_path) == 2.0
|
||||
assert mark_dead_workers(str(tmp_path)) == ()
|
||||
finally:
|
||||
alive.kill()
|
||||
alive.wait(timeout=30)
|
||||
|
||||
def test_leaves_counters_of_exited_workers_alone(self, tmp_path: Path) -> None:
|
||||
(tmp_path / "counter_424242.db").touch()
|
||||
(tmp_path / "histogram_424242.db").touch()
|
||||
assert mark_dead_workers(str(tmp_path)) == ()
|
||||
assert sorted(p.name for p in tmp_path.glob("*.db")) == ["counter_424242.db", "histogram_424242.db"]
|
||||
|
||||
def test_keeps_live_gauges_of_workers_it_may_not_signal(self, tmp_path: Path) -> None:
|
||||
"""Signal 0 to pid 1 raises PermissionError for an unprivileged proxy; that pid is alive, not dead."""
|
||||
(tmp_path / "gauge_livesum_1.db").touch()
|
||||
assert mark_dead_workers(str(tmp_path)) == ()
|
||||
assert (tmp_path / "gauge_livesum_1.db").exists()
|
||||
|
||||
|
||||
class TestWipeDirectory:
|
||||
def test_deletes_all_db_files(self, tmp_path):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
"""The separate metrics server must aggregate PROMETHEUS_MULTIPROC_DIR, expose only /metrics, and follow its
|
||||
parent's lifetime.
|
||||
"""The separate metrics server must aggregate PROMETHEUS_MULTIPROC_DIR, expose /metrics plus a probe-friendly
|
||||
/health, and follow its parent's lifetime.
|
||||
|
||||
Everything here runs on loopback against a child of this test process; no LLM keys or external network.
|
||||
"""
|
||||
|
|
@ -91,7 +91,10 @@ def test_metrics_app_aggregates_multiproc_dir_and_reports_pid(tmp_path: Path, mo
|
|||
assert metrics.headers[PID_HEADER] == str(os.getpid())
|
||||
assert 'litellm_requests_metric_total{model="gpt-5"} 5.0' in metrics.text
|
||||
|
||||
assert client.get("/health").status_code == 404
|
||||
health: Final = client.get("/health")
|
||||
assert health.status_code == 200
|
||||
assert health.json() == {"status": "healthy", "multiproc_dir": str(tmp_path)}
|
||||
assert client.get("/docs").status_code == 404
|
||||
|
||||
empty: Final = TestClient(build_metrics_app(str(other_dir))).get("/metrics")
|
||||
assert empty.status_code == 200
|
||||
|
|
|
|||
|
|
@ -223,6 +223,59 @@ def test_gating_matches_the_monolithic_entrypoint_and_get_secret_bool(
|
|||
assert monolith[1] == ("args=litellm --port 4000" if traced else "args=--port 4000")
|
||||
|
||||
|
||||
def test_wipes_the_prometheus_multiproc_dir_before_uvicorn_forks(tmp_path: Path) -> None:
|
||||
"""A restarted container inherits the emptyDir of its predecessor, whose worker pids it may reuse, so the
|
||||
stale .db files must be gone before any worker opens the one carrying its own pid."""
|
||||
multiproc_dir = tmp_path / "multiproc"
|
||||
multiproc_dir.mkdir()
|
||||
(multiproc_dir / "gauge_livesum_7.db").write_bytes(b"stale")
|
||||
(multiproc_dir / "counter_7.db").write_bytes(b"stale")
|
||||
(multiproc_dir / "keep.txt").write_text("not a sample")
|
||||
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
_write_stubs(bin_dir, ("uvicorn",))
|
||||
record = tmp_path / "record.txt"
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
|
||||
"RECORD": str(record),
|
||||
"PROMETHEUS_MULTIPROC_DIR": str(multiproc_dir),
|
||||
}
|
||||
env.pop("USE_DDTRACE", None)
|
||||
result = subprocess.run(
|
||||
["sh", str(COMPONENT_ENTRYPOINT), "uvicorn", "gateway.main:app"],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}"
|
||||
assert sorted(p.name for p in multiproc_dir.iterdir()) == ["keep.txt"]
|
||||
assert record.read_text().splitlines()[0] == "exec=uvicorn"
|
||||
|
||||
|
||||
def test_creates_a_missing_prometheus_multiproc_dir(tmp_path: Path) -> None:
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
_write_stubs(bin_dir, ("uvicorn",))
|
||||
missing = tmp_path / "multiproc"
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
|
||||
"RECORD": str(tmp_path / "record.txt"),
|
||||
"PROMETHEUS_MULTIPROC_DIR": str(missing),
|
||||
}
|
||||
env.pop("USE_DDTRACE", None)
|
||||
result = subprocess.run(
|
||||
["sh", str(COMPONENT_ENTRYPOINT), "uvicorn", "gateway.main:app"], env=env, capture_output=True, text=True
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}"
|
||||
assert missing.is_dir()
|
||||
|
||||
|
||||
def _copied_script(dockerfile: Path, image_path: str) -> Path:
|
||||
"""Resolve the repo file a Dockerfile `COPY`s to `image_path`, so tests run what the image ships."""
|
||||
matches = _COPY_RE.findall(dockerfile.read_text())
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue