From c28c4235bdddc455d4cba84565955294df6111e7 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 10 Sep 2026 04:25:32 +0000 Subject: [PATCH] feat(helm): opt-in spend-worker sidecar and gateway-container HPA metric in the componentized chart Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- helm/litellm/templates/_helpers.tpl | 31 +++ .../litellm/templates/gateway/deployment.yaml | 53 ++++- helm/litellm/templates/gateway/hpa.yaml | 10 + helm/litellm/tests/spend_worker_tests.yaml | 189 ++++++++++++++++++ helm/litellm/values.yaml | 36 ++++ 5 files changed, 317 insertions(+), 2 deletions(-) create mode 100644 helm/litellm/tests/spend_worker_tests.yaml diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index c459512c7b9..d268d46068d 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -443,3 +443,34 @@ ImplementationSpecific {{- end -}} {{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}} + +{{/* +Directory of the spend worker's unix socket, shared by the gateway and +spend-worker containers through an emptyDir. Empty when the sidecar is off +or gateway.spendWorker.address is a tcp://127.0.0.1: address. +*/}} +{{- define "litellm.gateway.spendWorkerSocketDir" -}} +{{- if and .Values.gateway.spendWorker.enabled (hasPrefix "unix://" .Values.gateway.spendWorker.address) -}} +{{- dir (trimPrefix "unix://" .Values.gateway.spendWorker.address) -}} +{{- end -}} +{{- end -}} + +{{/* +LITELLM_SPEND_WORKER_* env shared by the producer (gateway container) and the +consumer (spend-worker container), so both agree on the transport and the +shutdown drain window. +*/}} +{{- define "litellm.gateway.spendWorkerEnv" -}} +{{- with .Values.gateway.spendWorker }} +- name: LITELLM_SPEND_WORKER_ENABLED + value: "true" +- name: LITELLM_SPEND_WORKER_ADDRESS + value: {{ .address | quote }} +- name: LITELLM_SPEND_WORKER_BUFFER_SIZE + value: {{ .bufferSize | quote }} +- name: LITELLM_SPEND_WORKER_ON_UNAVAILABLE + value: {{ .onUnavailable | quote }} +- name: LITELLM_SPEND_WORKER_DRAIN_TIMEOUT_SECONDS + value: {{ .drainTimeoutSeconds | quote }} +{{- end }} +{{- end -}} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 9cb6b07e77b..2bde10aef11 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -71,8 +71,11 @@ spec: - name: PROMETHEUS_MULTIPROC_DIR value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} {{- end }} + {{- if .Values.gateway.spendWorker.enabled }} + {{- include "litellm.gateway.spendWorkerEnv" . | nindent 12 }} + {{- end }} {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.spendWorkerSocketDir" .) }} volumeMounts: {{- if .Values.gateway.config.create }} - name: gateway-config @@ -83,6 +86,10 @@ spec: - name: prometheus-multiproc mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} {{- end }} + {{- if include "litellm.gateway.spendWorkerSocketDir" . }} + - name: spend-worker-socket + mountPath: {{ include "litellm.gateway.spendWorkerSocketDir" . }} + {{- end }} {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} {{- end }} @@ -142,10 +149,47 @@ spec: resources: {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }} {{- end }} + {{- if .Values.gateway.spendWorker.enabled }} + - name: spend-worker + image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + command: + - python + - -m + - gateway.spend_worker + env: + {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} + {{- if .Values.gateway.config.create }} + - name: CONFIG_FILE_PATH + value: /app/config/config.yaml + {{- end }} + {{- include "litellm.gateway.spendWorkerEnv" . | nindent 12 }} + - name: LITELLM_JOB_ROLE + value: spend_worker + {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} + {{- if or .Values.gateway.config.create (include "litellm.gateway.spendWorkerSocketDir" .) }} + volumeMounts: + {{- if .Values.gateway.config.create }} + - name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + {{- end }} + {{- if include "litellm.gateway.spendWorkerSocketDir" . }} + - name: spend-worker-socket + mountPath: {{ include "litellm.gateway.spendWorkerSocketDir" . }} + {{- end }} + {{- end }} + resources: + {{- toYaml .Values.gateway.spendWorker.resources | nindent 12 }} + {{- end }} {{- with .Values.gateway.extraContainers }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }} + {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.spendWorkerSocketDir" .) }} volumes: {{- if .Values.gateway.config.create }} - name: gateway-config @@ -156,6 +200,11 @@ spec: - name: prometheus-multiproc emptyDir: {} {{- end }} + {{- if include "litellm.gateway.spendWorkerSocketDir" . }} + - name: spend-worker-socket + emptyDir: + sizeLimit: 1Mi + {{- end }} {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} {{- end }} diff --git a/helm/litellm/templates/gateway/hpa.yaml b/helm/litellm/templates/gateway/hpa.yaml index 1023baf16f1..a5bf7099fd2 100644 --- a/helm/litellm/templates/gateway/hpa.yaml +++ b/helm/litellm/templates/gateway/hpa.yaml @@ -15,6 +15,15 @@ spec: maxReplicas: {{ .Values.gateway.hpa.maxReplicas }} metrics: {{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- if and .Values.gateway.spendWorker.enabled .Values.gateway.spendWorker.scaleOnGatewayContainerCpu }} + - type: ContainerResource + containerResource: + name: cpu + container: gateway + target: + type: Utilization + averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- else }} - type: Resource resource: name: cpu @@ -22,6 +31,7 @@ spec: type: Utilization averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} {{- end }} + {{- end }} {{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }} - type: Resource resource: diff --git a/helm/litellm/tests/spend_worker_tests.yaml b/helm/litellm/tests/spend_worker_tests.yaml new file mode 100644 index 00000000000..5b5e3eea2fa --- /dev/null +++ b/helm/litellm/tests/spend_worker_tests.yaml @@ -0,0 +1,189 @@ +suite: test gateway spend worker sidecar +templates: + - gateway/configmap.yaml + - gateway/deployment.yaml + - gateway/hpa.yaml +values: + - ./values/required.yaml +tests: + - it: adds no sidecar, env, volume or container metric when the spend worker is off + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_SPEND_WORKER_ENABLED + value: "true" + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.volumes + content: + name: spend-worker-socket + any: true + template: gateway/deployment.yaml + - equal: + path: spec.metrics[0].type + value: Resource + template: gateway/hpa.yaml + + - it: runs the spend worker as a sidecar sharing env, config and a unix socket emptyDir, and scales on the gateway container only + set: + gateway.spendWorker.enabled: true + gateway.spendWorker.bufferSize: 250 + gateway.spendWorker.onUnavailable: drop + gateway.image.tag: v1.102.0 + gateway.numWorkers: 4 + gateway.extraEnv: + - name: LITELLM_PGBOUNCER_ENABLED + value: "true" + gateway.envSecrets: + - litellm-license + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_SPEND_WORKER_ADDRESS + value: unix:///var/run/litellm/spend-worker.sock + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_SPEND_WORKER_BUFFER_SIZE + value: "250" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_SPEND_WORKER_ON_UNAVAILABLE + value: drop + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: spend-worker-socket + mountPath: /var/run/litellm + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].name + value: spend-worker + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].image + value: ghcr.io/berriai/litellm-gateway:v1.102.0 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].command + value: + - python + - -m + - gateway.spend_worker + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_JOB_ROLE + value: spend_worker + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: CONFIG_FILE_PATH + value: /app/config/config.yaml + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: DATABASE_HOST + value: postgres.example.com + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_SPEND_WORKER_ADDRESS + value: unix:///var/run/litellm/spend-worker.sock + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.containers[1].env + content: + name: NUM_WORKERS + any: true + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].envFrom + value: + - secretRef: + name: litellm-license + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: spend-worker-socket + mountPath: /var/run/litellm + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].resources.limits.cpu + value: "1" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.volumes + content: + name: spend-worker-socket + emptyDir: + sizeLimit: 1Mi + template: gateway/deployment.yaml + - equal: + path: spec.metrics[0] + value: + type: ContainerResource + containerResource: + name: cpu + container: gateway + target: + type: Utilization + averageUtilization: 70 + template: gateway/hpa.yaml + + - it: uses loopback tcp without a socket volume and keeps the pod-wide cpu metric when asked + set: + gateway.spendWorker.enabled: true + gateway.spendWorker.address: tcp://127.0.0.1:4010 + gateway.spendWorker.scaleOnGatewayContainerCpu: false + asserts: + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_SPEND_WORKER_ADDRESS + value: tcp://127.0.0.1:4010 + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.volumes + content: + name: spend-worker-socket + any: true + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: spend-worker-socket + any: true + template: gateway/deployment.yaml + - equal: + path: spec.metrics[0].type + value: Resource + template: gateway/hpa.yaml diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 5e44aecc286..b0ed47c0982 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -284,6 +284,42 @@ gateway: memory: 128Mi limits: memory: 512Mi + # Opt-in `spend-worker` sidecar (same image, `python -m gateway.spend_worker`) + # that runs the post-response spend pipeline (cost calculation, spend logs, + # spend counters, budget reservation reconciliation) so the uvicorn workers + # only serialise a compact event over loopback and go back to serving + # requests. It shares the pod's env, proxy config, in-container pgbouncer and + # Redis spend buffer, so the per-pod DB connection budget is unchanged. + # Delivery is at-most-once inside the pod: events already handed over are + # lost if the sidecar dies before writing them; events the workers cannot + # hand over follow `onUnavailable`. + spendWorker: + enabled: false + # unix:////.sock (the becomes a shared emptyDir) or + # tcp://127.0.0.1: + address: unix:///var/run/litellm/spend-worker.sock + # Events each uvicorn worker holds in memory while the sidecar is slow or + # restarting. + bufferSize: 1000 + # fallback: run the pipeline in the worker when the sidecar is unreachable + # or the buffer is full (spend stays exact, that request costs gateway CPU + # again). drop: count and discard the event instead (spend under-reports). + onUnavailable: fallback + # How long the workers keep pushing buffered events on shutdown, and how + # long the sidecar keeps serving open connections after SIGTERM. + drainTimeoutSeconds: 10 + # Sized independently of the gateway container; the pipeline is CPU bound. + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + # With hpa.targetCPUUtilizationPercentage set, scale on an autoscaling/v2 + # ContainerResource metric of the `gateway` container only, so the + # sidecar's CPU never drives inference replicas. Needs Kubernetes 1.30+. + scaleOnGatewayContainerCpu: true # Prometheus Operator ServiceMonitor for the gateway pods. Scrapes the # `-metrics` Service, so it requires metricsServer above (the http # port serves /metrics/ behind virtual-key auth). Every pod is its own scrape