feat(helm): opt-in spend-worker sidecar and gateway-container HPA metric in the componentized chart

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

helm/litellm/templates/_helpers.tpl

@@ -443,3 +443,34 @@ ImplementationSpecific
 {{- end -}}
 
 {{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}}
+
+{{/*
+Directory of the spend worker's unix socket, shared by the gateway and
+spend-worker containers through an emptyDir. Empty when the sidecar is off
+or gateway.spendWorker.address is a tcp://127.0.0.1:<port> address.
+*/}}
+{{- define "litellm.gateway.spendWorkerSocketDir" -}}
+{{- if and .Values.gateway.spendWorker.enabled (hasPrefix "unix://" .Values.gateway.spendWorker.address) -}}
+{{- dir (trimPrefix "unix://" .Values.gateway.spendWorker.address) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+LITELLM_SPEND_WORKER_* env shared by the producer (gateway container) and the
+consumer (spend-worker container), so both agree on the transport and the
+shutdown drain window.
+*/}}
+{{- define "litellm.gateway.spendWorkerEnv" -}}
+{{- with .Values.gateway.spendWorker }}
+- name: LITELLM_SPEND_WORKER_ENABLED
+  value: "true"
+- name: LITELLM_SPEND_WORKER_ADDRESS
+  value: {{ .address | quote }}
+- name: LITELLM_SPEND_WORKER_BUFFER_SIZE
+  value: {{ .bufferSize | quote }}
+- name: LITELLM_SPEND_WORKER_ON_UNAVAILABLE
+  value: {{ .onUnavailable | quote }}
+- name: LITELLM_SPEND_WORKER_DRAIN_TIMEOUT_SECONDS
+  value: {{ .drainTimeoutSeconds | quote }}
+{{- end }}
+{{- end -}}

helm/litellm/templates/gateway/deployment.yaml

@@ -71,8 +71,11 @@ spec:
             - name: PROMETHEUS_MULTIPROC_DIR
               value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
             {{- end }}
+            {{- if .Values.gateway.spendWorker.enabled }}
+            {{- include "litellm.gateway.spendWorkerEnv" . | nindent 12 }}
+            {{- end }}
           {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }}
+          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.spendWorkerSocketDir" .) }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
@@ -83,6 +86,10 @@ spec:
             - name: prometheus-multiproc
               mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
             {{- end }}
+            {{- if include "litellm.gateway.spendWorkerSocketDir" . }}
+            - name: spend-worker-socket
+              mountPath: {{ include "litellm.gateway.spendWorkerSocketDir" . }}
+            {{- end }}
             {{- if .Values.billingMetrics.enabled }}
             {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
             {{- end }}
@@ -142,10 +149,47 @@ spec:
           resources:
             {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }}
         {{- end }}
+        {{- if .Values.gateway.spendWorker.enabled }}
+        - name: spend-worker
+          image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
+          {{- with .Values.gateway.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          command:
+            - python
+            - -m
+            - gateway.spend_worker
+          env:
+            {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
+            {{- if .Values.gateway.config.create }}
+            - name: CONFIG_FILE_PATH
+              value: /app/config/config.yaml
+            {{- end }}
+            {{- include "litellm.gateway.spendWorkerEnv" . | nindent 12 }}
+            - name: LITELLM_JOB_ROLE
+              value: spend_worker
+          {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
+          {{- if or .Values.gateway.config.create (include "litellm.gateway.spendWorkerSocketDir" .) }}
+          volumeMounts:
+            {{- if .Values.gateway.config.create }}
+            - name: gateway-config
+              mountPath: /app/config/config.yaml
+              subPath: config.yaml
+            {{- end }}
+            {{- if include "litellm.gateway.spendWorkerSocketDir" . }}
+            - name: spend-worker-socket
+              mountPath: {{ include "litellm.gateway.spendWorkerSocketDir" . }}
+            {{- end }}
+          {{- end }}
+          resources:
+            {{- toYaml .Values.gateway.spendWorker.resources | nindent 12 }}
+        {{- end }}
       {{- with .Values.gateway.extraContainers }}
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
-      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }}
+      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.spendWorkerSocketDir" .) }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
@@ -156,6 +200,11 @@ spec:
         - name: prometheus-multiproc
           emptyDir: {}
         {{- end }}
+        {{- if include "litellm.gateway.spendWorkerSocketDir" . }}
+        - name: spend-worker-socket
+          emptyDir:
+            sizeLimit: 1Mi
+        {{- end }}
         {{- if .Values.billingMetrics.enabled }}
         {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
         {{- end }}

helm/litellm/templates/gateway/hpa.yaml

@@ -15,6 +15,15 @@ spec:
   maxReplicas: {{ .Values.gateway.hpa.maxReplicas }}
   metrics:
     {{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }}
+    {{- if and .Values.gateway.spendWorker.enabled .Values.gateway.spendWorker.scaleOnGatewayContainerCpu }}
+    - type: ContainerResource
+      containerResource:
+        name: cpu
+        container: gateway
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
+    {{- else }}
     - type: Resource
       resource:
         name: cpu
@@ -22,6 +31,7 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
     {{- end }}
+    {{- end }}
     {{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
     - type: Resource
       resource:

helm/litellm/tests/spend_worker_tests.yaml

@@ -0,0 +1,189 @@
+suite: test gateway spend worker sidecar
+templates:
+  - gateway/configmap.yaml
+  - gateway/deployment.yaml
+  - gateway/hpa.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: adds no sidecar, env, volume or container metric when the spend worker is off
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.containers
+          count: 1
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_SPEND_WORKER_ENABLED
+            value: "true"
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: spend-worker-socket
+          any: true
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.metrics[0].type
+          value: Resource
+        template: gateway/hpa.yaml
+
+  - it: runs the spend worker as a sidecar sharing env, config and a unix socket emptyDir, and scales on the gateway container only
+    set:
+      gateway.spendWorker.enabled: true
+      gateway.spendWorker.bufferSize: 250
+      gateway.spendWorker.onUnavailable: drop
+      gateway.image.tag: v1.102.0
+      gateway.numWorkers: 4
+      gateway.extraEnv:
+        - name: LITELLM_PGBOUNCER_ENABLED
+          value: "true"
+      gateway.envSecrets:
+        - litellm-license
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_SPEND_WORKER_ADDRESS
+            value: unix:///var/run/litellm/spend-worker.sock
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_SPEND_WORKER_BUFFER_SIZE
+            value: "250"
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_SPEND_WORKER_ON_UNAVAILABLE
+            value: drop
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: spend-worker-socket
+            mountPath: /var/run/litellm
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: spend-worker
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].image
+          value: ghcr.io/berriai/litellm-gateway:v1.102.0
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].command
+          value:
+            - python
+            - -m
+            - gateway.spend_worker
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_JOB_ROLE
+            value: spend_worker
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: CONFIG_FILE_PATH
+            value: /app/config/config.yaml
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: DATABASE_HOST
+            value: postgres.example.com
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_SPEND_WORKER_ADDRESS
+            value: unix:///var/run/litellm/spend-worker.sock
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: NUM_WORKERS
+          any: true
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].envFrom
+          value:
+            - secretRef:
+                name: litellm-license
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: gateway-config
+            mountPath: /app/config/config.yaml
+            subPath: config.yaml
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: spend-worker-socket
+            mountPath: /var/run/litellm
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].resources.limits.cpu
+          value: "1"
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: spend-worker-socket
+            emptyDir:
+              sizeLimit: 1Mi
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.metrics[0]
+          value:
+            type: ContainerResource
+            containerResource:
+              name: cpu
+              container: gateway
+              target:
+                type: Utilization
+                averageUtilization: 70
+        template: gateway/hpa.yaml
+
+  - it: uses loopback tcp without a socket volume and keeps the pod-wide cpu metric when asked
+    set:
+      gateway.spendWorker.enabled: true
+      gateway.spendWorker.address: tcp://127.0.0.1:4010
+      gateway.spendWorker.scaleOnGatewayContainerCpu: false
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_SPEND_WORKER_ADDRESS
+            value: tcp://127.0.0.1:4010
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: spend-worker-socket
+          any: true
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: spend-worker-socket
+          any: true
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.metrics[0].type
+          value: Resource
+        template: gateway/hpa.yaml

helm/litellm/values.yaml

@@ -284,6 +284,42 @@ gateway:
         memory: 128Mi
       limits:
         memory: 512Mi
+  # Opt-in `spend-worker` sidecar (same image, `python -m gateway.spend_worker`)
+  # that runs the post-response spend pipeline (cost calculation, spend logs,
+  # spend counters, budget reservation reconciliation) so the uvicorn workers
+  # only serialise a compact event over loopback and go back to serving
+  # requests. It shares the pod's env, proxy config, in-container pgbouncer and
+  # Redis spend buffer, so the per-pod DB connection budget is unchanged.
+  # Delivery is at-most-once inside the pod: events already handed over are
+  # lost if the sidecar dies before writing them; events the workers cannot
+  # hand over follow `onUnavailable`.
+  spendWorker:
+    enabled: false
+    # unix:///<dir>/<file>.sock (the <dir> becomes a shared emptyDir) or
+    # tcp://127.0.0.1:<port>
+    address: unix:///var/run/litellm/spend-worker.sock
+    # Events each uvicorn worker holds in memory while the sidecar is slow or
+    # restarting.
+    bufferSize: 1000
+    # fallback: run the pipeline in the worker when the sidecar is unreachable
+    # or the buffer is full (spend stays exact, that request costs gateway CPU
+    # again). drop: count and discard the event instead (spend under-reports).
+    onUnavailable: fallback
+    # How long the workers keep pushing buffered events on shutdown, and how
+    # long the sidecar keeps serving open connections after SIGTERM.
+    drainTimeoutSeconds: 10
+    # Sized independently of the gateway container; the pipeline is CPU bound.
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: "1"
+        memory: 2Gi
+    # With hpa.targetCPUUtilizationPercentage set, scale on an autoscaling/v2
+    # ContainerResource metric of the `gateway` container only, so the
+    # sidecar's CPU never drives inference replicas. Needs Kubernetes 1.30+.
+    scaleOnGatewayContainerCpu: true
   # Prometheus Operator ServiceMonitor for the gateway pods. Scrapes the
   # `<gateway>-metrics` Service, so it requires metricsServer above (the http
   # port serves /metrics/ behind virtual-key auth). Every pod is its own scrape