scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-12 23:01:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 88

feat(infra): scale gateway on per-pod RPS and TPS in Helm and Terraform (#40479)

Browse source 
* feat(infra): scale gateway on per-pod RPM and TPM in Helm and Terraform

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(helm): require the metrics server before rendering the gateway ServiceMonitor

The http port serves /metrics/ behind virtual-key auth, so a ServiceMonitor
pointed at it only collects 401s and the RPM/TPM HPA metrics never appear

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* feat(infra): express gateway HPA, KEDA and ECS workload targets per second

Rename the per-pod request and token targets in both Helm charts and the
AWS module from per minute to per second, and shorten the recommended
Prometheus rate window to [1m] with no * 60 so the adapter and KEDA
signals are what the HPA compares against. ECS keeps CloudWatch's
60-second aggregation: the ALB target is 60x the per-second variable and
the token metric math divides the period Sum by 60 before dividing by
the running task count.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-09-10 13:36:23 -07:00 • committed by GitHub
parent b0d66a15b8
commit a9cec50960
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 954 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
helm/litellm-helm/templates/hpa.yaml
View file

@ -33,4 +33,22 @@ spec:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- with .Values.autoscaling.targetRequestsPerSecond }}
- type: Pods
pods:
metric:
name: litellm_requests_per_second
target:
type: AverageValue
averageValue: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- with .Values.autoscaling.targetTokensPerSecond }}
- type: Pods
pods:
metric:
name: litellm_tokens_per_second
target:
type: AverageValue
averageValue: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- end }}

21
helm/litellm-helm/templates/keda.yaml
View file

@ -23,6 +23,27 @@ spec:
triggers:
{{- with .Values.keda.triggers }}
{{- toYaml . | nindent 2 }}
{{- end }}
{{- $prom := .Values.keda.prometheus }}
{{- if or $prom.requestsPerSecond $prom.tokensPerSecond }}
{{- if not $prom.serverAddress }}
{{- fail "keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set" }}
{{- end }}
{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s%s" (include "litellm.fullname" .) (ternary "-metrics" "" .Values.metricsServer.enabled)) }}
{{- with $prom.requestsPerSecond }}
- type: prometheus
metadata:
serverAddress: {{ $prom.serverAddress | quote }}
threshold: {{ toJson . | trimAll "\"" | quote }}
query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }}
{{- end }}
{{- with $prom.tokensPerSecond }}
- type: prometheus
metadata:
serverAddress: {{ $prom.serverAddress | quote }}
threshold: {{ toJson . | trimAll "\"" | quote }}
query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }}
{{- end }}
{{- end }}
advanced:
restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }}

78
helm/litellm-helm/tests/hpa_tests.yaml
View file

@ -61,6 +61,84 @@ tests:
- equal: { path: "spec.metrics[1].resource.name", value: memory }
- equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 }
- it: "renders no workload metrics by default"
set:
autoscaling.enabled: true
autoscaling.targetMemoryUtilizationPercentage: 80
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- notContains: { path: spec.metrics, content: { type: Pods }, any: true }
- it: "adds a requests-per-second Pods metric after the cpu metric"
set:
autoscaling.enabled: true
autoscaling.targetRequestsPerSecond: 90
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
- equal:
path: "spec.metrics[1]"
value:
type: Pods
pods:
metric: { name: litellm_requests_per_second }
target: { type: AverageValue, averageValue: "90" }
- it: "adds a tokens-per-second Pods metric on its own"
set:
autoscaling.enabled: true
autoscaling.targetTokensPerSecond: 6M
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- equal:
path: "spec.metrics[1]"
value:
type: Pods
pods:
metric: { name: litellm_tokens_per_second }
target: { type: AverageValue, averageValue: "6M" }
- notContains:
path: spec.metrics
content: { type: Pods, pods: { metric: { name: litellm_requests_per_second } } }
any: true
- it: "renders requests, tokens, cpu and memory metrics together"
set:
autoscaling.enabled: true
autoscaling.targetMemoryUtilizationPercentage: 80
autoscaling.targetRequestsPerSecond: 90
autoscaling.targetTokensPerSecond: 6000000
asserts:
- lengthEqual: { path: spec.metrics, count: 4 }
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
- equal: { path: "spec.metrics[1].resource.name", value: memory }
- equal: { path: "spec.metrics[2].pods.metric.name", value: litellm_requests_per_second }
- equal: { path: "spec.metrics[2].pods.target.averageValue", value: "90" }
- equal: { path: "spec.metrics[3].pods.metric.name", value: litellm_tokens_per_second }
- equal: { path: "spec.metrics[3].pods.target.averageValue", value: "6000000" }
- it: "scales on workload metrics alone when the cpu target is cleared"
set:
autoscaling.enabled: true
autoscaling.targetCPUUtilizationPercentage: null
autoscaling.targetRequestsPerSecond: 90
autoscaling.targetTokensPerSecond: 6000000
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- notContains: { path: spec.metrics, content: { type: Resource }, any: true }
- equal: { path: "spec.metrics[0].pods.metric.name", value: litellm_requests_per_second }
- equal: { path: "spec.metrics[1].pods.metric.name", value: litellm_tokens_per_second }
- notMatchRegexRaw: { pattern: per_minute }
- it: "ignores the per-minute keys, which the chart never shipped"
set:
autoscaling.enabled: true
autoscaling.targetRequestsPerMinute: 5400
autoscaling.targetTokensPerMinute: 360000000
asserts:
- lengthEqual: { path: spec.metrics, count: 1 }
- notContains: { path: spec.metrics, content: { type: Pods }, any: true }
- it: "renders no hpa when autoscaling is disabled"
asserts:
- hasDocuments: { count: 0 }

106
helm/litellm-helm/tests/keda_tests.yaml Normal file
View file

@ -0,0 +1,106 @@
suite: "keda"
templates:
- keda.yaml
release:
name: rel
namespace: llm
tests:
- it: "renders no scaled object by default"
asserts:
- hasDocuments: { count: 0 }
- it: "passes user triggers through and adds no prometheus triggers by default"
set:
keda.enabled: true
keda.triggers:
- type: cpu
metricType: Utilization
metadata: { value: "60" }
asserts:
- isKind: { of: ScaledObject }
- equal:
path: spec.triggers
value:
- type: cpu
metricType: Utilization
metadata: { value: "60" }
- it: "scales on release-wide requests per second divided by the per-replica target"
set:
keda.enabled: true
keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090
keda.prometheus.requestsPerSecond: 90
asserts:
- lengthEqual: { path: spec.triggers, count: 1 }
- equal:
path: "spec.triggers[0]"
value:
type: prometheus
metadata:
serverAddress: http://prometheus-operated.monitoring.svc:9090
threshold: "90"
query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm"}[1m]))
- it: "scales on tokens per second on its own"
set:
keda.enabled: true
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.tokensPerSecond: 6000000
asserts:
- lengthEqual: { path: spec.triggers, count: 1 }
- equal: { path: "spec.triggers[0].type", value: prometheus }
- equal: { path: "spec.triggers[0].metadata.threshold", value: "6000000" }
- equal:
path: "spec.triggers[0].metadata.query"
value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm"}[1m]))
- it: "appends requests and tokens triggers after user triggers and selects the metrics service job"
set:
keda.enabled: true
metricsServer.enabled: true
keda.triggers:
- type: cpu
metricType: Utilization
metadata: { value: "60" }
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.requestsPerSecond: 90
keda.prometheus.tokensPerSecond: 6000000
asserts:
- lengthEqual: { path: spec.triggers, count: 3 }
- equal: { path: "spec.triggers[0].type", value: cpu }
- equal: { path: "spec.triggers[1].metadata.threshold", value: "90" }
- equal:
path: "spec.triggers[1].metadata.query"
value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m]))
- equal: { path: "spec.triggers[2].metadata.threshold", value: "6000000" }
- equal:
path: "spec.triggers[2].metadata.query"
value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m]))
- notMatchRegexRaw: { pattern: "\\* *60|per_minute|PerMinute" }
- it: "ignores the per-minute keys, which the chart never shipped"
set:
keda.enabled: true
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.requestsPerMinute: 5400
keda.prometheus.tokensPerMinute: 360000000
asserts:
- isKind: { of: ScaledObject }
- isNullOrEmpty: { path: spec.triggers }
- it: "refuses a workload target without a prometheus server address"
set:
keda.enabled: true
keda.prometheus.requestsPerSecond: 90
asserts:
- failedTemplate:
errorMessage: keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set
- it: "yields to the hpa when both autoscalers are enabled"
set:
autoscaling.enabled: true
keda.enabled: true
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.requestsPerSecond: 90
asserts:
- hasDocuments: { count: 0 }

36
helm/litellm-helm/values.yaml
View file

@ -222,6 +222,25 @@ autoscaling:
# Memory is a floor to provision under 'resources', not a signal to scale on.
# targetMemoryUtilizationPercentage: 80
# behavior: {}
# Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics
# named `litellm_requests_per_second` and `litellm_tokens_per_second` with an
# AverageValue target, alongside whichever resource targets are set (the HPA
# follows the metric asking for the most replicas). A Prometheus Adapter must
# serve those two names on custom.metrics.k8s.io from the proxy's counters,
# grouped by the scrape target's `pod` label (enable serviceMonitor below so
# every pod is scraped on its own):
# litellm_requests_per_second:
# sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
# litellm_tokens_per_second:
# sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
# rate() over [1m] is already per second, so no `* 60`. How fast the HPA
# reacts is set by that window, the scrape interval and the HPA sync period
# (15s by default), not by the unit: keep serviceMonitor.interval at 15s or
# faster so a 1m window holds at least 4 samples. averageValue takes SI
# suffixes, so "6M" is six million tokens per second per pod. Tokens are
# counted when a response completes, so TPS trails long streams.
targetRequestsPerSecond: ""
targetTokensPerSecond: ""
# Autoscaling with keda is mutually exclusive with hpa
keda:
@ -243,6 +262,23 @@ keda:
# metricName: http_requests_total
# threshold: '100'
# query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
# First-class Prometheus triggers on the proxy's own request and token
# counters, appended to `triggers`. Each target is the per-second load one
# replica should carry: KEDA divides the release-wide
# `sum(rate(<counter>[1m]))` by it to pick the replica count. Thresholds
# are plain numbers (KEDA parses them as floats, no SI suffixes). The
# queries select samples by the release namespace and the `job` label the
# chart's ServiceMonitor produces (the metrics Service name), so enable
# serviceMonitor below together with metricsServer: the http port serves
# /metrics/ behind virtual-key auth and answers an unauthenticated scrape
# with 401. Reaction time comes from the [1m] window, the scrape interval
# and pollingInterval above, so keep both at 15s or faster. Tokens are
# counted at completion, so TPS trails long streams. serverAddress is
# required once either target is set.
prometheus:
serverAddress: ""
requestsPerSecond: ""
tokensPerSecond: ""
behavior: {}
# scaleDown:
# stabilizationWindowSeconds: 300

18
helm/litellm/templates/gateway/hpa.yaml
View file

@ -30,6 +30,24 @@ spec:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- with .Values.gateway.hpa.targetRequestsPerSecond }}
- type: Pods
pods:
metric:
name: litellm_requests_per_second
target:
type: AverageValue
averageValue: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- with .Values.gateway.hpa.targetTokensPerSecond }}
- type: Pods
pods:
metric:
name: litellm_tokens_per_second
target:
type: AverageValue
averageValue: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- with .Values.gateway.hpa.behavior }}
behavior:
{{- toYaml . | nindent 4 }}

28
helm/litellm/templates/gateway/servicemonitor.yaml Normal file
View file

@ -0,0 +1,28 @@
{{- if and .Values.gateway.enabled .Values.gateway.serviceMonitor.enabled }}
{{- if not .Values.gateway.metricsServer.enabled }}
{{- fail "gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled: the http port serves /metrics/ behind virtual-key auth, so an unauthenticated scrape gets 401" }}
{{- end }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
{{- with .Values.gateway.serviceMonitor.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
{{- include "litellm.gateway.selectorLabels" . | nindent 6 }}
namespaceSelector:
matchNames:
- {{ .Release.Namespace | quote }}
endpoints:
- port: metrics
path: /metrics/
interval: {{ .Values.gateway.serviceMonitor.interval }}
scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }}
scheme: http
{{- end }}

213
helm/litellm/tests/hpa_workload_metrics_tests.yaml Normal file
View file

@ -0,0 +1,213 @@
suite: test gateway HPA per-pod requests-per-second and tokens-per-second targets
templates:
- gateway/hpa.yaml
- gateway/servicemonitor.yaml
values:
- ./values/required.yaml
tests:
- it: scales on CPU and memory only by default
template: gateway/hpa.yaml
asserts:
- equal:
path: spec.metrics
value:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 70
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 80
- it: adds a requests-per-second Pods metric next to the resource metrics
template: gateway/hpa.yaml
set:
gateway.hpa.targetRequestsPerSecond: 90
asserts:
- lengthEqual:
path: spec.metrics
count: 3
- contains:
path: spec.metrics
content:
type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 70
- equal:
path: spec.metrics[2]
value:
type: Pods
pods:
metric:
name: litellm_requests_per_second
target:
type: AverageValue
averageValue: "90"
- notContains:
path: spec.metrics
content:
type: Pods
pods:
metric:
name: litellm_tokens_per_second
any: true
- it: adds a tokens-per-second Pods metric on its own
template: gateway/hpa.yaml
set:
gateway.hpa.targetTokensPerSecond: 6M
asserts:
- lengthEqual:
path: spec.metrics
count: 3
- equal:
path: spec.metrics[2]
value:
type: Pods
pods:
metric:
name: litellm_tokens_per_second
target:
type: AverageValue
averageValue: "6M"
- notContains:
path: spec.metrics
content:
type: Pods
pods:
metric:
name: litellm_requests_per_second
any: true
- it: renders requests and tokens targets together and keeps CPU and memory
template: gateway/hpa.yaml
set:
gateway.hpa.targetRequestsPerSecond: 90
gateway.hpa.targetTokensPerSecond: 6000000
asserts:
- lengthEqual:
path: spec.metrics
count: 4
- equal:
path: spec.metrics[0].resource.name
value: cpu
- equal:
path: spec.metrics[1].resource.name
value: memory
- equal:
path: spec.metrics[2].pods.metric.name
value: litellm_requests_per_second
- equal:
path: spec.metrics[3].pods.metric.name
value: litellm_tokens_per_second
- equal:
path: spec.metrics[3].pods.target.averageValue
value: "6000000"
- it: scales on workload metrics alone when the resource targets are cleared
template: gateway/hpa.yaml
set:
gateway.hpa.targetCPUUtilizationPercentage: null
gateway.hpa.targetMemoryUtilizationPercentage: null
gateway.hpa.targetRequestsPerSecond: 90
gateway.hpa.targetTokensPerSecond: 6000000
asserts:
- lengthEqual:
path: spec.metrics
count: 2
- notContains:
path: spec.metrics
content:
type: Resource
any: true
- equal:
path: spec.metrics[0].pods.metric.name
value: litellm_requests_per_second
- equal:
path: spec.metrics[1].pods.metric.name
value: litellm_tokens_per_second
- notMatchRegexRaw:
pattern: per_minute
- it: ignores the per-minute keys, which the chart never shipped
template: gateway/hpa.yaml
set:
gateway.hpa.targetRequestsPerMinute: 5400
gateway.hpa.targetTokensPerMinute: 360000000
asserts:
- lengthEqual:
path: spec.metrics
count: 2
- notContains:
path: spec.metrics
content:
type: Pods
any: true
- it: renders no ServiceMonitor by default
template: gateway/servicemonitor.yaml
asserts:
- hasDocuments:
count: 0
- it: refuses a ServiceMonitor without the metrics server, whose http port needs a bearer token
template: gateway/servicemonitor.yaml
set:
gateway.serviceMonitor.enabled: true
asserts:
- failedTemplate:
errorPattern: gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled
- it: scrapes each gateway pod through the metrics port
template: gateway/servicemonitor.yaml
release:
name: rel
namespace: llm
set:
gateway.serviceMonitor.enabled: true
gateway.metricsServer.enabled: true
gateway.serviceMonitor.labels:
release: kube-prometheus-stack
asserts:
- isKind:
of: ServiceMonitor
- equal:
path: metadata.labels.release
value: kube-prometheus-stack
- equal:
path: spec.selector.matchLabels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: rel
app.kubernetes.io/component: gateway
- equal:
path: spec.namespaceSelector.matchNames
value:
- llm
- equal:
path: spec.endpoints
value:
- port: metrics
path: /metrics/
interval: 15s
scrapeTimeout: 10s
scheme: http
- it: honours a custom scrape interval
template: gateway/servicemonitor.yaml
set:
gateway.serviceMonitor.enabled: true
gateway.serviceMonitor.interval: 30s
gateway.metricsServer.enabled: true
asserts:
- equal:
path: spec.endpoints[0].interval
value: 30s

29
helm/litellm/values.yaml
View file

@ -284,6 +284,16 @@ gateway:
memory: 128Mi
limits:
memory: 512Mi
# Prometheus Operator ServiceMonitor for the gateway pods. Scrapes the
# `<gateway>-metrics` Service, so it requires metricsServer above (the http
# port serves /metrics/ behind virtual-key auth). Every pod is its own scrape
# target, so the samples carry the `pod` label the per-pod autoscaling
# queries below group by.
serviceMonitor:
enabled: false
labels: {}
interval: 15s
scrapeTimeout: 10s
image:
repository: ghcr.io/berriai/litellm-gateway
tag: "" # defaults to .Chart.AppVersion
@ -340,6 +350,25 @@ gateway:
# policies:
# - { type: Percent, value: 100, periodSeconds: 30 }
behavior: {}
# Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics
# named `litellm_requests_per_second` and `litellm_tokens_per_second` with an
# AverageValue target. They coexist with the CPU/memory targets above: the
# HPA scales on whichever metric asks for the most replicas. Kubernetes has
# no idea what a token is, so a Prometheus Adapter must serve those two
# names on custom.metrics.k8s.io from the proxy's counters, grouped by the
# scrape target's `pod` label (enable serviceMonitor above):
# litellm_requests_per_second:
# sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
# litellm_tokens_per_second:
# sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
# rate() over [1m] is already per second, so no `* 60`. How fast the HPA
# reacts is set by that window, the scrape interval and the HPA sync period
# (15s by default), not by the unit: keep serviceMonitor.interval at 15s or
# faster so a 1m window holds at least 4 samples. averageValue takes SI
# suffixes, so "6M" is six million tokens per second per pod. Tokens are
# counted when a response completes, so TPS trails long streams.
targetRequestsPerSecond: ""
targetTokensPerSecond: ""
# PodDisruptionBudget for the gateway pods. Set exactly one of
# `minAvailable` / `maxUnavailable` (minAvailable wins if both are set;
# enabling without either falls back to `maxUnavailable: 1`). Disabled by

55
terraform/litellm/aws/README.md
View file

@ -258,6 +258,61 @@ gateway_metrics_port = 4001
gateway_metrics_scrape_cidrs = ["10.0.0.0/16"]
```
### Scaling the gateway on requests and tokens
By default the gateway service target-tracks CPU (`gateway_cpu_target`) and
memory (`gateway_memory_target`). Two more targets add workload signals next
to them. Application Auto Scaling evaluates every attached policy and follows
the one asking for the most tasks, so the resource policies keep working as a
floor while requests or tokens drive scale-out
Both targets are per task per second, the way load is usually quoted (1k
rps, 75M tok/s). CloudWatch is the limit on how fast they react: target
tracking evaluates every metric, predefined or custom, aggregated over
60-second periods and has no period setting, so ECS reacts on a roughly
one-minute cadence whatever unit the variable is written in. The Kubernetes
charts get a faster signal because the Prometheus `rate()` window and scrape
interval are theirs to shorten
`gateway_target_requests_per_second` adds an `ALBRequestCountPerTarget`
policy on the gateway target group. The ALB publishes that metric as requests
per minute per registered task, so the policy's target value is 60 times the
variable: 90 rps becomes a target of 5,400 per minute. No agent or sidecar is
needed
`gateway_target_tokens_per_second` adds a metric-math policy over a
CloudWatch metric of the gateway's `litellm_total_tokens_metric_total`
counter and the service's `RunningTaskCount` from Container Insights. Nothing
native to ECS carries token throughput, so you publish that metric yourself
with the CloudWatch agent's Prometheus scraper pointed at the metrics sidecar
above. The agent emits the delta of a counter between scrapes, so `Sum` over
the 60-second period is the tokens served in that minute; the expression
divides by 60 (`tokens_per_second`) and then by the task count
(`tokens_per_second_per_task`). Tokens are counted when a response completes,
so long streams show up late in this signal. `gateway_tokens_metric` tells the
policy where the agent publishes: the namespace, the metric name (defaults to
the counter name) and the dimensions from your `metric_declaration`
```hcl
gateway_metrics_port = 4001
gateway_target_requests_per_second = 90
gateway_target_tokens_per_second = 6000000
gateway_tokens_metric = {
namespace = "LiteLLM/Prometheus"
dimensions = { ClusterName = "acme-litellm-prod", TaskDefinitionFamily = "acme-litellm-prod-gateway" }
}
```
Worked example for the request policy: 1,000 rps across 10 tasks is 100 rps
per task (the ALB reports it as 6,000 per minute per target) against a target
of 90 (5,400), so target tracking sizes the service to
`ceil(10 * 100 / 90) = 12` tasks. The token policy does the same arithmetic:
ten tasks handle 4,200,000,000 tokens in a minute, `tokens / 60` is
70,000,000 tokens per second and `tokens_per_second / running_tasks` is
7,000,000 against a target of 6,000,000, so the service grows to
`ceil(10 * 7000000 / 6000000) = 12`. Container Insights must be enabled on the
cluster for `RunningTaskCount` to exist
## Tenant deployment
Every resource the stack creates is named `${tenant}-litellm-${env}` (or

99
terraform/litellm/aws/autoscaling.tf
View file

@ -52,6 +52,105 @@ resource "aws_appautoscaling_policy" "gateway_memory" {
}
}
resource "aws_appautoscaling_policy" "gateway_requests" {
count = var.gateway_autoscaling_enabled && var.gateway_target_requests_per_second > 0 ? 1 : 0
name = "${local.name}-gateway-requests"
policy_type = "TargetTrackingScaling"
service_namespace = aws_appautoscaling_target.gateway[0].service_namespace
resource_id = aws_appautoscaling_target.gateway[0].resource_id
scalable_dimension = aws_appautoscaling_target.gateway[0].scalable_dimension
target_tracking_scaling_policy_configuration {
predefined_metric_specification {
predefined_metric_type = "ALBRequestCountPerTarget"
resource_label = "${aws_lb.this.arn_suffix}/${aws_lb_target_group.gateway.arn_suffix}"
}
# ALBRequestCountPerTarget is a per-minute count
target_value = var.gateway_target_requests_per_second * 60
}
}
resource "aws_appautoscaling_policy" "gateway_tokens" {
count = var.gateway_autoscaling_enabled && var.gateway_target_tokens_per_second > 0 ? 1 : 0
name = "${local.name}-gateway-tokens"
policy_type = "TargetTrackingScaling"
service_namespace = aws_appautoscaling_target.gateway[0].service_namespace
resource_id = aws_appautoscaling_target.gateway[0].resource_id
scalable_dimension = aws_appautoscaling_target.gateway[0].scalable_dimension
lifecycle {
precondition {
condition = var.gateway_tokens_metric != null
error_message = "gateway_tokens_metric is required when gateway_target_tokens_per_second > 0."
}
}
target_tracking_scaling_policy_configuration {
target_value = var.gateway_target_tokens_per_second
# target tracking has no period setting and always aggregates over 60s
customized_metric_specification {
metrics {
id = "tokens"
return_data = false
metric_stat {
stat = "Sum"
metric {
namespace = var.gateway_tokens_metric.namespace
metric_name = var.gateway_tokens_metric.name
dynamic "dimensions" {
for_each = var.gateway_tokens_metric.dimensions
content {
name = dimensions.key
value = dimensions.value
}
}
}
}
}
metrics {
id = "running_tasks"
return_data = false
metric_stat {
stat = "Average"
metric {
namespace = "ECS/ContainerInsights"
metric_name = "RunningTaskCount"
dimensions {
name = "ClusterName"
value = aws_ecs_cluster.this.name
}
dimensions {
name = "ServiceName"
value = aws_ecs_service.gateway.name
}
}
}
}
metrics {
id = "tokens_per_second"
expression = "tokens / 60"
return_data = false
}
metrics {
id = "tokens_per_second_per_task"
expression = "tokens_per_second / running_tasks"
label = "Tokens per second per gateway task"
return_data = true
}
}
}
}
# ---------- Backend ----------
resource "aws_appautoscaling_target" "backend" {
count = var.backend_autoscaling_enabled ? 1 : 0

194
terraform/litellm/aws/tests/workload_autoscaling.tftest.hcl Normal file
View file

@ -0,0 +1,194 @@
# Plan-only coverage for the gateway request and token autoscaling policies.
# Offline via mock_provider, same as byo_infrastructure.tftest.hcl.
mock_provider "aws" {
mock_data "aws_iam_policy_document" {
defaults = {
json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"
}
}
}
mock_provider "random" {}
variables {
region = "us-east-1"
tenant = "acme"
env = "test"
allow_plaintext_alb = true
azs = ["us-east-1a", "us-east-1b"]
}
run "defaults_scale_on_cpu_and_memory_only" {
command = plan
assert {
condition = alltrue([
length(aws_appautoscaling_policy.gateway_cpu) == 1,
length(aws_appautoscaling_policy.gateway_memory) == 1,
length(aws_appautoscaling_policy.gateway_requests) == 0,
length(aws_appautoscaling_policy.gateway_tokens) == 0,
])
error_message = "Request and token policies must be absent by default while the CPU and memory policies stay."
}
}
run "requests_per_second_adds_an_alb_request_count_policy" {
command = plan
variables {
gateway_target_requests_per_second = 90
}
assert {
condition = length(aws_appautoscaling_policy.gateway_requests) == 1 && length(aws_appautoscaling_policy.gateway_tokens) == 0
error_message = "A request target alone must add exactly the request policy."
}
assert {
condition = alltrue([
aws_appautoscaling_policy.gateway_requests[0].name == "acme-litellm-test-gateway-requests",
aws_appautoscaling_policy.gateway_requests[0].policy_type == "TargetTrackingScaling",
aws_appautoscaling_policy.gateway_requests[0].service_namespace == "ecs",
aws_appautoscaling_policy.gateway_requests[0].resource_id == "service/acme-litellm-test/acme-litellm-test-gateway",
aws_appautoscaling_policy.gateway_requests[0].scalable_dimension == "ecs:service:DesiredCount",
])
error_message = "The request policy must be a target-tracking policy on the gateway service's desired count."
}
assert {
condition = alltrue([
one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).target_value == 5400,
one(one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).predefined_metric_specification).predefined_metric_type == "ALBRequestCountPerTarget",
length(one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).customized_metric_specification) == 0,
])
error_message = "The request policy must track ALBRequestCountPerTarget at 60 times the configured requests per second per task."
}
}
run "tokens_per_second_adds_a_metric_math_policy" {
command = plan
variables {
gateway_target_tokens_per_second = 6000000
gateway_tokens_metric = {
namespace = "LiteLLM/Prometheus"
dimensions = { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" }
}
}
assert {
condition = length(aws_appautoscaling_policy.gateway_tokens) == 1 && length(aws_appautoscaling_policy.gateway_requests) == 0
error_message = "A token target alone must add exactly the token policy."
}
assert {
condition = alltrue([
aws_appautoscaling_policy.gateway_tokens[0].name == "acme-litellm-test-gateway-tokens",
aws_appautoscaling_policy.gateway_tokens[0].resource_id == "service/acme-litellm-test/acme-litellm-test-gateway",
one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).target_value == 6000000,
length(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).predefined_metric_specification) == 0,
])
error_message = "The token policy must track a customized metric at the configured tokens per second per task."
}
assert {
condition = alltrue([
length(one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics) == 4,
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].id == "tokens",
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].return_data == false,
one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).stat == "Sum",
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).namespace == "LiteLLM/Prometheus",
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).metric_name == "litellm_total_tokens_metric_total",
{ for d in one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).dimensions : d.name => d.value } == { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" },
])
error_message = "The first metric must sum the published token counter deltas under the configured namespace and dimensions."
}
assert {
condition = alltrue([
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].id == "running_tasks",
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].return_data == false,
one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).stat == "Average",
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).namespace == "ECS/ContainerInsights",
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).metric_name == "RunningTaskCount",
{ for d in one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).dimensions : d.name => d.value } == { ClusterName = "acme-litellm-test", ServiceName = "acme-litellm-test-gateway" },
])
error_message = "The second metric must read the gateway service's Container Insights RunningTaskCount."
}
assert {
condition = alltrue([
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second"].expression == "tokens / 60",
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second"].return_data == false,
])
error_message = "The 60s period Sum must be divided by 60 to yield tokens per second."
}
assert {
condition = alltrue([
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second_per_task"].expression == "tokens_per_second / running_tasks",
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second_per_task"].return_data == true,
length([for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m if m.return_data]) == 1,
])
error_message = "Only the per-task tokens per second may return data to the scaling policy."
}
}
run "tokens_per_second_needs_the_metric_location" {
command = plan
variables {
gateway_target_tokens_per_second = 6000000
}
expect_failures = [
aws_appautoscaling_policy.gateway_tokens,
]
}
run "requests_and_tokens_scale_next_to_cpu_and_memory" {
command = plan
variables {
gateway_target_requests_per_second = 90
gateway_target_tokens_per_second = 6000000
gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" }
}
assert {
condition = alltrue([
length(aws_appautoscaling_policy.gateway_cpu) == 1,
length(aws_appautoscaling_policy.gateway_memory) == 1,
length(aws_appautoscaling_policy.gateway_requests) == 1,
length(aws_appautoscaling_policy.gateway_tokens) == 1,
one(aws_appautoscaling_policy.gateway_cpu[0].target_tracking_scaling_policy_configuration).target_value == 70,
one(aws_appautoscaling_policy.gateway_memory[0].target_tracking_scaling_policy_configuration).target_value == 80,
])
error_message = "Workload policies must coexist with the CPU and memory policies at their default targets."
}
assert {
condition = length(one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).dimensions) == 0
error_message = "Omitting dimensions must query the token metric without any."
}
}
run "workload_targets_are_ignored_when_autoscaling_is_off" {
command = plan
variables {
gateway_autoscaling_enabled = false
gateway_target_requests_per_second = 90
gateway_target_tokens_per_second = 6000000
gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" }
}
assert {
condition = alltrue([
length(aws_appautoscaling_target.gateway) == 0,
length(aws_appautoscaling_policy.gateway_requests) == 0,
length(aws_appautoscaling_policy.gateway_tokens) == 0,
])
error_message = "Disabling gateway autoscaling must drop the workload policies with the target."
}
}

41
terraform/litellm/aws/variables.tf
View file

@ -272,6 +272,47 @@ variable "gateway_memory_target" {
default = 80
}
variable "gateway_target_requests_per_second" {
description = <<-EOT
Requests per second one gateway task should serve. Adds an
ALBRequestCountPerTarget target-tracking policy next to the CPU/memory
ones (Application Auto Scaling follows whichever asks for more tasks).
CloudWatch publishes that metric as a 1-minute count, so the policy
targets 60x this value and ECS reacts on a ~1 minute cadence. 0 skips
the policy.
EOT
type = number
default = 0
}
variable "gateway_target_tokens_per_second" {
description = <<-EOT
Tokens per second one gateway task should serve. Adds a target-tracking
policy on gateway_tokens_metric summed over each 60s period, divided by
60 and by the service's Container Insights RunningTaskCount. Tokens are
counted when a response completes, so the signal trails long streams.
0 skips the policy.
EOT
type = number
default = 0
}
variable "gateway_tokens_metric" {
description = <<-EOT
CloudWatch metric carrying the gateway's litellm_total_tokens_metric_total
counter, as published by the CloudWatch agent's Prometheus scraper (it
emits the delta between scrapes, so Sum over a period is the tokens
served in it). Required when gateway_target_tokens_per_second > 0.
dimensions must match the metric_declaration the agent publishes with.
EOT
type = object({
namespace = string
name = optional(string, "litellm_total_tokens_metric_total")
dimensions = optional(map(string), {})
})
default = null
}
variable "backend_autoscaling_enabled" {
description = "Toggle Application Auto Scaling target-tracking on the backend service."
type = bool

18
terraform/litellm/gcp/README.md
View file

@ -238,6 +238,24 @@ this with `litellm_license`. To tune the export cadence, set
Behavior matches the AWS stack 1:1; the variable names are identical
### Autoscaling
Cloud Run scales the gateway on request concurrency (plus its built-in CPU
target), not on a metric you attach. Each instance takes up to
`gateway_max_instance_request_concurrency` requests at once (default 80)
and Cloud Run adds instances between `gateway_min_instances` and
`gateway_max_instances` when the in-flight count fills up. That is the
request-rate signal for this stack: lower the concurrency for LLM streams
that hold a worker for tens of seconds, since a stream counts as one request
for as long as it is open
There is no tokens-per-second path here. Cloud Run's autoscaler has no
custom-metric input, so the `litellm_total_tokens_metric_total` counter the
proxy exposes cannot drive it. If you need token-based scaling on GCP, run
the gateway on GKE with the Helm chart's `targetTokensPerSecond` (see
"Dependencies only" below) rather than wiring the counter into Cloud
Monitoring, which the autoscaler would ignore
## Tenant deployment
Every resource the stack creates is named `${tenant}-litellm-${env}` (or