scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-12 23:01:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 88

feat(infra): scale gateway on per-pod RPM and TPM in Helm and Terraform

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-09 22:35:41 +00:00
parent 264fc82dc1
commit fdf5d99f9d
14 changed files with 853 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
helm/litellm-helm/templates/hpa.yaml
View file

@ -33,4 +33,22 @@ spec:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- with .Values.autoscaling.targetRequestsPerMinute }}
- type: Pods
pods:
metric:
name: litellm_requests_per_minute
target:
type: AverageValue
averageValue: {{ . | quote }}
{{- end }}
{{- with .Values.autoscaling.targetTokensPerMinute }}
- type: Pods
pods:
metric:
name: litellm_tokens_per_minute
target:
type: AverageValue
averageValue: {{ . | quote }}
{{- end }}
{{- end }}

21
helm/litellm-helm/templates/keda.yaml
View file

@ -23,6 +23,27 @@ spec:
triggers:
{{- with .Values.keda.triggers }}
{{- toYaml . | nindent 2 }}
{{- end }}
{{- $prom := .Values.keda.prometheus }}
{{- if or $prom.requestsPerMinute $prom.tokensPerMinute }}
{{- if not $prom.serverAddress }}
{{- fail "keda.prometheus.serverAddress is required when keda.prometheus.requestsPerMinute or tokensPerMinute is set" }}
{{- end }}
{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s%s" (include "litellm.fullname" .) (ternary "-metrics" "" .Values.metricsServer.enabled)) }}
{{- with $prom.requestsPerMinute }}
- type: prometheus
metadata:
serverAddress: {{ $prom.serverAddress | quote }}
threshold: {{ . | quote }}
query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m])) * 60" $selector | quote }}
{{- end }}
{{- with $prom.tokensPerMinute }}
- type: prometheus
metadata:
serverAddress: {{ $prom.serverAddress | quote }}
threshold: {{ . | quote }}
query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m])) * 60" $selector | quote }}
{{- end }}
{{- end }}
advanced:
restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }}

68
helm/litellm-helm/tests/hpa_tests.yaml
View file

@ -61,6 +61,74 @@ tests:
- equal: { path: "spec.metrics[1].resource.name", value: memory }
- equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 }
- it: "renders no workload metrics by default"
set:
autoscaling.enabled: true
autoscaling.targetMemoryUtilizationPercentage: 80
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- notContains: { path: spec.metrics, content: { type: Pods }, any: true }
- it: "adds a requests-per-minute Pods metric after the cpu metric"
set:
autoscaling.enabled: true
autoscaling.targetRequestsPerMinute: 600
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
- equal:
path: "spec.metrics[1]"
value:
type: Pods
pods:
metric: { name: litellm_requests_per_minute }
target: { type: AverageValue, averageValue: "600" }
- it: "adds a tokens-per-minute Pods metric on its own"
set:
autoscaling.enabled: true
autoscaling.targetTokensPerMinute: 400k
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- equal:
path: "spec.metrics[1]"
value:
type: Pods
pods:
metric: { name: litellm_tokens_per_minute }
target: { type: AverageValue, averageValue: "400k" }
- notContains:
path: spec.metrics
content: { type: Pods, pods: { metric: { name: litellm_requests_per_minute } } }
any: true
- it: "renders requests, tokens, cpu and memory metrics together"
set:
autoscaling.enabled: true
autoscaling.targetMemoryUtilizationPercentage: 80
autoscaling.targetRequestsPerMinute: 600
autoscaling.targetTokensPerMinute: 400000
asserts:
- lengthEqual: { path: spec.metrics, count: 4 }
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
- equal: { path: "spec.metrics[1].resource.name", value: memory }
- equal: { path: "spec.metrics[2].pods.metric.name", value: litellm_requests_per_minute }
- equal: { path: "spec.metrics[2].pods.target.averageValue", value: "600" }
- equal: { path: "spec.metrics[3].pods.metric.name", value: litellm_tokens_per_minute }
- equal: { path: "spec.metrics[3].pods.target.averageValue", value: "400000" }
- it: "scales on workload metrics alone when the cpu target is cleared"
set:
autoscaling.enabled: true
autoscaling.targetCPUUtilizationPercentage: null
autoscaling.targetRequestsPerMinute: 600
autoscaling.targetTokensPerMinute: 400000
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- notContains: { path: spec.metrics, content: { type: Resource }, any: true }
- equal: { path: "spec.metrics[0].pods.metric.name", value: litellm_requests_per_minute }
- equal: { path: "spec.metrics[1].pods.metric.name", value: litellm_tokens_per_minute }
- it: "renders no hpa when autoscaling is disabled"
asserts:
- hasDocuments: { count: 0 }

95
helm/litellm-helm/tests/keda_tests.yaml Normal file
View file

@ -0,0 +1,95 @@
suite: "keda"
templates:
- keda.yaml
release:
name: rel
namespace: llm
tests:
- it: "renders no scaled object by default"
asserts:
- hasDocuments: { count: 0 }
- it: "passes user triggers through and adds no prometheus triggers by default"
set:
keda.enabled: true
keda.triggers:
- type: cpu
metricType: Utilization
metadata: { value: "60" }
asserts:
- isKind: { of: ScaledObject }
- equal:
path: spec.triggers
value:
- type: cpu
metricType: Utilization
metadata: { value: "60" }
- it: "scales on release-wide requests per minute divided by the per-replica target"
set:
keda.enabled: true
keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090
keda.prometheus.requestsPerMinute: 600
asserts:
- lengthEqual: { path: spec.triggers, count: 1 }
- equal:
path: "spec.triggers[0]"
value:
type: prometheus
metadata:
serverAddress: http://prometheus-operated.monitoring.svc:9090
threshold: "600"
query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm"}[1m])) * 60
- it: "scales on tokens per minute on its own"
set:
keda.enabled: true
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.tokensPerMinute: 400000
asserts:
- lengthEqual: { path: spec.triggers, count: 1 }
- equal: { path: "spec.triggers[0].type", value: prometheus }
- equal: { path: "spec.triggers[0].metadata.threshold", value: "400000" }
- equal:
path: "spec.triggers[0].metadata.query"
value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm"}[1m])) * 60
- it: "appends requests and tokens triggers after user triggers and selects the metrics service job"
set:
keda.enabled: true
metricsServer.enabled: true
keda.triggers:
- type: cpu
metricType: Utilization
metadata: { value: "60" }
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.requestsPerMinute: 600
keda.prometheus.tokensPerMinute: 400000
asserts:
- lengthEqual: { path: spec.triggers, count: 3 }
- equal: { path: "spec.triggers[0].type", value: cpu }
- equal: { path: "spec.triggers[1].metadata.threshold", value: "600" }
- equal:
path: "spec.triggers[1].metadata.query"
value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) * 60
- equal: { path: "spec.triggers[2].metadata.threshold", value: "400000" }
- equal:
path: "spec.triggers[2].metadata.query"
value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) * 60
- it: "refuses a workload target without a prometheus server address"
set:
keda.enabled: true
keda.prometheus.requestsPerMinute: 600
asserts:
- failedTemplate:
errorMessage: keda.prometheus.serverAddress is required when keda.prometheus.requestsPerMinute or tokensPerMinute is set
- it: "yields to the hpa when both autoscalers are enabled"
set:
autoscaling.enabled: true
keda.enabled: true
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.requestsPerMinute: 600
asserts:
- hasDocuments: { count: 0 }

23
helm/litellm-helm/values.yaml
View file

@ -222,6 +222,18 @@ autoscaling:
# Memory is a floor to provision under 'resources', not a signal to scale on.
# targetMemoryUtilizationPercentage: 80
# behavior: {}
# Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics
# named `litellm_requests_per_minute` and `litellm_tokens_per_minute` with an
# AverageValue target, alongside whichever resource targets are set (the HPA
# follows the metric asking for the most replicas). A Prometheus Adapter must
# serve those two names on custom.metrics.k8s.io from the proxy's request and
# token counters, grouped by the scrape target's `pod` label; enable
# serviceMonitor below so every pod is scraped on its own. Adapter rules and
# the replica arithmetic:
# https://docs.litellm.ai/docs/proxy/deploy#scale-on-requests-and-tokens-per-pod
# Tokens are counted when a response completes, so TPM trails long streams.
targetRequestsPerMinute: ""
targetTokensPerMinute: ""
# Autoscaling with keda is mutually exclusive with hpa
keda:
@ -243,6 +255,17 @@ keda:
# metricName: http_requests_total
# threshold: '100'
# query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
# First-class Prometheus triggers on the proxy's own request and token
# counters, appended to `triggers`. Each target is the load one replica should
# carry: KEDA divides the release-wide rate by it to pick the replica count.
# The queries select samples by the release namespace and the `job` label the
# chart's ServiceMonitor produces (the metrics Service name), so enable
# serviceMonitor below. Tokens are counted at completion, so TPM trails long
# streams. serverAddress is required once either target is set.
prometheus:
serverAddress: ""
requestsPerMinute: ""
tokensPerMinute: ""
behavior: {}
# scaleDown:
# stabilizationWindowSeconds: 300

18
helm/litellm/templates/gateway/hpa.yaml
View file

@ -30,6 +30,24 @@ spec:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- with .Values.gateway.hpa.targetRequestsPerMinute }}
- type: Pods
pods:
metric:
name: litellm_requests_per_minute
target:
type: AverageValue
averageValue: {{ . | quote }}
{{- end }}
{{- with .Values.gateway.hpa.targetTokensPerMinute }}
- type: Pods
pods:
metric:
name: litellm_tokens_per_minute
target:
type: AverageValue
averageValue: {{ . | quote }}
{{- end }}
{{- with .Values.gateway.hpa.behavior }}
behavior:
{{- toYaml . | nindent 4 }}

25
helm/litellm/templates/gateway/servicemonitor.yaml Normal file
View file

@ -0,0 +1,25 @@
{{- if and .Values.gateway.enabled .Values.gateway.serviceMonitor.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
{{- with .Values.gateway.serviceMonitor.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
{{- include "litellm.gateway.selectorLabels" . | nindent 6 }}
namespaceSelector:
matchNames:
- {{ .Release.Namespace | quote }}
endpoints:
- port: {{ ternary "metrics" "http" .Values.gateway.metricsServer.enabled }}
path: /metrics/
interval: {{ .Values.gateway.serviceMonitor.interval }}
scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }}
scheme: http
{{- end }}

190
helm/litellm/tests/hpa_workload_metrics_tests.yaml Normal file
View file

@ -0,0 +1,190 @@
suite: test gateway HPA per-pod requests and tokens targets
templates:
- gateway/hpa.yaml
- gateway/servicemonitor.yaml
values:
- ./values/required.yaml
tests:
- it: scales on CPU and memory only by default
template: gateway/hpa.yaml
asserts:
- equal:
path: spec.metrics
value:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 70
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 80
- it: adds a requests-per-minute Pods metric next to the resource metrics
template: gateway/hpa.yaml
set:
gateway.hpa.targetRequestsPerMinute: 600
asserts:
- lengthEqual:
path: spec.metrics
count: 3
- contains:
path: spec.metrics
content:
type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 70
- equal:
path: spec.metrics[2]
value:
type: Pods
pods:
metric:
name: litellm_requests_per_minute
target:
type: AverageValue
averageValue: "600"
- notContains:
path: spec.metrics
content:
type: Pods
pods:
metric:
name: litellm_tokens_per_minute
any: true
- it: adds a tokens-per-minute Pods metric on its own
template: gateway/hpa.yaml
set:
gateway.hpa.targetTokensPerMinute: 400k
asserts:
- lengthEqual:
path: spec.metrics
count: 3
- equal:
path: spec.metrics[2]
value:
type: Pods
pods:
metric:
name: litellm_tokens_per_minute
target:
type: AverageValue
averageValue: "400k"
- notContains:
path: spec.metrics
content:
type: Pods
pods:
metric:
name: litellm_requests_per_minute
any: true
- it: renders requests and tokens targets together and keeps CPU and memory
template: gateway/hpa.yaml
set:
gateway.hpa.targetRequestsPerMinute: 600
gateway.hpa.targetTokensPerMinute: 400000
asserts:
- lengthEqual:
path: spec.metrics
count: 4
- equal:
path: spec.metrics[0].resource.name
value: cpu
- equal:
path: spec.metrics[1].resource.name
value: memory
- equal:
path: spec.metrics[2].pods.metric.name
value: litellm_requests_per_minute
- equal:
path: spec.metrics[3].pods.metric.name
value: litellm_tokens_per_minute
- equal:
path: spec.metrics[3].pods.target.averageValue
value: "400000"
- it: scales on workload metrics alone when the resource targets are cleared
template: gateway/hpa.yaml
set:
gateway.hpa.targetCPUUtilizationPercentage: null
gateway.hpa.targetMemoryUtilizationPercentage: null
gateway.hpa.targetRequestsPerMinute: 600
gateway.hpa.targetTokensPerMinute: 400000
asserts:
- lengthEqual:
path: spec.metrics
count: 2
- notContains:
path: spec.metrics
content:
type: Resource
any: true
- equal:
path: spec.metrics[0].pods.metric.name
value: litellm_requests_per_minute
- equal:
path: spec.metrics[1].pods.metric.name
value: litellm_tokens_per_minute
- it: renders no ServiceMonitor by default
template: gateway/servicemonitor.yaml
asserts:
- hasDocuments:
count: 0
- it: scrapes each gateway pod through the http port when the metrics sidecar is off
template: gateway/servicemonitor.yaml
release:
name: rel
namespace: llm
set:
gateway.serviceMonitor.enabled: true
gateway.serviceMonitor.labels:
release: kube-prometheus-stack
asserts:
- isKind:
of: ServiceMonitor
- equal:
path: metadata.labels.release
value: kube-prometheus-stack
- equal:
path: spec.selector.matchLabels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: rel
app.kubernetes.io/component: gateway
- equal:
path: spec.namespaceSelector.matchNames
value:
- llm
- equal:
path: spec.endpoints
value:
- port: http
path: /metrics/
interval: 15s
scrapeTimeout: 10s
scheme: http
- it: scrapes the metrics sidecar port when the metrics server is on
template: gateway/servicemonitor.yaml
set:
gateway.serviceMonitor.enabled: true
gateway.serviceMonitor.interval: 30s
gateway.metricsServer.enabled: true
asserts:
- equal:
path: spec.endpoints[0].port
value: metrics
- equal:
path: spec.endpoints[0].interval
value: 30s

21
helm/litellm/values.yaml
View file

@ -284,6 +284,16 @@ gateway:
memory: 128Mi
limits:
memory: 512Mi
# Prometheus Operator ServiceMonitor for the gateway pods. Scrapes the
# `<gateway>-metrics` Service when metricsServer is enabled, otherwise the
# gateway Service on the http port. Either way every pod is its own scrape
# target, so the samples carry the `pod` label the per-pod autoscaling
# queries below group by.
serviceMonitor:
enabled: false
labels: {}
interval: 15s
scrapeTimeout: 10s
image:
repository: ghcr.io/berriai/litellm-gateway
tag: "" # defaults to .Chart.AppVersion
@ -340,6 +350,17 @@ gateway:
# policies:
# - { type: Percent, value: 100, periodSeconds: 30 }
behavior: {}
# Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics
# named `litellm_requests_per_minute` and `litellm_tokens_per_minute` with an
# AverageValue target. They coexist with the CPU/memory targets above: the
# HPA scales on whichever metric asks for the most replicas. Kubernetes has
# no idea what a token is, so a Prometheus Adapter must serve those two
# names on custom.metrics.k8s.io from the proxy's request and token
# counters; the adapter rules and the replica arithmetic are documented at
# https://docs.litellm.ai/docs/proxy/deploy#scale-on-requests-and-tokens-per-pod
# Tokens are counted when a response completes, so TPM trails long streams.
targetRequestsPerMinute: ""
targetTokensPerMinute: ""
# PodDisruptionBudget for the gateway pods. Set exactly one of
# `minAvailable` / `maxUnavailable` (minAvailable wins if both are set;
# enabling without either falls back to `maxUnavailable: 1`). Disabled by

41
terraform/litellm/aws/README.md
View file

@ -258,6 +258,47 @@ gateway_metrics_port = 4001
gateway_metrics_scrape_cidrs = ["10.0.0.0/16"]
```
### Scaling the gateway on requests and tokens
By default the gateway service target-tracks CPU (`gateway_cpu_target`) and
memory (`gateway_memory_target`). Two more targets add workload signals next
to them. Application Auto Scaling evaluates every attached policy and follows
the one asking for the most tasks, so the resource policies keep working as a
floor while requests or tokens drive scale-out
`gateway_requests_per_target` adds an `ALBRequestCountPerTarget` policy on
the gateway target group. The ALB counts requests per minute per registered
task, so a value of 600 means "keep each task at about 10 requests per
second". No agent or sidecar is needed
`gateway_tokens_per_target` adds a metric-math policy that divides a
CloudWatch metric of the gateway's `litellm_total_tokens_metric_total`
counter by the service's `RunningTaskCount` from Container Insights. Nothing
native to ECS carries token throughput, so you publish that metric yourself
with the CloudWatch agent's Prometheus scraper pointed at the metrics sidecar
above. The agent emits the delta of a counter between scrapes, so `Sum` over a
minute is tokens per minute. Tokens are counted when a response completes, so
long streams show up late in this signal. `gateway_tokens_metric` tells the
policy where the agent publishes: the namespace, the metric name (defaults to
the counter name) and the dimensions from your `metric_declaration`
```hcl
gateway_metrics_port = 4001
gateway_requests_per_target = 600
gateway_tokens_per_target = 400000
gateway_tokens_metric = {
namespace = "LiteLLM/Prometheus"
dimensions = { ClusterName = "acme-litellm-prod", TaskDefinitionFamily = "acme-litellm-prod-gateway" }
}
```
Worked example for the token policy: three tasks handle 1,800,000 tokens in a
minute, so `tokens_per_minute / running_tasks` is 600,000 against a target of
400,000. Target tracking sizes the service to `ceil(3 * 600000 / 400000) = 5`
tasks. The request policy does the same arithmetic with the ALB's per-target
count. Container Insights must be enabled on the cluster for `RunningTaskCount`
to exist
## Tenant deployment
Every resource the stack creates is named `${tenant}-litellm-${env}` (or

91
terraform/litellm/aws/autoscaling.tf
View file

@ -52,6 +52,97 @@ resource "aws_appautoscaling_policy" "gateway_memory" {
}
}
resource "aws_appautoscaling_policy" "gateway_requests" {
count = var.gateway_autoscaling_enabled && var.gateway_requests_per_target > 0 ? 1 : 0
name = "${local.name}-gateway-requests"
policy_type = "TargetTrackingScaling"
service_namespace = aws_appautoscaling_target.gateway[0].service_namespace
resource_id = aws_appautoscaling_target.gateway[0].resource_id
scalable_dimension = aws_appautoscaling_target.gateway[0].scalable_dimension
target_tracking_scaling_policy_configuration {
predefined_metric_specification {
predefined_metric_type = "ALBRequestCountPerTarget"
resource_label = "${aws_lb.this.arn_suffix}/${aws_lb_target_group.gateway.arn_suffix}"
}
target_value = var.gateway_requests_per_target
}
}
resource "aws_appautoscaling_policy" "gateway_tokens" {
count = var.gateway_autoscaling_enabled && var.gateway_tokens_per_target > 0 ? 1 : 0
name = "${local.name}-gateway-tokens"
policy_type = "TargetTrackingScaling"
service_namespace = aws_appautoscaling_target.gateway[0].service_namespace
resource_id = aws_appautoscaling_target.gateway[0].resource_id
scalable_dimension = aws_appautoscaling_target.gateway[0].scalable_dimension
lifecycle {
precondition {
condition = var.gateway_tokens_metric != null
error_message = "gateway_tokens_metric is required when gateway_tokens_per_target > 0."
}
}
target_tracking_scaling_policy_configuration {
target_value = var.gateway_tokens_per_target
customized_metric_specification {
metrics {
id = "tokens_per_minute"
return_data = false
metric_stat {
stat = "Sum"
metric {
namespace = var.gateway_tokens_metric.namespace
metric_name = var.gateway_tokens_metric.name
dynamic "dimensions" {
for_each = var.gateway_tokens_metric.dimensions
content {
name = dimensions.key
value = dimensions.value
}
}
}
}
}
metrics {
id = "running_tasks"
return_data = false
metric_stat {
stat = "Average"
metric {
namespace = "ECS/ContainerInsights"
metric_name = "RunningTaskCount"
dimensions {
name = "ClusterName"
value = aws_ecs_cluster.this.name
}
dimensions {
name = "ServiceName"
value = aws_ecs_service.gateway.name
}
}
}
}
metrics {
id = "tokens_per_minute_per_task"
expression = "tokens_per_minute / running_tasks"
label = "Tokens per minute per gateway task"
return_data = true
}
}
}
}
# ---------- Backend ----------
resource "aws_appautoscaling_target" "backend" {
count = var.backend_autoscaling_enabled ? 1 : 0

186
terraform/litellm/aws/tests/workload_autoscaling.tftest.hcl Normal file
View file

@ -0,0 +1,186 @@
# Plan-only coverage for the gateway request and token autoscaling policies.
# Offline via mock_provider, same as byo_infrastructure.tftest.hcl.
mock_provider "aws" {
mock_data "aws_iam_policy_document" {
defaults = {
json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"
}
}
}
mock_provider "random" {}
variables {
region = "us-east-1"
tenant = "acme"
env = "test"
allow_plaintext_alb = true
azs = ["us-east-1a", "us-east-1b"]
}
run "defaults_scale_on_cpu_and_memory_only" {
command = plan
assert {
condition = alltrue([
length(aws_appautoscaling_policy.gateway_cpu) == 1,
length(aws_appautoscaling_policy.gateway_memory) == 1,
length(aws_appautoscaling_policy.gateway_requests) == 0,
length(aws_appautoscaling_policy.gateway_tokens) == 0,
])
error_message = "Request and token policies must be absent by default while the CPU and memory policies stay."
}
}
run "requests_per_target_adds_an_alb_request_count_policy" {
command = plan
variables {
gateway_requests_per_target = 600
}
assert {
condition = length(aws_appautoscaling_policy.gateway_requests) == 1 && length(aws_appautoscaling_policy.gateway_tokens) == 0
error_message = "A request target alone must add exactly the request policy."
}
assert {
condition = alltrue([
aws_appautoscaling_policy.gateway_requests[0].name == "acme-litellm-test-gateway-requests",
aws_appautoscaling_policy.gateway_requests[0].policy_type == "TargetTrackingScaling",
aws_appautoscaling_policy.gateway_requests[0].service_namespace == "ecs",
aws_appautoscaling_policy.gateway_requests[0].resource_id == "service/acme-litellm-test/acme-litellm-test-gateway",
aws_appautoscaling_policy.gateway_requests[0].scalable_dimension == "ecs:service:DesiredCount",
])
error_message = "The request policy must be a target-tracking policy on the gateway service's desired count."
}
assert {
condition = alltrue([
one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).target_value == 600,
one(one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).predefined_metric_specification).predefined_metric_type == "ALBRequestCountPerTarget",
length(one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).customized_metric_specification) == 0,
])
error_message = "The request policy must track ALBRequestCountPerTarget at the configured requests per minute per task."
}
}
run "tokens_per_target_adds_a_metric_math_policy" {
command = plan
variables {
gateway_tokens_per_target = 400000
gateway_tokens_metric = {
namespace = "LiteLLM/Prometheus"
dimensions = { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" }
}
}
assert {
condition = length(aws_appautoscaling_policy.gateway_tokens) == 1 && length(aws_appautoscaling_policy.gateway_requests) == 0
error_message = "A token target alone must add exactly the token policy."
}
assert {
condition = alltrue([
aws_appautoscaling_policy.gateway_tokens[0].name == "acme-litellm-test-gateway-tokens",
aws_appautoscaling_policy.gateway_tokens[0].resource_id == "service/acme-litellm-test/acme-litellm-test-gateway",
one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).target_value == 400000,
length(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).predefined_metric_specification) == 0,
])
error_message = "The token policy must track a customized metric at the configured tokens per minute per task."
}
assert {
condition = alltrue([
length(one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics) == 3,
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].id == "tokens_per_minute",
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].return_data == false,
one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].metric_stat).stat == "Sum",
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].metric_stat).metric).namespace == "LiteLLM/Prometheus",
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].metric_stat).metric).metric_name == "litellm_total_tokens_metric_total",
{ for d in one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].metric_stat).metric).dimensions : d.name => d.value } == { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" },
])
error_message = "The first metric must sum the published token counter deltas under the configured namespace and dimensions."
}
assert {
condition = alltrue([
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].id == "running_tasks",
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].return_data == false,
one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).stat == "Average",
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).namespace == "ECS/ContainerInsights",
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).metric_name == "RunningTaskCount",
{ for d in one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).dimensions : d.name => d.value } == { ClusterName = "acme-litellm-test", ServiceName = "acme-litellm-test-gateway" },
])
error_message = "The second metric must read the gateway service's Container Insights RunningTaskCount."
}
assert {
condition = alltrue([
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute_per_task"].id == "tokens_per_minute_per_task",
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute_per_task"].expression == "tokens_per_minute / running_tasks",
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute_per_task"].return_data == true,
])
error_message = "Only the per-task division may return data to the scaling policy."
}
}
run "tokens_per_target_needs_the_metric_location" {
command = plan
variables {
gateway_tokens_per_target = 400000
}
expect_failures = [
aws_appautoscaling_policy.gateway_tokens,
]
}
run "requests_and_tokens_scale_next_to_cpu_and_memory" {
command = plan
variables {
gateway_requests_per_target = 600
gateway_tokens_per_target = 400000
gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" }
}
assert {
condition = alltrue([
length(aws_appautoscaling_policy.gateway_cpu) == 1,
length(aws_appautoscaling_policy.gateway_memory) == 1,
length(aws_appautoscaling_policy.gateway_requests) == 1,
length(aws_appautoscaling_policy.gateway_tokens) == 1,
one(aws_appautoscaling_policy.gateway_cpu[0].target_tracking_scaling_policy_configuration).target_value == 70,
one(aws_appautoscaling_policy.gateway_memory[0].target_tracking_scaling_policy_configuration).target_value == 80,
])
error_message = "Workload policies must coexist with the CPU and memory policies at their default targets."
}
assert {
condition = length(one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].metric_stat).metric).dimensions) == 0
error_message = "Omitting dimensions must query the token metric without any."
}
}
run "workload_targets_are_ignored_when_autoscaling_is_off" {
command = plan
variables {
gateway_autoscaling_enabled = false
gateway_requests_per_target = 600
gateway_tokens_per_target = 400000
gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" }
}
assert {
condition = alltrue([
length(aws_appautoscaling_target.gateway) == 0,
length(aws_appautoscaling_policy.gateway_requests) == 0,
length(aws_appautoscaling_policy.gateway_tokens) == 0,
])
error_message = "Disabling gateway autoscaling must drop the workload policies with the target."
}
}

38
terraform/litellm/aws/variables.tf
View file

@ -272,6 +272,44 @@ variable "gateway_memory_target" {
default = 80
}
variable "gateway_requests_per_target" {
description = <<-EOT
Requests per minute one gateway task should serve. Adds an
ALBRequestCountPerTarget target-tracking policy next to the CPU/memory
ones (Application Auto Scaling follows whichever asks for more tasks).
0 skips the policy.
EOT
type = number
default = 0
}
variable "gateway_tokens_per_target" {
description = <<-EOT
Tokens per minute one gateway task should serve. Adds a target-tracking
policy on gateway_tokens_metric divided by the service's Container
Insights RunningTaskCount. Tokens are counted when a response completes,
so the signal trails long streams. 0 skips the policy.
EOT
type = number
default = 0
}
variable "gateway_tokens_metric" {
description = <<-EOT
CloudWatch metric carrying the gateway's litellm_total_tokens_metric_total
counter, as published by the CloudWatch agent's Prometheus scraper (it
emits the delta between scrapes, so Sum over a minute is tokens per
minute). Required when gateway_tokens_per_target > 0. dimensions must
match the metric_declaration the agent publishes with.
EOT
type = object({
namespace = string
name = optional(string, "litellm_total_tokens_metric_total")
dimensions = optional(map(string), {})
})
default = null
}
variable "backend_autoscaling_enabled" {
description = "Toggle Application Auto Scaling target-tracking on the backend service."
type = bool

18
terraform/litellm/gcp/README.md
View file

@ -238,6 +238,24 @@ this with `litellm_license`. To tune the export cadence, set
Behavior matches the AWS stack 1:1; the variable names are identical
### Autoscaling
Cloud Run scales the gateway on request concurrency (plus its built-in CPU
target), not on a metric you attach. Each instance takes up to
`gateway_max_instance_request_concurrency` requests at once (default 80)
and Cloud Run adds instances between `gateway_min_instances` and
`gateway_max_instances` when the in-flight count fills up. That is the
request-rate signal for this stack: lower the concurrency for LLM streams
that hold a worker for tens of seconds, since a stream counts as one request
for as long as it is open
There is no tokens-per-minute path here. Cloud Run's autoscaler has no
custom-metric input, so the `litellm_total_tokens_metric_total` counter the
proxy exposes cannot drive it. If you need token-based scaling on GCP, run
the gateway on GKE with the Helm chart's `targetTokensPerMinute` (see
"Dependencies only" below) rather than wiring the counter into Cloud
Monitoring, which the autoscaler would ignore
## Tenant deployment
Every resource the stack creates is named `${tenant}-litellm-${env}` (or