scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 81

feat: support custom hpa metrics in litellm helm chart

Browse source
This commit is contained in:
MarshHawk 2026-06-03 09:51:07 -04:00
parent 35f6961526
commit 5bf2d9bcd6
3 changed files with 192 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
deploy/charts/litellm-helm/templates/hpa.yaml
View file

@ -16,7 +16,11 @@ spec:
behavior:
{{- toYaml .Values.autoscaling.behavior | nindent 4 }}
{{- end }}
{{- if coalesce .Values.autoscaling.metrics .Values.autoscaling.targetCPUUtilizationPercentage .Values.autoscaling.targetMemoryUtilizationPercentage }}
metrics:
{{- with .Values.autoscaling.metrics }}
{{- toYaml . | nindent 4 }}
{{- else }}
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
@ -33,4 +37,6 @@ spec:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

207
deploy/charts/litellm-helm/tests/hpa_tests.yaml
View file

@ -1,36 +1,199 @@
suite: "hpa with behavior"
suite: test hpa
templates:
- hpa.yaml
tests:
- it: "renders behavior when set"
- it: should render custom autoscaling metrics and behavior
template: hpa.yaml
set:
autoscaling.enabled: true
autoscaling.behavior:
scaleUp:
stabilizationWindowSeconds: 60
policies:
autoscaling:
enabled: true
minReplicas: 2
maxReplicas: 10
metrics:
- type: Object
object:
describedObject:
apiVersion: v1
kind: Service
name: litellm-helm
metric:
name: litellm_requests_per_second
target:
type: AverageValue
averageValue: "20"
behavior:
scaleUp:
stabilizationWindowSeconds: 0
selectPolicy: Max
policies:
- type: Percent
value: 100
periodSeconds: 60
- type: Pods
value: 4
periodSeconds: 60
scaleDown:
stabilizationWindowSeconds: 300
selectPolicy: Max
policies:
- type: Percent
value: 50
periodSeconds: 60
asserts:
- isKind:
of: HorizontalPodAutoscaler
- equal:
path: spec.minReplicas
value: 2
- equal:
path: spec.maxReplicas
value: 10
- equal:
path: spec.metrics
value:
- type: Object
object:
describedObject:
apiVersion: v1
kind: Service
name: litellm-helm
metric:
name: litellm_requests_per_second
target:
type: AverageValue
averageValue: "20"
- equal:
path: spec.behavior
value:
scaleUp:
stabilizationWindowSeconds: 0
selectPolicy: Max
policies:
- type: Percent
value: 100
periodSeconds: 60
- type: Pods
value: 4
periodSeconds: 60
scaleDown:
stabilizationWindowSeconds: 300
selectPolicy: Max
policies:
- type: Percent
value: 50
periodSeconds: 60
- it: should keep rendering cpu and memory metrics when custom metrics are not set
template: hpa.yaml
set:
autoscaling:
enabled: true
targetCPUUtilizationPercentage: 75
targetMemoryUtilizationPercentage: 70
asserts:
- isKind:
of: HorizontalPodAutoscaler
- equal:
path: spec.metrics
value:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 75
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 70
- isNull:
path: spec.behavior
- it: should prefer custom metrics over cpu and memory targets
template: hpa.yaml
set:
autoscaling:
enabled: true
targetCPUUtilizationPercentage: 75
targetMemoryUtilizationPercentage: 70
metrics:
- type: Object
object:
describedObject:
apiVersion: v1
kind: Service
name: litellm-helm
metric:
name: litellm_requests_per_second
target:
type: AverageValue
averageValue: "20"
asserts:
- isKind:
of: HorizontalPodAutoscaler
- equal:
path: spec.metrics
value:
- type: Object
object:
describedObject:
apiVersion: v1
kind: Service
name: litellm-helm
metric:
name: litellm_requests_per_second
target:
type: AverageValue
averageValue: "20"
- it: should render behavior when set
template: hpa.yaml
set:
autoscaling:
enabled: true
behavior:
scaleUp:
stabilizationWindowSeconds: 60
policies:
- type: Pods
value: 2
periodSeconds: 60
scaleDown:
stabilizationWindowSeconds: 90
policies:
scaleDown:
stabilizationWindowSeconds: 90
policies:
- type: Pods
value: 1
periodSeconds: 60
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 }
- equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 }
- isKind:
of: HorizontalPodAutoscaler
- equal:
path: spec.behavior.scaleUp.stabilizationWindowSeconds
value: 60
- equal:
path: spec.behavior.scaleDown.stabilizationWindowSeconds
value: 90
---
suite: "hpa without behavior"
templates:
- hpa.yaml
tests:
- it: "does not render behavior when not set"
- it: should omit metrics when no metric sources are set
template: hpa.yaml
set:
autoscaling.enabled: true
autoscaling:
enabled: true
targetCPUUtilizationPercentage: null
targetMemoryUtilizationPercentage: null
metrics: []
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- isNull: { path: spec.behavior }
- isKind:
of: HorizontalPodAutoscaler
- isNull:
path: spec.metrics
- it: should not render when autoscaling is disabled
template: hpa.yaml
set:
autoscaling.enabled: false
asserts:
- hasDocuments:
count: 0

1
deploy/charts/litellm-helm/values.yaml
View file

@ -184,6 +184,7 @@ autoscaling:
maxReplicas: 100
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80
metrics: []
# behavior: {}
# Autoscaling with keda is mutually exclusive with hpa