diff --git a/helm/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml index 3959d85edf3..a3cb388ffc6 100644 --- a/helm/litellm-helm/Chart.yaml +++ b/helm/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.2 +version: 1.1.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/litellm-helm/tests/hpa_tests.yaml b/helm/litellm-helm/tests/hpa_tests.yaml index ec18c3591d3..cd062dd5971 100644 --- a/helm/litellm-helm/tests/hpa_tests.yaml +++ b/helm/litellm-helm/tests/hpa_tests.yaml @@ -1,4 +1,4 @@ -suite: "hpa with behavior" +suite: "hpa" templates: - hpa.yaml tests: @@ -23,14 +23,44 @@ tests: - equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 } - equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 } ---- -suite: "hpa without behavior" -templates: - - hpa.yaml -tests: - it: "does not render behavior when not set" set: autoscaling.enabled: true asserts: - isKind: { of: HorizontalPodAutoscaler } - isNull: { path: spec.behavior } + + - it: "scales on cpu at the documented 60 percent by default" + set: + autoscaling.enabled: true + asserts: + - isKind: { of: HorizontalPodAutoscaler } + - equal: { path: "spec.metrics[0].resource.name", value: cpu } + - equal: { path: "spec.metrics[0].resource.target.type", value: Utilization } + - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 } + + - it: "does not scale on memory by default" + set: + autoscaling.enabled: true + asserts: + - lengthEqual: { path: spec.metrics, count: 1 } + + - it: "honours an explicit cpu target override" + set: + autoscaling.enabled: true + autoscaling.targetCPUUtilizationPercentage: 75 + asserts: + - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 } + + - it: "renders a memory metric only when a memory target is set" + set: + autoscaling.enabled: true + autoscaling.targetMemoryUtilizationPercentage: 80 + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - equal: { path: "spec.metrics[1].resource.name", value: memory } + - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 } + + - it: "renders no hpa when autoscaling is disabled" + asserts: + - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index f8df98de102..637be2322e3 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -200,7 +200,16 @@ autoscaling: enabled: false minReplicas: 1 maxReplicas: 100 - targetCPUUtilizationPercentage: 80 + # 60 is the documented recommendation. See "Recommended Machine Specifications" + # in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe + # above only after up to failureThreshold x periodSeconds = 300 seconds, so a target + # high enough to trip near saturation adds capacity minutes after it was needed. + targetCPUUtilizationPercentage: 60 + # Deliberately left unset rather than given a value. The prisma query engine's + # resident memory is a high-water mark that ratchets to the pod's worst-ever write + # and is never returned, so a memory target reads the largest write a pod ever did + # rather than what it is doing now, and replicas ratchet up without scaling back in. + # Memory is a floor to provision under 'resources', not a signal to scale on. # targetMemoryUtilizationPercentage: 80 # behavior: {}