mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
Merge remote-tracking branch 'origin/litellm_hpa_rpm_tpm' into litellm_perf_1k_rps_sidecar
This commit is contained in:
yassin
commit
bff8ee7a4b
13 changed files with 258 additions and 168 deletions
|
|
@ -43,22 +43,22 @@ spec:
|
|||
type: Utilization
|
||||
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
|
||||
{{- end }}
|
||||
{{- with .Values.autoscaling.targetRequestsPerMinute }}
|
||||
{{- with .Values.autoscaling.targetRequestsPerSecond }}
|
||||
- type: Pods
|
||||
pods:
|
||||
metric:
|
||||
name: litellm_requests_per_minute
|
||||
name: litellm_requests_per_second
|
||||
target:
|
||||
type: AverageValue
|
||||
averageValue: {{ . | quote }}
|
||||
averageValue: {{ toJson . | trimAll "\"" | quote }}
|
||||
{{- end }}
|
||||
{{- with .Values.autoscaling.targetTokensPerMinute }}
|
||||
{{- with .Values.autoscaling.targetTokensPerSecond }}
|
||||
- type: Pods
|
||||
pods:
|
||||
metric:
|
||||
name: litellm_tokens_per_minute
|
||||
name: litellm_tokens_per_second
|
||||
target:
|
||||
type: AverageValue
|
||||
averageValue: {{ . | quote }}
|
||||
averageValue: {{ toJson . | trimAll "\"" | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -25,24 +25,24 @@ spec:
|
|||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- $prom := .Values.keda.prometheus }}
|
||||
{{- if or $prom.requestsPerMinute $prom.tokensPerMinute }}
|
||||
{{- if or $prom.requestsPerSecond $prom.tokensPerSecond }}
|
||||
{{- if not $prom.serverAddress }}
|
||||
{{- fail "keda.prometheus.serverAddress is required when keda.prometheus.requestsPerMinute or tokensPerMinute is set" }}
|
||||
{{- fail "keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set" }}
|
||||
{{- end }}
|
||||
{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s%s" (include "litellm.fullname" .) (ternary "-metrics" "" .Values.metricsServer.enabled)) }}
|
||||
{{- with $prom.requestsPerMinute }}
|
||||
{{- with $prom.requestsPerSecond }}
|
||||
- type: prometheus
|
||||
metadata:
|
||||
serverAddress: {{ $prom.serverAddress | quote }}
|
||||
threshold: {{ . | quote }}
|
||||
query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m])) * 60" $selector | quote }}
|
||||
threshold: {{ toJson . | trimAll "\"" | quote }}
|
||||
query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }}
|
||||
{{- end }}
|
||||
{{- with $prom.tokensPerMinute }}
|
||||
{{- with $prom.tokensPerSecond }}
|
||||
- type: prometheus
|
||||
metadata:
|
||||
serverAddress: {{ $prom.serverAddress | quote }}
|
||||
threshold: {{ . | quote }}
|
||||
query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m])) * 60" $selector | quote }}
|
||||
threshold: {{ toJson . | trimAll "\"" | quote }}
|
||||
query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
advanced:
|
||||
|
|
|
|||
|
|
@ -69,10 +69,10 @@ tests:
|
|||
- lengthEqual: { path: spec.metrics, count: 2 }
|
||||
- notContains: { path: spec.metrics, content: { type: Pods }, any: true }
|
||||
|
||||
- it: "adds a requests-per-minute Pods metric after the cpu metric"
|
||||
- it: "adds a requests-per-second Pods metric after the cpu metric"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
autoscaling.targetRequestsPerMinute: 600
|
||||
autoscaling.targetRequestsPerSecond: 90
|
||||
asserts:
|
||||
- lengthEqual: { path: spec.metrics, count: 2 }
|
||||
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
|
||||
|
|
@ -81,13 +81,13 @@ tests:
|
|||
value:
|
||||
type: Pods
|
||||
pods:
|
||||
metric: { name: litellm_requests_per_minute }
|
||||
target: { type: AverageValue, averageValue: "600" }
|
||||
metric: { name: litellm_requests_per_second }
|
||||
target: { type: AverageValue, averageValue: "90" }
|
||||
|
||||
- it: "adds a tokens-per-minute Pods metric on its own"
|
||||
- it: "adds a tokens-per-second Pods metric on its own"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
autoscaling.targetTokensPerMinute: 400k
|
||||
autoscaling.targetTokensPerSecond: 6M
|
||||
asserts:
|
||||
- lengthEqual: { path: spec.metrics, count: 2 }
|
||||
- equal:
|
||||
|
|
@ -95,39 +95,49 @@ tests:
|
|||
value:
|
||||
type: Pods
|
||||
pods:
|
||||
metric: { name: litellm_tokens_per_minute }
|
||||
target: { type: AverageValue, averageValue: "400k" }
|
||||
metric: { name: litellm_tokens_per_second }
|
||||
target: { type: AverageValue, averageValue: "6M" }
|
||||
- notContains:
|
||||
path: spec.metrics
|
||||
content: { type: Pods, pods: { metric: { name: litellm_requests_per_minute } } }
|
||||
content: { type: Pods, pods: { metric: { name: litellm_requests_per_second } } }
|
||||
any: true
|
||||
|
||||
- it: "renders requests, tokens, cpu and memory metrics together"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
autoscaling.targetMemoryUtilizationPercentage: 80
|
||||
autoscaling.targetRequestsPerMinute: 600
|
||||
autoscaling.targetTokensPerMinute: 400000
|
||||
autoscaling.targetRequestsPerSecond: 90
|
||||
autoscaling.targetTokensPerSecond: 6000000
|
||||
asserts:
|
||||
- lengthEqual: { path: spec.metrics, count: 4 }
|
||||
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
|
||||
- equal: { path: "spec.metrics[1].resource.name", value: memory }
|
||||
- equal: { path: "spec.metrics[2].pods.metric.name", value: litellm_requests_per_minute }
|
||||
- equal: { path: "spec.metrics[2].pods.target.averageValue", value: "600" }
|
||||
- equal: { path: "spec.metrics[3].pods.metric.name", value: litellm_tokens_per_minute }
|
||||
- equal: { path: "spec.metrics[3].pods.target.averageValue", value: "400000" }
|
||||
- equal: { path: "spec.metrics[2].pods.metric.name", value: litellm_requests_per_second }
|
||||
- equal: { path: "spec.metrics[2].pods.target.averageValue", value: "90" }
|
||||
- equal: { path: "spec.metrics[3].pods.metric.name", value: litellm_tokens_per_second }
|
||||
- equal: { path: "spec.metrics[3].pods.target.averageValue", value: "6000000" }
|
||||
|
||||
- it: "scales on workload metrics alone when the cpu target is cleared"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
autoscaling.targetCPUUtilizationPercentage: null
|
||||
autoscaling.targetRequestsPerMinute: 600
|
||||
autoscaling.targetTokensPerMinute: 400000
|
||||
autoscaling.targetRequestsPerSecond: 90
|
||||
autoscaling.targetTokensPerSecond: 6000000
|
||||
asserts:
|
||||
- lengthEqual: { path: spec.metrics, count: 2 }
|
||||
- notContains: { path: spec.metrics, content: { type: Resource }, any: true }
|
||||
- equal: { path: "spec.metrics[0].pods.metric.name", value: litellm_requests_per_minute }
|
||||
- equal: { path: "spec.metrics[1].pods.metric.name", value: litellm_tokens_per_minute }
|
||||
- equal: { path: "spec.metrics[0].pods.metric.name", value: litellm_requests_per_second }
|
||||
- equal: { path: "spec.metrics[1].pods.metric.name", value: litellm_tokens_per_second }
|
||||
- notMatchRegexRaw: { pattern: per_minute }
|
||||
|
||||
- it: "ignores the per-minute keys, which the chart never shipped"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
autoscaling.targetRequestsPerMinute: 5400
|
||||
autoscaling.targetTokensPerMinute: 360000000
|
||||
asserts:
|
||||
- lengthEqual: { path: spec.metrics, count: 1 }
|
||||
- notContains: { path: spec.metrics, content: { type: Pods }, any: true }
|
||||
|
||||
- it: "renders no hpa when autoscaling is disabled"
|
||||
asserts:
|
||||
|
|
|
|||
|
|
@ -25,11 +25,11 @@ tests:
|
|||
metricType: Utilization
|
||||
metadata: { value: "60" }
|
||||
|
||||
- it: "scales on release-wide requests per minute divided by the per-replica target"
|
||||
- it: "scales on release-wide requests per second divided by the per-replica target"
|
||||
set:
|
||||
keda.enabled: true
|
||||
keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090
|
||||
keda.prometheus.requestsPerMinute: 600
|
||||
keda.prometheus.requestsPerSecond: 90
|
||||
asserts:
|
||||
- lengthEqual: { path: spec.triggers, count: 1 }
|
||||
- equal:
|
||||
|
|
@ -38,21 +38,21 @@ tests:
|
|||
type: prometheus
|
||||
metadata:
|
||||
serverAddress: http://prometheus-operated.monitoring.svc:9090
|
||||
threshold: "600"
|
||||
query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm"}[1m])) * 60
|
||||
threshold: "90"
|
||||
query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm"}[1m]))
|
||||
|
||||
- it: "scales on tokens per minute on its own"
|
||||
- it: "scales on tokens per second on its own"
|
||||
set:
|
||||
keda.enabled: true
|
||||
keda.prometheus.serverAddress: http://prom:9090
|
||||
keda.prometheus.tokensPerMinute: 400000
|
||||
keda.prometheus.tokensPerSecond: 6000000
|
||||
asserts:
|
||||
- lengthEqual: { path: spec.triggers, count: 1 }
|
||||
- equal: { path: "spec.triggers[0].type", value: prometheus }
|
||||
- equal: { path: "spec.triggers[0].metadata.threshold", value: "400000" }
|
||||
- equal: { path: "spec.triggers[0].metadata.threshold", value: "6000000" }
|
||||
- equal:
|
||||
path: "spec.triggers[0].metadata.query"
|
||||
value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm"}[1m])) * 60
|
||||
value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm"}[1m]))
|
||||
|
||||
- it: "appends requests and tokens triggers after user triggers and selects the metrics service job"
|
||||
set:
|
||||
|
|
@ -63,33 +63,44 @@ tests:
|
|||
metricType: Utilization
|
||||
metadata: { value: "60" }
|
||||
keda.prometheus.serverAddress: http://prom:9090
|
||||
keda.prometheus.requestsPerMinute: 600
|
||||
keda.prometheus.tokensPerMinute: 400000
|
||||
keda.prometheus.requestsPerSecond: 90
|
||||
keda.prometheus.tokensPerSecond: 6000000
|
||||
asserts:
|
||||
- lengthEqual: { path: spec.triggers, count: 3 }
|
||||
- equal: { path: "spec.triggers[0].type", value: cpu }
|
||||
- equal: { path: "spec.triggers[1].metadata.threshold", value: "600" }
|
||||
- equal: { path: "spec.triggers[1].metadata.threshold", value: "90" }
|
||||
- equal:
|
||||
path: "spec.triggers[1].metadata.query"
|
||||
value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) * 60
|
||||
- equal: { path: "spec.triggers[2].metadata.threshold", value: "400000" }
|
||||
value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m]))
|
||||
- equal: { path: "spec.triggers[2].metadata.threshold", value: "6000000" }
|
||||
- equal:
|
||||
path: "spec.triggers[2].metadata.query"
|
||||
value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) * 60
|
||||
value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m]))
|
||||
- notMatchRegexRaw: { pattern: "\\* *60|per_minute|PerMinute" }
|
||||
|
||||
- it: "ignores the per-minute keys, which the chart never shipped"
|
||||
set:
|
||||
keda.enabled: true
|
||||
keda.prometheus.serverAddress: http://prom:9090
|
||||
keda.prometheus.requestsPerMinute: 5400
|
||||
keda.prometheus.tokensPerMinute: 360000000
|
||||
asserts:
|
||||
- isKind: { of: ScaledObject }
|
||||
- isNullOrEmpty: { path: spec.triggers }
|
||||
|
||||
- it: "refuses a workload target without a prometheus server address"
|
||||
set:
|
||||
keda.enabled: true
|
||||
keda.prometheus.requestsPerMinute: 600
|
||||
keda.prometheus.requestsPerSecond: 90
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: keda.prometheus.serverAddress is required when keda.prometheus.requestsPerMinute or tokensPerMinute is set
|
||||
errorMessage: keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set
|
||||
|
||||
- it: "yields to the hpa when both autoscalers are enabled"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
keda.enabled: true
|
||||
keda.prometheus.serverAddress: http://prom:9090
|
||||
keda.prometheus.requestsPerMinute: 600
|
||||
keda.prometheus.requestsPerSecond: 90
|
||||
asserts:
|
||||
- hasDocuments: { count: 0 }
|
||||
|
|
|
|||
|
|
@ -265,17 +265,24 @@ autoscaling:
|
|||
# targetMemoryUtilizationPercentage: 80
|
||||
# behavior: {}
|
||||
# Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics
|
||||
# named `litellm_requests_per_minute` and `litellm_tokens_per_minute` with an
|
||||
# named `litellm_requests_per_second` and `litellm_tokens_per_second` with an
|
||||
# AverageValue target, alongside whichever resource targets are set (the HPA
|
||||
# follows the metric asking for the most replicas). A Prometheus Adapter must
|
||||
# serve those two names on custom.metrics.k8s.io from the proxy's request and
|
||||
# token counters, grouped by the scrape target's `pod` label; enable
|
||||
# serviceMonitor below so every pod is scraped on its own. Adapter rules and
|
||||
# the replica arithmetic:
|
||||
# https://docs.litellm.ai/docs/proxy/deploy#scale-on-requests-and-tokens-per-pod
|
||||
# Tokens are counted when a response completes, so TPM trails long streams.
|
||||
targetRequestsPerMinute: ""
|
||||
targetTokensPerMinute: ""
|
||||
# serve those two names on custom.metrics.k8s.io from the proxy's counters,
|
||||
# grouped by the scrape target's `pod` label (enable serviceMonitor below so
|
||||
# every pod is scraped on its own):
|
||||
# litellm_requests_per_second:
|
||||
# sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
|
||||
# litellm_tokens_per_second:
|
||||
# sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
|
||||
# rate() over [1m] is already per second, so no `* 60`. How fast the HPA
|
||||
# reacts is set by that window, the scrape interval and the HPA sync period
|
||||
# (15s by default), not by the unit: keep serviceMonitor.interval at 15s or
|
||||
# faster so a 1m window holds at least 4 samples. averageValue takes SI
|
||||
# suffixes, so "6M" is six million tokens per second per pod. Tokens are
|
||||
# counted when a response completes, so TPS trails long streams.
|
||||
targetRequestsPerSecond: ""
|
||||
targetTokensPerSecond: ""
|
||||
|
||||
# Autoscaling with keda is mutually exclusive with hpa
|
||||
keda:
|
||||
|
|
@ -298,18 +305,22 @@ keda:
|
|||
# threshold: '100'
|
||||
# query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
|
||||
# First-class Prometheus triggers on the proxy's own request and token
|
||||
# counters, appended to `triggers`. Each target is the load one replica should
|
||||
# carry: KEDA divides the release-wide rate by it to pick the replica count.
|
||||
# The queries select samples by the release namespace and the `job` label the
|
||||
# counters, appended to `triggers`. Each target is the per-second load one
|
||||
# replica should carry: KEDA divides the release-wide
|
||||
# `sum(rate(<counter>[1m]))` by it to pick the replica count. Thresholds
|
||||
# are plain numbers (KEDA parses them as floats, no SI suffixes). The
|
||||
# queries select samples by the release namespace and the `job` label the
|
||||
# chart's ServiceMonitor produces (the metrics Service name), so enable
|
||||
# serviceMonitor below together with metricsServer: the http port serves
|
||||
# /metrics/ behind virtual-key auth and answers an unauthenticated scrape
|
||||
# with 401. Tokens are counted at completion, so TPM trails long streams.
|
||||
# serverAddress is required once either target is set.
|
||||
# with 401. Reaction time comes from the [1m] window, the scrape interval
|
||||
# and pollingInterval above, so keep both at 15s or faster. Tokens are
|
||||
# counted at completion, so TPS trails long streams. serverAddress is
|
||||
# required once either target is set.
|
||||
prometheus:
|
||||
serverAddress: ""
|
||||
requestsPerMinute: ""
|
||||
tokensPerMinute: ""
|
||||
requestsPerSecond: ""
|
||||
tokensPerSecond: ""
|
||||
behavior: {}
|
||||
# scaleDown:
|
||||
# stabilizationWindowSeconds: 300
|
||||
|
|
|
|||
|
|
@ -40,23 +40,23 @@ spec:
|
|||
type: Utilization
|
||||
averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
|
||||
{{- end }}
|
||||
{{- with .Values.gateway.hpa.targetRequestsPerMinute }}
|
||||
{{- with .Values.gateway.hpa.targetRequestsPerSecond }}
|
||||
- type: Pods
|
||||
pods:
|
||||
metric:
|
||||
name: litellm_requests_per_minute
|
||||
name: litellm_requests_per_second
|
||||
target:
|
||||
type: AverageValue
|
||||
averageValue: {{ . | quote }}
|
||||
averageValue: {{ toJson . | trimAll "\"" | quote }}
|
||||
{{- end }}
|
||||
{{- with .Values.gateway.hpa.targetTokensPerMinute }}
|
||||
{{- with .Values.gateway.hpa.targetTokensPerSecond }}
|
||||
- type: Pods
|
||||
pods:
|
||||
metric:
|
||||
name: litellm_tokens_per_minute
|
||||
name: litellm_tokens_per_second
|
||||
target:
|
||||
type: AverageValue
|
||||
averageValue: {{ . | quote }}
|
||||
averageValue: {{ toJson . | trimAll "\"" | quote }}
|
||||
{{- end }}
|
||||
{{- with .Values.gateway.hpa.behavior }}
|
||||
behavior:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
suite: test gateway HPA per-pod requests and tokens targets
|
||||
suite: test gateway HPA per-pod requests-per-second and tokens-per-second targets
|
||||
templates:
|
||||
- gateway/hpa.yaml
|
||||
- gateway/servicemonitor.yaml
|
||||
|
|
@ -24,10 +24,10 @@ tests:
|
|||
type: Utilization
|
||||
averageUtilization: 80
|
||||
|
||||
- it: adds a requests-per-minute Pods metric next to the resource metrics
|
||||
- it: adds a requests-per-second Pods metric next to the resource metrics
|
||||
template: gateway/hpa.yaml
|
||||
set:
|
||||
gateway.hpa.targetRequestsPerMinute: 600
|
||||
gateway.hpa.targetRequestsPerSecond: 90
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.metrics
|
||||
|
|
@ -47,23 +47,23 @@ tests:
|
|||
type: Pods
|
||||
pods:
|
||||
metric:
|
||||
name: litellm_requests_per_minute
|
||||
name: litellm_requests_per_second
|
||||
target:
|
||||
type: AverageValue
|
||||
averageValue: "600"
|
||||
averageValue: "90"
|
||||
- notContains:
|
||||
path: spec.metrics
|
||||
content:
|
||||
type: Pods
|
||||
pods:
|
||||
metric:
|
||||
name: litellm_tokens_per_minute
|
||||
name: litellm_tokens_per_second
|
||||
any: true
|
||||
|
||||
- it: adds a tokens-per-minute Pods metric on its own
|
||||
- it: adds a tokens-per-second Pods metric on its own
|
||||
template: gateway/hpa.yaml
|
||||
set:
|
||||
gateway.hpa.targetTokensPerMinute: 400k
|
||||
gateway.hpa.targetTokensPerSecond: 6M
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.metrics
|
||||
|
|
@ -74,24 +74,24 @@ tests:
|
|||
type: Pods
|
||||
pods:
|
||||
metric:
|
||||
name: litellm_tokens_per_minute
|
||||
name: litellm_tokens_per_second
|
||||
target:
|
||||
type: AverageValue
|
||||
averageValue: "400k"
|
||||
averageValue: "6M"
|
||||
- notContains:
|
||||
path: spec.metrics
|
||||
content:
|
||||
type: Pods
|
||||
pods:
|
||||
metric:
|
||||
name: litellm_requests_per_minute
|
||||
name: litellm_requests_per_second
|
||||
any: true
|
||||
|
||||
- it: renders requests and tokens targets together and keeps CPU and memory
|
||||
template: gateway/hpa.yaml
|
||||
set:
|
||||
gateway.hpa.targetRequestsPerMinute: 600
|
||||
gateway.hpa.targetTokensPerMinute: 400000
|
||||
gateway.hpa.targetRequestsPerSecond: 90
|
||||
gateway.hpa.targetTokensPerSecond: 6000000
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.metrics
|
||||
|
|
@ -104,21 +104,21 @@ tests:
|
|||
value: memory
|
||||
- equal:
|
||||
path: spec.metrics[2].pods.metric.name
|
||||
value: litellm_requests_per_minute
|
||||
value: litellm_requests_per_second
|
||||
- equal:
|
||||
path: spec.metrics[3].pods.metric.name
|
||||
value: litellm_tokens_per_minute
|
||||
value: litellm_tokens_per_second
|
||||
- equal:
|
||||
path: spec.metrics[3].pods.target.averageValue
|
||||
value: "400000"
|
||||
value: "6000000"
|
||||
|
||||
- it: scales on workload metrics alone when the resource targets are cleared
|
||||
template: gateway/hpa.yaml
|
||||
set:
|
||||
gateway.hpa.targetCPUUtilizationPercentage: null
|
||||
gateway.hpa.targetMemoryUtilizationPercentage: null
|
||||
gateway.hpa.targetRequestsPerMinute: 600
|
||||
gateway.hpa.targetTokensPerMinute: 400000
|
||||
gateway.hpa.targetRequestsPerSecond: 90
|
||||
gateway.hpa.targetTokensPerSecond: 6000000
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.metrics
|
||||
|
|
@ -130,10 +130,27 @@ tests:
|
|||
any: true
|
||||
- equal:
|
||||
path: spec.metrics[0].pods.metric.name
|
||||
value: litellm_requests_per_minute
|
||||
value: litellm_requests_per_second
|
||||
- equal:
|
||||
path: spec.metrics[1].pods.metric.name
|
||||
value: litellm_tokens_per_minute
|
||||
value: litellm_tokens_per_second
|
||||
- notMatchRegexRaw:
|
||||
pattern: per_minute
|
||||
|
||||
- it: ignores the per-minute keys, which the chart never shipped
|
||||
template: gateway/hpa.yaml
|
||||
set:
|
||||
gateway.hpa.targetRequestsPerMinute: 5400
|
||||
gateway.hpa.targetTokensPerMinute: 360000000
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.metrics
|
||||
count: 2
|
||||
- notContains:
|
||||
path: spec.metrics
|
||||
content:
|
||||
type: Pods
|
||||
any: true
|
||||
|
||||
- it: renders no ServiceMonitor by default
|
||||
template: gateway/servicemonitor.yaml
|
||||
|
|
|
|||
|
|
@ -387,16 +387,24 @@ gateway:
|
|||
# - { type: Percent, value: 100, periodSeconds: 30 }
|
||||
behavior: {}
|
||||
# Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics
|
||||
# named `litellm_requests_per_minute` and `litellm_tokens_per_minute` with an
|
||||
# named `litellm_requests_per_second` and `litellm_tokens_per_second` with an
|
||||
# AverageValue target. They coexist with the CPU/memory targets above: the
|
||||
# HPA scales on whichever metric asks for the most replicas. Kubernetes has
|
||||
# no idea what a token is, so a Prometheus Adapter must serve those two
|
||||
# names on custom.metrics.k8s.io from the proxy's request and token
|
||||
# counters; the adapter rules and the replica arithmetic are documented at
|
||||
# https://docs.litellm.ai/docs/proxy/deploy#scale-on-requests-and-tokens-per-pod
|
||||
# Tokens are counted when a response completes, so TPM trails long streams.
|
||||
targetRequestsPerMinute: ""
|
||||
targetTokensPerMinute: ""
|
||||
# names on custom.metrics.k8s.io from the proxy's counters, grouped by the
|
||||
# scrape target's `pod` label (enable serviceMonitor above):
|
||||
# litellm_requests_per_second:
|
||||
# sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
|
||||
# litellm_tokens_per_second:
|
||||
# sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
|
||||
# rate() over [1m] is already per second, so no `* 60`. How fast the HPA
|
||||
# reacts is set by that window, the scrape interval and the HPA sync period
|
||||
# (15s by default), not by the unit: keep serviceMonitor.interval at 15s or
|
||||
# faster so a 1m window holds at least 4 samples. averageValue takes SI
|
||||
# suffixes, so "6M" is six million tokens per second per pod. Tokens are
|
||||
# counted when a response completes, so TPS trails long streams.
|
||||
targetRequestsPerSecond: ""
|
||||
targetTokensPerSecond: ""
|
||||
# PodDisruptionBudget for the gateway pods. Set exactly one of
|
||||
# `minAvailable` / `maxUnavailable` (minAvailable wins if both are set;
|
||||
# enabling without either falls back to `maxUnavailable: 1`). Disabled by
|
||||
|
|
|
|||
|
|
@ -266,38 +266,52 @@ to them. Application Auto Scaling evaluates every attached policy and follows
|
|||
the one asking for the most tasks, so the resource policies keep working as a
|
||||
floor while requests or tokens drive scale-out
|
||||
|
||||
`gateway_requests_per_target` adds an `ALBRequestCountPerTarget` policy on
|
||||
the gateway target group. The ALB counts requests per minute per registered
|
||||
task, so a value of 600 means "keep each task at about 10 requests per
|
||||
second". No agent or sidecar is needed
|
||||
Both targets are per task per second, the way load is usually quoted (1k
|
||||
rps, 75M tok/s). CloudWatch is the limit on how fast they react: target
|
||||
tracking evaluates every metric, predefined or custom, aggregated over
|
||||
60-second periods and has no period setting, so ECS reacts on a roughly
|
||||
one-minute cadence whatever unit the variable is written in. The Kubernetes
|
||||
charts get a faster signal because the Prometheus `rate()` window and scrape
|
||||
interval are theirs to shorten
|
||||
|
||||
`gateway_tokens_per_target` adds a metric-math policy that divides a
|
||||
`gateway_target_requests_per_second` adds an `ALBRequestCountPerTarget`
|
||||
policy on the gateway target group. The ALB publishes that metric as requests
|
||||
per minute per registered task, so the policy's target value is 60 times the
|
||||
variable: 90 rps becomes a target of 5,400 per minute. No agent or sidecar is
|
||||
needed
|
||||
|
||||
`gateway_target_tokens_per_second` adds a metric-math policy over a
|
||||
CloudWatch metric of the gateway's `litellm_total_tokens_metric_total`
|
||||
counter by the service's `RunningTaskCount` from Container Insights. Nothing
|
||||
counter and the service's `RunningTaskCount` from Container Insights. Nothing
|
||||
native to ECS carries token throughput, so you publish that metric yourself
|
||||
with the CloudWatch agent's Prometheus scraper pointed at the metrics sidecar
|
||||
above. The agent emits the delta of a counter between scrapes, so `Sum` over a
|
||||
minute is tokens per minute. Tokens are counted when a response completes, so
|
||||
long streams show up late in this signal. `gateway_tokens_metric` tells the
|
||||
above. The agent emits the delta of a counter between scrapes, so `Sum` over
|
||||
the 60-second period is the tokens served in that minute; the expression
|
||||
divides by 60 (`tokens_per_second`) and then by the task count
|
||||
(`tokens_per_second_per_task`). Tokens are counted when a response completes,
|
||||
so long streams show up late in this signal. `gateway_tokens_metric` tells the
|
||||
policy where the agent publishes: the namespace, the metric name (defaults to
|
||||
the counter name) and the dimensions from your `metric_declaration`
|
||||
|
||||
```hcl
|
||||
gateway_metrics_port = 4001
|
||||
gateway_requests_per_target = 600
|
||||
gateway_tokens_per_target = 400000
|
||||
gateway_metrics_port = 4001
|
||||
gateway_target_requests_per_second = 90
|
||||
gateway_target_tokens_per_second = 6000000
|
||||
gateway_tokens_metric = {
|
||||
namespace = "LiteLLM/Prometheus"
|
||||
dimensions = { ClusterName = "acme-litellm-prod", TaskDefinitionFamily = "acme-litellm-prod-gateway" }
|
||||
}
|
||||
```
|
||||
|
||||
Worked example for the token policy: three tasks handle 1,800,000 tokens in a
|
||||
minute, so `tokens_per_minute / running_tasks` is 600,000 against a target of
|
||||
400,000. Target tracking sizes the service to `ceil(3 * 600000 / 400000) = 5`
|
||||
tasks. The request policy does the same arithmetic with the ALB's per-target
|
||||
count. Container Insights must be enabled on the cluster for `RunningTaskCount`
|
||||
to exist
|
||||
Worked example for the request policy: 1,000 rps across 10 tasks is 100 rps
|
||||
per task (the ALB reports it as 6,000 per minute per target) against a target
|
||||
of 90 (5,400), so target tracking sizes the service to
|
||||
`ceil(10 * 100 / 90) = 12` tasks. The token policy does the same arithmetic:
|
||||
ten tasks handle 4,200,000,000 tokens in a minute, `tokens / 60` is
|
||||
70,000,000 tokens per second and `tokens_per_second / running_tasks` is
|
||||
7,000,000 against a target of 6,000,000, so the service grows to
|
||||
`ceil(10 * 7000000 / 6000000) = 12`. Container Insights must be enabled on the
|
||||
cluster for `RunningTaskCount` to exist
|
||||
|
||||
## Tenant deployment
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ resource "aws_appautoscaling_policy" "gateway_memory" {
|
|||
}
|
||||
|
||||
resource "aws_appautoscaling_policy" "gateway_requests" {
|
||||
count = var.gateway_autoscaling_enabled && var.gateway_requests_per_target > 0 ? 1 : 0
|
||||
count = var.gateway_autoscaling_enabled && var.gateway_target_requests_per_second > 0 ? 1 : 0
|
||||
name = "${local.name}-gateway-requests"
|
||||
policy_type = "TargetTrackingScaling"
|
||||
service_namespace = aws_appautoscaling_target.gateway[0].service_namespace
|
||||
|
|
@ -65,12 +65,13 @@ resource "aws_appautoscaling_policy" "gateway_requests" {
|
|||
predefined_metric_type = "ALBRequestCountPerTarget"
|
||||
resource_label = "${aws_lb.this.arn_suffix}/${aws_lb_target_group.gateway.arn_suffix}"
|
||||
}
|
||||
target_value = var.gateway_requests_per_target
|
||||
# ALBRequestCountPerTarget is a per-minute count
|
||||
target_value = var.gateway_target_requests_per_second * 60
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_appautoscaling_policy" "gateway_tokens" {
|
||||
count = var.gateway_autoscaling_enabled && var.gateway_tokens_per_target > 0 ? 1 : 0
|
||||
count = var.gateway_autoscaling_enabled && var.gateway_target_tokens_per_second > 0 ? 1 : 0
|
||||
name = "${local.name}-gateway-tokens"
|
||||
policy_type = "TargetTrackingScaling"
|
||||
service_namespace = aws_appautoscaling_target.gateway[0].service_namespace
|
||||
|
|
@ -80,16 +81,17 @@ resource "aws_appautoscaling_policy" "gateway_tokens" {
|
|||
lifecycle {
|
||||
precondition {
|
||||
condition = var.gateway_tokens_metric != null
|
||||
error_message = "gateway_tokens_metric is required when gateway_tokens_per_target > 0."
|
||||
error_message = "gateway_tokens_metric is required when gateway_target_tokens_per_second > 0."
|
||||
}
|
||||
}
|
||||
|
||||
target_tracking_scaling_policy_configuration {
|
||||
target_value = var.gateway_tokens_per_target
|
||||
target_value = var.gateway_target_tokens_per_second
|
||||
|
||||
# target tracking has no period setting and always aggregates over 60s
|
||||
customized_metric_specification {
|
||||
metrics {
|
||||
id = "tokens_per_minute"
|
||||
id = "tokens"
|
||||
return_data = false
|
||||
|
||||
metric_stat {
|
||||
|
|
@ -134,9 +136,15 @@ resource "aws_appautoscaling_policy" "gateway_tokens" {
|
|||
}
|
||||
|
||||
metrics {
|
||||
id = "tokens_per_minute_per_task"
|
||||
expression = "tokens_per_minute / running_tasks"
|
||||
label = "Tokens per minute per gateway task"
|
||||
id = "tokens_per_second"
|
||||
expression = "tokens / 60"
|
||||
return_data = false
|
||||
}
|
||||
|
||||
metrics {
|
||||
id = "tokens_per_second_per_task"
|
||||
expression = "tokens_per_second / running_tasks"
|
||||
label = "Tokens per second per gateway task"
|
||||
return_data = true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,11 +32,11 @@ run "defaults_scale_on_cpu_and_memory_only" {
|
|||
}
|
||||
}
|
||||
|
||||
run "requests_per_target_adds_an_alb_request_count_policy" {
|
||||
run "requests_per_second_adds_an_alb_request_count_policy" {
|
||||
command = plan
|
||||
|
||||
variables {
|
||||
gateway_requests_per_target = 600
|
||||
gateway_target_requests_per_second = 90
|
||||
}
|
||||
|
||||
assert {
|
||||
|
|
@ -57,19 +57,19 @@ run "requests_per_target_adds_an_alb_request_count_policy" {
|
|||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).target_value == 600,
|
||||
one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).target_value == 5400,
|
||||
one(one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).predefined_metric_specification).predefined_metric_type == "ALBRequestCountPerTarget",
|
||||
length(one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).customized_metric_specification) == 0,
|
||||
])
|
||||
error_message = "The request policy must track ALBRequestCountPerTarget at the configured requests per minute per task."
|
||||
error_message = "The request policy must track ALBRequestCountPerTarget at 60 times the configured requests per second per task."
|
||||
}
|
||||
}
|
||||
|
||||
run "tokens_per_target_adds_a_metric_math_policy" {
|
||||
run "tokens_per_second_adds_a_metric_math_policy" {
|
||||
command = plan
|
||||
|
||||
variables {
|
||||
gateway_tokens_per_target = 400000
|
||||
gateway_target_tokens_per_second = 6000000
|
||||
gateway_tokens_metric = {
|
||||
namespace = "LiteLLM/Prometheus"
|
||||
dimensions = { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" }
|
||||
|
|
@ -85,21 +85,21 @@ run "tokens_per_target_adds_a_metric_math_policy" {
|
|||
condition = alltrue([
|
||||
aws_appautoscaling_policy.gateway_tokens[0].name == "acme-litellm-test-gateway-tokens",
|
||||
aws_appautoscaling_policy.gateway_tokens[0].resource_id == "service/acme-litellm-test/acme-litellm-test-gateway",
|
||||
one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).target_value == 400000,
|
||||
one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).target_value == 6000000,
|
||||
length(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).predefined_metric_specification) == 0,
|
||||
])
|
||||
error_message = "The token policy must track a customized metric at the configured tokens per minute per task."
|
||||
error_message = "The token policy must track a customized metric at the configured tokens per second per task."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
length(one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics) == 3,
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].id == "tokens_per_minute",
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].return_data == false,
|
||||
one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].metric_stat).stat == "Sum",
|
||||
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].metric_stat).metric).namespace == "LiteLLM/Prometheus",
|
||||
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].metric_stat).metric).metric_name == "litellm_total_tokens_metric_total",
|
||||
{ for d in one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].metric_stat).metric).dimensions : d.name => d.value } == { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" },
|
||||
length(one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics) == 4,
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].id == "tokens",
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].return_data == false,
|
||||
one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).stat == "Sum",
|
||||
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).namespace == "LiteLLM/Prometheus",
|
||||
one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).metric_name == "litellm_total_tokens_metric_total",
|
||||
{ for d in one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).dimensions : d.name => d.value } == { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" },
|
||||
])
|
||||
error_message = "The first metric must sum the published token counter deltas under the configured namespace and dimensions."
|
||||
}
|
||||
|
|
@ -118,19 +118,27 @@ run "tokens_per_target_adds_a_metric_math_policy" {
|
|||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute_per_task"].id == "tokens_per_minute_per_task",
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute_per_task"].expression == "tokens_per_minute / running_tasks",
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute_per_task"].return_data == true,
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second"].expression == "tokens / 60",
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second"].return_data == false,
|
||||
])
|
||||
error_message = "Only the per-task division may return data to the scaling policy."
|
||||
error_message = "The 60s period Sum must be divided by 60 to yield tokens per second."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second_per_task"].expression == "tokens_per_second / running_tasks",
|
||||
{ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second_per_task"].return_data == true,
|
||||
length([for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m if m.return_data]) == 1,
|
||||
])
|
||||
error_message = "Only the per-task tokens per second may return data to the scaling policy."
|
||||
}
|
||||
}
|
||||
|
||||
run "tokens_per_target_needs_the_metric_location" {
|
||||
run "tokens_per_second_needs_the_metric_location" {
|
||||
command = plan
|
||||
|
||||
variables {
|
||||
gateway_tokens_per_target = 400000
|
||||
gateway_target_tokens_per_second = 6000000
|
||||
}
|
||||
|
||||
expect_failures = [
|
||||
|
|
@ -142,9 +150,9 @@ run "requests_and_tokens_scale_next_to_cpu_and_memory" {
|
|||
command = plan
|
||||
|
||||
variables {
|
||||
gateway_requests_per_target = 600
|
||||
gateway_tokens_per_target = 400000
|
||||
gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" }
|
||||
gateway_target_requests_per_second = 90
|
||||
gateway_target_tokens_per_second = 6000000
|
||||
gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" }
|
||||
}
|
||||
|
||||
assert {
|
||||
|
|
@ -160,7 +168,7 @@ run "requests_and_tokens_scale_next_to_cpu_and_memory" {
|
|||
}
|
||||
|
||||
assert {
|
||||
condition = length(one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_minute"].metric_stat).metric).dimensions) == 0
|
||||
condition = length(one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).dimensions) == 0
|
||||
error_message = "Omitting dimensions must query the token metric without any."
|
||||
}
|
||||
}
|
||||
|
|
@ -169,10 +177,10 @@ run "workload_targets_are_ignored_when_autoscaling_is_off" {
|
|||
command = plan
|
||||
|
||||
variables {
|
||||
gateway_autoscaling_enabled = false
|
||||
gateway_requests_per_target = 600
|
||||
gateway_tokens_per_target = 400000
|
||||
gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" }
|
||||
gateway_autoscaling_enabled = false
|
||||
gateway_target_requests_per_second = 90
|
||||
gateway_target_tokens_per_second = 6000000
|
||||
gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" }
|
||||
}
|
||||
|
||||
assert {
|
||||
|
|
|
|||
|
|
@ -272,23 +272,26 @@ variable "gateway_memory_target" {
|
|||
default = 80
|
||||
}
|
||||
|
||||
variable "gateway_requests_per_target" {
|
||||
variable "gateway_target_requests_per_second" {
|
||||
description = <<-EOT
|
||||
Requests per minute one gateway task should serve. Adds an
|
||||
Requests per second one gateway task should serve. Adds an
|
||||
ALBRequestCountPerTarget target-tracking policy next to the CPU/memory
|
||||
ones (Application Auto Scaling follows whichever asks for more tasks).
|
||||
0 skips the policy.
|
||||
CloudWatch publishes that metric as a 1-minute count, so the policy
|
||||
targets 60x this value and ECS reacts on a ~1 minute cadence. 0 skips
|
||||
the policy.
|
||||
EOT
|
||||
type = number
|
||||
default = 0
|
||||
}
|
||||
|
||||
variable "gateway_tokens_per_target" {
|
||||
variable "gateway_target_tokens_per_second" {
|
||||
description = <<-EOT
|
||||
Tokens per minute one gateway task should serve. Adds a target-tracking
|
||||
policy on gateway_tokens_metric divided by the service's Container
|
||||
Insights RunningTaskCount. Tokens are counted when a response completes,
|
||||
so the signal trails long streams. 0 skips the policy.
|
||||
Tokens per second one gateway task should serve. Adds a target-tracking
|
||||
policy on gateway_tokens_metric summed over each 60s period, divided by
|
||||
60 and by the service's Container Insights RunningTaskCount. Tokens are
|
||||
counted when a response completes, so the signal trails long streams.
|
||||
0 skips the policy.
|
||||
EOT
|
||||
type = number
|
||||
default = 0
|
||||
|
|
@ -298,9 +301,9 @@ variable "gateway_tokens_metric" {
|
|||
description = <<-EOT
|
||||
CloudWatch metric carrying the gateway's litellm_total_tokens_metric_total
|
||||
counter, as published by the CloudWatch agent's Prometheus scraper (it
|
||||
emits the delta between scrapes, so Sum over a minute is tokens per
|
||||
minute). Required when gateway_tokens_per_target > 0. dimensions must
|
||||
match the metric_declaration the agent publishes with.
|
||||
emits the delta between scrapes, so Sum over a period is the tokens
|
||||
served in it). Required when gateway_target_tokens_per_second > 0.
|
||||
dimensions must match the metric_declaration the agent publishes with.
|
||||
EOT
|
||||
type = object({
|
||||
namespace = string
|
||||
|
|
|
|||
|
|
@ -249,10 +249,10 @@ request-rate signal for this stack: lower the concurrency for LLM streams
|
|||
that hold a worker for tens of seconds, since a stream counts as one request
|
||||
for as long as it is open
|
||||
|
||||
There is no tokens-per-minute path here. Cloud Run's autoscaler has no
|
||||
There is no tokens-per-second path here. Cloud Run's autoscaler has no
|
||||
custom-metric input, so the `litellm_total_tokens_metric_total` counter the
|
||||
proxy exposes cannot drive it. If you need token-based scaling on GCP, run
|
||||
the gateway on GKE with the Helm chart's `targetTokensPerMinute` (see
|
||||
the gateway on GKE with the Helm chart's `targetTokensPerSecond` (see
|
||||
"Dependencies only" below) rather than wiring the counter into Cloud
|
||||
Monitoring, which the autoscaler would ignore
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue