mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
9e18526887
* feat(deploy): expose SSE keepalive, pre-call checks and a metrics sidecar in Helm and Terraform Typed reliability values on both Helm charts and the AWS/GCP Terraform modules, a dedicated ClusterIP Service for the separate Prometheus port, a /health route on the metrics server and dead-worker pruning so the aggregate does not keep stale multiprocess samples. Resolves LIT-7142 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(deploy): drop reliability config from Helm and Terraform, keep only the metrics sidecar Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): cover startup pruning of dead workers' live gauges and unsignalable pids Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin <yassin@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
106 lines
3 KiB
YAML
106 lines
3 KiB
YAML
suite: separate metrics server
|
|
templates:
|
|
- configmap-litellm.yaml
|
|
- deployment.yaml
|
|
- service.yaml
|
|
- service-metrics.yaml
|
|
- servicemonitor.yaml
|
|
tests:
|
|
- it: should not expose a metrics port or PROMETHEUS_METRICS_PORT by default
|
|
asserts:
|
|
- notContains:
|
|
path: spec.template.spec.containers[0].ports
|
|
content:
|
|
name: metrics
|
|
any: true
|
|
template: deployment.yaml
|
|
- notContains:
|
|
path: spec.template.spec.containers[0].env
|
|
content:
|
|
name: PROMETHEUS_METRICS_PORT
|
|
any: true
|
|
template: deployment.yaml
|
|
- lengthEqual:
|
|
path: spec.ports
|
|
count: 1
|
|
template: service.yaml
|
|
- hasDocuments:
|
|
count: 0
|
|
template: service-metrics.yaml
|
|
|
|
- it: should scrape the proxy port when the metrics server is disabled
|
|
template: servicemonitor.yaml
|
|
set:
|
|
serviceMonitor.enabled: true
|
|
asserts:
|
|
- equal:
|
|
path: spec.endpoints[0].port
|
|
value: http
|
|
|
|
- it: should wire the separate metrics server through container, a ClusterIP metrics service and servicemonitor
|
|
set:
|
|
metricsServer.enabled: true
|
|
metricsServer.port: 4101
|
|
serviceMonitor.enabled: true
|
|
service.type: LoadBalancer
|
|
asserts:
|
|
- contains:
|
|
path: spec.template.spec.containers[0].env
|
|
content:
|
|
name: PROMETHEUS_METRICS_PORT
|
|
value: "4101"
|
|
template: deployment.yaml
|
|
- contains:
|
|
path: spec.template.spec.containers[0].ports
|
|
content:
|
|
name: metrics
|
|
containerPort: 4101
|
|
protocol: TCP
|
|
template: deployment.yaml
|
|
- lengthEqual:
|
|
path: spec.ports
|
|
count: 1
|
|
template: service.yaml
|
|
- equal:
|
|
path: spec.type
|
|
value: LoadBalancer
|
|
template: service.yaml
|
|
- equal:
|
|
path: metadata.name
|
|
value: RELEASE-NAME-litellm-metrics
|
|
template: service-metrics.yaml
|
|
- equal:
|
|
path: spec.type
|
|
value: ClusterIP
|
|
template: service-metrics.yaml
|
|
- equal:
|
|
path: spec.ports
|
|
value:
|
|
- port: 4101
|
|
targetPort: metrics
|
|
protocol: TCP
|
|
name: metrics
|
|
template: service-metrics.yaml
|
|
- equal:
|
|
path: spec.selector
|
|
value:
|
|
app.kubernetes.io/name: litellm
|
|
app.kubernetes.io/instance: RELEASE-NAME
|
|
template: service-metrics.yaml
|
|
- equal:
|
|
path: spec.endpoints[0].port
|
|
value: metrics
|
|
template: servicemonitor.yaml
|
|
- equal:
|
|
path: spec.endpoints[0].path
|
|
value: /metrics/
|
|
template: servicemonitor.yaml
|
|
|
|
- it: should reject a metrics port equal to the proxy port
|
|
template: deployment.yaml
|
|
set:
|
|
metricsServer.enabled: true
|
|
metricsServer.port: 4000
|
|
asserts:
|
|
- failedTemplate:
|
|
errorMessage: metricsServer.port must differ from service.port
|