From b1fd20f4cdc8f7a9b4e0c886a277465aedb05ec8 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 1 Aug 2026 14:13:17 -0700 Subject: [PATCH] fix(helm): give gateway and backend probes an explicit timeoutSeconds (#35497) The gateway and backend probes omitted timeoutSeconds, so kubelet applied its 1s default. Both containers run a single uvicorn worker (the gateway defaults NUM_WORKERS to 1; the backend passes no --workers at all), so each pod is one asyncio event loop and its per-request latency under closed-loop saturation rises by queueing (~57-62ms serial vs ~6s at 100 concurrent users against one replica). Both /health/readiness and /health/liveliness then time out on the stage cluster while the pod is serving traffic correctly, which exposes the deployment to losing a healthy pod from its load balancer during a burst and to restarting a merely busy one. Readiness now gets timeoutSeconds 10, equal to periodSeconds and above the measured saturated latency, and keeps failureThreshold 3. kubelet drives each probe from a time.Ticker of periodSeconds rather than sleeping between attempts, and coalesces ticks that arrive mid-probe, so the interval between probe starts is max(periodSeconds, probeDuration) and not their sum. Keeping timeoutSeconds <= periodSeconds is what holds that interval at 10s, so three consecutive failures still evict a genuinely wedged pod in ~30s. Liveness gets the same timeout plus failureThreshold 6: /health/liveliness is an in-memory flag check, so a timeout there only ever means event-loop starvation, which a restart makes worse, and it now needs ~90s of sustained unresponsiveness to fire. The ui container keeps the default. It is nginx serving a Next.js static export, so / is a file off disk with no application runtime that could queue behind saturated work, and nothing measured suggests it needs more than 1s. --- helm/litellm/tests/probe_tests.yaml | 106 ++++++++++++++++++++++++++++ helm/litellm/values.yaml | 6 ++ 2 files changed, 112 insertions(+) create mode 100644 helm/litellm/tests/probe_tests.yaml diff --git a/helm/litellm/tests/probe_tests.yaml b/helm/litellm/tests/probe_tests.yaml new file mode 100644 index 00000000000..a04709db2f5 --- /dev/null +++ b/helm/litellm/tests/probe_tests.yaml @@ -0,0 +1,106 @@ +suite: test liveness and readiness probe timeouts +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - backend/deployment.yaml +values: + - ./values/required.yaml +tests: + - it: gateway probes set an explicit timeout that outlasts a saturated event loop + template: gateway/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe + value: + httpGet: + path: /health/liveliness + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 6 + - equal: + path: spec.template.spec.containers[0].readinessProbe + value: + httpGet: + path: /health/readiness + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 10 + + - it: backend probes set an explicit timeout that outlasts a saturated event loop + template: backend/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe + value: + httpGet: + path: /health/liveliness + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 6 + - equal: + path: spec.template.spec.containers[0].readinessProbe + value: + httpGet: + path: /health/readiness + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 10 + + - it: no single-event-loop component is left on the kubernetes default 1s probe timeout + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + asserts: + - isNotNullOrEmpty: + path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds + - isNotNullOrEmpty: + path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds + - equal: + path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds + value: 10 + - equal: + path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds + value: 10 + + - it: gateway liveness tolerates a longer outage than readiness before acting + template: gateway/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.failureThreshold + value: 6 + - notExists: + path: spec.template.spec.containers[0].readinessProbe.failureThreshold + + - it: probe timeouts and thresholds stay overridable per component + template: gateway/deployment.yaml + set: + gateway.readinessProbe.timeoutSeconds: 3 + gateway.readinessProbe.periodSeconds: 20 + gateway.livenessProbe.timeoutSeconds: 4 + gateway.livenessProbe.failureThreshold: 3 + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe + value: + httpGet: + path: /health/readiness + port: http + initialDelaySeconds: 5 + periodSeconds: 20 + timeoutSeconds: 3 + - equal: + path: spec.template.spec.containers[0].livenessProbe + value: + httpGet: + path: /health/liveliness + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 4 + failureThreshold: 3 diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 48e55a6805b..cd377667602 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -216,10 +216,13 @@ gateway: httpGet: { path: /health/liveliness, port: http } initialDelaySeconds: 10 periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 6 readinessProbe: httpGet: { path: /health/readiness, port: http } initialDelaySeconds: 5 periodSeconds: 10 + timeoutSeconds: 10 hpa: enabled: true minReplicas: 1 @@ -309,10 +312,13 @@ backend: httpGet: { path: /health/liveliness, port: http } initialDelaySeconds: 10 periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 6 readinessProbe: httpGet: { path: /health/readiness, port: http } initialDelaySeconds: 5 periodSeconds: 10 + timeoutSeconds: 10 hpa: enabled: true minReplicas: 1