From 5fdb0860ec51544def55bdd0a56e7472df5be3f9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:02:37 -0700 Subject: [PATCH] fix(helm): route /debug/memory/summary to the gateway so the memory gate reads the serving workers On the release gate the e2e tests only see the nginx router, and the chart's ingress sent /debug/memory/summary to the backend catch-all, so the RSS check measured the backend pod instead of the gateway workers that serve the failing requests. Render it as an Exact gateway path next to /test, name the host in the summary response so workers behind one origin never collide on pid alone, and key the harness readings by (origin, hostname, pid) --- helm/litellm/templates/ingress.yaml | 11 ++++++++++- .../tests/ingress_controller_tests.yaml | 10 ++++++++++ .../tests/ingress_extra_paths_tests.yaml | 11 +++++++++++ litellm/proxy/common_utils/debug_utils.py | 3 +++ tests/e2e/models.py | 4 +++- .../e2e/router/test_reliability_memory_e2e.py | 19 ++++++++++++------- .../proxy/common_utils/test_debug_utils.py | 16 ++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 1 + 8 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 tests/test_litellm/proxy/common_utils/test_debug_utils.py diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index 732564b280f..d42558b9396 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -89,7 +89,7 @@ at "/" Prefix would swallow the whole backend management API) instead of adding to it. */}} -{{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}} +{{- $builtinPathKeys := list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix" -}} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -129,6 +129,8 @@ spec: # --- Gateway data plane --- # Exact /test only (see the $gatewayPrefixes comment above); # /test/* MCP management endpoints fall to the backend catch-all. + # Exact /debug/memory/summary reads a serving worker's RSS (the e2e memory + # gate); the rest of /debug/* stays on the backend. - path: /test pathType: Exact backend: @@ -136,6 +138,13 @@ spec: name: {{ $gatewayName }} port: number: {{ $gatewayPort }} + - path: /debug/memory/summary + pathType: Exact + backend: + service: + name: {{ $gatewayName }} + port: + number: {{ $gatewayPort }} {{- range $gatewayPrefixes }} {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }} {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }} diff --git a/helm/litellm/tests/ingress_controller_tests.yaml b/helm/litellm/tests/ingress_controller_tests.yaml index 40790ba674a..aa30db3c9c1 100644 --- a/helm/litellm/tests/ingress_controller_tests.yaml +++ b/helm/litellm/tests/ingress_controller_tests.yaml @@ -97,6 +97,16 @@ tests: name: RELEASE-NAME-litellm-gateway port: number: 4000 + - contains: + path: spec.rules[0].http.paths + content: + path: /debug/memory/summary + pathType: Exact + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 - equal: path: spec.rules[0].http.paths[-1] value: diff --git a/helm/litellm/tests/ingress_extra_paths_tests.yaml b/helm/litellm/tests/ingress_extra_paths_tests.yaml index fc7d5943278..1305af15ae2 100644 --- a/helm/litellm/tests/ingress_extra_paths_tests.yaml +++ b/helm/litellm/tests/ingress_extra_paths_tests.yaml @@ -288,6 +288,17 @@ tests: - failedTemplate: errorMessage: "ingress.extraPaths[0]: path /test with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it" + - it: rejects an entry that would take over the exact /debug/memory/summary route + set: + ingress.enabled: true + ingress.extraPaths: + - path: /debug/memory/summary + pathType: Exact + service: backend + asserts: + - failedTemplate: + errorMessage: "ingress.extraPaths[0]: path /debug/memory/summary with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it" + - it: allows a built-in path under a different pathType, which is a distinct rule set: ingress.enabled: true diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index 554a6ae8d1a..7ea80166e9f 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -3,6 +3,7 @@ import asyncio import gc import json import os +import socket import sys import tracemalloc from collections import Counter @@ -241,6 +242,7 @@ async def get_memory_summary( Returns: - worker_pid: Process ID + - hostname: Host (the pod on Kubernetes) the worker runs on - status: Overall health based on memory usage - memory: Process memory usage and RAM info - caches: Cache item counts and descriptions @@ -340,6 +342,7 @@ async def get_memory_summary( return { "worker_pid": os.getpid(), + "hostname": socket.gethostname(), "status": health_status, "memory": process_memory, "caches": { diff --git a/tests/e2e/models.py b/tests/e2e/models.py index c7904966653..c36cc2909c6 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1282,9 +1282,11 @@ class ProcessMemory(BaseModel): class MemorySummaryResponse(BaseModel): """GET /debug/memory/summary (master key). One worker's resident memory, keyed by - its pid so readings behind a load balancer can be told apart per pod.""" + its hostname (the pod name on Kubernetes) and pid so readings behind a load + balancer can be told apart per worker; older proxies omit the hostname.""" worker_pid: int + hostname: str | None = None status: str memory: ProcessMemory diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/router/test_reliability_memory_e2e.py index 6a975722801..1c4114223d9 100644 --- a/tests/e2e/router/test_reliability_memory_e2e.py +++ b/tests/e2e/router/test_reliability_memory_e2e.py @@ -79,15 +79,19 @@ class FailedCall: call_id: str | None +WorkerKey = tuple[str, str | None, int] + + @dataclass(frozen=True, slots=True) class RssReading: replica: str + hostname: str | None worker_pid: int ram_usage_mb: float @property - def worker(self) -> tuple[str, int]: - return (self.replica, self.worker_pid) + def worker(self) -> WorkerKey: + return (self.replica, self.hostname, self.worker_pid) @dataclass(frozen=True, slots=True) @@ -135,14 +139,14 @@ def _fail_many(proxy: ProxyClient, key: str, model: str, override: RouterSetting def _read_rss_everywhere_after_pause(proxy: ProxyClient) -> tuple[RssReading, ...]: time.sleep(MEMORY_RSS_SAMPLE_INTERVAL_SECONDS) return tuple( - RssReading(replica, body.worker_pid, body.memory.ram_usage_mb) + RssReading(replica, body.hostname, body.worker_pid, body.memory.ram_usage_mb) for replica, result in proxy.memory_summary_everywhere().items() for body in (unwrap(result),) if body.memory.ram_usage_mb is not None ) -def _settled_rss_per_worker(proxy: ProxyClient) -> Mapping[tuple[str, int], RssReading]: +def _settled_rss_per_worker(proxy: ProxyClient) -> Mapping[WorkerKey, RssReading]: readings: Final = tuple( reading for _ in range(MEMORY_RSS_SETTLE_SAMPLES) for reading in _read_rss_everywhere_after_pause(proxy) ) @@ -156,7 +160,7 @@ def _settled_rss_per_worker(proxy: ProxyClient) -> Mapping[tuple[str, int], RssR def _heaviest_worker_growth( - warm: Mapping[tuple[str, int], RssReading], after: Mapping[tuple[str, int], RssReading] + warm: Mapping[WorkerKey, RssReading], after: Mapping[WorkerKey, RssReading] ) -> WorkerGrowth: growths: Final = tuple(WorkerGrowth(warm[worker], after[worker]) for worker in warm.keys() & after.keys()) assert growths, ( @@ -232,8 +236,9 @@ def test_failing_requests_do_not_grow_rss_or_stored_request( assert heaviest.growth_mb <= MEMORY_RSS_BUDGET_MB, ( f"proxy RSS grew {heaviest.growth_mb:.1f} MB over a second batch of {MEMORY_REQUESTS_PER_PHASE} failing " f"requests ({MEMORY_RETRIES_PER_REQUEST} retries each plus a fallback) after an identical warmup batch, " - f"past the {MEMORY_RSS_BUDGET_MB:.0f} MB budget: worker pid {heaviest.warm.worker_pid} at " - f"{heaviest.warm.replica} settled at {heaviest.warm.ram_usage_mb:.1f} MB warm and " + f"past the {MEMORY_RSS_BUDGET_MB:.0f} MB budget: worker pid {heaviest.warm.worker_pid} on " + f"{heaviest.warm.hostname or 'an unnamed host'} behind {heaviest.warm.replica} settled at " + f"{heaviest.warm.ram_usage_mb:.1f} MB warm and " f"{heaviest.after.ram_usage_mb:.1f} MB after; failing requests are leaking memory the way the " f"v1.100.0 retry breadcrumbs did" ) diff --git a/tests/test_litellm/proxy/common_utils/test_debug_utils.py b/tests/test_litellm/proxy/common_utils/test_debug_utils.py new file mode 100644 index 00000000000..728fe02981e --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_debug_utils.py @@ -0,0 +1,16 @@ +import os +import socket + +import pytest + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.debug_utils import get_memory_summary + + +@pytest.mark.asyncio +async def test_memory_summary_names_the_host_and_worker_that_answered() -> None: + summary = await get_memory_summary(UserAPIKeyAuth()) + + assert summary["hostname"] == socket.gethostname() + assert summary["worker_pid"] == os.getpid() + assert summary["memory"]["ram_usage_mb"] > 0 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6826cded6f5..494e3936772 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4136,6 +4136,7 @@ export interface paths { * * Returns: * - worker_pid: Process ID + * - hostname: Host (the pod on Kubernetes) the worker runs on * - status: Overall health based on memory usage * - memory: Process memory usage and RAM info * - caches: Cache item counts and descriptions