mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
6e90733ffc
* test(perf): gate the gateway's own overhead header, redis latency and postgres load in the perf cases * test(perf): keep the gateway_added_seconds limits next to the header ones --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
65 lines
2.8 KiB
Python
65 lines
2.8 KiB
Python
"""The gateway's own `/metrics`, read before and after a run for the numbers nothing outside the gateway
|
|
can see. Today that is `litellm_redis_latency`, the histogram customers' Grafana panels show when Redis
|
|
slows the gateway down. The gateway's latency and request counts are read per request from its response
|
|
headers instead (`setup/wire.py`), which gives percentiles where a histogram delta only gives a mean.
|
|
|
|
RSS and CPU are not here: a multi-worker gateway serves `/metrics` through the prometheus multiprocess
|
|
collector, which drops `process_resident_memory_bytes` and `process_cpu_seconds_total`, so those stay
|
|
on psutil (`proxy_usage.py`)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from collections.abc import Iterable
|
|
from dataclasses import dataclass
|
|
from itertools import chain
|
|
from typing import Final
|
|
|
|
from e2e_http import NoBody
|
|
from prometheus_client.parser import text_string_to_metric_families
|
|
from prometheus_client.samples import Sample
|
|
from proxy_client import ProxyClient
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class HistogramTotals:
|
|
"""`_sum` and `_count` of one histogram across every label set, enough for a mean over a window."""
|
|
|
|
total: float
|
|
count: float
|
|
|
|
def minus(self, earlier: HistogramTotals) -> HistogramTotals:
|
|
return HistogramTotals(total=self.total - earlier.total, count=self.count - earlier.count)
|
|
|
|
@property
|
|
def mean(self) -> float | None:
|
|
return self.total / self.count if self.count > 0 else None
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class GatewayTelemetry:
|
|
redis_latency: HistogramTotals
|
|
"""`litellm_redis_latency`, seconds per Redis call the gateway made."""
|
|
|
|
def minus(self, earlier: GatewayTelemetry) -> GatewayTelemetry:
|
|
return GatewayTelemetry(redis_latency=self.redis_latency.minus(earlier.redis_latency))
|
|
|
|
|
|
def _value(samples: Iterable[Sample], name: str) -> float:
|
|
return sum(sample.value for sample in samples if sample.name == name and math.isfinite(sample.value))
|
|
|
|
|
|
def _histogram(samples: tuple[Sample, ...], family: str) -> HistogramTotals:
|
|
return HistogramTotals(total=_value(samples, f"{family}_sum"), count=_value(samples, f"{family}_count"))
|
|
|
|
|
|
def parse_gateway_telemetry(text: str) -> GatewayTelemetry:
|
|
samples: Final = tuple(chain.from_iterable(family.samples for family in text_string_to_metric_families(text)))
|
|
return GatewayTelemetry(redis_latency=_histogram(samples, "litellm_redis_latency"))
|
|
|
|
|
|
def read_gateway_telemetry(proxy: ProxyClient) -> GatewayTelemetry | None:
|
|
"""One `/metrics` scrape, or None when the gateway does not serve it (no `prometheus` callback), in which
|
|
case every limit on a gateway-reported metric fails as missing rather than reading a silent zero."""
|
|
scrape: Final = proxy.probe("/metrics", params=NoBody())
|
|
return parse_gateway_telemetry(scrape.body) if scrape.status_code == 200 else None
|