mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-15 04:19:08 +00:00
fix(helm): meter the backend too, it serves the MCP transport
Scoping billingMetrics to the gateway was wrong. Applying each component's own
route allowlist to the proxy app shows the split is 75 billable routes on the
gateway and one on the backend: /{mcp_server_name}/mcp, the named-server MCP
transport, which writes a SpendLogs row on success. Metering only the gateway
would have silently dropped every MCP transport call from the counter, an
undercount proportional to a customer's MCP traffic.
The backend deployment now renders the same env and mounts the same read-only
cert secret. The migrations job still gets neither; it runs prisma and serves no
traffic, and a test pins that.
helm unittest -f 'tests/*.yaml' helm/litellm # 25 passed
helm unittest -f 'tests/*.yaml' helm/litellm-helm # 69 passed
This also aligns the chart with the terraform templates, which inject the
credentials into both components.
This commit is contained in:
parent
a5b0f7cf2c
commit
37c6bad289
4 changed files with 67 additions and 14 deletions
|
|
@ -44,10 +44,13 @@ helm.sh/chart: {{ include "litellm.chart" . }}
|
|||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Enterprise billable-request metering. Gateway only: the gateway is the component
|
||||
that serves billable traffic. The client certificate identifies the deployment to
|
||||
LiteLLM's collector, so it is mounted read-only from an existing Secret rather
|
||||
than passed through the environment.
|
||||
Enterprise billable-request metering. Wired into gateway and backend, not the
|
||||
migrations job. The gateway serves nearly all billable traffic, but the backend
|
||||
keeps the named-server MCP transport (/{mcp_server_name}/mcp), which writes a
|
||||
SpendLogs row, so metering only the gateway would silently drop that traffic.
|
||||
The client certificate identifies the deployment to LiteLLM's collector, so it is
|
||||
mounted read-only from an existing Secret rather than passed through the
|
||||
environment.
|
||||
*/}}
|
||||
{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
|
||||
{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
|
||||
|
|
|
|||
|
|
@ -44,14 +44,20 @@ spec:
|
|||
- name: CONFIG_FILE_PATH
|
||||
value: /app/config/config.yaml
|
||||
{{- end }}
|
||||
{{- if .Values.billingMetrics.enabled }}
|
||||
{{- include "litellm.billingMetricsEnv" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- include "litellm.envFrom" .Values.backend | nindent 10 }}
|
||||
{{- if or .Values.gateway.config.create .Values.backend.volumeMounts }}
|
||||
{{- if or .Values.gateway.config.create .Values.backend.volumeMounts .Values.billingMetrics.enabled }}
|
||||
volumeMounts:
|
||||
{{- if .Values.gateway.config.create }}
|
||||
- name: gateway-config
|
||||
mountPath: /app/config/config.yaml
|
||||
subPath: config.yaml
|
||||
{{- end }}
|
||||
{{- if .Values.billingMetrics.enabled }}
|
||||
{{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.backend.volumeMounts }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
|
|
@ -66,13 +72,16 @@ spec:
|
|||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.backend.resources | nindent 12 }}
|
||||
{{- if or .Values.gateway.config.create .Values.backend.volumes }}
|
||||
{{- if or .Values.gateway.config.create .Values.backend.volumes .Values.billingMetrics.enabled }}
|
||||
volumes:
|
||||
{{- if .Values.gateway.config.create }}
|
||||
- name: gateway-config
|
||||
configMap:
|
||||
name: {{ include "litellm.gateway.fullname" . }}-config
|
||||
{{- end }}
|
||||
{{- if .Values.billingMetrics.enabled }}
|
||||
{{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.backend.volumes }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
suite: test billingMetrics wiring on the gateway
|
||||
suite: test billingMetrics wiring on gateway and backend
|
||||
templates:
|
||||
- gateway/deployment.yaml
|
||||
- gateway/configmap.yaml
|
||||
- backend/deployment.yaml
|
||||
- migrations-job.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
tests:
|
||||
|
|
@ -158,9 +159,46 @@ tests:
|
|||
secret:
|
||||
secretName: billing-mtls
|
||||
|
||||
# Only the gateway serves billable traffic; the backend must not mount the cert.
|
||||
- it: does not touch the backend when enabled
|
||||
# The backend keeps the named-server MCP transport (/{mcp_server_name}/mcp),
|
||||
# which writes a SpendLogs row, so it must meter too or that traffic is lost.
|
||||
- it: meters the backend as well, since it serves the MCP transport
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
billingMetrics:
|
||||
enabled: true
|
||||
secretName: billing-mtls
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_BILLING_METRICS_ENDPOINT
|
||||
value: https://telemetry.litellm.ai
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
content:
|
||||
name: billing-metrics-mtls
|
||||
mountPath: /etc/litellm/billing-mtls
|
||||
readOnly: true
|
||||
- contains:
|
||||
path: spec.template.spec.volumes
|
||||
content:
|
||||
name: billing-metrics-mtls
|
||||
secret:
|
||||
secretName: billing-mtls
|
||||
|
||||
- it: leaves the backend alone when metering is off
|
||||
template: backend/deployment.yaml
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_BILLING_METRICS_ENDPOINT
|
||||
value: https://telemetry.litellm.ai
|
||||
|
||||
# The migrations job runs prisma and serves no traffic; it must never receive
|
||||
# the client key.
|
||||
- it: never mounts the billing cert on the migrations job
|
||||
template: migrations-job.yaml
|
||||
set:
|
||||
billingMetrics:
|
||||
enabled: true
|
||||
|
|
@ -171,6 +209,8 @@ tests:
|
|||
content:
|
||||
name: LITELLM_BILLING_METRICS_ENDPOINT
|
||||
value: https://telemetry.litellm.ai
|
||||
- isNull:
|
||||
path: spec.template.spec.volumes
|
||||
|
||||
- it: fails loudly when enabled without a secretName
|
||||
template: gateway/deployment.yaml
|
||||
|
|
|
|||
|
|
@ -73,11 +73,12 @@ masterKey:
|
|||
secretName: litellm-master-key-secret # name of a Secret containing the master key
|
||||
secretKey: master-key
|
||||
|
||||
# Optional: enterprise billable-request metering. When enabled, the gateway
|
||||
# counts successful requests to inference, MCP, and A2A endpoints and pushes
|
||||
# them to LiteLLM's collector over mutual TLS. Requires an enterprise license.
|
||||
# The client certificate identifies the deployment, so it is mounted read-only
|
||||
# from an existing Secret and never passed through the environment.
|
||||
# Optional: enterprise billable-request metering. When enabled, the gateway and
|
||||
# backend count successful requests to inference, MCP, and A2A endpoints and push
|
||||
# them to LiteLLM's collector over mutual TLS. Both components serve billable
|
||||
# routes: the backend keeps the named-server MCP transport. Requires an
|
||||
# enterprise license. The client certificate identifies the deployment, so it is
|
||||
# mounted read-only from an existing Secret and never passed through the env.
|
||||
billingMetrics:
|
||||
enabled: false
|
||||
endpoint: https://telemetry.litellm.ai # collector to push the counter to
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue