scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

Merge branch 'litellm_internal_staging' into litellm_vertex_opus_global_passthrough

Browse source
This commit is contained in:
Mateo Wang 2026-07-16 14:51:29 -07:00 • committed by GitHub
commit 01cab1ad65
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
336 changed files with 13728 additions and 7884 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -5,6 +5,8 @@ on:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
permissions:
contents: read

6
.github/workflows/zizmor.yml vendored
View file

@ -4,7 +4,11 @@ on:
push:
branches: [main, litellm_internal_staging]
pull_request:
branches: [main, litellm_internal_staging]
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

2
docker/build_from_pip/Dockerfile.build_from_pip
View file

@ -36,7 +36,7 @@ RUN uv venv --python python && \
"opentelemetry-api==1.28.0" \
"opentelemetry-sdk==1.28.0" \
"opentelemetry-exporter-otlp==1.28.0" \
"ddtrace==2.19.0" \
"ddtrace==4.11.0" \
"sentry-sdk==2.21.0" \
"mangum==0.17.0" \
"azure-ai-contentsafety==1.0.0" \

8
enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py
View file

@ -113,6 +113,10 @@ class PagerDutyAlerting(SlackAlerting):
user_api_key_spend=_meta.get("user_api_key_spend"),
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
user_api_key_user_spend=_meta.get("user_api_key_user_spend"),
user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"),
user_api_key_team_spend=_meta.get("user_api_key_team_spend"),
user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"),
user_api_key_org_id=_meta.get("user_api_key_org_id"),
user_api_key_org_alias=_meta.get("user_api_key_org_alias"),
user_api_key_team_id=_meta.get("user_api_key_team_id"),
@ -196,6 +200,10 @@ class PagerDutyAlerting(SlackAlerting):
if user_api_key_dict.budget_reset_at
else None
),
user_api_key_user_spend=user_api_key_dict.user_spend,
user_api_key_user_max_budget=user_api_key_dict.user_max_budget,
user_api_key_team_spend=user_api_key_dict.team_spend,
user_api_key_team_max_budget=user_api_key_dict.team_max_budget,
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_org_alias=user_api_key_dict.organization_alias,
user_api_key_team_id=user_api_key_dict.team_id,

5
helm/litellm/templates/NOTES.txt
View file

@ -46,4 +46,9 @@ Reminders:
- gateway.config.proxy_config (rendered into a ConfigMap and mounted at
/app/config/config.yaml; gateway reads it via
CONFIG_FILE_PATH)
- {component}.pdb.{enabled,minAvailable,maxUnavailable} (per-component PodDisruptionBudget; disabled by
default — with hpa.minReplicas of 1, minAvailable: 1
would block node drains)
- {component}.topologySpreadConstraints (standard k8s list, e.g. spread replicas across
topology.kubernetes.io/zone)
- Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend.

46
helm/litellm/templates/_helpers.tpl
View file

@ -295,6 +295,52 @@ harmless no-op for the Job and authoritative for the app pods.
{{- end }}
{{- end -}}
{{/*
PodDisruptionBudget shared by gateway, backend, and ui.
Invoke with a dict:
(dict "root" $ "component" .Values.gateway "componentName" "gateway"
"fullname" (include "litellm.gateway.fullname" .)
"selectorLabels" (include "litellm.gateway.selectorLabels" .))
Renders nothing unless both the component and its `pdb.enabled` are on.
Only one of minAvailable / maxUnavailable should be set; if both are,
minAvailable wins. If neither is set, falls back to `maxUnavailable: 1` so
an enabled-but-unconfigured PDB still permits node drains.
"Set" means non-nil and non-empty-string, so an explicit 0 (e.g.
`maxUnavailable: 0` to forbid all voluntary disruptions) is honored rather
than silently replaced by the fallback.
*/}}
{{- define "litellm.pdb" -}}
{{- $root := .root -}}
{{- $component := .component -}}
{{- $min := $component.pdb.minAvailable -}}
{{- $max := $component.pdb.maxUnavailable -}}
{{- $minSet := not (or (kindIs "invalid" $min) (eq (printf "%v" $min) "")) -}}
{{- $maxSet := not (or (kindIs "invalid" $max) (eq (printf "%v" $max) "")) -}}
{{- if and $component.enabled $component.pdb $component.pdb.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ .fullname }}
labels:
{{- include "litellm.commonLabels" $root | nindent 4 }}
app.kubernetes.io/component: {{ .componentName }}
spec:
selector:
matchLabels:
{{- .selectorLabels | nindent 6 }}
{{- if $minSet }}
minAvailable: {{ $min }}
{{- else if $maxSet }}
maxUnavailable: {{ $max }}
{{- else }}
maxUnavailable: 1
{{- end }}
{{- end }}
{{- end -}}
{{/*
Renders `envFrom:` block for a component's `envConfigMaps` / `envSecrets`
lists. Each entry is a resource name; the chart wires the whole ConfigMap /

4
helm/litellm/templates/backend/deployment.yaml
View file

@ -98,4 +98,8 @@ spec:
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.backend.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

6
helm/litellm/templates/backend/poddisruptionbudget.yaml Normal file
View file

@ -0,0 +1,6 @@
{{- include "litellm.pdb" (dict
"root" $
"component" .Values.backend
"componentName" "backend"
"fullname" (include "litellm.backend.fullname" .)
"selectorLabels" (include "litellm.backend.selectorLabels" .)) }}

4
helm/litellm/templates/gateway/deployment.yaml
View file

@ -100,4 +100,8 @@ spec:
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

6
helm/litellm/templates/gateway/poddisruptionbudget.yaml Normal file
View file

@ -0,0 +1,6 @@
{{- include "litellm.pdb" (dict
"root" $
"component" .Values.gateway
"componentName" "gateway"
"fullname" (include "litellm.gateway.fullname" .)
"selectorLabels" (include "litellm.gateway.selectorLabels" .)) }}

4
helm/litellm/templates/ui/deployment.yaml
View file

@ -76,4 +76,8 @@ spec:
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.ui.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

6
helm/litellm/templates/ui/poddisruptionbudget.yaml Normal file
View file

@ -0,0 +1,6 @@
{{- include "litellm.pdb" (dict
"root" $
"component" .Values.ui
"componentName" "ui"
"fullname" (include "litellm.ui.fullname" .)
"selectorLabels" (include "litellm.ui.selectorLabels" .)) }}

188
helm/litellm/tests/pdb_topology_spread_tests.yaml Normal file
View file

@ -0,0 +1,188 @@
suite: test pod disruption budgets and topology spread constraints
templates:
- gateway/poddisruptionbudget.yaml
- backend/poddisruptionbudget.yaml
- ui/poddisruptionbudget.yaml
- gateway/deployment.yaml
- gateway/configmap.yaml
- backend/deployment.yaml
- ui/deployment.yaml
values:
- ./values/required.yaml
tests:
- it: renders no PDB by default
templates:
- gateway/poddisruptionbudget.yaml
- backend/poddisruptionbudget.yaml
- ui/poddisruptionbudget.yaml
asserts:
- hasDocuments:
count: 0
- it: gateway PDB uses minAvailable and matches the gateway selector labels
template: gateway/poddisruptionbudget.yaml
set:
gateway.pdb.enabled: true
gateway.pdb.minAvailable: 1
asserts:
- isKind:
of: PodDisruptionBudget
- equal:
path: apiVersion
value: policy/v1
- equal:
path: metadata.name
value: RELEASE-NAME-litellm-gateway
- equal:
path: spec.minAvailable
value: 1
- notExists:
path: spec.maxUnavailable
- equal:
path: spec.selector.matchLabels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: gateway
- it: backend PDB uses maxUnavailable when minAvailable is unset
template: backend/poddisruptionbudget.yaml
set:
backend.pdb.enabled: true
backend.pdb.maxUnavailable: 25%
asserts:
- equal:
path: spec.maxUnavailable
value: 25%
- notExists:
path: spec.minAvailable
- equal:
path: spec.selector.matchLabels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: backend
- it: minAvailable wins when both minAvailable and maxUnavailable are set
template: gateway/poddisruptionbudget.yaml
set:
gateway.pdb.enabled: true
gateway.pdb.minAvailable: 2
gateway.pdb.maxUnavailable: 1
asserts:
- equal:
path: spec.minAvailable
value: 2
- notExists:
path: spec.maxUnavailable
- it: an explicit maxUnavailable 0 is honored instead of the fallback
template: backend/poddisruptionbudget.yaml
set:
backend.pdb.enabled: true
backend.pdb.maxUnavailable: 0
asserts:
- equal:
path: spec.maxUnavailable
value: 0
- notExists:
path: spec.minAvailable
- it: an explicit minAvailable 0 is honored and beats a set maxUnavailable
template: gateway/poddisruptionbudget.yaml
set:
gateway.pdb.enabled: true
gateway.pdb.minAvailable: 0
gateway.pdb.maxUnavailable: 1
asserts:
- equal:
path: spec.minAvailable
value: 0
- notExists:
path: spec.maxUnavailable
- it: enabled PDB with neither knob set falls back to maxUnavailable 1
template: ui/poddisruptionbudget.yaml
set:
ui.pdb.enabled: true
asserts:
- equal:
path: spec.maxUnavailable
value: 1
- notExists:
path: spec.minAvailable
- equal:
path: spec.selector.matchLabels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: ui
- it: renders no PDB for a disabled component even when its pdb is enabled
template: gateway/poddisruptionbudget.yaml
set:
gateway.enabled: false
gateway.pdb.enabled: true
asserts:
- hasDocuments:
count: 0
- it: deployments omit topologySpreadConstraints by default
templates:
- gateway/deployment.yaml
- backend/deployment.yaml
- ui/deployment.yaml
asserts:
- notExists:
path: spec.template.spec.topologySpreadConstraints
- it: gateway deployment renders configured topologySpreadConstraints
template: gateway/deployment.yaml
set:
gateway.topologySpreadConstraints:
- maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app.kubernetes.io/component: gateway
asserts:
- equal:
path: spec.template.spec.topologySpreadConstraints
value:
- maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app.kubernetes.io/component: gateway
- it: backend deployment renders configured topologySpreadConstraints
template: backend/deployment.yaml
set:
backend.topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/component: backend
asserts:
- equal:
path: spec.template.spec.topologySpreadConstraints[0].topologyKey
value: kubernetes.io/hostname
- equal:
path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable
value: DoNotSchedule
- it: ui deployment renders configured topologySpreadConstraints
template: ui/deployment.yaml
set:
ui.topologySpreadConstraints:
- maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
asserts:
- equal:
path: spec.template.spec.topologySpreadConstraints[0].topologyKey
value: topology.kubernetes.io/zone

32
helm/litellm/values.yaml
View file

@ -190,10 +190,28 @@ gateway:
maxReplicas: 10
targetCPUUtilizationPercentage: 70
targetMemoryUtilizationPercentage: 80
# PodDisruptionBudget for the gateway pods. Set exactly one of
# `minAvailable` / `maxUnavailable` (minAvailable wins if both are set;
# enabling without either falls back to `maxUnavailable: 1`). Disabled by
# default: with the default hpa.minReplicas of 1, a `minAvailable: 1` PDB
# would block node drains entirely.
pdb:
enabled: false
minAvailable: ""
maxUnavailable: ""
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
# Standard k8s topologySpreadConstraints for the gateway pods, e.g. to
# spread replicas across zones:
# - maxSkew: 1
# topologyKey: topology.kubernetes.io/zone
# whenUnsatisfiable: ScheduleAnyway
# labelSelector:
# matchLabels:
# app.kubernetes.io/component: gateway
topologySpreadConstraints: []
# ---------- backend (UI / management API) ----------
backend:
@ -233,10 +251,17 @@ backend:
minReplicas: 1
maxReplicas: 4
targetCPUUtilizationPercentage: 70
# Same shape as gateway.pdb.
pdb:
enabled: false
minAvailable: ""
maxUnavailable: ""
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
# Same shape as gateway.topologySpreadConstraints.
topologySpreadConstraints: []
# ---------- ui (Next.js static dashboard) ----------
ui:
@ -279,7 +304,14 @@ ui:
minReplicas: 1
maxReplicas: 3
targetCPUUtilizationPercentage: 80
# Same shape as gateway.pdb.
pdb:
enabled: false
minAvailable: ""
maxUnavailable: ""
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
# Same shape as gateway.topologySpreadConstraints.
topologySpreadConstraints: []

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260715000000_add_issuer_to_mcp_server_table/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "issuer" TEXT;

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -325,6 +325,7 @@ model LiteLLM_MCPServerTable {
command String?
args String[] @default([])
env Json? @default("{}")
issuer String?
authorization_url String?
token_url String?
registration_url String?

6
litellm/_redis.py
View file

@ -688,10 +688,8 @@ def get_redis_connection_pool(
elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(redis_connect_func._gcp_service_account)
connection_class = async_redis.Connection
if redis_kwargs.pop("ssl", False):
connection_class = async_redis.SSLConnection
redis_kwargs["connection_class"] = connection_class
if redis_kwargs.pop("ssl", None):
redis_kwargs["connection_class"] = async_redis.SSLConnection
return async_redis.BlockingConnectionPool(timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs)

1
litellm/constants.py
View file

@ -1496,6 +1496,7 @@ MAX_TEAM_LIST_LIMIT = int(os.getenv("MAX_TEAM_LIST_LIMIT", 20))
MAX_POLICY_ESTIMATE_IMPACT_ROWS = int(os.getenv("MAX_POLICY_ESTIMATE_IMPACT_ROWS", 1000))
DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7))
LENGTH_OF_LITELLM_GENERATED_KEY = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16))
MINIMUM_CUSTOM_KEY_LENGTH = int(os.getenv("MINIMUM_CUSTOM_KEY_LENGTH", 16))
SECRET_MANAGER_REFRESH_INTERVAL = int(os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400))
LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
"default_internal_user_params",

4
litellm/exceptions.py
View file

@ -966,11 +966,15 @@ class BudgetExceededError(Exception):
max_budget: float,
message: Optional[str] = None,
llm_provider: Optional[str] = None,
entity_type: Optional[str] = None,
entity_id: Optional[str] = None,
):
self.current_cost = current_cost
self.max_budget = max_budget
self.status_code = 429
self.llm_provider = llm_provider or ""
self.entity_type = entity_type
self.entity_id = entity_id
# Surface unified rate-limit fields without joining the RateLimitError
# hierarchy so existing `except BudgetExceededError:` handlers keep
# working; custom callbacks reading StandardLoggingPayload pick these

79
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -267,29 +267,10 @@ class LangfuseOtelLogger(OpenTelemetry):
# If no keys, return default from env (likely logging to console or something else)
return OpenTelemetryConfig.from_env()
# Determine endpoint - default to US cloud
langfuse_host = LangfuseOtelLogger._get_langfuse_otel_host()
if langfuse_host:
# If LANGFUSE_HOST is provided, construct OTEL endpoint from it
if not langfuse_host.startswith("http"):
langfuse_host = "https://" + langfuse_host
endpoint = f"{langfuse_host.rstrip('/')}/api/public/otel"
verbose_logger.debug(f"Using Langfuse OTEL endpoint from host: {endpoint}")
else:
# Default to US cloud endpoint
endpoint = LANGFUSE_CLOUD_US_ENDPOINT
verbose_logger.debug(f"Using Langfuse US cloud endpoint: {endpoint}")
auth_header = LangfuseOtelLogger._get_langfuse_authorization_header(
public_key=public_key, secret_key=secret_key
)
otlp_auth_headers = f"Authorization={auth_header}"
return OpenTelemetryConfig(
exporter="otlp_http",
endpoint=endpoint,
headers=otlp_auth_headers,
return LangfuseOtelLogger._build_langfuse_otel_config(
public_key=public_key,
secret_key=secret_key,
langfuse_host=LangfuseOtelLogger._get_langfuse_otel_host(),
)
@staticmethod
@ -316,33 +297,36 @@ class LangfuseOtelLogger(OpenTelemetry):
"LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY must be set for Langfuse OpenTelemetry integration."
)
# Determine endpoint - default to US cloud
langfuse_host = LangfuseOtelLogger._get_langfuse_otel_host()
return LangfuseOtelLogger._build_langfuse_otel_config(
public_key=public_key,
secret_key=secret_key,
langfuse_host=LangfuseOtelLogger._get_langfuse_otel_host(),
)
@staticmethod
def _build_langfuse_otel_config(
public_key: str, secret_key: str, langfuse_host: Optional[str]
) -> "OpenTelemetryConfig":
"""
Builds an OTLP HTTP config pointing at the Langfuse OTEL endpoint for the
given host (US cloud when no host is provided), authorized with the given keys.
"""
if langfuse_host:
# If LANGFUSE_HOST is provided, construct OTEL endpoint from it
if not langfuse_host.startswith("http"):
langfuse_host = "https://" + langfuse_host
endpoint = f"{langfuse_host.rstrip('/')}/api/public/otel"
normalized_host = langfuse_host if langfuse_host.startswith("http") else f"https://{langfuse_host}"
endpoint = f"{normalized_host.rstrip('/')}/api/public/otel"
verbose_logger.debug(f"Using Langfuse OTEL endpoint from host: {endpoint}")
else:
# Default to US cloud endpoint
endpoint = LANGFUSE_CLOUD_US_ENDPOINT
verbose_logger.debug(f"Using Langfuse US cloud endpoint: {endpoint}")
auth_header = LangfuseOtelLogger._get_langfuse_authorization_header(
public_key=public_key, secret_key=secret_key
)
otlp_auth_headers = f"Authorization={auth_header}"
# Prevent modification of global env vars which causes leakage
# os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
# os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers
return OpenTelemetryConfig(
exporter="otlp_http",
endpoint=endpoint,
headers=otlp_auth_headers,
headers=f"Authorization={auth_header}",
)
@staticmethod
@ -378,6 +362,29 @@ class LangfuseOtelLogger(OpenTelemetry):
return dynamic_headers
def construct_dynamic_otel_config(
self, standard_callback_dynamic_params: StandardCallbackDynamicParams
) -> Optional["OpenTelemetryConfig"]:
"""
Build a full per-request OTLP config from team/key dynamic Langfuse credentials.
Key-scoped credentials must define the export target, not just the auth
headers: without this, a proxy with no global LANGFUSE_* env vars keeps its
init-time fallback exporter (console), so key-level langfuse_otel silently
never reaches Langfuse.
"""
public_key = standard_callback_dynamic_params.get("langfuse_public_key")
secret_key = standard_callback_dynamic_params.get("langfuse_secret_key")
if not public_key or not secret_key:
return None
langfuse_host = standard_callback_dynamic_params.get("langfuse_host") or self._get_langfuse_otel_host()
return LangfuseOtelLogger._build_langfuse_otel_config(
public_key=public_key,
secret_key=secret_key,
langfuse_host=langfuse_host,
)
def create_litellm_proxy_request_started_span(
self,
start_time: datetime,

111
litellm/integrations/opentelemetry.py
View file

@ -28,6 +28,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
parse_semconv_opt_in,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.secret_managers.main import get_secret_bool, str_to_bool
from litellm.types.services import ServiceLoggerPayload
from litellm.types.utils import (
@ -948,12 +949,22 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
Returns:
Tracer: The tracer to use for this request
"""
dynamic_config = self._get_dynamic_otel_config_from_kwargs(kwargs)
if dynamic_config is not None:
verbose_logger.debug(
"[OTEL DEBUG] Using DYNAMIC config tracer with endpoint: %s",
dynamic_config.endpoint,
)
return self._get_tracer_with_dynamic_config(dynamic_config)
dynamic_headers = self._get_dynamic_otel_headers_from_kwargs(kwargs)
if dynamic_headers is not None:
# Create spans using a temporary tracer with dynamic headers
tracer_to_use = self._get_tracer_with_dynamic_headers(dynamic_headers)
verbose_logger.debug("[OTEL DEBUG] Using DYNAMIC tracer with headers: %s", dynamic_headers)
verbose_logger.debug(
"[OTEL DEBUG] Using DYNAMIC tracer with headers: %s", redact_string(str(dynamic_headers))
)
else:
# For langfuse_otel without dynamic headers, create a provider with env var credentials
if hasattr(self, "callback_name") and self.callback_name == "langfuse_otel":
@ -989,6 +1000,32 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
return dynamic_headers if dynamic_headers else None
def _get_dynamic_otel_config_from_kwargs(self, kwargs: dict) -> Optional[OpenTelemetryConfig]:
"""Extract a full dynamic exporter config from kwargs if available."""
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = kwargs.get(
"standard_callback_dynamic_params"
)
if not standard_callback_dynamic_params:
return None
return self.construct_dynamic_otel_config(standard_callback_dynamic_params=standard_callback_dynamic_params)
def _get_tracer_with_dynamic_config(self, dynamic_config: OpenTelemetryConfig):
"""Create (or reuse) a tracer whose exporter target comes from a per-request config."""
from opentelemetry.sdk.trace import TracerProvider
cache_key = f"dynamic_config:{dynamic_config.exporter}:{dynamic_config.endpoint}:{dynamic_config.headers}"
if cache_key in self._tracer_provider_cache:
return self._tracer_provider_cache[cache_key].get_tracer(LITELLM_TRACER_NAME)
temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config))
temp_provider.add_span_processor(self._get_span_processor(config_override=dynamic_config))
self._tracer_provider_cache[cache_key] = temp_provider
return temp_provider.get_tracer(LITELLM_TRACER_NAME)
def _get_tracer_with_dynamic_headers(self, dynamic_headers: dict):
"""Create a temporary tracer with dynamic headers for this request only."""
from opentelemetry.sdk.trace import TracerProvider
@ -1020,6 +1057,19 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
"""
return None
def construct_dynamic_otel_config(
self, standard_callback_dynamic_params: StandardCallbackDynamicParams
) -> Optional[OpenTelemetryConfig]:
"""
Construct a full exporter config from standard callback dynamic params.
Override this when team/key dynamic params must control the export
target (exporter kind + endpoint), not just the request headers. When
this returns a config, it takes precedence over
construct_dynamic_otel_headers for the request.
"""
return None
#########################################################
# End of Team/Key Based Logging Control Flow
#########################################################
@ -2747,7 +2797,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
verbose_logger.debug("OpenTelemetry: No parent context found, creating root span")
return None, None
def _get_span_processor(self, dynamic_headers: Optional[dict] = None):
def _get_span_processor(
self,
dynamic_headers: Optional[dict] = None,
config_override: Optional[OpenTelemetryConfig] = None,
):
from opentelemetry.sdk.trace.export import (
BatchSpanProcessor,
ConsoleSpanExporter,
@ -2755,40 +2809,45 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
SpanExporter,
)
otel_exporter = config_override.exporter if config_override else self.OTEL_EXPORTER
otel_endpoint = config_override.endpoint if config_override else self.OTEL_ENDPOINT
otel_headers = config_override.headers if config_override else self.OTEL_HEADERS
verbose_logger.debug(
"OpenTelemetry Logger, initializing span processor \nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s",
self.OTEL_EXPORTER,
self.OTEL_ENDPOINT,
self.OTEL_HEADERS,
"OpenTelemetry Logger, initializing span processor \nexporter: %s\nendpoint: %s\nheaders: %s",
otel_exporter,
otel_endpoint,
redact_string(str(otel_headers)),
)
_split_otel_headers = OpenTelemetry._get_headers_dictionary(headers=dynamic_headers or self.OTEL_HEADERS)
_split_otel_headers = OpenTelemetry._get_headers_dictionary(headers=dynamic_headers or otel_headers)
if dynamic_headers:
verbose_logger.debug(
"[OTEL DEBUG] Creating span processor with DYNAMIC headers: %s",
{k: v[:20] + "..." if len(str(v)) > 20 else v for k, v in _split_otel_headers.items()},
redact_string(str(_split_otel_headers)),
)
elif config_override:
verbose_logger.debug(
"[OTEL DEBUG] Creating span processor with DYNAMIC config, endpoint: %s",
otel_endpoint,
)
else:
verbose_logger.debug("[OTEL DEBUG] Creating span processor with GLOBAL headers")
if hasattr(self.OTEL_EXPORTER, "export"): # Check if it has the export method that SpanExporter requires
if hasattr(otel_exporter, "export"): # Check if it has the export method that SpanExporter requires
verbose_logger.debug(
"OpenTelemetry: intiializing SpanExporter. Value of OTEL_EXPORTER: %s",
self.OTEL_EXPORTER,
otel_exporter,
)
return SimpleSpanProcessor(cast(SpanExporter, self.OTEL_EXPORTER))
return SimpleSpanProcessor(cast(SpanExporter, otel_exporter))
if self.OTEL_EXPORTER == "console":
if otel_exporter == "console":
verbose_logger.debug(
"OpenTelemetry: intiializing console exporter. Value of OTEL_EXPORTER: %s",
self.OTEL_EXPORTER,
otel_exporter,
)
return BatchSpanProcessor(ConsoleSpanExporter())
elif (
self.OTEL_EXPORTER == "otlp_http"
or self.OTEL_EXPORTER == "http/protobuf"
or self.OTEL_EXPORTER == "http/json"
):
elif otel_exporter == "otlp_http" or otel_exporter == "http/protobuf" or otel_exporter == "http/json":
try:
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter as OTLPSpanExporterHTTP,
@ -2801,13 +2860,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
verbose_logger.debug(
"OpenTelemetry: intiializing http exporter. Value of OTEL_EXPORTER: %s",
self.OTEL_EXPORTER,
otel_exporter,
)
normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "traces")
normalized_endpoint = self._normalize_otel_endpoint(otel_endpoint, "traces")
return BatchSpanProcessor(
OTLPSpanExporterHTTP(endpoint=normalized_endpoint, headers=_split_otel_headers),
)
elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc":
elif otel_exporter == "otlp_grpc" or otel_exporter == "grpc":
try:
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
OTLPSpanExporter as OTLPSpanExporterGRPC,
@ -2820,16 +2879,16 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
verbose_logger.debug(
"OpenTelemetry: intiializing grpc exporter. Value of OTEL_EXPORTER: %s",
self.OTEL_EXPORTER,
otel_exporter,
)
normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "traces")
normalized_endpoint = self._normalize_otel_endpoint(otel_endpoint, "traces")
return BatchSpanProcessor(
OTLPSpanExporterGRPC(endpoint=normalized_endpoint, headers=_split_otel_headers),
)
else:
verbose_logger.debug(
"OpenTelemetry: intiializing console exporter. Value of OTEL_EXPORTER: %s",
self.OTEL_EXPORTER,
otel_exporter,
)
return BatchSpanProcessor(ConsoleSpanExporter())
@ -2841,7 +2900,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
"OpenTelemetry Logger, initializing log exporter \nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s",
self.OTEL_EXPORTER,
self.OTEL_ENDPOINT,
self.OTEL_HEADERS,
redact_string(str(self.OTEL_HEADERS)),
)
_split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS)
@ -2928,7 +2987,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
"OpenTelemetry Logger, initializing metric reader\nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s",
self.OTEL_EXPORTER,
self.OTEL_ENDPOINT,
self.OTEL_HEADERS,
redact_string(str(self.OTEL_HEADERS)),
)
_split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS)

2
litellm/litellm_core_utils/dd_tracing.py
View file

@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union
from litellm.secret_managers.main import get_secret_bool
if TYPE_CHECKING:
from ddtrace.tracer import Tracer as DD_TRACER
from ddtrace.trace import Tracer as DD_TRACER
else:
DD_TRACER = Any

18
litellm/litellm_core_utils/litellm_logging.py
View file

@ -38,6 +38,7 @@ from litellm import (
)
from litellm._logging import _is_debugging_on, _redact_string, verbose_logger
from litellm.exceptions import (
BudgetExceededError,
validate_rate_limit_category,
validate_rate_limit_type,
)
@ -4597,6 +4598,10 @@ class StandardLoggingPayloadSetup:
user_api_key_spend=None,
user_api_key_max_budget=None,
user_api_key_budget_reset_at=None,
user_api_key_user_spend=None,
user_api_key_user_max_budget=None,
user_api_key_team_spend=None,
user_api_key_team_max_budget=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_org_alias=None,
@ -4943,6 +4948,7 @@ class StandardLoggingPayloadSetup:
rate_limit_category = validate_rate_limit_category(getattr(original_exception, "category", None))
rate_limit_type = validate_rate_limit_type(getattr(original_exception, "rate_limit_type", None))
budget_error = original_exception if isinstance(original_exception, BudgetExceededError) else None
return StandardLoggingPayloadErrorInformation(
error_code=error_status,
@ -4952,6 +4958,10 @@ class StandardLoggingPayloadSetup:
error_message=error_message,
error_rate_limit_category=rate_limit_category,
error_rate_limit_type=rate_limit_type,
error_budget_entity_type=budget_error.entity_type if budget_error else None,
error_budget_entity_id=budget_error.entity_id if budget_error else None,
error_budget_limit=budget_error.max_budget if budget_error else None,
error_budget_spend=budget_error.current_cost if budget_error else None,
)
@staticmethod
@ -5428,6 +5438,10 @@ def get_standard_logging_metadata(
user_api_key_spend=None,
user_api_key_max_budget=None,
user_api_key_budget_reset_at=None,
user_api_key_user_spend=None,
user_api_key_user_max_budget=None,
user_api_key_team_spend=None,
user_api_key_team_max_budget=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_org_alias=None,
@ -5527,6 +5541,10 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
user_api_key_team_id=str("test_team"),
user_api_key_user_id=str("test_user"),
user_api_key_team_alias=str("test_team_alias"),
user_api_key_user_spend=None,
user_api_key_user_max_budget=None,
user_api_key_team_spend=None,
user_api_key_team_max_budget=None,
user_api_key_org_id=None,
spend_logs_metadata=None,
requester_ip_address=str("127.0.0.1"),

4
litellm/litellm_core_utils/secret_redaction.py
View file

@ -9,6 +9,8 @@ secrets from strings without depending on the logging-configuration module.
import re
from typing import List
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
_REDACTED = "REDACTED"
@ -30,7 +32,7 @@ def _build_secret_patterns() -> "re.Pattern[str]":
# Basic auth headers
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
# OpenAI / Anthropic sk- prefixed keys
r"sk-[A-Za-z0-9\-_]{20,}",
rf"sk-[A-Za-z0-9\-_]{{{MINIMUM_CUSTOM_KEY_LENGTH - len('sk-')},}}",
# Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
# x-api-key / api-key header values (handles 'key': 'value' dict repr)

40
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -467,6 +467,7 @@ class ChunkProcessor:
cache_read_input_tokens: Optional[int] = None
completion_tokens_details: Optional[CompletionTokensDetails] = None
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
cost: Optional[float] = None
if "prompt_tokens" in usage_chunk:
prompt_tokens = usage_chunk.get("prompt_tokens", 0) or 0
@ -476,6 +477,8 @@ class ChunkProcessor:
cache_creation_input_tokens = usage_chunk.get("cache_creation_input_tokens")
if "cache_read_input_tokens" in usage_chunk:
cache_read_input_tokens = usage_chunk.get("cache_read_input_tokens")
if "cost" in usage_chunk:
cost = usage_chunk.get("cost")
if hasattr(usage_chunk, "completion_tokens_details"):
if isinstance(usage_chunk.completion_tokens_details, dict):
completion_tokens_details = CompletionTokensDetails(**usage_chunk.completion_tokens_details)
@ -494,6 +497,7 @@ class ChunkProcessor:
"cache_read_input_tokens": cache_read_input_tokens,
"completion_tokens_details": completion_tokens_details,
"prompt_tokens_details": prompt_tokens_details,
"cost": cost,
}
def count_reasoning_tokens(self, response: ModelResponse) -> Optional[int]:
@ -512,6 +516,22 @@ class ChunkProcessor:
return reasoning_tokens
@staticmethod
def _extract_usage_chunk(chunk: dict[str, Any] | ModelResponse | ModelResponseStream) -> Usage | None:
usage_chunk: Usage | dict[str, Any] | None = None
if hasattr(chunk, "usage") and chunk.usage is not None:
usage_chunk = chunk.usage
elif "usage" in chunk:
usage_chunk = chunk["usage"]
elif (isinstance(chunk, ModelResponse) or isinstance(chunk, ModelResponseStream)) and hasattr(
chunk, "_hidden_params"
):
usage_chunk = chunk._hidden_params.get("usage", None)
if isinstance(usage_chunk, dict):
return Usage(**usage_chunk)
return usage_chunk
def _calculate_usage_per_chunk(
self,
chunks: List[Union[Dict[str, Any], ModelResponse]],
@ -548,18 +568,12 @@ class ChunkProcessor:
# is last-wins, so without preserving this separately the 1h breakdown is
# lost and 1h cache writes get billed at the 5m rate.
cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
cost: Optional[float] = None
for chunk in chunks:
usage_chunk: Optional[Usage] = None
if "usage" in chunk:
usage_chunk = chunk["usage"]
elif (isinstance(chunk, ModelResponse) or isinstance(chunk, ModelResponseStream)) and hasattr(
chunk, "_hidden_params"
):
usage_chunk = chunk._hidden_params.get("usage", None)
usage_chunk = self._extract_usage_chunk(chunk)
if usage_chunk is not None:
if isinstance(usage_chunk, dict):
usage_chunk = Usage(**usage_chunk)
usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk)
if usage_chunk_dict["prompt_tokens"] is not None and usage_chunk_dict["prompt_tokens"] > 0:
prompt_tokens = usage_chunk_dict["prompt_tokens"]
@ -610,6 +624,9 @@ class ChunkProcessor:
prompt_tokens_details, cache_creation_token_details
)
if usage_chunk_dict["cost"] is not None:
cost = usage_chunk_dict["cost"]
prompt_tokens_details = self._attach_cache_creation_token_details(
prompt_tokens_details, cache_creation_token_details
)
@ -629,6 +646,7 @@ class ChunkProcessor:
web_search_requests=web_search_requests,
completion_tokens_details=completion_tokens_details,
prompt_tokens_details=prompt_tokens_details,
cost=cost,
)
@staticmethod
@ -727,6 +745,7 @@ class ChunkProcessor:
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = calculated_usage_per_chunk[
"prompt_tokens_details"
]
cost: Optional[float] = calculated_usage_per_chunk["cost"]
try:
returned_usage.prompt_tokens = prompt_tokens or token_counter(model=model, messages=messages)
@ -784,6 +803,9 @@ class ChunkProcessor:
else:
returned_usage.prompt_tokens_details.web_search_requests = web_search_requests
if cost is not None:
setattr(returned_usage, "cost", cost)
# Return a new usage object with the new values
returned_usage = Usage(**returned_usage.model_dump())

275
litellm/litellm_core_utils/streaming_handler.py
View file

@ -962,10 +962,11 @@ class CustomStreamWrapper:
if self.custom_llm_provider == "bedrock" and "trace" in model_response:
return model_response
# Default - return StopIteration
if hasattr(model_response, "usage"):
self.chunks.append(model_response)
raise StopIteration
# Don't raise StopIteration here - some providers (like OpenRouter)
# send usage/cost data in chunks after the finish_reason chunk
if hasattr(model_response, "usage") and model_response.usage is not None:
return model_response
return
# flush any remaining holding chunk
if len(self.holding_chunk) > 0:
if model_response.choices[0].delta.content is None:
@ -1474,12 +1475,16 @@ class CustomStreamWrapper:
self.tool_call = True
if hasattr(chunk, "usage") and chunk.usage is not None:
model_response.usage = chunk.usage
## RETURN ARG
return self.return_processed_chunk_logic(
result = self.return_processed_chunk_logic(
completion_obj=completion_obj,
model_response=model_response, # type: ignore
response_obj=response_obj,
)
return result
except StopIteration:
raise StopIteration
@ -1686,6 +1691,21 @@ class CustomStreamWrapper:
model_response.choices[0].finish_reason = "tool_calls"
return model_response
@staticmethod
def _propagate_usage_cost_to_hidden_params(
response: "ModelResponse",
) -> None:
"""
If the assembled response carries a provider-reported cost on
usage.cost, copy it into _hidden_params so litellm's cost
calculator uses it instead of a token-based estimate.
"""
_usage = getattr(response, "usage", None)
if _usage is not None and hasattr(_usage, "cost") and _usage.cost is not None:
if "additional_headers" not in response._hidden_params:
response._hidden_params["additional_headers"] = {}
response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(_usage.cost)
def __next__(self) -> "ModelResponseStream":
cache_hit = False
if self.custom_llm_provider is not None and self.custom_llm_provider == "cached_response":
@ -1741,6 +1761,10 @@ class CustomStreamWrapper:
# hasattr(response, "usage") is always True — must check
# `is not None` to avoid running this path on every chunk.
if getattr(response, "usage", None) is not None:
usage_to_preserve = response.usage
if usage_to_preserve:
response._hidden_params["usage"] = usage_to_preserve
obj_dict = response.model_dump()
if "usage" in obj_dict:
@ -1789,6 +1813,8 @@ class CustomStreamWrapper:
response = self.model_response_creator()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response)
setattr(
response,
"usage",
@ -1974,97 +2000,7 @@ class CustomStreamWrapper:
self.chunks.append(processed_chunk)
return processed_chunk
except (StopAsyncIteration, StopIteration):
if self.sent_last_chunk is True:
# log the final chunk with accurate streaming values
try:
complete_streaming_response = litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages,
logging_obj=self.logging_obj,
)
except Exception as e:
# see sync __next__: a raise from stream_chunk_builder inside this
# except handler escapes __anext__ and drops the request from SpendLogs.
# Recover best-effort usage from the raw chunks so cost is still tracked
verbose_logger.warning(
"stream_chunk_builder raised at end-of-stream (%s); logging best-effort usage from chunks.",
str(e),
)
try:
complete_streaming_response = self.model_response_creator(
chunk={"usage": calculate_total_usage(chunks=self.chunks)}
)
except Exception:
complete_streaming_response = None
response = self.model_response_creator()
if complete_streaming_response is not None:
setattr(
response,
"usage",
getattr(complete_streaming_response, "usage"),
)
try:
_copy = complete_streaming_response.model_copy(deep=True)
except RuntimeError:
_copy = complete_streaming_response.model_copy()
asyncio.create_task(
self.async_cache_streaming_response(
processed_chunk=_copy,
cache_hit=cache_hit,
)
)
# Update hidden_params with final usage from
# stream_chunk_builder (see sync __next__ for full comment).
if (
self.stream_options is None
and complete_streaming_response is not None
and self._last_returned_hidden_params is not None
):
final_usage = getattr(complete_streaming_response, "usage", None)
if final_usage is not None:
self._last_returned_hidden_params["usage"] = final_usage
if self.sent_stream_usage is False and self.send_stream_usage is True:
self.sent_stream_usage = True
return response
_deferred_cb = getattr(
self.logging_obj,
"_on_deferred_stream_complete",
None,
)
if _deferred_cb is not None:
# Proxy has post-call guardrails. Store the assembled
# response so the outer streaming consumer
# (ProxyLogging.async_post_call_streaming_iterator_hook)
# can fire the deferred callback AFTER all guardrail
# end-of-stream blocks complete. Scheduling here via
# create_task would race with unified_guardrail's
# end-of-stream block for short-stream providers.
self.logging_obj._deferred_stream_complete_args = ( # type: ignore[attr-defined]
complete_streaming_response,
cache_hit,
)
else:
# prefer_async_handlers routes CustomLogger to async_success_handler
# when consumers use ``async for`` on sync-SDK streams. Legacy string
# callbacks still run via executor.submit inside dispatch_success_handlers.
asyncio.create_task(
self.logging_obj.dispatch_success_handlers(
complete_streaming_response,
cache_hit=cache_hit,
start_time=None,
end_time=None,
prefer_async_handlers=True,
)
)
raise StopAsyncIteration # Re-raise StopIteration
else:
self.sent_last_chunk = True
processed_chunk = self.finish_reason_handler()
return processed_chunk
return await self._finalize_completed_stream(cache_hit=cache_hit)
except httpx.TimeoutException as e: # if httpx read timeout error occues
traceback_exception = traceback.format_exc()
## ADD DEBUG INFORMATION - E.G. LITELLM REQUEST TIMEOUT
@ -2079,20 +2015,122 @@ class CustomStreamWrapper:
# Handle any exceptions that might occur during streaming
asyncio.create_task(self.logging_obj.async_failure_handler(e, traceback_exception))
self._handle_stream_fallback_error(e)
except (httpx.ReadError, httpx.RemoteProtocolError) as e:
if self.received_finish_reason is None:
self._log_stream_failure_and_raise(e)
return await self._finalize_completed_stream(cache_hit=cache_hit)
except Exception as e:
traceback_exception = traceback.format_exc()
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
threading.Thread(
target=self.logging_obj.failure_handler,
args=(e, traceback_exception),
).start() # log response
# Handle any exceptions that might occur during streaming
asyncio.create_task(
self.logging_obj.async_failure_handler(e, traceback_exception) # type: ignore
self._log_stream_failure_and_raise(e)
async def _finalize_completed_stream(self, cache_hit: bool) -> "ModelResponseStream":
if self.sent_last_chunk is True:
# log the final chunk with accurate streaming values
try:
complete_streaming_response = litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages,
logging_obj=self.logging_obj,
)
self._handle_stream_fallback_error(e)
except Exception as e:
# see sync __next__: a raise from stream_chunk_builder inside this
# except handler escapes __anext__ and drops the request from SpendLogs.
# Recover best-effort usage from the raw chunks so cost is still tracked
verbose_logger.warning(
"stream_chunk_builder raised at end-of-stream (%s); logging best-effort usage from chunks.",
str(e),
)
try:
complete_streaming_response = self.model_response_creator(
chunk={"usage": calculate_total_usage(chunks=self.chunks)}
)
except Exception:
complete_streaming_response = None
response = self.model_response_creator()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response)
setattr(
response,
"usage",
getattr(complete_streaming_response, "usage"),
)
try:
_copy = complete_streaming_response.model_copy(deep=True)
except RuntimeError:
_copy = complete_streaming_response.model_copy()
asyncio.create_task(
self.async_cache_streaming_response(
processed_chunk=_copy,
cache_hit=cache_hit,
)
)
# Update hidden_params with final usage from
# stream_chunk_builder (see sync __next__ for full comment).
if (
self.stream_options is None
and complete_streaming_response is not None
and self._last_returned_hidden_params is not None
):
final_usage = getattr(complete_streaming_response, "usage", None)
if final_usage is not None:
self._last_returned_hidden_params["usage"] = final_usage
if self.sent_stream_usage is False and self.send_stream_usage is True:
self.sent_stream_usage = True
return response
_deferred_cb = getattr(
self.logging_obj,
"_on_deferred_stream_complete",
None,
)
if _deferred_cb is not None:
# Proxy has post-call guardrails. Store the assembled
# response so the outer streaming consumer
# (ProxyLogging.async_post_call_streaming_iterator_hook)
# can fire the deferred callback AFTER all guardrail
# end-of-stream blocks complete. Scheduling here via
# create_task would race with unified_guardrail's
# end-of-stream block for short-stream providers.
self.logging_obj._deferred_stream_complete_args = ( # type: ignore[attr-defined]
complete_streaming_response,
cache_hit,
)
else:
# prefer_async_handlers routes CustomLogger to async_success_handler
# when consumers use ``async for`` on sync-SDK streams. Legacy string
# callbacks still run via executor.submit inside dispatch_success_handlers.
asyncio.create_task(
self.logging_obj.dispatch_success_handlers(
complete_streaming_response,
cache_hit=cache_hit,
start_time=None,
end_time=None,
prefer_async_handlers=True,
)
)
raise StopAsyncIteration # Re-raise StopIteration
else:
self.sent_last_chunk = True
processed_chunk = self.finish_reason_handler()
return processed_chunk
def _log_stream_failure_and_raise(self, e: Exception) -> NoReturn:
traceback_exception = traceback.format_exc()
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
threading.Thread(
target=self.logging_obj.failure_handler,
args=(e, traceback_exception),
).start() # log response
# Handle any exceptions that might occur during streaming
asyncio.create_task(
self.logging_obj.async_failure_handler(e, traceback_exception) # type: ignore
)
self._handle_stream_fallback_error(e)
def _record_partial_usage_for_failure(self) -> None:
"""
@ -2228,12 +2266,16 @@ def calculate_total_usage(chunks: List[ModelResponse]) -> Usage:
"""Assume most recent usage chunk has total usage uptil then."""
prompt_tokens: int = 0
completion_tokens: int = 0
latest_usage_chunk = None
for chunk in chunks:
if "usage" in chunk and chunk["usage"] is not None:
if "prompt_tokens" in chunk["usage"]:
prompt_tokens = chunk["usage"].get("prompt_tokens", 0) or 0
if "completion_tokens" in chunk["usage"]:
completion_tokens = chunk["usage"].get("completion_tokens", 0) or 0
usage = chunk["usage"]
latest_usage_chunk = usage
if "prompt_tokens" in usage:
prompt_tokens = usage.get("prompt_tokens", 0) or 0
if "completion_tokens" in usage:
completion_tokens = usage.get("completion_tokens", 0) or 0
returned_usage_chunk = Usage(
prompt_tokens=prompt_tokens,
@ -2241,6 +2283,15 @@ def calculate_total_usage(chunks: List[ModelResponse]) -> Usage:
total_tokens=prompt_tokens + completion_tokens,
)
if latest_usage_chunk is not None:
latest_cost = (
latest_usage_chunk.get("cost")
if isinstance(latest_usage_chunk, dict)
else getattr(latest_usage_chunk, "cost", None)
)
if latest_cost is not None:
returned_usage_chunk.cost = latest_cost
return returned_usage_chunk

12
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -1403,11 +1403,6 @@ class LiteLLMAnthropicMessagesAdapter:
assert isinstance(thinking, str)
assert isinstance(signature, str)
if thinking and signature:
raise ValueError(
"Both `thinking` and `signature` in a single streaming chunk isn't supported."
)
return "thinking", ChatCompletionThinkingBlock(
type="thinking", thinking=thinking, signature=signature
)
@ -1463,17 +1458,14 @@ class LiteLLMAnthropicMessagesAdapter:
if choice.delta.reasoning_content is not None:
reasoning_content += choice.delta.reasoning_content
if reasoning_content and reasoning_signature:
raise ValueError("Both `reasoning` and `signature` in a single streaming chunk isn't supported.")
if partial_json is not None:
return "input_json_delta", ContentJsonBlockDelta(type="input_json_delta", partial_json=partial_json)
elif reasoning_content:
return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
elif reasoning_signature:
return "signature_delta", ContentThinkingSignatureBlockDelta(
type="signature_delta", signature=reasoning_signature
)
elif reasoning_content:
return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
else:
return "text_delta", ContentTextBlockDelta(type="text_delta", text=text)

25
litellm/llms/custom_httpx/aiohttp_transport.py
View file

@ -85,29 +85,12 @@ class AiohttpResponseStream(httpx.AsyncByteStream):
try:
async for chunk in self._aiohttp_response.content.iter_chunked(self.CHUNK_SIZE):
yield chunk
except (
aiohttp.ClientPayloadError,
aiohttp.client_exceptions.ClientPayloadError,
) as e:
# Handle incomplete transfers more gracefully
# Log the error but don't re-raise if we've already yielded some data
verbose_logger.debug(f"Transfer incomplete, but continuing: {e}")
# If the error is due to incomplete transfer encoding, we can still
# return what we've received so far, similar to how httpx handles it
return
except RuntimeError as e:
# Some providers (e.g., SSE streams) may close the connection
# causing aiohttp StreamReader to raise a generic RuntimeError
# with message "Connection closed.". Treat this as a graceful
# end-of-stream so downstream consumers don't error.
if "Connection closed" in str(e):
verbose_logger.debug("Upstream closed streaming connection; ending iterator gracefully")
return
raise
if "Connection closed" not in str(e):
raise
raise httpx.ReadError(str(e)) from e
except aiohttp.http_exceptions.TransferEncodingError as e:
# Handle transfer encoding errors gracefully
verbose_logger.debug(f"Transfer encoding error, but continuing: {e}")
return
raise httpx.ReadError(str(e)) from e
except Exception:
# For other exceptions, use the normal mapping
with map_aiohttp_exceptions():

1
litellm/models/mcp_server.py
View file

@ -79,6 +79,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
command: Optional[str] = None
args: List[str] = Field(default_factory=list)
env: Dict[str, str] = Field(default_factory=dict)
issuer: Optional[str] = None
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None

36
litellm/proxy/_experimental/mcp_server/db.py
View file

@ -48,6 +48,7 @@ if TYPE_CHECKING:
_AUTH_FLOW_SCOPED_FIELDS: frozenset = frozenset(
{
"issuer",
"authorization_url",
"token_url",
"registration_url",
@ -60,6 +61,13 @@ _AUTH_FLOW_SCOPED_FIELDS: frozenset = frozenset(
}
)
def _blank_to_none(value: Optional[str]) -> Optional[str]:
if not isinstance(value, str):
return None
return value.strip() or None
# Token-exchange settings with dedicated columns that also exist on
# ``MCPCredentials`` as a legacy shape (rows and REST callers that predate the
# columns). Every write lifts blob values into the columns and strips them from
@ -697,13 +705,15 @@ async def update_mcp_server(
# of being reset to a schema default (transport=sse, allow_all_keys=False...).
data_dict = _prepare_mcp_server_data(data, exclude_unset=True, fields_set=fields_set)
# Pre-fetch existing record once if we need it for auth_type or credential logic
# Pre-fetch existing record once if we need it for auth_type, url, or credential logic
existing = None
has_credentials = "credentials" in data_dict and data_dict["credentials"] is not None
# An explicit token-exchange column write (set or clear) also migrates the
# legacy blob copies below, so the existing row is needed for those updates.
explicit_te_write = bool(_TOKEN_EXCHANGE_COLUMN_FIELDS & data_dict.keys())
if data.auth_type or has_credentials or explicit_te_write:
url_provided = "url" in data_dict and data_dict["url"] is not None
issuer_provided = "issuer" in data_dict
if data.auth_type or has_credentials or explicit_te_write or url_provided or issuer_provided:
existing = await MCPServerRepository(prisma_client).table.find_unique(where={"server_id": data.server_id})
auth_type_changed = bool(
@ -711,13 +721,30 @@ async def update_mcp_server(
and existing
and _credential_auth_class(existing.auth_type) != _credential_auth_class(data.auth_type)
)
# A url change re-points the server at a potentially different upstream, so any discovered or
# trust-on-first-use OAuth endpoints/issuer belong to the old upstream and must re-discover.
url_changed = bool(url_provided and existing and existing.url != data_dict["url"])
old_issuer = _blank_to_none(getattr(existing, "issuer", None)) if existing else None
issuer_changed = bool(
issuer_provided and old_issuer is not None and _blank_to_none(data_dict.get("issuer")) != old_issuer
)
# Clear stale credentials when auth_type changes but no new credentials provided
if auth_type_changed and "credentials" not in data_dict:
data_dict["credentials"] = None
if auth_type_changed:
data_dict.update({field: None for field in _AUTH_FLOW_SCOPED_FIELDS if field not in data_dict})
if auth_type_changed or url_changed or issuer_changed:
# Clear each auth-flow-scoped field that the caller either omitted (partial update) or
# resubmitted unchanged. The edit form re-sends every field, so a stale issuer/endpoint
# belonging to the old upstream would otherwise survive a url/auth_type change and win in the
# resolution merge; only a genuinely new submitted value is kept.
data_dict.update(
{
field: None
for field in _AUTH_FLOW_SCOPED_FIELDS
if field not in data_dict or data_dict[field] == getattr(existing, field, None)
}
)
# An explicit column write that does not touch credentials must still migrate
# the row's legacy blob copies: lift values for columns the caller left
@ -1181,6 +1208,7 @@ def mcp_oauth_token_identity(server: object) -> tuple[object, ...]:
getattr(server, "spec_path", None),
getattr(server, "auth_type", None),
getattr(server, "oauth2_flow", None),
getattr(server, "issuer", None),
getattr(server, "authorization_url", None),
getattr(server, "token_url", None),
getattr(server, "registration_url", None),

442
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -186,6 +186,104 @@ _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES: tuple[MCPAuth, ...] = (
)
def _blank_to_none(value: str | None) -> str | None:
"""Collapse an absent, empty, or whitespace-only string to ``None``.
OAuth endpoint fields are consumed by truthiness-based merges (``row or discovered``) and by the
corroboration gate. A whitespace-only value is truthy to ``or`` but is not a usable endpoint, so
without this the merge would keep the blank value for redirects while the gate treats it as
unpinned and backfills the other fields, yielding a broken half-discovered config. Normalizing
the pinned fields once, at each build entry point, gives every downstream consumer a single
notion of "blank" so those code paths cannot disagree.
"""
if not isinstance(value, str):
return None
return value.strip() or None
def _uses_issuer_anchor(manual_issuer: str | None, is_discovery_auth_type: bool) -> bool:
"""Whether the endpoints are authoritatively anchored to an admin-pinned issuer (RFC 8414 §3.3).
This is the trust/provenance property, distinct from whether the ``issuer`` field is merely
populated: a trust-on-first-use discovered issuer sets ``issuer`` for token identity but is NOT
anchored, so its endpoints stay resource-rooted. Anchoring holds only when the issuer was pinned
(present on the row/config) on a discovery auth type. Every consumer of "is this anchored" reads
this one definition, so the answer cannot diverge across build paths.
"""
return _blank_to_none(manual_issuer) is not None and is_discovery_auth_type
def _endpoints_yield_to_issuer(
issuer: str | None,
is_discovery_auth_type: bool,
authorization_url: str | None,
token_url: str | None,
registration_url: str | None,
) -> tuple[str | None, str | None, str | None]:
"""The single rule that makes an admin-configured ``issuer`` the sole authoritative endpoint
source (RFC 8414 §3.3): when it is set for a discovery auth type, the stored/manual
``authorization_url``/``token_url``/``registration_url`` do not apply. They neither anchor nor
short-circuit discovery, never override the issuer document in the merge, and never substitute for
it when the issuer fetch fails (fail-closed). Returns the endpoint values that remain in force,
i.e. all ``None`` when issuer-anchored, else the inputs unchanged. Called at every resolution site
so the invariant holds in one place instead of being re-derived per merge.
"""
if issuer is not None and is_discovery_auth_type:
return None, None, None
return authorization_url, token_url, registration_url
def _normalized_authorize_endpoint(url: str) -> str:
"""Compare authorize endpoints on scheme, host, and path only. The default port is elided and
the host is lowercased so ``https://IDP.example.com:443/authorize/`` and
``https://idp.example.com/authorize`` are the same identity; query and trailing slash are not."""
parsed = urlparse(url)
scheme = parsed.scheme.lower()
host = (parsed.hostname or "").lower()
default_port = {"https": 443, "http": 80}.get(scheme)
try:
port = parsed.port
except ValueError:
port = None
authority = host if port is None or port == default_port else f"{host}:{port}"
return f"{scheme}://{authority}{parsed.path.rstrip('/')}"
def _issuer_matches(claimed_issuer: object, configured_issuer: str) -> bool:
"""RFC 8414 §3.3 issuer equality between the metadata document's self-attested ``issuer`` and the
admin-configured issuer, tolerant only of URL-insignificant differences (scheme/host case, the
default port, a trailing slash). A non-string or empty claimed issuer never matches, so a
document that omits ``issuer`` fails closed under issuer-anchored discovery.
"""
if not isinstance(claimed_issuer, str) or not claimed_issuer:
return False
return _normalized_authorize_endpoint(claimed_issuer) == _normalized_authorize_endpoint(configured_issuer)
def _endpoints_corroborate_authorization_url(
source_authorization_url: str | None,
trusted_authorization_url: str | None,
) -> bool:
"""Whether a source's ``token_url``/``registration_url`` may be paired with a trusted authorize
endpoint. This is the single trust rule for adopting OAuth endpoints from any non-manual source.
Discovery is rooted at the MCP resource (RFC 9728), so a compromised upstream can advertise an
attacker-run authorization server. When ``authorization_url`` is admin-pinned, pairing it with a
``token_url`` from a different source is the RFC 9700 authorization-server mix-up: the user signs
in at the trusted authorize endpoint while the gateway redeems the code, with the stored client
secret and PKCE verifier, at the attacker's token endpoint. Endpoints are trustworthy together
only when they share an authorization server, so a source's endpoints are adopted only when the
same source advertised an ``authorization_endpoint`` matching the pinned value. With no pinned
value (``trusted_authorization_url is None``) there is nothing to protect: the authorize endpoint
comes from the same source as the token endpoint, so they corroborate each other by construction.
"""
if not (trusted_authorization_url and trusted_authorization_url.strip()):
return True
return bool(source_authorization_url) and _normalized_authorize_endpoint(
source_authorization_url
) == _normalized_authorize_endpoint(trusted_authorization_url)
def _carry_forward_resolved_oauth_endpoints(new_server: MCPServer, previous_server: MCPServer | None) -> None:
"""Keep the last known good OAuth endpoints when a rebuild's re-discovery comes back empty.
@ -193,26 +291,98 @@ def _carry_forward_resolved_oauth_endpoints(new_server: MCPServer, previous_serv
during re-discovery downgrades a working server (``authorization_url`` set) to a broken one
(``None``, /authorize 400s) with no configuration change. Mirrors the ``short_prefix``
carry-forward. Skipped when the server's ``url`` or ``auth_type`` changed, since the previous
endpoints may then belong to a different upstream. ``registration_url`` IS carried here even
though ``_persist_discovered_oauth_endpoints`` refuses to write it to the row: carrying only
restores the same in-memory value the previous build already ran with, while persisting it
would flip ``_dcr_bridge_relays_client_registration`` (which keys off the stored column) for
dcr_bridge servers that never had one configured.
endpoints may then belong to a different upstream. ``registration_url`` IS carried even though
``_persist_discovered_oauth_endpoints`` refuses to write it to the row: carrying only restores
the same in-memory value the previous build already ran with, while persisting it would flip
``_dcr_bridge_relays_client_registration`` (which keys off the stored column) for dcr_bridge
servers that never had one configured.
Carry-forward is a non-manual endpoint source, so the same trust rule as discovery applies: the
previous ``token_url``/``registration_url``/``scopes`` are carried only when the previous
``authorization_url`` corroborates the authorize endpoint this build will use, i.e. when the
incoming build has no pinned authorize endpoint (``None`` -> we adopt the previous one too, a
consistent group) or pins the same one. An admin re-pointing ``authorization_url`` to a different
server must not keep serving the old server's token endpoint or granted scopes.
When the server is issuer-anchored (``issuer_is_anchored`` -- a pinned issuer on a discovery auth
type), the endpoints come solely from the §3.3-validated issuer document, so carry-forward is
skipped entirely for its endpoints: a failed issuer fetch leaves them ``None`` and must stay
``None`` (fail-closed), never resurrected from the previous registry entry. A merely discovered
(trust-on-first-use) issuer is NOT anchored -- ``issuer`` is set for token identity but the
endpoints are resource-rooted, so they still carry forward as last-known-good, gated by the
corroboration check below like any other resource-rooted server. Scopes stay resource-driven and
can carry either way.
"""
if previous_server is None:
return
if previous_server.url != new_server.url or previous_server.auth_type != new_server.auth_type:
return
if new_server.issuer_is_anchored:
# Endpoints come solely from the §3.3-validated issuer document; a failed fetch stays
# fail-closed and must not be resurrected from the previous entry. Only the resource-driven
# scopes carry as last-known-good.
if not new_server.scopes and previous_server.scopes:
new_server.scopes = previous_server.scopes
return
may_carry = _endpoints_corroborate_authorization_url(
previous_server.authorization_url, new_server.authorization_url
)
if new_server.authorization_url is None and previous_server.authorization_url:
new_server.authorization_url = previous_server.authorization_url
if new_server.token_url is None and previous_server.token_url:
if may_carry and new_server.token_url is None and previous_server.token_url:
new_server.token_url = previous_server.token_url
if new_server.registration_url is None and previous_server.registration_url:
if may_carry and new_server.registration_url is None and previous_server.registration_url:
new_server.registration_url = previous_server.registration_url
if not new_server.scopes and previous_server.scopes:
if may_carry and not new_server.scopes and previous_server.scopes:
new_server.scopes = previous_server.scopes
def _restrict_discovery_to_corroborated_authorization_server(
metadata: MCPOAuthMetadata | None,
manual_authorization_url: str | None,
server_identifier: str,
is_dcr_bridge: bool,
) -> MCPOAuthMetadata | None:
"""Reject discovered token/registration endpoints a manually pinned authorize endpoint cannot
vouch for (the RFC 9700 authorization-server mix-up).
Discovery is rooted at the MCP resource, so a compromised upstream can advertise an attacker
``token_endpoint``: with ``authorization_url`` admin-pinned but ``token_url`` blank, the merge
would pair the trusted authorize endpoint with that attacker token endpoint, and the gateway would
post the authorization code and client secret there. So the discovered ``token_url`` and
``registration_url`` are kept only if the document corroborates the pin (its
``authorization_endpoint`` matches). ``scopes`` are deliberately NOT gated here: per the MCP
authorization spec Scope Selection Strategy and RFC 9700 §2.3, the scopes a client requests are
resource-driven (the WWW-Authenticate challenge or the RFC 9728 protected-resource
``scopes_supported``), and scope inflation by a compromised resource is bounded by the
authorization server and user consent (RFC 6749 §3.3), not by the client second-guessing the
request. With no pin there is no trust anchor to protect, so discovery is returned as-is.
"""
if metadata is None or not (manual_authorization_url and manual_authorization_url.strip()):
return metadata
if _endpoints_corroborate_authorization_url(metadata.authorization_url, manual_authorization_url):
return metadata
if not metadata.token_url and not metadata.registration_url:
return metadata
bridge_note = (
" The discovered registration_url is rejected with it, so this dcr_bridge server stays on the"
" short-circuit registration arm."
if is_dcr_bridge and metadata.registration_url
else ""
)
verbose_logger.warning(
"MCP OAuth discovery for server %s advertised authorization_endpoint %s, which does not match the "
"manually configured authorization_url %s; rejecting the discovered token_url/registration_url so "
"authorization codes and client credentials only follow the configured authorization server. "
"Configure Token URL manually if the mismatch is intentional.%s",
server_identifier,
_normalized_authorize_endpoint(metadata.authorization_url) if metadata.authorization_url else "<absent>",
_normalized_authorize_endpoint(manual_authorization_url),
bridge_note,
)
return metadata.model_copy(update={"token_url": None, "registration_url": None})
def invalidate_user_env_vars_cache(user_id: str, server_id: str) -> None:
"""Drop a cached entry after the user stores or clears their env var values
so the next request reads the fresh value instead of a stale one."""
@ -1026,36 +1196,68 @@ class MCPServerManager:
)
auth_type = server_config.get("auth_type", None)
if server_url and (
auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
manual_issuer = _blank_to_none(server_config.get("issuer"))
manual_authorization_url = _blank_to_none(server_config.get("authorization_url"))
manual_token_url = _blank_to_none(server_config.get("token_url"))
manual_registration_url = _blank_to_none(server_config.get("registration_url"))
is_discovery_auth_type = auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type)
manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer(
manual_issuer,
is_discovery_auth_type,
manual_authorization_url,
manual_token_url,
manual_registration_url,
)
should_discover = bool(server_url) and (
is_discovery_auth_type
or self._obo_needs_endpoint_discovery(
auth_type,
server_config.get("token_exchange_endpoint"),
server_config.get("token_url"),
manual_token_url,
)
):
)
if not should_discover:
mcp_oauth_metadata = None
elif manual_issuer is not None and is_discovery_auth_type:
mcp_oauth_metadata = await self._fetch_issuer_anchored_oauth_metadata(manual_issuer, server_url)
else:
mcp_oauth_metadata = await self._descovery_metadata(
server_url=server_url,
allow_origin_fallback=auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES,
allow_origin_fallback=is_discovery_auth_type,
)
if use_issuer_anchor:
gated_oauth_metadata = mcp_oauth_metadata
elif is_discovery_auth_type:
gated_oauth_metadata = _restrict_discovery_to_corroborated_authorization_server(
mcp_oauth_metadata,
manual_authorization_url,
server_name or server_id,
bool(server_config.get("dcr_bridge")),
)
else:
mcp_oauth_metadata = None
gated_oauth_metadata = mcp_oauth_metadata
# Filter blank scopes (e.g. YAML ``scopes: [""]``) the same way the DB-build path does, so
# an all-blank list normalizes to None rather than a ``("",)`` tuple that skips the
# entra_obo fail-closed scope precondition and POSTs an empty scope to the IdP.
resolved_scopes = self._extract_scopes(server_config.get("scopes")) or (
mcp_oauth_metadata.scopes if mcp_oauth_metadata else None
gated_oauth_metadata.scopes if gated_oauth_metadata else None
)
resolved_authorization_url = server_config.get("authorization_url") or (
mcp_oauth_metadata.authorization_url if mcp_oauth_metadata else None
resolved_authorization_url = manual_authorization_url or (
gated_oauth_metadata.authorization_url if gated_oauth_metadata else None
)
resolved_token_url = server_config.get("token_url") or (
mcp_oauth_metadata.token_url if mcp_oauth_metadata else None
resolved_token_url = manual_token_url or (gated_oauth_metadata.token_url if gated_oauth_metadata else None)
resolved_registration_url = manual_registration_url or (
gated_oauth_metadata.registration_url if gated_oauth_metadata else None
)
resolved_registration_url = server_config.get("registration_url") or (
mcp_oauth_metadata.registration_url if mcp_oauth_metadata else None
discovered_issuer = (
gated_oauth_metadata.discovered_issuer
if gated_oauth_metadata and not gated_oauth_metadata.from_origin_fallback
else None
)
effective_issuer = manual_issuer or discovered_issuer
config_oauth2_flow = server_config.get("oauth2_flow", None)
if auth_type == MCPAuth.oauth2 and config_oauth2_flow not in (
@ -1104,6 +1306,8 @@ class MCPServerManager:
client_secret=server_config.get("client_secret", None),
oauth2_flow=self._explicit_oauth2_flow(config_oauth2_flow),
scopes=resolved_scopes,
issuer=effective_issuer,
issuer_is_anchored=use_issuer_anchor,
authorization_url=resolved_authorization_url,
token_url=resolved_token_url,
registration_url=resolved_registration_url,
@ -1364,6 +1568,52 @@ class MCPServerManager:
decrypt_global_env_var_values(env_vars_list)
return env_vars_list
async def _resolve_table_oauth_metadata(
self,
*,
mcp_server: LiteLLM_MCPServerTable,
auth_type: MCPAuthType,
server_url: Optional[str],
manual_issuer: Optional[str],
manual_authorization_url: Optional[str],
manual_token_url: Optional[str],
is_discovery_auth_type: bool,
use_issuer_anchor: bool,
scopes: Optional[list[str]],
token_exchange_endpoint: Optional[str],
) -> Optional[MCPOAuthMetadata]:
has_all_upstream_oauth_fields = bool(manual_authorization_url and manual_token_url and scopes)
needs_discovery = bool(server_url) and (
(is_discovery_auth_type and not has_all_upstream_oauth_fields)
or self._obo_needs_endpoint_discovery(auth_type, token_exchange_endpoint, manual_token_url)
)
if not needs_discovery:
mcp_oauth_metadata: Optional[MCPOAuthMetadata] = None
elif use_issuer_anchor and manual_issuer is not None:
mcp_oauth_metadata = await self._fetch_issuer_anchored_oauth_metadata(manual_issuer, server_url)
else:
mcp_oauth_metadata = await self._descovery_metadata(
server_url=server_url, # type: ignore[arg-type]
allow_origin_fallback=is_discovery_auth_type,
)
if needs_discovery and not use_issuer_anchor and mcp_oauth_metadata is None:
verbose_logger.warning(
"MCP OAuth discovery yielded no metadata for server %s (%s); "
"OAuth endpoints/scopes stay unresolved until a rebuild succeeds",
mcp_server.server_id,
server_url,
)
if use_issuer_anchor:
return mcp_oauth_metadata
if is_discovery_auth_type:
return _restrict_discovery_to_corroborated_authorization_server(
mcp_oauth_metadata,
manual_authorization_url,
mcp_server.server_id,
bool(getattr(mcp_server, "dcr_bridge", None)),
)
return mcp_oauth_metadata
async def build_mcp_server_from_table(
self,
mcp_server: LiteLLM_MCPServerTable,
@ -1447,32 +1697,38 @@ class MCPServerManager:
auth_type = cast(MCPAuthType, mcp_server.auth_type)
server_url = mcp_server.url
needs_discovery = bool(server_url) and (
(auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES and not mcp_server.authorization_url)
or self._obo_needs_endpoint_discovery(
auth_type,
mcp_server.token_exchange_endpoint
or (credentials_dict.get("token_exchange_endpoint") if credentials_dict else None),
mcp_server.token_url,
)
manual_issuer = _blank_to_none(mcp_server.issuer)
manual_authorization_url = _blank_to_none(mcp_server.authorization_url)
manual_token_url = _blank_to_none(mcp_server.token_url)
manual_registration_url = _blank_to_none(mcp_server.registration_url)
is_discovery_auth_type = auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type)
manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer(
manual_issuer, is_discovery_auth_type, manual_authorization_url, manual_token_url, manual_registration_url
)
mcp_oauth_metadata = (
await self._descovery_metadata(
server_url=server_url, # type: ignore[arg-type]
allow_origin_fallback=auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES,
)
if needs_discovery
token_exchange_endpoint = mcp_server.token_exchange_endpoint or (
credentials_dict.get("token_exchange_endpoint") if credentials_dict else None
)
gated_oauth_metadata = await self._resolve_table_oauth_metadata(
mcp_server=mcp_server,
auth_type=auth_type,
server_url=server_url,
manual_issuer=manual_issuer,
manual_authorization_url=manual_authorization_url,
manual_token_url=manual_token_url,
is_discovery_auth_type=is_discovery_auth_type,
use_issuer_anchor=use_issuer_anchor,
scopes=scopes,
token_exchange_endpoint=token_exchange_endpoint,
)
resolved_scopes = scopes or (gated_oauth_metadata.scopes if gated_oauth_metadata else None)
discovered_issuer = (
gated_oauth_metadata.discovered_issuer
if gated_oauth_metadata and not gated_oauth_metadata.from_origin_fallback
else None
)
if needs_discovery and mcp_oauth_metadata is None:
verbose_logger.warning(
"MCP OAuth discovery yielded no metadata for server %s (%s); "
"OAuth endpoints stay unresolved until a rebuild succeeds",
mcp_server.server_id,
server_url,
)
resolved_scopes = scopes or (mcp_oauth_metadata.scopes if mcp_oauth_metadata else None)
effective_issuer = manual_issuer or discovered_issuer
new_server = MCPServer(
server_id=mcp_server.server_id,
@ -1492,9 +1748,11 @@ class MCPServerManager:
client_secret=client_secret_value or getattr(mcp_server, "client_secret", None),
oauth2_flow=self._explicit_oauth2_flow(getattr(mcp_server, "oauth2_flow", None)),
scopes=resolved_scopes,
authorization_url=mcp_server.authorization_url or getattr(mcp_oauth_metadata, "authorization_url", None),
token_url=mcp_server.token_url or getattr(mcp_oauth_metadata, "token_url", None),
registration_url=mcp_server.registration_url or getattr(mcp_oauth_metadata, "registration_url", None),
issuer=effective_issuer,
issuer_is_anchored=use_issuer_anchor,
authorization_url=manual_authorization_url or getattr(gated_oauth_metadata, "authorization_url", None),
token_url=manual_token_url or getattr(gated_oauth_metadata, "token_url", None),
registration_url=manual_registration_url or getattr(gated_oauth_metadata, "registration_url", None),
token_endpoint_auth_method=(
credentials_dict.get("token_endpoint_auth_method") if credentials_dict else None
),
@ -1545,16 +1803,18 @@ class MCPServerManager:
await self._persist_discovered_obo_token_url(
server_id=mcp_server.server_id,
auth_type=auth_type,
existing_token_url=mcp_server.token_url,
existing_token_url=manual_token_url,
discovered_token_url=new_server.token_url,
)
await self._persist_discovered_oauth_endpoints(
server_id=mcp_server.server_id,
auth_type=auth_type,
existing_authorization_url=mcp_server.authorization_url,
existing_token_url=mcp_server.token_url,
existing_issuer=manual_issuer,
existing_authorization_url=manual_authorization_url,
existing_token_url=manual_token_url,
existing_scopes=scopes,
metadata=mcp_oauth_metadata,
metadata=gated_oauth_metadata,
is_issuer_anchored=use_issuer_anchor,
)
return new_server
@ -1598,10 +1858,12 @@ class MCPServerManager:
*,
server_id: str,
auth_type: MCPAuthType | None,
existing_issuer: str | None,
existing_authorization_url: str | None,
existing_token_url: str | None,
existing_scopes: list[str] | None,
metadata: MCPOAuthMetadata | None,
is_issuer_anchored: bool = False,
) -> None:
"""Write freshly discovered OAuth endpoints back onto the DB row.
@ -1615,19 +1877,37 @@ class MCPServerManager:
because ``_dcr_bridge_relays_client_registration`` keys off that column. Best-effort: a
failed write re-discovers on the next build. Scopes go through ``update_mcp_server`` so
they merge into the credentials blob without touching the stored client credentials.
For an issuer-anchored server (``is_issuer_anchored``) the endpoints are re-derived from the
§3.3-validated issuer document on every build, so they are NOT persisted into the endpoint
columns: persisting them would make the next build see populated endpoints and treat them as
authoritative stored values, defeating the "endpoints come solely from the issuer" invariant.
Only the resource-driven scopes are persisted for such servers.
"""
if auth_type not in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES:
return
if metadata is None or metadata.from_origin_fallback:
return
issuer_update = (
{"issuer": metadata.discovered_issuer} if metadata.discovered_issuer and not existing_issuer else {}
)
authorization_url_update = (
{"authorization_url": metadata.authorization_url}
if metadata.authorization_url and not existing_authorization_url
if metadata.authorization_url and not existing_authorization_url and not is_issuer_anchored
else {}
)
token_url_update = (
{"token_url": metadata.token_url}
if metadata.token_url and not existing_token_url and not is_issuer_anchored
else {}
)
token_url_update = {"token_url": metadata.token_url} if metadata.token_url and not existing_token_url else {}
scopes_update = {"credentials": {"scopes": metadata.scopes}} if metadata.scopes and not existing_scopes else {}
updates: dict[str, object] = {**authorization_url_update, **token_url_update, **scopes_update}
updates: dict[str, object] = {
**issuer_update,
**authorization_url_update,
**token_url_update,
**scopes_update,
}
if not updates:
return
from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # db.py imports this module at load
@ -3200,8 +3480,41 @@ class MCPServerManager:
return metadata
return None
async def _fetch_issuer_anchored_oauth_metadata(
self, issuer: str, server_url: Optional[str]
) -> Optional[MCPOAuthMetadata]:
"""RFC 8414 issuer-anchored discovery for the OAuth endpoints, with resource-driven scopes.
Fetch authorization-server metadata from the admin-configured issuer's own origin and adopt
its ``token_endpoint``/``registration_endpoint`` only when the document self-attests that same
issuer (RFC 8414 §3.3). Because the trust anchor is the pinned issuer rather than anything the
MCP resource advertises, the endpoints are authoritative for that issuer and cannot be
substituted by a compromised resource. Fails closed (returns None) on a §3.3 mismatch or a
fetch failure. The issuer is passed as its own ``server_url`` so the endpoint fetch is treated
as same-authority and is not subject to the resource-scoped SSRF shortcut.
Scopes are NOT taken from the issuer document. Per the MCP authorization spec Scope Selection
Strategy and RFC 9728, the scopes a client requests are resource-driven (the WWW-Authenticate
challenge or the protected-resource ``scopes_supported``), so the resource's advertised scopes
are fetched separately and used; the resource can influence only the requested scope, which
the authorization server and user consent bound (RFC 6749 §3.3), never the token endpoint.
"""
metadata = await self._fetch_single_authorization_server_metadata(issuer, issuer, require_issuer=issuer)
if metadata is None:
verbose_logger.warning(
"MCP OAuth issuer-anchored discovery for issuer %s yielded no metadata whose issuer "
"matched (RFC 8414 §3.3); OAuth endpoints stay unresolved until a rebuild succeeds",
issuer,
)
return None
resource_metadata = (
await self._descovery_metadata(server_url, allow_origin_fallback=False) if server_url else None
)
resource_scopes = resource_metadata.scopes if resource_metadata else None
return metadata.model_copy(update={"scopes": resource_scopes})
async def _fetch_single_authorization_server_metadata(
self, issuer_url: str, server_url: str
self, issuer_url: str, server_url: str, require_issuer: Optional[str] = None
) -> Optional[MCPOAuthMetadata]:
try:
parsed = urlparse(issuer_url)
@ -3245,20 +3558,33 @@ class MCPServerManager:
)
continue
scopes = self._extract_scopes(data.get("scopes_supported"))
claimed_issuer = data.get("issuer")
verbose_logger.debug(
"Authorization server metadata from %s: issuer=%s grant_types_supported=%s "
"token_endpoint_auth_methods_supported=%s",
url,
data.get("issuer"),
claimed_issuer,
data.get("grant_types_supported"),
data.get("token_endpoint_auth_methods_supported"),
)
if require_issuer is not None and not _issuer_matches(claimed_issuer, require_issuer):
verbose_logger.warning(
"MCP OAuth issuer-anchored discovery: metadata at %s self-attests issuer %r, which "
"does not match the configured issuer %r (RFC 8414 §3.3); rejecting so a compromised "
"resource cannot substitute an attacker authorization server",
url,
claimed_issuer,
require_issuer,
)
continue
scopes = self._extract_scopes(data.get("scopes_supported"))
metadata = MCPOAuthMetadata(
scopes=scopes,
authorization_url=data.get("authorization_endpoint"),
token_url=data.get("token_endpoint"),
registration_url=data.get("registration_endpoint"),
discovered_issuer=claimed_issuer if isinstance(claimed_issuer, str) and claimed_issuer else None,
)
if any(
@ -4979,6 +5305,7 @@ class MCPServerManager:
command=getattr(server, "command", None),
args=getattr(server, "args", None) or [],
env=getattr(server, "env", None) or {},
issuer=server.issuer,
authorization_url=server.authorization_url,
token_url=server.token_url,
registration_url=server.registration_url,
@ -5088,6 +5415,7 @@ class MCPServerManager:
command=getattr(server, "command", None),
args=getattr(server, "args", None) or [],
env=getattr(server, "env", None) or {},
issuer=server.issuer,
authorization_url=server.authorization_url,
token_url=server.token_url,
registration_url=server.registration_url,

1
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -1138,6 +1138,7 @@ if MCP_AVAILABLE:
static_headers=request.static_headers,
client_id=client_id,
client_secret=client_secret,
issuer=request.issuer,
token_url=request.token_url,
scopes=scopes,
authorization_url=request.authorization_url,

105
litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py
View file

@ -4,6 +4,7 @@ Semantic MCP Tool Filtering using semantic-router
Filters MCP tools semantically for /chat/completions and /responses endpoints.
"""
import asyncio
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from litellm._logging import verbose_logger
@ -76,6 +77,7 @@ class SemanticMCPToolFilter:
self.tool_router: Optional["SemanticRouter"] = None
self.context_window_error: Optional[str] = None
self._tool_map: Dict[str, Any] = {} # MCPTool objects or OpenAI function dicts
self._index_sync_lock = asyncio.Lock()
async def build_router_from_mcp_registry(self) -> None:
"""Build semantic router from all MCP tools in the registry (no auth checks)."""
@ -182,6 +184,81 @@ class SemanticMCPToolFilter:
return
raise
def _has_tools_missing_from_index(self, tools: list[Any]) -> bool:
"""Allocation-free check for any named tool not yet in the semantic index."""
return any(name and name not in self._tool_map for name in (self._extract_tool_info(t)[0] for t in tools))
def _tools_missing_from_index(self, tools: list[Any]) -> dict[str, Any]:
"""Map name -> tool for every named tool not yet in the semantic index."""
return {
name: tool
for name, tool in ((self._extract_tool_info(t)[0], t) for t in tools)
if name and name not in self._tool_map
}
async def _ensure_tools_indexed(self, available_tools: list[Any]) -> None:
"""
Index request-time tools the startup build never saw.
The startup index lists every registered MCP server WITHOUT per-user
credentials, so servers requiring per-user auth (interactive OAuth
tokens, user-scoped env vars) contribute zero routes. Tools reaching
the filter came through an authenticated expansion; without indexing
them here they can never be selected, so requests either bypass
filtering entirely (N->N) or lose every tool to unrelated matches.
Runs async-only (no synchronous embedding on the request path) and
never writes shared error state: an embedding failure here raises and
is scoped to the requesting call, so one request's oversized tool
description cannot poison the filter for other users on the worker.
"""
from semantic_router.routers import SemanticRouter
from semantic_router.routers.base import Route
from litellm.router_strategy.auto_router.litellm_encoder import (
LiteLLMRouterEncoder,
)
if not self._has_tools_missing_from_index(available_tools):
return
async with self._index_sync_lock:
missing = self._tools_missing_from_index(available_tools)
if not missing:
return
descriptions = {name: self._extract_tool_info(tool)[1] for name, tool in missing.items()}
routes = [
Route(
name=name,
description=description,
utterances=[description],
score_threshold=self.similarity_threshold,
)
for name, description in descriptions.items()
]
if self.tool_router is None:
router = SemanticRouter(
routes=[],
encoder=LiteLLMRouterEncoder(
litellm_router_instance=self.router_instance,
model_name=self.embedding_model,
score_threshold=self.similarity_threshold,
),
auto_sync="local",
top_k=self.top_k,
)
await router.aadd(routes)
self.tool_router = router
else:
await self.tool_router.aadd(routes)
self._tool_map.update(missing)
verbose_logger.info(
f"Semantic tool filter indexed {len(routes)} request-time tools missing from the startup index"
)
async def filter_tools(
self,
query: str,
@ -216,22 +293,34 @@ class SemanticMCPToolFilter:
if not query or not query.strip():
return available_tools
# Router should be built on startup - if not, something went wrong
if self.tool_router is None:
verbose_logger.warning("Router not initialized - was build_router_from_mcp_registry() called on startup?")
return available_tools
# Run semantic filtering
try:
await self._ensure_tools_indexed(available_tools)
if self.tool_router is None:
verbose_logger.warning("Semantic router could not be built from the request's tools")
return available_tools
available_names = [name for name in (self._extract_tool_info(t)[0] for t in available_tools) if name]
if not available_names:
return available_tools
limit = top_k or self.top_k
matches = self.tool_router(text=query, limit=limit)
if self.tool_router.top_k < limit:
self.tool_router.top_k = limit
matches = self.tool_router(text=query, limit=limit, route_filter=available_names)
matched_tool_names = self._extract_tool_names_from_matches(matches)
if not matched_tool_names:
return available_tools
return self._get_tools_by_names(matched_tool_names, available_tools)
filtered_tools = self._get_tools_by_names(matched_tool_names, available_tools)
if not filtered_tools:
return available_tools
return filtered_tools
except SemanticToolFilterContextWindowError:
raise
except Exception as e:
if _is_context_window_error(e):
verbose_logger.error(
@ -240,7 +329,7 @@ class SemanticMCPToolFilter:
)
raise SemanticToolFilterContextWindowError(
embedding_model=self.embedding_model,
stage="the user query",
stage="the user query or the MCP tool descriptions being indexed",
original_error=str(e),
) from e
verbose_logger.error(f"Semantic tool filter failed: {e}", exc_info=True)

6
litellm/proxy/_types.py
View file

@ -1263,6 +1263,7 @@ class NewMCPServerRequest(LiteLLMPydanticObjectBase):
command: Optional[str] = None
args: List[str] = Field(default_factory=list)
env: Dict[str, str] = Field(default_factory=dict)
issuer: Optional[str] = None
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
@ -1368,6 +1369,7 @@ class UpdateMCPServerRequest(LiteLLMPydanticObjectBase):
command: Optional[str] = None
args: List[str] = Field(default_factory=list)
env: Dict[str, str] = Field(default_factory=dict)
issuer: Optional[str] = None
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
@ -2367,6 +2369,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
None,
description="If True, stores request messages and responses in spend logs. Default is False.",
)
disable_auto_add_proxy_admin_to_teams: bool | None = Field(
None,
description="By default, the user calling /team/new is automatically added to the new team as a team admin. If True, proxy admins are no longer auto-added; members explicitly listed in members_with_roles are unaffected. Default is False.",
)
maximum_spend_logs_retention_period: Optional[str] = Field(
None,
description="Maximum retention period for spend logs (e.g., '7d' for 7 days). Logs older than this will be deleted.",

26
litellm/proxy/auth/auth_checks.py
View file

@ -361,7 +361,11 @@ def _global_proxy_budget_check(global_proxy_spend: Optional[float], skip_budget_
and route != "/models"
):
if math.isfinite(litellm.max_budget) and global_proxy_spend > litellm.max_budget:
raise litellm.BudgetExceededError(current_cost=global_proxy_spend, max_budget=litellm.max_budget)
raise litellm.BudgetExceededError(
current_cost=global_proxy_spend,
max_budget=litellm.max_budget,
entity_type=Litellm_EntityType.PROXY.value,
)
_GUARDRAIL_MODIFICATION_KEYS: tuple = (
@ -648,6 +652,8 @@ async def common_checks(
current_cost=user_spend,
max_budget=user_budget,
message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_spend}, Budget={user_budget}",
entity_type=Litellm_EntityType.USER.value,
entity_id=user_object.user_id,
)
# Each scope reads a distinct counter key with no cross-scope ordering
@ -1093,6 +1099,8 @@ async def _check_end_user_budget(
current_cost=end_user_spend,
max_budget=end_user_budget,
message=f"ExceededBudget: End User={end_user_obj.user_id} over budget. Spend={end_user_spend}, Budget={end_user_budget}",
entity_type=Litellm_EntityType.END_USER.value,
entity_id=end_user_obj.user_id,
)
@ -3552,6 +3560,8 @@ async def _virtual_key_max_budget_check(
current_cost=spend,
max_budget=valid_token.max_budget,
message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}",
entity_type=Litellm_EntityType.KEY.value,
entity_id=valid_token.token,
)
@ -3593,6 +3603,8 @@ async def _virtual_key_multi_budget_check(
f"ExceededBudget: Key over {w['budget_duration']} budget. "
f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}"
),
entity_type=Litellm_EntityType.KEY.value,
entity_id=valid_token.token,
)
@ -3824,6 +3836,8 @@ async def _check_team_member_budget(
current_cost=team_member_spend,
max_budget=team_member_budget,
message=f"Budget has been exceeded! User={valid_token.user_id} in Team={team_object.team_id} Current cost: {team_member_spend}, Max budget: {team_member_budget}",
entity_type=Litellm_EntityType.TEAM_MEMBER.value,
entity_id=f"{valid_token.user_id}:{team_object.team_id}",
)
@ -3923,6 +3937,8 @@ async def _team_max_budget_check(
current_cost=spend,
max_budget=team_object.max_budget,
message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}",
entity_type=Litellm_EntityType.TEAM.value,
entity_id=team_object.team_id,
)
@ -3960,6 +3976,8 @@ async def _team_multi_budget_check(
f"ExceededBudget: Team={team_object.team_id} over {w['budget_duration']} budget. "
f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}"
),
entity_type=Litellm_EntityType.TEAM.value,
entity_id=team_object.team_id,
)
@ -4081,6 +4099,8 @@ async def _project_max_budget_check(
current_cost=project_object.spend,
max_budget=max_budget,
message=f"Budget has been exceeded! Project={project_object.project_id} Current cost: {project_object.spend}, Max budget: {max_budget}",
entity_type=Litellm_EntityType.PROJECT.value,
entity_id=project_object.project_id,
)
@ -4269,6 +4289,8 @@ async def _organization_max_budget_check(
current_cost=org_spend,
max_budget=org_max_budget,
message=f"Budget has been exceeded! Organization={org_id} Current cost: {org_spend}, Max budget: {org_max_budget}",
entity_type=Litellm_EntityType.ORGANIZATION.value,
entity_id=org_id,
)
@ -4326,6 +4348,8 @@ async def _tag_max_budget_check(
current_cost=tag_spend,
max_budget=tag_object.litellm_budget_table.max_budget,
message=f"Budget has been exceeded! Tag={tag_name} Current cost: {tag_spend}, Max budget: {tag_object.litellm_budget_table.max_budget}",
entity_type=Litellm_EntityType.TAG.value,
entity_id=tag_name,
)

4
litellm/proxy/auth/auth_utils.py
View file

@ -10,7 +10,7 @@ from fastapi import HTTPException, Request, status
import litellm
from litellm import Router, provider_list
from litellm._logging import verbose_proxy_logger
from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
from litellm.proxy._types import *
@ -1533,4 +1533,6 @@ def get_model_from_request(
def abbreviate_api_key(api_key: str) -> str:
if len(api_key) < MINIMUM_CUSTOM_KEY_LENGTH:
return "sk-..."
return f"sk-...{api_key[-4:]}"

2
litellm/proxy/auth/user_api_key_auth.py
View file

@ -1797,6 +1797,8 @@ async def _user_api_key_auth_builder(
raise litellm.BudgetExceededError(
current_cost=team_member_spend,
max_budget=team_member_budget,
entity_type=Litellm_EntityType.TEAM_MEMBER.value,
entity_id=f"{valid_token.user_id}:{valid_token.team_id}",
)
# Check 3. If token is expired

103
litellm/proxy/client/cli/README.md
View file

@ -471,6 +471,109 @@ The token minted by `lite login` is a short-lived, per-session agent credential,
The credential is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); run `lite login` again to refresh it, which also re-reads your latest team and user settings. It does not appear in the Keys UI and cannot be rotated or revoked mid-session. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while it's still fresh and fails once it expires -- there is no silent renewal, so a long-running session needs a fresh `lite login` once a day. `lite claude`, `lite codex`, and `lite opencode` work with it on a default deployment; `EXPERIMENTAL_UI_LOGIN` is not required. If you need a long-lived, rotatable key that shows up in the Keys UI, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY` instead.
### Route Every Claude Code Session Through the Proxy
`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it.
Two things need to already be true: you've run `lite login`, since the apiKeyHelper depends on that stored token, and the proxy is already reachable, since `lite up` does not start one for you.
```bash
lite login
litellm --config litellm/proxy/dev_config.yaml &
lite up
```
`lite up` runs in the foreground and blocks. Press Ctrl-C to stop it, which restores the original settings file and exits. If the process is ever killed uncleanly instead -- `kill -9`, a crash -- the settings file is left patched, and `lite down` is the manual recovery path: run it at any later point to restore from the same backup.
This is a one-time file patch and restore, not a live traffic interceptor. A Claude Code session already running before `lite up` started keeps whatever `ANTHROPIC_BASE_URL` and token it loaded at its own startup, and a session still running when `lite up` stops keeps routing through the proxy until it exits; only sessions *started* while the patch is in effect are affected, and only *new* sessions after a restore go back to Anthropic directly.
Cursor is not supported: it has no equivalent file-based config to hot-patch this way, since its model routing lives in its own app storage and is configured through its GUI.
### QA Complexity-Based Auto-Routing Against Your Real Proxy
`lite autoroute` lets you try LiteLLM's complexity-based auto-routing -- picking a cheaper or more expensive model depending on how complex a prompt looks -- against models your key already has access to on your real, running proxy, without editing that proxy's `config.yaml` and without any real request ever bypassing it. It builds a second, throwaway proxy locally that forwards every request back to your real proxy, and points Claude Code at that local proxy for the duration of the session.
#### Install the CLI
`lite autoroute up` builds and runs a throwaway litellm proxy locally, so unlike the rest of this CLI it needs the proxy server runtime, not just the thin `litellm[cli]` client. Install `litellm[proxy]` (which ships the `lite` command too) with a single curl command -- no existing Python tooling required, `uv` is bootstrapped automatically if missing:
```bash
curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh
```
To QA an unreleased branch or commit instead of the latest PyPI release, set `LITELLM_CLI_REF`:
```bash
curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/<branch-or-commit>/scripts/install.sh | \
LITELLM_CLI_REF=<branch-or-commit> sh
```
The thin `scripts/install-cli.sh` installs only `litellm[cli]`, which is enough for `lite login`, `lite claude`, and `lite up`, but not for `lite autoroute up`; running it against a `litellm[cli]` install fails fast with a message telling you to install the proxy runtime.
Point the CLI at your real proxy and key before running any `lite model-groups` or `lite autoroute` command -- like every other command in this CLI, they read `LITELLM_PROXY_URL`/`LITELLM_PROXY_API_KEY` (or `--base-url`/`--api-key`), no `lite login` required:
```bash
export LITELLM_PROXY_URL=http://localhost:4000
export LITELLM_PROXY_API_KEY=sk-...
```
#### List Your Accessible Model Groups
```bash
lite model-groups list [--format table|json]
```
Lists the model groups your key can reach on the proxy, via `/model_group/info`, along with each group's mode (`chat`, `embedding`, etc.) and per-token pricing. This is also what `lite autoroute configure` uses internally to discover what it can offer you.
#### Configure the Auto-Router
```bash
lite autoroute configure
```
An interactive wizard. It runs the same model-group discovery as above, splits the results into chat-capable and embedding-capable pools, and asks you to assign one or more models from the chat pool to each of the four complexity tiers -- SIMPLE, MEDIUM, COMPLEX, REASONING. Each tier's picker is a type-to-filter fuzzy search (fzf-style) rather than a scrollable numbered list, so it stays usable even with hundreds of model groups: type a substring to narrow the list, tab to toggle a model into the selection, enter to confirm (assigning more than one model to a tier is exactly when this matters -- complexity_router picks randomly among a tier's pool per request, and adaptive mode specifically depends on having more than one candidate to choose from). From there it optionally offers: classifying prompt complexity with an LLM (again picked from your discovered pool) instead of the free built-in heuristic scorer, semantic keyword matching for tier assignment (needs an embedding model from the pool), and adaptive (bandit-based) selection layered on top of tiering.
The wizard writes the result to `~/.litellm/autorouter/config.yaml` with `0600` permissions, since the file embeds your real proxy API key. Every model referenced anywhere in that config -- tier targets, the classifier model, the embedding model -- becomes its own `litellm_proxy/<model-name>` deployment whose `api_base` and `api_key` point back at your real proxy. That is the trick that keeps your real proxy's config untouched: every actual network call this generates, whether it is the routed completion, an LLM-classifier call, or an embedding call, forwards transparently through your real, already-running proxy with your real key.
You do not need to tell Claude Code to request `autorouter` by name yourself: `lite autoroute up` also sets `ANTHROPIC_DEFAULT_SONNET_MODEL`, `ANTHROPIC_DEFAULT_HAIKU_MODEL`, and `ANTHROPIC_DEFAULT_OPUS_MODEL` to `autorouter` in `~/.claude/settings.json`, so every one of Claude Code's own model tiers requests it directly regardless of `/model` or whatever it defaults to otherwise. (A bare `model_name: "*"` deployment looks like the obvious way to catch any request instead, but litellm's Router looks up auto-router deployments by the literal requested model string with no wildcard resolution, so a `"*"` entry would never actually match real traffic -- these env var overrides are what makes it work.)
You must run `configure` at least once before `up`; running `up` first fails with a clear error telling you to configure first.
#### Launch the Ephemeral Auto-Router Proxy
```bash
lite autoroute up
```
Starts a local, throwaway litellm proxy on a random free port, running the config `configure` generated, with a freshly-minted random API key baked in for this session only (your real proxy key never leaves the generated config -- it only appears there, forwarding to your real proxy). It waits for the ephemeral proxy to report healthy, then patches `~/.claude/settings.json` the same way `lite up` does, except with a static `ANTHROPIC_AUTH_TOKEN` env var instead of an `apiKeyHelper`, since this key is short-lived and self-issued rather than something needing SSO refresh. Any `claude` session started afterward, from any terminal, routes through the ephemeral proxy.
`lite autoroute up` runs in the foreground and streams the ephemeral proxy's own log file into your terminal, so you can watch its routing decisions -- which tier and model got picked for each request -- as you use Claude Code normally. Press Ctrl-C (or send SIGTERM) to stop it; this kills the child proxy process and restores your original Claude Code settings, in that order.
#### Recover From an Unclean Shutdown
```bash
lite autoroute down
```
If the `lite autoroute up` process dies uncleanly -- `kill -9`, a crash -- rather than being stopped with Ctrl-C, `down` is the manual recovery path: it kills any leftover ephemeral proxy process found via a recorded pid file and restores Claude Code's settings from whatever backup is on disk.
#### Example
```bash
lite autoroute configure
lite autoroute up
# use Claude Code as normal in another terminal; routing decisions stream live
lite autoroute down # only needed if `up` was killed uncleanly instead of Ctrl-C'd
```
#### Caveats
Adaptive mode's learned state does not persist across `lite autoroute up` sessions -- there is no local database, so every session starts adaptive selection cold. A Claude Code session already running before `up` started, or still running when it stops, keeps whatever settings it loaded at its own startup; like `lite up`, this is a one-time file patch and restore, not a live traffic interceptor. Only Claude Code is supported, for the same reason as `lite up`: no other supported agent (for example Cursor) has an equivalent hot-patchable config file.
A session that outlives `up` (or is still running the moment you stop it) keeps sending requests, master key included, to that now-freed loopback port until you restart it. Once the ephemeral proxy process exits, nothing stops another local account on the same machine from binding that same port and receiving those requests instead -- unlike `lite up`'s `apiKeyHelper`, which is re-resolved per request, `autoroute`'s master key is a static value, so whoever receives them gets a live-looking token along with the prompt content. Restart any Claude Code session before you consider the machine clean, run `lite autoroute down` promptly rather than leaving a stopped session's settings patched, and do not run `lite autoroute up` on a shared or multi-tenant host.
Do not run `lite up` and `lite autoroute up` at the same time. Each patches `~/.claude/settings.json` and keeps its own separate backup, with no coordination between them: whichever one you stop or crash out of last is the one whose backup gets restored, which can silently leave the *other* mode's settings (a static master key and a now-dead loopback URL, or a stale `apiKeyHelper`) active. Run `lite down` or `lite autoroute down` (whichever applies) before switching to the other mode.
## Environment Variables
The CLI respects the following environment variables:

5
litellm/proxy/client/cli/commands/agents.py
View file

@ -212,7 +212,7 @@ def _is_interactive() -> bool:
return sys.stdin.isatty()
def _resolve_api_key(ctx: click.Context) -> str:
def resolve_api_key(ctx: click.Context) -> str:
base_url = ctx.obj["base_url"]
api_key = ctx.obj.get("api_key")
if api_key:
@ -238,7 +238,7 @@ _SKIP_VERIFY_HELP = "Skip the pre-launch key check against the proxy."
def _launch(ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify: bool) -> None:
base_url = ctx.obj["base_url"]
started_interactive = _is_interactive()
api_key = _resolve_api_key(ctx)
api_key = resolve_api_key(ctx)
display_name, _ = agent_profile(binary)
click.echo(f"litellm: routing {display_name} through proxy at {base_url.rstrip('/')}")
@ -288,5 +288,6 @@ __all__ = [
"agent_launch_args",
"verify_proxy_key",
"agent_profile",
"resolve_api_key",
"AgentRunError",
]

62
litellm/proxy/client/cli/commands/auth.py
View file

@ -72,7 +72,7 @@ def display_teams_table(teams: List[Dict[str, Any]]) -> None:
console = Console()
if not teams:
console.print("❌ No teams found for your user.")
console.print("No teams found for your user.")
return
table = Table(title="Available Teams")
@ -162,7 +162,7 @@ def display_interactive_team_selection(teams: List[Dict[str, Any]], selected_ind
# Clear the screen using Rich's method
console.clear()
console.print("🎯 Select a Team (Use ↑↓ arrows, Enter to select, 'q' to skip):\n")
console.print("Select a Team (Use up/down arrows, Enter to select, 'q' to skip):\n")
for i, team in enumerate(teams):
team_alias = team.get("team_alias") or "N/A"
@ -184,7 +184,7 @@ def display_interactive_team_selection(teams: List[Dict[str, Any]], selected_ind
# Highlight the selected item
if i == selected_index:
console.print(f"➤ [bold cyan]{team_alias}[/bold cyan] ({team_id})")
console.print(f"> [bold cyan]{team_alias}[/bold cyan] ({team_id})")
console.print(f" Models: [yellow]{models_str}[/yellow]")
console.print(f" Budget: [blue]{budget_str}[/blue]\n")
else:
@ -220,15 +220,13 @@ def prompt_team_selection(teams: List[Dict[str, Any]]) -> Optional[Dict[str, Any
# Clear screen and show selection
console = Console()
console.clear()
click.echo(
f"✅ Selected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})"
)
click.echo(f"Selected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})")
return selected_team
elif key == "quit" or key == "escape":
# Clear screen
console = Console()
console.clear()
click.echo("ℹ️ Team selection skipped.")
click.echo("Team selection skipped.")
return None
elif key is None:
# If we can't get key input, fall back to simple selection
@ -237,7 +235,7 @@ def prompt_team_selection(teams: List[Dict[str, Any]]) -> Optional[Dict[str, Any
except KeyboardInterrupt:
console = Console()
console.clear()
click.echo("\n❌ Team selection cancelled.")
click.echo("\nTeam selection cancelled.")
return None
except Exception:
# If interactive mode fails, fall back to simple selection
@ -265,15 +263,15 @@ def prompt_team_selection_fallback(
if 0 <= index < len(teams):
selected_team = teams[index]
click.echo(
f"\n✅ Selected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})"
f"\nSelected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})"
)
return selected_team
else:
click.echo(f"❌ Invalid selection. Please enter a number between 1 and {len(teams)}")
click.echo(f"Invalid selection. Please enter a number between 1 and {len(teams)}")
except ValueError:
click.echo("❌ Invalid input. Please enter a number or 'skip'")
click.echo("Invalid input. Please enter a number or 'skip'")
except KeyboardInterrupt:
click.echo("\n❌ Team selection cancelled.")
click.echo("\nTeam selection cancelled.")
return None
@ -437,7 +435,7 @@ def _poll_for_authentication(base_url: str, key_id: str, poll_secret: str) -> Op
user_id = data.get("user_id")
normalized_teams: List[Dict[str, Any]] = _normalize_teams(teams, team_details)
if not normalized_teams:
click.echo("⚠️ No teams available for selection.")
click.echo("Warning: No teams available for selection.")
return None
# User has multiple teams - let them select
@ -457,7 +455,7 @@ def _poll_for_authentication(base_url: str, key_id: str, poll_secret: str) -> Op
"team_id": None, # Set by server in JWT
}
click.echo("❌ Team selection cancelled or JWT generation failed.")
click.echo("Team selection cancelled or JWT generation failed.")
return None
# JWT is ready (single team or team already selected)
@ -468,7 +466,7 @@ def _poll_for_authentication(base_url: str, key_id: str, poll_secret: str) -> Op
# Show which team was assigned
if team_id and len(teams) == 1:
click.echo(f"\n✅ Automatically assigned to team: {team_id}")
click.echo(f"\nAutomatically assigned to team: {team_id}")
if api_key:
return {
@ -494,19 +492,19 @@ def _handle_team_selection_during_polling(
The JWT token with the selected team, or None if selection was skipped
"""
if not teams:
click.echo("ℹ️ No teams found. You can create or join teams using the web interface.")
click.echo("No teams found. You can create or join teams using the web interface.")
return None
click.echo("\n" + "=" * 60)
click.echo("📋 Select a team for your CLI session...")
click.echo("Select a team for your CLI session...")
team_id = _render_and_prompt_for_team_selection(teams)
if not team_id:
click.echo("ℹ️ No team selected.")
click.echo("No team selected.")
return None
click.echo(f"\n🔄 Generating JWT for team: {team_id}")
click.echo(f"\nGenerating JWT for team: {team_id}")
poll_url = f"{base_url}/sso/cli/poll/{key_id}?team_id={team_id}"
data = _poll_for_ready_data(
@ -520,7 +518,7 @@ def _handle_team_selection_during_polling(
return None
jwt_token = data.get("key")
if jwt_token:
click.echo(f"✅ Successfully generated JWT for team: {team_id}")
click.echo(f"Successfully generated JWT for team: {team_id}")
return jwt_token
return None
@ -568,14 +566,14 @@ def _render_and_prompt_for_team_selection(teams: List[Dict[str, Any]]) -> Option
selected_team = teams[index]
team_id = str(selected_team.get("team_id"))
team_alias = selected_team.get("team_alias") or team_id
click.echo(f"\n✅ Selected team: {team_alias} ({team_id})")
click.echo(f"\nSelected team: {team_alias} ({team_id})")
return team_id
click.echo(f"❌ Invalid selection. Please enter a number between 1 and {len(teams)}")
click.echo(f"Invalid selection. Please enter a number between 1 and {len(teams)}")
except ValueError:
click.echo("❌ Invalid input. Please enter a number or 'skip'")
click.echo("Invalid input. Please enter a number or 'skip'")
except KeyboardInterrupt:
click.echo("\n❌ Team selection cancelled.")
click.echo("\nTeam selection cancelled.")
return None
@ -628,7 +626,7 @@ def login(ctx: click.Context):
}
)
click.echo("\n✅ Login successful!")
click.echo("\nLogin successful!")
click.echo(f"JWT Token: {api_key[:20]}...")
click.echo("You can now use the CLI without specifying --api-key")
@ -637,7 +635,7 @@ def login(ctx: click.Context):
show_commands()
return
else:
click.echo("❌ Authentication timed out. Please try again.")
click.echo("Authentication timed out. Please try again.")
click.echo(
"The proxy never reported the browser sign-in as finished. If you did complete it, "
"check the proxy logs for /sso/callback errors and confirm SSO is configured on the proxy."
@ -645,10 +643,10 @@ def login(ctx: click.Context):
return
except KeyboardInterrupt:
click.echo("\n❌ Authentication cancelled by user.")
click.echo("\nAuthentication cancelled by user.")
return
except Exception as e:
click.echo(f"❌ Authentication failed: {e}")
click.echo(f"Authentication failed: {e}")
return
@ -656,7 +654,7 @@ def login(ctx: click.Context):
def logout():
"""Logout and clear stored authentication"""
clear_token()
click.echo("✅ Logged out successfully. Authentication token cleared.")
click.echo("Logged out successfully. Authentication token cleared.")
@click.command(name="print-token")
@ -703,10 +701,10 @@ def whoami():
token_data = load_token()
if not token_data:
click.echo("❌ Not authenticated. Run 'lite login' to authenticate.")
click.echo("Not authenticated. Run 'lite login' to authenticate.")
return
click.echo("✅ Authenticated")
click.echo("Authenticated")
click.echo(f"User Email: {token_data.get('user_email', 'Unknown')}")
click.echo(f"User ID: {token_data.get('user_id', 'Unknown')}")
click.echo(f"User Role: {token_data.get('user_role', 'Unknown')}")
@ -717,7 +715,7 @@ def whoami():
click.echo(f"Token age: {age_hours:.1f} hours")
if age_hours > CLI_JWT_EXPIRATION_HOURS:
click.echo(f"⚠️ Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.")
click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.")
@click.group(name="auth")

0
tests/e2e/claude_code/_publisher_unit_tests/__init__.py → litellm/proxy/client/cli/commands/autoroute/__init__.py
View file

207
litellm/proxy/client/cli/commands/autoroute/commands.py Normal file
View file

@ -0,0 +1,207 @@
import atexit
import json
import secrets
import signal
import threading
from types import FrameType
import click
import yaml
from pydantic import JsonValue, TypeAdapter, ValidationError
from ..up import CLAUDE_SETTINGS_PATH, UpError, load_json_or_empty, restore_claude_settings, write_backup
from ..up import BackupRecord as ClaudeBackupRecord
from .process import (
AUTOROUTE_DIR,
CONFIG_PATH,
LOG_PATH,
PidRecord,
ProcessLaunchError,
allocate_free_port,
clear_pid_record,
is_running,
launch_proxy,
missing_proxy_runtime_modules,
poll_liveliness,
read_pid_record,
secure_create,
stream_log,
terminate,
write_pid_record,
)
from .settings import merge_claude_settings_static_token
from .wizard import run_configure_wizard
AUTOROUTE_BACKUP_PATH = AUTOROUTE_DIR / "claude_settings_backup.json"
_GENERATED_CONFIG_ADAPTER = TypeAdapter(dict[str, JsonValue])
def _mint_and_embed_master_key() -> str:
"""Generate a fresh key for this session and write it into the generated config.yaml.
Must go under general_settings, not litellm_settings -- the proxy server only ever
reads general_settings.master_key (proxy_server.py:4530) to authenticate requests. A
key placed under litellm_settings is silently ignored, leaving the ephemeral proxy with
no real auth: any request reaches it regardless of the token Claude Code sends.
"""
master_key = secrets.token_urlsafe(32)
with open(CONFIG_PATH, "r") as f:
try:
generated = _GENERATED_CONFIG_ADAPTER.validate_python(yaml.safe_load(f))
except (yaml.YAMLError, ValidationError):
raise click.ClickException(
f"{CONFIG_PATH} is empty or corrupt. Run `lite autoroute configure` again to regenerate it."
)
general_settings = generated.get("general_settings")
updated_settings: dict[str, JsonValue] = {
**(general_settings if isinstance(general_settings, dict) else {}),
"master_key": master_key,
}
updated: dict[str, JsonValue] = {**generated, "general_settings": updated_settings}
with secure_create(CONFIG_PATH) as f:
yaml.safe_dump(updated, f, sort_keys=False)
return master_key
@click.group(name="autoroute")
def autoroute_group() -> None:
"""QA complexity-based auto-routing against models your key can already use"""
@autoroute_group.command("configure")
@click.pass_context
def configure(ctx: click.Context) -> None:
"""Discover accessible models and generate an ephemeral auto-router config"""
run_configure_wizard(ctx)
@autoroute_group.command("up")
def up() -> None:
"""Launch the ephemeral auto-router proxy and route Claude Code through it"""
if not CONFIG_PATH.exists():
raise click.ClickException("No config found. Run `lite autoroute configure` first.")
missing = missing_proxy_runtime_modules()
if missing:
raise click.ClickException(
"lite autoroute up launches a local litellm proxy, which needs the proxy runtime that the "
f"thin `litellm[cli]` install does not include (missing: {', '.join(missing)}). Install the "
"proxy runtime with `uv tool install --force 'litellm[proxy]'`, or to QA a branch, "
"`curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/<branch>/scripts/install.sh | "
"LITELLM_CLI_REF=<branch> sh`."
)
try:
existing_pid = read_pid_record()
except UpError as e:
raise click.ClickException(str(e))
if existing_pid is not None and is_running(existing_pid.pid):
raise click.ClickException(
"An ephemeral proxy is already running (lite autoroute up looks already active). "
"Run `lite autoroute down` first."
)
if AUTOROUTE_BACKUP_PATH.exists():
raise click.ClickException(
f"{AUTOROUTE_BACKUP_PATH} already exists -- `lite autoroute up` looks like it's already "
"running (or crashed without cleanup). Run `lite autoroute down` first."
)
master_key = _mint_and_embed_master_key()
port = allocate_free_port()
base_url = f"http://127.0.0.1:{port}"
process = launch_proxy(CONFIG_PATH, port, LOG_PATH)
write_pid_record(PidRecord(pid=process.pid, port=port, config_path=str(CONFIG_PATH), log_path=str(LOG_PATH)))
try:
poll_liveliness(base_url, LOG_PATH, process)
except ProcessLaunchError as e:
terminate(process.pid)
clear_pid_record()
raise click.ClickException(str(e))
try:
original_existed = CLAUDE_SETTINGS_PATH.exists()
original_settings = load_json_or_empty(CLAUDE_SETTINGS_PATH)
write_backup(
ClaudeBackupRecord(existed=original_existed, content=original_settings if original_existed else None),
AUTOROUTE_BACKUP_PATH,
)
merged = merge_claude_settings_static_token(original_settings, base_url, master_key)
CLAUDE_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True)
with secure_create(CLAUDE_SETTINGS_PATH) as f:
json.dump(merged, f, indent=2)
except UpError as e:
terminate(process.pid)
clear_pid_record()
raise click.ClickException(str(e))
click.echo(f"litellm: ephemeral auto-router proxy up at {base_url} (pid {process.pid})")
click.echo("Claude Code sessions started now will route through it. Press Ctrl-C to stop and restore.")
stop_event = threading.Event()
restored = threading.Lock()
def _teardown() -> None:
if not restored.acquire(blocking=False):
return
terminate(process.pid)
clear_pid_record()
try:
restore_claude_settings(CLAUDE_SETTINGS_PATH, AUTOROUTE_BACKUP_PATH)
except UpError as e:
# Runs from atexit/a signal handler too, outside Click's own exception
# handling -- raising here would only produce an unhandled-exception
# warning on stderr, not a clean message.
click.echo(str(e), err=True)
return
click.echo("\nStopped ephemeral proxy and restored Claude Code settings.")
click.echo(
f"Restart any Claude Code session still open from this session, or another local account could "
f"bind the now-free port {port} and receive its requests. Do not use `lite autoroute up` on a "
f"shared or multi-tenant host."
)
def _handle_signal(_signum: int, _frame: FrameType | None) -> None:
stop_event.set()
signal.signal(signal.SIGINT, _handle_signal)
signal.signal(signal.SIGTERM, _handle_signal)
atexit.register(_teardown)
log_thread = threading.Thread(target=stream_log, args=(LOG_PATH, stop_event), daemon=True)
log_thread.start()
stop_event.wait()
_teardown()
@autoroute_group.command("down")
def down() -> None:
"""Restore Claude Code settings and stop a leftover ephemeral proxy, if any"""
try:
record: PidRecord | None = read_pid_record()
except UpError as e:
# down is the crash-recovery path -- a corrupt pid record must not block it; clear the
# unusable record and keep going rather than leaving the user with no way to clean up.
click.echo(f"{e} Clearing it and continuing cleanup.", err=True)
record = None
if record is not None and is_running(record.pid):
terminate(record.pid)
click.echo(f"Stopped leftover ephemeral proxy (pid {record.pid}).")
clear_pid_record()
try:
restored = restore_claude_settings(CLAUDE_SETTINGS_PATH, AUTOROUTE_BACKUP_PATH)
except UpError as e:
raise click.ClickException(str(e))
if restored is None:
click.echo("Nothing to restore.")
elif restored.existed:
click.echo(f"Restored {CLAUDE_SETTINGS_PATH} to its original contents.")
else:
click.echo(f"Removed {CLAUDE_SETTINGS_PATH} (it did not exist before `lite autoroute up`).")
__all__ = ["autoroute_group"]

249
litellm/proxy/client/cli/commands/autoroute/config.py Normal file
View file

@ -0,0 +1,249 @@
from typing import Literal, Union
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter
TIER_NAMES: tuple[str, ...] = ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")
AUTOROUTER_MODEL_NAME = "autorouter"
class ConfigGenerationError(Exception):
"""Raised when an AutorouteConfig references a model the discovery step didn't find."""
class DiscoveredModel(BaseModel):
model_config = ConfigDict(frozen=True)
name: str
mode: str = "chat"
input_cost_per_token: float | None = None
output_cost_per_token: float | None = None
class _RawModelGroup(BaseModel):
model_config = ConfigDict(extra="ignore")
model_group: str
# Optional: some real deployments return an explicit `"mode": null` for models that
# were registered without a mode (seen for embedding models like voyage-4-large).
# ModelGroupInfo's own "chat" default (litellm/types/router.py) only applies when the
# key is missing entirely, not when it's present as null, so this must tolerate None.
mode: str | None = "chat"
input_cost_per_token: float | None = None
output_cost_per_token: float | None = None
_RAW_MODEL_GROUPS_ADAPTER = TypeAdapter(list[_RawModelGroup])
def parse_discovered_models(raw: list[JsonValue]) -> tuple[DiscoveredModel, ...]:
"""Validate a raw `/model_group/info` response into typed models."""
parsed = _RAW_MODEL_GROUPS_ADAPTER.validate_python(raw)
return tuple(
DiscoveredModel(
name=group.model_group,
# A null mode means the server genuinely doesn't know what this model does;
# "unknown" (rather than guessing "chat") keeps it out of both chat_models()
# and embedding_models() instead of risking a wrong-mode deployment.
mode=group.mode or "unknown",
input_cost_per_token=group.input_cost_per_token,
output_cost_per_token=group.output_cost_per_token,
)
for group in parsed
)
def chat_models(models: tuple[DiscoveredModel, ...]) -> tuple[DiscoveredModel, ...]:
return tuple(m for m in models if m.mode == "chat")
def embedding_models(models: tuple[DiscoveredModel, ...]) -> tuple[DiscoveredModel, ...]:
return tuple(m for m in models if m.mode == "embedding")
class HeuristicClassifier(BaseModel):
model_config = ConfigDict(frozen=True)
kind: Literal["heuristic"] = "heuristic"
class LLMClassifier(BaseModel):
model_config = ConfigDict(frozen=True)
kind: Literal["llm"] = "llm"
model: str
timeout_ms: int = 3000
ClassifierChoice = Union[HeuristicClassifier, LLMClassifier]
class NoSemanticMatching(BaseModel):
model_config = ConfigDict(frozen=True)
kind: Literal["none"] = "none"
class KeywordTierRule(BaseModel):
model_config = ConfigDict(frozen=True)
keywords: tuple[str, ...]
tier: str
# Satisfies complexity_router's "semantic matching requires non-empty keyword_tier_rules"
# invariant with a sane starting point; the wizard lets the user override these per tier.
DEFAULT_KEYWORD_TIER_RULES: tuple[KeywordTierRule, ...] = (
KeywordTierRule(keywords=("hi", "hello", "thanks"), tier="SIMPLE"),
KeywordTierRule(keywords=("explain", "how does"), tier="MEDIUM"),
KeywordTierRule(keywords=("refactor", "implement", "debug"), tier="COMPLEX"),
KeywordTierRule(keywords=("step by step", "think through", "prove"), tier="REASONING"),
)
class SemanticMatching(BaseModel):
model_config = ConfigDict(frozen=True)
kind: Literal["semantic"] = "semantic"
embedding_model: str
match_threshold: float = 0.5
keyword_tier_rules: tuple[KeywordTierRule, ...] = DEFAULT_KEYWORD_TIER_RULES
SemanticMatchingChoice = Union[NoSemanticMatching, SemanticMatching]
class AutorouteConfig(BaseModel):
model_config = ConfigDict(frozen=True)
base_url: str
api_key: str
# Each tier maps to a pool of one or more models; complexity_router picks randomly among
# them per request (or, in adaptive mode, learns which to prefer within the pool).
tiers: dict[str, tuple[str, ...]]
default_model: str
classifier: ClassifierChoice = Field(default_factory=HeuristicClassifier)
semantic_matching: SemanticMatchingChoice = Field(default_factory=NoSemanticMatching)
adaptive: bool = False
def validate_config(config: AutorouteConfig, discovered: tuple[DiscoveredModel, ...]) -> None:
"""Raise ConfigGenerationError if config references a model discovery didn't return."""
chat_names: frozenset[str] = frozenset(m.name for m in chat_models(discovered))
embedding_names: frozenset[str] = frozenset(m.name for m in embedding_models(discovered))
for tier, models in config.tiers.items():
for model in models:
if model not in chat_names:
raise ConfigGenerationError(f"Tier {tier} references unknown chat model '{model}'")
if config.default_model not in chat_names:
raise ConfigGenerationError(f"default_model '{config.default_model}' is not a known chat model")
if isinstance(config.classifier, LLMClassifier) and config.classifier.model not in chat_names:
raise ConfigGenerationError(f"classifier model '{config.classifier.model}' is not a known chat model")
if (
isinstance(config.semantic_matching, SemanticMatching)
and config.semantic_matching.embedding_model not in embedding_names
):
raise ConfigGenerationError(
f"embedding model '{config.semantic_matching.embedding_model}' is not a known embedding model"
)
def _litellm_proxy_deployment(name: str, base_url: str, api_key: str) -> dict[str, JsonValue]:
return {
"model_name": name,
"litellm_params": {
"model": f"litellm_proxy/{name}",
"api_base": base_url,
"api_key": api_key,
},
}
def build_generated_model_list(config: AutorouteConfig) -> list[JsonValue]:
"""Build the model_list for the ephemeral proxy's config.yaml.
Every real model referenced anywhere (tier targets, classifier, embedding) is deduplicated
to exactly one `litellm_proxy/<name>` deployment forwarding to the customer's real proxy,
plus one `auto_router/complexity_router` deployment tying the tiers together.
"""
referenced_names = {model for models in config.tiers.values() for model in models}
referenced_names.add(config.default_model)
if isinstance(config.classifier, LLMClassifier):
referenced_names.add(config.classifier.model)
if isinstance(config.semantic_matching, SemanticMatching):
referenced_names.add(config.semantic_matching.embedding_model)
proxy_deployments = [
_litellm_proxy_deployment(name, config.base_url, config.api_key) for name in sorted(referenced_names)
]
complexity_router_config: dict[str, JsonValue] = {
"tiers": {tier: list(models) for tier, models in config.tiers.items()},
"default_model": config.default_model,
}
if isinstance(config.classifier, LLMClassifier):
complexity_router_config["classifier_type"] = "llm"
complexity_router_config["classifier_llm_config"] = {
"model": config.classifier.model,
"timeout_ms": config.classifier.timeout_ms,
}
if isinstance(config.semantic_matching, SemanticMatching):
complexity_router_config["semantic_keyword_matching"] = True
complexity_router_config["embedding_model"] = config.semantic_matching.embedding_model
complexity_router_config["match_threshold"] = config.semantic_matching.match_threshold
complexity_router_config["keyword_tier_rules"] = [
{"keywords": list(rule.keywords), "tier": rule.tier} for rule in config.semantic_matching.keyword_tier_rules
]
if config.adaptive:
complexity_router_config["adaptive"] = True
auto_router_litellm_params: dict[str, JsonValue] = {
"model": "auto_router/complexity_router",
"complexity_router_config": complexity_router_config,
}
# A bare "*" model_name looks like the obvious way to catch every request Claude Code
# might send regardless of which model it thinks it's using, but Router's auto-router
# registry is keyed by the literal requested model string (router.py:10711-10717), not
# resolved through pattern/wildcard matching first -- so a "*" entry here would only ever
# match a client that literally sends model="*", never an actual wildcard catch-all. Callers
# instead need to make Claude Code request this "autorouter" name directly (see
# ANTHROPIC_DEFAULT_*_MODEL in settings.py's merge_claude_settings_static_token).
return [
*proxy_deployments,
{"model_name": AUTOROUTER_MODEL_NAME, "litellm_params": auto_router_litellm_params},
]
def build_generated_proxy_config(config: AutorouteConfig, master_key: str) -> dict[str, JsonValue]:
"""Full config.yaml content for the ephemeral proxy, including its own auth key.
master_key must live under general_settings, not litellm_settings -- the proxy server
only ever reads general_settings.master_key (proxy_server.py:4530) to authenticate
requests; a key placed under litellm_settings is silently ignored, leaving the proxy
with no real auth at all.
"""
return {
"model_list": build_generated_model_list(config),
"general_settings": {"master_key": master_key},
}
__all__ = [
"AUTOROUTER_MODEL_NAME",
"TIER_NAMES",
"AutorouteConfig",
"ClassifierChoice",
"ConfigGenerationError",
"DEFAULT_KEYWORD_TIER_RULES",
"DiscoveredModel",
"HeuristicClassifier",
"KeywordTierRule",
"LLMClassifier",
"NoSemanticMatching",
"SemanticMatching",
"SemanticMatchingChoice",
"build_generated_model_list",
"build_generated_proxy_config",
"chat_models",
"embedding_models",
"parse_discovered_models",
"validate_config",
]

190
litellm/proxy/client/cli/commands/autoroute/process.py Normal file
View file

@ -0,0 +1,190 @@
import contextlib
import importlib.util
import json
import os
import signal
import socket
import subprocess
import sys
import threading
import time
from dataclasses import dataclass
from pathlib import Path
import click
import requests
from pydantic import TypeAdapter, ValidationError
from ..up import UpError, secure_create
AUTOROUTE_DIR = Path.home() / ".litellm" / "autorouter"
CONFIG_PATH = AUTOROUTE_DIR / "config.yaml"
LOG_PATH = AUTOROUTE_DIR / "proxy.log"
PID_RECORD_PATH = AUTOROUTE_DIR / "proxy.pid.json"
class ProcessLaunchError(Exception):
"""Raised when the ephemeral proxy subprocess fails to come up healthy."""
@dataclass(frozen=True, slots=True)
class PidRecord:
pid: int
port: int
config_path: str
log_path: str
_PID_RECORD_ADAPTER = TypeAdapter(PidRecord)
_PROXY_RUNTIME_MODULES: tuple[str, ...] = ("fastapi", "uvicorn", "backoff", "orjson", "websockets", "apscheduler")
def missing_proxy_runtime_modules() -> tuple[str, ...]:
"""Proxy-server modules that ``lite autoroute up`` needs but the thin CLI install lacks.
``launch_proxy`` runs the full ``litellm.proxy.proxy_cli`` server, whose dependencies live in
the ``proxy`` extra, not the ``cli`` extra that installs the ``lite`` command. On a thin
``litellm[cli]`` install the subprocess dies with a bare ``ModuleNotFoundError``; detecting the
gap here lets ``up`` fail with an actionable message instead.
"""
return tuple(name for name in _PROXY_RUNTIME_MODULES if importlib.util.find_spec(name) is None)
def allocate_free_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(("127.0.0.1", 0))
return int(sock.getsockname()[1])
def launch_proxy(config_path: Path, port: int, log_path: Path) -> "subprocess.Popen[bytes]":
log_path.parent.mkdir(parents=True, exist_ok=True)
with open(log_path, "w") as log_file:
return subprocess.Popen(
[
sys.executable,
"-m",
"litellm.proxy.proxy_cli",
"--config",
str(config_path),
"--port",
str(port),
"--host",
"127.0.0.1",
],
stdout=log_file,
stderr=subprocess.STDOUT,
)
def _tail(log_path: Path, lines: int = 40) -> str:
if not log_path.exists():
return "(no log output captured)"
return "\n".join(log_path.read_text(errors="replace").splitlines()[-lines:])
def poll_liveliness(base_url: str, log_path: Path, process: "subprocess.Popen[bytes]", timeout: float = 30.0) -> None:
"""Poll /health/liveliness until it responds, the process dies, or timeout elapses."""
deadline = time.monotonic() + timeout
url = base_url.rstrip("/") + "/health/liveliness"
while time.monotonic() < deadline:
if process.poll() is not None:
raise ProcessLaunchError(
f"Ephemeral proxy exited early (code {process.returncode}). Last log lines:\n{_tail(log_path)}"
)
with contextlib.suppress(requests.RequestException):
if requests.get(url, timeout=2).status_code == 200:
return
time.sleep(0.5)
raise ProcessLaunchError(
f"Ephemeral proxy never became healthy within {timeout}s. Last log lines:\n{_tail(log_path)}"
)
def write_pid_record(record: PidRecord, path: Path | None = None) -> None:
resolved_path = path if path is not None else PID_RECORD_PATH
resolved_path.parent.mkdir(parents=True, exist_ok=True)
with open(resolved_path, "w") as f:
json.dump(
{"pid": record.pid, "port": record.port, "config_path": record.config_path, "log_path": record.log_path},
f,
indent=2,
)
def read_pid_record(path: Path | None = None) -> PidRecord | None:
resolved_path = path if path is not None else PID_RECORD_PATH
if not resolved_path.exists():
return None
with open(resolved_path, "r") as f:
content = f.read()
try:
return _PID_RECORD_ADAPTER.validate_json(content)
except ValidationError:
raise UpError(f"{resolved_path} contains invalid or unexpected JSON; cannot proceed safely.")
def clear_pid_record(path: Path | None = None) -> None:
resolved_path = path if path is not None else PID_RECORD_PATH
resolved_path.unlink(missing_ok=True)
def is_running(pid: int) -> bool:
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
def terminate(pid: int, grace_period: float = 5.0) -> None:
"""Terminate a process by pid, escalating from SIGTERM to SIGKILL if needed."""
if not is_running(pid):
return
with contextlib.suppress(ProcessLookupError):
os.kill(pid, signal.SIGTERM)
deadline = time.monotonic() + grace_period
while time.monotonic() < deadline and is_running(pid):
time.sleep(0.2)
if is_running(pid):
with contextlib.suppress(ProcessLookupError):
os.kill(pid, signal.SIGKILL)
def stream_log(log_path: Path, stop_event: threading.Event) -> None:
"""Print new lines appended to log_path until stop_event is set. Blocks the calling thread."""
while not log_path.exists() and not stop_event.is_set():
time.sleep(0.1)
if stop_event.is_set() or not log_path.exists():
return
with open(log_path, "r") as f:
while not stop_event.is_set():
line = f.readline()
if line:
click.echo(line, nl=False)
else:
time.sleep(0.2)
__all__ = [
"AUTOROUTE_DIR",
"CONFIG_PATH",
"LOG_PATH",
"PID_RECORD_PATH",
"PidRecord",
"ProcessLaunchError",
"allocate_free_port",
"clear_pid_record",
"is_running",
"launch_proxy",
"missing_proxy_runtime_modules",
"poll_liveliness",
"read_pid_record",
"secure_create",
"stream_log",
"terminate",
"write_pid_record",
]

46
litellm/proxy/client/cli/commands/autoroute/settings.py Normal file
View file

@ -0,0 +1,46 @@
from pydantic import JsonValue
from .config import AUTOROUTER_MODEL_NAME
ENV_KEY = "env"
API_KEY_HELPER_KEY = "apiKeyHelper"
ANTHROPIC_API_KEY_KEY = "ANTHROPIC_API_KEY"
ANTHROPIC_AUTH_TOKEN_KEY = "ANTHROPIC_AUTH_TOKEN"
ANTHROPIC_BASE_URL_KEY = "ANTHROPIC_BASE_URL"
# Force every one of Claude Code's own model tiers to request the auto-router by name.
# Router's auto-router registry is keyed by the literal requested model string
# (litellm/router.py:10711-10717) with no wildcard/pattern resolution, so a bare "*"
# model_name can never work as a catch-all -- these overrides are what actually makes
# Claude Code send "autorouter" regardless of /model or its own version-specific defaults.
ANTHROPIC_DEFAULT_MODEL_ENV_KEYS = (
"ANTHROPIC_DEFAULT_SONNET_MODEL",
"ANTHROPIC_DEFAULT_HAIKU_MODEL",
"ANTHROPIC_DEFAULT_OPUS_MODEL",
)
def merge_claude_settings_static_token(
settings: dict[str, JsonValue], base_url: str, auth_token: str
) -> dict[str, JsonValue]:
"""Return a new settings dict wired to a local ephemeral proxy with a static token.
Unlike up.py's merge_claude_settings (which sets apiKeyHelper for a long-lived, real
remote proxy needing refreshable SSO tokens), this proxy is ephemeral and its key was just
minted for this session, so a plain env var is simpler and correct. Any existing
apiKeyHelper is cleared so it can't fight with the static token.
"""
raw_env = settings.get(ENV_KEY, {})
base_env = raw_env if isinstance(raw_env, dict) else {}
env: dict[str, JsonValue] = {
**base_env,
ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"),
ANTHROPIC_AUTH_TOKEN_KEY: auth_token,
**{key: AUTOROUTER_MODEL_NAME for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS},
}
env.pop(ANTHROPIC_API_KEY_KEY, None)
merged: dict[str, JsonValue] = {**settings, ENV_KEY: env}
merged.pop(API_KEY_HELPER_KEY, None)
return merged
__all__ = ["merge_claude_settings_static_token"]

150
litellm/proxy/client/cli/commands/autoroute/wizard.py Normal file
View file

@ -0,0 +1,150 @@
import sys
from pathlib import Path
import click
import yaml
from InquirerPy import inquirer
from InquirerPy.base.control import Choice
from .... import Client
from .config import (
DEFAULT_KEYWORD_TIER_RULES,
TIER_NAMES,
AutorouteConfig,
ConfigGenerationError,
DiscoveredModel,
HeuristicClassifier,
KeywordTierRule,
LLMClassifier,
NoSemanticMatching,
SemanticMatching,
build_generated_model_list,
chat_models,
embedding_models,
parse_discovered_models,
validate_config,
)
from .process import CONFIG_PATH, secure_create
def _is_interactive() -> bool:
return sys.stdin.isatty()
def _fuzzy_pick(models: tuple[DiscoveredModel, ...], prompt_label: str, multiselect: bool) -> list[str]:
"""Type-to-filter picker over a (possibly huge) model pool, using InquirerPy's fzf-style fuzzy prompt.
A plain numbered table + typed index does not scale past a handful of models -- proxies with
hundreds of model groups made that interaction unusable. This lets the user narrow the pool by
typing a substring instead of scrolling/counting.
Assumes the caller already checked interactivity (run_configure_wizard does, once, up front) --
checking here too would check the wrong thing under test, where InquirerPy is driven through its
own injected input/output rather than the real process stdin.
"""
choices = [Choice(value=model.name, name=model.name) for model in models]
toggle_hint = "tab to toggle, " if multiselect else ""
while True:
result = inquirer.fuzzy(
message=f"{prompt_label}: type to filter, {toggle_hint}enter to confirm",
choices=choices,
multiselect=multiselect,
max_height="70%",
).execute()
selected = result if multiselect else [result]
if selected:
return selected
click.echo("Select at least one model.")
def _render_and_prompt_for_model(models: tuple[DiscoveredModel, ...], prompt_label: str) -> str:
return _fuzzy_pick(models, prompt_label, multiselect=False)[0]
def _render_and_prompt_for_models(models: tuple[DiscoveredModel, ...], prompt_label: str) -> tuple[str, ...]:
return tuple(_fuzzy_pick(models, prompt_label, multiselect=True))
def _parse_keywords(raw: str) -> tuple[str, ...]:
return tuple(keyword.strip() for keyword in raw.split(",") if keyword.strip())
def _prompt_for_keyword_tier_rules() -> tuple[KeywordTierRule, ...]:
"""Let the user supply the semantic-matching keywords per tier, since matching those
keywords against the request is the whole point of enabling it. Each prompt is prefilled
with the built-in default, so pressing enter keeps it."""
click.echo("\nEnter example keywords/phrases per tier (comma-separated); press enter to keep the default:")
defaults = {rule.tier: rule.keywords for rule in DEFAULT_KEYWORD_TIER_RULES}
def _rule_for(tier: str) -> KeywordTierRule:
default_keywords = defaults.get(tier, ())
raw = click.prompt(f" {tier} keywords", default=", ".join(default_keywords), show_default=True)
return KeywordTierRule(keywords=_parse_keywords(raw) or default_keywords, tier=tier)
return tuple(_rule_for(tier) for tier in TIER_NAMES)
def run_configure_wizard(ctx: click.Context) -> Path:
"""Discover the caller's accessible models, walk them through tier assignment, write config."""
base_url = ctx.obj["base_url"]
api_key = ctx.obj["api_key"]
client = Client(base_url=base_url, api_key=api_key)
raw_groups = client.model_groups.info()
if not isinstance(raw_groups, list):
raise click.ClickException(
f"Unexpected response from /model_group/info: expected a list, got {type(raw_groups).__name__}"
)
discovered = parse_discovered_models(raw_groups)
chat_pool = chat_models(discovered)
embedding_pool = embedding_models(discovered)
if not chat_pool:
raise click.ClickException("Your key has no chat-capable models available on this proxy.")
if not _is_interactive():
raise click.ClickException("`lite autoroute configure` requires an interactive terminal.")
click.echo("Assign model(s) to each complexity tier (from what your key can access):")
tiers = {tier: _render_and_prompt_for_models(chat_pool, tier) for tier in TIER_NAMES}
default_model = tiers["MEDIUM"][0]
classifier = HeuristicClassifier()
if click.confirm("\nUse an LLM classifier instead of the free heuristic scorer?", default=False):
classifier_model = _render_and_prompt_for_model(chat_pool, "LLM classifier")
classifier = LLMClassifier(model=classifier_model)
semantic_matching = NoSemanticMatching()
if embedding_pool and click.confirm("\nEnable semantic keyword matching?", default=False):
embedding_model = _render_and_prompt_for_model(embedding_pool, "semantic embeddings")
keyword_tier_rules = _prompt_for_keyword_tier_rules()
semantic_matching = SemanticMatching(embedding_model=embedding_model, keyword_tier_rules=keyword_tier_rules)
adaptive = click.confirm("\nEnable adaptive (bandit) selection on top of tiering?", default=False)
config = AutorouteConfig(
base_url=base_url,
api_key=api_key,
tiers=tiers,
default_model=default_model,
classifier=classifier,
semantic_matching=semantic_matching,
adaptive=adaptive,
)
try:
validate_config(config, discovered)
except ConfigGenerationError as e:
raise click.ClickException(str(e))
model_list = build_generated_model_list(config)
CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
with secure_create(CONFIG_PATH) as f:
yaml.safe_dump({"model_list": model_list}, f, sort_keys=False)
click.echo(f"\nWrote {CONFIG_PATH}")
for tier, models in tiers.items():
click.echo(f" {tier}: {', '.join(models)}")
return CONFIG_PATH
__all__ = ["run_configure_wizard"]

2
litellm/proxy/client/cli/commands/chat.py
View file

@ -150,7 +150,7 @@ def chat(
f"Max Tokens: [yellow]{max_tokens or 'unlimited'}[/yellow]\n\n"
f"Type your messages and press Enter. Type '/quit' or '/exit' to end the session.\n"
f"Type '/help' for more commands.",
title="🤖 Chat Session",
title="Chat Session",
)
)

2
litellm/proxy/client/cli/commands/encryption.py
View file

@ -32,7 +32,7 @@ def migrate(ctx: click.Context, check_only: bool, dry_run: bool):
Requires the proxy to be started with
``general_settings.encryption_algorithm: aes-256-gcm``. Idempotent and
resumable — safe to re-run after an interruption.
resumable; safe to re-run after an interruption.
Examples:
litellm-proxy encryption migrate --check # attestation scan, no writes

4
litellm/proxy/client/cli/commands/keys.py
View file

@ -309,12 +309,12 @@ def _import_keys_to_destination(
imported_count += 1
key_alias = key.get("key_alias", "N/A")
click.echo(f"✓ Imported key: {key_alias}")
click.echo(f"Imported key: {key_alias}")
except Exception as e:
failed_count += 1
key_alias = key.get("key_alias", "N/A")
click.echo(f"✗ Failed to import key {key_alias}: {str(e)}", err=True)
click.echo(f"Failed to import key {key_alias}: {str(e)}", err=True)
return imported_count, failed_count

57
litellm/proxy/client/cli/commands/model_groups.py Normal file
View file

@ -0,0 +1,57 @@
from typing import Literal
import click
import rich
import rich.table
from ... import Client
def create_client(ctx: click.Context) -> Client:
return Client(base_url=ctx.obj["base_url"], api_key=ctx.obj["api_key"])
@click.group(name="model-groups")
def model_groups() -> None:
"""Inspect model groups your key can access on the proxy"""
@model_groups.command("list")
@click.option(
"--format",
"output_format",
type=click.Choice(["table", "json"]),
default="table",
help="Output format (table or json)",
)
@click.pass_context
def list_model_groups(ctx: click.Context, output_format: Literal["table", "json"]) -> None:
"""List model groups accessible to your key, with mode and pricing"""
client = create_client(ctx)
groups = client.model_groups.info()
if not isinstance(groups, list):
raise click.ClickException(
f"Unexpected response from /model_group/info: expected a list, got {type(groups).__name__}"
)
if output_format == "json":
rich.print_json(data=groups)
return
table = rich.table.Table(title="Accessible Model Groups")
table.add_column("Model", style="cyan")
table.add_column("Mode", style="green")
table.add_column("Input $/token", style="yellow")
table.add_column("Output $/token", style="yellow")
for group in groups:
table.add_row(
str(group.get("model_group", "")),
str(group.get("mode", "chat")),
str(group.get("input_cost_per_token", "")),
str(group.get("output_cost_per_token", "")),
)
rich.print(table)
__all__ = ["model_groups"]

20
litellm/proxy/client/cli/commands/teams.py
View file

@ -21,7 +21,7 @@ def display_teams_table(teams: List[Dict[str, Any]]) -> None:
console = Console()
if not teams:
console.print("❌ No teams found for your user.")
console.print("No teams found for your user.")
return
table = Table(title="Available Teams")
@ -91,10 +91,10 @@ def available(ctx: click.Context):
teams = client.teams.get_available()
if teams:
console = Console()
console.print("\n🎯 Available Teams to Join:")
console.print("\nAvailable Teams to Join:")
display_teams_table(teams)
else:
click.echo("ℹ️ No available teams to join.")
click.echo("No available teams to join.")
except requests.exceptions.HTTPError as e:
click.echo(f"Error: HTTP {e.response.status_code}", err=True)
error_body = e.response.json()
@ -113,7 +113,7 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
api_key = ctx.obj["api_key"]
if not api_key:
click.echo("❌ No API key found. Please login first using 'litellm login'")
click.echo("No API key found. Please login first using 'litellm login'")
raise click.Abort()
try:
@ -122,7 +122,7 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
teams = client.teams.list()
if not teams:
click.echo("❌ No teams found for your user.")
click.echo("No teams found for your user.")
return
# Use interactive selection from auth module
@ -133,14 +133,14 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
if selected_team:
team_id = selected_team.get("team_id")
else:
click.echo("❌ Operation cancelled.")
click.echo("Operation cancelled.")
return
# Update the key with the selected team
if team_id:
click.echo(f"\n🔄 Assigning your key to team: {team_id}")
click.echo(f"\nAssigning your key to team: {team_id}")
client.keys.update(key=api_key, team_id=team_id)
click.echo(f"✅ Successfully assigned key to team: {team_id}")
click.echo(f"Successfully assigned key to team: {team_id}")
# Show team details if available
teams = client.teams.list()
@ -148,9 +148,9 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
if team.get("team_id") == team_id:
models = team.get("models", [])
if models:
click.echo(f"🎯 You can now access models: {', '.join(models)}")
click.echo(f"You can now access models: {', '.join(models)}")
else:
click.echo("🎯 You can now access all available models")
click.echo("You can now access all available models")
break
except requests.exceptions.HTTPError as e:

283
litellm/proxy/client/cli/commands/up.py Normal file
View file

@ -0,0 +1,283 @@
import atexit
import contextlib
import json
import os
import shlex
import shutil
import signal
import sys
import threading
from dataclasses import dataclass
from pathlib import Path
from types import FrameType
from typing import IO, Iterator, Mapping
import click
from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh
from .agents import AgentRunError, resolve_api_key, verify_proxy_key
from .auth import load_token, login
ENV_KEY = "env"
API_KEY_HELPER_KEY = "apiKeyHelper"
ANTHROPIC_BASE_URL_KEY = "ANTHROPIC_BASE_URL"
ANTHROPIC_API_KEY_KEY = "ANTHROPIC_API_KEY"
CLAUDE_SETTINGS_PATH = Path.home() / ".claude" / "settings.json"
BACKUP_PATH = Path.home() / ".litellm" / "claude_settings_backup.json"
class UpError(Exception):
"""Raised for any user-actionable failure while starting/stopping interception."""
@dataclass(frozen=True, slots=True)
class BackupRecord:
"""Snapshot of ~/.claude/settings.json taken right before `lite up` patches it."""
existed: bool
content: dict[str, JsonValue] | None
_SETTINGS_ADAPTER = TypeAdapter(dict[str, JsonValue])
_BACKUP_RECORD_ADAPTER = TypeAdapter(BackupRecord)
def load_json_or_empty(path: Path) -> dict[str, JsonValue]:
if not path.exists():
return {}
with open(path, "r") as f:
content = f.read()
if not content.strip():
return {}
try:
return _SETTINGS_ADAPTER.validate_json(content)
except ValidationError:
raise UpError(f"{path} contains invalid JSON (or its root is not an object); cannot proceed safely.")
def merge_claude_settings(
settings: Mapping[str, JsonValue], base_url: str, api_key_helper: str
) -> dict[str, JsonValue]:
"""Return a new settings dict wired to route Claude Code through the proxy.
Only env.ANTHROPIC_BASE_URL and the top-level apiKeyHelper are overridden; a
stray env.ANTHROPIC_API_KEY is dropped so it cannot outrank the helper-issued
token (same reasoning as build_agent_env in agents.py). Every other key is
preserved untouched.
"""
raw_env = settings.get(ENV_KEY, {})
base_env = raw_env if isinstance(raw_env, dict) else {}
env = {**base_env, ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/")}
env.pop(ANTHROPIC_API_KEY_KEY, None)
return {**settings, ENV_KEY: env, API_KEY_HELPER_KEY: api_key_helper}
@contextlib.contextmanager
def secure_create(path: Path) -> Iterator[IO[str]]:
"""Open path for writing with mode 0600 fixed up before any content is written.
A plain `open(path, "w")` creates a *new* file at the umask-derived default (commonly 0644)
and leaves it world- or group-readable until a later `chmod` call catches up -- a real window
in which a file holding a credential is readable by another local account. Passing the mode to
`os.open` closes that window for a brand-new file, but `O_CREAT`'s mode argument is only
applied on creation: if the file already exists its old, broader permissions carry over
untouched. `os.fchmod` right after opening -- before a single byte of the new content is
written -- covers both cases.
"""
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
os.fchmod(fd, 0o600)
f: IO[str] = os.fdopen(fd, "w")
try:
yield f
finally:
f.close()
def write_backup(record: BackupRecord, backup_path: Path | None = None) -> None:
path = backup_path if backup_path is not None else BACKUP_PATH
path.parent.mkdir(exist_ok=True)
with secure_create(path) as f:
json.dump({"existed": record.existed, "content": record.content}, f, indent=2)
def read_backup(backup_path: Path | None = None) -> BackupRecord | None:
path = backup_path if backup_path is not None else BACKUP_PATH
if not path.exists():
return None
with open(path, "r") as f:
content = f.read()
try:
return _BACKUP_RECORD_ADAPTER.validate_json(content)
except ValidationError:
raise UpError(f"{path} contains invalid or unexpected JSON; cannot restore from it safely.")
def restore_claude_settings(settings_path: Path | None = None, backup_path: Path | None = None) -> BackupRecord | None:
"""Restore settings_path from the backup at backup_path, then delete the backup.
Returns the restored record, or None if there was nothing to restore.
"""
resolved_settings_path = settings_path if settings_path is not None else CLAUDE_SETTINGS_PATH
resolved_backup_path = backup_path if backup_path is not None else BACKUP_PATH
record = read_backup(resolved_backup_path)
if record is None:
return None
if record.existed and record.content is not None:
resolved_settings_path.parent.mkdir(parents=True, exist_ok=True)
with open(resolved_settings_path, "w") as f:
json.dump(record.content, f, indent=2)
elif resolved_settings_path.exists():
resolved_settings_path.unlink()
resolved_backup_path.unlink()
return record
def resolve_api_key_helper(base_url: str) -> str:
"""Build the shell command Claude Code should run for its apiKeyHelper.
Resolves `lite` to an absolute path so the helper works regardless of the
PATH visible to whatever subprocess Claude Code spawns it from. Passing
--base-url explicitly (rather than relying on the bare invocation Claude
Code would otherwise use) makes `print-token` enforce that the cached
token was actually issued for this proxy -- without it, a token minted
for a different, previously-logged-into proxy would be handed to
whichever server `up` currently points at.
"""
lite_path = shutil.which("lite")
if lite_path is None:
raise UpError(
"Could not find `lite` on your PATH. Claude Code's apiKeyHelper needs "
"an absolute path to it, so `lite up` cannot continue."
)
return f"{shlex.quote(lite_path)} auth print-token --base-url {shlex.quote(base_url)}"
def _ensure_fresh_login(ctx: click.Context) -> None:
base_url = ctx.obj["base_url"].rstrip("/")
token_data = load_token()
if token_data and token_data.get("base_url") == base_url and is_cli_token_fresh(token_data):
return
if not sys.stdin.isatty():
raise UpError(
"No fresh LiteLLM login found for this proxy. Run `lite login` first (apiKeyHelper "
"reads this token on every Claude Code request)."
)
click.echo("No fresh LiteLLM login found for this proxy; starting login...")
ctx.invoke(login)
token_data = load_token()
if not token_data or token_data.get("base_url") != base_url or not is_cli_token_fresh(token_data):
raise UpError("Login did not produce a usable token; cannot start `lite up`.")
def _restore_and_report() -> None:
record = restore_claude_settings()
if record is None:
click.echo("Nothing to restore.")
return
if record.existed:
click.echo(f"Restored {CLAUDE_SETTINGS_PATH} to its original contents.")
else:
click.echo(f"Removed {CLAUDE_SETTINGS_PATH} (it did not exist before `lite up`).")
@click.command(name="up")
@click.pass_context
def up(ctx: click.Context) -> None:
"""Route every Claude Code session through your LiteLLM proxy until stopped.
Patches ~/.claude/settings.json so Claude Code picks up the proxy on its own
next startup, from any terminal -- no need to launch it through `lite`.
Press Ctrl-C to stop and restore your original settings. Assumes the proxy
is already running (this does not start one for you). Cursor is not
supported: it has no equivalent file-based config to patch.
"""
base_url = ctx.obj["base_url"]
try:
_ensure_fresh_login(ctx)
api_key = resolve_api_key(ctx)
verify_proxy_key(base_url, api_key)
if BACKUP_PATH.exists():
raise UpError(
f"{BACKUP_PATH} already exists -- `lite up` looks like it's already "
"running (or crashed without cleanup). Run `lite down` first."
)
api_key_helper = resolve_api_key_helper(base_url)
original_existed = CLAUDE_SETTINGS_PATH.exists()
original_settings = load_json_or_empty(CLAUDE_SETTINGS_PATH)
write_backup(
BackupRecord(
existed=original_existed,
content=original_settings if original_existed else None,
)
)
CLAUDE_SETTINGS_PATH.parent.mkdir(exist_ok=True)
merged = merge_claude_settings(original_settings, base_url, api_key_helper)
with open(CLAUDE_SETTINGS_PATH, "w") as f:
json.dump(merged, f, indent=2)
except (AgentRunError, UpError) as e:
raise click.ClickException(str(e))
click.echo(f"litellm: routing Claude Code through proxy at {base_url.rstrip('/')}")
click.echo("Press Ctrl-C to stop and restore your original settings.")
stop_event = threading.Event()
restored = threading.Lock()
def _handle_signal(_signum: int, _frame: FrameType | None) -> None:
stop_event.set()
def _restore_once() -> None:
if not restored.acquire(blocking=False):
return
try:
_restore_and_report()
except UpError as e:
# Runs from atexit/a signal handler, outside Click's own exception
# handling -- raising here would only produce an unhandled-exception
# warning on stderr, not a clean message.
click.echo(str(e), err=True)
signal.signal(signal.SIGINT, _handle_signal)
signal.signal(signal.SIGTERM, _handle_signal)
atexit.register(_restore_once)
stop_event.wait()
_restore_once()
@click.command(name="down")
def down() -> None:
"""Restore ~/.claude/settings.json if a `lite up` session left it patched.
Use this after a `lite up` process was killed uncleanly (e.g. `kill -9`)
instead of stopped with Ctrl-C.
"""
try:
_restore_and_report()
except UpError as e:
raise click.ClickException(str(e))
__all__ = [
"BACKUP_PATH",
"CLAUDE_SETTINGS_PATH",
"BackupRecord",
"UpError",
"down",
"load_json_or_empty",
"merge_claude_settings",
"read_backup",
"resolve_api_key_helper",
"restore_claude_settings",
"up",
"write_backup",
]

14
litellm/proxy/client/cli/interface.py
View file

@ -27,13 +27,13 @@ def styled_prompt():
verbose_logger.debug(f"Error getting terminal size: {e}")
click.echo("\n" * 3)
# Unicode box drawing characters
top_left = "┌"
top_right = "┐"
bottom_left = "└"
bottom_right = "┘"
horizontal = "─"
vertical = "│"
# ASCII box drawing characters
top_left = "+"
top_right = "+"
bottom_left = "+"
bottom_right = "+"
horizontal = "-"
vertical = "|"
# Create the box with increased width
width = 80

10
litellm/proxy/client/cli/main.py
View file

@ -9,15 +9,18 @@ from litellm.proxy.client.health import HealthManagementClient
from .commands.agents import agent_commands
from .commands.auth import auth_group, get_stored_api_key, login, logout, whoami
from .commands.autoroute.commands import autoroute_group
from .commands.chat import chat
from .commands.credentials import credentials
from .commands.encryption import encryption
from .commands.http import http
from .commands.keys import keys
from .commands.model_groups import model_groups
# local imports
from .commands.models import models
from .commands.teams import teams
from .commands.up import down, up
from .commands.users import users
from .interface import interactive_shell
@ -131,6 +134,13 @@ cli.add_command(users)
# Add a top-level command per coding agent (claude, codex, opencode, ...)
for agent_command in agent_commands():
cli.add_command(agent_command)
# Add the up/down commands (route Claude Code through the local LiteLLM proxy)
cli.add_command(up)
cli.add_command(down)
# Add the model-groups command group (discover models your key can access)
cli.add_command(model_groups)
# Add the autoroute command group (QA auto-routing against your real proxy)
cli.add_command(autoroute_group, name="autoroute")
if __name__ == "__main__":

12
litellm/proxy/db/db_spend_update_writer.py
View file

@ -27,7 +27,7 @@ from typing import (
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching import DualCache, RedisCache
from litellm.caching import RedisCache
from litellm.constants import (
DB_SPEND_UPDATE_JOB_NAME,
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME,
@ -44,7 +44,6 @@ from litellm.proxy._types import (
DailyUserSpendTransaction,
DBSpendUpdateTransactions,
Litellm_EntityType,
LiteLLM_UserTable,
SpendLogsMetadata,
SpendLogsPayload,
SpendUpdateQueueItem,
@ -137,7 +136,6 @@ class DBSpendUpdateWriter:
disable_spend_logs,
litellm_proxy_budget_name,
prisma_client,
user_api_key_cache,
)
from litellm.proxy.utils import ProxyUpdateSpend, hash_token
@ -195,7 +193,6 @@ class DBSpendUpdateWriter:
org_id=org_id,
end_user_id=end_user_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
litellm_proxy_budget_name=litellm_proxy_budget_name,
payload=payload,
)
@ -326,7 +323,6 @@ class DBSpendUpdateWriter:
org_id: Optional[str],
end_user_id: Optional[str],
prisma_client: Optional[PrismaClient],
user_api_key_cache: DualCache,
litellm_proxy_budget_name: Optional[str],
payload: SpendLogsPayload,
):
@ -345,7 +341,6 @@ class DBSpendUpdateWriter:
response_cost=response_cost,
user_id=user_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
litellm_proxy_budget_name=litellm_proxy_budget_name,
end_user_id=end_user_id,
)
@ -510,7 +505,6 @@ class DBSpendUpdateWriter:
response_cost: Optional[float],
user_id: Optional[str],
prisma_client: Optional[PrismaClient],
user_api_key_cache: DualCache,
litellm_proxy_budget_name: Optional[str],
end_user_id: Optional[str] = None,
):
@ -518,10 +512,6 @@ class DBSpendUpdateWriter:
- Update that user's row
- Update litellm-proxy-budget row (global proxy spend)
"""
## if an end-user is passed in, do an upsert - we can't guarantee they already exist in db
existing_user_obj = await user_api_key_cache.async_get_cache(key=user_id)
if existing_user_obj is not None and isinstance(existing_user_obj, dict):
existing_user_obj = LiteLLM_UserTable(**existing_user_obj)
try:
if prisma_client is not None: # update
user_ids = [user_id]

212
litellm/proxy/db/query_engine_reaper.py Normal file
View file

@ -0,0 +1,212 @@
"""Supervisor-side reaper for orphaned Prisma query-engine processes.
Each proxy worker owns a Prisma query-engine subprocess whose only cleanup
hook is an in-process ``atexit`` handler. When a multi-worker supervisor
(uvicorn's multiprocess manager, the gunicorn arbiter) force-kills a hung or
crashed worker, that handler never runs: the engine reparents to the nearest
subreaper (PID 1 in a container, which is the supervisor itself under the
standard docker entrypoint) and keeps its database connection pool
established forever, while the replacement worker opens a fresh pool. Over
repeated worker deaths the active database connections grow without bound.
The reaper runs only in the supervisor process, where a query-engine process
can never be a legitimate direct child: workers own their engines, and the
supervisor never starts one. Any direct child whose command name begins with
``query-engine`` is therefore an adopted orphan and is terminated
(SIGTERM, bounded grace, SIGKILL) and reaped. On Linux the supervisor also
marks itself a child subreaper so orphans reparent to it even when it is not
PID 1.
Linux-only by construction (``/proc`` scan, ``prctl``); a no-op elsewhere.
"""
import ctypes
import os
import signal
import sys
import threading
import time
from typing import Optional
from litellm._logging import verbose_proxy_logger
QUERY_ENGINE_COMM_PREFIX = "query-engine"
REAPER_SCAN_INTERVAL_SECONDS = 5.0
SIGTERM_GRACE_SECONDS = 10.0
PR_SET_CHILD_SUBREAPER = 36
def set_child_subreaper() -> bool:
"""Mark this process as a child subreaper so orphaned descendants
reparent to it instead of PID 1. Best-effort: when it fails (or on
non-Linux) the reaper still covers the containerized case where the
supervisor already is PID 1."""
if not sys.platform.startswith("linux"):
return False
try:
libc = ctypes.CDLL(None, use_errno=True)
result: int = libc.prctl( # pyright: ignore[reportAny] # ctypes types foreign calls as Any; default restype is c_int
PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0
)
return result == 0
except (OSError, AttributeError):
return False
def _read_comm_and_ppid(pid: int, proc_root: str) -> Optional[tuple[str, int]]:
try:
with open(f"{proc_root}/{pid}/stat", encoding="ascii", errors="replace") as stat_file:
data = stat_file.read()
except (FileNotFoundError, ProcessLookupError, PermissionError, OSError):
return None
lparen = data.find("(")
rparen = data.rfind(")")
if lparen == -1 or rparen == -1 or rparen < lparen:
return None
comm = data[lparen + 1 : rparen]
fields = data[rparen + 2 :].split()
if len(fields) < 2:
return None
try:
ppid = int(fields[1])
except ValueError:
return None
return comm, ppid
def list_orphaned_engine_pids(parent_pid: int, proc_root: str = "/proc") -> tuple[int, ...]:
"""PIDs of direct children of ``parent_pid`` whose command name marks
them as Prisma query engines. In the supervisor these are always
adopted orphans: live engines are children of workers, not of the
supervisor."""
try:
entries = os.listdir(proc_root)
except (FileNotFoundError, OSError):
return ()
candidate_pids = (int(entry) for entry in entries if entry.isdigit())
return tuple(
pid
for pid in candidate_pids
if (info := _read_comm_and_ppid(pid, proc_root)) is not None
and info[1] == parent_pid
and info[0].startswith(QUERY_ENGINE_COMM_PREFIX)
)
def _try_reap(pid: int) -> bool:
try:
reaped_pid, _ = os.waitpid(pid, os.WNOHANG)
except ChildProcessError:
return True
except OSError:
return True
return reaped_pid == pid
def _send_signal(pid: int, signum: int) -> None:
try:
os.kill(pid, signum)
except (ProcessLookupError, PermissionError, OSError):
pass
def _await_reaped(pids: tuple[int, ...], timeout_seconds: float) -> tuple[int, ...]:
"""Poll until every PID is reaped or the shared deadline passes.
Returns the PIDs still alive at the deadline."""
deadline = time.monotonic() + timeout_seconds
remaining = pids
while remaining and time.monotonic() < deadline:
remaining = tuple(pid for pid in remaining if not _try_reap(pid))
if remaining:
time.sleep(0.2)
return remaining
def terminate_and_reap(pid: int, grace_seconds: float = SIGTERM_GRACE_SECONDS) -> None:
"""SIGTERM the orphaned engine, escalate to SIGKILL after the grace
period, and reap it so it does not linger as a zombie."""
terminate_and_reap_all((pid,), grace_seconds=grace_seconds)
def terminate_and_reap_all(
pids: tuple[int, ...],
grace_seconds: float = SIGTERM_GRACE_SECONDS,
) -> None:
"""Terminate a batch of orphaned engines concurrently: SIGTERM all of
them, share one grace period, SIGKILL the stragglers, and reap. The
shared deadline keeps cleanup time bounded when several workers die
at once instead of paying the grace period once per orphan."""
for pid in pids:
verbose_proxy_logger.warning(
"Reaping orphaned prisma query-engine PID %s (its worker process exited without cleanup).",
pid,
)
_send_signal(pid, signal.SIGTERM)
survivors = _await_reaped(pids, grace_seconds)
if not survivors:
return
for pid in survivors:
verbose_proxy_logger.warning(
"Orphaned prisma query-engine PID %s did not exit within %.1fs of SIGTERM; sending SIGKILL.",
pid,
grace_seconds,
)
_send_signal(pid, signal.SIGKILL)
unkillable = _await_reaped(survivors, 5.0)
for pid in unkillable:
verbose_proxy_logger.error(
"Orphaned prisma query-engine PID %s survived SIGKILL; will retry on the next scan.",
pid,
)
def reap_orphaned_engines(parent_pid: int, proc_root: str = "/proc") -> tuple[int, ...]:
"""One scan-and-reap pass. Returns the PIDs it acted on."""
orphaned_pids = list_orphaned_engine_pids(parent_pid, proc_root=proc_root)
if orphaned_pids:
terminate_and_reap_all(orphaned_pids)
return orphaned_pids
def _reaper_loop(parent_pid: int) -> None:
while True:
try:
reap_orphaned_engines(parent_pid)
except Exception as scan_error: # noqa: BLE001 # reaper thread must survive any scan failure
verbose_proxy_logger.debug("Orphaned query-engine scan failed: %s", scan_error)
time.sleep(REAPER_SCAN_INTERVAL_SECONDS)
REAPER_THREAD_NAME = "litellm-orphan-query-engine-reaper"
def start_query_engine_reaper() -> Optional[threading.Thread]:
"""Start the reaper daemon thread in the supervisor process.
Must only be called from a process that never hosts the proxy app
itself (uvicorn with ``workers > 1``, the gunicorn arbiter): with a
single in-process uvicorn worker the query engine is a legitimate
direct child and must not be touched. Idempotent: a reaper already
running in this process is returned instead of starting a second one.
"""
if not sys.platform.startswith("linux"):
return None
existing = next(
(thread for thread in threading.enumerate() if thread.name == REAPER_THREAD_NAME),
None,
)
if existing is not None:
return existing
set_child_subreaper()
reaper_thread = threading.Thread(
target=_reaper_loop,
args=(os.getpid(),),
daemon=True,
name=REAPER_THREAD_NAME,
)
reaper_thread.start()
verbose_proxy_logger.info(
"Started orphaned prisma query-engine reaper in supervisor process %s.",
os.getpid(),
)
return reaper_thread

22
litellm/proxy/guardrails/guardrail_registry.py
View file

@ -494,6 +494,7 @@ class InMemoryGuardrailHandler:
guardrail_id=guardrail.get("guardrail_id"),
guardrail_name=guardrail["guardrail_name"],
litellm_params=litellm_params,
guardrail_info=guardrail.get("guardrail_info"),
)
# store references to the guardrail in memory
@ -612,6 +613,27 @@ class InMemoryGuardrailHandler:
"""
return self._sources.get(guardrail_id)
def list_config_guardrails(self) -> List[Guardrail]:
"""
List in-memory guardrails owned by config.yaml.
DB-sourced entries are excluded: a read surface that also queries the DB
would double-count live ones, and a DB-sourced entry that's missing from
the DB is stale (deleted on another pod, awaiting reconciliation here).
"""
return [g for gid, g in self.IN_MEMORY_GUARDRAILS.items() if self._sources.get(gid) == "config"]
def get_config_guardrail_by_id(self, guardrail_id: str) -> Optional[Guardrail]:
"""
Get a config-owned in-memory guardrail by its ID, or None.
Mirrors the fallback in get_guardrail_info: a DB-sourced in-memory entry
that missed the DB lookup is stale and must not be surfaced.
"""
if self._sources.get(guardrail_id) != "config":
return None
return self.IN_MEMORY_GUARDRAILS.get(guardrail_id)
def reconcile_db_guardrails(self, db_guardrail_ids: Set[str]) -> List[str]:
"""
Drop in-memory entries that originated from the DB but are no longer

68
litellm/proxy/guardrails/usage_endpoints.py
View file

@ -137,10 +137,26 @@ def _chart_from_metrics(metrics: Any) -> List[Dict[str, Any]]:
return [{"date": d, "passed": v["passed"], "blocked": v["blocked"]} for d, v in sorted(chart_by_date.items())]
def _get_guardrail_field(g: Any, field: str) -> Any:
"""Read `field` off a guardrail whether it's a Prisma row (attr) or a dict/TypedDict (key)."""
if isinstance(g, dict):
return g.get(field)
return getattr(g, field, None)
def _to_dict(value: Any) -> Dict[str, Any]:
"""Coerce a pydantic model (e.g. LitellmParams) / dict value into a plain dict."""
if isinstance(value, BaseModel):
return value.model_dump(exclude_none=True)
if isinstance(value, dict):
return value
return {}
def _get_guardrail_attrs(g: Any) -> tuple[Any, str]:
"""Get (guardrail_id, display_name) from guardrail - handles Prisma model or dict."""
gid = getattr(g, "guardrail_id", None) or (g.get("guardrail_id") if isinstance(g, dict) else None)
name = getattr(g, "guardrail_name", None) or (g.get("guardrail_name") if isinstance(g, dict) else None)
gid = _get_guardrail_field(g, "guardrail_id")
name = _get_guardrail_field(g, "guardrail_name")
return gid, (name or gid or "")
@ -163,9 +179,9 @@ def _guardrail_overview_rows(
break
req, blocked = a["requests"], a["blocked"]
fail_rate = (100.0 * blocked / req) if req else 0.0
litellm_params = (g.litellm_params or {}) if isinstance(g.litellm_params, dict) else {}
litellm_params = _to_dict(_get_guardrail_field(g, "litellm_params"))
provider = str(litellm_params.get("guardrail", "Unknown"))
guardrail_info = (g.guardrail_info or {}) if isinstance(g.guardrail_info, dict) else {}
guardrail_info = _to_dict(_get_guardrail_field(g, "guardrail_info"))
gtype = str(guardrail_info.get("type", "Guardrail"))
prev_fail = 0.0
for k in lookup_keys:
@ -262,9 +278,15 @@ async def guardrails_usage_overview(
end = end_date or now.strftime("%Y-%m-%d")
start = start_date or (now - timedelta(days=7)).strftime("%Y-%m-%d")
from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER
try:
# Guardrails from DB
guardrails = await GuardrailsRepository(prisma_client).table.find_many()
db_guardrails = await GuardrailsRepository(prisma_client).table.find_many()
seen_ids = {gid for g in db_guardrails if (gid := _get_guardrail_field(g, "guardrail_id")) is not None}
config_guardrails = [
g for g in IN_MEMORY_GUARDRAIL_HANDLER.list_config_guardrails() if g.get("guardrail_id") not in seen_ids
]
guardrails: List[Any] = [*db_guardrails, *config_guardrails]
# Daily metrics in range
metrics = await DailyGuardrailMetricsRepository(prisma_client).table.find_many(
@ -321,16 +343,18 @@ async def guardrails_usage_detail(
end = end_date or now.strftime("%Y-%m-%d")
start = start_date or (now - timedelta(days=7)).strftime("%Y-%m-%d")
guardrail = await GuardrailsRepository(prisma_client).table.find_unique(where={"guardrail_id": guardrail_id})
if not guardrail:
from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER
guardrail: Any = await GuardrailsRepository(prisma_client).table.find_unique(where={"guardrail_id": guardrail_id})
if guardrail is None:
guardrail = IN_MEMORY_GUARDRAIL_HANDLER.get_config_guardrail_by_id(guardrail_id=guardrail_id)
if guardrail is None:
from fastapi import HTTPException
raise HTTPException(status_code=404, detail="Guardrail not found")
# Metrics are keyed by logical name (from spend log metadata), not UUID
logical_id = getattr(guardrail, "guardrail_name", None) or (
guardrail.get("guardrail_name") if isinstance(guardrail, dict) else None
)
logical_id = _get_guardrail_field(guardrail, "guardrail_name")
metric_ids = [i for i in (logical_id, guardrail_id) if i]
metrics = await DailyGuardrailMetricsRepository(prisma_client).table.find_many(
@ -367,17 +391,9 @@ async def guardrails_usage_detail(
{"date": d, "passed": v["passed"], "blocked": v["blocked"], "score": None}
for d, v in sorted(ts_by_date.items())
]
_litellm_params = getattr(guardrail, "litellm_params", None) or (
guardrail.get("litellm_params") if isinstance(guardrail, dict) else None
)
litellm_params = _litellm_params if isinstance(_litellm_params, dict) else {}
_guardrail_info = getattr(guardrail, "guardrail_info", None) or (
guardrail.get("guardrail_info") if isinstance(guardrail, dict) else None
)
guardrail_info = _guardrail_info if isinstance(_guardrail_info, dict) else {}
_guardrail_name = getattr(guardrail, "guardrail_name", None) or (
guardrail.get("guardrail_name") if isinstance(guardrail, dict) else None
)
litellm_params = _to_dict(_get_guardrail_field(guardrail, "litellm_params"))
guardrail_info = _to_dict(_get_guardrail_field(guardrail, "guardrail_info"))
_guardrail_name = _get_guardrail_field(guardrail, "guardrail_name")
return UsageDetailResponse(
guardrail_id=guardrail_id,
@ -548,11 +564,15 @@ async def guardrails_usage_logs(
# Query by both so we match regardless of which was written.
effective_guardrail_ids: List[str] = [guardrail_id] if guardrail_id else []
if guardrail_id:
guardrail = await GuardrailsRepository(prisma_client).table.find_unique(
from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER
guardrail: Any = await GuardrailsRepository(prisma_client).table.find_unique(
where={"guardrail_id": guardrail_id}
)
if guardrail is None:
guardrail = IN_MEMORY_GUARDRAIL_HANDLER.get_config_guardrail_by_id(guardrail_id=guardrail_id)
if guardrail:
logical_name = getattr(guardrail, "guardrail_name", None)
logical_name = _get_guardrail_field(guardrail, "guardrail_name")
if logical_name and logical_name not in effective_guardrail_ids:
effective_guardrail_ids.append(logical_name)

6
litellm/proxy/hooks/model_max_budget_limiter.py
View file

@ -5,7 +5,7 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import Span
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import (
@ -76,6 +76,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
message=f"LiteLLM Virtual Key: {user_api_key_dict.token}, key_alias: {user_api_key_dict.key_alias}, exceeded budget for model={model}",
current_cost=_current_spend,
max_budget=_current_model_budget_info.max_budget,
entity_type=Litellm_EntityType.KEY.value,
entity_id=user_api_key_dict.token,
)
return True
@ -140,6 +142,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
message=f"LiteLLM End User: {end_user_id}, exceeded budget for model={model}",
current_cost=_current_spend,
max_budget=_current_model_budget_info.max_budget,
entity_type=Litellm_EntityType.END_USER.value,
entity_id=end_user_id,
)
return True

4
litellm/proxy/litellm_pre_call_utils.py
View file

@ -949,6 +949,10 @@ class LiteLLMProxyRequestSetup:
user_api_key_alias=user_api_key_dict.key_alias,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_user_spend=user_api_key_dict.user_spend,
user_api_key_user_max_budget=user_api_key_dict.user_max_budget,
user_api_key_team_spend=user_api_key_dict.team_spend,
user_api_key_team_max_budget=user_api_key_dict.team_max_budget,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_project_id=user_api_key_dict.project_id,
user_api_key_project_alias=user_api_key_dict.project_alias,

24
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -32,6 +32,7 @@ from litellm._uuid import uuid
from litellm.constants import (
LENGTH_OF_LITELLM_GENERATED_KEY,
LITELLM_PROXY_ADMIN_NAME,
MINIMUM_CUSTOM_KEY_LENGTH,
UI_SESSION_TOKEN_TEAM_ID,
)
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
@ -1022,6 +1023,14 @@ async def _common_key_generation_helper(
detail={"error": f"Invalid key format. LiteLLM Virtual Key must start with 'sk-'. Received: {_masked}"},
)
if data.key is not None and len(data.key) < MINIMUM_CUSTOM_KEY_LENGTH:
raise HTTPException(
status_code=400,
detail={
"error": f"Invalid key format. LiteLLM Virtual Key must be at least {MINIMUM_CUSTOM_KEY_LENGTH} characters long."
},
)
# check org key limits - done here to handle inheriting org id from team
if data.organization_id is not None:
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
@ -1474,7 +1483,7 @@ async def generate_key_fn(
Parameters:
- duration: Optional[str] - Specify the length of time the token is valid for. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
- key_alias: Optional[str] - User defined key alias
- key: Optional[str] - User defined key value. If not set, a 16-digit unique sk-key is created for you.
- key: Optional[str] - User defined key value. Must start with 'sk-' and be at least 16 characters long. If not set, a 16-digit unique sk-key is created for you.
- team_id: Optional[str] - The team id of the key
- user_id: Optional[str] - The user id of the key
- agent_id: Optional[str] - The agent id associated with the key.
@ -1688,7 +1697,7 @@ async def generate_service_account_key_fn(
Parameters:
- duration: Optional[str] - Specify the length of time the token is valid for. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
- key_alias: Optional[str] - User defined key alias
- key: Optional[str] - User defined key value. If not set, a 16-digit unique sk-key is created for you.
- key: Optional[str] - User defined key value. Must start with 'sk-' and be at least 16 characters long. If not set, a 16-digit unique sk-key is created for you.
- team_id: Optional[str] - The team id of the key
- user_id: Optional[str] - [NON-FUNCTIONAL] THIS WILL BE IGNORED. The user id of the key
- budget_id: Optional[str] - The budget id associated with the key. Created by calling `/budget/new`.
@ -4356,7 +4365,6 @@ async def get_new_token(data: Optional[RegenerateKeyRequest]) -> str:
if data and data.new_key is not None:
# Reject custom key values if disabled by admin
await _check_custom_key_allowed(data.new_key)
new_token = data.new_key
if not data.new_key.startswith("sk-"):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
@ -4364,6 +4372,12 @@ async def get_new_token(data: Optional[RegenerateKeyRequest]) -> str:
"error": "New key must start with 'sk-'. This is to distinguish a key hash (used by litellm for logging / internal logic) from the actual key."
},
)
if len(data.new_key) < MINIMUM_CUSTOM_KEY_LENGTH:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={"error": f"New key must be at least {MINIMUM_CUSTOM_KEY_LENGTH} characters long."},
)
new_token = data.new_key
else:
new_token = f"sk-{secrets.token_urlsafe(LENGTH_OF_LITELLM_GENERATED_KEY)}"
return new_token
@ -4470,7 +4484,7 @@ async def _execute_virtual_key_regeneration(
new_token = await get_new_token(data=data)
new_token_hash = hash_token(new_token)
new_token_key_name = f"sk-...{new_token[-4:]}"
new_token_key_name = abbreviate_api_key(api_key=new_token)
update_data = {"token": new_token_hash, "key_name": new_token_key_name}
non_default_values = {}
@ -4550,7 +4564,7 @@ async def regenerate_key_fn(
- data: Optional[RegenerateKeyRequest] - Request body containing optional parameters to update
- key: Optional[str] - The key to regenerate.
- new_master_key: Optional[str] - The new master key to use, if key is the master key.
- new_key: Optional[str] - The new key to use, if key is not the master key. If both set, new_master_key will be used.
- new_key: Optional[str] - The new key to use, if key is not the master key. Must start with 'sk-' and be at least 16 characters long. If both set, new_master_key will be used.
- key_alias: Optional[str] - User-friendly key alias
- user_id: Optional[str] - User ID associated with key
- team_id: Optional[str] - Team ID associated with key

3
litellm/proxy/management_endpoints/mcp_management_endpoints.py
View file

@ -536,6 +536,7 @@ if MCP_AVAILABLE:
sanitized.env = {}
sanitized.command = None
sanitized.args = []
sanitized.issuer = None
sanitized.authorization_url = None
sanitized.token_url = None
sanitized.registration_url = None
@ -581,6 +582,7 @@ if MCP_AVAILABLE:
sanitized.teams = []
sanitized.env_vars = None
sanitized.issuer = None
sanitized.authorization_url = None
sanitized.token_url = None
sanitized.registration_url = None
@ -686,6 +688,7 @@ if MCP_AVAILABLE:
command=payload.command,
args=payload.args,
env=payload.env,
issuer=payload.issuer,
authorization_url=payload.authorization_url,
token_url=payload.token_url,
registration_url=payload.registration_url,

26
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -14,7 +14,7 @@ import json
import math
import traceback
from datetime import datetime, timezone
from typing import Annotated, Any, Dict, List, Optional, Tuple, Union, cast
from typing import Annotated, Any, Dict, List, Mapping, Optional, Tuple, Union, cast
import fastapi
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
@ -894,6 +894,17 @@ def _check_team_budget_update_authority(
)
def _should_auto_add_team_creator(
user_api_key_dict: UserAPIKeyAuth,
general_settings: Mapping[str, object],
) -> bool:
if user_api_key_dict.user_id is None:
return False
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
return True
return general_settings.get("disable_auto_add_proxy_admin_to_teams") is not True
#### TEAM MANAGEMENT ####
@router.post(
"/team/new",
@ -997,6 +1008,7 @@ async def new_team(
from litellm.proxy.proxy_server import (
_license_check,
create_audit_log_for_update,
general_settings,
litellm_proxy_admin_name,
prisma_client,
user_api_key_cache,
@ -1123,13 +1135,11 @@ async def new_team(
user_api_key_cache=user_api_key_cache,
)
if user_api_key_dict.user_id is not None:
creating_user_in_list = False
for member in data.members_with_roles:
if member.user_id == user_api_key_dict.user_id:
creating_user_in_list = True
if creating_user_in_list is False:
if _should_auto_add_team_creator(user_api_key_dict, general_settings):
creating_user_in_list = any(
member.user_id == user_api_key_dict.user_id for member in data.members_with_roles
)
if not creating_user_in_list:
data.members_with_roles.append(Member(role="admin", user_id=user_api_key_dict.user_id))
_check_passthrough_routes_caller_permission(data, user_api_key_dict, entity="team")

4
litellm/proxy/proxy_cli.py
View file

@ -15,6 +15,7 @@ from dotenv import load_dotenv
import litellm
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
from litellm.proxy.db.query_engine_reaper import start_query_engine_reaper
from litellm.secret_managers.main import get_secret_bool
if TYPE_CHECKING:
@ -495,6 +496,7 @@ class ProxyInitializationHelpers:
gunicorn_options["certfile"] = ssl_certfile_path
gunicorn_options["keyfile"] = ssl_keyfile_path
start_query_engine_reaper()
StandaloneApplication(app=app, options=gunicorn_options).run() # Run gunicorn
@staticmethod
@ -1261,6 +1263,8 @@ def run_server(
if reload:
ProxyInitializationHelpers._configure_dev_reload(uvicorn_args, config)
if num_workers > 1:
start_query_engine_reaper()
uvicorn.run(
**uvicorn_args,
workers=num_workers,

17
litellm/proxy/proxy_server.py
View file

@ -4425,6 +4425,15 @@ class ProxyConfig:
litellm.default_max_internal_user_budget = float(value)
if litellm.max_internal_user_budget is None:
litellm.max_internal_user_budget = litellm.default_max_internal_user_budget
elif key == "default_internal_user_params" and isinstance(value, dict):
litellm.default_internal_user_params = (
{**value, "max_budget": float(value["max_budget"])}
if value.get("max_budget") is not None
else value
)
verbose_proxy_logger.debug(
f"{blue_color_code} setting litellm.{key}={_redact_general_setting_value(key, litellm.default_internal_user_params, is_full_admin=False)}{reset_color_code}"
)
elif key == "custom_provider_map":
from litellm.utils import custom_llm_setup
@ -5785,6 +5794,13 @@ class ProxyConfig:
# For other types, convert to bool
general_settings["store_prompts_in_spend_logs"] = bool(value)
if "disable_auto_add_proxy_admin_to_teams" in _general_settings:
value = _general_settings["disable_auto_add_proxy_admin_to_teams"]
if isinstance(value, str):
general_settings["disable_auto_add_proxy_admin_to_teams"] = value.lower() == "true"
else:
general_settings["disable_auto_add_proxy_admin_to_teams"] = value if value is None else bool(value)
## STORE MODEL IN DB ##
if "store_model_in_db" in _general_settings:
value = _general_settings["store_model_in_db"]
@ -14898,6 +14914,7 @@ async def get_config_list(
"mcp_required_fields": {"type": "List"},
"cancel_on_disconnect": {"type": "Boolean"},
"skip_user_budget_on_team_key": {"type": "Boolean"},
"disable_auto_add_proxy_admin_to_teams": {"type": "Boolean"},
}
return_val = []

4
litellm/proxy/route_llm_request.py
View file

@ -362,6 +362,8 @@ async def route_request(
for _key in _MOCK_TESTING_KWARG_NAMES:
data.pop(_key, None)
data.pop("enable_tag_filtering", None)
team_id = get_team_id_from_data(data)
router_model_names = llm_router.model_names if llm_router is not None else []
is_proxy_admin_without_team = team_id is None and _is_proxy_admin_request(data)
@ -409,6 +411,8 @@ async def route_request(
"num_retries",
"timeout",
"model_group_retry_policy",
"routing_strategy",
"enable_tag_filtering",
]
# Merge override settings into data (only if not already set in request)

1
litellm/proxy/schema.prisma
View file

@ -325,6 +325,7 @@ model LiteLLM_MCPServerTable {
command String?
args String[] @default([])
env Json? @default("{}")
issuer String?
authorization_url String?
token_url String?
registration_url String?

16
litellm/proxy/spend_tracking/budget_reservation.py
View file

@ -4,7 +4,7 @@ import asyncio
import json
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Sequence, cast
from typing import Any, Dict, List, Mapping, Optional, Sequence, cast
import litellm
from litellm._logging import verbose_proxy_logger
@ -12,6 +12,7 @@ from litellm.caching import DualCache
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
from litellm.proxy._types import (
Litellm_EntityType,
LiteLLM_TeamMembership,
LiteLLM_TeamTable,
LiteLLM_UserTable,
@ -36,6 +37,17 @@ class _BudgetCounter:
window_start: Optional[datetime] = None
_COUNTER_ENTITY_TYPES: Mapping[str, str] = {
"Key": Litellm_EntityType.KEY.value,
"Team": Litellm_EntityType.TEAM.value,
"TeamMember": Litellm_EntityType.TEAM_MEMBER.value,
"User": Litellm_EntityType.USER.value,
"EndUser": Litellm_EntityType.END_USER.value,
"Tag": Litellm_EntityType.TAG.value,
"Organization": Litellm_EntityType.ORGANIZATION.value,
}
class _CounterReservationUnavailable(Exception):
def __init__(
self,
@ -108,6 +120,8 @@ async def _apply_over_budget_reservation_policy(
f"Current cost: {current_spend}, "
f"Max budget: {counter.max_budget}"
),
entity_type=_COUNTER_ENTITY_TYPES.get(counter.entity_type),
entity_id=counter.spend_log_entity_id or counter.entity_id,
)

12
litellm/responses/streaming_iterator.py
View file

@ -718,6 +718,12 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
except StopAsyncIteration:
# Normal end of stream - don't log as failure
raise
except (httpx.ReadError, httpx.RemoteProtocolError) as e:
self.finished = True
if self.completed_response is None:
self._handle_failure(e)
raise
raise StopAsyncIteration from e
except httpx.HTTPError as e:
# Handle HTTP errors
self.finished = True
@ -794,6 +800,12 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
except StopIteration:
# Normal end of stream - don't log as failure
raise
except (httpx.ReadError, httpx.RemoteProtocolError) as e:
self.finished = True
if self.completed_response is None:
self._handle_failure(e)
raise
raise StopIteration from e
except httpx.HTTPError as e:
# Handle HTTP errors
self.finished = True

109
litellm/router.py
View file

@ -251,6 +251,15 @@ else:
PreRoutingHookResponse = Any
def _cost_value_as_float(value: Union[str, int, float, None]) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except (TypeError, ValueError):
return None
class RoutingArgs(enum.Enum):
ttl = 60 # 1min (RPM/TPM expire key)
@ -622,6 +631,8 @@ class Router:
routing_strategy_args=routing_strategy_args,
)
self._init_routing_groups(self._routing_groups_input)
self._override_selectors: dict[str, Any] = {}
self._override_selectors_lock = threading.Lock()
self.access_groups = None
## USAGE TRACKING ##
if isinstance(litellm._async_success_callback, list):
@ -893,7 +904,9 @@ class Router:
self._unregister_router_selectors(
[getattr(self, attr, None) for attr in self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.values()]
+ list(getattr(self, "_override_selectors", {}).values())
)
self._override_selectors = {}
self.leastbusy_logger: Optional[LeastBusyLoggingHandler] = None
self.lowesttpm_logger: Optional[LowestTPMLoggingHandler] = None
@ -983,12 +996,67 @@ class Router:
{strategy_value: group_selector} if group_selector is not None else {}
)
def _get_routing_context(self, model: str) -> Tuple[Optional[str], Optional[Any]]:
_OVERRIDABLE_ROUTING_STRATEGIES: frozenset[str] = frozenset({"simple-shuffle", *_DEFAULT_SELECTOR_ATTR_BY_STRATEGY})
def _get_request_routing_strategy_override(self, request_kwargs: Optional[dict]) -> Optional[str]:
"""
Reads a per-request `routing_strategy` override (forwarded by the proxy
from key/team `router_settings`) out of the request kwargs.
Only strategies with a per-request-capable selector are honored;
anything else (unknown strings, `lar1`, `provider-budget-routing`) is
ignored with a warning so a bad value stored on a key or team can
never take down that caller's traffic.
"""
if not request_kwargs:
return None
raw_strategy = request_kwargs.get("routing_strategy")
if raw_strategy is None:
return None
strategy = self._normalize_strategy(raw_strategy) if isinstance(raw_strategy, (str, RoutingStrategy)) else None
if not isinstance(strategy, str) or strategy not in self._OVERRIDABLE_ROUTING_STRATEGIES:
verbose_router_logger.warning(
"Ignoring per-request routing_strategy override '%s'; supported overrides: %s.",
raw_strategy,
sorted(self._OVERRIDABLE_ROUTING_STRATEGIES),
)
return None
return strategy
def _get_override_strategy_selector(self, strategy: str) -> Optional[Any]:
"""
Returns the selector for a per-request strategy override.
Reuses the default group's selector when the override matches the
router's configured strategy (so shared state keeps accumulating in
one place); otherwise lazily builds one selector per strategy and
caches it for the router's lifetime so its usage/latency state
persists across requests.
"""
if strategy == self._normalize_strategy(self.routing_strategy):
attr = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy)
return getattr(self, attr, None) if attr is not None else None
with self._override_selectors_lock:
if strategy not in self._override_selectors:
self._override_selectors[strategy] = self._build_strategy_selector(
strategy=strategy,
routing_strategy_args={},
)
return self._override_selectors[strategy]
def _get_routing_context(
self, model: str, request_kwargs: Optional[dict] = None
) -> tuple[Optional[str], Optional[Any]]:
"""
Resolves the routing strategy and selector to use for the given model.
Every model belongs to exactly one group: an explicit entry from
`routing_groups`, or the implicit `"default"` group driven by the
A per-request `routing_strategy` in `request_kwargs` (forwarded by the
proxy from key/team `router_settings`) takes precedence over both the
model's routing group and the router's top-level strategy, since it is
the most specific expression of caller intent.
Otherwise every model belongs to exactly one group: an explicit entry
from `routing_groups`, or the implicit `"default"` group driven by the
router's top-level `routing_strategy` / `routing_strategy_args`.
`self.routing_strategy` may be either a string or a `RoutingStrategy`
@ -996,6 +1064,11 @@ class Router:
string here. Downstream call sites and `_select_deployment_*` arms
compare against string literals.
"""
override = self._get_request_routing_strategy_override(request_kwargs)
if override is not None:
verbose_router_logger.debug("routing_group=request-override model=%s strategy=%s", model, override)
return override, self._get_override_strategy_selector(override)
group_name = self._model_to_group.get(model)
if group_name is None:
strategy = self._normalize_strategy(self.routing_strategy)
@ -5936,7 +6009,7 @@ class Router:
input_kwargs: dict,
) -> Optional[Any]:
"""Same-model-group retry after a failed deployment; returns None if not applicable."""
strategy, _ = self._get_routing_context(original_model_group)
strategy, _ = self._get_routing_context(original_model_group, kwargs)
if strategy != "simple-shuffle":
return None
@ -8750,8 +8823,8 @@ class Router:
# Get mode from database model_info if available, otherwise default to "chat"
db_model_info = model.get("model_info", {})
mode = db_model_info.get("mode", "chat")
input_cost_per_token = db_model_info.get("input_cost_per_token")
output_cost_per_token = db_model_info.get("output_cost_per_token")
input_cost_per_token = _cost_value_as_float(db_model_info.get("input_cost_per_token"))
output_cost_per_token = _cost_value_as_float(db_model_info.get("output_cost_per_token"))
model_info = ModelMapInfo(
key=model_group,
@ -8802,16 +8875,18 @@ class Router:
)
):
model_group_info.max_output_tokens = model_info["max_output_tokens"]
if model_info.get("input_cost_per_token", None) is not None and (
_input_cost_per_token = _cost_value_as_float(model_info.get("input_cost_per_token"))
if _input_cost_per_token is not None and (
model_group_info.input_cost_per_token is None
or (model_info["input_cost_per_token"] or 0.0) > (model_group_info.input_cost_per_token or 0.0)
or _input_cost_per_token > (model_group_info.input_cost_per_token or 0.0)
):
model_group_info.input_cost_per_token = model_info["input_cost_per_token"]
if model_info.get("output_cost_per_token", None) is not None and (
model_group_info.input_cost_per_token = _input_cost_per_token
_output_cost_per_token = _cost_value_as_float(model_info.get("output_cost_per_token"))
if _output_cost_per_token is not None and (
model_group_info.output_cost_per_token is None
or (model_info["output_cost_per_token"] or 0.0) > (model_group_info.output_cost_per_token or 0.0)
or _output_cost_per_token > (model_group_info.output_cost_per_token or 0.0)
):
model_group_info.output_cost_per_token = model_info["output_cost_per_token"]
model_group_info.output_cost_per_token = _output_cost_per_token
if (
model_info.get("supports_parallel_function_calling", None) is not None
and model_info["supports_parallel_function_calling"] is True # type: ignore
@ -9703,6 +9778,7 @@ class Router:
"retry_policy",
"model_group_alias",
"enable_weighted_failover",
"enable_tag_filtering",
]
for var in vars_to_include:
@ -9739,6 +9815,7 @@ class Router:
"model_group_retry_policy",
"model_group_alias",
"enable_weighted_failover",
"enable_tag_filtering",
]
_int_settings = [
@ -10462,7 +10539,7 @@ class Router:
# Resolve the strategy and logger AFTER the pre-routing hook, since
# the hook can replace `model` and routing-group lookup must key
# off the final model name.
strategy, strategy_selector = self._get_routing_context(model)
strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
healthy_deployments = await self.async_get_healthy_deployments(
model=model,
@ -10606,7 +10683,7 @@ class Router:
# 5. Apply load balancing strategy
start_time = time.perf_counter()
strategy, strategy_selector = self._get_routing_context(model)
strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
if strategy == "simple-shuffle":
return simple_shuffle(
llm_router_instance=self,
@ -10893,7 +10970,7 @@ class Router:
cooldown_list=_cooldown_list,
)
strategy, strategy_selector = self._get_routing_context(model)
strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
if strategy == "simple-shuffle":
# if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm
############## Check 'weight' param set for weighted pick #################
@ -11033,7 +11110,7 @@ class Router:
)
# 6. Apply load balancing strategy
strategy, strategy_selector = self._get_routing_context(model)
strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
if strategy == "simple-shuffle":
return simple_shuffle(
llm_router_instance=self,

8
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -98,9 +98,9 @@ def _sanitize_user_api_key_auth(auth: Any) -> Any:
return auth
def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any] | None:
def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
if not metadata:
return metadata
return {}
return {
k: _sanitize_user_api_key_auth(v) if k == "user_api_key_auth" else v
for k, v in metadata.items()
@ -763,8 +763,8 @@ class ComplexityRouter(CustomLogger):
# embedding call. Forwarding it would let the embedding's cost callback finalize the
# reservation, so the routed completion's own callback then skips incrementing the
# key/team budget. Key/team attribution fields are preserved for spend logging.
metadata = _classifier_call_metadata(request_kwargs.get("metadata")) or {}
litellm_metadata = _classifier_call_metadata(request_kwargs.get("litellm_metadata")) or {}
metadata = _classifier_call_metadata(request_kwargs.get("metadata"))
litellm_metadata = _classifier_call_metadata(request_kwargs.get("litellm_metadata"))
query_vector = (
await encoder.aencode_queries([user_message], metadata=metadata, litellm_metadata=litellm_metadata)
)[0]

7
litellm/router_strategy/complexity_router/config.py
View file

@ -363,10 +363,13 @@ class ComplexityRouterConfig(BaseModel):
# Session affinity: pin the first turn's routed model for the rest of the session
session_affinity: bool = Field(
default=False,
default=True,
description=(
"When True and a session_id is resolvable on the request, pin the model chosen on the "
"session's first turn and reuse it for every later turn, skipping re-classification."
"session's first turn and reuse it for every later turn, skipping re-classification. "
"On by default so multi-turn sessions stay on one model, preserving provider prompt "
"caches and avoiding cross-model conversation-history errors. Set False to reclassify "
"every turn."
),
)
session_affinity_ttl_seconds: int = Field(

8
litellm/router_strategy/tag_based_routing.py
View file

@ -160,8 +160,14 @@ async def get_deployments_for_tag(
Returns a list of deployments that match the requested model and tags in the request.
Executes tag based filtering based on the tags in request metadata and the tags on the deployments
Runs when the router-level `enable_tag_filtering` is True or the request carries
`enable_tag_filtering=True` (set from key/team router_settings by the proxy).
A request-level False never disables a router-level True, so per-request settings
cannot escape an operator's global tag-routing policy.
"""
if llm_router_instance.enable_tag_filtering is not True:
request_enable_tag_filtering = request_kwargs.get("enable_tag_filtering") if request_kwargs else None
if request_enable_tag_filtering is not True and llm_router_instance.enable_tag_filtering is not True:
return healthy_deployments
if request_kwargs is None:

1
litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -14,3 +14,4 @@ class UsagePerChunk(TypedDict):
web_search_requests: Optional[int]
completion_tokens_details: Optional[CompletionTokensDetails]
prompt_tokens_details: Optional[PromptTokensDetailsWrapper]
cost: Optional[float]

13
litellm/types/mcp_server/mcp_server_manager.py
View file

@ -17,9 +17,20 @@ MCPInfo = Dict[str, Any]
class MCPOAuthMetadata(BaseModel):
scopes: Optional[List[str]] = None
"""Resource-driven scopes for the authorization request: the RFC 9728 protected-resource
``scopes_supported``, or the ``scope`` from the WWW-Authenticate 401 challenge when the resource
supplied one, else the authorization server's ``scopes_supported``. This is the scope value a
client requests per the MCP authorization spec Scope Selection Strategy; scope minimization and
inflation control are the authorization server's and user's job at consent (RFC 6749 §3.3), not
the client's."""
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
discovered_issuer: Optional[str] = None
"""The ``issuer`` the authorization-server metadata document self-attests (RFC 8414). Persisted
trust-on-first-use as the server's ``issuer`` when none is configured, so that later rebuilds
anchor discovery on it (RFC 8414 §3.3) and a subsequently compromised resource cannot re-point
it. Never overwrites an admin-configured issuer."""
from_origin_fallback: bool = False
"""True when the metadata came from guessing the resource origin as its authorization
server rather than from an RFC 9728/8414-advertised document. Guessed endpoints are
@ -54,6 +65,8 @@ class MCPServer(BaseModel):
# OAuth-specific fields
client_id: Optional[str] = None
client_secret: Optional[str] = None
issuer: Optional[str] = None
issuer_is_anchored: bool = False
scopes: Optional[List[str]] = None
authorization_url: Optional[str] = None
token_url: Optional[str] = None

1
litellm/types/router.py
View file

@ -117,6 +117,7 @@ class UpdateRouterConfig(BaseModel):
fallbacks: Optional[List[dict]] = None
context_window_fallbacks: Optional[List[dict]] = None
model_group_alias: Optional[Dict[str, Union[str, Dict]]] = {}
enable_tag_filtering: Optional[bool] = None
model_config = ConfigDict(protected_namespaces=())

20
litellm/types/utils.py
View file

@ -1798,14 +1798,17 @@ class ModelResponseStream(ModelResponseBase):
else:
created = created
usage_to_set = None
if "usage" in kwargs and kwargs["usage"] is not None:
if isinstance(kwargs["usage"], dict):
kwargs["usage"] = Usage(**kwargs["usage"])
usage_to_set = Usage(**kwargs["usage"])
kwargs["usage"] = usage_to_set
elif isinstance(kwargs["usage"], BaseModel):
dump = (
kwargs["usage"].model_dump() if hasattr(kwargs["usage"], "model_dump") else kwargs["usage"].dict()
)
kwargs["usage"] = Usage(**dump)
usage_to_set = Usage(**dump)
kwargs["usage"] = usage_to_set
kwargs["id"] = id
kwargs["created"] = created
@ -1814,6 +1817,9 @@ class ModelResponseStream(ModelResponseBase):
super().__init__(**kwargs)
if usage_to_set is not None:
self.usage = usage_to_set
def __contains__(self, key):
# Define custom behavior for the 'in' operator
return hasattr(self, key)
@ -2469,6 +2475,10 @@ class StandardLoggingUserAPIKeyMetadata(TypedDict):
user_api_key_spend: Optional[float]
user_api_key_max_budget: Optional[float]
user_api_key_budget_reset_at: Optional[str]
user_api_key_user_spend: Optional[float]
user_api_key_user_max_budget: Optional[float]
user_api_key_team_spend: Optional[float]
user_api_key_team_max_budget: Optional[float]
user_api_key_org_id: Optional[str]
user_api_key_org_alias: Optional[str]
user_api_key_team_id: Optional[str]
@ -2687,6 +2697,10 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False):
# hints). Lets dashboards split rate-limit failures by cause without
# parsing free-text error messages.
error_rate_limit_type: Optional[str]
error_budget_entity_type: Optional[str]
error_budget_entity_id: Optional[str]
error_budget_limit: Optional[float]
error_budget_spend: Optional[float]
class GuardrailMode(TypedDict, total=False):
@ -3136,6 +3150,7 @@ all_litellm_params = (
"use_client",
"id",
"fallbacks",
"routing_strategy",
"azure",
"headers",
"model_list",
@ -3207,6 +3222,7 @@ all_litellm_params = (
"shared_session",
"search_tool_name",
"order",
"enable_tag_filtering",
"enable_json_schema_validation",
"use_xai_oauth",
"_litellm_rate_limit_descriptors",

7
pyproject.toml
View file

@ -66,6 +66,7 @@ proxy = [
"litellm-enterprise==0.1.50",
"RestrictedPython>=8.1,<9.0",
"rich>=13.9.4,<14.0",
"InquirerPy>=0.3.4,<1.0",
"polars>=1.38.1,<2.0",
"soundfile>=0.12.1,<1.0",
"pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'",
@ -74,11 +75,12 @@ proxy = [
]
# Thin client install for the `lite` CLI on developer laptops. The CLI's heavy
# imports (fastapi, cryptography, ...) are all guarded, so it runs on the base
# SDK plus just these three; none of the server runtime in `proxy` is pulled in.
# SDK plus just these four; none of the server runtime in `proxy` is pulled in.
cli = [
"rich>=13.9.4,<14.0",
"pyyaml>=6.0.3,<7.0",
"requests>=2.32.0,<3.0",
"InquirerPy>=0.3.4,<1.0",
]
extra_proxy = [
"prisma>=0.11.0,<1.0",
@ -129,8 +131,7 @@ proxy-runtime = [
"opentelemetry-sdk==1.28.0",
"opentelemetry-exporter-otlp==1.28.0",
"opentelemetry-instrumentation-fastapi==0.49b0",
"ddtrace>=2.19.0,<3.0; python_version < '3.14'",
"ddtrace>=4.0.0,<5.0; python_version >= '3.14'",
"ddtrace>=4.8.2,<5.0",
"sentry-sdk>=2.21.0,<3.0",
"mangum>=0.17.0,<1.0",
"azure-ai-contentsafety>=1.0.0,<2.0",

1
schema.prisma
View file

@ -325,6 +325,7 @@ model LiteLLM_MCPServerTable {
command String?
args String[] @default([])
env Json? @default("{}")
issuer String?
authorization_url String?
token_url String?
registration_url String?

19
scripts/install-cli.sh
View file

@ -11,12 +11,21 @@
# Python itself (honouring litellm's requires-python), downloading a managed one
# when the host has no suitable interpreter.
#
# To try an unreleased branch instead of the latest PyPI release (for example, to
# QA a CLI feature before it ships), set LITELLM_CLI_REF to a branch, tag, or commit:
# curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/<branch>/scripts/install-cli.sh | \
# LITELLM_CLI_REF=<branch> sh
#
# NOTE: set -e without pipefail for POSIX sh compatibility (dash on Ubuntu/Debian
# ignores the shebang when invoked as `sh` and does not support `pipefail`).
set -eu
# NOTE: before merging, this must stay as "litellm[cli]" to install from PyPI.
LITELLM_PACKAGE="litellm[cli]"
# Defaults to the PyPI release; LITELLM_CLI_REF opts into installing from source instead.
if [ -n "${LITELLM_CLI_REF:-}" ]; then
LITELLM_PACKAGE="litellm[cli] @ git+https://github.com/BerriAI/litellm.git@${LITELLM_CLI_REF}"
else
LITELLM_PACKAGE="litellm[cli]"
fi
UV_VERSION="0.10.9"
# ── colours ────────────────────────────────────────────────────────────────
@ -90,7 +99,11 @@ fi
# otherwise download a managed one. Either way uv honours litellm's requires-python,
# so a too-old (3.9) or too-new (3.14+) system Python is skipped, not forced.
echo ""
header "Installing litellm[cli]…"
if [ -n "${LITELLM_CLI_REF:-}" ]; then
header "Installing litellm[cli] from ${LITELLM_CLI_REF}…"
else
header "Installing litellm[cli]…"
fi
echo ""
"$UV_BIN" tool install --python-preference system --force "${LITELLM_PACKAGE}" \

20
scripts/install.sh
View file

@ -5,12 +5,24 @@
# Needs only curl: uv is bootstrapped if missing, and uv provisions a compatible
# Python itself (reusing a suitable system one, else downloading a managed build).
#
# To install from an unreleased branch, tag, or commit instead of the latest PyPI
# release, set LITELLM_CLI_REF:
# curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/<branch>/scripts/install.sh | \
# LITELLM_CLI_REF=<branch> sh
#
# NOTE: set -e without pipefail for POSIX sh compatibility (dash on Ubuntu/Debian
# ignores the shebang when invoked as `sh` and does not support `pipefail`).
set -eu
# NOTE: before merging, this must stay as "litellm[proxy]" to install from PyPI.
LITELLM_PACKAGE="litellm[proxy]"
# LITELLM_CLI_REF opts into installing from a branch, tag, or commit instead (for
# example, to QA lite autoroute against an unreleased branch, which needs this proxy
# runtime, not the thin litellm[cli] install).
if [ -n "${LITELLM_CLI_REF:-}" ]; then
LITELLM_PACKAGE="litellm[proxy] @ git+https://github.com/BerriAI/litellm.git@${LITELLM_CLI_REF}"
else
LITELLM_PACKAGE="litellm[proxy]"
fi
UV_VERSION="0.10.9"
# ── colours ────────────────────────────────────────────────────────────────
@ -81,7 +93,11 @@ fi
# ── install ────────────────────────────────────────────────────────────────
echo ""
header "Installing litellm[proxy]…"
if [ -n "${LITELLM_CLI_REF:-}" ]; then
header "Installing litellm[proxy] from ${LITELLM_CLI_REF}…"
else
header "Installing litellm[proxy]…"
fi
echo ""
# --python-preference system: reuse a compatible system Python when present,

13
tests/e2e/CLAUDE.md
View file

@ -171,3 +171,16 @@ other.<area>.<case>.<assertion>
e.g. other.auth.jwt.valid_token_allows
other.lifecycle.readiness.reports_db
```
## Hard Rules
- no monkeypatching, mock tests or unit tests of any kind. if a contributor asks you to write an end to end test, do NOT stage a unit test with it. if you find a product gap, call it out in the PR description
- use model management endpoints to create new models for a test. this could be in a conftest / inline for each test. ask the user what they want.
- do not overengineer a test, i need you to write readable, clean code of what would look like a natural user scenario
- when it comes to typing an input schema for an api endpoint, have it type X = A | B | C ... where X = exhaustive union of all supported input schemas and A, B, C typically are composed by a base type. types are only pretty for a api request / response body. make sure to compose types instead of repeating the same base attributes over and over again.
- use the docker-compose to your advantage and spin up a local proxy, make sure all tests pass. if a test fails due to an internally found issue, let users know to create a linear ticket for it.
- do not use xfail markers, tests should be written in a form that the end user expects it to pass

30
tests/e2e/claude_code/_basic_messaging.py
View file

@ -27,19 +27,20 @@ collecting this module as a test file.
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Any, Callable, Mapping, Sequence
import pytest
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
DriverResult,
failure_diagnostic,
run_claude_models_parallel,
)
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
ClaudeRunner = Callable[..., Mapping[str, DriverResult | ClaudeCLIError]]
# Floor on the number of `stream_event` records (with delta payloads)
# we expect to see when the proxy actually streams. With
@ -79,6 +80,8 @@ def run_basic_messaging_cell(
models: Sequence[str],
prompt: str,
verify_streaming: bool = False,
env: Mapping[str, str] | None = None,
runner: ClaudeRunner = run_claude_models_parallel,
) -> None:
"""Run the shared `basic_messaging_*` × <provider> cell body.
@ -99,28 +102,13 @@ def run_basic_messaging_cell(
streamed reply to a single ``assistant`` event in
``--print --output-format stream-json`` mode).
"""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
compat_result.set(
{
"status": "fail",
"error": (
f"missing required env: set {PROXY_BASE_URL_ENV} and "
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
),
}
)
pytest.fail(
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured",
pytrace=False,
)
base_url, api_key = require_proxy(compat_result, env=env)
extra_args: Sequence[str] = (
("--include-partial-messages",) if verify_streaming else ()
)
outcomes = run_claude_models_parallel(
outcomes = runner(
models=models,
prompt=prompt,
base_url=base_url,

2
tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json
View file

@ -26,7 +26,7 @@
"providers": {
"anthropic": {
"status": "fail",
"error": "[claude-sonnet-4-6] tool call dropped"
"error": "[claude-sonnet-4-5] tool call dropped"
},
"bedrock_invoke": {
"status": "not_applicable",

6
tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json
View file

@ -10,7 +10,7 @@
{
"feature_id": "basic_messaging_non_streaming",
"provider": "anthropic",
"nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-sonnet-4-6]",
"nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-sonnet-4-5]",
"result": {"status": "pass"}
},
{
@ -28,8 +28,8 @@
{
"feature_id": "tool_use",
"provider": "anthropic",
"nodeid": "tests/e2e/claude_code/tool_use/test_anthropic.py::test_x[claude-sonnet-4-6]",
"result": {"status": "fail", "error": "[claude-sonnet-4-6] tool call dropped"}
"nodeid": "tests/e2e/claude_code/tool_use/test_anthropic.py::test_x[claude-sonnet-4-5]",
"result": {"status": "fail", "error": "[claude-sonnet-4-5] tool call dropped"}
},
{
"feature_id": "tool_use",

2
tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py
View file

@ -393,7 +393,7 @@ def test_build_matrix_6x5_grid_matches_published_sample():
feature_ids = [feature["id"] for feature in manifest["features"]]
providers = manifest["providers"]
models = ["claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-7"]
models = ["claude-haiku-4-5", "claude-sonnet-4-5", "claude-opus-4-7"]
results = []
for feature_id in feature_ids:

22
tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py
View file

@ -16,8 +16,8 @@ from pathlib import Path
import pytest
import yaml
REPO_ROOT = Path(__file__).resolve().parents[1]
MANIFEST_PATH = REPO_ROOT / "manifest.yaml"
SUITE_ROOT = Path(__file__).resolve().parents[1]
MANIFEST_PATH = SUITE_ROOT / "manifest.yaml"
# The PRD's "Features in v0" section, in row order.
EXPECTED_FEATURE_IDS = [
@ -90,14 +90,14 @@ def test_manifest_every_feature_has_human_readable_name(manifest):
@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS)
def test_feature_directory_exists(feature_id):
feature_dir = REPO_ROOT / feature_id
feature_dir = SUITE_ROOT / feature_id
assert feature_dir.is_dir(), f"missing feature directory: {feature_dir}"
@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS)
@pytest.mark.parametrize("provider", EXPECTED_PROVIDERS)
def test_per_provider_test_file_exists(feature_id, provider):
test_file = REPO_ROOT / feature_id / f"test_{provider}.py"
test_file = SUITE_ROOT / feature_id / f"test_{provider}.py"
assert test_file.is_file(), f"missing per-provider test file: {test_file}"
@ -106,7 +106,7 @@ def test_feature_directory_has_init_file(feature_id):
"""Each feature directory needs an __init__.py so pytest collects
the per-provider test files as a package — matches the layout
established by `basic_messaging_non_streaming/`."""
init_file = REPO_ROOT / feature_id / "__init__.py"
init_file = SUITE_ROOT / feature_id / "__init__.py"
assert init_file.is_file(), f"missing __init__.py: {init_file}"
@ -117,7 +117,7 @@ def test_feature_directory_has_init_file(feature_id):
# a broken post-v0 directory still fails CI.
@pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS)
def test_every_manifest_feature_has_directory(feature_id):
feature_dir = REPO_ROOT / feature_id
feature_dir = SUITE_ROOT / feature_id
assert feature_dir.is_dir(), (
f"manifest declares {feature_id!r} but {feature_dir} is missing — "
"feature_id MUST match its on-disk directory (see manifest.yaml header)."
@ -126,7 +126,7 @@ def test_every_manifest_feature_has_directory(feature_id):
@pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS)
def test_every_manifest_feature_has_init_file(feature_id):
init_file = REPO_ROOT / feature_id / "__init__.py"
init_file = SUITE_ROOT / feature_id / "__init__.py"
assert init_file.is_file(), f"missing __init__.py: {init_file}"
@ -137,7 +137,7 @@ def test_every_manifest_feature_has_per_provider_test_file(feature_id, provider)
backed by a per-provider test file. Without this check, a missing
file silently becomes a `not_tested` cell in the published matrix
rather than a CI failure surfacing the layout drift."""
test_file = REPO_ROOT / feature_id / f"test_{provider}.py"
test_file = SUITE_ROOT / feature_id / f"test_{provider}.py"
assert test_file.is_file(), f"missing per-provider test file: {test_file}"
@ -151,8 +151,8 @@ def test_per_provider_test_file_imports_and_parametrizes_three_models(
use plain aliases or per-provider-suffixed aliases (e.g.
`claude-opus-4-7-bedrock-invoke`), so we check for the tier
substrings rather than exact alias names."""
text = (REPO_ROOT / feature_id / f"test_{provider}.py").read_text()
for tier in ("haiku-4-5", "sonnet-4-6", "opus-4-7"):
text = (SUITE_ROOT / feature_id / f"test_{provider}.py").read_text()
for tier in ("haiku-4-5", "sonnet-4-5", "opus-4-7"):
assert (
tier in text
), f"{feature_id}/test_{provider}.py does not reference {tier}"
@ -171,7 +171,7 @@ def test_azure_test_file_drives_the_proxy(feature_id):
that wraps them — both shapes drive the proxy, and we don't want
this layout pin to block legitimate de-duplication of test bodies.
"""
text = (REPO_ROOT / feature_id / "test_azure.py").read_text()
text = (SUITE_ROOT / feature_id / "test_azure.py").read_text()
assert "run_claude" in text or "run_basic_messaging_cell" in text, (
f"{feature_id}/test_azure.py must drive the claude CLI via run_claude() "
"or a shared helper that wraps it; the not_applicable stub was removed "

86
tests/e2e/claude_code/_compat_models.py Normal file
View file

@ -0,0 +1,86 @@
"""Load the claude_code compat matrix's deployment list from
``test_config.yaml``.
``test_config.yaml`` is the ground-truth config the stage deployment
uses; parsing it at fixture time means a change there (new tier, tier
retirement, provider swap, endpoint rename) reaches the fixture with
no extra edit. A drift-check test asserts every ``*_MODELS`` list
referenced by the compat cells is covered by the yaml, so a cell that
adds a probe for a name the yaml doesn't know about fails loudly at
collection instead of at 400-time.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Mapping
import yaml
from models import LiteLLMParamsBody
CONFIG_PATH = Path(__file__).resolve().parent / "test_config.yaml"
@dataclass(frozen=True, slots=True)
class CompatDeployment:
model_name: str
litellm_params: LiteLLMParamsBody
# The yaml uses ``vertex_ai_*`` for the vertex project/location fields
# (that is the spelling the proxy config file historically standardized
# on), while ``LiteLLMParamsBody`` names them without the ``_ai`` infix
# (matching the proxy's DB column). Both spellings resolve at call time
# on the proxy side, but pydantic silently drops unknown fields, so a
# raw ``LiteLLMParamsBody(**entry)`` would produce a body with the
# vertex project stripped - the resulting deployment 400s at
# ``/v1/messages`` with "Invalid model name". Normalize the yaml keys
# to the pydantic names in one place.
_YAML_TO_PYDANTIC_ALIASES = {
"vertex_ai_project": "vertex_project",
"vertex_ai_location": "vertex_location",
"vertex_ai_credentials": "vertex_credentials",
}
def _normalize_params(raw: Mapping[str, object]) -> dict[str, object]:
return {_YAML_TO_PYDANTIC_ALIASES.get(k, k): v for k, v in raw.items()}
ConfigReader = Callable[[Path], str]
def _default_reader(path: Path) -> str:
return path.read_text()
def load_all_deployments(
config_path: Path = CONFIG_PATH,
reader: ConfigReader = _default_reader,
) -> tuple[CompatDeployment, ...]:
"""Every deployment declared in the yaml, in file order."""
doc = yaml.safe_load(reader(config_path))
model_list = doc.get("model_list") or []
return tuple(
CompatDeployment(
model_name=entry["model_name"],
litellm_params=LiteLLMParamsBody(
**_normalize_params(entry["litellm_params"])
),
)
for entry in model_list
)
def all_expected_model_names(
*,
config_path: Path = CONFIG_PATH,
reader: ConfigReader = _default_reader,
) -> frozenset[str]:
"""Every virtual name the compat matrix declares - the ground truth
the cells are supposed to probe. Used by the drift-check test."""
return frozenset(
d.model_name for d in load_all_deployments(config_path, reader)
)

94
tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py
View file

@ -1,20 +1,19 @@
"""Unit tests for the shared `run_basic_messaging_cell` helper.
These tests mock `run_claude_models_parallel` so they exercise the
helper's branching (env-missing guard, per-model pass/fail/empty-text,
streaming wire check) without spawning the real CLI. The streaming
check is the regression we care about: a proxy that buffers the
upstream stream must turn the cell red, not green.
These tests inject a fake ``ClaudeRunner`` and a fake env mapping so
they exercise the helper's branching (env-missing guard, per-model
pass/fail/empty-text, streaming wire check) without spawning the real
CLI or touching ``os.environ``. The streaming check is the regression
we care about: a proxy that buffers the upstream stream must turn the
cell red, not green.
"""
from __future__ import annotations
import os
from typing import Any, Dict, List, Mapping, Optional, Sequence
import pytest
from claude_code import _basic_messaging
from claude_code._basic_messaging import (
MIN_STREAM_DELTA_EVENTS,
_count_stream_event_deltas,
@ -23,6 +22,12 @@ from claude_code._basic_messaging import (
from claude_code.cli_driver import DriverResult
_PROXY_ENV: Mapping[str, str] = {
"LITELLM_PROXY_URL": "http://localhost:4000",
"LITELLM_MASTER_KEY": "sk-test",
}
class _FakeResult:
"""Stand-in for the test's `compat_result` fixture.
@ -82,16 +87,17 @@ def _buffered_events() -> List[Dict[str, Any]]:
]
def _install_fake_runner(monkeypatch, *, outcomes_by_model):
"""Patch `run_claude_models_parallel` to return canned outcomes.
def _make_fake_runner(*, outcomes_by_model):
"""Build an injectable runner that returns canned outcomes and
records the kwargs the helper passed in.
Captures the kwargs the cell passed in so tests can assert on
`extra_args` (which is how the streaming variant opts into
`--include-partial-messages`).
"""
Returns a ``(runner, captured)`` pair; ``captured`` is a dict the
test can assert against without any global mutation, which is why
we prefer DI over ``monkeypatch.setattr``: the helper takes a
``runner=`` kwarg, so tests bind their fake directly."""
captured: Dict[str, Any] = {}
def fake(*, models, prompt, base_url, api_key, extra_args=None, **_kwargs):
def runner(*, models, prompt, base_url, api_key, extra_args=None, **_kwargs):
captured["models"] = list(models)
captured["prompt"] = prompt
captured["base_url"] = base_url
@ -99,14 +105,7 @@ def _install_fake_runner(monkeypatch, *, outcomes_by_model):
captured["extra_args"] = list(extra_args) if extra_args else []
return {model: outcomes_by_model[model] for model in models}
monkeypatch.setattr(_basic_messaging, "run_claude_models_parallel", fake)
return captured
@pytest.fixture(autouse=True)
def _proxy_env(monkeypatch):
monkeypatch.setenv("LITELLM_PROXY_BASE_URL", "http://localhost:4000")
monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-test")
return runner, captured
def test_count_stream_event_deltas_only_counts_records_with_event_payload():
@ -123,28 +122,30 @@ def test_count_stream_event_deltas_only_counts_records_with_event_payload():
assert _count_stream_event_deltas(events) == 2
def test_verify_streaming_passes_when_proxy_streams(monkeypatch):
def test_verify_streaming_passes_when_proxy_streams():
fake_result = _FakeResult()
model = "claude-haiku-4-5"
outcome = DriverResult(text="1\n2\n3", events=_streamed_events(n_deltas=5))
captured = _install_fake_runner(monkeypatch, outcomes_by_model={model: outcome})
runner, captured = _make_fake_runner(outcomes_by_model={model: outcome})
run_basic_messaging_cell(
compat_result=fake_result,
models=[model],
prompt="Count from 1 to 5, one number per line.",
verify_streaming=True,
env=_PROXY_ENV,
runner=runner,
)
assert captured["extra_args"] == ["--include-partial-messages"]
assert fake_result.rows == [{"status": "pass"}]
def test_verify_streaming_fails_when_proxy_buffers(monkeypatch):
def test_verify_streaming_fails_when_proxy_buffers():
fake_result = _FakeResult()
model = "claude-haiku-4-5"
outcome = DriverResult(text="1\n2\n3", events=_buffered_events())
_install_fake_runner(monkeypatch, outcomes_by_model={model: outcome})
runner, _captured = _make_fake_runner(outcomes_by_model={model: outcome})
with pytest.raises(pytest.fail.Exception):
run_basic_messaging_cell(
@ -152,7 +153,9 @@ def test_verify_streaming_fails_when_proxy_buffers(monkeypatch):
models=[model],
prompt="Count from 1 to 5, one number per line.",
verify_streaming=True,
)
env=_PROXY_ENV,
runner=runner,
)
assert len(fake_result.rows) == 1
row = fake_result.rows[0]
@ -161,33 +164,35 @@ def test_verify_streaming_fails_when_proxy_buffers(monkeypatch):
assert f"< {MIN_STREAM_DELTA_EVENTS}" in row["error"]
def test_non_streaming_variant_omits_partial_messages_flag(monkeypatch):
def test_non_streaming_variant_omits_partial_messages_flag():
"""Default `verify_streaming=False` keeps the non-streaming wire identical."""
fake_result = _FakeResult()
model = "claude-haiku-4-5"
outcome = DriverResult(text="pong", events=_buffered_events())
captured = _install_fake_runner(monkeypatch, outcomes_by_model={model: outcome})
runner, captured = _make_fake_runner(outcomes_by_model={model: outcome})
run_basic_messaging_cell(
compat_result=fake_result,
models=[model],
prompt="Reply with the single word 'pong' and nothing else.",
env=_PROXY_ENV,
runner=runner,
)
assert captured["extra_args"] == []
assert fake_result.rows == [{"status": "pass"}]
def test_verify_streaming_requires_all_models_to_stream(monkeypatch):
def test_verify_streaming_requires_all_models_to_stream():
"""If any one tier buffers, the cell fails — same all-must-pass shape as
the non-streaming check."""
fake_result = _FakeResult()
outcomes = {
"claude-haiku-4-5": DriverResult(text="ok", events=_streamed_events(5)),
"claude-sonnet-4-6": DriverResult(text="ok", events=_buffered_events()),
"claude-sonnet-4-5": DriverResult(text="ok", events=_buffered_events()),
"claude-opus-4-7": DriverResult(text="ok", events=_streamed_events(5)),
}
_install_fake_runner(monkeypatch, outcomes_by_model=outcomes)
runner, _captured = _make_fake_runner(outcomes_by_model=outcomes)
with pytest.raises(pytest.fail.Exception):
run_basic_messaging_cell(
@ -195,7 +200,30 @@ def test_verify_streaming_requires_all_models_to_stream(monkeypatch):
models=list(outcomes.keys()),
prompt="Count from 1 to 5, one number per line.",
verify_streaming=True,
)
env=_PROXY_ENV,
runner=runner,
)
statuses = [row["status"] for row in fake_result.rows]
assert statuses == ["pass", "fail", "pass"]
def test_missing_proxy_env_hard_fails_regardless_of_runner():
"""The env guard fires before the runner is called, and takes the
env from the injected mapping (not os.environ). Passing an empty
env dict must hard-fail even if a happy runner is bound."""
fake_result = _FakeResult()
runner, captured = _make_fake_runner(outcomes_by_model={})
with pytest.raises(pytest.fail.Exception):
run_basic_messaging_cell(
compat_result=fake_result,
models=["claude-haiku-4-5"],
prompt="whatever",
env={},
runner=runner,
)
assert captured == {}, "runner must not be called when env resolution fails"

4
tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py
View file

@ -140,7 +140,7 @@ def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch):
monkeypatch.setenv("HOME", "/home/runner")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "totally-secret")
monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-proxy-only")
monkeypatch.setenv("AZURE_FOUNDRY_API_KEY", "azure-secret")
monkeypatch.setenv("AZURE_AI_API_KEY", "azure-secret")
monkeypatch.setenv("VERTEXAI_CREDENTIALS", '{"private_key": "leak"}')
monkeypatch.setenv("GITHUB_TOKEN", "ghs_xxx")
@ -156,7 +156,7 @@ def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch):
assert env["PATH"] == "/usr/bin:/usr/local/bin"
assert "AWS_SECRET_ACCESS_KEY" not in env
assert "ANTHROPIC_API_KEY" not in env
assert "AZURE_FOUNDRY_API_KEY" not in env
assert "AZURE_AI_API_KEY" not in env
assert "VERTEXAI_CREDENTIALS" not in env
assert "GITHUB_TOKEN" not in env

38
tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py
View file

@ -20,6 +20,10 @@ from typing import Any, Dict, List, Mapping, Optional
import pytest
from claude_code._env import (
PRIMARY_API_KEY_ENV,
PRIMARY_BASE_URL_ENV,
)
from claude_code._passthrough import (
ANTHROPIC_PASSTHROUGH_BASE_PATH,
CLIENT_SIDE_AWS_REGION,
@ -33,8 +37,8 @@ from claude_code._passthrough import (
from claude_code.cli_driver import ClaudeCLIError, DriverResult
PROXY_ENV = {
"LITELLM_PROXY_BASE_URL": "http://localhost:4000",
"LITELLM_PROXY_API_KEY": "sk-test",
PRIMARY_BASE_URL_ENV: "http://localhost:4000",
PRIMARY_API_KEY_ENV: "sk-test",
}
@ -73,7 +77,29 @@ def test_env_missing_guard_reports_fail_and_aborts():
)
assert fake_result.single is not None
assert fake_result.single["status"] == "fail"
assert "LITELLM_PROXY_BASE_URL" in fake_result.single["error"]
assert PRIMARY_BASE_URL_ENV in fake_result.single["error"]
assert PRIMARY_API_KEY_ENV in fake_result.single["error"]
def test_suite_wide_env_reaches_the_proxy():
"""Passthrough cells resolve via the same suite-wide env names as
every other e2e cell, so EKS wiring that only exports
LITELLM_PROXY_URL + LITELLM_MASTER_KEY reaches the ALB."""
fake_result = _FakeResult()
captured: Dict[str, Any] = {}
outcome = DriverResult(text="pong")
run_passthrough_cell(
compat_result=fake_result,
models=["claude-haiku-4-5"],
prompt="ping",
run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured),
env=PROXY_ENV,
)
assert captured["base_url"] == "http://localhost:4000"
assert captured["api_key"] == "sk-test"
assert fake_result.rows == [{"status": "pass"}]
def test_anthropic_base_path_appended_to_normalized_proxy_url():
@ -87,7 +113,7 @@ def test_anthropic_base_path_appended_to_normalized_proxy_url():
prompt="ping",
passthrough_base_path=ANTHROPIC_PASSTHROUGH_BASE_PATH,
run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured),
env={**PROXY_ENV, "LITELLM_PROXY_BASE_URL": "http://localhost:4000/"},
env={**PROXY_ENV, PRIMARY_BASE_URL_ENV: "http://localhost:4000/"},
)
assert captured["base_url"] == "http://localhost:4000/anthropic"
@ -111,7 +137,7 @@ def test_extra_env_builder_receives_normalized_base_and_is_forwarded():
prompt="ping",
build_extra_env=build,
run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured),
env={**PROXY_ENV, "LITELLM_PROXY_BASE_URL": "http://localhost:4000/"},
env={**PROXY_ENV, PRIMARY_BASE_URL_ENV: "http://localhost:4000/"},
)
assert seen_bases == ["http://localhost:4000"]
@ -124,7 +150,7 @@ def test_per_model_failures_reported_individually():
captured: Dict[str, Any] = {}
outcomes = {
"claude-haiku-4-5": DriverResult(text="pong"),
"claude-sonnet-4-6": ClaudeCLIError("claude CLI timed out after 120s"),
"claude-sonnet-4-5": ClaudeCLIError("claude CLI timed out after 120s"),
"claude-opus-4-7": DriverResult(text="", exit_code=1),
}

4
tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py
View file

@ -60,10 +60,10 @@ from claude_code.rate_limiter import (
"model, expected",
[
("claude-haiku-4-5", PROVIDER_ANTHROPIC),
("claude-sonnet-4-6", PROVIDER_ANTHROPIC),
("claude-sonnet-4-5", PROVIDER_ANTHROPIC),
("claude-opus-4-7", PROVIDER_ANTHROPIC),
("claude-haiku-4-5-azure", PROVIDER_AZURE),
("claude-sonnet-4-6-azure", PROVIDER_AZURE),
("claude-sonnet-4-5-azure", PROVIDER_AZURE),
("claude-opus-4-7-vertex", PROVIDER_VERTEX_AI),
("claude-haiku-4-5-bedrock-converse", PROVIDER_BEDROCK_CONVERSE),
("claude-haiku-4-5-bedrock-invoke", PROVIDER_BEDROCK_INVOKE),

74
tests/e2e/claude_code/_env.py Normal file
View file

@ -0,0 +1,74 @@
"""Proxy-env resolution for the claude_code compat cells.
Uses the same ``LITELLM_PROXY_URL`` / ``LITELLM_MASTER_KEY`` names as
``e2e_config.py`` and the rest of ``tests/e2e/``. Everything under
``claude_code/`` goes through ``resolve_proxy`` / ``require_proxy`` here
so the naming lives in one place.
"""
from __future__ import annotations
import os
from typing import Mapping, NamedTuple
import pytest
class ProxyConfig(NamedTuple):
base_url: str
api_key: str
PRIMARY_BASE_URL_ENV = "LITELLM_PROXY_URL"
PRIMARY_API_KEY_ENV = "LITELLM_MASTER_KEY"
def resolve_proxy_from(mapping: Mapping[str, str]) -> ProxyConfig | None:
"""Pure resolver: takes an env mapping, returns a ProxyConfig if
both a base URL and an API key are present, else None. Extracted so
tests can exercise it without mutating ``os.environ``."""
base_url = mapping.get(PRIMARY_BASE_URL_ENV) or None
api_key = mapping.get(PRIMARY_API_KEY_ENV) or None
if not base_url or not api_key:
return None
return ProxyConfig(base_url=base_url, api_key=api_key)
def resolve_proxy(env: Mapping[str, str] | None = None) -> ProxyConfig | None:
"""Convenience wrapper that defaults to ``os.environ``. Prefer
calling ``resolve_proxy_from(env)`` from tests so nothing has to
reach into the process environment."""
return resolve_proxy_from(os.environ if env is None else env)
def _fail_missing_proxy_env(compat_result) -> None:
compat_result.set(
{
"status": "fail",
"error": (
f"missing required env: set {PRIMARY_BASE_URL_ENV} and "
f"{PRIMARY_API_KEY_ENV} to point at a running LiteLLM proxy"
),
}
)
pytest.fail(
f"{PRIMARY_BASE_URL_ENV} / {PRIMARY_API_KEY_ENV} not configured",
pytrace=False,
)
def require_proxy(
compat_result,
*,
env: Mapping[str, str] | None = None,
) -> ProxyConfig:
"""Return the proxy config (base URL + master key), or hard-fail
the test.
``env`` is injected for tests; production callers pass nothing and
the process env is used. This keeps tests off ``monkeypatch.setenv``
for a check that is a pure function of its inputs."""
cfg = resolve_proxy(env)
if cfg is None:
_fail_missing_proxy_env(compat_result)
return cfg

28
tests/e2e/claude_code/_passthrough.py
View file

@ -54,20 +54,17 @@ them unset.
from __future__ import annotations
import os
from typing import Any, Callable, Dict, Mapping, Optional, Sequence
import pytest
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
failure_diagnostic,
run_claude_models_parallel,
)
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
ANTHROPIC_PASSTHROUGH_BASE_PATH = "/anthropic"
CLIENT_SIDE_AWS_REGION = "us-east-1"
@ -140,32 +137,15 @@ def run_passthrough_cell(
the trailing-slash-normalized proxy base URL and returns the
provider-mode env for the CLI subprocess.
"""
environ = env if env is not None else os.environ
base_url = environ.get(PROXY_BASE_URL_ENV)
api_key = environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
compat_result.set(
{
"status": "fail",
"error": (
f"missing required env: set {PROXY_BASE_URL_ENV} and "
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
),
}
)
pytest.fail(
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured",
pytrace=False,
)
proxy_base = base_url.rstrip("/")
proxy = require_proxy(compat_result, env=env)
proxy_base = proxy.base_url.rstrip("/")
extra_env = dict(build_extra_env(proxy_base)) if build_extra_env else None
outcomes = run_models(
models=models,
prompt=prompt,
base_url=proxy_base + passthrough_base_path,
api_key=api_key,
api_key=proxy.api_key,
extra_env=extra_env,
)

29
tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py
View file

@ -35,8 +35,7 @@ from typing import Iterable
import pytest
REPO_ROOT = Path(__file__).resolve().parents[4]
CLAUDE_CODE_DIR = REPO_ROOT / "tests" / "e2e" / "claude_code"
CLAUDE_CODE_DIR = Path(__file__).resolve().parents[1]
# Feature directories whose cells drive the `Bash` built-in tool. Add
# new entries here when a new Bash-using feature is added; the test
@ -74,14 +73,14 @@ def _has_bare_bash_token(text: str) -> bool:
@pytest.mark.parametrize(
"cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT))
"cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(CLAUDE_CODE_DIR))
)
def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None:
"""The cell must pass `Bash(echo pong)` as the allow rule, not the
unrestricted `Bash` value that was originally flagged."""
text = cell.read_text()
assert '"Bash(echo pong)"' in text, (
f"{cell.relative_to(REPO_ROOT)} must restrict `--allowed-tools` to "
f"{cell.relative_to(CLAUDE_CODE_DIR)} must restrict `--allowed-tools` to "
f'`Bash(echo pong)` (exact-match pattern). Unrestricted `"Bash"` '
f"grants arbitrary host command execution to model-controlled "
f"tool_use blocks, which can read `docker inspect compat-proxy` "
@ -95,7 +94,7 @@ def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None:
# in text` short-circuits to True and lets a stray bare `"Bash"`
# slip through silently.
assert not _has_bare_bash_token(text), (
f"{cell.relative_to(REPO_ROOT)} still references the unrestricted "
f"{cell.relative_to(CLAUDE_CODE_DIR)} still references the unrestricted "
f'`"Bash"` value outside the `"Bash(echo pong)"` allow rule — '
f"sweep it out before merging."
)
@ -130,7 +129,7 @@ def test_has_bare_bash_token_ignores_unrelated_substrings():
@pytest.mark.parametrize(
"cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT))
"cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(CLAUDE_CODE_DIR))
)
def test_bash_cell_uses_dontask_permission_mode(cell: Path) -> None:
"""The cell must pair the allow rule with `--permission-mode dontAsk`
@ -139,9 +138,25 @@ def test_bash_cell_uses_dontask_permission_mode(cell: Path) -> None:
succeed without ever surfacing the security issue)."""
text = cell.read_text()
assert '"--permission-mode"' in text and '"dontAsk"' in text, (
f"{cell.relative_to(REPO_ROOT)} must pass `--permission-mode dontAsk` "
f"{cell.relative_to(CLAUDE_CODE_DIR)} must pass `--permission-mode dontAsk` "
f"alongside the `Bash(echo pong)` allow rule. Without dontAsk, "
f"commands outside the allow rule fall back to the default ask-"
f"mode behavior, which in `--print` (headless) mode is non-"
f"interactive — defeating the explicit-allow contract."
)
def test_claude_code_dir_anchor_is_layout_independent() -> None:
"""CLAUDE_CODE_DIR must resolve to the `claude_code/` directory that
contains this test file, regardless of how deep the repository is
mounted. The previous anchor `Path(__file__).resolve().parents[4]`
baked in the host layout (repo root sits four levels up) and broke
when the suite runs inside the stage container, where tests/e2e/ is
mounted at /app/e2e/ so `parents[4]` resolves to filesystem root and
the BASH_FEATURE_DIRS assertion looks for `/tests/e2e/claude_code/
tool_use`. Anchoring at `parents[1]` (the sibling of this file's
parent) is the same directory in both layouts.
"""
assert CLAUDE_CODE_DIR.name == "claude_code"
assert CLAUDE_CODE_DIR.is_dir()
assert (CLAUDE_CODE_DIR / "_pr_gate_unit_tests" / Path(__file__).name).is_file()

166
tests/e2e/claude_code/_pr_gate_unit_tests/test_compat_models.py Normal file
View file

@ -0,0 +1,166 @@
"""Regression tests for the compat-model registration loader.
The compat cells hardcode virtual model names like ``claude-sonnet-4-5``
and expect them to be registered on the proxy before the cell runs. The
session fixture in ``conftest.py`` reads ``test_config.yaml`` and POSTs
those deployments via ``/model/new``. These tests pin the invariants
that make that safe:
- The yaml declares an entry for every virtual name a cell references -
otherwise a cell probes a name the fixture never registered, and the
cell hits an ``Invalid model name`` 400 that is much harder to trace.
- The ``vertex_ai_*`` yaml keys get normalized to the ``vertex_*``
pydantic-body names before ``LiteLLMParamsBody(**)`` sees them, so
the vertex project/location aren't silently dropped by pydantic's
``extra="ignore"`` default.
"""
from __future__ import annotations
import re
from pathlib import Path
import pytest
from claude_code._compat_models import (
all_expected_model_names,
load_all_deployments,
)
CLAUDE_CODE_DIR = Path(__file__).resolve().parents[1]
def _cell_declared_model_names() -> frozenset[str]:
"""Every ``"claude-*"`` model name a compat cell hardcodes in a
``*_MODELS`` list. Uses a simple regex rather than importing every
cell because the cells depend on the harness which depends on env
the unit-test run does not have."""
pattern = re.compile(r'"(claude-[a-zA-Z0-9._-]+)"')
found: set[str] = set()
for path in CLAUDE_CODE_DIR.glob("*/test_*.py"):
if path.parent.name.startswith("_"):
continue
for match in pattern.finditer(path.read_text()):
name = match.group(1)
# Skip upstream model references (they carry a version
# suffix or the ``anthropic/`` provider prefix - we only
# want proxy-side virtual names here).
if "/" in name or "@" in name:
continue
found.add(name)
return frozenset(found)
def test_yaml_covers_every_cell_declared_model_name() -> None:
"""Every ``"claude-..."`` string a cell probes must have a
corresponding ``model_list`` entry in ``test_config.yaml``. A new
cell that adds a probe for a name the yaml doesn't know fails this
test - the alternative is a 400 at runtime that is much harder to
diagnose."""
yaml_names = all_expected_model_names()
cell_names = _cell_declared_model_names()
missing = cell_names - yaml_names
assert not missing, (
f"compat cells reference model names not declared in "
f"test_config.yaml: {sorted(missing)}. Add a matching "
f"model_list entry so the session fixture can register them."
)
def test_yaml_has_no_unused_declarations() -> None:
"""Every declaration in ``test_config.yaml`` is referenced by at
least one cell. A yaml entry no test exercises is dead
configuration and drift-prone; delete it or add the cell."""
yaml_names = all_expected_model_names()
cell_names = _cell_declared_model_names()
unused = yaml_names - cell_names
assert not unused, (
f"test_config.yaml declares model names no cell references: "
f"{sorted(unused)}. Delete them or add the cell."
)
def test_load_returns_fifteen_deployments() -> None:
"""The compat matrix is 3 tiers x 5 provider surfaces = 15. Pin the
count so a future edit to the yaml can't silently drop a tier."""
assert len(load_all_deployments()) == 15
def test_deployments_are_hashable_and_frozen() -> None:
"""``CompatDeployment`` is frozen so tests cannot accidentally
mutate the shared list mid-session."""
d = load_all_deployments()[0]
with pytest.raises((AttributeError, TypeError)):
d.model_name = "mutated" # type: ignore[misc]
def test_vertex_yaml_keys_populate_pydantic_body() -> None:
"""The yaml spells vertex fields ``vertex_ai_project`` /
``vertex_ai_location`` but ``LiteLLMParamsBody`` names them
``vertex_project`` / ``vertex_location``. Without the alias
normalization the pydantic body silently drops the yaml keys, and
the deployment gets registered with no vertex project - a real
incident the drift regressed twice historically."""
all_deployments = load_all_deployments()
vertex = [
d for d in all_deployments if d.model_name.endswith("-vertex")
]
assert vertex, "no vertex deployments found in yaml"
for d in vertex:
assert d.litellm_params.vertex_project, (
f"{d.model_name} lost its vertex_project after normalization"
)
assert d.litellm_params.vertex_location, (
f"{d.model_name} lost its vertex_location after normalization"
)
def test_vertex_deployments_keep_use_in_pass_through() -> None:
"""Vertex passthrough cells need the deployment registered with
``use_in_pass_through: true`` so the proxy wires project/location
credentials into the /vertex_ai passthrough router. ``LiteLLMParamsBody``
defaults to ``extra="ignore"``, so a missing field on the body silently
strips the yaml flag and every vertex passthrough cell fails at runtime
with "No credentials found on proxy for project_name=..."."""
vertex = [
d
for d in load_all_deployments()
if d.model_name.endswith("-vertex")
]
assert vertex, "no vertex deployments found in yaml"
for d in vertex:
assert d.litellm_params.use_in_pass_through is True, (
f"{d.model_name} lost use_in_pass_through after load; "
f"serialized body would be "
f"{d.litellm_params.model_dump(exclude_none=True)}"
)
def test_yaml_litellm_params_are_all_known_body_fields() -> None:
"""Every key under ``litellm_params`` in ``test_config.yaml`` must map
to a ``LiteLLMParamsBody`` field (after the vertex alias rewrite).
Without this pin, a new yaml flag can land in the fixture config and
be silently dropped by pydantic before ``/model/new`` ever sees it."""
import yaml
from models import LiteLLMParamsBody
from claude_code._compat_models import (
CONFIG_PATH,
_YAML_TO_PYDANTIC_ALIASES,
)
known = frozenset(LiteLLMParamsBody.model_fields)
doc = yaml.safe_load(CONFIG_PATH.read_text())
model_list = doc.get("model_list") or []
unknown = tuple(
(entry["model_name"], key)
for entry in model_list
for key in entry["litellm_params"]
if _YAML_TO_PYDANTIC_ALIASES.get(key, key) not in known
)
assert not unknown, (
f"test_config.yaml litellm_params keys not on LiteLLMParamsBody "
f"(will be silently dropped at register time): {unknown}"
)

162
tests/e2e/claude_code/_pr_gate_unit_tests/test_env_resolution.py Normal file
View file

@ -0,0 +1,162 @@
"""Regression tests for ``claude_code/_env.py``.
Pin the resolution rules so a future edit cannot silently reintroduce
a private spelling that makes every ``claude_code`` cell fail with
"not configured" even when the surrounding e2e suite has a live proxy
configured under the suite-wide ``LITELLM_PROXY_URL`` /
``LITELLM_MASTER_KEY`` names.
"""
from __future__ import annotations
import pytest
from claude_code._env import (
PRIMARY_API_KEY_ENV,
PRIMARY_BASE_URL_ENV,
ProxyConfig,
require_proxy,
resolve_proxy_from,
)
class _CompatResultStub:
"""Minimal stand-in for the compat_result fixture used by cells."""
def __init__(self) -> None:
self.calls: list[dict[str, str]] = []
def set(self, payload: dict[str, str]) -> None:
self.calls.append(payload)
def test_primary_env_names_match_suite_wide_config() -> None:
"""The names claude_code reads must exactly match the ones
``e2e_config.py`` reads for the rest of the suite. Anything else
silently reintroduces the drift this refactor cleaned up."""
assert PRIMARY_BASE_URL_ENV == "LITELLM_PROXY_URL"
assert PRIMARY_API_KEY_ENV == "LITELLM_MASTER_KEY"
def test_returns_none_when_no_env_is_set() -> None:
assert resolve_proxy_from({}) is None
def test_returns_none_when_only_url_is_set() -> None:
assert (
resolve_proxy_from({PRIMARY_BASE_URL_ENV: "http://localhost:4000"}) is None
)
def test_returns_none_when_only_key_is_set() -> None:
assert resolve_proxy_from({PRIMARY_API_KEY_ENV: "sk-1234"}) is None
def test_primary_pair_resolves() -> None:
cfg = resolve_proxy_from(
{
PRIMARY_BASE_URL_ENV: "http://localhost:4000",
PRIMARY_API_KEY_ENV: "sk-1234",
}
)
assert cfg == ProxyConfig("http://localhost:4000", "sk-1234")
def test_legacy_pair_is_ignored() -> None:
"""``LITELLM_PROXY_BASE_URL`` / ``LITELLM_PROXY_API_KEY`` are not
accepted. A runner that only exports those must fail closed rather
than silently use a private spelling that the rest of the suite
does not know about."""
assert (
resolve_proxy_from(
{
"LITELLM_PROXY_BASE_URL": "http://legacy:4000",
"LITELLM_PROXY_API_KEY": "sk-legacy",
}
)
is None
)
def test_empty_string_env_is_treated_as_unset() -> None:
"""``os.environ.get`` on an exported-but-empty var returns "" which
is falsy. The resolver must treat that as unset so a shell that
accidentally exports ``LITELLM_PROXY_URL=`` doesn't turn into a
"" base_url that hits the wrong endpoint."""
assert (
resolve_proxy_from(
{PRIMARY_BASE_URL_ENV: "", PRIMARY_API_KEY_ENV: "sk-1234"}
)
is None
)
def test_require_proxy_fails_with_helpful_message_when_env_empty() -> None:
"""The error the user sees must name the suite-wide env vars so
they know exactly what to export."""
compat = _CompatResultStub()
with pytest.raises(pytest.fail.Exception) as excinfo:
require_proxy(compat, env={})
assert PRIMARY_BASE_URL_ENV in str(excinfo.value)
assert PRIMARY_API_KEY_ENV in str(excinfo.value)
assert compat.calls and compat.calls[0]["status"] == "fail"
assert PRIMARY_BASE_URL_ENV in compat.calls[0]["error"]
assert PRIMARY_API_KEY_ENV in compat.calls[0]["error"]
assert "LITELLM_PROXY_BASE_URL" not in compat.calls[0]["error"]
def test_require_proxy_returns_config_when_primary_env_supplied() -> None:
cfg = require_proxy(
_CompatResultStub(),
env={
PRIMARY_BASE_URL_ENV: "http://localhost:4000",
PRIMARY_API_KEY_ENV: "sk-1234",
},
)
assert cfg == ProxyConfig("http://localhost:4000", "sk-1234")
class TestControlGatewayFollowsResolvedProxy:
"""The session fixture that registers the compat deployments must talk
to the *same* proxy the cells do.
Building its Gateway off ``e2e_config``'s own env read instead of the
resolved ``ProxyConfig`` would send ``/model/new`` to whatever
``e2e_config`` defaults to when the process env is empty, while the
cells drive a different host — so registration silently lands
somewhere else and every cell 400s with "Invalid model name".
"""
RESOLVED = ProxyConfig("http://eks-alb.internal:4000", "sk-eks")
def _gateway(self):
from claude_code.conftest import _build_control_gateway
return _build_control_gateway(self.RESOLVED)
def test_management_calls_go_to_the_resolved_host_and_key(self) -> None:
control = self._gateway().transport.control
assert control.base_url == self.RESOLVED.base_url
assert control.master_key == self.RESOLVED.api_key
def test_both_planes_share_the_one_address_the_cells_use(self) -> None:
"""The deployment is fronted by a single address that routes
management and LLM paths itself, so a resolved proxy pins both."""
transport = self._gateway().transport
assert transport.data.base_url == self.RESOLVED.base_url
assert transport.data.master_key == self.RESOLVED.api_key
assert transport.control.base_url == transport.data.base_url
def test_require_proxy_leaves_compat_result_untouched_on_success() -> None:
"""A successful resolution must NOT append a spurious fail entry.
Would have silently poisoned every compat cell's result rows."""
compat = _CompatResultStub()
require_proxy(
compat,
env={
PRIMARY_BASE_URL_ENV: "http://localhost:4000",
PRIMARY_API_KEY_ENV: "sk-1234",
},
)
assert compat.calls == []

95
tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_pytest_scrubs_env.py
View file

@ -1,95 +0,0 @@
"""Pin: the cron `pytest` invocation must run under `env -i`.
The systemd service `litellm-compat-matrix.service` loads provider
credentials (`ANTHROPIC_API_KEY`, `AWS_BEARER_TOKEN_BEDROCK`,
`AZURE_FOUNDRY_API_KEY`, `VERTEXAI_*`) and the agent-shin GitHub token
(`AGENT_SHIN_GITHUB_TOKEN`) into `run_daily.sh`'s environment from
`/etc/litellm-compat-matrix.env`. Pytest only needs to talk to the
loopback proxy at `127.0.0.1:${PROXY_PORT}` and has no legitimate reason
to see provider creds in its own `os.environ`. Leaving them in would
let a test under `tests/e2e/claude_code/` read them via `os.environ` and
exfiltrate them, and would also let a model-directed `Read` tool call
during a PDF/vision cell reach `/proc/<pytest-pid>/environ`. The
PR-gate's pytest step in `.circleci/config.yml` already runs under
`env -i`; this pin enforces the same scrub on the cron path.
"""
from __future__ import annotations
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[4]
RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh"
def _pytest_invocation_block() -> str:
"""Return only the executable lines around the pytest invocation.
Comment text in run_daily.sh explains *why* certain credential
names must not appear, so a naïve substring scan over the whole
region would false-positive on the rationale itself. Strip lines
whose first non-space character is `#`.
"""
body = RUN_DAILY.read_text()
start = body.index('log "running pytest"')
end = body.index("PYTEST_EXIT=$?", start)
return "\n".join(
line for line in body[start:end].splitlines()
if line.lstrip()[:1] != "#"
)
def test_pytest_invocation_wraps_in_env_i() -> None:
block = _pytest_invocation_block()
assert "env -i" in block, (
"run_daily.sh: the pytest invocation must run under `env -i` so "
"PR-controlled test code under tests/e2e/claude_code/ cannot read "
"provider/agent-shin credentials out of the systemd service "
"environment, and so a model-directed `Read` tool call cannot "
"reach /proc/<pytest-pid>/environ to pull them out."
)
assert block.index("env -i") < block.index('"${WORKTREE_UV}" run pytest'), (
"run_daily.sh: `env -i` must precede the pytest invocation; "
"otherwise pytest inherits the full credential-bearing env."
)
def test_pytest_invocation_env_i_excludes_provider_secrets() -> None:
block = _pytest_invocation_block()
for forbidden in (
"ANTHROPIC_API_KEY",
"AWS_BEARER_TOKEN_BEDROCK",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"VERTEXAI_CREDENTIALS",
"VERTEXAI_PROJECT",
"VERTEXAI_LOCATION",
"AZURE_FOUNDRY_API_KEY",
"AZURE_FOUNDRY_API_BASE",
"GITHUB_TOKEN",
"AGENT_SHIN_GITHUB_TOKEN",
):
assert forbidden not in block, (
f"run_daily.sh: the pytest-step `env -i` allowlist must not "
f"pass {forbidden} through. Found it inside the pytest "
f"invocation block."
)
def test_pytest_invocation_passes_proxy_url_and_key_explicitly() -> None:
block = _pytest_invocation_block()
assert "LITELLM_PROXY_BASE_URL=" in block, (
"run_daily.sh: the pytest `env -i` block must still pass "
"LITELLM_PROXY_BASE_URL so the test suite knows where to find "
"the loopback proxy."
)
assert "LITELLM_PROXY_API_KEY=" in block, (
"run_daily.sh: the pytest `env -i` block must still pass "
"LITELLM_PROXY_API_KEY so the test suite can authenticate to "
"the loopback proxy."
)
assert "COMPAT_RESULTS_PATH=" in block, (
"run_daily.sh: the pytest `env -i` block must still pass "
"COMPAT_RESULTS_PATH so the conftest writes the per-cell "
"tagged-union artifact to the script-managed path."
)

289
tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_release_pagination.py
View file

@ -1,289 +0,0 @@
"""Regression tests for the GitHub release pagination in `run_daily.sh`.
The cron job resolves "newest LiteLLM v*-stable" via the GitHub Releases
API. A previous version of the loop broke as soon as the current page
contained ANY v*-stable tag. The Releases endpoint orders by
`created_at`, NOT by semver, so a backport on an older series cut today
(e.g. v1.80.1-stable) can land on an earlier page than a higher-version
release cut two weeks ago (e.g. v1.83.0-stable). The early-break would
silently pin the cron to a stale tag because the higher-version release
on a later page never made it into the merged set the final `sort_by`
consumed.
These tests pin two things:
1. The buggy early-break-on-first-stable pattern must not return.
2. The loop still terminates early on the standard "empty page" guard
so a quiet release feed doesn't burn API quota.
The shell loop itself is exercised end-to-end with a fake `curl` that
serves canned page JSON, demonstrating that the resolved tag is the
highest-semver stable across all pages even when the highest tag lives
on page 2+.
"""
from __future__ import annotations
import os
import shutil
import subprocess
import textwrap
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[4]
RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh"
# The extracted snippet starts AFTER `log`/`die` are defined in run_daily.sh,
# so the test harness has to provide its own stubs. Without them, a failure
# inside the snippet (e.g. jq returning an empty LITELLM_VERSION) would crash
# with `bash: die: command not found` (exit 127) instead of the intended
# diagnostic, making test failures unnecessarily hard to debug.
_PREAMBLE = (
"set -Eeuo pipefail\n"
"log() { printf '==> %s\\n' \"$*\" >&2; }\n"
"die() { printf 'ERROR: %s\\n' \"$*\" >&2; exit 1; }\n"
)
def test_run_daily_does_not_early_break_on_first_stable_page() -> None:
"""The regex pattern `select(test("...stable$"))] | length > 0` followed
by `break` is exactly the buggy early-stop. If it ever returns the
cron will silently start testing against a stale stable tag.
"""
body = RUN_DAILY.read_text()
assert (
"length > 0" not in body
or "break" not in body
or (
# If both substrings exist, make sure they aren't both inside the
# same release-pagination loop. The current loop only contains
# a `break` for the empty-page guard, not for any "length > 0"
# condition.
not _shares_loop_body(body, "length > 0", "break")
)
), (
"run_daily.sh contains the old early-break-on-stable pattern. The "
"Releases endpoint orders by created_at, not semver, so breaking "
"on first-stable-seen can miss higher-versioned releases sitting "
"on later pages."
)
def _shares_loop_body(body: str, needle_a: str, needle_b: str) -> bool:
"""Heuristic: do both needles live inside a `for page in ...; do ... done`
block? Used as a defensive guard for the static check above."""
in_loop = False
saw_a = False
saw_b = False
for line in body.splitlines():
stripped = line.strip()
if stripped.startswith("for page in"):
in_loop = True
saw_a = False
saw_b = False
continue
if in_loop and stripped == "done":
if saw_a and saw_b:
return True
in_loop = False
continue
if in_loop:
if needle_a in line:
saw_a = True
if needle_b in line:
saw_b = True
return False
def test_run_daily_keeps_empty_page_break_guard() -> None:
"""The empty-page break is the only break that should remain in the
pagination loop — without it a quiet release feed wastes API quota
walking past the last real page."""
body = RUN_DAILY.read_text()
assert "jq 'length' \"${PAGE_JSON}\"" in body, (
"run_daily.sh must still detect empty pages via `jq 'length' "
"${PAGE_JSON}`; without this the loop walks the full 5-page cap "
"even when there are no more releases."
)
assert (
'== "0"' in body
), 'The empty-page guard must compare jq\'s length output to "0".'
def _make_fake_curl(scratch: Path, pages: dict[int, str]) -> Path:
"""Build a fake `curl` shim that serves the canned page JSON for
each `page=N` request and an empty array for any page past the
last canned one.
The shim mimics just enough of curl's CLI surface for the cron
script: it accepts the headers + URL we pass, ignores everything
we don't need, and writes the canned body to either stdout or the
--output target if one is given.
"""
pages_dir = scratch / "pages"
pages_dir.mkdir()
for page_num, body in pages.items():
(pages_dir / f"page{page_num}.json").write_text(body)
curl_path = scratch / "curl"
curl_path.write_text(
textwrap.dedent(
f"""\
#!/usr/bin/env bash
# Fake curl for run_daily.sh release pagination tests. Serves
# page JSON from {pages_dir} keyed by the `page=` query value,
# and returns "[]" for pages past the last canned one (which
# is exactly how the real GitHub API behaves past the end).
url=""
output=""
while [[ $# -gt 0 ]]; do
case "$1" in
-fsS|-fsSL|-H|-o|--output)
if [[ "$1" == "-o" || "$1" == "--output" ]]; then
output="$2"; shift 2
elif [[ "$1" == "-H" ]]; then
shift 2
else
shift
fi
;;
http*)
url="$1"; shift
;;
*)
shift
;;
esac
done
page="$(printf '%s' "$url" | sed -n 's/.*[?&]page=\\([0-9]*\\).*/\\1/p')"
[[ -z "$page" ]] && page=1
file="{pages_dir}/page${{page}}.json"
if [[ -f "$file" ]]; then
if [[ -n "$output" ]]; then cp "$file" "$output"; else cat "$file"; fi
else
if [[ -n "$output" ]]; then printf '[]' > "$output"; else printf '[]'; fi
fi
"""
)
)
curl_path.chmod(0o755)
return curl_path
def _extract_resolution_snippet() -> str:
"""Pull the pagination + sort_by + assignment block out of run_daily.sh
so the test exercises the actual production code path (not a copy).
The block is everything from the GH_AUTH_HEADER setup down through
the LITELLM_VERSION emission.
"""
body = RUN_DAILY.read_text()
start = body.index("GH_AUTH_HEADER=()")
end = body.index('log "resolved litellm:')
return body[start:end]
@pytest.mark.skipif(shutil.which("jq") is None, reason="jq not available")
def test_run_daily_resolves_highest_semver_across_pages(tmp_path: Path) -> None:
"""End-to-end: drive the actual run_daily.sh pagination loop with a
fake curl whose page 1 contains a freshly-cut LOW-version backport
(v1.80.1-stable) and page 2 contains a two-weeks-old HIGH-version
release (v1.83.0-stable). The correct behavior is to resolve
v1.83.0-stable. The pre-fix behavior would resolve v1.80.1-stable
because the early-break consumed only page 1.
"""
pages = {
# Page 1: most-recently-created releases. The order here matches
# what /releases?page=1 returns: created-at descending. The
# freshly-cut v1.80.1-stable backport sits at the top, plus a
# bunch of non-stable releases.
1: """[
{"tag_name": "v1.84.0-nightly.1"},
{"tag_name": "v1.80.1-stable"},
{"tag_name": "v1.84.0-nightly.0"}
]""",
# Page 2: older releases. The HIGHER-version stable lives here
# because it was cut two weeks ago, before the v1.80.1 backport.
2: """[
{"tag_name": "v1.83.0-rc.5"},
{"tag_name": "v1.83.0-stable"},
{"tag_name": "v1.82.4-stable"}
]""",
# Page 3+: empty -> the loop's empty-page guard fires here.
}
fake_curl_dir = tmp_path / "shim"
fake_curl_dir.mkdir()
_make_fake_curl(fake_curl_dir, pages)
workdir = tmp_path / "work"
workdir.mkdir()
snippet = _extract_resolution_snippet()
script = (
_PREAMBLE
+ f"WORKDIR={workdir!s}\n"
+ snippet
+ 'printf "%s" "${LITELLM_VERSION}"\n'
)
env = {
**os.environ,
"PATH": f"{fake_curl_dir}:{os.environ.get('PATH', '')}",
}
# Make sure the loop hits the fake curl, not the system one.
env.pop("GITHUB_TOKEN", None)
result = subprocess.run(
["bash", "-c", script],
capture_output=True,
text=True,
env=env,
check=True,
)
assert result.stdout == "v1.83.0-stable", (
f"Expected the highest-semver stable across pages 1-2, got "
f"{result.stdout!r}. stderr={result.stderr!r}"
)
@pytest.mark.skipif(shutil.which("jq") is None, reason="jq not available")
def test_run_daily_terminates_on_empty_page(tmp_path: Path) -> None:
"""The empty-page guard must fire so we don't always walk all 5
pages. With a single populated page and an empty page 2 we should
stop after fetching page 2 (the first empty response)."""
pages = {1: '[{"tag_name": "v1.50.0-stable"}]'}
fake_curl_dir = tmp_path / "shim"
fake_curl_dir.mkdir()
_make_fake_curl(fake_curl_dir, pages)
workdir = tmp_path / "work"
workdir.mkdir()
snippet = _extract_resolution_snippet()
script = (
_PREAMBLE
+ f"WORKDIR={workdir!s}\n"
+ snippet
+ 'printf "%s" "${LITELLM_VERSION}"\n'
)
env = {
**os.environ,
"PATH": f"{tmp_path}/shim:{os.environ.get('PATH', '')}",
}
env.pop("GITHUB_TOKEN", None)
result = subprocess.run(
["bash", "-c", script],
capture_output=True,
text=True,
env=env,
check=True,
)
assert result.stdout == "v1.50.0-stable"
# Only pages 1 and 2 should have been fetched (2 is empty -> break).
assert (workdir / "releases.page2.json").exists()
assert not (workdir / "releases.page3.json").exists(), (
"Empty-page guard didn't fire — the loop kept walking past the "
"first empty response."
)

92
tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py
View file

@ -1,92 +0,0 @@
"""Pin: the cron `claude --version` probe must run under `env -i`.
The systemd service `litellm-compat-matrix.service` loads provider
credentials (`ANTHROPIC_API_KEY`, `AWS_BEARER_TOKEN_BEDROCK`,
`AZURE_FOUNDRY_API_KEY`) and the agent-shin GitHub token
(`AGENT_SHIN_GITHUB_TOKEN`) into `run_daily.sh`'s environment from
`/etc/litellm-compat-matrix.env`. Running the npm-installed `claude`
binary directly there would hand that full env to package code, so a
compromised `@anthropic-ai/claude-code` release could read those
secrets out of `os.environ` before the proxy or test harness ever
starts. The version probe must be wrapped in `env -i` with a minimal
PATH/HOME/USER/TERM/LANG/LC_ALL/TMPDIR allowlist — matching the
PR-gate's resolver/npm-install/pytest scrubs.
"""
from __future__ import annotations
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[4]
RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh"
def _version_probe_block() -> str:
body = RUN_DAILY.read_text()
start = body.index("CLAUDE_CODE_VERSION=")
end = body.index('[[ -n "${CLAUDE_CODE_VERSION}" ]]', start)
return body[start:end]
def test_version_probe_wraps_claude_in_env_i() -> None:
block = _version_probe_block()
assert "env -i" in block, (
"run_daily.sh: the `claude --version` probe must run under "
"`env -i` so a compromised @anthropic-ai/claude-code package "
"cannot read provider/GitHub credentials out of the systemd "
"service environment."
)
assert block.index("env -i") < block.index("claude --version"), (
"run_daily.sh: `env -i` must precede `claude --version`; "
"otherwise the binary inherits the full credential-bearing env."
)
def test_version_probe_env_i_excludes_provider_secrets() -> None:
block = _version_probe_block()
for forbidden in (
"ANTHROPIC_API_KEY",
"AWS_BEARER_TOKEN_BEDROCK",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"VERTEXAI_CREDENTIALS",
"AZURE_FOUNDRY_API_KEY",
"GITHUB_TOKEN",
"AGENT_SHIN_GITHUB_TOKEN",
):
assert forbidden not in block, (
f"run_daily.sh: the version-probe `env -i` allowlist must "
f"not pass {forbidden} through. Found it inside the probe "
f"block."
)
def test_version_probe_uses_isolated_home_not_runtime_user_home() -> None:
"""Pin: the `claude --version` probe runs under a fresh empty HOME.
`ProtectHome=read-only` in the systemd unit allows reads of the
runtime user's real home directory. If the probe's `env -i`
block forwards `HOME=${HOME}`, a compromised `claude` package
can `os.path.expanduser("~/.config/gh/hosts.yml")` or
`os.path.expanduser("~/.ssh/...")` and exfiltrate the contents
before the proxy or test harness ever starts. The probe must
point HOME at a per-run tmpdir under `${WORKDIR}` so the CLI
sees an empty HOME instead.
"""
block = _version_probe_block()
body = RUN_DAILY.read_text()
assert "CLAUDE_PROBE_HOME=" in body, (
"run_daily.sh: must define a `CLAUDE_PROBE_HOME` per-run tmpdir "
"for the `claude --version` probe so the CLI never sees the "
"runtime user's real $HOME."
)
assert 'HOME="${CLAUDE_PROBE_HOME}"' in block, (
"run_daily.sh: the probe's `env -i` block must set HOME to "
"the per-run isolated tmpdir, not to the runtime user's $HOME."
)
assert 'HOME="${HOME}"' not in block, (
"run_daily.sh: the probe's `env -i` block must not forward the "
"runtime user's $HOME to `claude --version`. Use the isolated "
"$CLAUDE_PROBE_HOME tmpdir instead."
)

Some files were not shown because too many files have changed in this diff Show more