diff --git a/Dockerfile b/Dockerfile
index 915daff999c..9ad9ab31b65 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,8 @@
# Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
# Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@@ -68,8 +68,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
- npm install -g npm@11.12.1 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
+ npm install -g npm@11.14.0 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
GLOBAL="$(npm root -g)" && \
for pkg in tar glob @isaacs/brace-expansion brace-expansion minimatch diff picomatch; do \
name="${pkg##*/}"; \
@@ -85,17 +85,17 @@ ENV PATH="/app/.venv/bin:${PATH}"
COPY --from=builder /app /app
# Prisma binaries live in $HOME/.cache (default prisma-python location),
-# which is /root/.cache here. Copy them from the builder so they survive
-# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
-# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
-COPY --from=builder /root/.cache /root/.cache
+# which is /root/.cache here. Copy only the Prisma subdirs — copying the
+# whole /root/.cache drags in the uv build cache (~660 MB, includes a
+# setuptools wheel that surfaces as a CVE finding even though it's not
+# on the runtime sys.path).
+COPY --from=builder /root/.cache/prisma /root/.cache/prisma
+COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
EXPOSE 4000/tcp
-COPY docker/supervisord.conf /etc/supervisord.conf
-
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]
diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml
index a9b41e0d8e0..25f69080878 100644
--- a/deploy/charts/litellm-helm/templates/deployment.yaml
+++ b/deploy/charts/litellm-helm/templates/deployment.yaml
@@ -100,6 +100,16 @@ spec:
- name: DATABASE_URL
value: {{ .Values.db.url | quote }}
{{- end }}
+ {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
+ - name: DATABASE_URL_READ_REPLICA
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.db.secret.name }}
+ key: {{ .Values.db.secret.readReplicaUrlKey }}
+ {{- else if .Values.db.readReplicaUrl }}
+ - name: DATABASE_URL_READ_REPLICA
+ value: {{ .Values.db.readReplicaUrl | quote }}
+ {{- end }}
- name: PROXY_MASTER_KEY
valueFrom:
secretKeyRef:
@@ -129,12 +139,6 @@ spec:
value: {{ $val | quote }}
{{- end }}
{{- end }}
- {{- if .Values.separateHealthApp }}
- - name: SEPARATE_HEALTH_APP
- value: "1"
- - name: SEPARATE_HEALTH_PORT
- value: {{ .Values.separateHealthPort | default "8081" | quote }}
- {{- end }}
{{- with .Values.extraEnvVars }}
{{- toYaml . | nindent 12 }}
{{- end }}
@@ -175,15 +179,10 @@ spec:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
- {{- if .Values.separateHealthApp }}
- - name: health
- containerPort: {{ .Values.separateHealthPort | default 8081 }}
- protocol: TCP
- {{- end }}
livenessProbe:
httpGet:
path: {{ .Values.livenessProbe.path | quote }}
- port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+ port: "http"
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
@@ -192,7 +191,7 @@ spec:
readinessProbe:
httpGet:
path: {{ .Values.readinessProbe.path | quote }}
- port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+ port: "http"
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
@@ -201,7 +200,7 @@ spec:
startupProbe:
httpGet:
path: {{ .Values.startupProbe.path | quote }}
- port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+ port: "http"
initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.startupProbe.periodSeconds }}
timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml
index 3dd1112bea7..9c7c013341b 100644
--- a/deploy/charts/litellm-helm/values.yaml
+++ b/deploy/charts/litellm-helm/values.yaml
@@ -88,12 +88,6 @@ service:
# optionally specify loadBalancerClass
# loadBalancerClass: tailscale
-# Separate health app configuration
-# When enabled, health checks will use a separate port and the application
-# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
-separateHealthApp: false
-separateHealthPort: 8081
-
# Probes for LiteLLM gateway container
livenessProbe:
path: /health/liveliness
@@ -258,6 +252,26 @@ db:
passwordKey: password
# Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
endpointKey: ""
+ # Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this
+ # secret key instead of db.readReplicaUrl. Prefer this over the plain
+ # value: read-replica URLs typically embed credentials, and a value
+ # written to db.readReplicaUrl ends up visible in the rendered pod spec
+ # and the Helm release secret.
+ readReplicaUrlKey: ""
+
+ # Optional read-replica routing. When set, the proxy sends read-only
+ # queries (find_*, count, group_by, query_raw/_first) to this URL while
+ # writes continue to go to db.url. Useful for Aurora-style clusters with
+ # separate reader/writer endpoints. Leave empty to keep single-DB behavior.
+ # When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed
+ # alongside the writer (host/port/user/db are parsed from this URL once
+ # at startup; only the IAM token rotates).
+ #
+ # If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over
+ # this field — the plain value is rendered into the pod spec and the
+ # Helm release secret. This field is intended for credential-less URLs
+ # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime).
+ readReplicaUrl: ""
# Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
# The Stackgres Operator must already be installed within the target
diff --git a/docker-compose.yml b/docker-compose.yml
index 988860a7877..80e1f289aad 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -16,6 +16,11 @@ services:
- "4000:4000" # Map the container port to the host, change the host port if necessary
environment:
DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
+ # Optional: route read-only queries (find_*, count, group_by, query_raw/_first)
+ # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for
+ # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL
+ # is auto-refreshed alongside the writer.
+ # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm"
STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
env_file:
- .env # Load local .env file
diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database
index 671f374ca27..c84003a065f 100644
--- a/docker/Dockerfile.database
+++ b/docker/Dockerfile.database
@@ -66,7 +66,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
@@ -102,7 +102,5 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
EXPOSE 4000/tcp
-COPY docker/supervisord.conf /etc/supervisord.conf
-
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]
diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root
index 3fa73f42437..4de4a55981d 100644
--- a/docker/Dockerfile.non_root
+++ b/docker/Dockerfile.non_root
@@ -103,13 +103,12 @@ RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
- apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
+ apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
done
COPY --from=builder /app /app
COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
-COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
ENV PATH="/app/.venv/bin:${PATH}" \
PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
diff --git a/docker/prod_entrypoint.sh b/docker/prod_entrypoint.sh
index 28d1bdcc294..bd78bf6687b 100644
--- a/docker/prod_entrypoint.sh
+++ b/docker/prod_entrypoint.sh
@@ -1,14 +1,8 @@
#!/bin/sh
-if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
- export LITELLM_ARGS="$@"
- export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
- exec supervisord -c /etc/supervisord.conf
-fi
-
if [ "$USE_DDTRACE" = "true" ]; then
export DD_TRACE_OPENAI_ENABLED="False"
exec ddtrace-run litellm "$@"
else
exec litellm "$@"
-fi
\ No newline at end of file
+fi
diff --git a/docker/supervisord.conf b/docker/supervisord.conf
deleted file mode 100644
index ba9d99d18a5..00000000000
--- a/docker/supervisord.conf
+++ /dev/null
@@ -1,46 +0,0 @@
-[supervisord]
-nodaemon=true
-loglevel=info
-logfile=/tmp/supervisord.log
-pidfile=/tmp/supervisord.pid
-
-[group:litellm]
-programs=main,health
-
-[program:main]
-command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi'
-autostart=true
-autorestart=true
-startretries=3
-priority=1
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes = 0
-stderr_logfile_maxbytes = 0
-environment=PYTHONUNBUFFERED=true
-
-[program:health]
-command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
-autostart=true
-autorestart=true
-startretries=3
-priority=2
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes = 0
-stderr_logfile_maxbytes = 0
-environment=PYTHONUNBUFFERED=true
-
-[eventlistener:process_monitor]
-command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
-events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
-autostart=true
-autorestart=true
\ No newline at end of file
diff --git a/litellm-js/spend-logs/package-lock.json b/litellm-js/spend-logs/package-lock.json
index e9fc4b00f6a..e33079766c9 100644
--- a/litellm-js/spend-logs/package-lock.json
+++ b/litellm-js/spend-logs/package-lock.json
@@ -535,9 +535,9 @@
}
},
"node_modules/get-tsconfig": {
- "version": "4.13.0",
- "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz",
- "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==",
+ "version": "4.14.0",
+ "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz",
+ "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==",
"dev": true,
"license": "MIT",
"dependencies": {
diff --git a/litellm/__init__.py b/litellm/__init__.py
index 617c102fb85..cf05fc4c980 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -388,6 +388,7 @@ anthropic_beta_headers_url: str = os.getenv(
suppress_debug_info = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None
+s3_audit_callback_params: Optional[Dict] = None
datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None
datadog_params: Optional[Union[DatadogInitParams, Dict]] = None
aws_sqs_callback_params: Optional[Dict] = None
diff --git a/litellm/constants.py b/litellm/constants.py
index 1e96f8eac9a..072c2c358f7 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -161,6 +161,11 @@ MCP_STDIO_ALLOWED_COMMANDS: frozenset = frozenset(
| (set(_MCP_STDIO_EXTRA_COMMANDS.split(",")) - {""})
)
+# MCP OAuth2 Token Exchange (OBO) Defaults
+MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE = int(
+ os.getenv("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", "500")
+)
+
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
"x-litellm-semantic-filter-tools",
diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py
index e703a3956b9..0dc56b6a3bc 100644
--- a/litellm/experimental_mcp_client/client.py
+++ b/litellm/experimental_mcp_client/client.py
@@ -366,6 +366,8 @@ class MCPClient:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif self.auth_type == MCPAuth.token:
headers["Authorization"] = f"token {self._mcp_auth_value}"
+ elif self.auth_type == MCPAuth.oauth2_token_exchange:
+ headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif isinstance(self._mcp_auth_value, dict):
headers.update(self._mcp_auth_value)
# Note: aws_sigv4 auth is not handled here — SigV4 requires per-request
diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py
index 77833e5de0f..0af016feb26 100644
--- a/litellm/integrations/opentelemetry.py
+++ b/litellm/integrations/opentelemetry.py
@@ -57,6 +57,17 @@ LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
+CAPTURE_MODE_NO_CONTENT = "NO_CONTENT"
+CAPTURE_MODE_SPAN_ONLY = "SPAN_ONLY"
+CAPTURE_MODE_EVENT_ONLY = "EVENT_ONLY"
+CAPTURE_MODE_SPAN_AND_EVENT = "SPAN_AND_EVENT"
+_VALID_CAPTURE_MODES = {
+ CAPTURE_MODE_NO_CONTENT,
+ CAPTURE_MODE_SPAN_ONLY,
+ CAPTURE_MODE_EVENT_ONLY,
+ CAPTURE_MODE_SPAN_AND_EVENT,
+}
+
@dataclass
class OpenTelemetryConfig:
@@ -71,6 +82,9 @@ class OpenTelemetryConfig:
ignore_context_propagation: Optional[bool] = None
# When True, create a private TracerProvider instead of reusing or setting the global one.
skip_set_global: bool = False
+ # Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT.
+ # One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
+ capture_message_content: Optional[str] = None
def __post_init__(self) -> None:
# If endpoint is specified but exporter is still the default "console",
@@ -182,6 +196,9 @@ class OpenTelemetry(CustomLogger):
super().__init__(**kwargs)
self._init_metrics(meter_provider)
self._init_logs(logger_provider)
+ # Sample env-var / config / message_logging at init so subsequent
+ # _capture_in_span / _capture_in_event calls are deterministic.
+ self._capture_mode_cached = self._compute_capture_mode_from_init_state()
self._init_otel_logger_on_litellm_proxy()
@staticmethod
@@ -306,6 +323,62 @@ class OpenTelemetry(CustomLogger):
hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
)
+ def _compute_capture_mode_from_init_state(self) -> Optional[str]:
+ """Sample explicit settings at init. Returns the resolved mode or
+ None if nothing explicit is set (in which case the legacy
+ ``self.message_logging`` flag is consulted dynamically per request).
+
+ ``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention.
+ ``"false"``/``"0"`` map to ``NO_CONTENT``.
+ Unknown values are ignored.
+ """
+ explicit = self.config.capture_message_content or os.getenv(
+ "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
+ )
+ if not explicit:
+ return None
+ normalized = explicit.upper()
+ if normalized in ("TRUE", "1"):
+ return CAPTURE_MODE_EVENT_ONLY
+ if normalized in ("FALSE", "0"):
+ return CAPTURE_MODE_NO_CONTENT
+ if normalized in _VALID_CAPTURE_MODES:
+ return normalized
+ return None
+
+ def _resolve_capture_mode(self) -> str:
+ """Return the active capture mode for this request.
+
+ Precedence:
+ 1. ``litellm.turn_off_message_logging=True`` forces ``NO_CONTENT``
+ (kill-switch checked dynamically).
+ 2. Explicit setting sampled at init from
+ ``OpenTelemetryConfig.capture_message_content`` or
+ ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT``.
+ 3. Legacy ``self.message_logging`` (checked dynamically).
+ """
+ if litellm.turn_off_message_logging:
+ return CAPTURE_MODE_NO_CONTENT
+ if self._capture_mode_cached is not None:
+ return self._capture_mode_cached
+ return (
+ CAPTURE_MODE_SPAN_AND_EVENT
+ if self.message_logging
+ else CAPTURE_MODE_NO_CONTENT
+ )
+
+ def _capture_in_span(self) -> bool:
+ return self._resolve_capture_mode() in (
+ CAPTURE_MODE_SPAN_ONLY,
+ CAPTURE_MODE_SPAN_AND_EVENT,
+ )
+
+ def _capture_in_event(self) -> bool:
+ return self._resolve_capture_mode() in (
+ CAPTURE_MODE_EVENT_ONLY,
+ CAPTURE_MODE_SPAN_AND_EVENT,
+ )
+
def _init_tracing(self, tracer_provider):
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
@@ -825,8 +898,7 @@ class OpenTelemetry(CustomLogger):
from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode
- # only log raw LLM request/response if message_logging is on and not globally turned off
- if litellm.turn_off_message_logging or not self.message_logging:
+ if not self._capture_in_span():
return
litellm_params = kwargs.get("litellm_params", {})
@@ -1117,9 +1189,14 @@ class OpenTelemetry(CustomLogger):
}
if role == "tool" and msg.get("id"):
attrs["id"] = msg["id"]
- if self.message_logging and msg.get("content"):
+ capture_event_content = self._capture_in_event()
+ if capture_event_content and msg.get("content"):
attrs["gen_ai.prompt"] = msg["content"]
+ body = msg.copy()
+ if not capture_event_content:
+ body.pop("content", None)
+
log_record = SdkLogRecord(
timestamp=self._to_ns(datetime.now()),
trace_id=parent_ctx.trace_id,
@@ -1127,7 +1204,7 @@ class OpenTelemetry(CustomLogger):
trace_flags=parent_ctx.trace_flags,
severity_number=SeverityNumber.INFO,
severity_text="INFO",
- body=msg.copy(),
+ body=body,
attributes=attrs,
)
otel_logger.emit(log_record)
@@ -1141,14 +1218,15 @@ class OpenTelemetry(CustomLogger):
"finish_reason": choice.get("finish_reason"),
}
body_msg = choice.get("message", {})
- if self.message_logging and body_msg.get("content"):
+ capture_event_content = self._capture_in_event()
+ if capture_event_content and body_msg.get("content"):
attrs["message.content"] = body_msg["content"]
body = {
"index": idx,
"finish_reason": choice.get("finish_reason"),
"message": {"role": body_msg.get("role", "assistant")},
}
- if self.message_logging and body_msg.get("content"):
+ if capture_event_content and body_msg.get("content"):
body["message"]["content"] = body_msg["content"]
log_record = SdkLogRecord(
@@ -1674,9 +1752,7 @@ class OpenTelemetry(CustomLogger):
########## LLM Request Medssages / tools / content Attributes ###########
#########################################################################
- if litellm.turn_off_message_logging is True:
- return
- if self.message_logging is not True:
+ if not self._capture_in_span():
return
if optional_params.get("tools"):
diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py
index 08ce7ed8947..332e84dd07d 100644
--- a/litellm/integrations/s3_v2.py
+++ b/litellm/integrations/s3_v2.py
@@ -16,6 +16,7 @@ from litellm._logging import print_verbose, verbose_logger
from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_SECONDS
from litellm.integrations.s3 import get_s3_object_key
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
@@ -53,15 +54,25 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
+ s3_callback_params_override: Optional[dict] = None,
**kwargs,
):
try:
- verbose_logger.debug(
- f"in init s3 logger - s3_callback_params {litellm.s3_callback_params}"
- )
+ _masker = SensitiveDataMasker()
+ if s3_callback_params_override is not None:
+ verbose_logger.debug(
+ f"in init s3 logger (audit override) - "
+ f"{_masker.mask_dict(dict(s3_callback_params_override))}"
+ )
+ else:
+ verbose_logger.debug(
+ f"in init s3 logger - s3_callback_params "
+ f"{_masker.mask_dict(dict(litellm.s3_callback_params or {}))}"
+ )
# Initialize S3 params first to get the correct s3_verify value
self._init_s3_params(
+ params_source=s3_callback_params_override,
s3_bucket_name=s3_bucket_name,
s3_region_name=s3_region_name,
s3_api_version=s3_api_version,
@@ -139,94 +150,85 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
+ params_source: Optional[dict] = None,
):
"""
- Initialize the s3 params for this logging callback
+ Initialize the s3 params for this logging callback. Reads from
+ `params_source` if given (e.g. `s3_audit_callback_params` for the
+ audit-log instance), otherwise falls back to `litellm.s3_callback_params`.
+ Resolves `os.environ/X` markers into a local dict; never mutates the source.
"""
- litellm.s3_callback_params = litellm.s3_callback_params or {}
- # read in .env variables - example os.environ/AWS_BUCKET_NAME
- for key, value in litellm.s3_callback_params.items():
- if isinstance(value, str) and value.startswith("os.environ/"):
- litellm.s3_callback_params[key] = litellm.get_secret(value)
+ if params_source is None:
+ params_source = litellm.s3_callback_params or {}
+ params: dict = {
+ key: (
+ litellm.get_secret(value)
+ if isinstance(value, str) and value.startswith("os.environ/")
+ else value
+ )
+ for key, value in params_source.items()
+ }
- self.s3_bucket_name = (
- litellm.s3_callback_params.get("s3_bucket_name") or s3_bucket_name
- )
- self.s3_region_name = (
- litellm.s3_callback_params.get("s3_region_name") or s3_region_name
- )
- self.s3_api_version = (
- litellm.s3_callback_params.get("s3_api_version") or s3_api_version
- )
+ self.s3_bucket_name = params.get("s3_bucket_name") or s3_bucket_name
+ self.s3_region_name = params.get("s3_region_name") or s3_region_name
+ self.s3_api_version = params.get("s3_api_version") or s3_api_version
self.s3_use_ssl = (
- litellm.s3_callback_params.get("s3_use_ssl", True)
- if litellm.s3_callback_params.get("s3_use_ssl") is not None
+ params.get("s3_use_ssl", True)
+ if params.get("s3_use_ssl") is not None
else s3_use_ssl
)
self.s3_verify = (
- litellm.s3_callback_params.get("s3_verify")
- if litellm.s3_callback_params.get("s3_verify") is not None
+ params.get("s3_verify")
+ if params.get("s3_verify") is not None
else s3_verify
)
- self.s3_endpoint_url = (
- litellm.s3_callback_params.get("s3_endpoint_url") or s3_endpoint_url
- )
+ self.s3_endpoint_url = params.get("s3_endpoint_url") or s3_endpoint_url
self.s3_aws_access_key_id = (
- litellm.s3_callback_params.get("s3_aws_access_key_id")
- or s3_aws_access_key_id
+ params.get("s3_aws_access_key_id") or s3_aws_access_key_id
)
self.s3_aws_secret_access_key = (
- litellm.s3_callback_params.get("s3_aws_secret_access_key")
- or s3_aws_secret_access_key
+ params.get("s3_aws_secret_access_key") or s3_aws_secret_access_key
)
self.s3_aws_session_token = (
- litellm.s3_callback_params.get("s3_aws_session_token")
- or s3_aws_session_token
+ params.get("s3_aws_session_token") or s3_aws_session_token
)
self.s3_aws_session_name = (
- litellm.s3_callback_params.get("s3_aws_session_name") or s3_aws_session_name
+ params.get("s3_aws_session_name") or s3_aws_session_name
)
self.s3_aws_profile_name = (
- litellm.s3_callback_params.get("s3_aws_profile_name") or s3_aws_profile_name
+ params.get("s3_aws_profile_name") or s3_aws_profile_name
)
- self.s3_aws_role_name = (
- litellm.s3_callback_params.get("s3_aws_role_name") or s3_aws_role_name
- )
+ self.s3_aws_role_name = params.get("s3_aws_role_name") or s3_aws_role_name
self.s3_aws_web_identity_token = (
- litellm.s3_callback_params.get("s3_aws_web_identity_token")
- or s3_aws_web_identity_token
+ params.get("s3_aws_web_identity_token") or s3_aws_web_identity_token
)
self.s3_aws_sts_endpoint = (
- litellm.s3_callback_params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
+ params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
)
- self.s3_config = litellm.s3_callback_params.get("s3_config") or s3_config
- self.s3_path = litellm.s3_callback_params.get("s3_path") or s3_path
- # done reading litellm.s3_callback_params
+ self.s3_config = params.get("s3_config") or s3_config
+ self.s3_path = params.get("s3_path") or s3_path
self.s3_use_team_prefix = (
- bool(litellm.s3_callback_params.get("s3_use_team_prefix", False))
- or s3_use_team_prefix
+ bool(params.get("s3_use_team_prefix", False)) or s3_use_team_prefix
)
self.s3_use_key_prefix = (
- bool(litellm.s3_callback_params.get("s3_use_key_prefix", False))
- or s3_use_key_prefix
+ bool(params.get("s3_use_key_prefix", False)) or s3_use_key_prefix
)
self.s3_strip_base64_files = (
- bool(litellm.s3_callback_params.get("s3_strip_base64_files", False))
- or s3_strip_base64_files
+ bool(params.get("s3_strip_base64_files", False)) or s3_strip_base64_files
)
self.s3_use_virtual_hosted_style = (
- bool(litellm.s3_callback_params.get("s3_use_virtual_hosted_style", False))
+ bool(params.get("s3_use_virtual_hosted_style", False))
or s3_use_virtual_hosted_style
)
diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py
index b234e6c8f77..52269d705d0 100644
--- a/litellm/litellm_core_utils/prompt_templates/common_utils.py
+++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py
@@ -436,12 +436,21 @@ def update_messages_with_model_file_ids(
"""
Updates messages with model file ids.
+ For managed files (unified file IDs), uses model_file_id_mapping if it
+ resolves the id, otherwise decodes the base64-encoded unified file ID
+ and extracts the llm_output_file_id directly. Mirrors the Responses-API
+ sibling `update_responses_input_with_model_file_ids`.
+
model_file_id_mapping: Dict[str, Dict[str, str]] = {
"litellm_proxy/file_id": {
"model_id": "provider_file_id"
}
}
"""
+ from litellm.proxy.openai_files_endpoints.common_utils import (
+ _is_base64_encoded_unified_file_id,
+ convert_b64_uid_to_unified_uid,
+ )
for message in messages:
if message.get("role") == "user":
@@ -450,7 +459,13 @@ def update_messages_with_model_file_ids(
if isinstance(content, str):
continue
for c in content:
- if c["type"] == "file":
+ if not isinstance(c, dict):
+ # Content list items aren't always dicts. e.g.
+ # text_completion forwards a token-ids list/list-of-
+ # lists through this path. Skip non-dict items
+ # instead of indexing into them.
+ continue
+ if c.get("type") == "file":
file_object = cast(ChatCompletionFileObject, c)
file_object_file_field = file_object.get("file")
if not isinstance(file_object_file_field, dict):
@@ -468,9 +483,23 @@ def update_messages_with_model_file_ids(
if file_id:
provider_file_id = (
model_file_id_mapping.get(file_id, {}).get(model_id)
- or file_id
+ if model_file_id_mapping
+ else None
+ )
+ if (
+ not provider_file_id
+ and _is_base64_encoded_unified_file_id(file_id)
+ ):
+ unified_file_id = convert_b64_uid_to_unified_uid(
+ file_id
+ )
+ if "llm_output_file_id," in unified_file_id:
+ provider_file_id = unified_file_id.split(
+ "llm_output_file_id,"
+ )[1].split(";")[0]
+ file_object_file_field["file_id"] = (
+ provider_file_id or file_id
)
- file_object_file_field["file_id"] = provider_file_id
if format:
file_object_file_field["format"] = format
return messages
diff --git a/litellm/main.py b/litellm/main.py
index 3e31ab04ee1..051a82fdd19 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -1459,14 +1459,14 @@ def completion( # type: ignore # noqa: PLR0915
if eos_token:
custom_prompt_dict[model]["eos_token"] = eos_token
- if kwargs.get("model_file_id_mapping"):
- messages = update_messages_with_model_file_ids(
- messages=messages,
- model_id=kwargs.get("model_info", {}).get("id", None),
- model_file_id_mapping=cast(
- Dict[str, Dict[str, str]], kwargs.get("model_file_id_mapping")
- ),
- )
+ messages = update_messages_with_model_file_ids(
+ messages=messages,
+ model_id=kwargs.get("model_info", {}).get("id", None),
+ model_file_id_mapping=cast(
+ Dict[str, Dict[str, str]],
+ kwargs.get("model_file_id_mapping") or {},
+ ),
+ )
provider_config: Optional[BaseConfig] = None
if custom_llm_provider is not None and custom_llm_provider in [
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index d609a3d1f4b..76d2d35a53a 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -27187,6 +27187,20 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
+ "openrouter/qwen/qwen3.6-plus": {
+ "input_cost_per_token": 3.25e-07,
+ "litellm_provider": "openrouter",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 1.95e-06,
+ "source": "https://openrouter.ai/qwen/qwen3.6-plus",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"openrouter/qwen/qwen3.5-35b-a3b": {
"input_cost_per_token": 2.5e-07,
"litellm_provider": "openrouter",
@@ -34940,6 +34954,48 @@
"supports_vision": true,
"supports_web_search": true
},
+ "xai/grok-4.3": {
+ "cache_read_input_token_cost": 2e-07,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "input_cost_per_token": 1.25e-06,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true
+ },
+ "xai/grok-4.3-latest": {
+ "cache_read_input_token_cost": 2e-07,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "input_cost_per_token": 1.25e-06,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true
+ },
"xai/grok-beta": {
"input_cost_per_token": 5e-06,
"litellm_provider": "xai",
diff --git a/litellm/proxy/_experimental/mcp_server/auth/token_exchange.py b/litellm/proxy/_experimental/mcp_server/auth/token_exchange.py
new file mode 100644
index 00000000000..97a16ad3e15
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/auth/token_exchange.py
@@ -0,0 +1,196 @@
+"""
+OAuth 2.0 Token Exchange (RFC 8693) handler for MCP servers.
+
+Exchanges a user's incoming JWT (subject_token) for a scoped access token
+at an IDP's token exchange endpoint. The exchanged token is then used to
+authenticate requests to the upstream MCP server.
+
+See: https://datatracker.ietf.org/doc/html/rfc8693
+"""
+
+import asyncio
+import hashlib
+import weakref
+from typing import TYPE_CHECKING, Dict, Tuple
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.constants import (
+ MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
+ MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
+ MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
+ MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
+)
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+from litellm.types.llms.custom_http import httpxSpecialProvider
+
+if TYPE_CHECKING:
+ from litellm.types.mcp_server.mcp_server_manager import MCPServer
+
+# RFC 8693 grant type constant
+TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
+
+DEFAULT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token"
+
+
+class TokenExchangeHandler:
+ """Handles OAuth 2.0 Token Exchange (RFC 8693) for MCP servers.
+
+ Caches exchanged tokens keyed by ``hash(subject_token + server_id)`` so
+ repeated calls with the same user token skip the IDP round-trip.
+ """
+
+ def __init__(self) -> None:
+ self._cache = InMemoryCache(
+ max_size_in_memory=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
+ default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
+ )
+ # WeakValueDictionary so locks are GC'd once no coroutine holds a reference,
+ # preventing unbounded growth with many rotating user tokens.
+ self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = (
+ weakref.WeakValueDictionary()
+ )
+
+ def _get_lock(self, cache_key: str) -> asyncio.Lock:
+ lock = self._locks.get(cache_key)
+ if lock is None:
+ lock = asyncio.Lock()
+ self._locks[cache_key] = lock
+ return lock
+
+ @staticmethod
+ def _cache_key(subject_token: str, server_id: str) -> str:
+ raw = f"{subject_token}:{server_id}"
+ return hashlib.sha256(raw.encode()).hexdigest()
+
+ async def exchange_token(
+ self,
+ subject_token: str,
+ server: "MCPServer",
+ ) -> str:
+ """Exchange *subject_token* for a scoped access token.
+
+ Returns the exchanged ``access_token`` string (suitable for a
+ ``Bearer`` header).
+
+ Raises ``ValueError`` on configuration or IDP errors.
+ """
+ cache_key = self._cache_key(subject_token, server.server_id)
+
+ # Fast path
+ cached = self._cache.get_cache(cache_key)
+ if cached is not None:
+ return cached
+
+ # Slow path — one exchange at a time per (user, server) pair
+ async with self._get_lock(cache_key):
+ cached = self._cache.get_cache(cache_key)
+ if cached is not None:
+ return cached
+
+ token, ttl = await self._do_exchange(subject_token, server)
+ self._cache.set_cache(cache_key, token, ttl=ttl)
+ return token
+
+ async def _do_exchange(
+ self,
+ subject_token: str,
+ server: "MCPServer",
+ ) -> Tuple[str, int]:
+ """POST to the token exchange endpoint with RFC 8693 parameters.
+
+ Returns ``(access_token, ttl_seconds)``.
+ """
+ endpoint = server.token_exchange_endpoint or server.token_url
+ if not endpoint:
+ raise ValueError(
+ f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange "
+ f"but no token_exchange_endpoint or token_url configured"
+ )
+ if not server.client_id or not server.client_secret:
+ raise ValueError(
+ f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange "
+ f"but missing client_id or client_secret"
+ )
+
+ data: Dict[str, str] = {
+ "grant_type": TOKEN_EXCHANGE_GRANT_TYPE,
+ "subject_token": subject_token,
+ "subject_token_type": server.subject_token_type
+ or DEFAULT_SUBJECT_TOKEN_TYPE,
+ "client_id": server.client_id,
+ "client_secret": server.client_secret,
+ }
+ if server.audience:
+ data["audience"] = server.audience
+ if server.scopes:
+ data["scope"] = " ".join(server.scopes)
+
+ verbose_logger.debug(
+ "Exchanging token for MCP server %s at %s (audience=%s)",
+ server.server_id,
+ endpoint,
+ server.audience,
+ )
+
+ client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
+ try:
+ response = await client.post(endpoint, data=data)
+ response.raise_for_status()
+ except httpx.HTTPStatusError as exc:
+ verbose_logger.debug(
+ "Token exchange IDP error for MCP server %s (status %d)",
+ server.server_id,
+ exc.response.status_code,
+ )
+ raise ValueError(
+ f"Token exchange for MCP server '{server.server_id}' "
+ f"failed with status {exc.response.status_code}"
+ ) from exc
+
+ body = response.json()
+ if not isinstance(body, dict):
+ raise ValueError(
+ f"Token exchange response for MCP server '{server.server_id}' "
+ f"returned non-object JSON (got {type(body).__name__})"
+ )
+
+ access_token = body.get("access_token")
+ if not access_token:
+ raise ValueError(
+ f"Token exchange response for MCP server '{server.server_id}' "
+ f"missing 'access_token'"
+ )
+
+ raw_expires_in = body.get("expires_in")
+ try:
+ expires_in = (
+ int(raw_expires_in)
+ if raw_expires_in is not None
+ else MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
+ )
+ except (TypeError, ValueError):
+ expires_in = MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
+
+ ttl = max(
+ expires_in - MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
+ MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
+ )
+
+ verbose_logger.info(
+ "Token exchange succeeded for MCP server %s (expires in %ds)",
+ server.server_id,
+ expires_in,
+ )
+ return access_token, ttl
+
+ def invalidate(self, subject_token: str, server_id: str) -> None:
+ """Remove a cached exchanged token (e.g. after a 401)."""
+ cache_key = self._cache_key(subject_token, server_id)
+ self._cache.delete_cache(cache_key)
+
+
+# Module-level singleton
+mcp_token_exchange_handler = TokenExchangeHandler()
diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
index b3eca56a182..6ad731e7113 100644
--- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
+++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
@@ -411,6 +411,15 @@ class MCPServerManager:
aws_role_name=server_config.get("aws_role_name", None),
aws_session_name=server_config.get("aws_session_name", None),
instructions=server_config.get("instructions", None),
+ # Token Exchange (OBO) fields
+ token_exchange_endpoint=server_config.get(
+ "token_exchange_endpoint", None
+ ),
+ audience=server_config.get("audience", None),
+ subject_token_type=server_config.get(
+ "subject_token_type",
+ "urn:ietf:params:oauth:token-type:access_token",
+ ),
)
self._assign_unique_short_prefix(new_server)
self.config_mcp_servers[server_id] = new_server
@@ -766,6 +775,17 @@ class MCPServerManager:
aws_role_name=aws_creds.get("aws_role_name"),
aws_session_name=aws_creds.get("aws_session_name"),
instructions=mcp_server.instructions,
+ # Token Exchange (OBO) fields — read from credentials JSON blob
+ token_exchange_endpoint=(
+ credentials_dict.get("token_exchange_endpoint")
+ if credentials_dict
+ else None
+ ),
+ audience=(credentials_dict.get("audience") if credentials_dict else None),
+ subject_token_type=(
+ credentials_dict.get("subject_token_type") if credentials_dict else None
+ )
+ or "urn:ietf:params:oauth:token-type:access_token",
)
return new_server
@@ -1140,6 +1160,29 @@ class MCPServerManager:
#########################################################
# Methods that call the upstream MCP servers
#########################################################
+ @staticmethod
+ def _extract_bearer_token(
+ oauth2_headers: Optional[Dict[str, str]],
+ raw_headers: Optional[Dict[str, str]],
+ ) -> Optional[str]:
+ """Extract the bare Bearer token from oauth2_headers or raw_headers.
+
+ Returns the token string without the ``Bearer `` prefix, or ``None``
+ if no Authorization header is found.
+ """
+ auth_value: Optional[str] = None
+ if oauth2_headers and "Authorization" in oauth2_headers:
+ auth_value = oauth2_headers["Authorization"]
+ elif raw_headers:
+ # raw_headers may have lowercase keys depending on the ASGI server
+ normalized = {k.lower(): v for k, v in raw_headers.items()}
+ auth_value = normalized.get("authorization")
+ if auth_value:
+ if auth_value.startswith("Bearer "):
+ return auth_value[len("Bearer ") :]
+ return auth_value
+ return None
+
def _build_stdio_env(
self,
server: MCPServer,
@@ -1173,25 +1216,30 @@ class MCPServerManager:
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
stdio_env: Optional[Dict[str, str]] = None,
+ subject_token: Optional[str] = None,
) -> MCPClient:
"""
Create an MCPClient instance for the given server.
Auth resolution (single place for all auth logic):
1. ``mcp_auth_header`` — per-request/per-user override
- 2. OAuth2 client_credentials token — auto-fetched and cached
- 3. ``server.authentication_token`` — static token from config/DB
+ 2. OAuth2 Token Exchange (OBO) — exchange user token for scoped token
+ 3. OAuth2 client_credentials token — auto-fetched and cached
+ 4. ``server.authentication_token`` — static token from config/DB
Args:
server: The server configuration.
mcp_auth_header: Optional per-request auth override.
extra_headers: Additional headers to forward.
stdio_env: Environment variables for stdio transport.
+ subject_token: Optional user JWT for token exchange (OBO) flow.
Returns:
Configured MCP client instance.
"""
- auth_value = await resolve_mcp_auth(server, mcp_auth_header)
+ auth_value = await resolve_mcp_auth(
+ server, mcp_auth_header, subject_token=subject_token
+ )
transport = server.transport or MCPTransport.sse
@@ -2543,9 +2591,12 @@ class MCPServerManager:
if server_auth_header is None:
server_auth_header = mcp_auth_header
- # oauth2 headers
+ # Extract subject token for OAuth2 Token Exchange (OBO) flow
+ subject_token: Optional[str] = None
extra_headers: Optional[Dict[str, str]] = None
- if mcp_server.auth_type == MCPAuth.oauth2:
+ if mcp_server.auth_type == MCPAuth.oauth2_token_exchange:
+ subject_token = self._extract_bearer_token(oauth2_headers, raw_headers)
+ elif mcp_server.auth_type == MCPAuth.oauth2:
if mcp_server.has_client_credentials:
# For M2M OAuth servers, Authorization must come from token fetch.
extra_headers = None
@@ -2613,6 +2664,7 @@ class MCPServerManager:
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
stdio_env=stdio_env,
+ subject_token=subject_token,
)
call_tool_params = MCPCallToolRequestParams(
diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py
index 476e215666e..92ef57d8cd5 100644
--- a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py
+++ b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py
@@ -26,6 +26,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
+from litellm.proxy._experimental.mcp_server.auth import token_exchange
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
@@ -50,12 +51,23 @@ class MCPOAuth2TokenCache(InMemoryCache):
def _get_lock(self, server_id: str) -> asyncio.Lock:
return self._locks.setdefault(server_id, asyncio.Lock())
- async def async_get_token(self, server: "MCPServer") -> Optional[str]:
+ @staticmethod
+ def _has_client_credentials_config(server: "MCPServer") -> bool:
+ return bool(server.client_id and server.client_secret and server.token_url)
+
+ async def async_get_token(
+ self,
+ server: "MCPServer",
+ *,
+ require_client_credentials_flow: bool = True,
+ ) -> Optional[str]:
"""Return a valid access token, fetching or refreshing as needed.
Returns ``None`` when the server lacks client credentials config.
"""
- if not server.has_client_credentials:
+ if require_client_credentials_flow and not server.has_client_credentials:
+ return None
+ if not self._has_client_credentials_config(server):
return None
server_id = server.server_id
@@ -263,16 +275,38 @@ mcp_per_user_token_cache = MCPPerUserTokenCache()
async def resolve_mcp_auth(
server: "MCPServer",
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
+ subject_token: Optional[str] = None,
) -> Optional[Union[str, Dict[str, str]]]:
"""Resolve the auth value for an MCP server.
Priority:
1. ``mcp_auth_header`` — per-request/per-user override
- 2. OAuth2 client_credentials token — auto-fetched and cached
- 3. ``server.authentication_token`` — static token from config/DB
+ 2. OAuth2 Token Exchange (OBO / RFC 8693) — exchange user token for scoped token
+ 3. OAuth2 client_credentials token — auto-fetched and cached
+ 4. ``server.authentication_token`` — static token from config/DB
"""
if mcp_auth_header:
return mcp_auth_header
+ if server.has_token_exchange_config:
+ if subject_token:
+ return await token_exchange.mcp_token_exchange_handler.exchange_token(
+ subject_token, server
+ )
+ # No subject_token — fall back to client_credentials using the same client
+ # credentials and token_url so M2M scenarios still work.
+ if server.client_id and server.client_secret and server.token_url:
+ return await mcp_oauth2_token_cache.async_get_token(
+ server,
+ require_client_credentials_flow=False,
+ )
+ # OBO configured but no subject_token and missing client credentials — warn
+ # rather than silently proceeding unauthenticated.
+ verbose_logger.warning(
+ "MCP server '%s' is configured for token exchange (OBO) but no subject_token "
+ "was provided and client credentials (client_id/client_secret/token_url) are "
+ "incomplete. The request will proceed without authentication.",
+ server.server_id,
+ )
if server.has_client_credentials:
return await mcp_oauth2_token_cache.async_get_token(server)
return server.authentication_token
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 2c976479798..d58612d5054 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -353,8 +353,10 @@ class LiteLLMRoutes(enum.Enum):
# realtime
"/realtime",
"/v1/realtime",
+ "/openai/v1/realtime",
"/realtime?{model}",
"/v1/realtime?{model}",
+ "/openai/v1/realtime?{model}",
# responses API
"/responses",
"/v1/responses",
@@ -656,6 +658,13 @@ class LiteLLMRoutes(enum.Enum):
"/health/services",
] + info_routes
+ # Stateless validators on caller-supplied log data; source logs are
+ # already accessible via spend_tracking_routes, so no scope expansion.
+ compliance_check_routes = [
+ "/compliance/eu-ai-act",
+ "/compliance/gdpr",
+ ]
+
# Routes in `global_spend_tracking_routes` return proxy-wide spend across
# every team, customer, and api_key. They are intentionally NOT included
# here — non-admin roles must not see other tenants' spend. Admin roles go
@@ -675,6 +684,7 @@ class LiteLLMRoutes(enum.Enum):
]
+ spend_tracking_routes
+ key_management_routes
+ + compliance_check_routes
)
internal_user_view_only_routes = spend_tracking_routes
@@ -699,6 +709,8 @@ class LiteLLMRoutes(enum.Enum):
# Project read routes - endpoint scopes results to caller's teams (non-admin)
"/project/list",
"/project/info",
+ # Endpoint enforces proxy-admin vs team-admin model access itself.
+ "/health/test_connection",
# Invitation routes - org/team admins checked in endpoint via _user_has_admin_privileges
"/invitation/new",
"/invitation/delete",
diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index baa08537003..038d2d81277 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -1512,7 +1512,7 @@ class ProxyBaseLLMRequestProcessing:
status_code=result.status_code,
headers=HttpPassThroughEndpointHelpers.get_response_headers(
headers=result.headers,
- custom_headers=None,
+ custom_headers=dict(fastapi_response.headers),
),
)
diff --git a/litellm/proxy/common_utils/html_forms/jwt_display_template.py b/litellm/proxy/common_utils/html_forms/jwt_display_template.py
index 03dff78dba0..ea65fd0e28f 100644
--- a/litellm/proxy/common_utils/html_forms/jwt_display_template.py
+++ b/litellm/proxy/common_utils/html_forms/jwt_display_template.py
@@ -14,7 +14,7 @@ jwt_display_template = """
padding: 20px;
display: flex;
justify-content: center;
- align-items: center;
+ align-items: flex-start;
min-height: 100vh;
color: #333;
}
@@ -27,18 +27,18 @@ jwt_display_template = """
width: 800px;
max-width: 100%;
}
-
+
.logo-container {
text-align: center;
margin-bottom: 30px;
}
-
+
.logo {
font-size: 24px;
font-weight: 600;
color: #1e293b;
}
-
+
h2 {
margin: 0 0 10px;
color: #1e293b;
@@ -46,7 +46,14 @@ jwt_display_template = """
font-weight: 600;
text-align: center;
}
-
+
+ h3 {
+ margin: 0 0 12px;
+ color: #1e293b;
+ font-size: 18px;
+ font-weight: 600;
+ }
+
.subtitle {
color: #64748b;
margin: 0 0 20px;
@@ -58,15 +65,15 @@ jwt_display_template = """
background-color: #f1f5f9;
border-radius: 6px;
padding: 20px;
- margin-bottom: 30px;
+ margin-bottom: 20px;
border-left: 4px solid #2563eb;
}
-
+
.success-box {
background-color: #f0fdf4;
border-radius: 6px;
padding: 20px;
- margin-bottom: 30px;
+ margin-bottom: 20px;
border-left: 4px solid #16a34a;
}
@@ -78,7 +85,7 @@ jwt_display_template = """
font-weight: 600;
font-size: 16px;
}
-
+
.success-header {
display: flex;
align-items: center;
@@ -87,46 +94,53 @@ jwt_display_template = """
font-weight: 600;
font-size: 16px;
}
-
+
.info-header svg, .success-header svg {
margin-right: 8px;
}
-
+
.data-container {
margin-top: 20px;
}
-
+
.data-row {
display: flex;
border-bottom: 1px solid #e2e8f0;
padding: 12px 0;
}
-
+
.data-row:last-child {
border-bottom: none;
}
-
+
.data-label {
font-weight: 500;
color: #334155;
- width: 180px;
+ width: 220px;
flex-shrink: 0;
}
-
+
.data-value {
color: #475569;
word-break: break-all;
}
-
+
+ .empty-note {
+ color: #64748b;
+ font-style: italic;
+ margin: 0;
+ font-size: 14px;
+ }
+
.jwt-container {
background-color: #f8fafc;
border-radius: 6px;
padding: 15px;
- margin-top: 20px;
+ margin-top: 12px;
overflow-x: auto;
border: 1px solid #e2e8f0;
}
-
+
.jwt-text {
font-family: monospace;
white-space: pre-wrap;
@@ -134,7 +148,7 @@ jwt_display_template = """
margin: 0;
color: #334155;
}
-
+
.back-button {
display: inline-block;
background-color: #6466E9;
@@ -146,18 +160,18 @@ jwt_display_template = """
margin-top: 20px;
text-align: center;
}
-
+
.back-button:hover {
background-color: #4138C2;
text-decoration: none;
}
-
+
.buttons {
display: flex;
gap: 10px;
- margin-top: 20px;
+ margin-top: 12px;
}
-
+
.copy-button {
background-color: #e2e8f0;
color: #334155;
@@ -169,11 +183,11 @@ jwt_display_template = """
display: flex;
align-items: center;
}
-
+
.copy-button:hover {
background-color: #cbd5e1;
}
-
+
.copy-button svg {
margin-right: 6px;
}
@@ -188,7 +202,7 @@ jwt_display_template = """
SSO Debug Information
Results from the SSO authentication process.
-
+
The SSO authentication completed successfully. Below is the information returned by the provider.
-
-
-
-
-
+
+
Fields the proxy extracted into its internal user model.
+
+
+
+
+
+
Complete set of claims returned by the IdP's userinfo endpoint.
-
Loading...
+
Loading...
+
+
Decoded payload of the access token JWT (when the IdP issues one).
+
+
+