scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_lit5046_autorouter_savings_fixes

Browse source 
# Conflicts:
#	tests/test_litellm/router_strategy/test_complexity_router.py
This commit is contained in:
Tin Chi Lo 2026-08-01 14:31:20 -07:00
commit 10a2a29c66
856 changed files with 15564 additions and 6955 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.github/workflows/test-code-quality.yml vendored
View file

@ -7,13 +7,17 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
code-quality:

8
.github/workflows/test-unit-core-utils.yml vendored
View file

@ -7,6 +7,10 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
@ -14,8 +18,8 @@ permissions:
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
core-utils:

8
.github/workflows/test-unit-documentation.yml vendored
View file

@ -7,13 +7,17 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
documentation:

8
.github/workflows/test-unit-enterprise-routing.yml vendored
View file

@ -7,6 +7,10 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
@ -14,8 +18,8 @@ permissions:
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
enterprise-routing:

8
.github/workflows/test-unit-integrations.yml vendored
View file

@ -7,6 +7,10 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
@ -14,8 +18,8 @@ permissions:
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
integrations:

8
.github/workflows/test-unit-llm-providers.yml vendored
View file

@ -7,13 +7,17 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
vertex-ai:

8
.github/workflows/test-unit-misc.yml vendored
View file

@ -7,6 +7,10 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
@ -14,8 +18,8 @@ permissions:
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
misc:

8
.github/workflows/test-unit-proxy-auth.yml vendored
View file

@ -7,6 +7,10 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
@ -14,8 +18,8 @@ permissions:
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
proxy-auth:

8
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -7,13 +7,17 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
# rather than alphabetical letter ranges. Adding a new test file means adding it

8
.github/workflows/test-unit-proxy-endpoints.yml vendored
View file

@ -7,14 +7,18 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
proxy-endpoints:

8
.github/workflows/test-unit-proxy-infra.yml vendored
View file

@ -7,6 +7,10 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
@ -14,8 +18,8 @@ permissions:
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
proxy-infra:

8
.github/workflows/test-unit-proxy-legacy.yml vendored
View file

@ -7,13 +7,17 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
test:

8
.github/workflows/test-unit-responses-caching-types.yml vendored
View file

@ -7,6 +7,10 @@ on:
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
@ -14,8 +18,8 @@ permissions:
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
responses-caching-types:

20
basedpyright-code-budget.json
View file

@ -1,6 +1,6 @@
{
"reportAny": {
"limit": 31903
"limit": 29813
},
"reportArgumentType": {
"limit": 2645
@ -24,7 +24,7 @@
"limit": 42
},
"reportExplicitAny": {
"limit": 10214
"limit": 9473
},
"reportFunctionMemberAccess": {
"limit": 11
@ -33,7 +33,7 @@
"limit": 227
},
"reportIncompatibleMethodOverride": {
"limit": 78
"limit": 77
},
"reportIncompatibleVariableOverride": {
"limit": 12
@ -54,10 +54,10 @@
"limit": 0
},
"reportMissingParameterType": {
"limit": 5869
"limit": 5855
},
"reportMissingTypeArgument": {
"limit": 15861
"limit": 15852
},
"reportMissingTypeStubs": {
"limit": 41
@ -99,19 +99,19 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 45366
"limit": 45324
},
"reportUnknownLambdaType": {
"limit": 113
},
"reportUnknownMemberType": {
"limit": 40477
"limit": 40452
},
"reportUnknownParameterType": {
"limit": 20338
"limit": 20309
},
"reportUnknownVariableType": {
"limit": 32047
"limit": 31978
},
"reportUnnecessaryCast": {
"limit": 177
@ -123,7 +123,7 @@
"limit": 7
},
"reportUnnecessaryIsInstance": {
"limit": 1205
"limit": 1204
},
"reportUntypedBaseClass": {
"limit": 165

8
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -215,7 +215,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
if result:
return LiteLLM_ManagedFileTable(**result)
return LiteLLM_ManagedFileTable.model_validate(result)
## CHECK DB
db_object = await self.prisma_client.db.litellm_managedfiletable.find_first(
@ -223,7 +223,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
if db_object:
return LiteLLM_ManagedFileTable(**db_object.model_dump())
return LiteLLM_ManagedFileTable.model_validate(db_object.model_dump())
return None
async def delete_unified_file_id(
@ -349,7 +349,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if isinstance(batch.file_object, str)
else batch.file_object
)
batch_obj = LiteLLMBatch(**batch_data)
batch_obj = LiteLLMBatch.model_validate(batch_data)
batch_obj.id = batch.unified_object_id
batch_objects.append(batch_obj)
@ -382,7 +382,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"flat_model_file_ids": {"hasSome": model_object_ids},
}
)
return [OpenAIFileObject(**file_object.file_object) for file_object in file_ids]
return [OpenAIFileObject.model_validate(file_object.file_object) for file_object in file_ids]
async def check_managed_file_id_access(
self, data: Dict, user_api_key_dict: UserAPIKeyAuth

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.52"
version = "0.1.53"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.52"
version = "0.1.53"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

2
helm/litellm-helm/templates/migrations-job.yaml
View file

@ -35,6 +35,8 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- with .Values.migrationJob.extraInitContainers }}
initContainers:
{{- tpl (toYaml .) $ | nindent 8 }}

36
helm/litellm-helm/tests/migrations-job_tests.yaml
View file

@ -254,3 +254,39 @@ tests:
content:
name: sidecar-tpl
image: "ghcr.io/berriai/litellm-database:test"
- it: should render the pod-level securityContext from podSecurityContext
template: migrations-job.yaml
set:
migrationJob:
enabled: true
podSecurityContext:
fsGroup: 10000
runAsUser: 10000
runAsNonRoot: true
asserts:
- equal:
path: spec.template.spec.securityContext
value:
fsGroup: 10000
runAsUser: 10000
runAsNonRoot: true
- it: should keep the pod-level and container-level securityContext separate
template: migrations-job.yaml
set:
migrationJob:
enabled: true
podSecurityContext:
fsGroup: 10000
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
asserts:
- equal:
path: spec.template.spec.securityContext
value:
fsGroup: 10000
- equal:
path: spec.template.spec.containers[0].securityContext
value:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true

53
helm/litellm/templates/_helpers.tpl
View file

@ -138,6 +138,59 @@ is false the chart uses the provided name, or the namespace `default` SA.
{{- end -}}
{{- end -}}
{{/*
ServiceAccount name for the migrations Job.
The Job is a pre-install / pre-upgrade hook, so it is created before the
chart's ordinary resources. A ServiceAccount the chart creates is one of
those ordinary resources, which makes borrowing the backend name a cycle:
the hook pod is rejected because the account does not exist yet. So when
`serviceAccounts.backend.create` is true the Job falls back to the namespace
`default` account unless the operator names one that already exists. With
`create` false the backend name is either an operator-supplied existing
account or `default`, both of which are safe for the hook, so the Job keeps
sharing it.
`migrationJob.serviceAccountName` always wins when set, which is how a Job
that needs credentials of its own (IRSA / Workload Identity for IAM database
auth) gets them.
*/}}
{{- define "litellm.migrations.serviceAccountName" -}}
{{- if .Values.migrationJob.serviceAccountName -}}
{{ .Values.migrationJob.serviceAccountName }}
{{- else if .Values.serviceAccounts.backend.create -}}
default
{{- else -}}
{{ include "litellm.backend.serviceAccountName" . }}
{{- end -}}
{{- end -}}
{{/*
Extra pod labels for a component's Deployment, validated against its selector.
Invoke with a dict:
(dict "podLabels" .Values.gateway.podLabels "componentName" "gateway")
The three selector keys are also emitted on the pod template, so a podLabels
entry reusing one renders a duplicate YAML key whose later value wins. That
leaves the pod template no longer matching the (immutable) selector and the
apiserver rejects the Deployment. Fail at template time naming the key
instead, so the operator gets the reason here rather than an opaque
`selector does not match template labels` from the apiserver.
The migrations Job takes podLabels unvalidated: a Job's selector is generated
by the controller rather than declared, so nothing there can collide.
*/}}
{{- define "litellm.podLabels" -}}
{{- $componentName := .componentName -}}
{{- range $key, $value := .podLabels }}
{{- if has $key (list "app.kubernetes.io/name" "app.kubernetes.io/instance" "app.kubernetes.io/component") }}
{{- fail (printf "%s.podLabels cannot set %s: it is part of the Deployment's immutable selector" $componentName $key) }}
{{- end }}
{{- end }}
{{- toYaml .podLabels }}
{{- end -}}
{{/*
Master-key + database + redis env block — shared by gateway, backend, and the
migrations Job.

22
helm/litellm/templates/backend/deployment.yaml
View file

@ -23,9 +23,16 @@ spec:
{{- end }}
labels:
{{- include "litellm.backend.selectorLabels" . | nindent 8 }}
{{- with .Values.backend.podLabels }}
{{- include "litellm.podLabels" (dict "podLabels" . "componentName" "backend") | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }}
automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }}
{{- with .Values.backend.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
@ -34,6 +41,10 @@ spec:
- name: backend
image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.backend.image.pullPolicy }}
{{- with .Values.backend.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: http
containerPort: 4001
@ -70,8 +81,15 @@ spec:
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.backend.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.backend.resources | nindent 12 }}
{{- with .Values.backend.extraContainers }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- if or .Values.gateway.config.create .Values.backend.volumes .Values.billingMetrics.enabled }}
volumes:
{{- if .Values.gateway.config.create }}
@ -102,4 +120,8 @@ spec:
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- $gracePeriod := .Values.backend.terminationGracePeriodSeconds }}
{{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
terminationGracePeriodSeconds: {{ $gracePeriod }}
{{- end }}
{{- end }}

22
helm/litellm/templates/gateway/deployment.yaml
View file

@ -21,9 +21,16 @@ spec:
{{- end }}
labels:
{{- include "litellm.gateway.selectorLabels" . | nindent 8 }}
{{- with .Values.gateway.podLabels }}
{{- include "litellm.podLabels" (dict "podLabels" . "componentName" "gateway") | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }}
automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }}
{{- with .Values.gateway.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
@ -32,6 +39,10 @@ spec:
- name: gateway
image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
{{- with .Values.gateway.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: http
containerPort: 4000
@ -72,8 +83,15 @@ spec:
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.gateway.resources | nindent 12 }}
{{- with .Values.gateway.extraContainers }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled }}
volumes:
{{- if .Values.gateway.config.create }}
@ -104,4 +122,8 @@ spec:
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }}
{{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
terminationGracePeriodSeconds: {{ $gracePeriod }}
{{- end }}
{{- end }}

27
helm/litellm/templates/migrations-job.yaml
View file

@ -23,12 +23,21 @@ spec:
ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
template:
metadata:
{{- /* The Job's selector is generated by the controller rather than
declared, so podLabels may override a chart label here. Merge
instead of appending so an override replaces the key rather than
rendering it twice. */}}
{{- $chartLabels := merge (dict "app.kubernetes.io/component" "migrations") (fromYaml (include "litellm.commonLabels" .)) }}
labels:
{{- include "litellm.commonLabels" . | nindent 8 }}
app.kubernetes.io/component: migrations
{{- toYaml (merge (deepCopy .Values.migrationJob.podLabels) $chartLabels) | nindent 8 }}
spec:
restartPolicy: Never
serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }}
serviceAccountName: {{ include "litellm.migrations.serviceAccountName" . }}
automountServiceAccountToken: {{ .Values.migrationJob.automountServiceAccountToken }}
{{- with .Values.migrationJob.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
@ -37,10 +46,22 @@ spec:
- name: prisma-migrations
image: "{{ .Values.migrationJob.image.repository }}:{{ .Values.migrationJob.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.migrationJob.image.pullPolicy }}
{{- with .Values.migrationJob.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.migrationJob) | nindent 12 }}
{{- with .Values.migrationJob.volumeMounts }}
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.volumes }}
volumes:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

22
helm/litellm/templates/ui/deployment.yaml
View file

@ -18,9 +18,16 @@ spec:
{{- end }}
labels:
{{- include "litellm.ui.selectorLabels" . | nindent 8 }}
{{- with .Values.ui.podLabels }}
{{- include "litellm.podLabels" (dict "podLabels" . "componentName" "ui") | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "litellm.ui.serviceAccountName" . }}
automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }}
{{- with .Values.ui.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
@ -29,6 +36,10 @@ spec:
- name: ui
image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.ui.image.pullPolicy }}
{{- with .Values.ui.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: http
containerPort: 3000
@ -58,8 +69,15 @@ spec:
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.ui.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.ui.resources | nindent 12 }}
{{- with .Values.ui.extraContainers }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- with .Values.ui.volumes }}
volumes:
{{- toYaml . | nindent 8 }}
@ -80,4 +98,8 @@ spec:
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- $gracePeriod := .Values.ui.terminationGracePeriodSeconds }}
{{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
terminationGracePeriodSeconds: {{ $gracePeriod }}
{{- end }}
{{- end }}

169
helm/litellm/tests/migration_job_tests.yaml Normal file
View file

@ -0,0 +1,169 @@
suite: test migrations Job ServiceAccount resolution and pod hardening
templates:
- migrations-job.yaml
values:
- ./values/required.yaml
tests:
- it: borrows the namespace default account when no ServiceAccount is configured
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: default
- it: falls back to the namespace default account when the chart creates the backend ServiceAccount
set:
serviceAccounts.backend.create: true
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: default
- notEqual:
path: spec.template.spec.serviceAccountName
value: RELEASE-NAME-litellm-backend
- it: keeps sharing an existing backend ServiceAccount the chart does not create
set:
serviceAccounts.backend.create: false
serviceAccounts.backend.name: existing-backend-sa
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: existing-backend-sa
- it: prefers an explicit migration ServiceAccount over the created backend one
set:
serviceAccounts.backend.create: true
migrationJob.serviceAccountName: migrations-sa
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: migrations-sa
- it: prefers an explicit migration ServiceAccount over an existing backend one
set:
serviceAccounts.backend.create: false
serviceAccounts.backend.name: existing-backend-sa
migrationJob.serviceAccountName: migrations-sa
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: migrations-sa
- it: mounts no ServiceAccount token by default
asserts:
- equal:
path: spec.template.spec.automountServiceAccountToken
value: false
- it: mounts a ServiceAccount token when the operator asks for one
set:
migrationJob.automountServiceAccountToken: true
asserts:
- equal:
path: spec.template.spec.automountServiceAccountToken
value: true
- it: keeps the token off the Job when the backend disables automounting
set:
serviceAccounts.backend.create: true
serviceAccounts.backend.automount: false
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: default
- equal:
path: spec.template.spec.automountServiceAccountToken
value: false
- it: renders no hardening fields by default
asserts:
- isNull:
path: spec.template.spec.securityContext
- isNull:
path: spec.template.spec.containers[0].securityContext
- isNull:
path: spec.template.spec.volumes
- isNull:
path: spec.template.spec.containers[0].volumeMounts
- equal:
path: spec.template.metadata.labels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/managed-by: Helm
helm.sh/chart: litellm-0.1.0
app.kubernetes.io/component: migrations
- it: renders pod-level and container-level securityContext in their own scopes
set:
migrationJob.podSecurityContext:
runAsNonRoot: true
runAsUser: 65532
seccompProfile:
type: RuntimeDefault
migrationJob.securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
asserts:
- equal:
path: spec.template.spec.securityContext
value:
runAsNonRoot: true
runAsUser: 65532
seccompProfile:
type: RuntimeDefault
- equal:
path: spec.template.spec.containers[0].securityContext
value:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
- it: renders volumes on the pod and volumeMounts on the migration container
set:
migrationJob.volumes:
- name: tmp
emptyDir:
sizeLimit: 64Mi
migrationJob.volumeMounts:
- name: tmp
mountPath: /tmp
asserts:
- equal:
path: spec.template.spec.volumes
value:
- name: tmp
emptyDir:
sizeLimit: 64Mi
- equal:
path: spec.template.spec.containers[0].volumeMounts
value:
- name: tmp
mountPath: /tmp
- it: merges podLabels with the chart labels on the Job pod
set:
migrationJob.podLabels:
egress-policy: restricted
asserts:
- equal:
path: spec.template.metadata.labels['egress-policy']
value: restricted
- equal:
path: spec.template.metadata.labels['app.kubernetes.io/component']
value: migrations
- it: accepts a podLabel that reuses a chart label, since the Job selector is controller-generated
set:
migrationJob.podLabels:
app.kubernetes.io/component: batch-migrations
asserts:
- notFailedTemplate: {}
- equal:
path: spec.template.metadata.labels['app.kubernetes.io/component']
value: batch-migrations

298
helm/litellm/tests/pod_hardening_tests.yaml Normal file
View file

@ -0,0 +1,298 @@
suite: test pod hardening knobs on the component deployments
templates:
- gateway/deployment.yaml
- gateway/configmap.yaml
- backend/deployment.yaml
- ui/deployment.yaml
values:
- ./values/required.yaml
tests:
- it: gateway renders no hardening fields by default
template: gateway/deployment.yaml
asserts:
- isNull:
path: spec.template.spec.securityContext
- isNull:
path: spec.template.spec.containers[0].securityContext
- isNull:
path: spec.template.spec.containers[0].lifecycle
- isNull:
path: spec.template.spec.terminationGracePeriodSeconds
- lengthEqual:
path: spec.template.spec.containers
count: 1
- equal:
path: spec.template.metadata.labels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: gateway
- it: gateway renders pod-level and container-level securityContext in their own scopes
template: gateway/deployment.yaml
set:
gateway.podSecurityContext:
runAsNonRoot: true
runAsUser: 65532
fsGroup: 65532
seccompProfile:
type: RuntimeDefault
gateway.securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
asserts:
- equal:
path: spec.template.spec.securityContext
value:
runAsNonRoot: true
runAsUser: 65532
fsGroup: 65532
seccompProfile:
type: RuntimeDefault
- equal:
path: spec.template.spec.containers[0].securityContext
value:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
- it: gateway merges podLabels with the selector labels
template: gateway/deployment.yaml
set:
gateway.podLabels:
egress-policy: restricted
team: platform
asserts:
- equal:
path: spec.template.metadata.labels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: gateway
egress-policy: restricted
team: platform
- equal:
path: spec.selector.matchLabels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: gateway
- it: gateway rejects a podLabel that collides with the selector
template: gateway/deployment.yaml
set:
gateway.podLabels:
app.kubernetes.io/component: not-gateway
asserts:
- failedTemplate:
errorMessage: "gateway.podLabels cannot set app.kubernetes.io/component: it is part of the Deployment's immutable selector"
- it: backend rejects a podLabel that collides with the selector
template: backend/deployment.yaml
set:
backend.podLabels:
app.kubernetes.io/name: not-litellm
asserts:
- failedTemplate:
errorMessage: "backend.podLabels cannot set app.kubernetes.io/name: it is part of the Deployment's immutable selector"
- it: ui rejects a podLabel that collides with the selector
template: ui/deployment.yaml
set:
ui.podLabels:
app.kubernetes.io/instance: not-the-release
asserts:
- failedTemplate:
errorMessage: "ui.podLabels cannot set app.kubernetes.io/instance: it is part of the Deployment's immutable selector"
- it: gateway renders lifecycle hooks on the container
template: gateway/deployment.yaml
set:
gateway.lifecycle:
preStop:
httpGet:
path: /health/drain
port: 4000
asserts:
- equal:
path: spec.template.spec.containers[0].lifecycle
value:
preStop:
httpGet:
path: /health/drain
port: 4000
- it: gateway renders terminationGracePeriodSeconds on the pod spec
template: gateway/deployment.yaml
set:
gateway.terminationGracePeriodSeconds: 90
asserts:
- equal:
path: spec.template.spec.terminationGracePeriodSeconds
value: 90
- it: gateway honors an explicit terminationGracePeriodSeconds of zero
template: gateway/deployment.yaml
set:
gateway.terminationGracePeriodSeconds: 0
asserts:
- equal:
path: spec.template.spec.terminationGracePeriodSeconds
value: 0
- it: gateway appends extraContainers after the gateway container
template: gateway/deployment.yaml
set:
gateway.extraContainers:
- name: auth-sidecar
image: registry.example.com/auth-proxy:1.2.3
args:
- --upstream
- http://127.0.0.1:4000
asserts:
- lengthEqual:
path: spec.template.spec.containers
count: 2
- equal:
path: spec.template.spec.containers[0].name
value: gateway
- equal:
path: spec.template.spec.containers[1]
value:
name: auth-sidecar
image: registry.example.com/auth-proxy:1.2.3
args:
- --upstream
- http://127.0.0.1:4000
- it: gateway templates chart context inside extraContainers
template: gateway/deployment.yaml
set:
gateway.extraContainers:
- name: auth-sidecar
image: registry.example.com/auth-proxy:1.2.3
env:
- name: RELEASE
value: "{{ .Release.Name }}"
asserts:
- equal:
path: spec.template.spec.containers[1].env[0].value
value: RELEASE-NAME
- it: backend renders every hardening knob in the right scope
template: backend/deployment.yaml
set:
backend.podLabels:
egress-policy: restricted
backend.podSecurityContext:
runAsNonRoot: true
backend.securityContext:
readOnlyRootFilesystem: true
backend.lifecycle:
preStop:
exec:
command:
- sleep
- "5"
backend.terminationGracePeriodSeconds: 60
backend.extraContainers:
- name: auth-sidecar
image: registry.example.com/auth-proxy:1.2.3
asserts:
- equal:
path: spec.template.metadata.labels['egress-policy']
value: restricted
- equal:
path: spec.template.spec.securityContext
value:
runAsNonRoot: true
- equal:
path: spec.template.spec.containers[0].securityContext
value:
readOnlyRootFilesystem: true
- equal:
path: spec.template.spec.containers[0].lifecycle
value:
preStop:
exec:
command:
- sleep
- "5"
- equal:
path: spec.template.spec.terminationGracePeriodSeconds
value: 60
- equal:
path: spec.template.spec.containers[1].name
value: auth-sidecar
- it: ui renders every hardening knob in the right scope
template: ui/deployment.yaml
set:
ui.podLabels:
egress-policy: restricted
ui.podSecurityContext:
runAsNonRoot: true
fsGroup: 101
ui.securityContext:
readOnlyRootFilesystem: true
ui.lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- nginx -s quit
ui.terminationGracePeriodSeconds: 30
ui.extraContainers:
- name: auth-sidecar
image: registry.example.com/auth-proxy:1.2.3
asserts:
- equal:
path: spec.template.metadata.labels['egress-policy']
value: restricted
- equal:
path: spec.template.spec.securityContext
value:
runAsNonRoot: true
fsGroup: 101
- equal:
path: spec.template.spec.containers[0].securityContext
value:
readOnlyRootFilesystem: true
- equal:
path: spec.template.spec.containers[0].lifecycle
value:
preStop:
exec:
command:
- /bin/sh
- -c
- nginx -s quit
- equal:
path: spec.template.spec.terminationGracePeriodSeconds
value: 30
- equal:
path: spec.template.spec.containers[1].name
value: auth-sidecar
- it: backend and ui render no hardening fields by default
templates:
- backend/deployment.yaml
- ui/deployment.yaml
asserts:
- isNull:
path: spec.template.spec.securityContext
- isNull:
path: spec.template.spec.containers[0].securityContext
- isNull:
path: spec.template.spec.containers[0].lifecycle
- isNull:
path: spec.template.spec.terminationGracePeriodSeconds
- lengthEqual:
path: spec.template.spec.containers
count: 1

84
helm/litellm/values.yaml
View file

@ -57,6 +57,42 @@ migrationJob:
backoffLimit: 4
ttlSecondsAfterFinished: 120
resources: {}
# ServiceAccount for the Job pod only.
#
# The Job is a pre-install / pre-upgrade hook, so it runs before the chart's
# ordinary resources exist. With `serviceAccounts.backend.create: true` the
# backend ServiceAccount is one of those ordinary resources, so a Job that
# borrowed its name would reference an account that does not exist yet and
# the first install would fail with a forbidden pod creation. The name set
# here always wins; when it is empty the Job falls back to `default` if the
# chart creates the backend ServiceAccount, and to the backend
# ServiceAccount name otherwise (that name is either an existing account you
# supplied or `default`).
#
# Point this at a pre-existing ServiceAccount when the Job needs credentials
# of its own, e.g. the IRSA / Workload Identity annotations that
# `database.writer.useIAMAuth` relies on. That is also the upgrade path to
# watch: a release already running with `serviceAccounts.backend.create:
# true` used to hand the Job the created backend account on every upgrade,
# and now hands it `default` unless you name an account here.
serviceAccountName: ""
# The Job runs `prisma migrate deploy` against Postgres and never calls the
# K8s API, so it defaults to no projected ServiceAccount token, the same
# reasoning the ui SA above uses. Flip to true if your Job genuinely needs
# one; IAM database auth does not, since EKS Pod Identity injects its own
# projected token volume and GKE Workload Identity goes through the
# metadata server, neither of which is the default token mount.
automountServiceAccountToken: false
# Standard k8s pod-level and container-level securityContext for the Job
# pod. Same shape as gateway.podSecurityContext / gateway.securityContext.
podSecurityContext: {}
securityContext: {}
# Extra pod labels on the Job pod, merged into the chart's common labels.
podLabels: {}
# Additional volumes on the Job pod and volumeMounts on its container, e.g.
# the writable scratch space a read-only root filesystem needs.
volumes: []
volumeMounts: []
image:
repository: ghcr.io/berriai/litellm-migrations
tag: "" # defaults to .Chart.AppVersion
@ -200,6 +236,37 @@ gateway:
minAvailable: ""
maxUnavailable: ""
podAnnotations: {}
# Extra pod labels, merged into the chart's selector labels. Do not
# re-declare `app.kubernetes.io/name` / `instance` / `component` here: they
# form the Deployment's immutable selector.
podLabels: {}
# Pod-level securityContext, applied to every container in the pod
# (runAsNonRoot, runAsUser, fsGroup, seccompProfile, ...). Empty by default
# so the cluster's own defaults keep applying to existing installs; clusters
# enforcing a restricted Pod Security Standard usually want at least
# `runAsNonRoot: true` and `seccompProfile.type: RuntimeDefault`.
podSecurityContext: {}
# Container-level securityContext for the gateway container. Empty by
# default for the same reason. Example:
# allowPrivilegeEscalation: false
# readOnlyRootFilesystem: true
# capabilities:
# drop:
# - ALL
# `readOnlyRootFilesystem: true` needs writable scratch space; supply it
# through `volumes` / `volumeMounts` above rather than expecting the chart
# to guess the paths your workload writes to.
securityContext: {}
# Extra sidecar containers appended to the gateway pod, e.g. an auth or
# egress proxy. Rendered through `tpl`, so entries may reference chart
# values and release metadata.
extraContainers: []
# Container lifecycle hooks (postStart / preStop) for the gateway container.
lifecycle: {}
# Grace period the kubelet allows between SIGTERM and SIGKILL. Leave empty
# to inherit the Kubernetes default of 30s. Set it a few seconds above the
# proxy's GRACEFUL_SHUTDOWN_TIMEOUT when you use a draining preStop hook.
terminationGracePeriodSeconds: ""
nodeSelector: {}
tolerations: []
affinity: {}
@ -257,6 +324,13 @@ backend:
minAvailable: ""
maxUnavailable: ""
podAnnotations: {}
# Same shape as the gateway blocks of the same name.
podLabels: {}
podSecurityContext: {}
securityContext: {}
extraContainers: []
lifecycle: {}
terminationGracePeriodSeconds: ""
nodeSelector: {}
tolerations: []
affinity: {}
@ -310,6 +384,16 @@ ui:
minAvailable: ""
maxUnavailable: ""
podAnnotations: {}
# Same shape as the gateway blocks of the same name. The nginx runtime
# writes its pid, cache, and proxy temp files under the image's root
# filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs
# emptyDir volumes mounted over those paths.
podLabels: {}
podSecurityContext: {}
securityContext: {}
extraContainers: []
lifecycle: {}
terminationGracePeriodSeconds: ""
nodeSelector: {}
tolerations: []
affinity: {}

2
litellm/__init__.py
View file

@ -449,6 +449,8 @@ enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
custom_prometheus_metadata_labels: List[str] = []
custom_prometheus_tags: List[str] = []
prometheus_metrics_config: Optional[List] = None
prometheus_exclude_metrics: Optional[List[str]] = None
prometheus_exclude_labels: Optional[List[str]] = None
prometheus_emit_stream_label: bool = False
# Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on
# `litellm_proxy_failed_requests_metric`. Off by default to preserve the

10
litellm/constants.py
View file

@ -357,6 +357,15 @@ NON_LLM_CONNECTION_TIMEOUT = int(
MAX_EXCEPTION_MESSAGE_LENGTH = int(os.getenv("MAX_EXCEPTION_MESSAGE_LENGTH", 2000))
MAX_STRING_LENGTH_PROMPT_IN_DB = int(os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048))
BEDROCK_MAX_POLICY_SIZE = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75))
# One entry per distinct AWS credential-argument set. Per-user cost attribution passes the attributed
# identity as aws_session_name, so this bounds how many attributed identities keep a cached STS session.
BEDROCK_IAM_CACHE_MAX_ENTRIES = 1000
# Single-flight lock stripes over that cache. Only keys landing on the same stripe wait for each
# other, so a burst of distinct identities still resolves its credentials in parallel.
BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES = 64
# Retire a cached STS credential this many seconds before AWS expires it, so a request that reads it
# still has a usable credential for the whole call.
STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS = 60
BEDROCK_MIN_THINKING_BUDGET_TOKENS = int(os.getenv("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024))
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS = 1024
@ -1461,6 +1470,7 @@ SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS = float(
TOOL_SPEND_TOP_TOOLS = 100
SPEND_LOG_PARTITION_INTERVAL = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
SPEND_LOG_PARTITION_PRECREATE_AHEAD = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
SPEND_LOG_WRITE_BATCH_MAX_BYTES = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)))
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
SPEND_LOG_QUEUE_POLL_INTERVAL = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000))

113
litellm/integrations/prometheus.py
View file

@ -64,11 +64,48 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from prometheus_client.metrics import MetricWrapperBase
else:
AsyncIOScheduler = Any
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT = 5.0
_NON_ENUM_METRIC_LABELS: frozenset[str] = frozenset(
(
"guardrail_name",
"status",
"error_type",
"hook_type",
"purpose",
"file_type",
"result",
)
)
class _ExcludedLabelMetric:
"""Proxies a prometheus metric whose declared ``labelnames`` had globally
excluded labels removed, dropping those labels from every ``labels(...)``
call so the emitted arguments always match the metric's real label set."""
def __init__(
self,
metric: MetricWrapperBase,
original_labelnames: tuple[str, ...],
excluded_labels: frozenset[str],
) -> None:
self._metric = metric
self._original_labelnames = original_labelnames
self._excluded_labels = excluded_labels
def labels(self, *labelvalues: str, **labelkwargs: str) -> MetricWrapperBase:
values = labelvalues or tuple(labelkwargs[name] for name in self._original_labelnames)
kept_values = tuple(
value for name, value in zip(self._original_labelnames, values) if name not in self._excluded_labels
)
return self._metric.labels(*kept_values) if kept_values else self._metric
# Tiers a caller may name in a request, across the providers that accept the
# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and
# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which
@ -122,6 +159,8 @@ class PrometheusLogger(CustomLogger):
# Always initialize label_filters, even for non-premium users
self.label_filters = self._parse_prometheus_config()
self.exclude_metrics, self.exclude_labels = self._parse_exclude_config()
# Cache resolved label sets per metric. Several entries in
# ``PrometheusMetricLabels.get_labels`` read module-level toggles
# (e.g. ``litellm.prometheus_emit_stream_label``,
@ -696,6 +735,44 @@ class PrometheusLogger(CustomLogger):
self._pretty_print_prometheus_config(label_filters)
return label_filters
def _parse_exclude_config(self) -> tuple[frozenset[str], frozenset[str]]:
"""Parse and validate the global ``exclude_metrics`` / ``exclude_labels`` settings."""
from typing import get_args
import litellm
exclude_metrics = frozenset(litellm.prometheus_exclude_metrics or ())
exclude_labels = frozenset(litellm.prometheus_exclude_labels or ())
valid_metrics = frozenset(get_args(DEFINED_PROMETHEUS_METRICS))
invalid_metrics = sorted(exclude_metrics - valid_metrics)
valid_labels = self._all_defined_labels()
invalid_labels = sorted(exclude_labels - valid_labels)
errors = (
*(f"Invalid metric name in prometheus_exclude_metrics: {metric}" for metric in invalid_metrics),
*(f"Invalid label name in prometheus_exclude_labels: {label}" for label in invalid_labels),
)
if errors:
raise ValueError("Prometheus exclude configuration validation failed:\n" + "\n".join(errors))
return exclude_metrics, exclude_labels
@staticmethod
def _all_defined_labels() -> frozenset[str]:
"""Every label a metric can emit: enum labels, hard-coded labels, and configured custom labels / tags."""
import litellm
builtin_labels = frozenset(label.value for label in UserAPIKeyLabelNames)
custom_metadata_labels = frozenset(
_sanitize_prometheus_label_name(label) for label in litellm.custom_prometheus_metadata_labels
)
custom_tag_labels = frozenset(
_sanitize_prometheus_label_name(f"tag_{tag}") for tag in litellm.custom_prometheus_tags
)
return builtin_labels | _NON_ENUM_METRIC_LABELS | custom_metadata_labels | custom_tag_labels
def _validate_all_configurations(self, parsed_configs: List) -> ValidationResults:
"""Validate all metric configurations and return collected errors"""
metric_errors = []
@ -1015,6 +1092,9 @@ class PrometheusLogger(CustomLogger):
def _is_metric_enabled(self, metric_name: str) -> bool:
"""Check if a metric is enabled based on configuration"""
if metric_name in self.exclude_metrics:
return False
# If no specific configuration is provided, enable all metrics (default behavior)
if not hasattr(self, "enabled_metrics"):
return True
@ -1032,11 +1112,18 @@ class PrometheusLogger(CustomLogger):
# Extract metric name from the first argument or 'name' keyword argument
metric_name = args[0] if args else kwargs.get("name", "")
if self._is_metric_enabled(metric_name):
return metric_class(*args, **kwargs)
else:
if not self._is_metric_enabled(metric_name):
return NoOpMetric()
original_labelnames = tuple(kwargs.get("labelnames") or ())
if not (frozenset(original_labelnames) & self.exclude_labels):
return metric_class(*args, **kwargs)
kept = tuple(name for name in original_labelnames if name not in self.exclude_labels)
kept_kwargs = {**kwargs, "labelnames": kept} # mutable-ok: ** needs a mapping to override labelnames
real_metric = metric_class(*args, **kept_kwargs)
return _ExcludedLabelMetric(real_metric, original_labelnames, self.exclude_labels)
return factory
def get_labels_for_metric(self, metric_name: DEFINED_PROMETHEUS_METRICS) -> List[str]:
@ -1059,19 +1146,15 @@ class PrometheusLogger(CustomLogger):
# Get default labels for this metric from PrometheusMetricLabels
default_labels = PrometheusMetricLabels.get_labels(metric_name)
# If no label filtering is configured for this metric, use default labels
if metric_name not in self.label_filters:
self._cached_metric_labels[metric_name] = default_labels
return default_labels
resolved_labels = [
label
for label in default_labels
if label not in self.exclude_labels
and (metric_name not in self.label_filters or label in self.label_filters[metric_name])
]
# Get configured labels for this metric
configured_labels = self.label_filters[metric_name]
# Return intersection of configured and default labels to ensure we only use valid labels
filtered_labels = [label for label in default_labels if label in configured_labels]
self._cached_metric_labels[metric_name] = filtered_labels
return filtered_labels
self._cached_metric_labels[metric_name] = resolved_labels
return resolved_labels
@staticmethod
def _guardrail_is_additive(info: StandardLoggingGuardrailInformation) -> bool:

107
litellm/integrations/websearch_interception/handler.py
View file

@ -9,7 +9,8 @@ server-side using litellm router's search tools.
import asyncio
import math
import uuid
from typing import Any, Dict, List, Optional, Tuple, Union, cast
from collections.abc import AsyncIterator, Mapping
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
import litellm
from litellm._logging import verbose_logger
@ -29,19 +30,31 @@ from litellm.integrations.websearch_interception.transformation import (
WebSearchTransformation,
)
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.types.integrations.websearch_interception import (
WebSearchInterceptionConfig,
)
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
RESPONSES_AGENTIC_SURFACE,
AgenticLoopPlan,
AgenticLoopRequestPatch,
)
from litellm.types.integrations.websearch_interception import (
WebSearchInterceptionConfig,
)
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import CallTypes, LlmProviders
from litellm.utils import ProviderConfigManager
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
# Key used to flag, on per-request kwargs, that the originating client sent
# an Anthropic-native ``web_search_*`` tool — meaning the final response
# should include ``web_search_tool_result`` content blocks so the client
@ -94,8 +107,8 @@ class WebSearchInterceptionLogger(CustomLogger):
messages: List[Dict],
tools: Optional[List[Dict]],
custom_llm_provider: Optional[str],
kwargs: Optional[dict[str, Any]] = None,
) -> Optional[Dict[str, Any]]:
kwargs: Mapping[str, object] | None = None,
) -> dict[str, object] | None:
"""
Short-circuit web-search-only requests by executing the search directly.
@ -188,7 +201,7 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.error(f"WebSearchInterception: Short-circuit search failed: {e}")
search_result_text, structured = f"Search failed: {e}", None
content: List[Dict[str, Any]] = []
content: list[dict[str, object]] = []
if native_tool is not None:
tool_use_id = f"srvtoolu_{uuid.uuid4().hex}"
tool_name = native_tool.get("name") or "web_search"
@ -210,7 +223,7 @@ class WebSearchInterceptionLogger(CustomLogger):
# github_copilot, etc.) see the same payload they always have.
content.append({"type": "text", "text": search_result_text})
response: Dict[str, Any] = {
response: dict[str, object] = {
"id": f"msg_{str(uuid.uuid4())}",
"type": "message",
"role": "assistant",
@ -228,7 +241,9 @@ class WebSearchInterceptionLogger(CustomLogger):
)
return response
async def async_pre_call_deployment_hook(self, kwargs: Dict[str, Any], call_type: Optional[Any]) -> Optional[dict]:
async def async_pre_call_deployment_hook(
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
) -> Optional[dict]:
"""
Pre-call hook to convert native Anthropic web_search tools to regular tools.
@ -297,7 +312,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return kwargs
def _convert_responses_tools(self, kwargs: dict[str, Any], tools: list[dict[str, Any]]) -> dict | None:
def _convert_responses_tools(self, kwargs: Mapping[str, object], tools: list[dict[str, object]]) -> dict | None:
"""Convert Responses API web search tools to the LiteLLM standard function tool."""
if not any(is_web_search_tool_responses(tool) for tool in tools):
return None
@ -370,7 +385,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return tool.get("name")
@classmethod
def _sync_forced_tool_choice(cls, tool_choice: Any, converted_tools: list[dict[str, Any]]) -> Any:
def _sync_forced_tool_choice(cls, tool_choice: Any, converted_tools: list[dict[str, object]]) -> object:
"""Repoint a forced ``tool_choice`` at ``litellm_web_search`` when it
names a web-search tool that was just converted away.
@ -468,7 +483,7 @@ class WebSearchInterceptionLogger(CustomLogger):
async def async_should_run_agentic_loop(
self,
response: Any,
response: object,
model: str,
messages: List[Dict],
tools: Optional[List[Dict]],
@ -578,7 +593,7 @@ class WebSearchInterceptionLogger(CustomLogger):
async def async_should_run_chat_completion_agentic_loop(
self,
response: Any,
response: object,
model: str,
messages: List[Dict],
tools: Optional[List[Dict]],
@ -636,7 +651,7 @@ class WebSearchInterceptionLogger(CustomLogger):
async def async_should_run_responses_agentic_loop(
self,
response: Any,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
@ -687,13 +702,13 @@ class WebSearchInterceptionLogger(CustomLogger):
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
anthropic_messages_provider_config: Any,
response: object,
anthropic_messages_provider_config: "BaseAnthropicMessagesConfig | None",
anthropic_messages_optional_request_params: Dict,
logging_obj: Any,
logging_obj: "LiteLLMLoggingObj | None",
stream: bool,
kwargs: Dict,
) -> Any:
) -> "AnthropicMessagesResponse | AsyncIterator[object]":
"""
Execute agentic loop with WebSearch execution for Anthropic Messages API.
@ -721,10 +736,10 @@ class WebSearchInterceptionLogger(CustomLogger):
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
anthropic_messages_provider_config: Any,
response: object,
anthropic_messages_provider_config: "BaseAnthropicMessagesConfig | None",
anthropic_messages_optional_request_params: Dict,
logging_obj: Any,
logging_obj: "LiteLLMLoggingObj | None",
stream: bool,
kwargs: Dict,
) -> AgenticLoopPlan:
@ -764,7 +779,7 @@ class WebSearchInterceptionLogger(CustomLogger):
kwargs=kwargs,
)
metadata: Dict[str, Any] = {
metadata: dict[str, object] = {
"tool_type": "websearch",
"response_format": "anthropic",
}
@ -787,10 +802,10 @@ class WebSearchInterceptionLogger(CustomLogger):
async def async_post_agentic_loop_response_hook(
self,
response: Any,
response: object,
plan: AgenticLoopPlan,
kwargs: Dict,
) -> Any:
) -> object:
"""
Inject Anthropic-native ``web_search_tool_result`` blocks into the
final response when the originating client used a native
@ -810,9 +825,9 @@ class WebSearchInterceptionLogger(CustomLogger):
def _build_native_result_blocks(
tool_calls: List[Dict],
structured_results: List[Optional[SearchResponse]],
) -> List[Dict[str, Any]]:
) -> list[dict[str, object]]:
"""Build one ``web_search_tool_result`` block per tool_call."""
blocks: List[Dict[str, Any]] = []
blocks: list[dict[str, object]] = []
for i, tool_call in enumerate(tool_calls):
tool_use_id = tool_call.get("id") or ""
structured = structured_results[i] if i < len(structured_results) else None
@ -825,7 +840,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return blocks
@staticmethod
def _inject_native_blocks(response: Any, native_blocks: List[Dict[str, Any]]) -> Any:
def _inject_native_blocks(response: Any, native_blocks: list[dict[str, object]]) -> Any:
"""Prepend native blocks to response content, dict or object form."""
if not native_blocks:
return response
@ -849,12 +864,12 @@ class WebSearchInterceptionLogger(CustomLogger):
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
response: object,
optional_params: Dict,
logging_obj: Any,
logging_obj: "LiteLLMLoggingObj | None",
stream: bool,
kwargs: Dict,
) -> Any:
) -> "ModelResponse | CustomStreamWrapper":
"""
Execute agentic loop with WebSearch execution for Chat Completions API.
@ -884,9 +899,9 @@ class WebSearchInterceptionLogger(CustomLogger):
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
response: object,
optional_params: Dict,
logging_obj: Any,
logging_obj: "LiteLLMLoggingObj | None",
stream: bool,
kwargs: Dict,
) -> AgenticLoopPlan:
@ -911,9 +926,9 @@ class WebSearchInterceptionLogger(CustomLogger):
tools: dict,
model: str,
messages: list[dict],
response: Any,
response: object,
optional_params: dict,
logging_obj: Any,
logging_obj: "LiteLLMLoggingObj | None",
stream: bool,
kwargs: dict,
) -> AgenticLoopPlan:
@ -1023,7 +1038,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return []
@staticmethod
def _extract_search_text(result: Any) -> str:
def _extract_search_text(result: object) -> str:
if isinstance(result, Exception):
verbose_logger.error(f"WebSearchInterception: Responses search failed with error: {str(result)}")
return f"Search failed: {str(result)}"
@ -1091,10 +1106,10 @@ class WebSearchInterceptionLogger(CustomLogger):
tool_calls: List[Dict],
thinking_blocks: List[Dict],
anthropic_messages_optional_request_params: Dict,
logging_obj: Any,
logging_obj: "LiteLLMLoggingObj | None",
stream: bool,
kwargs: Dict,
) -> Any:
) -> "AnthropicMessagesResponse | AsyncIterator[object]":
"""Legacy path: execute search + build patch + run follow-up call."""
request_patch, structured_results = await self._build_anthropic_request_patch(
model=model,
@ -1118,7 +1133,7 @@ class WebSearchInterceptionLogger(CustomLogger):
if max_tokens is None:
max_tokens = cast(int, kwargs.get("max_tokens", 1024))
response = await anthropic_messages.acreate(
response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate(
max_tokens=max_tokens,
messages=request_patch.messages,
model=request_patch.model or model,
@ -1145,7 +1160,7 @@ class WebSearchInterceptionLogger(CustomLogger):
tool_calls: List[Dict],
thinking_blocks: List[Dict],
anthropic_messages_optional_request_params: Dict,
logging_obj: Any,
logging_obj: "LiteLLMLoggingObj | None",
kwargs: Dict,
) -> Tuple[AgenticLoopRequestPatch, List[Optional[SearchResponse]]]:
"""
@ -1238,7 +1253,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return patch, structured_results
async def _execute_search(
self, query: str, kwargs: Optional[dict[str, Any]] = None
self, query: str, kwargs: Mapping[str, object] | None = None
) -> Tuple[str, Optional[SearchResponse]]:
"""
Execute a single web search using router's search tools.
@ -1300,8 +1315,8 @@ class WebSearchInterceptionLogger(CustomLogger):
async def _authorize_search_tool(
self,
search_tool: dict[str, Any],
kwargs: Optional[dict[str, Any]],
search_tool: Mapping[str, object],
kwargs: Mapping[str, object] | None,
) -> None:
search_tool_name = search_tool.get("search_tool_name")
if not isinstance(search_tool_name, str) or not search_tool_name:
@ -1343,7 +1358,7 @@ class WebSearchInterceptionLogger(CustomLogger):
)
@staticmethod
def _get_user_api_key_auth_from_kwargs(kwargs: Optional[dict[str, Any]]) -> Any:
def _get_user_api_key_auth_from_kwargs(kwargs: Mapping[str, object] | None) -> "UserAPIKeyAuth | None":
if not kwargs:
return None
@ -1363,7 +1378,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return None
def _select_search_tool_from_router(self, llm_router: Any) -> Optional[dict[str, Any]]:
def _select_search_tool_from_router(self, llm_router: object) -> Optional[dict[str, Any]]:
if llm_router is None or not hasattr(llm_router, "search_tools"):
return None
search_tools = list(getattr(llm_router, "search_tools") or [])
@ -1405,11 +1420,11 @@ class WebSearchInterceptionLogger(CustomLogger):
messages: List[Dict],
tool_calls: List[Dict],
optional_params: Dict,
logging_obj: Any,
logging_obj: "LiteLLMLoggingObj | None",
stream: bool,
kwargs: Dict,
response_format: str = "openai",
) -> Any:
) -> "ModelResponse | CustomStreamWrapper":
"""Legacy path: execute search + build patch + run follow-up call."""
request_patch = await self._build_chat_completion_request_patch(
model=model,

4
litellm/litellm_core_utils/streaming_handler.py
View file

@ -131,8 +131,8 @@ class CustomStreamWrapper:
self.sent_last_chunk = False
self._stream_created_time: float = time.time()
litellm_params: GenericLiteLLMParams = GenericLiteLLMParams(
**self.logging_obj.model_call_details.get("litellm_params", {})
litellm_params: GenericLiteLLMParams = GenericLiteLLMParams.model_validate(
dict(**self.logging_obj.model_call_details.get("litellm_params", {}))
)
self.merge_reasoning_content_in_choices: bool = litellm_params.merge_reasoning_content_in_choices or False
self.sent_first_thinking_block = False

134
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -13,7 +13,8 @@ Mirrors Anthropic's native ``compact_20260112`` for non-Anthropic providers:
"""
import re
from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast
import litellm
from litellm._logging import verbose_logger
@ -23,6 +24,18 @@ from litellm.types.llms.anthropic import (
UsageIteration,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.proxy._types import UserAPIKeyAuth
from litellm.router import Router
from litellm.types.llms.anthropic import (
AllAnthropicToolsValues,
AnthopicMessagesAssistantMessageParam,
AnthropicMessagesUserMessageParam,
)
from litellm.types.llms.openai import ChatCompletionToolParam
from litellm.types.utils import ModelResponse
from ..constants import (
COMPACT_DEFAULT_INSTRUCTIONS,
COMPACT_DEFAULT_TRIGGER_TOKENS,
@ -98,9 +111,9 @@ def _read_summary_max_tokens_setting() -> int:
async def _check_summary_model_access(
user_api_key_auth: Any,
user_api_key_auth: Optional["UserAPIKeyAuth"],
summary_model: str,
llm_router: Any,
llm_router: Optional["Router"],
) -> bool:
"""Return True when every model-allowlist scope on the parent request is
satisfied for ``summary_model``.
@ -294,7 +307,7 @@ async def _check_summary_model_access(
async def _check_summary_model_budget(
user_api_key_auth: Any,
user_api_key_auth: Optional["UserAPIKeyAuth"],
summary_model: str,
) -> bool:
"""Return True when the caller is within their per-model budget for
@ -357,7 +370,7 @@ async def _check_summary_model_budget(
async def _check_summary_model_rate_limit(
user_api_key_auth: Any,
user_api_key_auth: Optional["UserAPIKeyAuth"],
summary_model: str,
) -> bool:
"""Return True when the caller is within their configured RPM/TPM limits
@ -433,7 +446,7 @@ async def _check_summary_model_rate_limit(
def _find_latest_compaction_index(
messages: List[Dict[str, Any]],
messages: List[Dict[str, object]],
) -> Tuple[Optional[int], Optional[int]]:
"""Return (message_index, block_index) of the most recent compaction block.
@ -453,7 +466,7 @@ def _find_latest_compaction_index(
def _slice_around_compaction_block(
messages: List[Dict[str, Any]],
) -> Tuple[List[Dict[str, Any]], Optional[Dict[str, Any]]]:
) -> Tuple[List[Dict[str, object]], Optional[Dict[str, object]]]:
"""Apply Anthropic's "drop everything before the compaction block" rule.
Returns ``(sliced_messages_with_compaction_block, compaction_block_dict)``
@ -468,27 +481,26 @@ def _slice_around_compaction_block(
original_msg = messages[msg_idx]
original_content = original_msg["content"]
compaction_block = cast(Dict[str, Any], original_content[blk_idx])
compaction_block = cast(Dict[str, object], original_content[blk_idx])
# Per Anthropic's contract everything before the compaction block is
# dropped, including earlier blocks within the same assistant message.
sliced_content = list(original_content[blk_idx:])
sliced_first_msg = {**original_msg, "content": sliced_content}
sliced_messages: List[Dict[str, Any]] = [sliced_first_msg]
sliced_messages: List[Dict[str, object]] = [{**original_msg, "content": sliced_content}]
sliced_messages.extend(messages[msg_idx + 1 :])
return sliced_messages, compaction_block
def _strip_compaction_blocks(
messages: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
messages: List[Dict[str, object]],
) -> List[Dict[str, object]]:
"""Drop any ``compaction`` content blocks from messages.
Used to build the downstream-bound message list — the adapter has no
concept of a compaction block, so it must not see one.
"""
cleaned: List[Dict[str, Any]] = []
cleaned: List[Dict[str, object]] = []
for msg in messages:
content = msg.get("content")
if not isinstance(content, list):
@ -503,9 +515,9 @@ def _strip_compaction_blocks(
def _augment_system_with_summary(
system: Optional[Union[str, List[Dict[str, Any]]]],
system: Optional[Union[str, List[Dict[str, object]]]],
summary_text: str,
) -> Union[str, List[Dict[str, Any]]]:
) -> Union[str, List[Dict[str, object]]]:
"""Prepend a "Previous conversation summary: ..." block to ``system``."""
prefix = f"{COMPACT_SUMMARY_SYSTEM_PREFIX}{summary_text}\n\n"
if system is None:
@ -522,7 +534,7 @@ def _augment_system_with_summary(
return [{"type": "text", "text": prefix.rstrip()}, *system]
def _resolve_trigger_tokens(edit_spec: Dict[str, Any]) -> Tuple[int, List[str]]:
def _resolve_trigger_tokens(edit_spec: Dict[str, object]) -> Tuple[int, List[str]]:
"""Validate and resolve ``trigger.value``.
Raises ``AnthropicContextManagementError`` if the explicitly-supplied value
@ -556,7 +568,7 @@ def _resolve_trigger_tokens(edit_spec: Dict[str, Any]) -> Tuple[int, List[str]]:
return value, warnings
def _build_summary_prompt(edit_spec: Dict[str, Any], tools: Optional[List[Dict[str, Any]]]) -> str:
def _build_summary_prompt(edit_spec: Dict[str, object], tools: Optional[List[Dict[str, object]]]) -> str:
custom = edit_spec.get("instructions")
if isinstance(custom, str) and custom.strip():
return custom
@ -567,8 +579,8 @@ def _build_summary_prompt(edit_spec: Dict[str, Any], tools: Optional[List[Dict[s
def _propagate_metadata(
parent_litellm_metadata: Optional[Dict[str, Any]],
) -> Dict[str, Any]:
parent_litellm_metadata: Optional[Mapping[str, object]],
) -> Dict[str, object]:
"""Extract the parent request's auth/spend-attribution fields for the summary subcall.
The proxy attaches ``user_api_key``, ``user_api_key_team_id`` etc. to
@ -579,7 +591,7 @@ def _propagate_metadata(
"""
if not parent_litellm_metadata:
return {}
propagated: Dict[str, Any] = {}
propagated: Dict[str, object] = {}
for key in _PROPAGATED_METADATA_KEYS:
if key in parent_litellm_metadata:
propagated[key] = parent_litellm_metadata[key]
@ -588,10 +600,10 @@ def _propagate_metadata(
def _count_effective_tokens(
model: str,
effective_messages: List[Dict[str, Any]],
compaction_block: Optional[Dict[str, Any]],
tools: Optional[List[Dict[str, Any]]],
system: Optional[Union[str, List[Dict[str, Any]]]] = None,
effective_messages: List[Dict[str, object]],
compaction_block: Optional[CompactionBlock],
tools: Optional[List[Dict[str, object]]],
system: Optional[Union[str, List[Dict[str, object]]]] = None,
) -> int:
"""Token-count the conversation as it will appear downstream.
@ -609,25 +621,32 @@ def _count_effective_tokens(
messages_without_compaction = _strip_compaction_blocks(effective_messages)
adapter = LiteLLMAnthropicMessagesAdapter()
try:
openai_shape = adapter.translate_anthropic_messages_to_openai(messages=cast(Any, messages_without_compaction))
openai_shape = adapter.translate_anthropic_messages_to_openai(
messages=cast(
"List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]]",
messages_without_compaction,
)
)
except Exception as e:
verbose_logger.debug(
"compact_20260112: anthropic→openai translation failed during token "
"count, falling back to raw messages: %s",
e,
)
openai_shape = cast(Any, messages_without_compaction)
openai_shape = messages_without_compaction
# Translate Anthropic-shaped tools (``input_schema``) to OpenAI-shaped
# tools (``{"type": "function", "function": {...}}``) so ``token_counter``
# gets a consistent format regardless of which counting path it uses.
# An inaccurate tool token count here could cause the polyfill to skip
# needed compaction or trigger unnecessary summarization.
openai_tools: Optional[List[Dict[str, Any]]] = None
openai_tools: Optional[List[Dict[str, object]]] = None
if tools:
try:
translated_tools, _ = adapter.translate_anthropic_tools_to_openai(tools=cast(Any, tools))
openai_tools = cast(List[Dict[str, Any]], translated_tools)
translated_tools, _ = adapter.translate_anthropic_tools_to_openai(
tools=cast("List[AllAnthropicToolsValues]", tools)
)
openai_tools = cast(List[Dict[str, object]], translated_tools)
except Exception as e:
verbose_logger.debug(
"compact_20260112: anthropic→openai tools translation failed "
@ -638,8 +657,8 @@ def _count_effective_tokens(
total = litellm.token_counter(
model=model,
messages=cast(Any, openai_shape),
tools=cast(Any, openai_tools),
messages=cast(List[Dict[str, object]], openai_shape),
tools=cast("Optional[List[ChatCompletionToolParam]]", openai_tools),
)
if compaction_block is not None:
content = compaction_block.get("content") or ""
@ -652,7 +671,7 @@ def _count_effective_tokens(
def _system_to_text(
system: Optional[Union[str, List[Dict[str, Any]]]],
system: Optional[Union[str, List[Dict[str, object]]]],
) -> str:
"""Flatten an Anthropic-style ``system`` value into a single string for
token counting. Returns ``""`` when ``system`` carries no text."""
@ -670,8 +689,8 @@ def _system_to_text(
def _select_last_user_question(
messages: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
messages: List[Dict[str, object]],
) -> List[Dict[str, object]]:
"""Pick the most recent ``user`` turn that is a real question.
Returns a one-element message list with any ``tool_result`` blocks
@ -735,10 +754,10 @@ def _system_to_openai_message(
def _build_summary_messages(
effective_messages: List[Dict[str, Any]],
effective_messages: List[Dict[str, object]],
prompt: str,
system: Optional[Union[str, List[Dict[str, Any]]]] = None,
) -> List[Dict[str, Any]]:
system: Optional[Union[str, List[Dict[str, object]]]] = None,
) -> List[Dict[str, object]]:
"""Build the OpenAI-shape message list for the summary call.
The caller's ``system`` prompt is prepended (the default summarization
@ -753,7 +772,10 @@ def _build_summary_messages(
stripped = _strip_compaction_blocks(effective_messages)
try:
openai_messages = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
messages=cast(Any, stripped)
messages=cast(
"List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]]",
stripped,
)
)
except Exception as e:
verbose_logger.warning(
@ -761,9 +783,9 @@ def _build_summary_messages(
"building summary call; falling back to raw shape: %s",
e,
)
openai_messages = cast(Any, stripped)
openai_messages = stripped
summary_messages: List[Dict[str, Any]] = []
summary_messages: List[Dict[str, object]] = []
system_message = _system_to_openai_message(system)
if system_message is not None:
summary_messages.append(system_message)
@ -783,7 +805,7 @@ def _build_summary_messages(
return summary_messages
def _is_user_message(msg: Any) -> bool:
def _is_user_message(msg: object) -> bool:
return isinstance(msg, dict) and msg.get("role") == "user"
@ -805,12 +827,12 @@ def _append_text_to_content(content: Any, extra_text: str) -> Any:
async def _call_summary_model(
*,
summary_model: str,
summary_messages: List[Dict[str, Any]],
metadata: Dict[str, Any],
summary_messages: List[Dict[str, object]],
metadata: Mapping[str, object],
llm_router: Any,
allowed_model_region: Optional[str] = None,
max_tokens: int = COMPACT_SUMMARY_MAX_TOKENS,
) -> Any:
) -> Union["ModelResponse", "CustomStreamWrapper"]:
"""Invoke the configured summary model.
Prefers ``llm_router.acompletion`` so the model alias resolves against the
@ -877,7 +899,7 @@ def _extract_response_text(response: Any) -> Optional[str]:
return None
def _extract_usage(response: Any) -> Tuple[int, int]:
def _extract_usage(response: object) -> Tuple[int, int]:
usage = getattr(response, "usage", None)
if usage is None:
return 0, 0
@ -889,8 +911,8 @@ def _extract_usage(response: Any) -> Tuple[int, int]:
def apply_client_compaction_block_history(
*,
messages: List[Dict[str, Any]],
system: Optional[Union[str, List[Dict[str, Any]]]],
messages: List[Dict[str, object]],
system: Optional[Union[str, List[Dict[str, object]]]],
) -> Optional[PolyfillResult]:
"""Honor client-sent compaction blocks without a ``compact_20260112`` edit.
@ -911,7 +933,7 @@ def apply_client_compaction_block_history(
)
prior_summary_text = prior_compaction_block.get("content") or ""
augmented_system: Union[str, List[Dict[str, Any]], None] = system
augmented_system: Union[str, List[Dict[str, object]], None] = system
if isinstance(prior_summary_text, str) and prior_summary_text:
augmented_system = _augment_system_with_summary(system, prior_summary_text)
verbose_logger.info(
@ -936,13 +958,13 @@ def apply_client_compaction_block_history(
async def apply_compact_20260112(
*,
model: str,
messages: List[Dict[str, Any]],
tools: Optional[List[Dict[str, Any]]],
system: Optional[Union[str, List[Dict[str, Any]]]],
edit_spec: Dict[str, Any],
litellm_metadata: Optional[Dict[str, Any]] = None,
llm_router: Any = None,
user_api_key_auth: Any = None,
messages: List[Dict[str, object]],
tools: Optional[List[Dict[str, object]]],
system: Optional[Union[str, List[Dict[str, object]]]],
edit_spec: Dict[str, object],
litellm_metadata: Optional[Mapping[str, object]] = None,
llm_router: Optional["Router"] = None,
user_api_key_auth: Optional["UserAPIKeyAuth"] = None,
) -> PolyfillResult:
"""Apply ``compact_20260112``; return a ``PolyfillResult``.
@ -971,7 +993,7 @@ async def apply_compact_20260112(
# non-Anthropic backends (which would reject them).
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
prior_summary_text = prior_compaction_block.get("content") if prior_compaction_block else None
augmented_system: Union[str, List[Dict[str, Any]], None] = system
augmented_system: Union[str, List[Dict[str, object]], None] = system
if isinstance(prior_summary_text, str) and prior_summary_text:
augmented_system = _augment_system_with_summary(system, prior_summary_text)
verbose_logger.info(

4
litellm/llms/azure/responses/transformation.py
View file

@ -66,7 +66,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
)
# Create ResponseReasoningItem object from the item data
reasoning_item = ResponseReasoningItem(**item_data)
reasoning_item = ResponseReasoningItem.model_validate(item_data)
# Convert back to dict with exclude_none=True to exclude None fields
dict_reasoning_item = reasoning_item.model_dump(exclude_none=True)
@ -346,4 +346,4 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIError
raise AzureOpenAIError(message=raw_response.text, status_code=raw_response.status_code)
return ResponsesAPIResponse(**raw_response_json)
return ResponsesAPIResponse.model_validate(raw_response_json)

149
litellm/llms/bedrock/base_aws_llm.py
View file

@ -5,6 +5,7 @@ import os
import re
import urllib.parse
from datetime import datetime
from threading import Lock
from typing import (
TYPE_CHECKING,
Any,
@ -24,10 +25,14 @@ from pydantic import BaseModel, ValidationError
from litellm._logging import verbose_logger
from litellm.caching.caching import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import (
BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES,
BEDROCK_IAM_CACHE_MAX_ENTRIES,
BEDROCK_INVOKE_PROVIDERS_LITERAL,
BEDROCK_MAX_POLICY_SIZE,
STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS,
)
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.secret_managers.main import get_secret, get_secret_str
@ -75,14 +80,28 @@ class AwsAuthError(Exception):
class BaseAWSLLM:
# Process-wide IAM credential cache (shared across instances — Bedrock passthrough is per-request).
# Storage is in-process memory only: default ``DualCache()`` has no Redis backend unless attached
# elsewhere. Entry TTL: static access-key + secret + region use ``_get_default_ttl_for_boto3_credentials``
# (~59 minutes); ambient env (``_auth_with_env_vars`` returns ``ttl=None``) uses ``InMemoryCache``'s
# ``default_ttl`` (600 seconds / 10 minutes); web identity STS credentials use
# ``_get_default_ttl_for_boto3_credentials`` (~59 minutes), keyed on all aws_* credential args
# plus ssl_verify. AssumeRole, profiles, and explicit session-token tuples are not cached — see
# ``get_credentials`` and ``_get_or_set_cached_credentials``.
_shared_iam_cache: ClassVar[DualCache] = DualCache()
# Storage is in-process memory only: no Redis backend unless attached elsewhere. Entry TTL: static
# access-key + secret + region use ``_get_default_ttl_for_boto3_credentials`` (~59 minutes); ambient
# env (``_auth_with_env_vars`` returns ``ttl=None``) uses ``InMemoryCache``'s ``default_ttl``
# (600 seconds / 10 minutes); web identity STS credentials use
# ``_get_default_ttl_for_boto3_credentials`` (~59 minutes); AssumeRole STS credentials expire with
# the STS session itself (Expiration minus a safety margin). All are keyed on all aws_* credential
# args plus ssl_verify, so ``aws_session_name`` scopes an entry to one attributed identity. Profiles
# and explicit session-token tuples are not cached — see ``get_credentials`` and
# ``_get_or_set_cached_credentials``. The bound is larger than ``InMemoryCache``'s default because
# per-user cost attribution puts one entry per attributed identity in this cache.
_shared_iam_cache: ClassVar[DualCache] = DualCache(
in_memory_cache=InMemoryCache(max_size_in_memory=BEDROCK_IAM_CACHE_MAX_ENTRIES)
)
# Striped single-flight locks over ``_shared_iam_cache``. Concurrent misses on one credential
# key would otherwise each issue their own STS call, which is the same thundering herd the cache
# exists to prevent, just moved to the miss window. Striping keeps distinct identities from
# serialising behind each other without a per-key registry that grows with the identity count.
# A cache hit holds its stripe only for the lookup itself.
_credential_fetch_locks: ClassVar[tuple[Lock, ...]] = tuple(
Lock() for _ in range(BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES)
)
def __init__(self) -> None:
self.iam_cache = BaseAWSLLM._shared_iam_cache
@ -140,19 +159,21 @@ class BaseAWSLLM:
Used for static access-key credentials, ambient credentials from
``_auth_with_env_vars`` (including when skipping AssumeRole because the runtime identity
already matches ``aws_role_name``), and web identity STS credentials (plain
non-refreshable ``Credentials`` cached ~59 min, inside the 3600s STS session).
already matches ``aws_role_name``), web identity STS credentials (plain
non-refreshable ``Credentials`` cached ~59 min, inside the 3600s STS session), and AssumeRole
STS credentials (cached for the lifetime of the STS session minus a safety margin).
AssumeRole, profiles, and explicit session-token tuples are not
cached here — shared ``Credentials`` / refresh state must not span logical sessions.
Profiles and explicit session-token tuples are not cached here — shared ``Credentials`` /
refresh state must not span logical sessions.
"""
cache_key = self.get_cache_key(credential_args)
_cached = self.iam_cache.get_cache(cache_key)
if _cached:
return _cached
credentials, ttl = credential_fetcher()
self.iam_cache.set_cache(cache_key, credentials, ttl=ttl)
return credentials
with self._credential_fetch_locks[hash(cache_key) % len(self._credential_fetch_locks)]:
_cached = self.iam_cache.get_cache(cache_key)
if _cached:
return _cached
credentials, ttl = credential_fetcher()
self.iam_cache.set_cache(cache_key, credentials, ttl=ttl)
return credentials
@staticmethod
def _is_auth_with_web_identity_token(
@ -269,8 +290,8 @@ class BaseAWSLLM:
# Credentials - boto3.Credentials
# cache ttl - Optional[int]. If None, the credentials are not cached. Some auth flows have no expiry time.
#
# iam_cache: static keys, ambient env (including skip-AssumeRole path), and web identity.
# Do not cache AssumeRole / profile / explicit session-token paths here.
# iam_cache: static keys, ambient env (including skip-AssumeRole path), web identity, and
# AssumeRole. Do not cache profile / explicit session-token paths here.
#########################################################
if self._is_auth_with_web_identity_token(
aws_web_identity_token,
@ -290,30 +311,20 @@ class BaseAWSLLM:
),
)
elif self._is_auth_with_aws_role(aws_role_name):
# Same role (IRSA/ECS/EC2): ambient creds via _get_or_set_cached_credentials like the
# default env branch; never pre-read cache (must run _is_already_running_as_role first).
if self._is_already_running_as_role(cast(str, aws_role_name), ssl_verify=ssl_verify):
verbose_logger.debug(
"Already running as target role %s, using ambient credentials",
aws_role_name,
)
return self._get_or_set_cached_credentials(args, self._auth_with_env_vars)
verbose_logger.debug("Using role assumption: calling _auth_with_aws_role")
# If aws_session_name is not provided, generate a default one
if aws_session_name is None:
aws_session_name = f"litellm-session-{int(datetime.now().timestamp())}"
credentials, _assume_ttl = self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_role_name=cast(str, aws_role_name),
aws_session_name=aws_session_name,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
ssl_verify=ssl_verify,
return self._get_or_set_cached_credentials(
args,
lambda: self._resolve_role_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_role_name=cast(str, aws_role_name),
aws_session_name=aws_session_name,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
ssl_verify=ssl_verify,
),
)
return credentials
elif self._is_auth_with_aws_profile(aws_profile_name):
credentials, _cache_ttl = self._auth_with_aws_profile(cast(str, aws_profile_name))
@ -1046,7 +1057,11 @@ class BaseAWSLLM:
return sts_client.assume_role(**assume_role_params)
def _extract_credentials_and_ttl(self, sts_response: dict) -> Tuple[Credentials, Optional[int]]:
"""Extract credentials and TTL from STS response."""
"""Extract credentials and TTL from STS response.
The TTL carries the same safety margin as the non-IRSA assume path, so a cached entry is
never handed out close enough to expiry to die mid-request.
"""
from botocore.credentials import Credentials
sts_credentials = sts_response["Credentials"]
@ -1057,10 +1072,52 @@ class BaseAWSLLM:
)
expiration_time = sts_credentials["Expiration"]
ttl = int((expiration_time - datetime.now(expiration_time.tzinfo)).total_seconds())
ttl = int(
(expiration_time - datetime.now(expiration_time.tzinfo)).total_seconds()
- STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS
)
return credentials, ttl
def _resolve_role_credentials(
self,
aws_access_key_id: str | None,
aws_secret_access_key: str | None,
aws_session_token: str | None,
aws_role_name: str,
aws_session_name: str | None,
aws_region_name: str | None,
aws_sts_endpoint: str | None,
aws_external_id: str | None,
ssl_verify: bool | str | None,
) -> tuple[Credentials, int | None]:
"""
Resolve credentials for a target role, either from the ambient identity or via sts:AssumeRole.
Both the ``sts:GetCallerIdentity`` probe and the assume itself run here, so a cache hit on the
caller's key skips both. ``aws_session_name`` defaults inside this fetcher rather than in
``get_credentials`` so the cache key stays stable when the caller does not supply one.
"""
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
verbose_logger.debug(
"Already running as target role %s, using ambient credentials",
aws_role_name,
)
return self._auth_with_env_vars()
verbose_logger.debug("Using role assumption: calling _auth_with_aws_role")
return self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_role_name=aws_role_name,
aws_session_name=aws_session_name or f"litellm-session-{int(datetime.now().timestamp())}",
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
ssl_verify=ssl_verify,
)
@tracer.wrap()
def _auth_with_aws_role(
self,
@ -1192,7 +1249,7 @@ class BaseAWSLLM:
sts_expiry = sts_credentials["Expiration"]
# Convert to timezone-aware datetime for comparison
current_time = datetime.now(sts_expiry.tzinfo)
sts_ttl = (sts_expiry - current_time).total_seconds() - 60
sts_ttl = (sts_expiry - current_time).total_seconds() - STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS
return credentials, sts_ttl
@tracer.wrap()

568
litellm/llms/custom_httpx/llm_http_handler.py
View file

File diff suppressed because it is too large Load diff

4
litellm/llms/manus/responses/transformation.py
View file

@ -217,7 +217,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
try:
response = ResponsesAPIResponse(**raw_response_json)
response = ResponsesAPIResponse.model_validate(raw_response_json)
except Exception:
verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct")
response = ResponsesAPIResponse.model_construct(**raw_response_json)
@ -305,7 +305,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
try:
response = ResponsesAPIResponse(**raw_response_json)
response = ResponsesAPIResponse.model_validate(raw_response_json)
except Exception:
verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct")
response = ResponsesAPIResponse.model_construct(**raw_response_json)

61
litellm/llms/openai/videos/transformation.py
View file

@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
from urllib.parse import quote
import httpx
from httpx._types import RequestFiles
from httpx._types import FileContent, FileTypes, RequestFiles
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
@ -128,7 +128,7 @@ class OpenAIVideoConfig(BaseVideoConfig):
# Handle input_reference parameter if provided
_input_reference = video_create_optional_request_params.get("input_reference")
data_without_files = {k: v for k, v in request_dict.items() if k not in ["input_reference"]}
files_list: List[Tuple[str, Any]] = []
files_list: List[Tuple[str, FileTypes]] = []
# Handle input_reference parameter
if _input_reference is not None:
@ -177,9 +177,7 @@ class OpenAIVideoConfig(BaseVideoConfig):
request_data: Optional[Dict] = None,
) -> VideoObject:
"""Transform the OpenAI video creation response."""
response_data = raw_response.json()
video_obj = VideoObject(**response_data) # type: ignore[arg-type]
video_obj = VideoObject.model_validate(raw_response.json())
if custom_llm_provider and video_obj.id:
video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model)
@ -223,7 +221,7 @@ class OpenAIVideoConfig(BaseVideoConfig):
url = f"{url}?variant={quote(variant, safe='')}"
# No additional data needed for GET content request
data: Dict[str, Any] = {}
data: Dict[str, object] = {}
return url, data
@ -274,10 +272,8 @@ class OpenAIVideoConfig(BaseVideoConfig):
"""
Transform the OpenAI video remix response.
"""
response_data = raw_response.json()
# Transform the response data
video_obj = VideoObject(**response_data) # type: ignore[arg-type]
video_obj = VideoObject.model_validate(raw_response.json())
if custom_llm_provider and video_obj.id:
video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None)
@ -392,7 +388,7 @@ class OpenAIVideoConfig(BaseVideoConfig):
url = f"{api_base.rstrip('/')}/{encoded_video_id}"
# No data needed for DELETE request
data: Dict[str, Any] = {}
data: Dict[str, object] = {}
return url, data
@ -404,10 +400,8 @@ class OpenAIVideoConfig(BaseVideoConfig):
"""
Transform the OpenAI video delete response.
"""
response_data = raw_response.json()
# Transform the response data
video_obj = VideoObject(**response_data) # type: ignore[arg-type] # type: ignore[arg-type]
video_obj = VideoObject.model_validate(raw_response.json())
return video_obj
@ -429,7 +423,7 @@ class OpenAIVideoConfig(BaseVideoConfig):
url = f"{api_base.rstrip('/')}/{encoded_video_id}"
# No additional data needed for GET request
data: Dict[str, Any] = {}
data: Dict[str, object] = {}
return url, data
@ -442,9 +436,8 @@ class OpenAIVideoConfig(BaseVideoConfig):
"""
Transform the OpenAI video retrieve response.
"""
response_data = raw_response.json()
# Transform the response data
video_obj = VideoObject(**response_data) # type: ignore[arg-type]
video_obj = VideoObject.model_validate(raw_response.json())
if custom_llm_provider and video_obj.id:
video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None)
@ -465,22 +458,22 @@ class OpenAIVideoConfig(BaseVideoConfig):
def transform_video_create_character_request(
self,
name: str,
video: Any,
video: FileContent,
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, list]:
url = f"{api_base.rstrip('/')}/characters"
files_list: List[Tuple[str, Any]] = [("name", (None, name))]
files_list: List[Tuple[str, FileTypes]] = [("name", (None, name))]
self._add_video_to_files(files_list, video, "video")
return url, files_list
def transform_video_create_character_response(
self,
raw_response: httpx.Response,
logging_obj: Any,
logging_obj: LiteLLMLoggingObj,
) -> CharacterObject:
return CharacterObject(**raw_response.json())
return CharacterObject.model_validate(raw_response.json())
def transform_video_get_character_request(
self,
@ -497,9 +490,9 @@ class OpenAIVideoConfig(BaseVideoConfig):
def transform_video_get_character_response(
self,
raw_response: httpx.Response,
logging_obj: Any,
logging_obj: LiteLLMLoggingObj,
) -> CharacterObject:
return CharacterObject(**raw_response.json())
return CharacterObject.model_validate(raw_response.json())
def transform_video_edit_request(
self,
@ -508,12 +501,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
extra_body: Optional[Dict[str, Any]] = None,
prefetched_source_data: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, object]] = None,
prefetched_source_data: Optional[Dict[str, object]] = None,
) -> Tuple[str, Dict]:
original_video_id = extract_original_video_id(video_id)
url = f"{api_base.rstrip('/')}/edits"
data: Dict[str, Any] = {"prompt": prompt, "video": {"id": original_video_id}}
data: Dict[str, object] = {"prompt": prompt, "video": {"id": original_video_id}}
if extra_body:
data.update(extra_body)
return url, data
@ -521,11 +514,11 @@ class OpenAIVideoConfig(BaseVideoConfig):
def transform_video_edit_response(
self,
raw_response: httpx.Response,
logging_obj: Any,
logging_obj: LiteLLMLoggingObj,
custom_llm_provider: Optional[str] = None,
request_data: Optional[Dict] = None,
) -> VideoObject:
video_obj = VideoObject(**raw_response.json())
video_obj = VideoObject.model_validate(raw_response.json())
if custom_llm_provider and video_obj.id:
video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None)
return video_obj
@ -538,11 +531,11 @@ class OpenAIVideoConfig(BaseVideoConfig):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
extra_body: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, object]] = None,
) -> Tuple[str, Dict]:
original_video_id = extract_original_video_id(video_id)
url = f"{api_base.rstrip('/')}/extensions"
data: Dict[str, Any] = {
data: Dict[str, object] = {
"prompt": prompt,
"seconds": seconds,
"video": {"id": original_video_id},
@ -554,10 +547,10 @@ class OpenAIVideoConfig(BaseVideoConfig):
def transform_video_extension_response(
self,
raw_response: httpx.Response,
logging_obj: Any,
logging_obj: LiteLLMLoggingObj,
custom_llm_provider: Optional[str] = None,
) -> VideoObject:
video_obj = VideoObject(**raw_response.json())
video_obj = VideoObject.model_validate(raw_response.json())
if custom_llm_provider and video_obj.id:
video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None)
return video_obj
@ -578,8 +571,8 @@ class OpenAIVideoConfig(BaseVideoConfig):
def _add_video_to_files(
self,
files_list: List[Tuple[str, Any]],
video: Any,
files_list: List[Tuple[str, FileTypes]],
video: FileContent,
field_name: str,
) -> None:
"""
@ -592,7 +585,7 @@ class OpenAIVideoConfig(BaseVideoConfig):
content_type = self._get_video_content_type(video=video, filename=filename)
files_list.append((field_name, (filename, video, content_type)))
def _get_video_content_type(self, video: Any, filename: str) -> str:
def _get_video_content_type(self, video: FileContent, filename: str) -> str:
guessed_content_type, _ = mimetypes.guess_type(filename)
if guessed_content_type and guessed_content_type.startswith("video/"):
return guessed_content_type

224
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -15,8 +15,19 @@ import re
import time
from collections.abc import Sequence
from contextlib import asynccontextmanager
from typing import Any, AsyncIterator, Callable, Literal, Optional, Union, cast
from urllib.parse import urlparse
from typing import (
TYPE_CHECKING,
Any,
AsyncIterator,
Callable,
Literal,
Optional,
TypeAlias,
TypedDict,
Union,
cast,
)
from urllib.parse import ParseResult, urlparse
import anyio
import httpx
@ -32,7 +43,7 @@ from mcp.types import (
ResourceTemplate,
)
from mcp.types import Tool as MCPTool
from pydantic import AnyUrl
from pydantic import AnyUrl, BaseModel
import litellm
from litellm._logging import verbose_logger
@ -139,10 +150,15 @@ from litellm.proxy._types import (
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl
from litellm.proxy.utils import ProxyLogging, get_server_root_path
from litellm.proxy.utils import PrismaClient, ProxyLogging, get_server_root_path
from litellm.repositories.table_repositories import MCPServerRepository
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE, MCPAuth, MCPStdioConfig
from litellm.types.mcp import (
DEFAULT_SUBJECT_TOKEN_TYPE,
MCPAuth,
MCPStdioConfig,
MCPTokenEndpointAuthMethod,
)
from litellm.types.mcp_server.mcp_server_manager import (
MCPInfo,
MCPOAuthMetadata,
@ -150,6 +166,14 @@ from litellm.types.mcp_server.mcp_server_manager import (
)
from litellm.types.utils import CallTypes
if TYPE_CHECKING:
from mcp.client.session import ClientSession
from mcp.shared.context import RequestContext
from mcp.types import CreateMessageRequestParams
from litellm.caching.caching import InMemoryCache
from litellm.types.mcp_server.mcp_toolset import MCPToolset
try:
from mcp.shared.tool_name_validation import (
SEP_986_URL,
@ -209,6 +233,95 @@ _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES: tuple[MCPAuth, ...] = (
_OAUTH_DISCOVERY_RETRY_BASE_SECONDS = 30.0
_OAUTH_DISCOVERY_RETRY_MAX_SECONDS = 900.0
_StringList: TypeAlias = list[str]
_StringMap: TypeAlias = dict[str, str]
_ToolParamMap: TypeAlias = dict[str, list[str]]
_EnvVarList: TypeAlias = list[dict[str, object]]
_InMemoryCacheDict: TypeAlias = dict[str, object]
_ToolArguments: TypeAlias = dict[str, object]
class MCPServerConfig(TypedDict, total=False):
"""Shape of a single ``mcp_servers`` entry in config.yaml, as consumed by
:meth:`MCPServerManager.load_servers_from_config`. Every key is optional: YAML supplies
whatever the admin wrote, and each read applies its own default."""
alias: str
description: str
mcp_info: MCPInfo
url: str
spec_path: str
transport: MCPTransportType
auth_type: MCPAuthType
authentication_token: str
auth_value: str
instructions: str
command: str
args: _StringList
env: _StringMap
client_id: str
client_secret: str
oauth2_flow: str
issuer: str
authorization_url: str
token_url: str
registration_url: str
token_endpoint_auth_method: MCPTokenEndpointAuthMethod
scopes: str | Sequence[str]
dcr_bridge: object
extra_headers: _StringList
allowed_tools: _StringList
disallowed_tools: _StringList
allowed_params: _ToolParamMap
access_groups: _StringList
static_headers: _StringMap
env_vars: _EnvVarList
allow_all_keys: bool
available_on_public_internet: bool
delegate_auth_to_upstream: bool
oauth_passthrough: bool
allow_sampling: bool
allow_elicitation: bool
aws_access_key_id: str
aws_secret_access_key: str
aws_session_token: str
aws_region_name: str
aws_service_name: str
aws_role_name: str
aws_session_name: str
token_exchange_endpoint: str
token_exchange_profile: str
audience: str
subject_token_type: str
upstream_resource: str
id_jag_resource_token_endpoint: str
id_jag_resource: str
client_private_key: str
client_private_key_id: str
client_assertion_signing_alg: str
timeout: float
max_concurrent_requests: int
class _ProtectedResourceMetadataPayload(TypedDict, total=False):
"""The RFC 9728 protected-resource metadata document fields this gateway reads."""
authorization_servers: Sequence[object]
scopes_supported: Sequence[str]
scopes: Sequence[str]
class _AuthorizationServerMetadataPayload(TypedDict, total=False):
"""The RFC 8414 / OpenID Discovery authorization-server metadata fields this gateway reads."""
issuer: str
authorization_endpoint: str
token_endpoint: str
registration_endpoint: str
scopes_supported: Sequence[str]
grant_types_supported: Sequence[str]
token_endpoint_auth_methods_supported: Sequence[str]
def _blank_to_none(value: str | None) -> str | None:
"""Collapse an absent, empty, or whitespace-only string to ``None``.
@ -968,7 +1081,7 @@ def _warn_internal_delegate_pkce_if_applicable(server: MCPServer, *, source: str
)
def _deserialize_json_dict(data: Any) -> Optional[dict[str, str]]:
def _deserialize_json_dict(data: str | _StringMap | None) -> Optional[dict[str, str]]:
"""
Deserialize optional JSON mappings stored in the database.
@ -1057,7 +1170,7 @@ def _normalize_mcp_server_cost_info(mcp_info: MCPInfo) -> None:
mcp_info["mcp_server_cost_info"] = normalized
def _create_sampling_callback(user_api_key_auth: Optional[Any] = None):
def _create_sampling_callback(user_api_key_auth: Optional[UserAPIKeyAuth] = None):
"""
Create a sampling callback for MCP ClientSession.
Returns a callable that handles sampling/createMessage requests from
@ -1066,7 +1179,10 @@ def _create_sampling_callback(user_api_key_auth: Optional[Any] = None):
if not MCP_SAMPLING_AVAILABLE:
return None
async def _sampling_callback(context, params):
async def _sampling_callback(
context: "RequestContext[ClientSession, object]",
params: "CreateMessageRequestParams",
):
import litellm
from litellm.proxy._experimental.mcp_server.sampling_handler import (
handle_sampling_create_message,
@ -1309,8 +1425,9 @@ class MCPServerManager:
if state is None:
return True
failures, attempted_at = state
backoff_multiplier: int = 2 ** max(failures - 1, 0)
delay = min(
_OAUTH_DISCOVERY_RETRY_BASE_SECONDS * (2 ** max(failures - 1, 0)),
_OAUTH_DISCOVERY_RETRY_BASE_SECONDS * backoff_multiplier,
_OAUTH_DISCOVERY_RETRY_MAX_SECONDS,
)
return (time.monotonic() - attempted_at) >= delay
@ -1324,7 +1441,7 @@ class MCPServerManager:
self._oauth_discovery_retry_state[server.server_id] = (failures + 1, time.monotonic())
def _remember_upstream_initialize_instructions(self, server: MCPServer, client: MCPClient) -> None:
raw = getattr(client, "_last_initialize_instructions", None)
raw: str | None = getattr(client, "_last_initialize_instructions", None)
if raw and str(raw).strip():
self._upstream_initialize_instructions_by_server_id[server.server_id] = str(raw).strip()
@ -1430,9 +1547,10 @@ class MCPServerManager:
# Track which aliases have been used to ensure only first occurrence is used
used_aliases = set()
for server_name, server_config in mcp_servers_config.items():
for server_name, raw_server_config in mcp_servers_config.items():
server_config: MCPServerConfig = raw_server_config
validate_mcp_server_name(server_name)
_mcp_info: dict[str, Any] = server_config.get("mcp_info", None) or {}
_mcp_info: MCPInfo = server_config.get("mcp_info", None) or {}
# Preserve all custom fields from config while setting defaults for core fields
mcp_info: MCPInfo = _mcp_info.copy()
# Set default values for core fields if not present
@ -1895,7 +2013,7 @@ class MCPServerManager:
mcp_server: LiteLLM_MCPServerTable,
*,
env_vars_are_encrypted: bool,
) -> Optional[list[dict[str, Any]]]:
) -> Optional[_EnvVarList]:
env_vars_list = _deserialize_json_list(getattr(mcp_server, "env_vars", None))
if env_vars_are_encrypted:
from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415
@ -2279,7 +2397,7 @@ class MCPServerManager:
async def _get_active_submitted_mcp_server_ids_for_user(
self, user_api_key_auth: UserAPIKeyAuth | None
) -> list[str]:
submitter_user_id = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None
submitter_user_id: str | None = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None
if not submitter_user_id:
return []
@ -2551,10 +2669,10 @@ class MCPServerManager:
try:
from litellm.proxy.proxy_server import user_api_key_cache
in_mem = getattr(user_api_key_cache, "in_memory_cache", None)
in_mem: InMemoryCache | None = getattr(user_api_key_cache, "in_memory_cache", None)
if in_mem is None:
return
cache_dict = getattr(in_mem, "cache_dict", {})
cache_dict: _InMemoryCacheDict = getattr(in_mem, "cache_dict", {})
if toolset_id is None:
keys_to_remove = [k for k in cache_dict if k.startswith("toolset_")]
else:
@ -2574,9 +2692,9 @@ class MCPServerManager:
async def get_toolset_by_name_cached(
self,
prisma_client: Any,
prisma_client: PrismaClient,
toolset_name: str,
) -> Optional[Any]:
) -> "Optional[MCPToolset]":
"""Return a toolset by name, cached in ``user_api_key_cache`` (Redis-backed
``DualCache`` in production) to avoid a DB hit on every routed request.
@ -2803,7 +2921,7 @@ class MCPServerManager:
and report ``unknown`` instead of a misleading ``unhealthy``.
"""
static_headers = server.static_headers
env_vars = getattr(server, "env_vars", None)
env_vars: _EnvVarList | None = getattr(server, "env_vars", None)
if not static_headers or not env_vars:
return False
_global_values, user_specs = parse_admin_env_vars(env_vars)
@ -2929,7 +3047,7 @@ class MCPServerManager:
"""
if user_api_key_auth is None:
return {}
user_id = getattr(user_api_key_auth, "user_id", None)
user_id: str | None = getattr(user_api_key_auth, "user_id", None)
if not user_id:
return {}
@ -2976,7 +3094,7 @@ class MCPServerManager:
match await provider.resolve_credentials(to_subject(user_api_key_auth, subject_token), spec):
case Ok(auth):
# NoOpAuth has no header_name and so never conflicts.
header_name = getattr(auth, "header_name", None)
header_name: str | None = getattr(auth, "header_name", None)
conflicts = bool(
header_name and extra_headers and any(key.lower() == header_name.lower() for key in extra_headers)
)
@ -3546,7 +3664,7 @@ class MCPServerManager:
self,
server: MCPServer,
prompt_name: str,
arguments: Optional[dict[str, Any]] = None,
arguments: Optional[dict[str, str]] = None,
mcp_auth_header: Optional[Union[str, dict[str, str]]] = None,
extra_headers: Optional[dict[str, str]] = None,
raw_headers: Optional[dict[str, str]] = None,
@ -3606,7 +3724,7 @@ class MCPServerManager:
and base_port == target_port
)
async def _fetch_oauth_discovery_url(self, url: str, server_url: str) -> Any:
async def _fetch_oauth_discovery_url(self, url: str, server_url: str) -> httpx.Response:
client = get_async_httpx_client(
llm_provider=httpxSpecialProvider.MCP,
params={"timeout": MCP_METADATA_TIMEOUT},
@ -3807,7 +3925,7 @@ class MCPServerManager:
try:
response = await self._fetch_oauth_discovery_url(resource_metadata_url, server_url)
response.raise_for_status()
data = response.json()
data: _ProtectedResourceMetadataPayload = response.json()
except SSRFError as exc:
verbose_logger.warning(
"MCP OAuth discovery: refusing to fetch resource metadata from %s "
@ -3932,7 +4050,7 @@ class MCPServerManager:
try:
response = await self._fetch_oauth_discovery_url(url, server_url)
response.raise_for_status()
data = response.json()
data: _AuthorizationServerMetadataPayload = response.json()
except SSRFError as exc:
verbose_logger.warning(
"MCP OAuth discovery: refusing to fetch authorization-server "
@ -3993,7 +4111,7 @@ class MCPServerManager:
@staticmethod
def _build_azure_authorization_server_metadata(
parsed_issuer_url: Any,
parsed_issuer_url: ParseResult,
) -> Optional[MCPOAuthMetadata]:
path_parts = [part for part in (parsed_issuer_url.path or "").split("/") if part]
if parsed_issuer_url.netloc not in _AZURE_ENTRA_HOSTS or len(path_parts) != 2 or path_parts[1] != "v2.0":
@ -4054,7 +4172,7 @@ class MCPServerManager:
"aws_session_name": credentials_dict.get("aws_session_name"),
}
def _extract_scopes(self, scopes_value: Any) -> Optional[list[str]]:
def _extract_scopes(self, scopes_value: str | Sequence[object] | None) -> Optional[list[str]]:
if isinstance(scopes_value, str):
scopes = [s.strip() for s in scopes_value.split() if s.strip()]
return scopes or None
@ -4292,7 +4410,7 @@ class MCPServerManager:
return match_known_tool_name(tool_name, server, server.allowed_tools or ()) is not None
return match_known_tool_name(tool_name, server, server.disallowed_tools or ()) is None
def validate_allowed_params(self, tool_name: str, arguments: dict[str, Any], server: MCPServer) -> None:
def validate_allowed_params(self, tool_name: str, arguments: _ToolArguments, server: MCPServer) -> None:
"""
Filter arguments to only include allowed parameters for the given tool.
@ -4373,7 +4491,7 @@ class MCPServerManager:
self,
server: MCPServer,
tool_name: str,
arguments: dict[str, Any],
arguments: _ToolArguments,
) -> CallToolResult:
"""
Call an OpenAPI tool handler directly.
@ -4436,13 +4554,18 @@ class MCPServerManager:
arguments: dict[str, Any],
server_name: str,
user_api_key_auth: Optional[UserAPIKeyAuth],
proxy_logging_obj: ProxyLogging,
proxy_logging_obj: ProxyLogging | None,
server: MCPServer,
raw_headers: Optional[dict[str, str]] = None,
) -> dict[str, Any]:
"""
Run pre-call checks and guardrail hooks for an MCP tool call.
Authorization runs unconditionally; only the guardrail hooks, which are
dispatched through ``proxy_logging_obj``, depend on a logger being
present. An absent logger must never be able to turn an authorization
decision into a no-op.
Returns a dict that may contain:
- "arguments": hook-modified tool arguments (only if changed)
- "extra_headers": headers injected by pre_mcp_call guardrail hooks
@ -4470,6 +4593,10 @@ class MCPServerManager:
server=server,
)
hook_result: dict[str, Any] = {}
if proxy_logging_obj is None:
return hook_result
# Extract incoming Bearer token from raw request headers so
# guardrails like MCPJWTSigner can verify + re-sign it (FR-5).
normalized_raw = {k.lower(): v for k, v in (raw_headers or {}).items()}
@ -4499,7 +4626,6 @@ class MCPServerManager:
# Convert to LLM format for existing guardrail compatibility
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(mcp_request_obj, pre_hook_kwargs)
hook_result: dict[str, Any] = {}
try:
# Use standard pre_call_hook
modified_data = await proxy_logging_obj.pre_call_hook(
@ -4529,7 +4655,7 @@ class MCPServerManager:
def _create_during_hook_task(
self,
name: str,
arguments: dict[str, Any],
arguments: _ToolArguments,
server_name_from_prefix: Optional[str],
user_api_key_auth: Optional[UserAPIKeyAuth],
proxy_logging_obj: ProxyLogging,
@ -4628,7 +4754,7 @@ class MCPServerManager:
self,
mcp_server: MCPServer,
original_tool_name: str,
arguments: dict[str, Any],
arguments: _ToolArguments,
tasks: list,
mcp_auth_header: Optional[str],
mcp_server_auth_headers: Optional[dict[str, dict[str, str]]],
@ -4982,7 +5108,7 @@ class MCPServerManager:
# shadow the resolver, double-resolving and hiding the per-server challenge.
return oauth2_headers
user_id = getattr(user_api_key_auth, "user_id", None)
user_id: str | None = getattr(user_api_key_auth, "user_id", None)
if not user_id:
return oauth2_headers
@ -5083,7 +5209,7 @@ class MCPServerManager:
self,
server_name: str,
name: str,
arguments: dict[str, Any],
arguments: _ToolArguments,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[dict[str, dict[str, str]]] = None,
@ -5125,19 +5251,17 @@ class MCPServerManager:
# Allow validation and modification of tool calls before execution
# Using standard pre_call_hook
#########################################################
hook_result: dict[str, Any] = {}
if proxy_logging_obj:
hook_result = await self.pre_call_tool_check(
name=name,
arguments=arguments,
server_name=server_name,
user_api_key_auth=user_api_key_auth,
proxy_logging_obj=proxy_logging_obj,
server=mcp_server,
raw_headers=raw_headers,
)
if "arguments" in hook_result:
arguments = hook_result["arguments"]
hook_result: dict[str, Any] = await self.pre_call_tool_check(
name=name,
arguments=arguments,
server_name=server_name,
user_api_key_auth=user_api_key_auth,
proxy_logging_obj=proxy_logging_obj,
server=mcp_server,
raw_headers=raw_headers,
)
if "arguments" in hook_result:
arguments = hook_result["arguments"]
# Prepare tasks for during hooks
tasks = []
@ -5324,7 +5448,7 @@ class MCPServerManager:
# Pending/rejected servers are excluded at the DB level so we never load them.
from litellm.proxy._experimental.mcp_server.db import LiteLLM_MCPServerTable
raw_rows = await MCPServerRepository(prisma_client).table.find_many(
raw_rows: Sequence[BaseModel] = await MCPServerRepository(prisma_client).table.find_many(
where={
"OR": [
{"approval_status": None},
@ -5830,7 +5954,7 @@ class MCPServerManager:
@staticmethod
def _env_vars_to_models(
env_vars: Optional[list[dict[str, Any]]],
env_vars: Optional[_EnvVarList],
) -> Optional[list[MCPEnvVar]]:
if env_vars is None:
return None

209
litellm/proxy/_experimental/mcp_server/sampling_handler.py
View file

@ -10,16 +10,23 @@ MCP Spec Reference:
https://modelcontextprotocol.io/specification/2025-11-25/client/sampling
"""
from typing import Any, Dict, List, Optional, Union
import typing
from collections.abc import Mapping, Sequence
from typing import Any, Dict, List, NamedTuple, Optional, Protocol, Union
if typing.TYPE_CHECKING:
from fastapi import Request
from mcp.client.session import ClientSession
from mcp.shared.context import RequestContext
from mcp.types import ContentBlock, SamplingMessageContentBlock
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.utils import ProxyLogging
from litellm._logging import verbose_logger
from fastapi import HTTPException
from litellm._logging import verbose_logger
# Guard imports that require the mcp package
try:
from mcp.types import (
@ -65,7 +72,7 @@ def _resolve_model_from_preferences(
import litellm
# Build list of available model names from proxy Router or litellm.model_list
available_model_names: list = []
available_model_names: list[str] = []
try:
from litellm.proxy.proxy_server import llm_router
@ -83,7 +90,7 @@ def _resolve_model_from_preferences(
available_model_names.append(entry)
if model_preferences and model_preferences.hints:
for hint in model_preferences.hints:
hint_name = getattr(hint, "name", None)
hint_name: str | None = getattr(hint, "name", None)
if not hint_name:
continue
# Try direct match first
@ -133,7 +140,7 @@ def _resolve_model_from_preferences(
)
return available_model_names[0]
# Last resort - use LiteLLM default or raise error
default_sampling_model = getattr(litellm, "default_mcp_sampling_model", None)
default_sampling_model: str | None = getattr(litellm, "default_mcp_sampling_model", None)
if default_sampling_model:
verbose_logger.debug(
"MCP sampling model resolution: using litellm.default_mcp_sampling_model='%s'",
@ -153,6 +160,13 @@ def _has_priorities(model_preferences: "ModelPreferences") -> bool:
)
class _ScoredModel(NamedTuple):
name: str
cost: float
max_output: float
output_tps: float
def _select_model_by_priority(
model_names: List[str],
model_preferences: "ModelPreferences",
@ -183,12 +197,12 @@ def _select_model_by_priority(
"""
import litellm as _litellm
cost_weight = getattr(model_preferences, "costPriority", None) or 0.0
speed_weight = getattr(model_preferences, "speedPriority", None) or 0.0
intel_weight = getattr(model_preferences, "intelligencePriority", None) or 0.0
cost_weight: float = getattr(model_preferences, "costPriority", None) or 0.0
speed_weight: float = getattr(model_preferences, "speedPriority", None) or 0.0
intel_weight: float = getattr(model_preferences, "intelligencePriority", None) or 0.0
# Gather raw metrics for each model
scored: List[Dict[str, Any]] = []
scored: list[_ScoredModel] = []
for name in model_names:
try:
info = _litellm.get_model_info(name)
@ -200,12 +214,12 @@ def _select_model_by_priority(
max_output = info.get("max_output_tokens") or info.get("max_tokens") or 0
output_tps = info.get("output_tokens_per_second") or 0.0
scored.append(
{
"name": name,
"cost": total_cost,
"max_output": max_output,
"output_tps": output_tps,
}
_ScoredModel(
name=name,
cost=total_cost,
max_output=max_output,
output_tps=output_tps,
)
)
if not scored:
@ -222,9 +236,9 @@ def _select_model_by_priority(
normed = [1.0 - n for n in normed]
return normed
costs = [s["cost"] for s in scored]
max_outputs = [float(s["max_output"]) for s in scored]
output_tps_values = [s["output_tps"] for s in scored]
costs = [s.cost for s in scored]
max_outputs = [float(s.max_output) for s in scored]
output_tps_values = [s.output_tps for s in scored]
# costPriority: lower cost → higher score (invert)
cost_scores = _normalise(costs, invert=True)
@ -243,7 +257,7 @@ def _select_model_by_priority(
score = cost_weight * cost_scores[i] + speed_weight * speed_scores[i] + intel_weight * intel_scores[i]
verbose_logger.debug(
"MCP priority scoring: model=%s cost_score=%.3f speed_score=%.3f intel_score=%.3f → weighted=%.3f",
entry["name"],
entry.name,
cost_scores[i],
speed_scores[i],
intel_scores[i],
@ -251,14 +265,14 @@ def _select_model_by_priority(
)
if score > best_score:
best_score = score
best_name = entry["name"]
best_name = entry.name
return best_name
def _convert_mcp_content_to_openai(
content: Any,
) -> Union[str, Dict[str, Any], List[Dict[str, Any]]]:
content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]",
) -> "str | dict[str, object] | list[dict[str, object]]":
"""
Convert MCP SamplingMessage content to OpenAI message content format.
Handles:
@ -283,7 +297,7 @@ def _convert_mcp_content_to_openai(
def _convert_single_content(
content: Any,
) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
) -> "dict[str, object] | list[dict[str, object]]":
"""Convert a single MCP content item to OpenAI format.
For text/image/audio content, returns a single content-part dict.
@ -339,7 +353,7 @@ def _convert_single_content(
# Marked so the message-level converter can emit it as a
# separate ``{"role": "tool", ...}`` message.
tool_use_id = getattr(content, "toolUseId", "")
nested_content = getattr(content, "content", [])
nested_content: Sequence[ContentBlock] = getattr(content, "content", [])
if isinstance(nested_content, list):
text_parts = [getattr(c, "text", str(c)) for c in nested_content if getattr(c, "type", None) == "text"]
result_text = "\n".join(text_parts) if text_parts else ""
@ -358,7 +372,7 @@ def _convert_single_content(
def _convert_mcp_messages_to_openai(
messages: List["SamplingMessage"],
system_prompt: Optional[str] = None,
) -> List[Dict[str, Any]]:
) -> "Sequence[Mapping[str, object]]":
"""
Convert MCP SamplingMessage list to OpenAI messages format.
MCP messages use:
@ -369,7 +383,7 @@ def _convert_mcp_messages_to_openai(
- role: "system" | "user" | "assistant" | "tool"
- content: str | list[content_part]
"""
openai_messages: List[Dict[str, Any]] = []
openai_messages: list[Mapping[str, object]] = []
# Add system prompt if provided
if system_prompt:
openai_messages.append({"role": "system", "content": system_prompt})
@ -380,7 +394,7 @@ def _convert_mcp_messages_to_openai(
if role == "assistant" and _has_tool_use(content):
tool_calls = _extract_tool_calls(content)
if tool_calls:
openai_msg: Dict[str, Any] = {
openai_msg: dict[str, object] = {
"role": "assistant",
"tool_calls": tool_calls,
}
@ -400,7 +414,7 @@ def _convert_mcp_messages_to_openai(
# tool_use / tool_result that slipped past the fast-path checks
# above (e.g. unexpected role, single non-list content).
converted = _convert_mcp_content_to_openai(content)
converted_parts = (
converted_parts: Sequence[Mapping[str, object]] = (
converted if isinstance(converted, list) else ([converted] if isinstance(converted, dict) else [])
)
@ -422,7 +436,7 @@ def _convert_mcp_messages_to_openai(
# Emit assistant message with tool_calls if any were found
if tool_call_markers:
openai_msg_tc: Dict[str, Any] = {
openai_msg_tc: dict[str, object] = {
"role": "assistant",
"tool_calls": tool_call_markers,
}
@ -442,21 +456,25 @@ def _convert_mcp_messages_to_openai(
return openai_messages
def _has_tool_use(content: Any) -> bool:
def _has_tool_use(content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]") -> bool:
"""Check if content contains ToolUseContent."""
if isinstance(content, list):
return any(getattr(c, "type", None) == "tool_use" for c in content)
return getattr(content, "type", None) == "tool_use"
content_type: str | None = getattr(content, "type", None)
return content_type == "tool_use"
def _has_tool_result(content: Any) -> bool:
def _has_tool_result(content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]") -> bool:
"""Check if content contains ToolResultContent."""
if isinstance(content, list):
return any(getattr(c, "type", None) == "tool_result" for c in content)
return getattr(content, "type", None) == "tool_result"
content_type: str | None = getattr(content, "type", None)
return content_type == "tool_result"
def _extract_tool_calls(content: Any) -> List[Dict[str, Any]]:
def _extract_tool_calls(
content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]",
) -> "Sequence[Mapping[str, object]]":
"""Extract OpenAI-format tool_calls from MCP ToolUseContent."""
import json
@ -477,7 +495,9 @@ def _extract_tool_calls(content: Any) -> List[Dict[str, Any]]:
return tool_calls
def _extract_text_parts(content: Any) -> Optional[str]:
def _extract_text_parts(
content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]",
) -> Optional[str]:
"""Extract text parts from mixed content."""
items = content if isinstance(content, list) else [content]
texts = []
@ -487,7 +507,9 @@ def _extract_text_parts(content: Any) -> Optional[str]:
return "\n".join(texts) if texts else None
def _extract_tool_results(content: Any) -> List[Dict[str, Any]]:
def _extract_tool_results(
content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]",
) -> "Sequence[Mapping[str, object]]":
"""Extract OpenAI-format tool messages from MCP ToolResultContent."""
items = content if isinstance(content, list) else [content]
results = []
@ -495,7 +517,7 @@ def _extract_tool_results(content: Any) -> List[Dict[str, Any]]:
if getattr(item, "type", None) == "tool_result":
tool_use_id = getattr(item, "toolUseId", "")
# Extract text from nested content
nested_content = getattr(item, "content", [])
nested_content: Sequence[ContentBlock] = getattr(item, "content", [])
if isinstance(nested_content, list):
text_parts = [getattr(c, "text", str(c)) for c in nested_content if getattr(c, "type", None) == "text"]
result_text = "\n".join(text_parts) if text_parts else ""
@ -513,7 +535,7 @@ def _extract_tool_results(content: Any) -> List[Dict[str, Any]]:
def _convert_mcp_tools_to_openai(
tools: Optional[List["Tool"]],
) -> Optional[List[Dict[str, Any]]]:
) -> "Sequence[Mapping[str, object]] | None":
"""
Convert MCP Tool definitions to OpenAI function calling format.
MCP Tool: {name, description, inputSchema}
@ -541,7 +563,7 @@ def _convert_mcp_tools_to_openai(
def _convert_mcp_tool_choice_to_openai(
tool_choice: Optional["ToolChoice"],
) -> Optional[Union[str, Dict[str, Any]]]:
) -> "str | None":
"""
Convert MCP ToolChoice to OpenAI tool_choice format.
MCP: {mode: "auto"} | {mode: "required"} | {mode: "none"}
@ -559,8 +581,32 @@ def _convert_mcp_tool_choice_to_openai(
return "auto"
class _SamplingResponseMessage(Protocol):
@property
def content(self) -> str | None: ...
@property
def tool_calls(self) -> Sequence[object] | None: ...
class _SamplingResponseChoice(Protocol):
@property
def message(self) -> _SamplingResponseMessage: ...
@property
def finish_reason(self) -> str | None: ...
class _SamplingCompletionResponse(Protocol):
@property
def choices(self) -> Sequence[_SamplingResponseChoice]: ...
@property
def model(self) -> str | None: ...
def _convert_openai_response_to_mcp_result(
response: Any,
response: _SamplingCompletionResponse,
model_name: str,
) -> Union["CreateMessageResult", "CreateMessageResultWithTools", "ErrorData"]:
"""
@ -593,12 +639,12 @@ def _convert_openai_response_to_mcp_result(
stop_reason = "maxTokens"
else:
stop_reason = "endTurn"
actual_model = getattr(response, "model", model_name) or model_name
actual_model: str = getattr(response, "model", model_name) or model_name
# Check if response has tool calls
tool_calls = getattr(message, "tool_calls", None)
if tool_calls:
# Build ToolUseContent items
content_parts: "List[Any]" = []
content_parts: list[SamplingMessageContentBlock] = []
# Include text content if present
if message.content:
content_parts.append(TextContent(type="text", text=message.content))
@ -636,7 +682,7 @@ def _convert_openai_response_to_mcp_result(
)
async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["ErrorData"]:
async def _check_model_access(model: str, user_api_key_auth: "UserAPIKeyAuth | None") -> Optional["ErrorData"]:
"""Enforce model-permission checks for MCP sampling requests.
Runs the same authorization checks as ``/chat/completions``:
@ -678,14 +724,14 @@ async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["E
try:
import litellm
from litellm.proxy.auth.auth_checks import (
_check_team_member_model_access,
can_key_call_model,
can_project_access_model,
can_team_access_model,
can_user_call_model,
can_project_access_model,
_check_team_member_model_access,
get_project_object,
get_team_object,
get_user_object,
get_project_object,
)
try:
@ -700,16 +746,20 @@ async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["E
llm_router=_llm_router,
)
_team_id = getattr(user_api_key_auth, "team_id", None)
_user_id = getattr(user_api_key_auth, "user_id", None)
_project_id = getattr(user_api_key_auth, "project_id", None)
_team_id: str | None = getattr(user_api_key_auth, "team_id", None)
_user_id: str | None = getattr(user_api_key_auth, "user_id", None)
_project_id: str | None = getattr(user_api_key_auth, "project_id", None)
try:
from litellm.proxy.proxy_server import (
prisma_client as _prisma_client,
user_api_key_cache as _user_api_key_cache,
)
from litellm.proxy.proxy_server import (
proxy_logging_obj as _proxy_logging_obj,
)
from litellm.proxy.proxy_server import (
user_api_key_cache as _user_api_key_cache,
)
except ImportError:
_prisma_client = None
_user_api_key_cache = None # type: ignore[assignment]
@ -799,7 +849,7 @@ async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["E
async def _run_budget_checks(
model: str,
user_api_key_auth: Any,
user_api_key_auth: "UserAPIKeyAuth",
raw_headers: Optional[Dict[str, str]] = None,
client_ip: Optional[str] = None,
) -> Optional["ErrorData"]:
@ -811,25 +861,33 @@ async def _run_budget_checks(
Returns None if all checks pass, or an ErrorData describing the denial.
"""
try:
from litellm.proxy.auth.auth_checks import common_checks
from litellm.proxy.proxy_server import (
general_settings,
llm_router as _llm_router,
prisma_client as _prisma_client,
proxy_logging_obj as _proxy_logging_obj,
user_api_key_cache as _user_api_key_cache,
)
import litellm
from litellm.proxy.auth.auth_checks import (
common_checks,
get_team_object,
get_user_object,
)
import litellm
from litellm.proxy.proxy_server import (
general_settings,
)
from litellm.proxy.proxy_server import (
llm_router as _llm_router,
)
from litellm.proxy.proxy_server import (
prisma_client as _prisma_client,
)
from litellm.proxy.proxy_server import (
proxy_logging_obj as _proxy_logging_obj,
)
from litellm.proxy.proxy_server import (
user_api_key_cache as _user_api_key_cache,
)
except ImportError as import_err:
verbose_logger.warning("MCP sampling: budget check imports unavailable: %s", import_err)
return None # Can't enforce budgets without the modules
_team_id = getattr(user_api_key_auth, "team_id", None)
_user_id = getattr(user_api_key_auth, "user_id", None)
_team_id: str | None = getattr(user_api_key_auth, "team_id", None)
_user_id: str | None = getattr(user_api_key_auth, "user_id", None)
team_obj = None
if _team_id and _prisma_client and _user_api_key_cache:
@ -889,7 +947,7 @@ async def _run_budget_checks(
# common_checks runs. _tag_max_budget_check inside common_checks only
# inspects request_body; without this pre-merge, header-supplied tags
# bypass per-tag budget enforcement (mirroring the regular auth path).
request_body: Dict[str, Any] = {"model": model}
request_body: dict[str, object] = {"model": model}
try:
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
@ -935,7 +993,7 @@ async def _run_budget_checks(
def _build_sampling_request(
raw_headers: Optional[Dict[str, str]] = None,
client_ip: Optional[str] = None,
) -> Any:
) -> "Request":
"""Build a synthetic FastAPI Request for sampling sub-calls.
Converts the original MCP connection's HTTP headers into ASGI
@ -961,7 +1019,7 @@ def _build_sampling_request(
from fastapi import Request
# --- Build ASGI headers ---
_scope_headers: list = [(b"content-type", b"application/json")]
_scope_headers: list[tuple[bytes, bytes]] = [(b"content-type", b"application/json")]
# Hop-by-hop headers that must NOT be forwarded into the
# synthetic request (they describe the original HTTP framing,
# not the logical request).
@ -1001,8 +1059,8 @@ def _build_sampling_request(
try:
import litellm.proxy.proxy_server as proxy_server
_proxy_host = getattr(proxy_server, "server_host", None)
_proxy_port = getattr(proxy_server, "server_port", None)
_proxy_host: str | None = getattr(proxy_server, "server_host", None)
_proxy_port: str | int | None = getattr(proxy_server, "server_port", None)
if _proxy_host:
_server_host = str(_proxy_host)
@ -1016,7 +1074,7 @@ def _build_sampling_request(
if client_ip:
_client_tuple = (client_ip, 0)
scope: Dict[str, Any] = {
scope: dict[str, object] = {
"type": "http",
"method": "POST",
"path": "/mcp/sampling/createMessage",
@ -1035,7 +1093,7 @@ def _build_sampling_request(
async def _build_completion_kwargs(
params: "CreateMessageRequestParams",
model: str,
user_api_key_auth: Any,
user_api_key_auth: "UserAPIKeyAuth",
raw_headers: Optional[Dict[str, str]],
client_ip: Optional[str],
) -> Dict[str, Any]:
@ -1078,7 +1136,7 @@ async def _build_completion_kwargs(
async def _run_guardrails_and_call_llm(
completion_kwargs: Dict[str, Any],
user_api_key_auth: Any,
user_api_key_auth: "UserAPIKeyAuth",
) -> Any:
try:
from litellm.proxy.proxy_server import proxy_logging_obj as _plo
@ -1111,10 +1169,10 @@ async def _run_guardrails_and_call_llm(
async def handle_sampling_create_message(
context: Any,
context: "RequestContext[ClientSession, object]",
params: "CreateMessageRequestParams",
default_model: Optional[str] = None,
user_api_key_auth: Optional[Any] = None,
user_api_key_auth: "UserAPIKeyAuth | None" = None,
raw_headers: Optional[Dict[str, str]] = None,
client_ip: Optional[str] = None,
) -> Union["CreateMessageResult", "CreateMessageResultWithTools", "ErrorData"]:
@ -1184,7 +1242,7 @@ async def handle_sampling_create_message(
client_ip=client_ip,
)
openai_messages = completion_kwargs["messages"]
openai_messages: Sequence[Mapping[str, object]] = completion_kwargs["messages"]
openai_tools = completion_kwargs.get("tools")
verbose_logger.debug(
"MCP sampling: calling litellm.acompletion with model=%s, num_messages=%d, has_tools=%s",
@ -1193,7 +1251,7 @@ async def handle_sampling_create_message(
bool(openai_tools),
)
response = await _run_guardrails_and_call_llm(
response: _SamplingCompletionResponse = await _run_guardrails_and_call_llm(
completion_kwargs=completion_kwargs,
user_api_key_auth=user_api_key_auth,
)
@ -1214,7 +1272,6 @@ async def handle_sampling_create_message(
RateLimitError,
ServiceUnavailableError,
)
from litellm.proxy._types import ProxyException
if isinstance(

48
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -2902,6 +2902,54 @@ if MCP_AVAILABLE:
# Deprecated: Local MCP Server Tool
#########################################################
else:
# Gate only what can actually dispatch. When the unprefixed name is
# not in the registry either, `_handle_local_mcp_tool` below reports
# 404 and nothing runs, so demanding a server here would turn every
# unknown tool name into a misleading 503.
if global_mcp_tool_registry.get_tool(original_tool_name) is not None:
# `mcp_server` is None here because the tool name is not in the
# tool -> server mapping, but the name still carries a prefix
# that the server-level check above compared against the
# caller's `allowed_mcp_servers` by exact `name`. So the named
# server is in that list and can carry the tool-level checks,
# even with the mapping cold. Resolve it from
# `allowed_mcp_servers` rather than the registry: the registry
# would happily return a server the caller holds no grant for,
# and matching anything other than `name` would accept a server
# the check never validated.
prefix_server = next(
(candidate for candidate in allowed_mcp_servers if candidate.name == server_name),
None,
)
if prefix_server is None:
# A non-empty prefix that passed the server-level check
# always matches here, so this arm only fires when the
# prefix was empty, which is exactly the case that check
# skips. Fail closed rather than dispatch with no server to
# evaluate a tool ceiling against.
raise HTTPException(
status_code=503,
detail=(
f"MCP server for tool '{original_tool_name}' is not available; "
"refusing to dispatch without authorization checks. "
"Retry once the server is registered."
),
)
from litellm.proxy.proxy_server import proxy_logging_obj
hook_result = await global_mcp_server_manager.pre_call_tool_check(
name=original_tool_name,
arguments=arguments,
server_name=server_name,
user_api_key_auth=user_api_key_auth,
proxy_logging_obj=proxy_logging_obj,
server=prefix_server,
raw_headers=raw_headers,
)
if "arguments" in hook_result:
arguments = hook_result["arguments"] # pyright: ignore[reportAny] # hook returns untyped args
local_content = await _handle_local_mcp_tool(original_tool_name, arguments)
response = CallToolResult(content=cast(Any, local_content), isError=False)

2
litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/404/index.html
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt
View file

@ -1,9 +1,9 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","/litellm-asset-prefix/_next/static/chunks/1ioy8obpggx93.js","/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","/litellm-asset-prefix/_next/static/chunks/1zr7rrk4wkmju.js","/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","/litellm-asset-prefix/_next/static/chunks/0fk0i3e2aixp7.js","/litellm-asset-prefix/_next/static/chunks/0zduf1gntl_f8.js","/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","/litellm-asset-prefix/_next/static/chunks/1a0bgy7kzrj91.js","/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","/litellm-asset-prefix/_next/static/chunks/0g8wwba6umbim.js","/litellm-asset-prefix/_next/static/chunks/1di-caw05k3tq.js","/litellm-asset-prefix/_next/static/chunks/00g6xfr4yow7h.js","/litellm-asset-prefix/_next/static/chunks/2kcxwg1mpncp6.js","/litellm-asset-prefix/_next/static/chunks/1fmx49l6q8v39.js","/litellm-asset-prefix/_next/static/chunks/0ww76lz_0cphv.js","/litellm-asset-prefix/_next/static/chunks/2uc2pi4ob086w.js","/litellm-asset-prefix/_next/static/chunks/2hu1vyy-5pv13.js","/litellm-asset-prefix/_next/static/chunks/2l25bmiiw9ixp.js","/litellm-asset-prefix/_next/static/chunks/1uz3jt-tj9lkf.js","/litellm-asset-prefix/_next/static/chunks/3drq2_k-jeio2.js","/litellm-asset-prefix/_next/static/chunks/0dsiq_ok1yngk.js","/litellm-asset-prefix/_next/static/chunks/3srzg1la93pwv.js","/litellm-asset-prefix/_next/static/chunks/17nqbxvhztf3k.js","/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","/litellm-asset-prefix/_next/static/chunks/0kap_rdm2-lem.js","/litellm-asset-prefix/_next/static/chunks/09l_m9l1emin2.js","/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","/litellm-asset-prefix/_next/static/chunks/323l6h8s7ahat.js","/litellm-asset-prefix/_next/static/chunks/112n0hv3cc2rg.js","/litellm-asset-prefix/_next/static/chunks/199uwr871eene.js","/litellm-asset-prefix/_next/static/chunks/105643dvf00hu.js","/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","/litellm-asset-prefix/_next/static/chunks/22iools_e0k44.js","/litellm-asset-prefix/_next/static/chunks/0am68mi9t9cb6.js","/litellm-asset-prefix/_next/static/chunks/23-g73xaw3kap.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"OutletBoundary"]
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","/litellm-asset-prefix/_next/static/chunks/28md7sjkucknx.js","/litellm-asset-prefix/_next/static/chunks/2x96scis66zmk.js","/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","/litellm-asset-prefix/_next/static/chunks/1azbeyb626rh5.js","/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","/litellm-asset-prefix/_next/static/chunks/0vvnul8uez9kj.js","/litellm-asset-prefix/_next/static/chunks/0g8wwba6umbim.js","/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","/litellm-asset-prefix/_next/static/chunks/0fk0i3e2aixp7.js","/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","/litellm-asset-prefix/_next/static/chunks/17gy9d71tfqhd.js","/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","/litellm-asset-prefix/_next/static/chunks/0kx52ovlpa34x.js","/litellm-asset-prefix/_next/static/chunks/3drq2_k-jeio2.js","/litellm-asset-prefix/_next/static/chunks/2vnpyhxoamx0f.js","/litellm-asset-prefix/_next/static/chunks/323l6h8s7ahat.js","/litellm-asset-prefix/_next/static/chunks/3ib18qm2ox61z.js","/litellm-asset-prefix/_next/static/chunks/112n0hv3cc2rg.js","/litellm-asset-prefix/_next/static/chunks/199uwr871eene.js","/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","/litellm-asset-prefix/_next/static/chunks/1abwfud5uqxxq.js","/litellm-asset-prefix/_next/static/chunks/1uz3jt-tj9lkf.js","/litellm-asset-prefix/_next/static/chunks/2hu1vyy-5pv13.js","/litellm-asset-prefix/_next/static/chunks/105643dvf00hu.js","/litellm-asset-prefix/_next/static/chunks/23-g73xaw3kap.js","/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","/litellm-asset-prefix/_next/static/chunks/09l_m9l1emin2.js","/litellm-asset-prefix/_next/static/chunks/31cwj7vkk3gfz.js","/litellm-asset-prefix/_next/static/chunks/111jj26rg98nb.js","/litellm-asset-prefix/_next/static/chunks/3pu9plov1btip.js","/litellm-asset-prefix/_next/static/chunks/1l1182ye657-m.js","/litellm-asset-prefix/_next/static/chunks/1lndy6n7cwvrq.js","/litellm-asset-prefix/_next/static/chunks/2kt_m68ln2fyr.js","/litellm-asset-prefix/_next/static/chunks/0onjeur4drmh-.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"OutletBoundary"]
7:"$Sreact.suspense"
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/00g6xfr4yow7h.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2kcxwg1mpncp6.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1fmx49l6q8v39.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0ww76lz_0cphv.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2uc2pi4ob086w.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2hu1vyy-5pv13.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/2l25bmiiw9ixp.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1uz3jt-tj9lkf.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3drq2_k-jeio2.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0dsiq_ok1yngk.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/3srzg1la93pwv.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/17nqbxvhztf3k.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0kap_rdm2-lem.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/09l_m9l1emin2.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/323l6h8s7ahat.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/112n0hv3cc2rg.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/199uwr871eene.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/105643dvf00hu.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/22iools_e0k44.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/0am68mi9t9cb6.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/23-g73xaw3kap.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"qXutWsQW5C1Pf62WxTkEI"}
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0kx52ovlpa34x.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3drq2_k-jeio2.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2vnpyhxoamx0f.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/323l6h8s7ahat.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3ib18qm2ox61z.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/112n0hv3cc2rg.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/199uwr871eene.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1abwfud5uqxxq.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/1uz3jt-tj9lkf.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/2hu1vyy-5pv13.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/105643dvf00hu.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/23-g73xaw3kap.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/09l_m9l1emin2.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/31cwj7vkk3gfz.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/111jj26rg98nb.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/3pu9plov1btip.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/1l1182ye657-m.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/1lndy6n7cwvrq.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/2kt_m68ln2fyr.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/0onjeur4drmh-.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"8bC_aTV0H1nUtrFDXkfaB"}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
8:null

10
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt
View file

@ -1,7 +1,7 @@
1:"$Sreact.fragment"
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","/litellm-asset-prefix/_next/static/chunks/1ioy8obpggx93.js","/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","/litellm-asset-prefix/_next/static/chunks/1zr7rrk4wkmju.js","/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","/litellm-asset-prefix/_next/static/chunks/0fk0i3e2aixp7.js","/litellm-asset-prefix/_next/static/chunks/0zduf1gntl_f8.js","/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","/litellm-asset-prefix/_next/static/chunks/1a0bgy7kzrj91.js","/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","/litellm-asset-prefix/_next/static/chunks/0g8wwba6umbim.js","/litellm-asset-prefix/_next/static/chunks/1di-caw05k3tq.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1ioy8obpggx93.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1zr7rrk4wkmju.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0fk0i3e2aixp7.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0zduf1gntl_f8.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/1a0bgy7kzrj91.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0g8wwba6umbim.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/1di-caw05k3tq.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"qXutWsQW5C1Pf62WxTkEI"}
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","/litellm-asset-prefix/_next/static/chunks/28md7sjkucknx.js","/litellm-asset-prefix/_next/static/chunks/2x96scis66zmk.js","/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","/litellm-asset-prefix/_next/static/chunks/1azbeyb626rh5.js","/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","/litellm-asset-prefix/_next/static/chunks/0vvnul8uez9kj.js","/litellm-asset-prefix/_next/static/chunks/0g8wwba6umbim.js","/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","/litellm-asset-prefix/_next/static/chunks/0fk0i3e2aixp7.js","/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","/litellm-asset-prefix/_next/static/chunks/17gy9d71tfqhd.js","/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/28md7sjkucknx.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2x96scis66zmk.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/1azbeyb626rh5.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0vvnul8uez9kj.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0g8wwba6umbim.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0fk0i3e2aixp7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/17gy9d71tfqhd.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"8bC_aTV0H1nUtrFDXkfaB"}
6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

36
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -1,6 +1,6 @@
1:"$Sreact.fragment"
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"MetadataBoundary"]
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"qXutWsQW5C1Pf62WxTkEI"}
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"8bC_aTV0H1nUtrFDXkfaB"}

14
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -1,9 +1,9 @@
1:"$Sreact.fragment"
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
4:I[557951,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"AuthProvider"]
5:I[339756,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
6:I[837457,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
4:I[557951,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"AuthProvider"]
5:I[339756,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
6:I[837457,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/3254j4ut19q6_.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3254j4ut19q6_.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"qXutWsQW5C1Pf62WxTkEI"}
:HL["/litellm-asset-prefix/_next/static/chunks/169bqf_mz3j8m.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/169bqf_mz3j8m.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/32_ulchi2_aad.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"8bC_aTV0H1nUtrFDXkfaB"}

4
litellm/proxy/_experimental/out/__next._tree.txt
View file

@ -1,4 +1,4 @@
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/3254j4ut19q6_.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/169bqf_mz3j8m.css","style"]
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"qXutWsQW5C1Pf62WxTkEI"}
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"8bC_aTV0H1nUtrFDXkfaB"}

0
litellm/proxy/_experimental/out/_next/static/qXutWsQW5C1Pf62WxTkEI/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/8bC_aTV0H1nUtrFDXkfaB/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/qXutWsQW5C1Pf62WxTkEI/_clientMiddlewareManifest.js → litellm/proxy/_experimental/out/_next/static/8bC_aTV0H1nUtrFDXkfaB/_clientMiddlewareManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/qXutWsQW5C1Pf62WxTkEI/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/8bC_aTV0H1nUtrFDXkfaB/_ssgManifest.js
View file

1
litellm/proxy/_experimental/out/_next/static/chunks/00g6xfr4yow7h.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/00lwtxl1k_z8t.js Normal file
View file

File diff suppressed because one or more lines are too long

4
litellm/proxy/_experimental/out/_next/static/chunks/10-c3gjiv7yt0.js → litellm/proxy/_experimental/out/_next/static/chunks/011as3ct2u0nu.js
View file

File diff suppressed because one or more lines are too long

10
litellm/proxy/_experimental/out/_next/static/chunks/02-alpsjfp5b7.js Normal file
View file

File diff suppressed because one or more lines are too long

7
litellm/proxy/_experimental/out/_next/static/chunks/025ocjcb8e481.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/03c3h-nx-fb3y.js Normal file
View file

File diff suppressed because one or more lines are too long

10
litellm/proxy/_experimental/out/_next/static/chunks/03kaz3d0v3z45.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/046q5lwe95zp6.js
View file

File diff suppressed because one or more lines are too long

31
litellm/proxy/_experimental/out/_next/static/chunks/048wkdcpsnwne.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/04y2hqzy08peg.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/054z32giulaw7.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/058j9m4b8p4wx.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/05id71gg6oywc.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/05ttqlxo9w0ow.js Normal file
View file

File diff suppressed because one or more lines are too long

7
litellm/proxy/_experimental/out/_next/static/chunks/06hxe45fjy7x7.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/06q8aep867ss7.js
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/_next/static/chunks/06sx0oh8weeph.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/08goggic_ad66.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/08k6jolcrw-uw.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/08yk0x-hh3ydk.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0965-angwdvwe.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0_mw8gm-qowti.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0_vlfpb8phl0v.js Normal file
View file

File diff suppressed because one or more lines are too long

10
litellm/proxy/_experimental/out/_next/static/chunks/0a-3zns09ja98.js
View file

File diff suppressed because one or more lines are too long

89
litellm/proxy/_experimental/out/_next/static/chunks/0ak2vacq91c7k.js Normal file
View file

File diff suppressed because one or more lines are too long

7
litellm/proxy/_experimental/out/_next/static/chunks/0ald9tfsbz2e-.js
View file

File diff suppressed because one or more lines are too long

7
litellm/proxy/_experimental/out/_next/static/chunks/0am68mi9t9cb6.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/3fqu7hpcrtg67.js → litellm/proxy/_experimental/out/_next/static/chunks/0b4c0ak_j76h0.js
View file

File diff suppressed because one or more lines are too long

7
litellm/proxy/_experimental/out/_next/static/chunks/0bzsgiuwi1q0e.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0cgtkk6qelb_j.js
View file

File diff suppressed because one or more lines are too long

10
litellm/proxy/_experimental/out/_next/static/chunks/0cj588tfl8vcq.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/0d398pudg-p7u.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0d6y38dwy2fvp.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0dh09yfknpuy3.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/0dsiq_ok1yngk.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0e3vwdd43gm8b.js Normal file
View file

File diff suppressed because one or more lines are too long

7
litellm/proxy/_experimental/out/_next/static/chunks/0eamb3kk74kws.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0euektnkbx78f.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0f_bhylcfohcm.js
View file

File diff suppressed because one or more lines are too long

4
litellm/proxy/_experimental/out/_next/static/chunks/3jgsbd_1fz8l8.js → litellm/proxy/_experimental/out/_next/static/chunks/0gylheyn-59ow.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0i25zatajbma2.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0jjbikxye1xv_.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/3e4fipm_mrl-n.js → litellm/proxy/_experimental/out/_next/static/chunks/0jl9haj4wjvj5.js
View file

File diff suppressed because one or more lines are too long

13
litellm/proxy/_experimental/out/_next/static/chunks/0k-74wqsm8dzt.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/0kap_rdm2-lem.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/007c8g8hmd9qz.js → litellm/proxy/_experimental/out/_next/static/chunks/0kpote6n3ff9a.js
View file

File diff suppressed because one or more lines are too long

7
litellm/proxy/_experimental/out/_next/static/chunks/0kx52ovlpa34x.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0lf6n96uy4q27.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0m-x8i06te864.js
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more