scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 81

Merge remote-tracking branch 'origin/litellm_internal_staging' into feature/fallback_streaming

Browse source
This commit is contained in:
lorenzbaraldi 2026-05-21 17:22:29 +02:00
commit 03c57ac0a1
538 changed files with 18305 additions and 1515 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
.circleci/config.yml
View file

@ -409,14 +409,25 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
mv coverage.xml auth_ui_unit_tests_coverage.xml
mv .coverage auth_ui_unit_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
@ -493,13 +504,24 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
mv coverage.xml router_unit_tests_coverage.xml
mv .coverage router_unit_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- router_unit_tests_coverage.xml
- router_unit_tests_coverage
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
docker:
- *python312_image
@ -2280,10 +2302,11 @@ jobs:
- run:
name: Combine Coverage
command: |
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
uv tool run --from 'coverage[toml]==7.10.6' coverage xml
- codecov/upload:
file: ./coverage.xml
flags: circleci
ui_build:
docker:
@ -2669,6 +2692,8 @@ workflows:
- local_testing_part1
- local_testing_part2
- litellm_assistants_api_testing
- litellm_router_unit_testing
- auth_ui_unit_tests
- db_migration_disable_update_check:
requires:
- build_docker_database_image

1
.github/workflows/_test-unit-base.yml vendored
View file

@ -132,4 +132,5 @@ jobs:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false

1
.github/workflows/_test-unit-services-base.yml vendored
View file

@ -186,4 +186,5 @@ jobs:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false

38
.github/workflows/test-unit-caching-redis.yml vendored
View file

@ -1,38 +0,0 @@
name: "Unit Tests: Caching (Redis)"
# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
# This prevents external PRs from accessing Redis credentials.
on:
push:
branches: [main, "litellm_*"]
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
caching-redis:
uses: ./.github/workflows/_test-unit-services-base.yml
with:
# Redis-only tests that do NOT require provider API keys.
# Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
# test_router_caching.py) are in Phase 3 integration workflows.
test-path: >-
tests/local_testing/test_dual_cache.py
tests/local_testing/test_redis_batch_optimizations.py
tests/local_testing/test_router_utils.py
workers: 2
reruns: 2
timeout-minutes: 20
enable-redis: true
enable-postgres: false
secrets:
REDIS_HOST: ${{ secrets.REDIS_HOST }}
REDIS_PORT: ${{ secrets.REDIS_PORT }}
REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
DATABASE_URL: ${{ secrets.DATABASE_URL }}
POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

19
.gitignore vendored
View file

@ -101,4 +101,23 @@ STABILIZATION_TODO.md
**/*.storageState.json
**/coverage
test-config
# ---------- Terraform ----------
# Provider binaries + module cache — regenerated by `terraform init`.
**/.terraform/
# State files often contain secrets (DB passwords, API keys snapshotted from
# data sources). Keep state in a remote backend, never in git.
*.tfstate
*.tfstate.*
*.tfstate.backup
# Plan files can also contain sensitive values (variables in plaintext).
*.tfplan
# User-specific variable inputs — example files (terraform.tfvars.example) are
# tracked because they end in .example, which doesn't match the glob below.
*.tfvars
*.auto.tfvars
crash.log
crash.*.log
# .terraform.lock.hcl is intentionally NOT ignored — it pins provider versions
# and should be committed.
.vscode

2
README.md
View file

@ -292,7 +292,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
| [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | | | | | | | |
| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [Databricks (`databricks`)](https://docs.litellm.ai/docs/providers/databricks) | ✅ | ✅ | ✅ | | | | | | | |
| [DataRobot (`datarobot`)](https://docs.litellm.ai/docs/providers/datarobot) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ | | | ✅ | | | | |

83
backend/Dockerfile Normal file
View file

@ -0,0 +1,83 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
# BuildKit cache mount (different filesystem).
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
# silently pulling a managed interpreter.
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_COMPILE_BYTECODE=1 \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}"
# Stage 1 — install dependencies only.
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
# Stage 2 — copy source and install the project + workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
RUN mkdir -p /home/nonroot && \
HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
chown -R nonroot:nonroot /home/nonroot/.cache
# ---------- Runtime ----------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
# /home/nonroot. We run the backend as that user
WORKDIR /app
ENV HOME=/home/nonroot \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
USER nonroot
EXPOSE 4001/tcp
ENTRYPOINT ["uvicorn", "backend.main:app"]
CMD ["--host", "0.0.0.0", "--port", "4001"]

51
backend/main.py Normal file
View file

@ -0,0 +1,51 @@
"""UI backend entrypoint.
Reuses the existing FastAPI app from `litellm.proxy.proxy_server` and trims its
route table to just the management/admin surface used by the dashboard. Purely
additive — no existing module is modified.
Run with:
uvicorn backend.main:app --host 0.0.0.0 --port 4001
"""
from contextlib import asynccontextmanager
from fastapi.routing import Mount
# See gateway/main.py for why we assemble DATABASE_URL(s) here before
# importing proxy_server.
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
DatabaseURLSettings.from_env().apply_to_env()
from litellm.proxy.proxy_server import app
from backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES
def _is_backend_route(route) -> bool:
"""Keep the route on the backend if its path is in the management surface."""
path = getattr(route, "path", None)
if path is None:
return False
if isinstance(route, Mount):
# Static UI mounts are served by the dedicated UI container, not here.
return False
if path in BACKEND_EXACT_PATHS:
return True
return any(path.startswith(prefix) for prefix in BACKEND_PATH_PREFIXES)
# See gateway/main.py for why the trim runs inside the lifespan instead of at
# module scope.
_proxy_lifespan = app.router.lifespan_context
@asynccontextmanager
async def _backend_lifespan(app_):
async with _proxy_lifespan(app_):
app_.router.routes = [r for r in app_.router.routes if _is_backend_route(r)]
yield
app.router.lifespan_context = _backend_lifespan

0
backend/routes/__init__.py Normal file
View file

135
backend/routes/allowlist.py Normal file
View file

@ -0,0 +1,135 @@
"""Path allowlist for the UI backend (control plane) component.
The backend exposes management/admin endpoints consumed by the UI: keys, users,
teams, orgs, customers, budgets, tags, workflows, model management, spend &
analytics, settings (router/cache/cost-tracking/fallbacks), SSO/onboarding,
audit logs, debug, enterprise admin, and UI bootstrap helpers (logo, favicon,
.well-known config).
Anything LLM data-plane is dropped — those run on the gateway component.
"""
BACKEND_PATH_PREFIXES: tuple[str, ...] = (
# Identity / access
"/key/",
"/v2/key/",
"/user/",
"/v2/user/",
"/team/",
"/v2/team/",
"/organization/",
"/customer/",
"/end_user/",
"/sso/",
"/login",
"/v2/login",
"/v3/login",
"/logout",
"/token",
"/onboarding/",
"/audit",
"/oauth/",
"/invitation/",
"/jwt/",
# Models & routing config
"/model/",
"/v1/model/info",
"/v2/model/",
"/model_group",
"/model_access_group/",
"/model_hub/",
"/v1/access_group",
"/access_group/",
"/router/",
"/router_settings",
"/adaptive_router/",
"/fallback",
"/fallbacks",
"/cache_settings",
"/cost_tracking",
"/cost/",
"/credentials",
"/credential",
"/provider/budgets",
# Tools / agents (registry & policy admin)
"/v1/tool/",
"/v1/agents",
# Guardrails admin
"/v2/guardrails/",
# MCP server admin + BYOK OAuth flow (UI-initiated) + dynamic per-server endpoints
"/v1/mcp/",
"/test/",
"/{mcp_server_name}/",
# Budgets / tags / workflows / memory mgmt
"/budget/",
"/tag/",
"/workflow/",
"/v1/workflows/",
"/project/",
"/memory/",
"/mcp/",
# Spend / analytics
"/spend/",
"/analytics/",
"/global/",
"/user_agent",
"/usage/",
"/daily/",
# CloudZero cost-export admin (init / settings / export / dry-run / delete)
"/cloudzero/",
# Caching admin
"/cache/",
"/caching/",
# Callbacks / hooks
"/active/callbacks",
"/callbacks",
"/team_callback",
# Alerting / email / IP allowlist
"/alerting/",
"/email/",
"/add/allowed_ip",
"/delete/allowed_ip",
"/get/",
# Enterprise admin
"/enterprise/",
# Debug / config / profiling
"/debug/",
"/config/",
"/memory-usage-in-mem-cache",
"/otel-spans",
"/lazy/",
"/in_product_nudges",
# Admin reload / schedule
"/reload/",
"/schedule/",
"/settings",
"/update/",
"/upload/",
# Dev / admin utilities
"/utils/",
# UI bootstrap helpers (assets the dashboard fetches)
"/get_logo_url",
"/get_image",
"/get_favicon",
"/.well-known/",
"/litellm/.well-known/",
"/ui_discovery/",
"/ui-config",
"/sso_settings",
"/public/",
"/robots.txt",
# Health (k8s probes)
"/health",
)
BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
{
"/",
"/routes",
"/openapi.json",
"/docs",
"/docs/oauth2-redirect",
"/redoc",
"/fallback/login",
}
)

10
codecov.yaml
View file

@ -3,6 +3,16 @@ codecov:
notify:
wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
# Uploads are flagged per workflow/shard (GHA) or "circleci". carryforward makes
# a re-upload of a flag replace its prior session instead of accumulating a
# conflicting one, and lets a commit reuse a flag from its parent when that flag
# was not re-uploaded. Required because the same commit can receive the
# push-triggered workflows more than once (re-runs / branches cut at the same
# SHA); flagless overlapping sessions made Codecov drop the largest files.
flag_management:
default_rules:
carryforward: true
component_management:
individual_components:
- component_id: "Router"

4
deploy/charts/litellm-helm/templates/hpa.yaml
View file

@ -12,6 +12,10 @@ spec:
name: {{ include "litellm.fullname" . }}
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
{{- if .Values.autoscaling.behavior }}
behavior:
{{- toYaml .Values.autoscaling.behavior | nindent 4 }}
{{- end }}
metrics:
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource

36
deploy/charts/litellm-helm/tests/hpa_tests.yaml Normal file
View file

@ -0,0 +1,36 @@
suite: "hpa with behavior"
templates:
- hpa.yaml
tests:
- it: "renders behavior when set"
set:
autoscaling.enabled: true
autoscaling.behavior:
scaleUp:
stabilizationWindowSeconds: 60
policies:
- type: Pods
value: 2
periodSeconds: 60
scaleDown:
stabilizationWindowSeconds: 90
policies:
- type: Pods
value: 1
periodSeconds: 60
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 }
- equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 }
---
suite: "hpa without behavior"
templates:
- hpa.yaml
tests:
- it: "does not render behavior when not set"
set:
autoscaling.enabled: true
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- isNull: { path: spec.behavior }

1
deploy/charts/litellm-helm/values.yaml
View file

@ -184,6 +184,7 @@ autoscaling:
maxReplicas: 100
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80
# behavior: {}
# Autoscaling with keda is mutually exclusive with hpa
keda:

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.40"
version = "0.1.41"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.40"
version = "0.1.41"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

83
gateway/Dockerfile Normal file
View file

@ -0,0 +1,83 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
# BuildKit cache mount (different filesystem).
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
# silently pulling a managed interpreter.
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_COMPILE_BYTECODE=1 \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}"
# Stage 1 — install dependencies only.
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
# Stage 2 — copy source and install the project + workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
RUN mkdir -p /home/nonroot && \
HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
chown -R nonroot:nonroot /home/nonroot/.cache
# ---------- Runtime ----------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
# /home/nonroot. We run the proxy as that user.
WORKDIR /app
ENV HOME=/home/nonroot \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
USER nonroot
EXPOSE 4000/tcp
ENTRYPOINT ["sh", "-c", "exec uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
CMD ["--host", "0.0.0.0", "--port", "4000"]

59
gateway/main.py Normal file
View file

@ -0,0 +1,59 @@
"""Gateway entrypoint.
Reuses the existing FastAPI app from `litellm.proxy.proxy_server` and trims its
route table to just the LLM data-plane surface. The trim is purely additive —
no existing module is modified, the full app continues to work via the legacy
entrypoint (`litellm.proxy.proxy_server:app`).
Run with:
uvicorn gateway.main:app --host 0.0.0.0 --port 4000
"""
from contextlib import asynccontextmanager
from fastapi.routing import Mount
# Assemble DATABASE_URL (+ DATABASE_URL_READ_REPLICA) from the discrete
# DATABASE_* env vars before proxy_server imports spin up Prisma. Handles
# both IAM (mint a token) and password auth, writer and reader. The standard
# CLI flow does this in proxy_cli.py; we bypass proxy_cli by uvicorn'ing the
# app directly, so without this Prisma initializes with the placeholder URL
# and every DB-needing endpoint returns "Database not connected".
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
DatabaseURLSettings.from_env().apply_to_env()
from litellm.proxy.proxy_server import app
from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES
def _is_gateway_route(route) -> bool:
"""Keep the route on the gateway if its path is in the LLM data-plane surface."""
path = getattr(route, "path", None)
if path is None:
return False
if isinstance(route, Mount):
# Gateway never serves the static UI or its asset bundles.
return False
if path in GATEWAY_EXACT_PATHS:
return True
return any(path.startswith(prefix) for prefix in GATEWAY_PATH_PREFIXES)
# Wrap proxy_server's existing lifespan so the route trim runs *after* its
# startup hooks (and any plugin code those hooks load) have had a chance to
# register routes. A module-load filter would miss routes added during
# startup; running inside the lifespan, after the inner __aenter__, catches
# them while still completing before uvicorn opens the listener.
_proxy_lifespan = app.router.lifespan_context
@asynccontextmanager
async def _gateway_lifespan(app_):
async with _proxy_lifespan(app_):
app_.router.routes = [r for r in app_.router.routes if _is_gateway_route(r)]
yield
app.router.lifespan_context = _gateway_lifespan

0
gateway/routes/__init__.py Normal file
View file

121
gateway/routes/allowlist.py Normal file
View file

@ -0,0 +1,121 @@
"""Path allowlist for the gateway component.
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
responses, vector stores, passthrough providers, realtime websockets, MCP
tool-call endpoints, and operational endpoints (/health, /metrics).
Any path not listed here is dropped from the gateway process so management/UI
endpoints don't ride on the same pods.
Versioned data-plane paths are enumerated explicitly rather than allowing a
blanket `/v1/` or `/v2/` prefix — those broad prefixes would otherwise also
match management routes like `/v1/access_group`, `/v1/tool/{tool_name}/logs`,
`/v2/key/info`, etc.
"""
GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
# OpenAI-compatible data-plane surface (versioned + unversioned)
"/v1/chat/",
"/chat/",
"/v1/completions",
"/completions",
"/v1/embeddings",
"/embeddings",
"/v1/moderations",
"/moderations",
"/v1/audio/",
"/audio/",
"/v1/images/",
"/images/",
"/v1/files",
"/files",
"/v1/batches",
"/batches",
"/v1/fine_tuning/",
"/fine_tuning/",
"/v1/fine-tuning/",
"/fine-tuning/",
"/v1/responses",
"/responses",
"/v1/threads",
"/threads",
"/v1/assistants",
"/assistants",
"/v1/vector_stores",
"/vector_stores",
"/v1/indexes",
"/v1/models",
"/models",
"/openai/",
"/engines/",
# Anthropic / agentic data-plane surface
"/v1/messages",
"/messages",
"/v1/skills",
"/v1/a2a/",
# LiteLLM-native LLM surface
"/v1/rerank",
"/v2/rerank",
"/rerank",
"/v1/ocr",
"/ocr",
"/v1/rag/",
"/rag/",
"/v1/video",
"/v1/videos",
"/video/",
"/videos",
"/v1/search",
"/search",
"/v1/containers",
"/containers",
"/v1/evals",
"/v1/memory",
"/queue/chat/",
# Google data plane (v1beta is the Google AI Studio version)
"/v1beta/",
"/interactions",
# Provider passthrough
"/anthropic/",
"/azure/",
"/azure_ai/",
"/aws/",
"/bedrock/",
"/cohere/",
"/gemini/",
"/google/",
"/vertex_ai/",
"/vertex-ai/",
"/assemblyai/",
"/eu.assemblyai/",
"/langfuse/",
"/vllm/",
"/mistral/",
"/groq/",
"/voyage/",
"/cursor/",
"/milvus/",
"/openai_passthrough/",
# Dynamic provider / toolset passthrough (path templates)
"/{provider}/",
"/toolset/",
# Realtime / streaming
"/v1/realtime",
"/realtime",
# Health & ops
"/health",
"/metrics",
)
GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
{
"/",
"/routes",
"/openapi.json",
"/docs",
"/docs/oauth2-redirect",
"/redoc",
"/test",
}
)

8
helm/litellm/Chart.yaml Normal file
View file

@ -0,0 +1,8 @@
apiVersion: v2
name: litellm
description: LiteLLM componentized — gateway, UI backend, and UI as separate services
type: application
version: 0.1.0
appVersion: "0.1.0"
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"

49
helm/litellm/templates/NOTES.txt Normal file
View file

@ -0,0 +1,49 @@
LiteLLM componentized — release {{ .Release.Name }} in namespace {{ .Release.Namespace }}.
Components:
{{- if .Values.gateway.enabled }}
- gateway : Service {{ include "litellm.gateway.fullname" . }} on port {{ .Values.gateway.service.port }}
{{- end }}
{{- if .Values.backend.enabled }}
- backend : Service {{ include "litellm.backend.fullname" . }} on port {{ .Values.backend.service.port }}
{{- end }}
{{- if .Values.ui.enabled }}
- ui : Service {{ include "litellm.ui.fullname" . }} on port {{ .Values.ui.service.port }}
{{- end }}
Port-forward examples:
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.gateway.fullname" . }} {{ .Values.gateway.service.port }}
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.backend.fullname" . }} {{ .Values.backend.service.port }}
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.ui.fullname" . }} {{ .Values.ui.service.port }}
Reminders:
- Sensitive values come from Secret references only. Before installing, set:
- masterKey.secretName (Secret with the proxy master key)
- database.writer.{host,port,dbname} (writer connection pieces)
- database.writer.passwordSecret.{name,usernameKey,passwordKey}
(Secret holding the writer DB username + password)
- database.writer.useIAMAuth: true (optional — chart sets IAM_TOKEN_DB_AUTH=true and
omits DATABASE_PASSWORD / DATABASE_URL so the proxy
mints the URL from an IAM token at startup)
- database.reader.host (optional — enables read-replica routing; reader
.passwordSecret.name is required when set, unless
.useIAMAuth is true)
- database.reader.useIAMAuth: true (optional, requires database.writer.useIAMAuth: true —
chart emits DATABASE_*_READ_REPLICA env vars and
omits DATABASE_PASSWORD_READ_REPLICA /
DATABASE_URL_READ_REPLICA so the proxy mints the
reader URL from an IAM token at startup)
- redis.passwordSecret.name (optional — set when redis.host is provided and the
cache requires auth)
- redis.cluster: true (optional — chart sets REDIS_CLUSTER_NODES from
redis.host / redis.port so the proxy's Cache()
constructs a RedisClusterCache; the cluster client
discovers remaining nodes from CLUSTER SLOTS)
- Per-component extras (gateway / backend / ui):
- {component}.extraEnv / envConfigMaps / envSecrets (the latter two are lists of resource names →
envFrom configMapRef / secretRef)
- {component}.logLevel (renders as LITELLM_LOG)
- gateway.config.proxy_config (rendered into a ConfigMap and mounted at
/app/config/config.yaml; gateway reads it via
CONFIG_FILE_PATH)
- Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend.

245
helm/litellm/templates/_helpers.tpl Normal file
View file

@ -0,0 +1,245 @@
{{/*
Common naming + label helpers shared by gateway, backend, and ui templates.
*/}}
{{- define "litellm.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.gateway.fullname" -}}
{{- printf "%s-gateway" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.backend.fullname" -}}
{{- printf "%s-backend" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.ui.fullname" -}}
{{- printf "%s-ui" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.commonLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
{{- end -}}
{{/*
Per-component selector labels — used in both Service selectors and Deployment matchLabels.
*/}}
{{- define "litellm.gateway.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: gateway
{{- end -}}
{{- define "litellm.backend.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: backend
{{- end -}}
{{- define "litellm.ui.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: ui
{{- end -}}
{{/*
Shared ServiceAccount name used by all three component Deployments. When
`serviceAccount.create` is true and `serviceAccount.name` is empty, default
to the chart fullname. When `create` is false, fall back to the provided
name or the namespace's `default` SA.
*/}}
{{- define "litellm.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{ default (include "litellm.fullname" .) .Values.serviceAccount.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
{{/*
Master-key + database + redis env block — shared by gateway, backend, and the
migrations Job.
Invoke with a dict: `(dict "root" $ "component" .Values.gateway)`. `root` is
the chart context (needed for .Values), `component` selects which component's
`extraEnv` / `logLevel` to render.
Sensitive values (master key, DB username + password, Redis password) come
only from referenced Secrets; the chart never accepts inline values for them.
The chart never assembles DATABASE_URL itself. It emits only the discrete
DATABASE_HOST/PORT/USER/NAME/SCHEMA (+ DATABASE_PASSWORD for password auth)
vars; the proxy's entrypoint (DatabaseURLSettings in
litellm/proxy/db/db_url_settings.py) builds the URL from them and
percent-encodes the credentials. Assembling the URL here via Kubernetes
`$(VAR)` substitution would embed the raw secret value, corrupting the URL
whenever the password contains a URL-reserved character (@, /, ?, %, +,
...) — as AWS RDS auto-generated passwords routinely do.
When `database.writer.useIAMAuth: true`, the chart injects
IAM_TOKEN_DB_AUTH=true and omits DATABASE_PASSWORD — the entrypoint mints
the URL from DATABASE_HOST/PORT/USER/NAME plus a short-lived IAM token
instead of a static password.
The read replica is opt-in via `database.reader.host`. The chart emits
DATABASE_HOST_READ_REPLICA / DATABASE_PORT_READ_REPLICA /
DATABASE_NAME_READ_REPLICA (+ DATABASE_SCHEMA_READ_REPLICA) for both auth
modes, plus DATABASE_USER_READ_REPLICA / DATABASE_PASSWORD_READ_REPLICA for
password auth. When `database.reader.useIAMAuth: true` it omits
DATABASE_PASSWORD_READ_REPLICA and the entrypoint mints the reader URL the
same way. Reader IAM only takes effect when the writer also uses IAM auth
(the proxy gates URL minting on IAM_TOKEN_DB_AUTH, which only the writer
sets).
*/}}
{{- define "litellm.serverEnv" -}}
{{- $root := .root -}}
{{- $component := .component -}}
- name: LITELLM_MASTER_KEY
valueFrom:
secretKeyRef:
name: {{ required "masterKey.secretName is required (the chart no longer accepts an inline master key)" $root.Values.masterKey.secretName }}
key: {{ $root.Values.masterKey.secretKey | default "master-key" }}
{{- if $component.logLevel }}
- name: LITELLM_LOG
value: {{ $component.logLevel | quote }}
{{- end }}
{{- with $root.Values.database.writer }}
- name: DATABASE_HOST
value: {{ required "database.writer.host is required" .host | quote }}
- name: DATABASE_PORT
value: {{ .port | default 5432 | quote }}
- name: DATABASE_USER
valueFrom:
secretKeyRef:
name: {{ required "database.writer.passwordSecret.name is required" .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
- name: DATABASE_NAME
value: {{ required "database.writer.dbname is required" .dbname | quote }}
{{- if .schema }}
- name: DATABASE_SCHEMA
value: {{ .schema | quote }}
{{- end }}
{{- if .useIAMAuth }}
- name: IAM_TOKEN_DB_AUTH
value: "true"
{{- else }}
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.passwordKey | default "password" }}
{{- end }}
{{- end }}
{{- with $root.Values.database.reader }}
{{- if .host }}
{{- if and .useIAMAuth (not $root.Values.database.writer.useIAMAuth) }}
{{- fail "database.reader.useIAMAuth requires database.writer.useIAMAuth: true (the proxy gates IAM URL minting on IAM_TOKEN_DB_AUTH, which is only set by the writer)" }}
{{- end }}
- name: DATABASE_HOST_READ_REPLICA
value: {{ .host | quote }}
- name: DATABASE_PORT_READ_REPLICA
value: {{ .port | default 5432 | quote }}
- name: DATABASE_NAME_READ_REPLICA
value: {{ required "database.reader.dbname is required when database.reader.host is set" .dbname | quote }}
{{- if .schema }}
- name: DATABASE_SCHEMA_READ_REPLICA
value: {{ .schema | quote }}
{{- end }}
{{- if .useIAMAuth }}
{{- if .passwordSecret.name }}
- name: DATABASE_USER_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
{{- end }}
{{- else }}
{{- if not .passwordSecret.name }}
{{- fail "database.reader.passwordSecret.name is required when database.reader.host is set" }}
{{- end }}
- name: DATABASE_USER_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
- name: DATABASE_PASSWORD_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.passwordKey | default "password" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
The migrations Job (helm.sh/hook: pre-upgrade) is the single owner of
`prisma migrate deploy`. Without this, every gateway/backend pod also runs
Prisma schema-update on startup and contends with the Job — and with each
other — for Prisma's Postgres advisory lock on the writer, which makes the
Job's `migrate deploy` intermittently block until its per-attempt timeout
and retry-exhaust. The Job's entrypoint (migrations/run.py) does not import
proxy_server and never reads DISABLE_SCHEMA_UPDATE, so emitting it here is a
harmless no-op for the Job and authoritative for the app pods.
*/}}
- name: DISABLE_SCHEMA_UPDATE
value: "true"
{{- if $root.Values.redis.host }}
- name: REDIS_HOST
value: {{ $root.Values.redis.host | quote }}
- name: REDIS_PORT
value: {{ $root.Values.redis.port | quote }}
{{- if $root.Values.redis.passwordSecret.name }}
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ $root.Values.redis.passwordSecret.name }}
key: {{ $root.Values.redis.passwordSecret.passwordKey | default "password" }}
{{- end }}
{{- if $root.Values.redis.cluster }}
{{/* The proxy's Cache() reads REDIS_CLUSTER_NODES as JSON and constructs a
RedisClusterCache when it's set (litellm/caching/caching.py:169-192).
We seed with the single configured endpoint — the cluster client
discovers the remaining nodes from CLUSTER SLOTS at startup. */}}
- name: REDIS_CLUSTER_NODES
value: {{ printf "[{\"host\":%q,\"port\":%v}]" $root.Values.redis.host (int $root.Values.redis.port) | quote }}
{{- end }}
{{- end }}
{{- with $component.extraEnv }}
{{ toYaml . }}
{{- end }}
{{- end -}}
{{/*
Renders `envFrom:` block for a component's `envConfigMaps` / `envSecrets`
lists. Each entry is a resource name; the chart wires the whole ConfigMap /
Secret into the container's env via configMapRef / secretRef.
Invoke with just the component dict, e.g. `.Values.gateway`. Emits nothing
when both lists are empty so the container spec stays clean.
*/}}
{{- define "litellm.envFrom" -}}
{{- $component := . -}}
{{- if or $component.envConfigMaps $component.envSecrets }}
envFrom:
{{- range $component.envConfigMaps }}
- configMapRef:
name: {{ . }}
{{- end }}
{{- range $component.envSecrets }}
- secretRef:
name: {{ . }}
{{- end }}
{{- end }}
{{- end -}}

60
helm/litellm/templates/backend/deployment.yaml Normal file
View file

@ -0,0 +1,60 @@
{{- if .Values.backend.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.backend.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
spec:
selector:
matchLabels:
{{- include "litellm.backend.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.backend.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.backend.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: backend
image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.backend.image.pullPolicy }}
ports:
- name: http
containerPort: 4001
protocol: TCP
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
{{- include "litellm.envFrom" .Values.backend | nindent 10 }}
{{- with .Values.backend.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.backend.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.backend.resources | nindent 12 }}
{{- with .Values.backend.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.backend.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.backend.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

33
helm/litellm/templates/backend/hpa.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- if and .Values.backend.enabled .Values.backend.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.backend.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.backend.fullname" . }}
minReplicas: {{ .Values.backend.hpa.minReplicas }}
maxReplicas: {{ .Values.backend.hpa.maxReplicas }}
metrics:
{{- if .Values.backend.hpa.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.backend.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.backend.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.backend.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

18
helm/litellm/templates/backend/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.backend.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.backend.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
spec:
type: {{ .Values.backend.service.type }}
ports:
- port: {{ .Values.backend.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.backend.selectorLabels" . | nindent 4 }}
{{- end }}

9
helm/litellm/templates/gateway/configmap.yaml Normal file
View file

@ -0,0 +1,9 @@
{{- if .Values.gateway.config.create }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "litellm.gateway.fullname" . }}-config
data:
config.yaml: |
{{ .Values.gateway.config.proxy_config | toYaml | indent 6 }}
{{- end }}

83
helm/litellm/templates/gateway/deployment.yaml Normal file
View file

@ -0,0 +1,83 @@
{{- if .Values.gateway.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
selector:
matchLabels:
{{- include "litellm.gateway.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
{{- if .Values.gateway.config.create }}
checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.gateway.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.gateway.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: gateway
image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
ports:
- name: http
containerPort: 4000
protocol: TCP
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
{{- if .Values.gateway.config.create }}
- name: CONFIG_FILE_PATH
value: /app/config/config.yaml
{{- end }}
{{- if .Values.gateway.numWorkers }}
- name: NUM_WORKERS
value: {{ .Values.gateway.numWorkers | quote }}
{{- end }}
{{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
{{- if .Values.gateway.config.create }}
volumeMounts:
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
{{- end }}
{{- with .Values.gateway.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.gateway.resources | nindent 12 }}
{{- if .Values.gateway.config.create }}
volumes:
- name: gateway-config
configMap:
name: {{ include "litellm.gateway.fullname" . }}-config
{{- end }}
{{- with .Values.gateway.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

33
helm/litellm/templates/gateway/hpa.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- if and .Values.gateway.enabled .Values.gateway.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.gateway.fullname" . }}
minReplicas: {{ .Values.gateway.hpa.minReplicas }}
maxReplicas: {{ .Values.gateway.hpa.maxReplicas }}
metrics:
{{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

18
helm/litellm/templates/gateway/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.gateway.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
type: {{ .Values.gateway.service.type }}
ports:
- port: {{ .Values.gateway.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.gateway.selectorLabels" . | nindent 4 }}
{{- end }}

153
helm/litellm/templates/ingress.yaml Normal file
View file

@ -0,0 +1,153 @@
{{- if .Values.ingress.enabled -}}
{{- $gatewayName := include "litellm.gateway.fullname" . -}}
{{- $backendName := include "litellm.backend.fullname" . -}}
{{- $uiName := include "litellm.ui.fullname" . -}}
{{- $gatewayPort := .Values.gateway.service.port -}}
{{- $backendPort := .Values.backend.service.port -}}
{{- $uiPort := .Values.ui.service.port -}}
{{/*
Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
Versioned paths are listed explicitly to avoid routing management routes
(e.g. /v1/access_group, /v2/key/info, /v1/tool/*, /v1/agents, /v1/workflows,
/v2/user/info, /v2/team/list, /v2/model/info, /v2/login, /v2/guardrails/*,
/v1/mcp/*) onto the gateway via a broad /v1 or /v2 prefix.
*/}}
{{- $gatewayPrefixes := list
"/v1/chat" "/chat" "/v1/completions" "/completions" "/v1/embeddings" "/embeddings"
"/v1/moderations" "/moderations" "/v1/audio" "/audio" "/v1/images" "/images"
"/v1/files" "/files" "/v1/batches" "/batches" "/v1/fine_tuning" "/fine_tuning"
"/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads"
"/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes"
"/v1/models" "/models" "/openai" "/engines"
"/v1/messages" "/messages" "/v1/skills" "/v1/a2a"
"/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag"
"/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
"/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
"/v1beta" "/interactions"
"/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/cohere" "/gemini" "/google"
"/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm"
"/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough"
"/toolset"
"/v1/realtime" "/realtime"
"/health" "/metrics"
-}}
{{/*
/test is gateway-only as an EXACT path (GATEWAY_EXACT_PATHS), but its
children /test/connection and /test/tools/list are MCP-server management
endpoints kept only on the backend ("/test/" in BACKEND_PATH_PREFIXES).
A Prefix match here would route /test/* to the gateway, which trims those
routes at startup -> 404. So /test is rendered as a standalone Exact path
and /test/* falls through to the backend catch-all.
*/}}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- with .Values.ingress.className }}
ingressClassName: {{ . | quote }}
{{- end }}
{{- with .Values.ingress.tls }}
tls:
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- {{- with .Values.ingress.host }}
host: {{ . | quote }}
{{- end }}
http:
paths:
# --- UI (Next.js static export) ---
- path: /
pathType: Exact
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /favicon.ico
pathType: Exact
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /litellm-asset-prefix
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /_next
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# /ui/* is where the Next.js SPA serves its login + dashboard
# routes (e.g. /ui/login). Without this, /ui/* falls into the
# catch-all → backend → 404.
- path: /ui
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# Next.js App Router (output: "export", basePath: "") emits the
# RSC/flight payload for every route as a ROOT-level <route>.txt
# (/index.txt, /teams.txt, /__next._tree.txt, ...). The client
# router fetches these on every soft navigation / prefetch as
# <route>.txt?_rsc=<hash> (the query string is irrelevant to path
# matching). They are not under /ui, /_next, or
# /litellm-asset-prefix, so without this rule they fall to the
# backend catch-all → 404 → client-side navigation never settles
# and the login flow spins in an infinite redirect loop
# (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt from the
# export; this rule only routes the request to it. Needs an
# ingress controller whose ImplementationSpecific path is a
# wildcard pattern (AWS ALB: `*` = 0+ chars); this chart targets
# the AWS Load Balancer Controller.
- path: /*.txt
pathType: ImplementationSpecific
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# --- Gateway data plane ---
# Exact /test only (see the $gatewayPrefixes comment above);
# /test/* MCP management endpoints fall to the backend catch-all.
- path: /test
pathType: Exact
backend:
service:
name: {{ $gatewayName }}
port:
number: {{ $gatewayPort }}
{{- range $gatewayPrefixes }}
- path: {{ . }}
pathType: Prefix
backend:
service:
name: {{ $gatewayName }}
port:
number: {{ $gatewayPort }}
{{- end }}
# --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) ---
- path: /
pathType: Prefix
backend:
service:
name: {{ $backendName }}
port:
number: {{ $backendPort }}
{{- end }}

46
helm/litellm/templates/migrations-job.yaml Normal file
View file

@ -0,0 +1,46 @@
{{- if .Values.migrationJob.enabled -}}
# Pre-install / pre-upgrade hook that runs `prisma migrate deploy` against
# the writer database before the gateway and backend Deployments are rolled
# out. Required because the gateway and backend both spin up Prisma at
# startup and assume the LiteLLM schema (LiteLLM_Config,
# LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...) already exists.
#
# Running this pre-upgrade closes the window where new application pods would
# otherwise serve traffic against the previous release's unmigrated schema.
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "litellm.fullname" . }}-migrations
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: migrations
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-delete-policy: before-hook-creation
helm.sh/hook-weight: "0"
spec:
backoffLimit: {{ .Values.migrationJob.backoffLimit }}
ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
template:
metadata:
labels:
{{- include "litellm.commonLabels" . | nindent 8 }}
app.kubernetes.io/component: migrations
spec:
restartPolicy: Never
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: prisma-migrations
image: "{{ .Values.migrationJob.image.repository }}:{{ .Values.migrationJob.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.migrationJob.image.pullPolicy }}
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.migrationJob) | nindent 12 }}
{{- with .Values.migrationJob.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- end }}

13
helm/litellm/templates/serviceaccount.yaml Normal file
View file

@ -0,0 +1,13 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "litellm.serviceAccountName" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
{{- end }}

70
helm/litellm/templates/ui/deployment.yaml Normal file
View file

@ -0,0 +1,70 @@
{{- if .Values.ui.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.ui.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: ui
spec:
selector:
matchLabels:
{{- include "litellm.ui.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.ui.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.ui.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: ui
image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.ui.image.pullPolicy }}
ports:
- name: http
containerPort: 3000
protocol: TCP
env:
{{- if .Values.ui.logLevel }}
- name: LITELLM_LOG
value: {{ .Values.ui.logLevel | quote }}
{{- end }}
{{- if .Values.ui.backendUrl }}
- name: LITELLM_BACKEND_URL
value: {{ .Values.ui.backendUrl | quote }}
{{- end }}
{{- with .Values.ui.extraEnv }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- include "litellm.envFrom" .Values.ui | nindent 10 }}
{{- with .Values.ui.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.ui.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.ui.resources | nindent 12 }}
{{- with .Values.ui.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.ui.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.ui.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

33
helm/litellm/templates/ui/hpa.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- if and .Values.ui.enabled .Values.ui.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.ui.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: ui
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.ui.fullname" . }}
minReplicas: {{ .Values.ui.hpa.minReplicas }}
maxReplicas: {{ .Values.ui.hpa.maxReplicas }}
metrics:
{{- if .Values.ui.hpa.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.ui.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.ui.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.ui.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

18
helm/litellm/templates/ui/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.ui.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.ui.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: ui
spec:
type: {{ .Values.ui.service.type }}
ports:
- port: {{ .Values.ui.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.ui.selectorLabels" . | nindent 4 }}
{{- end }}

225
helm/litellm/values.yaml Normal file
View file

@ -0,0 +1,225 @@
# LiteLLM helm chart values
nameOverride: ""
fullnameOverride: ""
imagePullSecrets: []
# Optional Ingress wiring the three component Services behind a single L7
# entrypoint. Required when serving the static UI bundle over the network.
ingress:
enabled: false
className: ""
annotations: {}
host: "" # optional; if set, becomes the rule's host
tls: []
# Shared ServiceAccount used by all three component Deployments. Set
# `create: true` to have the chart provision it (e.g. when wiring an EKS
# Pod Identity association by SA name). Set `name` to use an existing SA
# (chart-created or out-of-band). When both are empty / false, pods run
# with the namespace's `default` SA.
serviceAccount:
create: false
automount: true
annotations: {}
name: ""
# Pre-install / pre-upgrade Helm hook that runs `prisma migrate deploy`
# against the writer database, creating the LiteLLM schema (tables that
# gateway + backend assume exist at startup: LiteLLM_Config,
# LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...). Disable if your
# pipeline runs migrations out-of-band.
#
# Uses a dedicated `litellm-migrations` image (prisma CLI + the migration
# files from `litellm-proxy-extras`) instead of the backend image, so the
# Job doesn't drag in the rest of the proxy and doesn't run `prisma
# generate` — the migration engine doesn't need the generated client.
migrationJob:
enabled: true
backoffLimit: 4
ttlSecondsAfterFinished: 120
resources: {}
image:
repository: ghcr.io/berriai/litellm-migrations
tag: "" # defaults to .Chart.AppVersion
pullPolicy: IfNotPresent
# Extra env appended to the migration container. The migration entrypoint
# uses the v2 resolver by default (no diff-and-force recovery — avoids the
# schema thrashing seen during rolling deploys). To opt back into the v1
# resolver, append `- name: USE_V2_MIGRATION_RESOLVER` / `value: "false"`.
extraEnv: []
# Required: a master key used by gateway + backend to mint/verify proxy tokens.
# Must reference an existing Secret.
masterKey:
secretName: litellm-master-key-secret # name of a Secret containing the master key
secretKey: master-key
# External Postgres connection.
database:
writer:
host: ""
port: 5432
dbname: ""
schema: ""
useIAMAuth: false
passwordSecret:
name: litellm-writer-secret
usernameKey: username
passwordKey: password
# Optional read-replica routing. When `reader.host` is set, the proxy routes
# reads (find_*, count, group_by, query_raw/_first) to this endpoint while
# writes stay on the writer. Leave `reader.host` empty to disable.
reader:
host: ""
port: 5432
dbname: ""
schema: ""
useIAMAuth: false
passwordSecret:
name: litellm-reader-secret
usernameKey: username
passwordKey: password
# Optional Redis (caching, rate limiting). Leave host empty to disable.
#
# Set `cluster: true` for Redis Cluster mode (e.g. AWS ElastiCache Cluster,
# self-hosted Redis Cluster). The chart emits REDIS_CLUSTER_NODES from
# `host` / `port` as the single seed; the cluster client discovers the
# remaining nodes from CLUSTER SLOTS at startup.
redis:
cluster: false
host: ""
port: 6379
passwordSecret:
name: "" # Leave empty for auth-less Redis
passwordKey: password
# ---------- gateway (LLM data plane) ----------
gateway:
enabled: true
logLevel: INFO
# Number of uvicorn worker processes per gateway pod. Sets NUM_WORKERS,
# consumed by the gateway image entrypoint. Default is 1.
numWorkers: 1
extraEnv: [] # Add extra environment variables to the gateway
envConfigMaps: [] # Add extra environment variables to the gateway from config maps
envSecrets: [] # Add extra environment variables to the gateway from secrets
config:
create: true
proxy_config: {}
image:
repository: ghcr.io/berriai/litellm-gateway
tag: "" # defaults to .Chart.AppVersion
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 4000
resources:
requests:
cpu: "1"
memory: 4Gi
limits:
cpu: "2"
memory: 4Gi
livenessProbe:
httpGet: { path: /health/liveliness, port: http }
initialDelaySeconds: 10
periodSeconds: 15
readinessProbe:
httpGet: { path: /health/readiness, port: http }
initialDelaySeconds: 5
periodSeconds: 10
hpa:
enabled: true
minReplicas: 1
maxReplicas: 10
targetCPUUtilizationPercentage: 70
targetMemoryUtilizationPercentage: 80
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
# ---------- backend (UI / management API) ----------
backend:
enabled: true
logLevel: INFO
extraEnv: []
envConfigMaps: []
envSecrets: []
image:
repository: ghcr.io/berriai/litellm-backend
tag: ""
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 4001
resources:
requests:
cpu: "1"
memory: 4Gi
limits:
cpu: "2"
memory: 4Gi
livenessProbe:
httpGet: { path: /health/liveliness, port: http }
initialDelaySeconds: 10
periodSeconds: 15
readinessProbe:
httpGet: { path: /health/readiness, port: http }
initialDelaySeconds: 5
periodSeconds: 10
hpa:
enabled: true
minReplicas: 1
maxReplicas: 4
targetCPUUtilizationPercentage: 70
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
# ---------- ui (Next.js static dashboard) ----------
ui:
enabled: true
logLevel: INFO
extraEnv: []
envConfigMaps: []
envSecrets: []
image:
repository: ghcr.io/berriai/litellm-ui
tag: ""
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 3000
# The dashboard expects to know where to reach the backend API. Set this to
# the externally-routable URL (typically the ingress host + /api or similar).
backendUrl: ""
resources:
requests:
cpu: 500m
memory: 500Mi
limits:
cpu: "1"
memory: 1Gi
livenessProbe:
httpGet: { path: /, port: http }
initialDelaySeconds: 5
periodSeconds: 20
readinessProbe:
httpGet: { path: /, port: http }
initialDelaySeconds: 2
periodSeconds: 10
hpa:
enabled: false
minReplicas: 1
maxReplicas: 3
targetCPUUtilizationPercentage: 80
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}

9
license_cache.json
View file

File diff suppressed because one or more lines are too long

4
litellm-proxy-extras/litellm_proxy_extras/migrations/20260514120000_add_blocked_to_proxy_model_table/migration.sql Normal file
View file

@ -0,0 +1,4 @@
-- AlterTable
-- Adds the admin-toggleable pause flag used by the router's blocked filter and the
-- credential lookup helpers; defaults to false so existing rows behave unchanged.
ALTER TABLE "LiteLLM_ProxyModelTable" ADD COLUMN IF NOT EXISTS "blocked" BOOLEAN NOT NULL DEFAULT false;

5
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -48,9 +48,10 @@ model LiteLLM_CredentialsTable {
// Models on proxy
model LiteLLM_ProxyModelTable {
model_id String @id @default(uuid())
model_name String
model_name String
litellm_params Json
model_info Json?
model_info Json?
blocked Boolean @default(false)
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")

7
litellm/__init__.py
View file

@ -416,6 +416,7 @@ custom_prometheus_metadata_labels: List[str] = []
custom_prometheus_tags: List[str] = []
prometheus_metrics_config: Optional[List] = None
prometheus_emit_stream_label: bool = False
prometheus_user_budget_label_include_email_alias: bool = False
prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
@ -1880,6 +1881,12 @@ if TYPE_CHECKING:
from .llms.dashscope.chat.transformation import (
DashScopeChatConfig as DashScopeChatConfig,
)
from .llms.dashscope.embed.transformation import (
DashScopeEmbeddingConfig as DashScopeEmbeddingConfig,
)
from .llms.dashscope.rerank.transformation import (
DashScopeRerankConfig as DashScopeRerankConfig,
)
from .llms.moonshot.chat.transformation import (
MoonshotChatConfig as MoonshotChatConfig,
)

2
litellm/_redis.py
View file

@ -100,6 +100,8 @@ def _get_redis_cluster_kwargs(client=None):
"azure_tenant_id",
"azure_client_secret",
"max_connections",
"socket_timeout",
"socket_connect_timeout",
}
return available_args

70
litellm/caching/caching_handler.py
View file

@ -87,6 +87,16 @@ class CachingHandlerResponse(BaseModel):
in_memory_cache_obj = InMemoryCache()
def _is_chat_completion_cached_dict(cached_result: dict) -> bool:
cached_id = cached_result.get("id")
if isinstance(cached_id, str) and cached_id.startswith("chatcmpl"):
return True
obj = cached_result.get("object")
if isinstance(obj, str):
return obj.startswith("chat.completion")
return "choices" in cached_result
def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool:
"""
When stream=True, do not run success callbacks at cache-hit time.
@ -861,27 +871,47 @@ class LLMCachingHandler:
elif (call_type == "aresponses" or call_type == "responses") and isinstance(
cached_result, dict
):
from litellm.responses.streaming_iterator import (
CachedResponsesAPIStreamingIterator,
)
response_obj = ResponsesAPIResponse(**cached_result)
if (
hasattr(response_obj, "_hidden_params")
and response_obj._hidden_params is not None
and isinstance(response_obj._hidden_params, dict)
):
response_obj._hidden_params["cache_hit"] = True
if kwargs.get("stream", False) is True:
cached_result = CachedResponsesAPIStreamingIterator(
response=response_obj,
logging_obj=logging_obj,
request_data=kwargs,
call_type=call_type,
)
use_chat_completion_cache = _is_chat_completion_cached_dict(cached_result)
if use_chat_completion_cache:
if kwargs.get("stream", False) is True:
bridge_call_type = (
CallTypes.acompletion.value
if call_type == "aresponses"
else CallTypes.completion.value
)
cached_result = self._convert_cached_stream_response(
cached_result=cached_result,
call_type=bridge_call_type,
logging_obj=logging_obj,
model=model,
)
else:
cached_result = convert_to_model_response_object(
response_object=cached_result,
model_response_object=ModelResponse(),
)
else:
cached_result = response_obj
from litellm.responses.streaming_iterator import (
CachedResponsesAPIStreamingIterator,
)
response_obj = ResponsesAPIResponse(**cached_result)
if (
hasattr(response_obj, "_hidden_params")
and response_obj._hidden_params is not None
and isinstance(response_obj._hidden_params, dict)
):
response_obj._hidden_params["cache_hit"] = True
if kwargs.get("stream", False) is True:
cached_result = CachedResponsesAPIStreamingIterator(
response=response_obj,
logging_obj=logging_obj,
request_data=kwargs,
call_type=call_type,
)
else:
cached_result = response_obj
if (
hasattr(cached_result, "_hidden_params")

25
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -37,6 +37,15 @@ class ResponsesToCompletionBridgeHandler:
stream = litellm_params.get("stream", False)
return bool(stream)
@staticmethod
def _is_preformatted_cached_chat_stream(result: Any) -> bool:
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
return (
isinstance(result, CustomStreamWrapper)
and result.custom_llm_provider == "cached_response"
)
@staticmethod
def _coerce_response_object(
response_obj: Any,
@ -177,6 +186,8 @@ class ResponsesToCompletionBridgeHandler:
**request_data,
)
from litellm.types.utils import ModelResponse
stream = self._resolve_stream_flag(optional_params, litellm_params)
if isinstance(result, ResponsesAPIResponse):
return self.transformation_handler.transform_response(
@ -192,6 +203,8 @@ class ResponsesToCompletionBridgeHandler:
api_key=kwargs.get("api_key"),
json_mode=kwargs.get("json_mode"),
)
elif isinstance(result, ModelResponse):
return result
elif not stream:
responses_api_response = self._collect_response_from_stream(result)
return self.transformation_handler.transform_response(
@ -208,6 +221,10 @@ class ResponsesToCompletionBridgeHandler:
json_mode=kwargs.get("json_mode"),
)
else:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(
result, model, custom_llm_provider
)
completion_stream = self.transformation_handler.get_model_response_iterator(
streaming_response=result, # type: ignore
sync_stream=True,
@ -256,6 +273,8 @@ class ResponsesToCompletionBridgeHandler:
aresponses=True,
)
from litellm.types.utils import ModelResponse
stream = self._resolve_stream_flag(optional_params, litellm_params)
if isinstance(result, ResponsesAPIResponse):
return self.transformation_handler.transform_response(
@ -271,6 +290,8 @@ class ResponsesToCompletionBridgeHandler:
api_key=kwargs.get("api_key"),
json_mode=kwargs.get("json_mode"),
)
elif isinstance(result, ModelResponse):
return result
elif not stream:
responses_api_response = await self._collect_response_from_stream_async(
result
@ -289,6 +310,10 @@ class ResponsesToCompletionBridgeHandler:
json_mode=kwargs.get("json_mode"),
)
else:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(
result, model, custom_llm_provider
)
completion_stream = self.transformation_handler.get_model_response_iterator(
streaming_response=result, # type: ignore
sync_stream=False,

8
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -1141,6 +1141,14 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
event_type = parsed_chunk.get("type")
if isinstance(event_type, ResponsesAPIStreamEvents):
event_type = event_type.value
if parsed_chunk.get("object") == "chat.completion.chunk" or (
event_type is None
and isinstance(parsed_chunk.get("choices"), list)
and parsed_chunk.get("choices")
):
return ModelResponseStream(**parsed_chunk)
verbose_logger.debug(f"Chat provider: Processing event type: {event_type}")
if event_type == "response.created":

82
litellm/cost_calculator.py
View file

@ -173,17 +173,45 @@ def _cost_per_token_custom_pricing_helper(
prompt_tokens: float = 0,
completion_tokens: float = 0,
response_time_ms: Optional[float] = 0.0,
cached_tokens: float = 0,
cache_creation_tokens: float = 0,
### CUSTOM PRICING ###
custom_cost_per_token: Optional[CostPerToken] = None,
custom_cost_per_second: Optional[float] = None,
) -> Optional[Tuple[float, float]]:
"""Internal helper function for calculating cost, if custom pricing given"""
"""Internal helper function for calculating cost, if custom pricing given.
prompt_tokens is assumed to include both cached_tokens and cache_creation_tokens
(OpenAI-compatible convention). Anthropic-style usage where prompt_tokens excludes
cache tokens is handled at the caller (cost_per_token) before invoking this helper.
"""
if custom_cost_per_token is None and custom_cost_per_second is None:
return None
if custom_cost_per_token is not None:
input_cost = custom_cost_per_token["input_cost_per_token"] * prompt_tokens
output_cost = custom_cost_per_token["output_cost_per_token"] * completion_tokens
input_cost_per_token = custom_cost_per_token["input_cost_per_token"]
output_cost_per_token = custom_cost_per_token["output_cost_per_token"]
cache_read_input_token_cost = custom_cost_per_token.get(
"cache_read_input_token_cost",
input_cost_per_token,
)
cache_creation_input_token_cost = custom_cost_per_token.get(
"cache_creation_input_token_cost",
input_cost_per_token,
)
regular_prompt_tokens = max(
prompt_tokens - cached_tokens - cache_creation_tokens,
0,
)
input_cost = (
regular_prompt_tokens * input_cost_per_token
+ cached_tokens * cache_read_input_token_cost
+ cache_creation_tokens * cache_creation_input_token_cost
)
output_cost = completion_tokens * output_cost_per_token
return input_cost, output_cost
elif custom_cost_per_second is not None:
output_cost = custom_cost_per_second * response_time_ms / 1000 # type: ignore
@ -323,10 +351,56 @@ def cost_per_token( # noqa: PLR0915
)
## CUSTOM PRICING ##
# Normalize cache token counts across providers:
# - OpenAI-compatible: usage.prompt_tokens_details.cached_tokens
# (prompt_tokens already INCLUDES cached_tokens)
# - Anthropic: usage.cache_read_input_tokens / cache_creation_input_tokens
# (prompt_tokens does NOT include these — adjust before calling helper)
_cache_read_tokens: float = 0
_cache_creation_tokens: float = 0
_is_anthropic_style = False
if usage_object is not None:
_pt_details = getattr(usage_object, "prompt_tokens_details", None)
if _pt_details is not None:
_cache_read_tokens = float(getattr(_pt_details, "cached_tokens", 0) or 0)
# OpenAI-compatible providers report cache-write tokens under
# either `cache_write_tokens` (kimi-k2) or `cache_creation_tokens`.
# Mirror db_spend_update_writer to stay symmetric.
_cache_creation_tokens = float(
getattr(_pt_details, "cache_write_tokens", 0)
or getattr(_pt_details, "cache_creation_tokens", 0)
or 0
)
_anthropic_read = getattr(usage_object, "cache_read_input_tokens", None)
_anthropic_create = getattr(usage_object, "cache_creation_input_tokens", None)
if _anthropic_read is not None or _anthropic_create is not None:
_is_anthropic_style = True
if _anthropic_read is not None:
_cache_read_tokens = float(_anthropic_read)
if _anthropic_create is not None:
_cache_creation_tokens = float(_anthropic_create)
if not _cache_read_tokens and cache_read_input_tokens:
_cache_read_tokens = float(cache_read_input_tokens)
_is_anthropic_style = True
if not _cache_creation_tokens and cache_creation_input_tokens:
_cache_creation_tokens = float(cache_creation_input_tokens)
_is_anthropic_style = True
# Anthropic reports prompt_tokens as input_tokens (excluding cache tokens).
# Adjust so the helper's "prompt_tokens includes cache tokens" invariant holds.
_normalized_prompt_tokens = float(prompt_tokens)
if _is_anthropic_style:
_normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens
response_cost = _cost_per_token_custom_pricing_helper(
prompt_tokens=prompt_tokens,
prompt_tokens=_normalized_prompt_tokens,
completion_tokens=completion_tokens,
response_time_ms=response_time_ms,
cached_tokens=_cache_read_tokens,
cache_creation_tokens=_cache_creation_tokens,
custom_cost_per_second=custom_cost_per_second,
custom_cost_per_token=custom_cost_per_token,
)

4
litellm/exceptions.py
View file

@ -918,9 +918,11 @@ class GuardrailRaisedException(Exception):
guardrail_name: Optional[str] = None,
message: str = "",
should_wrap_with_default_message: bool = True,
status_code: int = 400,
):
default_message = f"Guardrail raised an exception, Guardrail: {guardrail_name}, Message: {message}"
self.guardrail_name = guardrail_name
self.status_code = status_code
self.message = default_message if should_wrap_with_default_message else message
super().__init__(self.message)
@ -930,12 +932,14 @@ class BlockedPiiEntityError(Exception):
self,
entity_type: str,
guardrail_name: Optional[str] = None,
status_code: int = 400,
):
"""
Raised when a blocked entity is detected by a guardrail.
"""
self.entity_type = entity_type
self.guardrail_name = guardrail_name
self.status_code = status_code
self.message = f"Blocked entity detected: {entity_type} by Guardrail: {guardrail_name}. This entity is not allowed to be used in this request."
super().__init__(self.message)

11
litellm/integrations/custom_guardrail.py
View file

@ -43,7 +43,11 @@ if TYPE_CHECKING:
dc = DualCache()
from litellm.exceptions import ModifyResponseException as ModifyResponseException
from litellm.exceptions import (
BlockedPiiEntityError,
GuardrailRaisedException,
ModifyResponseException,
)
class CustomGuardrail(CustomLogger):
@ -737,12 +741,15 @@ class CustomGuardrail(CustomLogger):
(this was logged previously as an API failure - guardrail_failed_to_respond).
Guardrails signal intentional blocks by raising:
- GuardrailRaisedException (generic guardrail API, tool permission)
- BlockedPiiEntityError (Presidio PII detection)
- HTTPException with status 400 (content policy violation)
- ModifyResponseException (passthrough mode violation)
"""
if isinstance(e, ModifyResponseException):
return True
if isinstance(e, (GuardrailRaisedException, BlockedPiiEntityError)):
return True
if (
HTTPException is not None
and isinstance(e, HTTPException)

143
litellm/integrations/opentelemetry.py
View file

@ -59,6 +59,11 @@ LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm")
LITELLM_METER_NAME = os.getenv("LITELLM_METER_NAME", "litellm")
LITELLM_LOGGER_NAME = os.getenv("LITELLM_LOGGER_NAME", "litellm")
LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
# OTel-standard names. status is also kept under error.code for back compat.
HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE = "http.response.status_code"
HTTP_ROUTE_ATTRIBUTE = "http.route"
URL_PATH_ATTRIBUTE = "url.path"
PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms"
# Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
@ -667,6 +672,31 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
parent_otel_span = user_api_key_dict.parent_otel_span
if parent_otel_span is not None:
parent_otel_span.set_status(Status(StatusCode.ERROR))
# Stamp structured error attrs on the SERVER span itself; the
# failure path otherwise only sets its status (_handle_failure
# records on the litellm_request child span). Inline import:
# litellm_logging <-> integrations is circular.
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
)
error_information = StandardLoggingPayloadSetup.get_error_information(
original_exception=original_exception,
traceback_str=traceback_str,
)
self._record_exception_on_span(
span=parent_otel_span,
kwargs={
"exception": original_exception,
"standard_logging_object": {"error_information": error_information},
},
)
# Pre-request latency (request_data carries the propagated
# metadata on the failure path; omitted if it failed before handoff).
self.set_preprocessing_duration_attribute(parent_otel_span, request_data)
_span_name = "Failed Proxy Server Request"
# Exception Logging Child Span
@ -703,6 +733,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
ctx, _ = self._get_span_context(kwargs, default_span=parent_span)
# Pre-request latency on the SERVER span (success path).
self.set_preprocessing_duration_attribute(parent_span, kwargs)
# http.response.status_code on the SERVER span (success path).
# A successful proxy response is HTTP 200; the failure path sets
# this from the error code in _record_exception_on_span.
self.set_response_status_code_attribute(parent_span, 200)
# 3. Guardrail span
self._create_guardrail_span(kwargs=kwargs, context=ctx)
@ -1069,8 +1107,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
"mcp_tool_call_metadata",
"vector_store_request_metadata",
]:
if md.get(key) is not None:
common_attrs[f"metadata.{key}"] = str(md[key])
value = md.get(key)
if value is None:
continue
if isinstance(value, (dict, list)):
common_attrs[f"metadata.{key}"] = safe_dumps(value)
else:
common_attrs[f"metadata.{key}"] = str(value)
# get hidden params
hidden_params = getattr(std_log, "hidden_params", None) or (std_log or {}).get(
@ -1627,6 +1670,19 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=error_information["error_code"],
)
# Also expose under the OTel-standard name as an int
# (error_code is a str, may be non-numeric).
_error_code_val = error_information["error_code"]
if _error_code_val is not None:
try:
self.safe_set_attribute(
span=span,
key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
value=int(_error_code_val),
)
except (ValueError, TypeError):
pass
if error_information.get("error_class"):
self.safe_set_attribute(
span=span,
@ -2889,3 +2945,86 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
context=self.get_traceparent_from_header(headers=headers),
kind=self.span_kind.SERVER,
)
def set_proxy_request_route_attributes(
self,
span: Optional[Span],
*,
url_path: Optional[str] = None,
http_route: Optional[str] = None,
) -> None:
"""
Set OTel-standard ``http.route`` / ``url.path`` on the proxy SERVER
span. Called from the auth path, the only point where both the
SERVER span and the request are in hand. No-op if span/value missing.
"""
if span is None:
return
if url_path:
self.safe_set_attribute(span=span, key=URL_PATH_ATTRIBUTE, value=url_path)
if http_route:
self.safe_set_attribute(
span=span, key=HTTP_ROUTE_ATTRIBUTE, value=http_route
)
def set_response_status_code_attribute(
self, span: Optional[Span], status_code: Optional[int]
) -> None:
"""
Set OTel-standard ``http.response.status_code`` (int) on the proxy
SERVER span. The failure path sets this from the error code in
``_record_exception_on_span``; this is the success-path counterpart
so the attribute is present on every SERVER span regardless of
outcome (required by the HTTP semconv, and needed for error-ratio /
status-breakdown dashboards). No-op if span/value missing.
"""
if span is None or status_code is None:
return
self.safe_set_attribute(
span=span,
key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
value=int(status_code),
)
def set_preprocessing_duration_attribute(
self, span: Optional[Span], container: Any
) -> None:
"""
Set ``litellm.preprocessing.duration_ms`` (proxy-receive -> first
provider handoff) on the proxy SERVER span. ``litellm_received_at``
rides request metadata; ``first_api_call_start_time`` is the
set-once first-handoff instant (retries/backoff excluded). Works
uniformly for the success (model_call_details) and failure
(request_data) containers. No-op if span/either anchor is missing.
"""
if span is None or not isinstance(container, dict):
return
received_at = None
# first_api_call_start_time is top-level (never in user metadata).
first_handoff = container.get("first_api_call_start_time")
_lp = container.get("litellm_params")
for _md in (
(_lp or {}).get("metadata") if isinstance(_lp, dict) else None,
container.get("metadata"),
container.get("litellm_metadata"),
):
if isinstance(_md, dict):
received_at = received_at or _md.get("litellm_received_at")
if received_at is None or first_handoff is None:
return
try:
start_ts = self._to_timestamp(received_at)
end_ts = self._to_timestamp(first_handoff)
except Exception:
return
if start_ts is None or end_ts is None:
return
duration_ms = (end_ts - start_ts) * 1000.0
# Clock skew → omit rather than emit a negative latency.
if duration_ms < 0:
return
self.safe_set_attribute(
span=span,
key=PREPROCESSING_DURATION_MS_ATTRIBUTE,
value=duration_ms,
)

6
litellm/integrations/prometheus.py
View file

@ -3540,6 +3540,10 @@ class PrometheusLogger(CustomLogger):
user_object.budget_reset_at = user_info.budget_reset_at
if user_object.max_budget is None and user_info.max_budget is not None:
user_object.max_budget = user_info.max_budget
if user_info.user_email is not None:
user_object.user_email = user_info.user_email
if user_info.user_alias is not None:
user_object.user_alias = user_info.user_alias
return user_object
@ -3556,6 +3560,8 @@ class PrometheusLogger(CustomLogger):
"""
enum_values = UserAPIKeyLabelValues(
user=user.user_id,
user_email=user.user_email or "",
user_alias=user.user_alias or "",
)
_labels = prometheus_label_factory(

67
litellm/interactions/litellm_responses_transformation/streaming_iterator.py
View file

@ -2,7 +2,7 @@
Streaming iterator for transforming Responses API stream to Interactions API stream.
"""
from typing import Any, AsyncIterator, Dict, Iterator, Optional, cast
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, cast
from litellm.responses.streaming_iterator import (
BaseResponsesAPIStreamingIterator,
@ -15,6 +15,7 @@ from litellm.types.interactions import (
InteractionsAPIStreamingResponse,
)
from litellm.types.llms.openai import (
ContentPartAddedEvent,
OutputTextDeltaEvent,
ResponseCompletedEvent,
ResponseCreatedEvent,
@ -51,6 +52,7 @@ class LiteLLMResponsesInteractionsStreamingIterator:
self.collected_text = ""
self.sent_interaction_start = False
self.sent_content_start = False
self._pending_events: List[InteractionsAPIStreamingResponse] = []
def _transform_responses_chunk_to_interactions_chunk(
self,
@ -80,7 +82,49 @@ class LiteLLMResponsesInteractionsStreamingIterator:
)
self.collected_text += delta_text
# Send interaction.start if not sent
# Fallback: emit interaction.start, and queue content.start carrying this
# delta so the first token is preserved in the stream.
if not self.sent_interaction_start:
self.sent_interaction_start = True
self.sent_content_start = True
self._pending_events.append(
InteractionsAPIStreamingResponse(
event_type="content.start",
id=getattr(responses_chunk, "item_id", None),
object="content",
delta={"type": "text", "text": delta_text},
)
)
return InteractionsAPIStreamingResponse(
event_type="interaction.start",
id=getattr(responses_chunk, "item_id", None)
or f"interaction_{id(self)}",
object="interaction",
status="in_progress",
model=self.model,
)
# Fallback: emit content.start if ContentPartAddedEvent never arrived
if not self.sent_content_start:
self.sent_content_start = True
return InteractionsAPIStreamingResponse(
event_type="content.start",
id=getattr(responses_chunk, "item_id", None),
object="content",
delta={"type": "text", "text": delta_text},
)
# Normal path: emit content.delta with type field
return InteractionsAPIStreamingResponse(
event_type="content.delta",
id=getattr(responses_chunk, "item_id", None),
object="content",
delta={"type": "text", "text": delta_text},
)
# Handle ContentPartAddedEvent -> content.start (arrives before text deltas)
if isinstance(responses_chunk, ContentPartAddedEvent):
# Fallback: emit interaction.start if ResponseCreatedEvent never arrived
if not self.sent_interaction_start:
self.sent_interaction_start = True
return InteractionsAPIStreamingResponse(
@ -91,8 +135,6 @@ class LiteLLMResponsesInteractionsStreamingIterator:
status="in_progress",
model=self.model,
)
# Send content.start if not sent
if not self.sent_content_start:
self.sent_content_start = True
return InteractionsAPIStreamingResponse(
@ -101,14 +143,7 @@ class LiteLLMResponsesInteractionsStreamingIterator:
object="content",
delta={"type": "text", "text": ""},
)
# Send content.delta
return InteractionsAPIStreamingResponse(
event_type="content.delta",
id=getattr(responses_chunk, "item_id", None),
object="content",
delta={"text": delta_text},
)
return None
# Handle ResponseCreatedEvent or ResponseInProgressEvent -> interaction.start
if isinstance(responses_chunk, (ResponseCreatedEvent, ResponseInProgressEvent)):
@ -172,6 +207,10 @@ class LiteLLMResponsesInteractionsStreamingIterator:
delattr(self, "_pending_interaction_complete")
return pending
# Drain events queued from a prior chunk (e.g. content.start emitted alongside
# the interaction.start fallback for the first OutputTextDeltaEvent).
if self._pending_events:
return self._pending_events.pop(0)
# Use a loop instead of recursion to avoid stack overflow
sync_iterator = cast(
SyncResponsesAPIStreamingIterator, self.responses_stream_iterator
@ -237,6 +276,10 @@ class LiteLLMResponsesInteractionsStreamingIterator:
delattr(self, "_pending_interaction_complete")
return pending
# Drain events queued from a prior chunk (e.g. content.start emitted alongside
# the interaction.start fallback for the first OutputTextDeltaEvent).
if self._pending_events:
return self._pending_events.pop(0)
# Use a loop instead of recursion to avoid stack overflow
async_iterator = cast(
ResponsesAPIStreamingIterator, self.responses_stream_iterator

10
litellm/litellm_core_utils/litellm_logging.py
View file

@ -1050,6 +1050,16 @@ class Logging(LiteLLMLoggingBaseClass):
)
self.model_call_details["api_call_start_time"] = datetime.datetime.now()
# Set-once first provider-handoff instant. api_call_start_time
# is overwritten on every retry, so it can't measure one-time
# preprocessing; pinning the first attempt excludes retry loops
# + backoff. Logging object only — must NOT go into
# litellm_params["metadata"] (caller request metadata, typed
# Dict[str, str], echoed downstream; a datetime breaks it).
if self.model_call_details.get("first_api_call_start_time") is None:
self.model_call_details["first_api_call_start_time"] = (
self.model_call_details["api_call_start_time"]
)
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
callbacks = litellm.input_callback + (self.dynamic_input_callbacks or [])
for callback in callbacks:

14
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -20,6 +20,7 @@ from typing import (
cast,
)
import litellm
from litellm import verbose_logger
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
@ -1170,9 +1171,16 @@ def migrate_file_to_image_url(
ChatCompletionImageUrlObject,
)
file_id = message["file"].get("file_id")
file_data = message["file"].get("file_data")
format = message["file"].get("format")
file_sub = message.get("file")
if file_sub is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider=None,
)
file_id = file_sub.get("file_id")
file_data = file_sub.get("file_data")
format = file_sub.get("format")
if not file_id and not file_data:
raise ValueError("file_id and file_data are both None")
image_url_object = ChatCompletionImageObject(

29
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -2057,9 +2057,16 @@ def anthropic_process_openai_file_message(
AnthropicMessagesContainerUploadParam,
]:
file_message = cast(ChatCompletionFileObject, message)
file_data = file_message["file"].get("file_data")
file_id = file_message["file"].get("file_id")
format = file_message["file"].get("format")
file_sub = file_message.get("file")
if file_sub is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="anthropic",
)
file_data = file_sub.get("file_data")
file_id = file_sub.get("file_id")
format = file_sub.get("format")
if file_data:
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=file_data,
@ -4879,7 +4886,13 @@ class BedrockConverseMessagesProcessor:
@staticmethod
def _process_file_message(message: ChatCompletionFileObject) -> BedrockContentBlock:
file_message = message["file"]
file_message = message.get("file")
if file_message is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="bedrock",
)
file_data = file_message.get("file_data")
file_id = file_message.get("file_id")
@ -4900,7 +4913,13 @@ class BedrockConverseMessagesProcessor:
async def _async_process_file_message(
message: ChatCompletionFileObject,
) -> BedrockContentBlock:
file_message = message["file"]
file_message = message.get("file")
if file_message is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="bedrock",
)
file_data = file_message.get("file_data")
file_id = file_message.get("file_id")
format = file_message.get("format")

26
litellm/llms/bedrock/batches/transformation.py
View file

@ -5,6 +5,7 @@ from typing import Any, Dict, List, Literal, Optional, Union, cast
from httpx import Headers, Response
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
@ -263,9 +264,32 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
cancelling_at=None,
cancelled_at=None,
request_counts=None,
metadata=original_request.get("metadata", {}),
metadata=self._get_openai_compatible_batch_metadata(
original_request.get("metadata", {})
),
)
@staticmethod
def _get_openai_compatible_batch_metadata(metadata: Any) -> Dict[str, str]:
"""
OpenAI Batch metadata only accepts string values.
"""
if not isinstance(metadata, dict):
return {}
sanitized_metadata: Dict[str, str] = {}
for key, value in metadata.items():
if key == "standard_logging_guardrail_information" or value is None:
continue
str_key = str(key)
if isinstance(value, str):
sanitized_metadata[str_key] = value
else:
sanitized_metadata[str_key] = safe_dumps(value)
return sanitized_metadata
def transform_retrieve_batch_request(
self,
batch_id: str,

4
litellm/llms/bedrock/chat/invoke_agent/transformation.py
View file

@ -299,9 +299,9 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
)
def _get_response_stream_shape(self):
from litellm.llms.bedrock.common_utils import BEDROCK_RESPONSE_STREAM_SHAPE
from litellm.llms.bedrock.common_utils import get_bedrock_response_stream_shape
return BEDROCK_RESPONSE_STREAM_SHAPE
return get_bedrock_response_stream_shape()
def _extract_response_content(self, events: InvokeAgentEventList) -> str:
"""Extract the final response content from parsed events."""

9
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -68,9 +68,9 @@ from litellm.utils import CustomStreamWrapper, get_secret
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import (
BEDROCK_RESPONSE_STREAM_SHAPE,
BedrockError,
ModelResponseIterator,
get_bedrock_response_stream_shape,
get_bedrock_tool_name,
)
@ -1828,7 +1828,8 @@ class AWSEventStreamDecoder:
yield self._chunk_parser(chunk_data=_data)
def _parse_message_from_event(self, event) -> Optional[str]:
if BEDROCK_RESPONSE_STREAM_SHAPE is None:
response_stream_shape = get_bedrock_response_stream_shape()
if response_stream_shape is None:
raise BedrockError(
status_code=500,
message=(
@ -1837,9 +1838,7 @@ class AWSEventStreamDecoder:
),
)
response_dict = event.to_response_dict()
parsed_response = self.parser.parse(
response_dict, BEDROCK_RESPONSE_STREAM_SHAPE
)
parsed_response = self.parser.parse(response_dict, response_stream_shape)
if response_dict["status_code"] != 200:
decoded_body = response_dict["body"].decode()

25
litellm/llms/bedrock/common_utils.py
View file

@ -4,6 +4,7 @@ from __future__ import annotations
Common utilities used across bedrock chat/embedding/image generation
"""
import functools
import json
import os
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
@ -963,10 +964,8 @@ def _load_bedrock_response_stream_shape():
"""
Load the ResponseStream shape from botocore's bundled bedrock-runtime schema.
Called once at module import time; the result is stored in
``BEDROCK_RESPONSE_STREAM_SHAPE`` and reused for the process lifetime.
Returns ``None`` if botocore is unavailable or the service model cannot be
loaded, so the module still imports cleanly.
loaded.
"""
try:
from botocore.loaders import Loader
@ -977,15 +976,22 @@ def _load_bedrock_response_stream_shape():
return ServiceModel(service_dict).shape_for("ResponseStream")
except Exception as e:
verbose_logger.warning(
"litellm: could not pre-load bedrock-runtime response stream shape "
"litellm: could not load bedrock-runtime response stream shape "
"— Bedrock event-stream decoding will be unavailable. Error: %s",
e,
)
return None
# Eagerly resolved once per process — avoids per-instance or per-request disk I/O.
BEDROCK_RESPONSE_STREAM_SHAPE = _load_bedrock_response_stream_shape()
@functools.lru_cache(maxsize=1)
def get_bedrock_response_stream_shape():
"""
Lazily load and cache the bedrock-runtime ResponseStream shape for the process.
Avoids importing botocore (and logging warnings) unless Bedrock event-stream
decoding is actually needed.
"""
return _load_bedrock_response_stream_shape()
class BedrockEventStreamDecoderBase:
@ -999,7 +1005,8 @@ class BedrockEventStreamDecoderBase:
self.parser = EventStreamJSONParser()
def _parse_message_from_event(self, event) -> Optional[str]:
if BEDROCK_RESPONSE_STREAM_SHAPE is None:
response_stream_shape = get_bedrock_response_stream_shape()
if response_stream_shape is None:
raise BedrockError(
status_code=500,
message=(
@ -1008,9 +1015,7 @@ class BedrockEventStreamDecoderBase:
),
)
response_dict = event.to_response_dict()
parsed_response = self.parser.parse(
response_dict, BEDROCK_RESPONSE_STREAM_SHAPE
)
parsed_response = self.parser.parse(response_dict, response_stream_shape)
if response_dict["status_code"] != 200:
decoded_body = response_dict["body"].decode()

2
litellm/llms/bedrock/embed/cohere_transformation.py
View file

@ -22,7 +22,7 @@ class BedrockCohereEmbeddingConfig:
) -> dict:
for k, v in non_default_params.items():
if k == "encoding_format":
optional_params["embedding_types"] = v
optional_params["embedding_types"] = v if isinstance(v, list) else [v]
elif k == "dimensions":
optional_params["output_dimension"] = v
return optional_params

28
litellm/llms/dashscope/common_utils.py Normal file
View file

@ -0,0 +1,28 @@
"""
Common utilities for the DashScope LLM provider.
"""
from typing import Optional
import httpx
from litellm.llms.base_llm.chat.transformation import BaseLLMException
class DashScopeError(BaseLLMException):
"""Exception class for DashScope provider errors."""
def __init__(
self,
status_code: int,
message: str,
headers: Optional[httpx.Headers] = None,
):
self.status_code = status_code
self.message = message
self.headers = headers or httpx.Headers()
super().__init__(
status_code=status_code,
message=message,
headers=dict(self.headers),
)

7
litellm/llms/dashscope/embed/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""
DashScope Embedding Module
"""
from .transformation import DashScopeEmbeddingConfig
__all__ = ["DashScopeEmbeddingConfig"]

191
litellm/llms/dashscope/embed/transformation.py Normal file
View file

@ -0,0 +1,191 @@
"""
Transformation logic from OpenAI /v1/embeddings format to DashScope's /v1/embeddings format.
Supports
- text-embedding-v4
- text-embedding-v3
Endpoint
- https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings
Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
"""
from typing import List, Optional, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
from litellm.types.utils import EmbeddingResponse, Usage
from ..common_utils import DashScopeError
DEFAULT_API_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
"""
Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
DashScope exposes an OpenAI-compatible /v1/embeddings endpoint, so the
request and response shapes are nearly identical to OpenAI's.
"""
def __init__(self) -> None:
pass
def get_supported_openai_params(self, model: str) -> List[str]:
# DashScope's compatible-mode embeddings API accepts the same params as OpenAI.
# `dimensions` / `encoding_format` are only honored by text-embedding-v3 / v4;
# earlier versions silently ignore them server-side.
return ["dimensions", "encoding_format", "user"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool = False,
) -> dict:
supported = self.get_supported_openai_params(model)
for k, v in non_default_params.items():
if v is None:
continue
if k in supported:
optional_params[k] = v
# unsupported params are dropped when drop_params=True;
# the upstream _check_valid_arg already raised UnsupportedParamsError
# for drop_params=False before this method is called.
return optional_params
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("DASHSCOPE_API_KEY")
if api_key is None:
raise ValueError(
"DashScope API key is required. Set 'DASHSCOPE_API_KEY' env var or pass api_key explicitly."
)
default_headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}",
}
return {**default_headers, **headers}
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
base = api_base or get_secret_str("DASHSCOPE_API_BASE") or DEFAULT_API_BASE
base = base.rstrip("/")
if base.endswith("/embeddings"):
return base
return f"{base}/embeddings"
def transform_embedding_request(
self,
model: str,
input: AllEmbeddingInputValues,
optional_params: dict,
headers: dict,
) -> dict:
data: dict = {
"model": model,
"input": input,
}
for key in ("dimensions", "encoding_format", "user"):
value = optional_params.get(key)
if value is not None:
data[key] = value
return data
def transform_embedding_response(
self,
model: str,
raw_response: httpx.Response,
model_response: EmbeddingResponse,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str],
request_data: dict,
optional_params: dict,
litellm_params: dict,
) -> EmbeddingResponse:
try:
response_json = raw_response.json()
except Exception as e:
raise DashScopeError(
status_code=raw_response.status_code,
message=f"Failed to parse DashScope response as JSON: {str(e)}",
)
logging_obj.post_call(
input=request_data.get("input"),
api_key=api_key,
additional_args={"complete_input_dict": request_data},
original_response=response_json,
)
if "error" in response_json:
error = response_json["error"]
message = (
error.get("message", str(error))
if isinstance(error, dict)
else str(error)
)
raise DashScopeError(
status_code=raw_response.status_code,
message=message,
)
model_response.object = "list"
model_response.data = response_json.get("data", [])
model_response.model = response_json.get("model", model)
usage = response_json.get("usage") or {}
prompt_tokens = usage.get("prompt_tokens", 0)
total_tokens = usage.get("total_tokens", prompt_tokens)
setattr(
model_response,
"usage",
Usage(
prompt_tokens=prompt_tokens,
completion_tokens=0,
total_tokens=total_tokens,
),
)
if "id" in response_json:
setattr(model_response, "id", response_json["id"])
return model_response
def get_error_class(
self,
error_message: str,
status_code: int,
headers: Union[dict, httpx.Headers],
) -> BaseLLMException:
if isinstance(headers, dict):
headers = httpx.Headers(headers)
return DashScopeError(
status_code=status_code,
message=error_message,
headers=headers,
)

7
litellm/llms/dashscope/rerank/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""
DashScope Rerank Module
"""
from .transformation import DashScopeRerankConfig
__all__ = ["DashScopeRerankConfig"]

241
litellm/llms/dashscope/rerank/transformation.py Normal file
View file

@ -0,0 +1,241 @@
"""
Transformation logic for DashScope's OpenAI-compatible /v1/reranks API.
Supports
- qwen3-rerank
(Other DashScope rerankers — gte-rerank-v2 / qwen3-vl-rerank — share the same
endpoint but have not been validated against this transformer. Behavior with
those models is undefined.)
Endpoint
- https://dashscope.aliyuncs.com/compatible-api/v1/reranks
Note: chat/embed live under `/compatible-mode/v1/`, but DashScope's rerank
route is exposed under `/compatible-api/v1/reranks` per the docs. Override
with `DASHSCOPE_API_BASE_RERANK` to point at a different host or path.
Empirically, qwen3-rerank accepts `return_documents=true` and echoes
`results[].document.text` back, even though the public docs list the flag
as supported only for gte-rerank-v2 / qwen3-vl-rerank.
Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api
"""
from typing import Any, Dict, List, Optional, Union
import httpx
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.rerank import (
OptionalRerankParams,
RerankBilledUnits,
RerankResponse,
RerankResponseMeta,
RerankTokens,
)
from ..common_utils import DashScopeError
DEFAULT_RERANK_URL = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
class DashScopeRerankConfig(BaseRerankConfig):
"""
Reference: https://help.aliyun.com/zh/model-studio/text-rerank-api
Targets DashScope's qwen3-rerank model. Request fields: model, query,
documents, top_n, return_documents. Response: results[].index,
results[].relevance_score, optionally results[].document.text (when
return_documents=true), plus a top-level usage.total_tokens counter.
"""
def __init__(self) -> None:
pass
def get_complete_url(
self,
api_base: Optional[str],
model: str,
optional_params: Optional[dict] = None,
) -> str:
if api_base is None:
api_base = get_secret_str("DASHSCOPE_API_BASE_RERANK") or DEFAULT_RERANK_URL
if api_base == DEFAULT_RERANK_URL:
return DEFAULT_RERANK_URL
cleaned = api_base.rstrip("/")
if cleaned.endswith("/reranks") or cleaned.endswith("/rerank"):
return cleaned
if cleaned.endswith("/v1"):
return f"{cleaned}/reranks"
# Unknown base: append /reranks rather than silently ignoring the caller's api_base.
return f"{cleaned}/reranks"
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
optional_params: Optional[dict] = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("DASHSCOPE_API_KEY")
if api_key is None:
raise ValueError(
"DashScope API key is required. Set 'DASHSCOPE_API_KEY' env var or pass api_key explicitly."
)
default_headers = {
"Authorization": f"Bearer {api_key}",
"accept": "application/json",
"content-type": "application/json",
}
return {**default_headers, **headers}
def get_supported_cohere_rerank_params(self, model: str) -> list:
return ["query", "documents", "top_n", "return_documents"]
def map_cohere_rerank_params(
self,
non_default_params: Optional[dict],
model: str,
drop_params: bool,
query: str,
documents: List[Union[str, Dict[str, Any]]],
custom_llm_provider: Optional[str] = None,
top_n: Optional[int] = None,
rank_fields: Optional[List[str]] = None,
return_documents: Optional[bool] = True,
max_chunks_per_doc: Optional[int] = None,
max_tokens_per_doc: Optional[int] = None,
) -> Dict:
# qwen3-rerank accepts query/documents/top_n/return_documents. The
# rest (rank_fields, max_*_per_doc) are silently dropped.
params: OptionalRerankParams = OptionalRerankParams(
query=query,
documents=documents,
)
if top_n is not None:
params["top_n"] = top_n
if return_documents is not None:
params["return_documents"] = return_documents
return dict(params)
def transform_rerank_request(
self,
model: str,
optional_rerank_params: Dict,
headers: dict,
litellm_params: Optional[dict] = None,
) -> dict:
if "query" not in optional_rerank_params:
raise ValueError("query is required for DashScope rerank")
if "documents" not in optional_rerank_params:
raise ValueError("documents is required for DashScope rerank")
request: Dict[str, Any] = {
"model": model,
"query": optional_rerank_params["query"],
"documents": optional_rerank_params["documents"],
}
if optional_rerank_params.get("top_n") is not None:
request["top_n"] = optional_rerank_params["top_n"]
if optional_rerank_params.get("return_documents") is not None:
request["return_documents"] = optional_rerank_params["return_documents"]
return request
def transform_rerank_response(
self,
model: str,
raw_response: httpx.Response,
model_response: RerankResponse,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str] = None,
request_data: Optional[dict] = None,
optional_params: Optional[dict] = None,
litellm_params: Optional[dict] = None,
) -> RerankResponse:
request_data = request_data or {}
optional_params = optional_params or {}
litellm_params = litellm_params or {}
try:
response_json = raw_response.json()
except Exception:
raise DashScopeError(
status_code=raw_response.status_code,
message=raw_response.text,
)
logging_obj.post_call(
input=request_data.get("query"),
api_key=api_key,
additional_args={"complete_input_dict": request_data},
original_response=response_json,
)
# DashScope error envelope: {"code": "...", "message": "...", "request_id": "..."}
if "code" in response_json and "results" not in response_json:
raise DashScopeError(
status_code=raw_response.status_code,
message=response_json.get("message", str(response_json)),
)
results = response_json.get("results")
if results is None:
raise DashScopeError(
status_code=raw_response.status_code,
message=f"No results in DashScope rerank response: {response_json}",
)
# qwen3-rerank returns:
# {"index": int, "relevance_score": float}
# plus, when return_documents=true was sent:
# "document": {"text": "..."}
# which already matches LiteLLM's RerankResponseDocument shape.
transformed_results: List[dict] = []
for r in results:
item: Dict[str, Any] = {
"index": r["index"],
"relevance_score": r["relevance_score"],
}
doc = r.get("document")
if isinstance(doc, dict):
item["document"] = doc
elif isinstance(doc, str):
# Defensive: spec says dict, but normalize string-shaped echoes.
item["document"] = {"text": doc}
transformed_results.append(item)
usage = response_json.get("usage") or {}
total_tokens = usage.get("total_tokens")
billed_units = RerankBilledUnits(total_tokens=total_tokens)
tokens = RerankTokens(input_tokens=total_tokens)
meta = RerankResponseMeta(billed_units=billed_units, tokens=tokens)
return RerankResponse(
id=response_json.get("id") or str(uuid.uuid4()),
results=transformed_results, # type: ignore
meta=meta,
)
def get_error_class(
self,
error_message: str,
status_code: int,
headers: Union[dict, httpx.Headers],
) -> BaseLLMException:
if isinstance(headers, dict):
headers = httpx.Headers(headers)
return DashScopeError(
status_code=status_code,
message=error_message,
headers=headers,
)

106
litellm/llms/deepseek/chat/transformation.py
View file

@ -2,13 +2,15 @@
Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completions`
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
@ -62,6 +64,48 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
return optional_params
def _fill_reasoning_content(
self, messages: List[AllMessageValues]
) -> List[AllMessageValues]:
"""
DeepSeek thinking mode requires `reasoning_content` to be passed back on
every assistant message in multi-turn conversations. If it is missing,
the API returns:
"The reasoning_content in the thinking mode must be passed back to the API."
For each assistant message that is missing `reasoning_content`:
1. Promote it from `provider_specific_fields["reasoning_content"]` if present
(LiteLLM stores provider-specific response fields there).
2. Otherwise inject a single space — the minimum value the API accepts.
"""
result: List[AllMessageValues] = []
for msg in messages:
if msg.get("role") == "assistant" and not msg.get("reasoning_content"):
patched = dict(cast(dict, msg))
provider_fields = patched.get("provider_specific_fields") or {}
stored = provider_fields.get("reasoning_content")
if stored:
patched["reasoning_content"] = stored
cleaned = dict(provider_fields)
cleaned.pop("reasoning_content", None)
patched["provider_specific_fields"] = cleaned
else:
litellm.verbose_logger.warning(
"DeepSeek thinking mode: assistant message is missing "
"`reasoning_content` and none was saved in "
"`provider_specific_fields`. A single-space placeholder "
"is being injected to satisfy API validation, but the "
"model will receive a blank reasoning chain for this turn, "
"which may silently degrade multi-turn response quality. "
"Preserve `reasoning_content` from the original assistant "
"response when building multi-turn conversation history."
)
patched["reasoning_content"] = " "
result.append(cast(AllMessageValues, patched))
else:
result.append(msg)
return result
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
@ -91,6 +135,66 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
messages=messages, model=model, is_async=False
)
def _thinking_mode_active(self, model: str, optional_params: dict) -> bool:
"""
Returns True only when thinking mode is actually active for this request:
- model supports reasoning (capability check)
- user explicitly passed thinking={"type": "enabled"} (opt-in check)
"""
return (
supports_reasoning(model=model, custom_llm_provider="deepseek")
and (optional_params.get("thinking") or {}).get("type") == "enabled"
)
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Ensures `reasoning_content` is forwarded on assistant messages for
multi-turn thinking-mode conversations (issue #28045).
Only runs when thinking mode is actually active - guarded by both
supports_reasoning() (model capability) and optional_params["thinking"]
(user explicitly enabled it), preventing spurious injection on models
like deepseek-v3.2 that support thinking as opt-in but not always-on.
"""
if self._thinking_mode_active(model=model, optional_params=optional_params):
messages = self._fill_reasoning_content(messages)
return super().transform_request(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
async def async_transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Async equivalent of transform_request — applies the same reasoning_content
fix for multi-turn thinking-mode conversations.
"""
if self._thinking_mode_active(model=model, optional_params=optional_params):
messages = self._fill_reasoning_content(messages)
return await super().async_transform_request(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:

133
litellm/llms/deepseek/messages/transformation.py Normal file
View file

@ -0,0 +1,133 @@
"""
DeepSeek Anthropic-compatible messages transformation config.
"""
from typing import Any, Dict, List, Optional, Tuple
import litellm
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
"""
DeepSeek exposes an Anthropic-compatible Messages API at
https://api.deepseek.com/anthropic.
It accepts the native Anthropic Messages conversation shape, including
thinking blocks in assistant history, but rejects Anthropic's explicit
custom-tool discriminator (`{"type": "custom"}`).
"""
@property
def custom_llm_provider(self) -> Optional[str]:
return "deepseek"
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
return api_key or get_secret_str("DEEPSEEK_API_KEY") or litellm.api_key
@staticmethod
def get_api_base(api_base: Optional[str] = None) -> str:
return (
api_base
or get_secret_str("DEEPSEEK_ANTHROPIC_API_BASE")
or get_secret_str("DEEPSEEK_API_BASE")
or "https://api.deepseek.com/anthropic"
)
def validate_anthropic_messages_environment(
self,
headers: dict,
model: str,
messages: List[Any],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> Tuple[dict, Optional[str]]:
dynamic_api_key = self.get_api_key(api_key=api_key)
if (
"x-api-key" not in headers
and "authorization" not in headers
and dynamic_api_key is not None
):
headers["x-api-key"] = dynamic_api_key
if "anthropic-version" not in headers:
headers["anthropic-version"] = "2023-06-01"
if "content-type" not in headers:
headers["content-type"] = "application/json"
headers = self._update_headers_with_anthropic_beta(
headers=headers,
optional_params=optional_params,
custom_llm_provider=self.custom_llm_provider or "deepseek",
)
return headers, api_base
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
base_url = self.get_api_base(api_base=api_base).rstrip("/")
if base_url.endswith("/v1/messages") and "/anthropic/" in base_url:
return base_url
if base_url.endswith("/v1/messages"):
base_url = base_url[: -len("/v1/messages")]
if base_url.endswith("/v1"):
base_url = base_url[: -len("/v1")]
if base_url.endswith("/beta"):
base_url = base_url[: -len("/beta")]
if not base_url.endswith("/anthropic") and "/anthropic/" not in base_url:
base_url = f"{base_url}/anthropic"
return f"{base_url}/v1/messages"
@staticmethod
def _sanitize_tools_for_deepseek(tools: Any) -> Any:
if not isinstance(tools, list):
return tools
sanitized_tools = []
for tool in tools:
if isinstance(tool, dict) and tool.get("type") == "custom":
sanitized_tool = dict(tool)
sanitized_tool.pop("type", None)
sanitized_tools.append(sanitized_tool)
else:
sanitized_tools.append(tool)
return sanitized_tools
def transform_anthropic_messages_request(
self,
model: str,
messages: List[Dict],
anthropic_messages_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Dict:
anthropic_messages_request = super().transform_anthropic_messages_request(
model=model,
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
if "tools" in anthropic_messages_request:
anthropic_messages_request["tools"] = self._sanitize_tools_for_deepseek(
anthropic_messages_request["tools"]
)
return anthropic_messages_request

24
litellm/llms/gemini/chat/transformation.py
View file

@ -1,5 +1,7 @@
from typing import List, Optional, cast
import litellm
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_generic_image_chunk_to_openai_image_obj,
convert_to_anthropic_image_obj,
@ -101,7 +103,10 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
return supported_params
def _transform_messages(
self, messages: List[AllMessageValues], model: Optional[str] = None
self,
messages: List[AllMessageValues],
model: Optional[str] = None,
litellm_params: Optional[dict] = None,
) -> List[ContentType]:
"""
Google AI Studio Gemini does not support HTTP/HTTPS URLs for files.
@ -141,14 +146,23 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
img_element["image_url"] = converted_image_url # type: ignore
elif element.get("type") == "file":
file_element = cast(ChatCompletionFileObject, element)
file_id = file_element["file"].get("file_id")
_file_field = file_element.get("file")
if _file_field is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=model,
llm_provider="gemini",
)
file_id = _file_field.get("file_id")
if file_id and ("http://" in file_id or "https://" in file_id):
# Convert HTTP/HTTPS file URL to base64 data
try:
base64_data = convert_url_to_base64(file_id)
file_element["file"]["file_data"] = base64_data # type: ignore
file_element["file"].pop("file_id", None) # type: ignore
_file_field["file_data"] = base64_data # type: ignore
_file_field.pop("file_id", None) # type: ignore
except Exception:
# If conversion fails, leave as is and let the API handle it
pass
return _gemini_convert_messages_with_history(messages=messages, model=model)
return _gemini_convert_messages_with_history(
messages=messages, model=model, litellm_params=litellm_params
)

8
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -287,7 +287,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
content_item["image_url"] = new_image_url_obj
elif content_item.get("type") == "file":
content_item = cast(ChatCompletionFileObject, content_item)
file_obj = content_item["file"]
file_obj = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj = ChatCompletionFileObjectFile(
**{ # type: ignore
k: v

20
litellm/llms/sagemaker/common_utils.py
View file

@ -1,3 +1,4 @@
import functools
import json
from typing import AsyncIterator, Iterator, List, Optional, Union
@ -22,14 +23,22 @@ def _load_sagemaker_response_stream_shape():
)
except Exception as e:
verbose_logger.warning(
"litellm: could not pre-load sagemaker-runtime response stream shape "
"litellm: could not load sagemaker-runtime response stream shape "
"— SageMaker event-stream decoding will be unavailable. Error: %s",
e,
)
return None
SAGEMAKER_RESPONSE_STREAM_SHAPE = _load_sagemaker_response_stream_shape()
@functools.lru_cache(maxsize=1)
def get_sagemaker_response_stream_shape():
"""
Lazily load and cache the sagemaker-runtime stream shape for the process.
Avoids importing botocore (and logging warnings) unless SageMaker event-stream
decoding is actually needed.
"""
return _load_sagemaker_response_stream_shape()
class SagemakerError(BaseLLMException):
@ -207,7 +216,8 @@ class AWSEventStreamDecoder:
verbose_logger.error(f"Final error parsing accumulated JSON: {e}")
def _parse_message_from_event(self, event) -> Optional[str]:
if SAGEMAKER_RESPONSE_STREAM_SHAPE is None:
response_stream_shape = get_sagemaker_response_stream_shape()
if response_stream_shape is None:
raise SagemakerError(
status_code=500,
message=(
@ -216,9 +226,7 @@ class AWSEventStreamDecoder:
),
)
response_dict = event.to_response_dict()
parsed_response = self.parser.parse(
response_dict, SAGEMAKER_RESPONSE_STREAM_SHAPE
)
parsed_response = self.parser.parse(response_dict, response_stream_shape)
if response_dict["status_code"] != 200:
raise ValueError(f"Bad response code, expected 200: {response_dict}")

503
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -6,13 +6,16 @@ Why separate file? Make it easy to see how transformation works
import json
import os
from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union, cast
import re
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast
from urllib.parse import quote
import httpx
from pydantic import BaseModel
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_get_image_mime_type_from_url,
)
@ -57,6 +60,45 @@ from ..common_utils import (
get_supports_system_message,
)
# Typed as Any to avoid introducing a module-load-time cyclic import to
# vertex_llm_base. The instance is lazily constructed by _get_vertex_base()
# the first time GCS metadata needs to be fetched.
_GCS_METADATA_VERTEX_BASE: Optional[Any] = None
# Shared sync client for GCS JSON API metadata reads so proxy/SSL settings
# from litellm's HTTP stack apply (see Greptile review on PR #27278).
_GCS_METADATA_HTTP_HANDLER: Optional[HTTPHandler] = None
_GEMINI_MIME_TYPE_ALIASES: Dict[str, str] = {
"image/jpg": "image/jpeg",
}
def _apply_gemini_mime_type_aliases(mime_type: str) -> str:
"""Normalize known MIME aliases only; does not consult the file-type registry.
Also strips MIME parameters (e.g. ``; charset=utf-8``) so that values
sourced from GCS object metadata (``contentType``) validate correctly.
"""
normalized = mime_type.split(";", 1)[0].strip().lower()
return _GEMINI_MIME_TYPE_ALIASES.get(normalized, normalized)
def _get_vertex_base() -> Any:
"""Lazily return the shared VertexBase instance to avoid a module-load-time cyclic import."""
global _GCS_METADATA_VERTEX_BASE
if _GCS_METADATA_VERTEX_BASE is None:
from ..vertex_llm_base import VertexBase
_GCS_METADATA_VERTEX_BASE = VertexBase()
return _GCS_METADATA_VERTEX_BASE
def _get_gcs_metadata_http_handler() -> HTTPHandler:
global _GCS_METADATA_HTTP_HANDLER
if _GCS_METADATA_HTTP_HANDLER is None:
_GCS_METADATA_HTTP_HANDLER = HTTPHandler(timeout=5.0)
return _GCS_METADATA_HTTP_HANDLER
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -171,12 +213,299 @@ def _apply_gemini_metadata(
return cast(PartType, part_dict)
def _parse_gs_uri(gs_uri: str) -> Tuple[str, str]:
if not gs_uri.startswith("gs://"):
raise ValueError(f"Invalid gs URI: {gs_uri}")
uri_without_scheme = gs_uri[5:] # drop gs://
uri_parts = uri_without_scheme.split("/", 1)
if len(uri_parts) != 2 or not uri_parts[0] or not uri_parts[1]:
raise ValueError(f"Invalid gs URI: {gs_uri}")
return uri_parts[0], uri_parts[1]
def _is_valid_gcs_bucket_name(bucket: str) -> bool:
"""
Validate bucket name against core GCS naming constraints.
"""
bucket_length = len(bucket)
max_bucket_length = 222 if "." in bucket else 63
if bucket_length < 3 or bucket_length > max_bucket_length:
return False
if "." in bucket and any(
len(label) == 0 or len(label) > 63 for label in bucket.split(".")
):
return False
if not re.fullmatch(r"[a-z0-9][a-z0-9._-]*[a-z0-9]", bucket):
return False
if ".." in bucket:
return False
if re.fullmatch(r"\d+\.\d+\.\d+\.\d+", bucket):
return False
return True
def _gs_uri_requires_content_type_metadata(url: str) -> bool:
"""
True when _process_gemini_media would call _get_gcs_object_content_type
(extension-less gs:// and no explicit format passed into that helper).
"""
if "gs://" not in url:
return False
extension_with_dot = os.path.splitext(url)[-1]
extension = extension_with_dot[1:] if extension_with_dot else ""
return len(extension) == 0
def _image_url_payload_may_need_sync_gcs_metadata_fetch(
raw_image_url: Any,
) -> bool:
"""
True when this image_url value (content-part image_url or assistant ``images[]``
entry) can trigger a blocking GCS metadata read for MIME resolution.
"""
fmt: Optional[str] = None
url: Optional[str] = None
if isinstance(raw_image_url, dict):
url = raw_image_url.get("url") # type: ignore[assignment]
if not isinstance(url, str):
return False
fmt = (
raw_image_url.get("format")
or raw_image_url.get("mime_type")
or raw_image_url.get("content_type")
)
elif isinstance(raw_image_url, str):
url = raw_image_url
else:
return False
if "gs://" not in url or fmt:
return False
return _gs_uri_requires_content_type_metadata(url)
def _openai_messages_may_need_sync_gcs_metadata_fetch(
messages: List[AllMessageValues],
) -> bool:
"""
Heuristic: True if any message part can trigger a blocking GCS JSON
metadata read inside _transform_request_body (extension-less gs:// without
explicit MIME hints). Covers user/system ``content`` parts and assistant
``images`` (same paths as ``_gemini_convert_messages_with_history``). Used
to decide whether ``async_transform_request_body`` should offload the sync
transform via ``asyncify``.
"""
for raw in messages:
msg: Any = raw
if not isinstance(msg, dict) and hasattr(msg, "model_dump"):
msg = msg.model_dump(exclude_none=False)
if not isinstance(msg, dict):
continue
images_field = msg.get("images")
if isinstance(images_field, list):
for image_item in images_field:
if not isinstance(image_item, dict):
continue
if _image_url_payload_may_need_sync_gcs_metadata_fetch(
image_item.get("image_url")
):
return True
content = msg.get("content")
if not isinstance(content, list):
continue
for item in content:
if not isinstance(item, dict):
continue
itype = item.get("type")
if itype == "image_url":
if _image_url_payload_may_need_sync_gcs_metadata_fetch(
item.get("image_url")
):
return True
elif itype == "file":
file_obj = item.get("file")
if not isinstance(file_obj, dict):
continue
fmt = (
file_obj.get("format")
or file_obj.get("mime_type")
or file_obj.get("content_type")
)
passed = file_obj.get("file_id") or file_obj.get("file_data")
if (
isinstance(passed, str)
and "gs://" in passed
and not fmt
and _gs_uri_requires_content_type_metadata(passed)
):
return True
return False
def _get_gcs_object_content_type(
image_url: str,
vertex_project: Optional[str] = None,
vertex_credentials: Optional[Any] = None,
) -> Optional[str]:
"""
Resolve content type from GCS object metadata.
Only attaches a Bearer token when the caller explicitly supplies Vertex
credentials, to avoid using the server's default Google credentials on
the Gemini API-key (Google AI Studio) path and being used as an oracle
for private GCS object metadata. Without explicit credentials we only
issue an anonymous request, which only succeeds for publicly-readable
objects.
"""
try:
bucket, object_name = _parse_gs_uri(image_url)
except ValueError:
return None
if not _is_valid_gcs_bucket_name(bucket):
return None
headers: Dict[str, str] = {}
explicit_vertex_auth_provided = (
vertex_project is not None or vertex_credentials is not None
)
if explicit_vertex_auth_provided:
try:
access_token, _ = _get_vertex_base().get_access_token(
credentials=vertex_credentials,
project_id=vertex_project,
)
headers["Authorization"] = f"Bearer {access_token}"
except Exception as e:
raise litellm.BadRequestError(
message=(
"Unable to fetch GCS metadata with provided Vertex credentials/project. "
f"Original error: {str(e)}"
),
model=None,
llm_provider="vertex_ai",
)
# Build the URL via httpx.URL with a fixed scheme/host and URL-encode both
# bucket and object so CodeQL does not flag the interpolation as a
# potential SSRF that could resolve to an arbitrary host.
encoded_bucket = quote(bucket, safe="")
encoded_object = quote(object_name, safe="")
metadata_url = httpx.URL(
scheme="https",
host="storage.googleapis.com",
path=f"/storage/v1/b/{encoded_bucket}/o/{encoded_object}",
params={"fields": "contentType"},
)
try:
response = _get_gcs_metadata_http_handler().get(
url=str(metadata_url),
headers=headers or None,
)
except httpx.RequestError as e:
if explicit_vertex_auth_provided:
raise litellm.BadRequestError(
message=(
"Unable to reach GCS JSON API for object metadata with provided "
f"Vertex credentials. {type(e).__name__}: {e}"
),
model=None,
llm_provider="vertex_ai",
) from e
return None
if response.is_error:
if explicit_vertex_auth_provided:
preview = (response.text or "")[:1024]
raise litellm.BadRequestError(
message=(
"Unable to read GCS object metadata with provided Vertex credentials. "
f"HTTP {response.status_code}. Response body (truncated): {preview!r}"
),
model=None,
llm_provider="vertex_ai",
)
return None
try:
payload = response.json()
except ValueError as e:
if explicit_vertex_auth_provided:
raise litellm.BadRequestError(
message=(
"GCS metadata response was not valid JSON when using provided "
f"Vertex credentials (HTTP {response.status_code}). Error: {e}"
),
model=None,
llm_provider="vertex_ai",
) from e
return None
if not isinstance(payload, dict):
if explicit_vertex_auth_provided:
raise litellm.BadRequestError(
message=(
"GCS metadata response was not a JSON object when using provided "
f"Vertex credentials (HTTP {response.status_code})."
),
model=None,
llm_provider="vertex_ai",
)
return None
content_type = payload.get("contentType")
if isinstance(content_type, str) and len(content_type) > 0:
return content_type
if explicit_vertex_auth_provided:
preview = (response.text or "")[:1024]
raise litellm.BadRequestError(
message=(
"GCS metadata JSON did not include a non-empty contentType field when "
f"using provided Vertex credentials (HTTP {response.status_code}). "
f"Body (truncated): {preview!r}"
),
model=None,
llm_provider="vertex_ai",
)
return None
def _normalize_and_validate_gemini_mime_type(
mime_type: str, model: Optional[str]
) -> str:
# Import lazily to avoid a module-level cyclic-import alert with
# litellm.types.files.
from litellm.types.files import get_file_extension_from_mime_type
normalized_mime_type = _apply_gemini_mime_type_aliases(mime_type)
try:
file_extension = get_file_extension_from_mime_type(normalized_mime_type)
file_type = get_file_type_from_extension(file_extension)
except ValueError:
raise litellm.BadRequestError(
message=f"File type not supported by gemini - {normalized_mime_type}",
model=model,
llm_provider="vertex_ai",
)
if not is_gemini_1_5_accepted_file_type(file_type):
raise litellm.BadRequestError(
message=f"File type not supported by gemini - {file_type}",
model=model,
llm_provider="vertex_ai",
)
return get_file_mime_type_for_file_type(file_type)
def _process_gemini_media(
image_url: str,
format: Optional[str] = None,
media_resolution_enum: Optional[Dict[str, str]] = None,
model: Optional[str] = None,
video_metadata: Optional[Dict[str, Any]] = None,
vertex_project: Optional[str] = None,
vertex_credentials: Optional[Any] = None,
) -> PartType:
"""
Given a media URL (image, audio, or video), return the appropriate PartType for Gemini
@ -193,20 +522,63 @@ def _process_gemini_media(
try:
# GCS URIs
if "gs://" in image_url:
# Figure out file type
extension_with_dot = os.path.splitext(image_url)[-1] # Ex: ".png"
extension = extension_with_dot[1:] # Ex: "png"
explicit_gcs_format = False
if not format:
file_type = get_file_type_from_extension(extension)
mime_type: Optional[str] = None
# For extension-less gs:// URIs, we cannot infer from path.
# If callers pass `format`/`mime_type`, this branch is skipped.
if extension:
file_type = get_file_type_from_extension(extension)
# Validate the file type is supported by Gemini
if not is_gemini_1_5_accepted_file_type(file_type):
raise Exception(f"File type not supported by gemini - {file_type}")
# Validate the file type is supported by Gemini
if not is_gemini_1_5_accepted_file_type(file_type):
raise litellm.BadRequestError(
message=f"File type not supported by gemini - {file_type}",
model=model,
llm_provider="vertex_ai",
)
mime_type = get_file_mime_type_for_file_type(file_type)
mime_type = get_file_mime_type_for_file_type(file_type)
else:
mime_type = _get_gcs_object_content_type(
image_url=image_url,
vertex_project=vertex_project,
vertex_credentials=vertex_credentials,
)
if mime_type is None:
raise litellm.BadRequestError(
message=(
f"Unable to determine mime type for gs URI: {image_url}. "
"This gs:// URI has no file extension and GCS metadata "
"lookup failed. Set it explicitly using image_url.format "
"(or image_url.mime_type/content_type) or "
"message.content[].file.format."
),
model=model,
llm_provider="vertex_ai",
)
else:
mime_type = format
explicit_gcs_format = True
if mime_type is None:
raise litellm.BadRequestError(
message=f"File type not supported by gemini - {image_url}",
model=model,
llm_provider="vertex_ai",
)
if explicit_gcs_format:
# Callers who pass format/mime_type explicitly for gs:// URIs
# rely on pass-through to Gemini (pre-PR behavior). Only apply
# known MIME aliases; skip litellm's file-type registry.
mime_type = _apply_gemini_mime_type_aliases(mime_type)
else:
mime_type = _normalize_and_validate_gemini_mime_type(
mime_type=mime_type,
model=model,
)
file_data = FileDataType(mime_type=mime_type, file_uri=image_url)
part: PartType = {"file_data": file_data}
return _apply_gemini_metadata(
@ -258,8 +630,6 @@ def _snake_to_camel(snake_str: str) -> str:
def _camel_to_snake(camel_str: str) -> str:
"""Convert camelCase to snake_case"""
import re
return re.sub(r"(?<!^)(?=[A-Z])", "_", camel_str).lower()
@ -311,6 +681,7 @@ def check_if_part_exists_in_parts(
def _gemini_convert_messages_with_history( # noqa: PLR0915
messages: List[AllMessageValues],
model: Optional[str] = None,
litellm_params: Optional[dict] = None,
) -> List[ContentType]:
"""
Converts given messages from OpenAI format to Gemini format
@ -326,6 +697,16 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
msg_i = 0
tool_call_responses = []
vertex_project = None
vertex_credentials = None
if litellm_params:
vertex_project = litellm_params.get("vertex_project") or litellm_params.get(
"vertex_ai_project"
)
vertex_credentials = litellm_params.get(
"vertex_credentials"
) or litellm_params.get("vertex_ai_credentials")
try:
while msg_i < len(messages):
user_content: List[PartType] = []
@ -351,20 +732,42 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
img_element = element
format: Optional[str] = None
media_resolution_enum: Optional[Dict[str, str]] = None
if isinstance(img_element["image_url"], dict):
image_url = img_element["image_url"]["url"]
format = img_element["image_url"].get("format")
detail = img_element["image_url"].get("detail")
raw_image_url = img_element.get("image_url")
if raw_image_url is None:
raise litellm.BadRequestError(
message="Invalid message content: element type is 'image_url' but 'image_url' field is missing ",
model=model,
llm_provider="vertex_ai",
)
if isinstance(raw_image_url, dict):
image_url = raw_image_url.get("url")
if image_url is None:
raise litellm.BadRequestError(
message="Invalid message content: element type is 'image_url' but 'url' field is missing inside 'image_url' ",
model=model,
llm_provider="vertex_ai",
)
# TypedDict does not declare mime_type/content_type;
# read via Dict[str, Any] for caller-provided MIME fields.
image_url_dict = cast(Dict[str, Any], raw_image_url)
format = (
image_url_dict.get("format")
or image_url_dict.get("mime_type")
or image_url_dict.get("content_type")
)
detail = image_url_dict.get("detail")
media_resolution_enum = (
_convert_detail_to_media_resolution_enum(detail)
)
else:
image_url = img_element["image_url"]
image_url = raw_image_url
_part = _process_gemini_media(
image_url=image_url,
format=format,
media_resolution_enum=media_resolution_enum,
model=model,
vertex_project=vertex_project,
vertex_credentials=vertex_credentials,
)
_parts.append(_part)
elif element["type"] == "input_audio":
@ -390,15 +793,31 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
image_url=openai_image_str,
format=audio_format_modified,
model=model,
vertex_project=vertex_project,
vertex_credentials=vertex_credentials,
)
_parts.append(_part)
elif element["type"] == "file":
file_element = cast(ChatCompletionFileObject, element)
file_id = file_element["file"].get("file_id")
format = file_element["file"].get("format")
file_data = file_element["file"].get("file_data")
detail = file_element["file"].get("detail")
video_metadata = file_element["file"].get("video_metadata")
_file_field = file_element.get("file")
if _file_field is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=model,
llm_provider="vertex_ai",
)
# TypedDict does not declare mime_type/content_type;
# read via Dict[str, Any] for caller-provided MIME fields.
file_dict = cast(Dict[str, Any], _file_field)
file_id = file_dict.get("file_id")
format = (
file_dict.get("format")
or file_dict.get("mime_type")
or file_dict.get("content_type")
)
file_data = file_dict.get("file_data")
detail = file_dict.get("detail")
video_metadata = file_dict.get("video_metadata")
passed_file = file_id or file_data
if passed_file is None:
raise Exception(
@ -417,13 +836,23 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
model=model,
media_resolution_enum=media_resolution_enum,
video_metadata=video_metadata,
vertex_project=vertex_project,
vertex_credentials=vertex_credentials,
)
_parts.append(_part)
except Exception:
raise Exception(
"Unable to determine mime type for file_id: {}, set this explicitly using message[{}].content[{}].file.format".format(
file_id, msg_i, element_idx
)
except litellm.BadRequestError:
raise
except Exception as e:
raise litellm.BadRequestError(
message=(
f"Unable to determine mime type for file: "
f"{file_id or 'provided data'}, set this explicitly "
f"using message[{msg_i}].content[{element_idx}].file.format "
f"(or file.mime_type/content_type). "
f"Original error: {str(e)}"
),
model=model,
llm_provider="vertex_ai",
)
user_content.extend(_parts)
elif _message_content is not None and isinstance(_message_content, str):
@ -528,7 +957,11 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
image_url_obj = image_item.get("image_url")
if isinstance(image_url_obj, dict):
assistant_image_url = image_url_obj.get("url")
format = image_url_obj.get("format")
format = (
image_url_obj.get("format")
or image_url_obj.get("mime_type")
or image_url_obj.get("content_type")
)
detail = image_url_obj.get("detail")
media_resolution_enum = (
_convert_detail_to_media_resolution_enum(detail)
@ -539,6 +972,8 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
format=format,
media_resolution_enum=media_resolution_enum,
model=model,
vertex_project=vertex_project,
vertex_credentials=vertex_credentials,
)
assistant_content.append(_part)
@ -713,11 +1148,11 @@ def _transform_request_body( # noqa: PLR0915
try:
if custom_llm_provider == "gemini":
content = litellm.GoogleAIStudioGeminiConfig()._transform_messages(
messages=messages, model=model
messages=messages, model=model, litellm_params=litellm_params
)
else:
content = litellm.VertexGeminiConfig()._transform_messages(
messages=messages, model=model
messages=messages, model=model, litellm_params=litellm_params
)
tools: Optional[Tools] = optional_params.pop("tools", None)
tool_choice: Optional[ToolConfig] = optional_params.pop("tool_choice", None)
@ -893,6 +1328,20 @@ async def async_transform_request_body(
vertex_auth_header=vertex_auth_header,
)
if _openai_messages_may_need_sync_gcs_metadata_fetch(messages):
# _transform_request_body may issue a sync httpx.get (up to 5s timeout)
# via _get_gcs_object_content_type to fetch GCS object metadata. Run the
# whole sync transformation on a worker thread so it does not block the
# async event loop.
return await asyncify(_transform_request_body)(
messages=messages,
model=model,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
cached_content=cached_content,
optional_params=optional_params,
)
return _transform_request_body(
messages=messages,
model=model,

39
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -2533,9 +2533,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return model_response
def _transform_messages(
self, messages: List[AllMessageValues], model: Optional[str] = None
self,
messages: List[AllMessageValues],
model: Optional[str] = None,
litellm_params: Optional[dict] = None,
) -> List[ContentType]:
return _gemini_convert_messages_with_history(messages=messages, model=model)
return _gemini_convert_messages_with_history(
messages=messages, model=model, litellm_params=litellm_params
)
def get_error_class(
self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers]
@ -3139,6 +3144,31 @@ class ModelResponseIterator:
self.cumulative_tool_call_index: int = 0
self.has_seen_tool_calls: bool = False
@staticmethod
def _check_streaming_error(chunk: dict) -> None:
"""Detect embedded errors (e.g. 429 RESOURCE_EXHAUSTED) in streaming chunks and raise VertexAIError."""
if "error" not in chunk:
return
error_data = chunk["error"]
if not isinstance(error_data, dict):
raise VertexAIError(
status_code=500,
message=f"Unexpected error format in mid-stream chunk: {error_data}",
)
raw_code = error_data.get("code", 500)
if raw_code is None:
raw_code = 500
try:
error_code = int(raw_code)
except (TypeError, ValueError):
error_code = 500
error_message = error_data.get("message", "Unknown error")
error_status = error_data.get("status", "UNKNOWN")
raise VertexAIError(
status_code=error_code,
message=f"{error_status} - {error_message}",
)
def _apply_stream_candidates(
self,
_candidates: List[Candidates],
@ -3256,6 +3286,11 @@ class ModelResponseIterator:
def chunk_parser(self, chunk: dict) -> Optional["ModelResponseStream"]:
try:
verbose_logger.debug(f"RAW GEMINI CHUNK: {chunk}")
# Detect mid-stream error chunks (e.g. 429 RESOURCE_EXHAUSTED).
# Vertex AI can return errors as HTTP 200 but with an "error" field in the SSE body.
self._check_streaming_error(chunk)
from litellm.types.utils import ModelResponseStream
processed_chunk = GenerateContentResponseBody(**chunk) # type: ignore

25
litellm/llms/vertex_ai/vertex_ai_partner_models/main.py
View file

@ -292,22 +292,15 @@ class VertexAIPartnerModels(VertexBase):
Returns:
Dict containing token count information
"""
try:
import vertexai
except Exception as e:
raise VertexAIError(
status_code=400,
message=f"""vertexai import failed please run `pip install -U "google-cloud-aiplatform>=1.38"`. Got error: {e}""",
)
if not (
hasattr(vertexai, "preview") or hasattr(vertexai.preview, "language_models")
):
raise VertexAIError(
status_code=400,
message="""Upgrade vertex ai. Run `pip install "google-cloud-aiplatform>=1.38"`""",
)
# Note: we intentionally do not import `vertexai` (the Gemini SDK shipped
# by `google-cloud-aiplatform`) on this path. Partner models such as
# Claude on Vertex use the Anthropic Messages API protocol directly via
# `:rawPredict`, and `VertexAIPartnerModelsTokenCounter` reaches that
# endpoint with an authenticated httpx client — it never touches the
# Gemini SDK. Requiring `google-cloud-aiplatform>=1.38` here turned a
# SDK-free Anthropic-protocol call into a hard dependency on the Gemini
# SDK (see #28084), breaking `/v1/messages/count_tokens` for Claude-on-
# Vertex on any LiteLLM install without that extra. Stay SDK-free.
try:
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import (
VertexAIPartnerModelsTokenCounter,

27
litellm/main.py
View file

@ -5720,6 +5720,33 @@ def embedding( # noqa: PLR0915
aembedding=aembedding,
headers=headers,
)
elif custom_llm_provider == "dashscope":
dashscope_key = (
api_key or litellm.api_key or get_secret_str("DASHSCOPE_API_KEY")
)
if dashscope_key is None:
raise ValueError(
"Missing API key for DashScope. Set DASHSCOPE_API_KEY environment variable or pass api_key parameter."
)
if extra_headers is not None and isinstance(extra_headers, dict):
headers = extra_headers
else:
headers = {}
response = base_llm_http_handler.embedding(
model=model,
input=input,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
logging_obj=logging,
api_base=api_base,
optional_params=optional_params,
litellm_params={},
model_response=EmbeddingResponse(),
api_key=dashscope_key,
client=client,
aembedding=aembedding,
headers=headers,
)
elif custom_llm_provider == "ovhcloud":
api_key = api_key or litellm.api_key or get_secret_str("OVHCLOUD_API_KEY")
api_base = (

374
litellm/model_prices_and_context_window_backup.json
View file

@ -2112,6 +2112,380 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"azure_ai/gpt-5.4": {
"cache_read_input_token_cost": 2.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_priority": 3e-05,
"output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-2026-03-05": {
"cache_read_input_token_cost": 2.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_priority": 3e-05,
"output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-pro": {
"cache_read_input_token_cost": 3e-06,
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
"cache_read_input_token_cost_priority": 6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_272k_tokens": 6e-05,
"input_cost_per_token_priority": 6e-05,
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00018,
"output_cost_per_token_above_272k_tokens": 0.00027,
"output_cost_per_token_priority": 0.00036,
"output_cost_per_token_above_272k_tokens_priority": 0.00054,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-pro",
"supported_endpoints": [
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-pro-2026-03-05": {
"cache_read_input_token_cost": 3e-06,
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
"cache_read_input_token_cost_priority": 6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_272k_tokens": 6e-05,
"input_cost_per_token_priority": 6e-05,
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00018,
"output_cost_per_token_above_272k_tokens": 0.00027,
"output_cost_per_token_priority": 0.00036,
"output_cost_per_token_above_272k_tokens_priority": 0.00054,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-pro",
"supported_endpoints": [
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-mini": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_above_272k_tokens": 1.5e-07,
"cache_read_input_token_cost_priority": 1.5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 3e-07,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_above_272k_tokens": 1.5e-06,
"input_cost_per_token_priority": 1.5e-06,
"input_cost_per_token_above_272k_tokens_priority": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"output_cost_per_token_above_272k_tokens": 6.75e-06,
"output_cost_per_token_priority": 9e-06,
"output_cost_per_token_above_272k_tokens_priority": 1.35e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-mini",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.4-mini-2026-03-17": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_above_272k_tokens": 1.5e-07,
"cache_read_input_token_cost_priority": 1.5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 3e-07,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_above_272k_tokens": 1.5e-06,
"input_cost_per_token_priority": 1.5e-06,
"input_cost_per_token_above_272k_tokens_priority": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"output_cost_per_token_above_272k_tokens": 6.75e-06,
"output_cost_per_token_priority": 9e-06,
"output_cost_per_token_above_272k_tokens_priority": 1.35e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-mini",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.4-nano": {
"cache_read_input_token_cost": 2e-08,
"cache_read_input_token_cost_above_272k_tokens": 4e-08,
"cache_read_input_token_cost_priority": 4e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-08,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
"input_cost_per_token_priority": 4e-07,
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"output_cost_per_token_above_272k_tokens": 1.875e-06,
"output_cost_per_token_priority": 2.5e-06,
"output_cost_per_token_above_272k_tokens_priority": 3.75e-06,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-nano",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.4-nano-2026-03-17": {
"cache_read_input_token_cost": 2e-08,
"cache_read_input_token_cost_above_272k_tokens": 4e-08,
"cache_read_input_token_cost_priority": 4e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-08,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
"input_cost_per_token_priority": 4e-07,
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"output_cost_per_token_above_272k_tokens": 1.875e-06,
"output_cost_per_token_priority": 2.5e-06,
"output_cost_per_token_above_272k_tokens_priority": 3.75e-06,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-nano",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/model_router": {
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 0,

62
litellm/proxy/_experimental/mcp_server/oauth_utils.py
View file

@ -30,23 +30,43 @@ _DEFAULT_PORTS = {"http": 80, "https": 443}
_TRUSTED_REDIRECT_ORIGINS_ENV = "MCP_TRUSTED_REDIRECT_ORIGINS"
_warned_invalid_proxy_base_url: Optional[str] = None
def _resolve_proxy_base_url_env() -> Optional[str]:
global _warned_invalid_proxy_base_url
configured = os.environ.get("PROXY_BASE_URL", "").strip()
if not configured:
return None
parsed = urlparse(configured)
if parsed.scheme in ("http", "https") and parsed.netloc:
normalized = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", ""))
return normalized.rstrip("/")
if _warned_invalid_proxy_base_url != configured:
verbose_logger.warning(
"PROXY_BASE_URL=%r is not a valid http(s) URL (missing scheme "
"or host) and will be ignored for MCP OAuth origin resolution. "
"Set it to a full URL like https://litellm.example.com.",
configured,
)
_warned_invalid_proxy_base_url = configured
return None
def get_request_base_url(request: Request) -> str:
"""
Get the base URL for the request, considering X-Forwarded-* headers.
X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured
when the request comes from a configured trusted proxy
(``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``).
Otherwise the request's literal ``base_url`` is returned, so an
untrusted caller cannot poison OAuth-discovery / redirect_uri values
by injecting headers.
Args:
request: FastAPI Request object
Returns:
The reconstructed base URL (e.g., "https://proxy.example.com")
Resolution order: ``PROXY_BASE_URL`` env var, then X-Forwarded-* when
the caller is a trusted proxy (``use_x_forwarded_for`` enabled AND
caller in ``mcp_trusted_proxy_ranges``), otherwise the request's
literal ``base_url``. Untrusted callers cannot poison OAuth-discovery
/ redirect_uri values by injecting headers.
"""
configured = _resolve_proxy_base_url_env()
if configured:
return configured
base_url = str(request.base_url).rstrip("/")
parsed = urlparse(base_url)
@ -284,4 +304,22 @@ def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
if _matches_trusted_origin_entry(redirect_netloc, entry):
return
verbose_logger.warning(
"MCP OAuth: rejecting redirect_uri %r as invalid_request. "
"Computed proxy base=%r (PROXY_BASE_URL=%r). "
"Inbound headers: X-Forwarded-Proto=%r X-Forwarded-Host=%r "
"X-Forwarded-Port=%r Host=%r. "
"Trusted-redirect-origins env=%r. "
"If this should be accepted, either align ingress X-Forwarded-* "
"with the browser URL, set PROXY_BASE_URL to your public origin, "
"or add the redirect_uri host to MCP_TRUSTED_REDIRECT_ORIGINS.",
redirect_uri,
proxy_base,
os.environ.get("PROXY_BASE_URL"),
request.headers.get("X-Forwarded-Proto"),
request.headers.get("X-Forwarded-Host"),
request.headers.get("X-Forwarded-Port"),
request.headers.get("Host"),
os.environ.get(_TRUSTED_REDIRECT_ORIGINS_ENV),
)
raise HTTPException(status_code=400, detail="invalid_request")

2
litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

6
litellm/proxy/_experimental/out/__next.__PAGE__.txt
View file

@ -1,10 +1,10 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/6188170a32c9a3c3.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/0279e5299e9f6e98.js","/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ca7a3fdb635fb7dc.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/934dbc43f8c1abde.js","/litellm-asset-prefix/_next/static/chunks/d6ab357d1bbb53f0.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4a0ccb5ed3d0c33f.js"],"default"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/6188170a32c9a3c3.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/e1a670efcb966aaa.js","/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ca7a3fdb635fb7dc.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/934dbc43f8c1abde.js","/litellm-asset-prefix/_next/static/chunks/d6ab357d1bbb53f0.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4a0ccb5ed3d0c33f.js"],"default"]
1a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
1b:"$Sreact.suspense"
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
@ -16,7 +16,7 @@ b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/61
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","async":true}]
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","async":true}]
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/0279e5299e9f6e98.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/e1a670efcb966aaa.js","async":true}]
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","async":true}]
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}]
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}]

6
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -3,4 +3,4 @@
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -5,4 +5,4 @@
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/__next._tree.txt
View file

@ -2,4 +2,4 @@
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}

0
litellm/proxy/_experimental/out/_next/static/bCuM4c6QggtFAyOTSKpXz/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/LpD6ruZoEpvYpT5IvMEoa/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/bCuM4c6QggtFAyOTSKpXz/_clientMiddlewareManifest.json → litellm/proxy/_experimental/out/_next/static/LpD6ruZoEpvYpT5IvMEoa/_clientMiddlewareManifest.json
View file

0
litellm/proxy/_experimental/out/_next/static/bCuM4c6QggtFAyOTSKpXz/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/LpD6ruZoEpvYpT5IvMEoa/_ssgManifest.js
View file

16
litellm/proxy/_experimental/out/_next/static/chunks/0279e5299e9f6e98.js → litellm/proxy/_experimental/out/_next/static/chunks/e1a670efcb966aaa.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_not-found.html
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_not-found.txt
View file

@ -10,7 +10,7 @@ b:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/li
d:I[168027,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"P":null,"b":"bCuM4c6QggtFAyOTSKpXz","c":["","_not-found"],"q":"","i":false,"f":[[["",{"children":["/_not-found",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],null,["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L9",null,{"children":"$La"}],["$","div",null,{"hidden":true,"children":["$","$Lb",null,{"children":["$","$7",null,{"name":"Next.Metadata","children":"$Lc"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$d","$undefined"],"S":true}
0:{"P":null,"b":"LpD6ruZoEpvYpT5IvMEoa","c":["","_not-found"],"q":"","i":false,"f":[[["",{"children":["/_not-found",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],null,["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L9",null,{"children":"$La"}],["$","div",null,{"hidden":true,"children":["$","$Lb",null,{"children":["$","$7",null,{"name":"Next.Metadata","children":"$Lc"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$d","$undefined"],"S":true}
a:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]
e:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
8:null

2
litellm/proxy/_experimental/out/_not-found/__next._full.txt
View file

@ -10,7 +10,7 @@ b:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/li
d:I[168027,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"P":null,"b":"bCuM4c6QggtFAyOTSKpXz","c":["","_not-found"],"q":"","i":false,"f":[[["",{"children":["/_not-found",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],null,["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L9",null,{"children":"$La"}],["$","div",null,{"hidden":true,"children":["$","$Lb",null,{"children":["$","$7",null,{"name":"Next.Metadata","children":"$Lc"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$d","$undefined"],"S":true}
0:{"P":null,"b":"LpD6ruZoEpvYpT5IvMEoa","c":["","_not-found"],"q":"","i":false,"f":[[["",{"children":["/_not-found",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],null,["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L9",null,{"children":"$La"}],["$","div",null,{"hidden":true,"children":["$","$Lb",null,{"children":["$","$7",null,{"name":"Next.Metadata","children":"$Lc"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$d","$undefined"],"S":true}
a:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]
e:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
8:null

2
litellm/proxy/_experimental/out/_not-found/__next._head.txt
View file

@ -3,4 +3,4 @@
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/_not-found/__next._index.txt
View file

@ -5,4 +5,4 @@
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/_not-found/__next._not-found.__PAGE__.txt
View file

@ -1,5 +1,5 @@
1:"$Sreact.fragment"
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
3:"$Sreact.suspense"
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],null,["$","$L2",null,{"children":["$","$3",null,{"name":"Next.MetadataOutlet","children":"$@4"}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],null,["$","$L2",null,{"children":["$","$3",null,{"name":"Next.MetadataOutlet","children":"$@4"}]}]]}],"loading":null,"isPartial":false}
4:null

2
litellm/proxy/_experimental/out/_not-found/__next._not-found.txt
View file

@ -1,4 +1,4 @@
1:"$Sreact.fragment"
2:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
3:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","template":["$","$L3",null,{}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","template":["$","$L3",null,{}]}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/_not-found/__next._tree.txt
View file

@ -1,3 +1,3 @@
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"/_not-found","paramType":null,"paramKey":"/_not-found","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"/_not-found","paramType":null,"paramKey":"/_not-found","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":false}},"isRootLayout":true},"staleTime":300}

2
litellm/proxy/_experimental/out/api-reference.html
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/api-reference.txt
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.api-reference.__PAGE__.txt
View file

@ -3,7 +3,7 @@
3:I[191905,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","/litellm-asset-prefix/_next/static/chunks/0377ae18aae60c57.js","/litellm-asset-prefix/_next/static/chunks/d7798a4e148be3fe.js","/litellm-asset-prefix/_next/static/chunks/1461020743acb21c.js","/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","/litellm-asset-prefix/_next/static/chunks/eea976cf4a05fc92.js","/litellm-asset-prefix/_next/static/chunks/120b47ad353e1fc9.js","/litellm-asset-prefix/_next/static/chunks/738c339383c3b4b6.js","/litellm-asset-prefix/_next/static/chunks/3356ae3643d24081.js","/litellm-asset-prefix/_next/static/chunks/2a5f4a7388e54210.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/6c4c97f1ea6e7d77.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
7:"$Sreact.suspense"
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/2a5f4a7388e54210.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/6c4c97f1ea6e7d77.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/2a5f4a7388e54210.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/6c4c97f1ea6e7d77.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false}
4:{}
5:{}
8:null

2
litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.api-reference.txt
View file

@ -1,4 +1,4 @@
1:"$Sreact.fragment"
2:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
3:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","template":["$","$L3",null,{}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","template":["$","$L3",null,{}]}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.txt
View file

@ -3,5 +3,5 @@
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","/litellm-asset-prefix/_next/static/chunks/0377ae18aae60c57.js","/litellm-asset-prefix/_next/static/chunks/d7798a4e148be3fe.js","/litellm-asset-prefix/_next/static/chunks/1461020743acb21c.js","/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","/litellm-asset-prefix/_next/static/chunks/eea976cf4a05fc92.js","/litellm-asset-prefix/_next/static/chunks/120b47ad353e1fc9.js","/litellm-asset-prefix/_next/static/chunks/738c339383c3b4b6.js","/litellm-asset-prefix/_next/static/chunks/3356ae3643d24081.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0377ae18aae60c57.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/d7798a4e148be3fe.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/1461020743acb21c.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/eea976cf4a05fc92.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/120b47ad353e1fc9.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/738c339383c3b4b6.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3356ae3643d24081.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0377ae18aae60c57.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/d7798a4e148be3fe.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/1461020743acb21c.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/eea976cf4a05fc92.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/120b47ad353e1fc9.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/738c339383c3b4b6.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3356ae3643d24081.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"loading":null,"isPartial":false}
6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

2
litellm/proxy/_experimental/out/api-reference/__next._full.txt
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/api-reference/__next._head.txt
View file

@ -3,4 +3,4 @@
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/api-reference/__next._index.txt
View file

@ -5,4 +5,4 @@
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}

Some files were not shown because too many files have changed in this diff Show more