scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge branch 'shin_agent_oss_staging_05_19_2026' into Support-xai_key

Browse source
This commit is contained in:
Krrish Dholakia 2026-05-18 19:53:28 -07:00 • committed by GitHub
commit bdf008c319
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
629 changed files with 20461 additions and 2407 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
.circleci/config.yml
View file

@ -228,7 +228,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm \
--cov=./litellm \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -293,7 +293,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm \
--cov=./litellm \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -409,14 +409,25 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
mv coverage.xml auth_ui_unit_tests_coverage.xml
mv .coverage auth_ui_unit_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
@ -493,13 +504,24 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
mv coverage.xml router_unit_tests_coverage.xml
mv .coverage router_unit_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- router_unit_tests_coverage.xml
- router_unit_tests_coverage
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
docker:
- *python312_image
@ -603,7 +625,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
@ -646,7 +668,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -688,7 +710,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
@ -732,7 +754,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--retries 3 --retry-delay 5"
@ -814,7 +836,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -856,7 +878,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -930,7 +952,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -972,7 +994,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1015,7 +1037,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1090,7 +1112,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
-n 4 \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1133,7 +1155,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1184,7 +1206,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1"
@ -2280,10 +2302,11 @@ jobs:
- run:
name: Combine Coverage
command: |
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
uv tool run --from 'coverage[toml]==7.10.6' coverage xml
- codecov/upload:
file: ./coverage.xml
flags: circleci
ui_build:
docker:
@ -2669,6 +2692,8 @@ workflows:
- local_testing_part1
- local_testing_part2
- litellm_assistants_api_testing
- litellm_router_unit_testing
- auth_ui_unit_tests
- db_migration_disable_update_check:
requires:
- build_docker_database_image

3
.github/workflows/_test-unit-base.yml vendored
View file

@ -91,7 +91,7 @@ jobs:
--reruns-delay 1 \
--dist=loadscope \
--durations=20 \
--cov=litellm \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
@ -132,4 +132,5 @@ jobs:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false

5
.github/workflows/_test-unit-services-base.yml vendored
View file

@ -132,7 +132,7 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20 \
--cov=litellm \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
@ -144,7 +144,7 @@ jobs:
--reruns-delay 1 \
--dist="${DIST}" \
--durations=20 \
--cov=litellm \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi
@ -186,4 +186,5 @@ jobs:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false

2
.github/workflows/test-mcp.yml vendored
View file

@ -43,4 +43,4 @@ jobs:
- name: Run MCP tests
run: |
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5

38
.github/workflows/test-unit-caching-redis.yml vendored
View file

@ -1,38 +0,0 @@
name: "Unit Tests: Caching (Redis)"
# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
# This prevents external PRs from accessing Redis credentials.
on:
push:
branches: [main, "litellm_*"]
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
caching-redis:
uses: ./.github/workflows/_test-unit-services-base.yml
with:
# Redis-only tests that do NOT require provider API keys.
# Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
# test_router_caching.py) are in Phase 3 integration workflows.
test-path: >-
tests/local_testing/test_dual_cache.py
tests/local_testing/test_redis_batch_optimizations.py
tests/local_testing/test_router_utils.py
workers: 2
reruns: 2
timeout-minutes: 20
enable-redis: true
enable-postgres: false
secrets:
REDIS_HOST: ${{ secrets.REDIS_HOST }}
REDIS_PORT: ${{ secrets.REDIS_PORT }}
REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
DATABASE_URL: ${{ secrets.DATABASE_URL }}
POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

19
.gitignore vendored
View file

@ -101,4 +101,23 @@ STABILIZATION_TODO.md
**/*.storageState.json
**/coverage
test-config
# ---------- Terraform ----------
# Provider binaries + module cache — regenerated by `terraform init`.
**/.terraform/
# State files often contain secrets (DB passwords, API keys snapshotted from
# data sources). Keep state in a remote backend, never in git.
*.tfstate
*.tfstate.*
*.tfstate.backup
# Plan files can also contain sensitive values (variables in plaintext).
*.tfplan
# User-specific variable inputs — example files (terraform.tfvars.example) are
# tracked because they end in .example, which doesn't match the glob below.
*.tfvars
*.auto.tfvars
crash.log
crash.*.log
# .terraform.lock.hcl is intentionally NOT ignored — it pins provider versions
# and should be committed.
.vscode

1
CLAUDE.md
View file

@ -117,6 +117,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only `<span>`, `<p>`, `<h*>` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
### MCP OAuth / OpenAPI Transport Mapping
- **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive — not `client_credentials`)** — LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database.
- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.

2
README.md
View file

@ -292,7 +292,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
| [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | | | | | | | |
| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [Databricks (`databricks`)](https://docs.litellm.ai/docs/providers/databricks) | ✅ | ✅ | ✅ | | | | | | | |
| [DataRobot (`datarobot`)](https://docs.litellm.ai/docs/providers/datarobot) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ | | | ✅ | | | | |

83
backend/Dockerfile Normal file
View file

@ -0,0 +1,83 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
# BuildKit cache mount (different filesystem).
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
# silently pulling a managed interpreter.
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_COMPILE_BYTECODE=1 \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}"
# Stage 1 — install dependencies only.
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
# Stage 2 — copy source and install the project + workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
RUN mkdir -p /home/nonroot && \
HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
chown -R nonroot:nonroot /home/nonroot/.cache
# ---------- Runtime ----------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
# /home/nonroot. We run the backend as that user
WORKDIR /app
ENV HOME=/home/nonroot \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
USER nonroot
EXPOSE 4001/tcp
ENTRYPOINT ["uvicorn", "backend.main:app"]
CMD ["--host", "0.0.0.0", "--port", "4001"]

51
backend/main.py Normal file
View file

@ -0,0 +1,51 @@
"""UI backend entrypoint.
Reuses the existing FastAPI app from `litellm.proxy.proxy_server` and trims its
route table to just the management/admin surface used by the dashboard. Purely
additive — no existing module is modified.
Run with:
uvicorn backend.main:app --host 0.0.0.0 --port 4001
"""
from contextlib import asynccontextmanager
from fastapi.routing import Mount
# See gateway/main.py for why we assemble DATABASE_URL(s) here before
# importing proxy_server.
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
DatabaseURLSettings.from_env().apply_to_env()
from litellm.proxy.proxy_server import app
from backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES
def _is_backend_route(route) -> bool:
"""Keep the route on the backend if its path is in the management surface."""
path = getattr(route, "path", None)
if path is None:
return False
if isinstance(route, Mount):
# Static UI mounts are served by the dedicated UI container, not here.
return False
if path in BACKEND_EXACT_PATHS:
return True
return any(path.startswith(prefix) for prefix in BACKEND_PATH_PREFIXES)
# See gateway/main.py for why the trim runs inside the lifespan instead of at
# module scope.
_proxy_lifespan = app.router.lifespan_context
@asynccontextmanager
async def _backend_lifespan(app_):
async with _proxy_lifespan(app_):
app_.router.routes = [r for r in app_.router.routes if _is_backend_route(r)]
yield
app.router.lifespan_context = _backend_lifespan

0
backend/routes/__init__.py Normal file
View file

135
backend/routes/allowlist.py Normal file
View file

@ -0,0 +1,135 @@
"""Path allowlist for the UI backend (control plane) component.
The backend exposes management/admin endpoints consumed by the UI: keys, users,
teams, orgs, customers, budgets, tags, workflows, model management, spend &
analytics, settings (router/cache/cost-tracking/fallbacks), SSO/onboarding,
audit logs, debug, enterprise admin, and UI bootstrap helpers (logo, favicon,
.well-known config).
Anything LLM data-plane is dropped — those run on the gateway component.
"""
BACKEND_PATH_PREFIXES: tuple[str, ...] = (
# Identity / access
"/key/",
"/v2/key/",
"/user/",
"/v2/user/",
"/team/",
"/v2/team/",
"/organization/",
"/customer/",
"/end_user/",
"/sso/",
"/login",
"/v2/login",
"/v3/login",
"/logout",
"/token",
"/onboarding/",
"/audit",
"/oauth/",
"/invitation/",
"/jwt/",
# Models & routing config
"/model/",
"/v1/model/info",
"/v2/model/",
"/model_group",
"/model_access_group/",
"/model_hub/",
"/v1/access_group",
"/access_group/",
"/router/",
"/router_settings",
"/adaptive_router/",
"/fallback",
"/fallbacks",
"/cache_settings",
"/cost_tracking",
"/cost/",
"/credentials",
"/credential",
"/provider/budgets",
# Tools / agents (registry & policy admin)
"/v1/tool/",
"/v1/agents",
# Guardrails admin
"/v2/guardrails/",
# MCP server admin + BYOK OAuth flow (UI-initiated) + dynamic per-server endpoints
"/v1/mcp/",
"/test/",
"/{mcp_server_name}/",
# Budgets / tags / workflows / memory mgmt
"/budget/",
"/tag/",
"/workflow/",
"/v1/workflows/",
"/project/",
"/memory/",
"/mcp/",
# Spend / analytics
"/spend/",
"/analytics/",
"/global/",
"/user_agent",
"/usage/",
"/daily/",
# CloudZero cost-export admin (init / settings / export / dry-run / delete)
"/cloudzero/",
# Caching admin
"/cache/",
"/caching/",
# Callbacks / hooks
"/active/callbacks",
"/callbacks",
"/team_callback",
# Alerting / email / IP allowlist
"/alerting/",
"/email/",
"/add/allowed_ip",
"/delete/allowed_ip",
"/get/",
# Enterprise admin
"/enterprise/",
# Debug / config / profiling
"/debug/",
"/config/",
"/memory-usage-in-mem-cache",
"/otel-spans",
"/lazy/",
"/in_product_nudges",
# Admin reload / schedule
"/reload/",
"/schedule/",
"/settings",
"/update/",
"/upload/",
# Dev / admin utilities
"/utils/",
# UI bootstrap helpers (assets the dashboard fetches)
"/get_logo_url",
"/get_image",
"/get_favicon",
"/.well-known/",
"/litellm/.well-known/",
"/ui_discovery/",
"/ui-config",
"/sso_settings",
"/public/",
"/robots.txt",
# Health (k8s probes)
"/health",
)
BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
{
"/",
"/routes",
"/openapi.json",
"/docs",
"/docs/oauth2-redirect",
"/redoc",
"/fallback/login",
}
)

10
codecov.yaml
View file

@ -3,6 +3,16 @@ codecov:
notify:
wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
# Uploads are flagged per workflow/shard (GHA) or "circleci". carryforward makes
# a re-upload of a flag replace its prior session instead of accumulating a
# conflicting one, and lets a commit reuse a flag from its parent when that flag
# was not re-uploaded. Required because the same commit can receive the
# push-triggered workflows more than once (re-runs / branches cut at the same
# SHA); flagless overlapping sessions made Codecov drop the largest files.
flag_management:
default_rules:
carryforward: true
component_management:
individual_components:
- component_id: "Router"

36
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -300,6 +300,42 @@ class CheckBatchCost:
custom_llm_provider=custom_llm_provider,
)
# CheckBatchCost bypasses async_post_call_success_hook, so convert raw
# output/error file IDs to managed base64 IDs before the DB write here.
managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
if managed_files_hook is not None:
from litellm.proxy._types import UserAPIKeyAuth
_minimal_auth = UserAPIKeyAuth(
user_id=job.created_by or "default-user-id",
team_id=getattr(job, "team_id", None),
)
for _file_attr in ["output_file_id", "error_file_id"]:
_raw_file_id = getattr(response, _file_attr, None)
if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
try:
_unified_file_id = managed_files_hook.get_unified_output_file_id(
output_file_id=_raw_file_id,
model_id=model_id,
model_name=str(model_name) if model_name else deployment_info.model_name or None,
)
await managed_files_hook.store_unified_file_id(
file_id=_unified_file_id,
file_object=None,
litellm_parent_otel_span=None,
model_mappings={model_id: _raw_file_id},
user_api_key_dict=_minimal_auth,
)
setattr(response, _file_attr, _unified_file_id)
verbose_proxy_logger.info(
f"CheckBatchCost: converted {_file_attr} "
f"{_raw_file_id!r} -> managed ID for batch {batch_id}"
)
except Exception as _e:
verbose_proxy_logger.warning(
f"CheckBatchCost: failed to create managed file ID for "
f"{_file_attr}={_raw_file_id!r}: {_e}"
)
# Pass deployment model_info so custom batch pricing
# (input_cost_per_token_batches etc.) is used for cost calc
deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.40"
version = "0.1.41"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.40"
version = "0.1.41"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

83
gateway/Dockerfile Normal file
View file

@ -0,0 +1,83 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
# BuildKit cache mount (different filesystem).
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
# silently pulling a managed interpreter.
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_COMPILE_BYTECODE=1 \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}"
# Stage 1 — install dependencies only.
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
# Stage 2 — copy source and install the project + workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
RUN mkdir -p /home/nonroot && \
HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
chown -R nonroot:nonroot /home/nonroot/.cache
# ---------- Runtime ----------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
# /home/nonroot. We run the proxy as that user.
WORKDIR /app
ENV HOME=/home/nonroot \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
USER nonroot
EXPOSE 4000/tcp
ENTRYPOINT ["sh", "-c", "exec uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
CMD ["--host", "0.0.0.0", "--port", "4000"]

59
gateway/main.py Normal file
View file

@ -0,0 +1,59 @@
"""Gateway entrypoint.
Reuses the existing FastAPI app from `litellm.proxy.proxy_server` and trims its
route table to just the LLM data-plane surface. The trim is purely additive —
no existing module is modified, the full app continues to work via the legacy
entrypoint (`litellm.proxy.proxy_server:app`).
Run with:
uvicorn gateway.main:app --host 0.0.0.0 --port 4000
"""
from contextlib import asynccontextmanager
from fastapi.routing import Mount
# Assemble DATABASE_URL (+ DATABASE_URL_READ_REPLICA) from the discrete
# DATABASE_* env vars before proxy_server imports spin up Prisma. Handles
# both IAM (mint a token) and password auth, writer and reader. The standard
# CLI flow does this in proxy_cli.py; we bypass proxy_cli by uvicorn'ing the
# app directly, so without this Prisma initializes with the placeholder URL
# and every DB-needing endpoint returns "Database not connected".
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
DatabaseURLSettings.from_env().apply_to_env()
from litellm.proxy.proxy_server import app
from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES
def _is_gateway_route(route) -> bool:
"""Keep the route on the gateway if its path is in the LLM data-plane surface."""
path = getattr(route, "path", None)
if path is None:
return False
if isinstance(route, Mount):
# Gateway never serves the static UI or its asset bundles.
return False
if path in GATEWAY_EXACT_PATHS:
return True
return any(path.startswith(prefix) for prefix in GATEWAY_PATH_PREFIXES)
# Wrap proxy_server's existing lifespan so the route trim runs *after* its
# startup hooks (and any plugin code those hooks load) have had a chance to
# register routes. A module-load filter would miss routes added during
# startup; running inside the lifespan, after the inner __aenter__, catches
# them while still completing before uvicorn opens the listener.
_proxy_lifespan = app.router.lifespan_context
@asynccontextmanager
async def _gateway_lifespan(app_):
async with _proxy_lifespan(app_):
app_.router.routes = [r for r in app_.router.routes if _is_gateway_route(r)]
yield
app.router.lifespan_context = _gateway_lifespan

0
gateway/routes/__init__.py Normal file
View file

121
gateway/routes/allowlist.py Normal file
View file

@ -0,0 +1,121 @@
"""Path allowlist for the gateway component.
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
responses, vector stores, passthrough providers, realtime websockets, MCP
tool-call endpoints, and operational endpoints (/health, /metrics).
Any path not listed here is dropped from the gateway process so management/UI
endpoints don't ride on the same pods.
Versioned data-plane paths are enumerated explicitly rather than allowing a
blanket `/v1/` or `/v2/` prefix — those broad prefixes would otherwise also
match management routes like `/v1/access_group`, `/v1/tool/{tool_name}/logs`,
`/v2/key/info`, etc.
"""
GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
# OpenAI-compatible data-plane surface (versioned + unversioned)
"/v1/chat/",
"/chat/",
"/v1/completions",
"/completions",
"/v1/embeddings",
"/embeddings",
"/v1/moderations",
"/moderations",
"/v1/audio/",
"/audio/",
"/v1/images/",
"/images/",
"/v1/files",
"/files",
"/v1/batches",
"/batches",
"/v1/fine_tuning/",
"/fine_tuning/",
"/v1/fine-tuning/",
"/fine-tuning/",
"/v1/responses",
"/responses",
"/v1/threads",
"/threads",
"/v1/assistants",
"/assistants",
"/v1/vector_stores",
"/vector_stores",
"/v1/indexes",
"/v1/models",
"/models",
"/openai/",
"/engines/",
# Anthropic / agentic data-plane surface
"/v1/messages",
"/messages",
"/v1/skills",
"/v1/a2a/",
# LiteLLM-native LLM surface
"/v1/rerank",
"/v2/rerank",
"/rerank",
"/v1/ocr",
"/ocr",
"/v1/rag/",
"/rag/",
"/v1/video",
"/v1/videos",
"/video/",
"/videos",
"/v1/search",
"/search",
"/v1/containers",
"/containers",
"/v1/evals",
"/v1/memory",
"/queue/chat/",
# Google data plane (v1beta is the Google AI Studio version)
"/v1beta/",
"/interactions",
# Provider passthrough
"/anthropic/",
"/azure/",
"/azure_ai/",
"/aws/",
"/bedrock/",
"/cohere/",
"/gemini/",
"/google/",
"/vertex_ai/",
"/vertex-ai/",
"/assemblyai/",
"/eu.assemblyai/",
"/langfuse/",
"/vllm/",
"/mistral/",
"/groq/",
"/voyage/",
"/cursor/",
"/milvus/",
"/openai_passthrough/",
# Dynamic provider / toolset passthrough (path templates)
"/{provider}/",
"/toolset/",
# Realtime / streaming
"/v1/realtime",
"/realtime",
# Health & ops
"/health",
"/metrics",
)
GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
{
"/",
"/routes",
"/openapi.json",
"/docs",
"/docs/oauth2-redirect",
"/redoc",
"/test",
}
)

8
helm/litellm/Chart.yaml Normal file
View file

@ -0,0 +1,8 @@
apiVersion: v2
name: litellm
description: LiteLLM componentized — gateway, UI backend, and UI as separate services
type: application
version: 0.1.0
appVersion: "0.1.0"
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"

49
helm/litellm/templates/NOTES.txt Normal file
View file

@ -0,0 +1,49 @@
LiteLLM componentized — release {{ .Release.Name }} in namespace {{ .Release.Namespace }}.
Components:
{{- if .Values.gateway.enabled }}
- gateway : Service {{ include "litellm.gateway.fullname" . }} on port {{ .Values.gateway.service.port }}
{{- end }}
{{- if .Values.backend.enabled }}
- backend : Service {{ include "litellm.backend.fullname" . }} on port {{ .Values.backend.service.port }}
{{- end }}
{{- if .Values.ui.enabled }}
- ui : Service {{ include "litellm.ui.fullname" . }} on port {{ .Values.ui.service.port }}
{{- end }}
Port-forward examples:
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.gateway.fullname" . }} {{ .Values.gateway.service.port }}
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.backend.fullname" . }} {{ .Values.backend.service.port }}
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.ui.fullname" . }} {{ .Values.ui.service.port }}
Reminders:
- Sensitive values come from Secret references only. Before installing, set:
- masterKey.secretName (Secret with the proxy master key)
- database.writer.{host,port,dbname} (writer connection pieces)
- database.writer.passwordSecret.{name,usernameKey,passwordKey}
(Secret holding the writer DB username + password)
- database.writer.useIAMAuth: true (optional — chart sets IAM_TOKEN_DB_AUTH=true and
omits DATABASE_PASSWORD / DATABASE_URL so the proxy
mints the URL from an IAM token at startup)
- database.reader.host (optional — enables read-replica routing; reader
.passwordSecret.name is required when set, unless
.useIAMAuth is true)
- database.reader.useIAMAuth: true (optional, requires database.writer.useIAMAuth: true —
chart emits DATABASE_*_READ_REPLICA env vars and
omits DATABASE_PASSWORD_READ_REPLICA /
DATABASE_URL_READ_REPLICA so the proxy mints the
reader URL from an IAM token at startup)
- redis.passwordSecret.name (optional — set when redis.host is provided and the
cache requires auth)
- redis.cluster: true (optional — chart sets REDIS_CLUSTER_NODES from
redis.host / redis.port so the proxy's Cache()
constructs a RedisClusterCache; the cluster client
discovers remaining nodes from CLUSTER SLOTS)
- Per-component extras (gateway / backend / ui):
- {component}.extraEnv / envConfigMaps / envSecrets (the latter two are lists of resource names →
envFrom configMapRef / secretRef)
- {component}.logLevel (renders as LITELLM_LOG)
- gateway.config.proxy_config (rendered into a ConfigMap and mounted at
/app/config/config.yaml; gateway reads it via
CONFIG_FILE_PATH)
- Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend.

245
helm/litellm/templates/_helpers.tpl Normal file
View file

@ -0,0 +1,245 @@
{{/*
Common naming + label helpers shared by gateway, backend, and ui templates.
*/}}
{{- define "litellm.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.gateway.fullname" -}}
{{- printf "%s-gateway" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.backend.fullname" -}}
{{- printf "%s-backend" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.ui.fullname" -}}
{{- printf "%s-ui" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.commonLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
{{- end -}}
{{/*
Per-component selector labels — used in both Service selectors and Deployment matchLabels.
*/}}
{{- define "litellm.gateway.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: gateway
{{- end -}}
{{- define "litellm.backend.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: backend
{{- end -}}
{{- define "litellm.ui.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: ui
{{- end -}}
{{/*
Shared ServiceAccount name used by all three component Deployments. When
`serviceAccount.create` is true and `serviceAccount.name` is empty, default
to the chart fullname. When `create` is false, fall back to the provided
name or the namespace's `default` SA.
*/}}
{{- define "litellm.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{ default (include "litellm.fullname" .) .Values.serviceAccount.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
{{/*
Master-key + database + redis env block — shared by gateway, backend, and the
migrations Job.
Invoke with a dict: `(dict "root" $ "component" .Values.gateway)`. `root` is
the chart context (needed for .Values), `component` selects which component's
`extraEnv` / `logLevel` to render.
Sensitive values (master key, DB username + password, Redis password) come
only from referenced Secrets; the chart never accepts inline values for them.
The chart never assembles DATABASE_URL itself. It emits only the discrete
DATABASE_HOST/PORT/USER/NAME/SCHEMA (+ DATABASE_PASSWORD for password auth)
vars; the proxy's entrypoint (DatabaseURLSettings in
litellm/proxy/db/db_url_settings.py) builds the URL from them and
percent-encodes the credentials. Assembling the URL here via Kubernetes
`$(VAR)` substitution would embed the raw secret value, corrupting the URL
whenever the password contains a URL-reserved character (@, /, ?, %, +,
...) — as AWS RDS auto-generated passwords routinely do.
When `database.writer.useIAMAuth: true`, the chart injects
IAM_TOKEN_DB_AUTH=true and omits DATABASE_PASSWORD — the entrypoint mints
the URL from DATABASE_HOST/PORT/USER/NAME plus a short-lived IAM token
instead of a static password.
The read replica is opt-in via `database.reader.host`. The chart emits
DATABASE_HOST_READ_REPLICA / DATABASE_PORT_READ_REPLICA /
DATABASE_NAME_READ_REPLICA (+ DATABASE_SCHEMA_READ_REPLICA) for both auth
modes, plus DATABASE_USER_READ_REPLICA / DATABASE_PASSWORD_READ_REPLICA for
password auth. When `database.reader.useIAMAuth: true` it omits
DATABASE_PASSWORD_READ_REPLICA and the entrypoint mints the reader URL the
same way. Reader IAM only takes effect when the writer also uses IAM auth
(the proxy gates URL minting on IAM_TOKEN_DB_AUTH, which only the writer
sets).
*/}}
{{- define "litellm.serverEnv" -}}
{{- $root := .root -}}
{{- $component := .component -}}
- name: LITELLM_MASTER_KEY
valueFrom:
secretKeyRef:
name: {{ required "masterKey.secretName is required (the chart no longer accepts an inline master key)" $root.Values.masterKey.secretName }}
key: {{ $root.Values.masterKey.secretKey | default "master-key" }}
{{- if $component.logLevel }}
- name: LITELLM_LOG
value: {{ $component.logLevel | quote }}
{{- end }}
{{- with $root.Values.database.writer }}
- name: DATABASE_HOST
value: {{ required "database.writer.host is required" .host | quote }}
- name: DATABASE_PORT
value: {{ .port | default 5432 | quote }}
- name: DATABASE_USER
valueFrom:
secretKeyRef:
name: {{ required "database.writer.passwordSecret.name is required" .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
- name: DATABASE_NAME
value: {{ required "database.writer.dbname is required" .dbname | quote }}
{{- if .schema }}
- name: DATABASE_SCHEMA
value: {{ .schema | quote }}
{{- end }}
{{- if .useIAMAuth }}
- name: IAM_TOKEN_DB_AUTH
value: "true"
{{- else }}
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.passwordKey | default "password" }}
{{- end }}
{{- end }}
{{- with $root.Values.database.reader }}
{{- if .host }}
{{- if and .useIAMAuth (not $root.Values.database.writer.useIAMAuth) }}
{{- fail "database.reader.useIAMAuth requires database.writer.useIAMAuth: true (the proxy gates IAM URL minting on IAM_TOKEN_DB_AUTH, which is only set by the writer)" }}
{{- end }}
- name: DATABASE_HOST_READ_REPLICA
value: {{ .host | quote }}
- name: DATABASE_PORT_READ_REPLICA
value: {{ .port | default 5432 | quote }}
- name: DATABASE_NAME_READ_REPLICA
value: {{ required "database.reader.dbname is required when database.reader.host is set" .dbname | quote }}
{{- if .schema }}
- name: DATABASE_SCHEMA_READ_REPLICA
value: {{ .schema | quote }}
{{- end }}
{{- if .useIAMAuth }}
{{- if .passwordSecret.name }}
- name: DATABASE_USER_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
{{- end }}
{{- else }}
{{- if not .passwordSecret.name }}
{{- fail "database.reader.passwordSecret.name is required when database.reader.host is set" }}
{{- end }}
- name: DATABASE_USER_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
- name: DATABASE_PASSWORD_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.passwordKey | default "password" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
The migrations Job (helm.sh/hook: pre-upgrade) is the single owner of
`prisma migrate deploy`. Without this, every gateway/backend pod also runs
Prisma schema-update on startup and contends with the Job — and with each
other — for Prisma's Postgres advisory lock on the writer, which makes the
Job's `migrate deploy` intermittently block until its per-attempt timeout
and retry-exhaust. The Job's entrypoint (migrations/run.py) does not import
proxy_server and never reads DISABLE_SCHEMA_UPDATE, so emitting it here is a
harmless no-op for the Job and authoritative for the app pods.
*/}}
- name: DISABLE_SCHEMA_UPDATE
value: "true"
{{- if $root.Values.redis.host }}
- name: REDIS_HOST
value: {{ $root.Values.redis.host | quote }}
- name: REDIS_PORT
value: {{ $root.Values.redis.port | quote }}
{{- if $root.Values.redis.passwordSecret.name }}
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ $root.Values.redis.passwordSecret.name }}
key: {{ $root.Values.redis.passwordSecret.passwordKey | default "password" }}
{{- end }}
{{- if $root.Values.redis.cluster }}
{{/* The proxy's Cache() reads REDIS_CLUSTER_NODES as JSON and constructs a
RedisClusterCache when it's set (litellm/caching/caching.py:169-192).
We seed with the single configured endpoint — the cluster client
discovers the remaining nodes from CLUSTER SLOTS at startup. */}}
- name: REDIS_CLUSTER_NODES
value: {{ printf "[{\"host\":%q,\"port\":%v}]" $root.Values.redis.host (int $root.Values.redis.port) | quote }}
{{- end }}
{{- end }}
{{- with $component.extraEnv }}
{{ toYaml . }}
{{- end }}
{{- end -}}
{{/*
Renders `envFrom:` block for a component's `envConfigMaps` / `envSecrets`
lists. Each entry is a resource name; the chart wires the whole ConfigMap /
Secret into the container's env via configMapRef / secretRef.
Invoke with just the component dict, e.g. `.Values.gateway`. Emits nothing
when both lists are empty so the container spec stays clean.
*/}}
{{- define "litellm.envFrom" -}}
{{- $component := . -}}
{{- if or $component.envConfigMaps $component.envSecrets }}
envFrom:
{{- range $component.envConfigMaps }}
- configMapRef:
name: {{ . }}
{{- end }}
{{- range $component.envSecrets }}
- secretRef:
name: {{ . }}
{{- end }}
{{- end }}
{{- end -}}

60
helm/litellm/templates/backend/deployment.yaml Normal file
View file

@ -0,0 +1,60 @@
{{- if .Values.backend.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.backend.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
spec:
selector:
matchLabels:
{{- include "litellm.backend.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.backend.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.backend.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: backend
image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.backend.image.pullPolicy }}
ports:
- name: http
containerPort: 4001
protocol: TCP
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
{{- include "litellm.envFrom" .Values.backend | nindent 10 }}
{{- with .Values.backend.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.backend.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.backend.resources | nindent 12 }}
{{- with .Values.backend.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.backend.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.backend.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

33
helm/litellm/templates/backend/hpa.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- if and .Values.backend.enabled .Values.backend.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.backend.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.backend.fullname" . }}
minReplicas: {{ .Values.backend.hpa.minReplicas }}
maxReplicas: {{ .Values.backend.hpa.maxReplicas }}
metrics:
{{- if .Values.backend.hpa.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.backend.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.backend.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.backend.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

18
helm/litellm/templates/backend/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.backend.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.backend.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
spec:
type: {{ .Values.backend.service.type }}
ports:
- port: {{ .Values.backend.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.backend.selectorLabels" . | nindent 4 }}
{{- end }}

9
helm/litellm/templates/gateway/configmap.yaml Normal file
View file

@ -0,0 +1,9 @@
{{- if .Values.gateway.config.create }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "litellm.gateway.fullname" . }}-config
data:
config.yaml: |
{{ .Values.gateway.config.proxy_config | toYaml | indent 6 }}
{{- end }}

83
helm/litellm/templates/gateway/deployment.yaml Normal file
View file

@ -0,0 +1,83 @@
{{- if .Values.gateway.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
selector:
matchLabels:
{{- include "litellm.gateway.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
{{- if .Values.gateway.config.create }}
checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.gateway.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.gateway.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: gateway
image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
ports:
- name: http
containerPort: 4000
protocol: TCP
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
{{- if .Values.gateway.config.create }}
- name: CONFIG_FILE_PATH
value: /app/config/config.yaml
{{- end }}
{{- if .Values.gateway.numWorkers }}
- name: NUM_WORKERS
value: {{ .Values.gateway.numWorkers | quote }}
{{- end }}
{{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
{{- if .Values.gateway.config.create }}
volumeMounts:
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
{{- end }}
{{- with .Values.gateway.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.gateway.resources | nindent 12 }}
{{- if .Values.gateway.config.create }}
volumes:
- name: gateway-config
configMap:
name: {{ include "litellm.gateway.fullname" . }}-config
{{- end }}
{{- with .Values.gateway.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

33
helm/litellm/templates/gateway/hpa.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- if and .Values.gateway.enabled .Values.gateway.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.gateway.fullname" . }}
minReplicas: {{ .Values.gateway.hpa.minReplicas }}
maxReplicas: {{ .Values.gateway.hpa.maxReplicas }}
metrics:
{{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

18
helm/litellm/templates/gateway/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.gateway.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
type: {{ .Values.gateway.service.type }}
ports:
- port: {{ .Values.gateway.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.gateway.selectorLabels" . | nindent 4 }}
{{- end }}

153
helm/litellm/templates/ingress.yaml Normal file
View file

@ -0,0 +1,153 @@
{{- if .Values.ingress.enabled -}}
{{- $gatewayName := include "litellm.gateway.fullname" . -}}
{{- $backendName := include "litellm.backend.fullname" . -}}
{{- $uiName := include "litellm.ui.fullname" . -}}
{{- $gatewayPort := .Values.gateway.service.port -}}
{{- $backendPort := .Values.backend.service.port -}}
{{- $uiPort := .Values.ui.service.port -}}
{{/*
Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
Versioned paths are listed explicitly to avoid routing management routes
(e.g. /v1/access_group, /v2/key/info, /v1/tool/*, /v1/agents, /v1/workflows,
/v2/user/info, /v2/team/list, /v2/model/info, /v2/login, /v2/guardrails/*,
/v1/mcp/*) onto the gateway via a broad /v1 or /v2 prefix.
*/}}
{{- $gatewayPrefixes := list
"/v1/chat" "/chat" "/v1/completions" "/completions" "/v1/embeddings" "/embeddings"
"/v1/moderations" "/moderations" "/v1/audio" "/audio" "/v1/images" "/images"
"/v1/files" "/files" "/v1/batches" "/batches" "/v1/fine_tuning" "/fine_tuning"
"/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads"
"/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes"
"/v1/models" "/models" "/openai" "/engines"
"/v1/messages" "/messages" "/v1/skills" "/v1/a2a"
"/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag"
"/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
"/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
"/v1beta" "/interactions"
"/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/cohere" "/gemini" "/google"
"/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm"
"/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough"
"/toolset"
"/v1/realtime" "/realtime"
"/health" "/metrics"
-}}
{{/*
/test is gateway-only as an EXACT path (GATEWAY_EXACT_PATHS), but its
children /test/connection and /test/tools/list are MCP-server management
endpoints kept only on the backend ("/test/" in BACKEND_PATH_PREFIXES).
A Prefix match here would route /test/* to the gateway, which trims those
routes at startup -> 404. So /test is rendered as a standalone Exact path
and /test/* falls through to the backend catch-all.
*/}}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- with .Values.ingress.className }}
ingressClassName: {{ . | quote }}
{{- end }}
{{- with .Values.ingress.tls }}
tls:
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- {{- with .Values.ingress.host }}
host: {{ . | quote }}
{{- end }}
http:
paths:
# --- UI (Next.js static export) ---
- path: /
pathType: Exact
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /favicon.ico
pathType: Exact
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /litellm-asset-prefix
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /_next
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# /ui/* is where the Next.js SPA serves its login + dashboard
# routes (e.g. /ui/login). Without this, /ui/* falls into the
# catch-all → backend → 404.
- path: /ui
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# Next.js App Router (output: "export", basePath: "") emits the
# RSC/flight payload for every route as a ROOT-level <route>.txt
# (/index.txt, /teams.txt, /__next._tree.txt, ...). The client
# router fetches these on every soft navigation / prefetch as
# <route>.txt?_rsc=<hash> (the query string is irrelevant to path
# matching). They are not under /ui, /_next, or
# /litellm-asset-prefix, so without this rule they fall to the
# backend catch-all → 404 → client-side navigation never settles
# and the login flow spins in an infinite redirect loop
# (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt from the
# export; this rule only routes the request to it. Needs an
# ingress controller whose ImplementationSpecific path is a
# wildcard pattern (AWS ALB: `*` = 0+ chars); this chart targets
# the AWS Load Balancer Controller.
- path: /*.txt
pathType: ImplementationSpecific
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# --- Gateway data plane ---
# Exact /test only (see the $gatewayPrefixes comment above);
# /test/* MCP management endpoints fall to the backend catch-all.
- path: /test
pathType: Exact
backend:
service:
name: {{ $gatewayName }}
port:
number: {{ $gatewayPort }}
{{- range $gatewayPrefixes }}
- path: {{ . }}
pathType: Prefix
backend:
service:
name: {{ $gatewayName }}
port:
number: {{ $gatewayPort }}
{{- end }}
# --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) ---
- path: /
pathType: Prefix
backend:
service:
name: {{ $backendName }}
port:
number: {{ $backendPort }}
{{- end }}

46
helm/litellm/templates/migrations-job.yaml Normal file
View file

@ -0,0 +1,46 @@
{{- if .Values.migrationJob.enabled -}}
# Pre-install / pre-upgrade hook that runs `prisma migrate deploy` against
# the writer database before the gateway and backend Deployments are rolled
# out. Required because the gateway and backend both spin up Prisma at
# startup and assume the LiteLLM schema (LiteLLM_Config,
# LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...) already exists.
#
# Running this pre-upgrade closes the window where new application pods would
# otherwise serve traffic against the previous release's unmigrated schema.
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "litellm.fullname" . }}-migrations
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: migrations
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-delete-policy: before-hook-creation
helm.sh/hook-weight: "0"
spec:
backoffLimit: {{ .Values.migrationJob.backoffLimit }}
ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
template:
metadata:
labels:
{{- include "litellm.commonLabels" . | nindent 8 }}
app.kubernetes.io/component: migrations
spec:
restartPolicy: Never
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: prisma-migrations
image: "{{ .Values.migrationJob.image.repository }}:{{ .Values.migrationJob.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.migrationJob.image.pullPolicy }}
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.migrationJob) | nindent 12 }}
{{- with .Values.migrationJob.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- end }}

13
helm/litellm/templates/serviceaccount.yaml Normal file
View file

@ -0,0 +1,13 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "litellm.serviceAccountName" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
{{- end }}

70
helm/litellm/templates/ui/deployment.yaml Normal file
View file

@ -0,0 +1,70 @@
{{- if .Values.ui.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.ui.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: ui
spec:
selector:
matchLabels:
{{- include "litellm.ui.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.ui.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.ui.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: ui
image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.ui.image.pullPolicy }}
ports:
- name: http
containerPort: 3000
protocol: TCP
env:
{{- if .Values.ui.logLevel }}
- name: LITELLM_LOG
value: {{ .Values.ui.logLevel | quote }}
{{- end }}
{{- if .Values.ui.backendUrl }}
- name: LITELLM_BACKEND_URL
value: {{ .Values.ui.backendUrl | quote }}
{{- end }}
{{- with .Values.ui.extraEnv }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- include "litellm.envFrom" .Values.ui | nindent 10 }}
{{- with .Values.ui.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.ui.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.ui.resources | nindent 12 }}
{{- with .Values.ui.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.ui.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.ui.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

33
helm/litellm/templates/ui/hpa.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- if and .Values.ui.enabled .Values.ui.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.ui.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: ui
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.ui.fullname" . }}
minReplicas: {{ .Values.ui.hpa.minReplicas }}
maxReplicas: {{ .Values.ui.hpa.maxReplicas }}
metrics:
{{- if .Values.ui.hpa.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.ui.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.ui.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.ui.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

18
helm/litellm/templates/ui/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.ui.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.ui.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: ui
spec:
type: {{ .Values.ui.service.type }}
ports:
- port: {{ .Values.ui.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.ui.selectorLabels" . | nindent 4 }}
{{- end }}

225
helm/litellm/values.yaml Normal file
View file

@ -0,0 +1,225 @@
# LiteLLM helm chart values
nameOverride: ""
fullnameOverride: ""
imagePullSecrets: []
# Optional Ingress wiring the three component Services behind a single L7
# entrypoint. Required when serving the static UI bundle over the network.
ingress:
enabled: false
className: ""
annotations: {}
host: "" # optional; if set, becomes the rule's host
tls: []
# Shared ServiceAccount used by all three component Deployments. Set
# `create: true` to have the chart provision it (e.g. when wiring an EKS
# Pod Identity association by SA name). Set `name` to use an existing SA
# (chart-created or out-of-band). When both are empty / false, pods run
# with the namespace's `default` SA.
serviceAccount:
create: false
automount: true
annotations: {}
name: ""
# Pre-install / pre-upgrade Helm hook that runs `prisma migrate deploy`
# against the writer database, creating the LiteLLM schema (tables that
# gateway + backend assume exist at startup: LiteLLM_Config,
# LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...). Disable if your
# pipeline runs migrations out-of-band.
#
# Uses a dedicated `litellm-migrations` image (prisma CLI + the migration
# files from `litellm-proxy-extras`) instead of the backend image, so the
# Job doesn't drag in the rest of the proxy and doesn't run `prisma
# generate` — the migration engine doesn't need the generated client.
migrationJob:
enabled: true
backoffLimit: 4
ttlSecondsAfterFinished: 120
resources: {}
image:
repository: ghcr.io/berriai/litellm-migrations
tag: "" # defaults to .Chart.AppVersion
pullPolicy: IfNotPresent
# Extra env appended to the migration container. The migration entrypoint
# uses the v2 resolver by default (no diff-and-force recovery — avoids the
# schema thrashing seen during rolling deploys). To opt back into the v1
# resolver, append `- name: USE_V2_MIGRATION_RESOLVER` / `value: "false"`.
extraEnv: []
# Required: a master key used by gateway + backend to mint/verify proxy tokens.
# Must reference an existing Secret.
masterKey:
secretName: litellm-master-key-secret # name of a Secret containing the master key
secretKey: master-key
# External Postgres connection.
database:
writer:
host: ""
port: 5432
dbname: ""
schema: ""
useIAMAuth: false
passwordSecret:
name: litellm-writer-secret
usernameKey: username
passwordKey: password
# Optional read-replica routing. When `reader.host` is set, the proxy routes
# reads (find_*, count, group_by, query_raw/_first) to this endpoint while
# writes stay on the writer. Leave `reader.host` empty to disable.
reader:
host: ""
port: 5432
dbname: ""
schema: ""
useIAMAuth: false
passwordSecret:
name: litellm-reader-secret
usernameKey: username
passwordKey: password
# Optional Redis (caching, rate limiting). Leave host empty to disable.
#
# Set `cluster: true` for Redis Cluster mode (e.g. AWS ElastiCache Cluster,
# self-hosted Redis Cluster). The chart emits REDIS_CLUSTER_NODES from
# `host` / `port` as the single seed; the cluster client discovers the
# remaining nodes from CLUSTER SLOTS at startup.
redis:
cluster: false
host: ""
port: 6379
passwordSecret:
name: "" # Leave empty for auth-less Redis
passwordKey: password
# ---------- gateway (LLM data plane) ----------
gateway:
enabled: true
logLevel: INFO
# Number of uvicorn worker processes per gateway pod. Sets NUM_WORKERS,
# consumed by the gateway image entrypoint. Default is 1.
numWorkers: 1
extraEnv: [] # Add extra environment variables to the gateway
envConfigMaps: [] # Add extra environment variables to the gateway from config maps
envSecrets: [] # Add extra environment variables to the gateway from secrets
config:
create: true
proxy_config: {}
image:
repository: ghcr.io/berriai/litellm-gateway
tag: "" # defaults to .Chart.AppVersion
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 4000
resources:
requests:
cpu: "1"
memory: 4Gi
limits:
cpu: "2"
memory: 4Gi
livenessProbe:
httpGet: { path: /health/liveliness, port: http }
initialDelaySeconds: 10
periodSeconds: 15
readinessProbe:
httpGet: { path: /health/readiness, port: http }
initialDelaySeconds: 5
periodSeconds: 10
hpa:
enabled: true
minReplicas: 1
maxReplicas: 10
targetCPUUtilizationPercentage: 70
targetMemoryUtilizationPercentage: 80
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
# ---------- backend (UI / management API) ----------
backend:
enabled: true
logLevel: INFO
extraEnv: []
envConfigMaps: []
envSecrets: []
image:
repository: ghcr.io/berriai/litellm-backend
tag: ""
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 4001
resources:
requests:
cpu: "1"
memory: 4Gi
limits:
cpu: "2"
memory: 4Gi
livenessProbe:
httpGet: { path: /health/liveliness, port: http }
initialDelaySeconds: 10
periodSeconds: 15
readinessProbe:
httpGet: { path: /health/readiness, port: http }
initialDelaySeconds: 5
periodSeconds: 10
hpa:
enabled: true
minReplicas: 1
maxReplicas: 4
targetCPUUtilizationPercentage: 70
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
# ---------- ui (Next.js static dashboard) ----------
ui:
enabled: true
logLevel: INFO
extraEnv: []
envConfigMaps: []
envSecrets: []
image:
repository: ghcr.io/berriai/litellm-ui
tag: ""
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 3000
# The dashboard expects to know where to reach the backend API. Set this to
# the externally-routable URL (typically the ingress host + /api or similar).
backendUrl: ""
resources:
requests:
cpu: 500m
memory: 500Mi
limits:
cpu: "1"
memory: 1Gi
livenessProbe:
httpGet: { path: /, port: http }
initialDelaySeconds: 5
periodSeconds: 20
readinessProbe:
httpGet: { path: /, port: http }
initialDelaySeconds: 2
periodSeconds: 10
hpa:
enabled: false
minReplicas: 1
maxReplicas: 3
targetCPUUtilizationPercentage: 80
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}

9
license_cache.json
View file

File diff suppressed because one or more lines are too long

7
litellm/__init__.py
View file

@ -417,6 +417,7 @@ custom_prometheus_metadata_labels: List[str] = []
custom_prometheus_tags: List[str] = []
prometheus_metrics_config: Optional[List] = None
prometheus_emit_stream_label: bool = False
prometheus_user_budget_label_include_email_alias: bool = False
prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
@ -1881,6 +1882,12 @@ if TYPE_CHECKING:
from .llms.dashscope.chat.transformation import (
DashScopeChatConfig as DashScopeChatConfig,
)
from .llms.dashscope.embed.transformation import (
DashScopeEmbeddingConfig as DashScopeEmbeddingConfig,
)
from .llms.dashscope.rerank.transformation import (
DashScopeRerankConfig as DashScopeRerankConfig,
)
from .llms.moonshot.chat.transformation import (
MoonshotChatConfig as MoonshotChatConfig,
)

2
litellm/_lazy_imports_registry.py
View file

@ -376,7 +376,6 @@ UTILS_MODULE_NAMES = (
"HTTPHandler",
"get_num_retries_from_retry_policy",
"reset_retry_policy",
"get_secret",
"get_coroutine_checker",
"get_litellm_logging_class",
"get_set_callbacks",
@ -1284,7 +1283,6 @@ _UTILS_MODULE_IMPORT_MAP = {
"litellm.router_utils.get_retry_from_policy",
"reset_retry_policy",
),
"get_secret": ("litellm.secret_managers.main", "get_secret"),
"get_coroutine_checker": (
"litellm.litellm_core_utils.cached_imports",
"get_coroutine_checker",

1
litellm/_logging.py
View file

@ -404,6 +404,7 @@ def _turn_on_debug():
def _disable_debugging():
"""Disable the package, router, and proxy verbose loggers."""
verbose_logger.disabled = True
verbose_router_logger.disabled = True
verbose_proxy_logger.disabled = True

23
litellm/batches/batch_utils.py
View file

@ -113,8 +113,11 @@ def _batch_cost_calculator(
"""
Calculate the cost of a batch based on the output file id
"""
# Handle Vertex AI with specialized method
if custom_llm_provider == "vertex_ai" and model_name:
if (
custom_llm_provider == "vertex_ai"
and model_name
and getattr(litellm, "disable_vertex_batch_output_transformation", False)
):
batch_cost, _ = calculate_vertex_ai_batch_cost_and_usage(
file_content_dictionary, model_name
)
@ -136,10 +139,13 @@ def calculate_vertex_ai_batch_cost_and_usage(
model_name: Optional[str] = None,
) -> Tuple[float, Usage]:
"""
Calculate both cost and usage from Vertex AI batch responses.
Calculate both cost and usage from raw Vertex AI batch responses.
Vertex AI batch output lines have format:
{"request": ..., "status": "", "response": {"candidates": [...], "usageMetadata": {...}}}
Used only when ``litellm.disable_vertex_batch_output_transformation = True``.
In that case the GCS predictions.jsonl is returned as-is, with each line in
the native Vertex format:
{"request": ..., "response": {"candidates": [...], "usageMetadata": {...}}}
usageMetadata contains promptTokenCount, candidatesTokenCount, totalTokenCount.
"""
@ -362,8 +368,11 @@ def _get_batch_job_total_usage_from_file_content(
"""
Get the tokens of a batch job from the file content
"""
# Handle Vertex AI with specialized method
if custom_llm_provider == "vertex_ai" and model_name:
if (
custom_llm_provider == "vertex_ai"
and model_name
and getattr(litellm, "disable_vertex_batch_output_transformation", False)
):
_, batch_usage = calculate_vertex_ai_batch_cost_and_usage(
file_content_dictionary, model_name
)

70
litellm/caching/caching_handler.py
View file

@ -87,6 +87,16 @@ class CachingHandlerResponse(BaseModel):
in_memory_cache_obj = InMemoryCache()
def _is_chat_completion_cached_dict(cached_result: dict) -> bool:
cached_id = cached_result.get("id")
if isinstance(cached_id, str) and cached_id.startswith("chatcmpl"):
return True
obj = cached_result.get("object")
if isinstance(obj, str):
return obj.startswith("chat.completion")
return "choices" in cached_result
def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool:
"""
When stream=True, do not run success callbacks at cache-hit time.
@ -861,27 +871,47 @@ class LLMCachingHandler:
elif (call_type == "aresponses" or call_type == "responses") and isinstance(
cached_result, dict
):
from litellm.responses.streaming_iterator import (
CachedResponsesAPIStreamingIterator,
)
response_obj = ResponsesAPIResponse(**cached_result)
if (
hasattr(response_obj, "_hidden_params")
and response_obj._hidden_params is not None
and isinstance(response_obj._hidden_params, dict)
):
response_obj._hidden_params["cache_hit"] = True
if kwargs.get("stream", False) is True:
cached_result = CachedResponsesAPIStreamingIterator(
response=response_obj,
logging_obj=logging_obj,
request_data=kwargs,
call_type=call_type,
)
use_chat_completion_cache = _is_chat_completion_cached_dict(cached_result)
if use_chat_completion_cache:
if kwargs.get("stream", False) is True:
bridge_call_type = (
CallTypes.acompletion.value
if call_type == "aresponses"
else CallTypes.completion.value
)
cached_result = self._convert_cached_stream_response(
cached_result=cached_result,
call_type=bridge_call_type,
logging_obj=logging_obj,
model=model,
)
else:
cached_result = convert_to_model_response_object(
response_object=cached_result,
model_response_object=ModelResponse(),
)
else:
cached_result = response_obj
from litellm.responses.streaming_iterator import (
CachedResponsesAPIStreamingIterator,
)
response_obj = ResponsesAPIResponse(**cached_result)
if (
hasattr(response_obj, "_hidden_params")
and response_obj._hidden_params is not None
and isinstance(response_obj._hidden_params, dict)
):
response_obj._hidden_params["cache_hit"] = True
if kwargs.get("stream", False) is True:
cached_result = CachedResponsesAPIStreamingIterator(
response=response_obj,
logging_obj=logging_obj,
request_data=kwargs,
call_type=call_type,
)
else:
cached_result = response_obj
if (
hasattr(cached_result, "_hidden_params")

25
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -37,6 +37,15 @@ class ResponsesToCompletionBridgeHandler:
stream = litellm_params.get("stream", False)
return bool(stream)
@staticmethod
def _is_preformatted_cached_chat_stream(result: Any) -> bool:
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
return (
isinstance(result, CustomStreamWrapper)
and result.custom_llm_provider == "cached_response"
)
@staticmethod
def _coerce_response_object(
response_obj: Any,
@ -177,6 +186,8 @@ class ResponsesToCompletionBridgeHandler:
**request_data,
)
from litellm.types.utils import ModelResponse
stream = self._resolve_stream_flag(optional_params, litellm_params)
if isinstance(result, ResponsesAPIResponse):
return self.transformation_handler.transform_response(
@ -192,6 +203,8 @@ class ResponsesToCompletionBridgeHandler:
api_key=kwargs.get("api_key"),
json_mode=kwargs.get("json_mode"),
)
elif isinstance(result, ModelResponse):
return result
elif not stream:
responses_api_response = self._collect_response_from_stream(result)
return self.transformation_handler.transform_response(
@ -208,6 +221,10 @@ class ResponsesToCompletionBridgeHandler:
json_mode=kwargs.get("json_mode"),
)
else:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(
result, model, custom_llm_provider
)
completion_stream = self.transformation_handler.get_model_response_iterator(
streaming_response=result, # type: ignore
sync_stream=True,
@ -256,6 +273,8 @@ class ResponsesToCompletionBridgeHandler:
aresponses=True,
)
from litellm.types.utils import ModelResponse
stream = self._resolve_stream_flag(optional_params, litellm_params)
if isinstance(result, ResponsesAPIResponse):
return self.transformation_handler.transform_response(
@ -271,6 +290,8 @@ class ResponsesToCompletionBridgeHandler:
api_key=kwargs.get("api_key"),
json_mode=kwargs.get("json_mode"),
)
elif isinstance(result, ModelResponse):
return result
elif not stream:
responses_api_response = await self._collect_response_from_stream_async(
result
@ -289,6 +310,10 @@ class ResponsesToCompletionBridgeHandler:
json_mode=kwargs.get("json_mode"),
)
else:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(
result, model, custom_llm_provider
)
completion_stream = self.transformation_handler.get_model_response_iterator(
streaming_response=result, # type: ignore
sync_stream=False,

8
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -1141,6 +1141,14 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
event_type = parsed_chunk.get("type")
if isinstance(event_type, ResponsesAPIStreamEvents):
event_type = event_type.value
if parsed_chunk.get("object") == "chat.completion.chunk" or (
event_type is None
and isinstance(parsed_chunk.get("choices"), list)
and parsed_chunk.get("choices")
):
return ModelResponseStream(**parsed_chunk)
verbose_logger.debug(f"Chat provider: Processing event type: {event_type}")
if event_type == "response.created":

34
litellm/cost_calculator.py
View file

@ -365,21 +365,21 @@ def cost_per_token( # noqa: PLR0915
if _pt_details is not None:
_cache_read_tokens = float(getattr(_pt_details, "cached_tokens", 0) or 0)
# OpenAI-compatible providers report cache-write tokens under
# either `cache_creation_tokens` or `cache_write_tokens` (kimi-k2
# uses the latter). Mirror db_spend_update_writer to stay symmetric.
# either `cache_write_tokens` (kimi-k2) or `cache_creation_tokens`.
# Mirror db_spend_update_writer to stay symmetric.
_cache_creation_tokens = float(
getattr(_pt_details, "cache_creation_tokens", 0)
or getattr(_pt_details, "cache_write_tokens", 0)
getattr(_pt_details, "cache_write_tokens", 0)
or getattr(_pt_details, "cache_creation_tokens", 0)
or 0
)
_anthropic_read = getattr(usage_object, "cache_read_input_tokens", None)
_anthropic_create = getattr(usage_object, "cache_creation_input_tokens", None)
if _anthropic_read or _anthropic_create:
if _anthropic_read is not None or _anthropic_create is not None:
_is_anthropic_style = True
if _anthropic_read:
if _anthropic_read is not None:
_cache_read_tokens = float(_anthropic_read)
if _anthropic_create:
if _anthropic_create is not None:
_cache_creation_tokens = float(_anthropic_create)
if not _cache_read_tokens and cache_read_input_tokens:
@ -2194,6 +2194,26 @@ def batch_cost_calculator(
)
except Exception:
model_info = None
elif not any(
model_info.get(k) is not None
for k in (
"input_cost_per_token_batches",
"input_cost_per_token",
"output_cost_per_token_batches",
"output_cost_per_token",
)
):
# model_info was provided (e.g. deployment metadata with only id/db_model)
# but carries no pricing fields. Fall back to the global pricing table so
# that standard model pricing is used instead of silently returning $0.
try:
global_info = litellm.get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
if global_info:
model_info = global_info
except Exception:
pass
if not model_info:
return 0.0, 0.0

4
litellm/exceptions.py
View file

@ -918,9 +918,11 @@ class GuardrailRaisedException(Exception):
guardrail_name: Optional[str] = None,
message: str = "",
should_wrap_with_default_message: bool = True,
status_code: int = 400,
):
default_message = f"Guardrail raised an exception, Guardrail: {guardrail_name}, Message: {message}"
self.guardrail_name = guardrail_name
self.status_code = status_code
self.message = default_message if should_wrap_with_default_message else message
super().__init__(self.message)
@ -930,12 +932,14 @@ class BlockedPiiEntityError(Exception):
self,
entity_type: str,
guardrail_name: Optional[str] = None,
status_code: int = 400,
):
"""
Raised when a blocked entity is detected by a guardrail.
"""
self.entity_type = entity_type
self.guardrail_name = guardrail_name
self.status_code = status_code
self.message = f"Blocked entity detected: {entity_type} by Guardrail: {guardrail_name}. This entity is not allowed to be used in this request."
super().__init__(self.message)

11
litellm/integrations/custom_guardrail.py
View file

@ -43,7 +43,11 @@ if TYPE_CHECKING:
dc = DualCache()
from litellm.exceptions import ModifyResponseException as ModifyResponseException
from litellm.exceptions import (
BlockedPiiEntityError,
GuardrailRaisedException,
ModifyResponseException,
)
class CustomGuardrail(CustomLogger):
@ -737,12 +741,15 @@ class CustomGuardrail(CustomLogger):
(this was logged previously as an API failure - guardrail_failed_to_respond).
Guardrails signal intentional blocks by raising:
- GuardrailRaisedException (generic guardrail API, tool permission)
- BlockedPiiEntityError (Presidio PII detection)
- HTTPException with status 400 (content policy violation)
- ModifyResponseException (passthrough mode violation)
"""
if isinstance(e, ModifyResponseException):
return True
if isinstance(e, (GuardrailRaisedException, BlockedPiiEntityError)):
return True
if (
HTTPException is not None
and isinstance(e, HTTPException)

292
litellm/integrations/opentelemetry.py
View file

@ -1,7 +1,7 @@
import os
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import datetime
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union, cast
import litellm
from litellm._logging import verbose_logger
@ -10,6 +10,12 @@ from litellm.integrations._types.open_inference import (
SpanAttributes,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
OTEL_SEMCONV_STABILITY_OPT_IN_ENV,
OTELGenAISemconvMixin,
OTELSemconvCategory,
parse_semconv_opt_in,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.secret_managers.main import get_secret_bool, str_to_bool
from litellm.types.services import ServiceLoggerPayload
@ -53,6 +59,11 @@ LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm")
LITELLM_METER_NAME = os.getenv("LITELLM_METER_NAME", "litellm")
LITELLM_LOGGER_NAME = os.getenv("LITELLM_LOGGER_NAME", "litellm")
LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
# OTel-standard names. status is also kept under error.code for back compat.
HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE = "http.response.status_code"
HTTP_ROUTE_ATTRIBUTE = "http.route"
URL_PATH_ATTRIBUTE = "url.path"
PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms"
# Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
@ -85,6 +96,7 @@ class OpenTelemetryConfig:
# Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT.
# One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
capture_message_content: Optional[str] = None
semconv_stability_opt_in: Set[OTELSemconvCategory] = field(default_factory=set)
def __post_init__(self) -> None:
# If endpoint is specified but exporter is still the default "console",
@ -110,6 +122,11 @@ class OpenTelemetryConfig:
self.ignore_context_propagation = str_to_bool(
os.getenv("OTEL_IGNORE_CONTEXT_PROPAGATION")
)
# Resolve the env opt-in once here so self.semconv_stability_opt_in is the
# single source of truth: the union of programmatic and env categories.
self.semconv_stability_opt_in |= parse_semconv_opt_in(
os.getenv(OTEL_SEMCONV_STABILITY_OPT_IN_ENV)
)
@classmethod
def from_env(cls):
@ -157,7 +174,7 @@ class OpenTelemetryConfig:
)
class OpenTelemetry(CustomLogger):
class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
def __init__(
self,
config: Optional[OpenTelemetryConfig] = None,
@ -655,6 +672,31 @@ class OpenTelemetry(CustomLogger):
parent_otel_span = user_api_key_dict.parent_otel_span
if parent_otel_span is not None:
parent_otel_span.set_status(Status(StatusCode.ERROR))
# Stamp structured error attrs on the SERVER span itself; the
# failure path otherwise only sets its status (_handle_failure
# records on the litellm_request child span). Inline import:
# litellm_logging <-> integrations is circular.
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
)
error_information = StandardLoggingPayloadSetup.get_error_information(
original_exception=original_exception,
traceback_str=traceback_str,
)
self._record_exception_on_span(
span=parent_otel_span,
kwargs={
"exception": original_exception,
"standard_logging_object": {"error_information": error_information},
},
)
# Pre-request latency (request_data carries the propagated
# metadata on the failure path; omitted if it failed before handoff).
self.set_preprocessing_duration_attribute(parent_otel_span, request_data)
_span_name = "Failed Proxy Server Request"
# Exception Logging Child Span
@ -691,6 +733,14 @@ class OpenTelemetry(CustomLogger):
ctx, _ = self._get_span_context(kwargs, default_span=parent_span)
# Pre-request latency on the SERVER span (success path).
self.set_preprocessing_duration_attribute(parent_span, kwargs)
# http.response.status_code on the SERVER span (success path).
# A successful proxy response is HTTP 200; the failure path sets
# this from the error code in _record_exception_on_span.
self.set_response_status_code_attribute(parent_span, 200)
# 3. Guardrail span
self._create_guardrail_span(kwargs=kwargs, context=ctx)
@ -979,13 +1029,14 @@ class OpenTelemetry(CustomLogger):
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
# Always create a new span
# The parent relationship is preserved through the context parameter
span = otel_tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=context,
)
span_kwargs: Dict[str, Any] = {
"name": self._get_span_name(kwargs),
"start_time": self._to_ns(start_time),
"context": context,
}
if self._gen_ai_semconv_latest_experimental:
span_kwargs["kind"] = self.span_kind.CLIENT
span = otel_tracer.start_span(**span_kwargs)
span.set_status(Status(StatusCode.OK))
self.set_attributes(span, kwargs, response_obj)
@ -998,6 +1049,10 @@ class OpenTelemetry(CustomLogger):
from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode
# raw_gen_ai_request is non-standard in semconv mode.
if self._gen_ai_semconv_latest_experimental:
return
if not self._capture_in_span():
return
@ -1023,7 +1078,11 @@ class OpenTelemetry(CustomLogger):
provider = params.get("custom_llm_provider", "Unknown")
common_attrs = {
"gen_ai.operation.name": "chat",
"gen_ai.operation.name": (
self._gen_ai_operation_name(kwargs)
if self._gen_ai_semconv_latest_experimental
else "chat"
),
"gen_ai.system": provider,
"gen_ai.request.model": kwargs.get("model"),
"gen_ai.framework": "litellm",
@ -1048,8 +1107,13 @@ class OpenTelemetry(CustomLogger):
"mcp_tool_call_metadata",
"vector_store_request_metadata",
]:
if md.get(key) is not None:
common_attrs[f"metadata.{key}"] = str(md[key])
value = md.get(key)
if value is None:
continue
if isinstance(value, (dict, list)):
common_attrs[f"metadata.{key}"] = safe_dumps(value)
else:
common_attrs[f"metadata.{key}"] = str(value)
# get hidden params
hidden_params = getattr(std_log, "hidden_params", None) or (std_log or {}).get(
@ -1246,6 +1310,24 @@ class OpenTelemetry(CustomLogger):
response_duration_seconds, attributes=common_attrs
)
@staticmethod
def _otel_log_types():
"""Resolve ``(LogRecord, SeverityNumber)`` across OTEL SDK versions.
``LogRecord`` moved out of ``opentelemetry.sdk._logs`` in OTEL >= 1.39.0
(open-telemetry/opentelemetry-python#4676). Imports stay function-local
because the SDK is an optional dependency.
"""
from opentelemetry._logs import SeverityNumber
try:
from opentelemetry.sdk._logs import LogRecord # OTEL < 1.39.0
except ImportError:
from opentelemetry.sdk._logs._internal import ( # OTEL >= 1.39.0
LogRecord,
)
return LogRecord, SeverityNumber
def _emit_semantic_logs(self, kwargs, response_obj, span: Span):
if not self.config.enable_events:
return
@ -1259,16 +1341,7 @@ class OpenTelemetry(CustomLogger):
# See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
# TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
from opentelemetry._logs import SeverityNumber
try:
from opentelemetry.sdk._logs import ( # type: ignore[attr-defined] # OTEL < 1.39.0
LogRecord as SdkLogRecord,
)
except ImportError:
from opentelemetry.sdk._logs._internal import (
LogRecord as SdkLogRecord, # type: ignore[attr-defined] # OTEL >= 1.39.0
)
SdkLogRecord, SeverityNumber = self._otel_log_types()
# Resolve through the handler's own LoggerProvider (which may be a
# private one when skip_set_global=True) rather than the module-level
@ -1280,6 +1353,16 @@ class OpenTelemetry(CustomLogger):
"custom_llm_provider", "Unknown"
)
if self._gen_ai_semconv_latest_experimental:
self._emit_inference_details_event(
kwargs=kwargs,
response_obj=response_obj,
provider=provider,
otel_logger=otel_logger,
parent_ctx=parent_ctx,
)
return
# per-message events
for msg in kwargs.get("messages", []):
role = msg.get("role", "user")
@ -1496,11 +1579,14 @@ class OpenTelemetry(CustomLogger):
if should_create_primary_span:
# Span 1: Request sent to litellm SDK
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
span = otel_tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=_parent_context,
)
span_kwargs: Dict[str, Any] = {
"name": self._get_span_name(kwargs),
"start_time": self._to_ns(start_time),
"context": _parent_context,
}
if self._gen_ai_semconv_latest_experimental:
span_kwargs["kind"] = self.span_kind.CLIENT
span = otel_tracer.start_span(**span_kwargs)
span.set_status(Status(StatusCode.ERROR))
self.set_attributes(span, kwargs, response_obj)
@ -1584,6 +1670,19 @@ class OpenTelemetry(CustomLogger):
value=error_information["error_code"],
)
# Also expose under the OTel-standard name as an int
# (error_code is a str, may be non-numeric).
_error_code_val = error_information["error_code"]
if _error_code_val is not None:
try:
self.safe_set_attribute(
span=span,
key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
value=int(_error_code_val),
)
except (ValueError, TypeError):
pass
if error_information.get("error_class"):
self.safe_set_attribute(
span=span,
@ -1782,11 +1881,21 @@ class OpenTelemetry(CustomLogger):
)
# The Generative AI Provider: Azure, OpenAI, etc.
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_SYSTEM.value,
value=litellm_params.get("custom_llm_provider", "Unknown"),
)
provider_name = litellm_params.get("custom_llm_provider", "Unknown")
# Latest-experimental semconv replaced gen_ai.system with
# gen_ai.provider.name; emit only the conformant key in that mode.
if self._gen_ai_semconv_latest_experimental:
self.safe_set_attribute(
span=span,
key="gen_ai.provider.name",
value=provider_name,
)
else:
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_SYSTEM.value,
value=provider_name,
)
# The maximum number of tokens the LLM generates for a request.
if optional_params.get("max_tokens"):
@ -1812,11 +1921,17 @@ class OpenTelemetry(CustomLogger):
value=optional_params.get("top_p"),
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_IS_STREAMING.value,
value=str(optional_params.get("stream", False)),
)
if self._gen_ai_semconv_latest_experimental:
# Semconv emits gen_ai.request.stream (only when streaming) via
# _set_semconv_request_attributes; skip the legacy llm.is_streaming.
self._set_semconv_request_attributes(span, optional_params)
self._set_semconv_cache_token_attributes(span, standard_logging_payload)
else:
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_IS_STREAMING.value,
value=str(optional_params.get("stream", False)),
)
if optional_params.get("user"):
self.safe_set_attribute(
@ -1937,14 +2052,18 @@ class OpenTelemetry(CustomLogger):
value=safe_dumps(transformed_system_instructions),
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_OPERATION_NAME.value,
value=(
if self._gen_ai_semconv_latest_experimental:
operation_name = self._gen_ai_operation_name(kwargs)
else:
operation_name = (
"chat"
if standard_logging_payload.get("call_type") == "completion"
else standard_logging_payload.get("call_type") or "chat"
),
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_OPERATION_NAME.value,
value=operation_name,
)
if standard_logging_payload.get("request_id"):
@ -2281,6 +2400,10 @@ class OpenTelemetry(CustomLogger):
if generation_name:
return generation_name
if self._gen_ai_semconv_latest_experimental:
model = kwargs.get("model") or "unknown"
return f"{self._gen_ai_operation_name(kwargs)} {model}"
return LITELLM_REQUEST_SPAN_NAME
def get_traceparent_from_header(self, headers):
@ -2822,3 +2945,86 @@ class OpenTelemetry(CustomLogger):
context=self.get_traceparent_from_header(headers=headers),
kind=self.span_kind.SERVER,
)
def set_proxy_request_route_attributes(
self,
span: Optional[Span],
*,
url_path: Optional[str] = None,
http_route: Optional[str] = None,
) -> None:
"""
Set OTel-standard ``http.route`` / ``url.path`` on the proxy SERVER
span. Called from the auth path, the only point where both the
SERVER span and the request are in hand. No-op if span/value missing.
"""
if span is None:
return
if url_path:
self.safe_set_attribute(span=span, key=URL_PATH_ATTRIBUTE, value=url_path)
if http_route:
self.safe_set_attribute(
span=span, key=HTTP_ROUTE_ATTRIBUTE, value=http_route
)
def set_response_status_code_attribute(
self, span: Optional[Span], status_code: Optional[int]
) -> None:
"""
Set OTel-standard ``http.response.status_code`` (int) on the proxy
SERVER span. The failure path sets this from the error code in
``_record_exception_on_span``; this is the success-path counterpart
so the attribute is present on every SERVER span regardless of
outcome (required by the HTTP semconv, and needed for error-ratio /
status-breakdown dashboards). No-op if span/value missing.
"""
if span is None or status_code is None:
return
self.safe_set_attribute(
span=span,
key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
value=int(status_code),
)
def set_preprocessing_duration_attribute(
self, span: Optional[Span], container: Any
) -> None:
"""
Set ``litellm.preprocessing.duration_ms`` (proxy-receive -> first
provider handoff) on the proxy SERVER span. ``litellm_received_at``
rides request metadata; ``first_api_call_start_time`` is the
set-once first-handoff instant (retries/backoff excluded). Works
uniformly for the success (model_call_details) and failure
(request_data) containers. No-op if span/either anchor is missing.
"""
if span is None or not isinstance(container, dict):
return
received_at = None
# first_api_call_start_time is top-level (never in user metadata).
first_handoff = container.get("first_api_call_start_time")
_lp = container.get("litellm_params")
for _md in (
(_lp or {}).get("metadata") if isinstance(_lp, dict) else None,
container.get("metadata"),
container.get("litellm_metadata"),
):
if isinstance(_md, dict):
received_at = received_at or _md.get("litellm_received_at")
if received_at is None or first_handoff is None:
return
try:
start_ts = self._to_timestamp(received_at)
end_ts = self._to_timestamp(first_handoff)
except Exception:
return
if start_ts is None or end_ts is None:
return
duration_ms = (end_ts - start_ts) * 1000.0
# Clock skew → omit rather than emit a negative latency.
if duration_ms < 0:
return
self.safe_set_attribute(
span=span,
key=PREPROCESSING_DURATION_MS_ATTRIBUTE,
value=duration_ms,
)

271
litellm/integrations/opentelemetry_utils/gen_ai_semconv.py Normal file
View file

@ -0,0 +1,271 @@
"""OTEL GenAI ``gen_ai_latest_experimental`` semantic conventions.
Setting ``OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental`` switches the
emitted traces to the experimental OTEL GenAI conventions
(https://opentelemetry.io/docs/specs/semconv/gen-ai/). Concretely, versus the
default LiteLLM output:
Request span:
- name is ``{operation} {model}`` (e.g. ``chat gpt-4``) instead of
``litellm_request``; span kind is ``CLIENT``.
- ``gen_ai.operation.name`` is the actual operation (``chat`` /
``text_completion`` / ``embeddings``) instead of always ``chat``.
- the provider is reported as ``gen_ai.provider.name``; the superseded
``gen_ai.system`` and the legacy ``llm.is_streaming`` are dropped.
- adds ``gen_ai.request.{frequency_penalty,presence_penalty,top_k,seed}``,
``gen_ai.request.stop_sequences`` (a string array),
``gen_ai.request.stream`` (only when streaming),
``gen_ai.request.choice.count`` (only when n > 1), and
``gen_ai.usage.cache_{creation,read}.input_tokens``.
- the non-standard ``raw_gen_ai_request`` child span is no longer created.
Events:
- the per-message ``gen_ai.content.prompt`` / per-choice
``gen_ai.content.completion`` log events are replaced by a single
``gen_ai.client.inference.operation.details`` log event carrying
``gen_ai.input.messages`` / ``gen_ai.output.messages`` (message content
included only when content capture is enabled).
"""
from datetime import datetime
from enum import Enum
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.integrations.opentelemetry import OpenTelemetryConfig
Span = Union[_Span, Any]
else:
Span = Any
# OTEL_SEMCONV_STABILITY_OPT_IN is a comma-separated list of category-specific
# opt-in values. See https://opentelemetry.io/docs/specs/semconv/gen-ai/
OTEL_SEMCONV_STABILITY_OPT_IN_ENV = "OTEL_SEMCONV_STABILITY_OPT_IN"
class OTELSemconvCategory(Enum):
GEN_AI_LATEST_EXPERIMENTAL = "gen_ai_latest_experimental"
# Reverse lookup: opt-in token string -> OTELSemconvCategory.
_SEMCONV_CATEGORY_BY_VALUE = {
category.value: category for category in OTELSemconvCategory
}
# LiteLLM optional_params key -> OTEL gen_ai semconv span attribute.
_SEMCONV_REQUEST_ATTRIBUTES = {
"frequency_penalty": "gen_ai.request.frequency_penalty",
"presence_penalty": "gen_ai.request.presence_penalty",
"top_k": "gen_ai.request.top_k",
"seed": "gen_ai.request.seed",
}
# usage_object key -> OTEL gen_ai semconv cache-token span attribute.
_SEMCONV_CACHE_TOKEN_ATTRIBUTES = {
"cache_creation_input_tokens": "gen_ai.usage.cache_creation.input_tokens",
"cache_read_input_tokens": "gen_ai.usage.cache_read.input_tokens",
}
# Name of the consolidated GenAI inference event (replaces the legacy
# per-message gen_ai.content.prompt / per-choice gen_ai.content.completion).
_INFERENCE_DETAILS_EVENT_NAME = "gen_ai.client.inference.operation.details"
def parse_semconv_opt_in(raw: Optional[str]) -> Set[OTELSemconvCategory]:
"""Parse the comma-separated OTEL_SEMCONV_STABILITY_OPT_IN value into the
set of recognized categories. Unknown tokens are ignored per the spec."""
if not raw:
return set()
return {
_SEMCONV_CATEGORY_BY_VALUE[token]
for token in (part.strip() for part in raw.split(","))
if token in _SEMCONV_CATEGORY_BY_VALUE
}
class OTELGenAISemconvMixin:
"""OTEL GenAI ``gen_ai_latest_experimental`` semantic-convention behavior.
Mixed into ``OpenTelemetry`` (its only host). Every member is internal to
the OTEL integration; the leading underscore marks "subsystem-internal",
not "class-private" (the host lives in a sibling module).
Members the host calls (the mixin -> host contract):
- ``_gen_ai_semconv_latest_experimental`` -- opt-in gate; guards every
semconv code path in ``opentelemetry.py``.
- ``_gen_ai_operation_name`` -- LiteLLM ``call_type`` -> spec
``gen_ai.operation.name``.
- ``_set_semconv_request_attributes`` /
``_set_semconv_cache_token_attributes`` -- add the ``gen_ai.request.*``
/ ``gen_ai.usage.cache_*`` span attributes.
- ``_emit_inference_details_event`` -- emit the consolidated event.
Helpers the host must provide (declared under ``TYPE_CHECKING`` below):
``config``, ``safe_set_attribute``, ``_capture_in_event``,
``_transform_messages_to_otel_semantic_conventions``,
``_transform_choices_to_otel_semantic_conventions``, ``_to_ns``,
``_otel_log_types``.
"""
if TYPE_CHECKING:
config: "OpenTelemetryConfig"
def safe_set_attribute(self, span: Span, key: str, value: Any) -> None: ...
def _capture_in_event(self) -> bool: ...
def _transform_messages_to_otel_semantic_conventions(
self, messages: Union[List[dict], str]
) -> List[dict]: ...
def _transform_choices_to_otel_semantic_conventions(
self, choices: List[dict]
) -> List[dict]: ...
def _to_ns(self, dt: datetime) -> int: ...
def _otel_log_types(self) -> Tuple[Any, Any]: ...
@property
def _gen_ai_semconv_latest_experimental(self) -> bool:
"""Whether the ``gen_ai_latest_experimental`` opt-in is active.
Every semconv behavior is gated on this; ``False`` => legacy output.
"""
return (
OTELSemconvCategory.GEN_AI_LATEST_EXPERIMENTAL
in self.config.semconv_stability_opt_in
)
@staticmethod
def _gen_ai_operation_name(kwargs: dict) -> str:
"""Map a LiteLLM ``call_type`` to spec ``gen_ai.operation.name``.
Substring match (e.g. ``aembedding`` -> ``embeddings``); defaults to
``chat``.
"""
call_type = kwargs.get("call_type", "") or ""
match call_type:
case s if "embedding" in s:
return "embeddings"
case s if "text_completion" in s:
return "text_completion"
case _:
return "chat"
def _set_semconv_request_attributes(
self, span: Span, optional_params: dict
) -> None:
"""Add ``gen_ai.request.*`` span attributes from ``optional_params``.
Covers the sampling params plus the conditionally-required
``stop_sequences`` / ``stream`` / ``choice.count`` per the spec.
"""
for source_key, semconv_key in _SEMCONV_REQUEST_ATTRIBUTES.items():
value = optional_params.get(source_key)
if value is not None:
self.safe_set_attribute(span=span, key=semconv_key, value=value)
stop = optional_params.get("stop")
if stop is not None:
# Spec types this as string[]. safe_set_attribute coerces to a
# primitive, so set the array directly via the span API.
stop_list = stop if isinstance(stop, list) else [stop]
span.set_attribute(
"gen_ai.request.stop_sequences", [str(s) for s in stop_list]
)
# Conditionally required: set only when the request is streaming.
if optional_params.get("stream"):
self.safe_set_attribute(span=span, key="gen_ai.request.stream", value=True)
# Conditionally required per spec ("if available and != 1"). Valid n is
# an int >= 1, so n > 1 is equivalent for conformant input while
# suppressing nonsensical values (0, negative, non-int).
n = optional_params.get("n")
if isinstance(n, int) and n > 1:
self.safe_set_attribute(
span=span, key="gen_ai.request.choice.count", value=n
)
def _set_semconv_cache_token_attributes(
self, span: Span, standard_logging_payload
) -> None:
"""Add ``gen_ai.usage.cache_*.input_tokens`` from the usage object.
No-op when the payload or the usage values are missing/zero.
"""
if not standard_logging_payload:
return
usage = (standard_logging_payload.get("metadata") or {}).get(
"usage_object"
) or {}
for source_key, semconv_key in _SEMCONV_CACHE_TOKEN_ATTRIBUTES.items():
value = usage.get(source_key)
if value:
self.safe_set_attribute(span=span, key=semconv_key, value=value)
def _build_inference_details_attrs(
self, kwargs: dict, response_obj: dict, provider: str
) -> Dict[str, Any]:
"""Build the attribute payload for the inference-details event.
Always includes provider/operation; input/output messages are added
only when content capture is enabled and non-empty. Mixin-internal.
"""
attrs: Dict[str, Any] = {
"event_name": _INFERENCE_DETAILS_EVENT_NAME,
"gen_ai.provider.name": provider,
"gen_ai.operation.name": self._gen_ai_operation_name(kwargs),
}
if not self._capture_in_event():
return attrs
input_messages = self._transform_messages_to_otel_semantic_conventions(
kwargs.get("messages") or []
)
output_messages = self._transform_choices_to_otel_semantic_conventions(
response_obj.get("choices", [])
)
if input_messages:
attrs["gen_ai.input.messages"] = safe_dumps(input_messages)
if output_messages:
attrs["gen_ai.output.messages"] = safe_dumps(output_messages)
return attrs
def _emit_inference_details_event(
self,
kwargs: dict,
response_obj: dict,
provider: str,
otel_logger,
parent_ctx,
) -> None:
"""Emit the consolidated ``gen_ai.client.inference.operation.details``
log event, correlated to the request span via ``parent_ctx``.
Replaces the legacy per-message / per-choice content events.
"""
LogRecord, SeverityNumber = self._otel_log_types()
log_record = LogRecord(
timestamp=self._to_ns(datetime.now()),
trace_id=parent_ctx.trace_id,
span_id=parent_ctx.span_id,
trace_flags=parent_ctx.trace_flags,
severity_number=SeverityNumber.INFO,
severity_text="INFO",
body=None,
attributes=self._build_inference_details_attrs(
kwargs, response_obj, provider
),
)
otel_logger.emit(log_record)

6
litellm/integrations/prometheus.py
View file

@ -3540,6 +3540,10 @@ class PrometheusLogger(CustomLogger):
user_object.budget_reset_at = user_info.budget_reset_at
if user_object.max_budget is None and user_info.max_budget is not None:
user_object.max_budget = user_info.max_budget
if user_info.user_email is not None:
user_object.user_email = user_info.user_email
if user_info.user_alias is not None:
user_object.user_alias = user_info.user_alias
return user_object
@ -3556,6 +3560,8 @@ class PrometheusLogger(CustomLogger):
"""
enum_values = UserAPIKeyLabelValues(
user=user.user_id,
user_email=user.user_email or "",
user_alias=user.user_alias or "",
)
_labels = prometheus_label_factory(

10
litellm/litellm_core_utils/litellm_logging.py
View file

@ -1050,6 +1050,16 @@ class Logging(LiteLLMLoggingBaseClass):
)
self.model_call_details["api_call_start_time"] = datetime.datetime.now()
# Set-once first provider-handoff instant. api_call_start_time
# is overwritten on every retry, so it can't measure one-time
# preprocessing; pinning the first attempt excludes retry loops
# + backoff. Logging object only — must NOT go into
# litellm_params["metadata"] (caller request metadata, typed
# Dict[str, str], echoed downstream; a datetime breaks it).
if self.model_call_details.get("first_api_call_start_time") is None:
self.model_call_details["first_api_call_start_time"] = (
self.model_call_details["api_call_start_time"]
)
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
callbacks = litellm.input_callback + (self.dynamic_input_callbacks or [])
for callback in callbacks:

14
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -20,6 +20,7 @@ from typing import (
cast,
)
import litellm
from litellm import verbose_logger
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
@ -1170,9 +1171,16 @@ def migrate_file_to_image_url(
ChatCompletionImageUrlObject,
)
file_id = message["file"].get("file_id")
file_data = message["file"].get("file_data")
format = message["file"].get("format")
file_sub = message.get("file")
if file_sub is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider=None,
)
file_id = file_sub.get("file_id")
file_data = file_sub.get("file_data")
format = file_sub.get("format")
if not file_id and not file_data:
raise ValueError("file_id and file_data are both None")
image_url_object = ChatCompletionImageObject(

29
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -2057,9 +2057,16 @@ def anthropic_process_openai_file_message(
AnthropicMessagesContainerUploadParam,
]:
file_message = cast(ChatCompletionFileObject, message)
file_data = file_message["file"].get("file_data")
file_id = file_message["file"].get("file_id")
format = file_message["file"].get("format")
file_sub = file_message.get("file")
if file_sub is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="anthropic",
)
file_data = file_sub.get("file_data")
file_id = file_sub.get("file_id")
format = file_sub.get("format")
if file_data:
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=file_data,
@ -4879,7 +4886,13 @@ class BedrockConverseMessagesProcessor:
@staticmethod
def _process_file_message(message: ChatCompletionFileObject) -> BedrockContentBlock:
file_message = message["file"]
file_message = message.get("file")
if file_message is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="bedrock",
)
file_data = file_message.get("file_data")
file_id = file_message.get("file_id")
@ -4900,7 +4913,13 @@ class BedrockConverseMessagesProcessor:
async def _async_process_file_message(
message: ChatCompletionFileObject,
) -> BedrockContentBlock:
file_message = message["file"]
file_message = message.get("file")
if file_message is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="bedrock",
)
file_data = file_message.get("file_data")
file_id = file_message.get("file_id")
format = file_message.get("format")

43
litellm/llms/anthropic/common_utils.py
View file

@ -832,6 +832,49 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(
data.pop("thinking", None)
def strip_empty_text_blocks_from_anthropic_messages(
messages: List[Any],
) -> List[Any]:
"""
Return a new message list with empty or whitespace-only ``{"type": "text"}``
content blocks removed.
Anthropic's API rejects requests containing such blocks with
``"messages: text content blocks must be non-empty"``, but assistant
messages from Anthropic routinely arrive with ``{"type": "text", "text": ""}``
alongside ``tool_use`` blocks (see anthropics/anthropic-sdk-python#461).
Multi-turn tool-use clients (e.g. Claude Code) loop these prior responses
back as conversation history, which then causes the next request to 400
on the unified ``/v1/messages`` path. ``/v1/chat/completions`` already
handles this in ``anthropic_messages_pt``; this helper provides the
equivalent guarantee for the native Anthropic Messages path.
Messages whose content is a list and becomes empty after stripping are
omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`.
The caller's list and its content blocks are never mutated; modified
messages are returned as shallow copies with a fresh content list.
"""
out: List[Any] = []
for m in messages:
if not isinstance(m, dict) or not isinstance(m.get("content"), list):
out.append(m)
continue
content = m["content"]
filtered = [b for b in content if not _is_empty_text_block(b)]
if len(filtered) == len(content):
out.append(m)
elif filtered:
out.append({**m, "content": filtered})
return out
def _is_empty_text_block(block: Any) -> bool:
if not isinstance(block, dict) or block.get("type") != "text":
return False
text = block.get("text")
return not isinstance(text, str) or not text.strip()
def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict:
openai_headers = {}
if "anthropic-ratelimit-requests-limit" in headers:

22
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -12,6 +12,9 @@ from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union, c
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
strip_empty_text_blocks_from_anthropic_messages,
)
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -188,8 +191,20 @@ async def anthropic_messages(
**kwargs,
) -> Union[AnthropicMessagesResponse, AsyncIterator]:
"""
Async: Make llm api request in Anthropic /messages API spec
Async: Make llm api request in Anthropic /messages API spec.
Runs the empty-text-block sanitizer before any backend dispatch.
"""
# Anthropic's API rejects requests containing empty / whitespace-only
# text content blocks with "messages: text content blocks must be
# non-empty". Multi-turn tool-use clients (e.g. Claude Code) routinely
# loop assistant responses that contain {"type": "text", "text": ""}
# alongside tool_use blocks back as conversation history, which then
# causes the next /v1/messages call to 400. /v1/chat/completions
# already handles this in anthropic_messages_pt; sanitize the native
# Anthropic Messages path here for the same guarantee. See #22930.
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
original_stream = stream or kwargs.get(
"_websearch_interception_converted_stream", False
)
@ -336,6 +351,11 @@ def anthropic_messages_handler(
"""
from litellm.types.utils import LlmProviders
# Sanitize empty text blocks here too so the sync entry point
# (litellm.messages.create -> anthropic_messages_handler) gets the same
# protection as the async wrapper. Idempotent when called twice.
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
metadata = validate_anthropic_api_metadata(metadata)
local_vars = locals()

26
litellm/llms/bedrock/batches/transformation.py
View file

@ -5,6 +5,7 @@ from typing import Any, Dict, List, Literal, Optional, Union, cast
from httpx import Headers, Response
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
@ -263,9 +264,32 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
cancelling_at=None,
cancelled_at=None,
request_counts=None,
metadata=original_request.get("metadata", {}),
metadata=self._get_openai_compatible_batch_metadata(
original_request.get("metadata", {})
),
)
@staticmethod
def _get_openai_compatible_batch_metadata(metadata: Any) -> Dict[str, str]:
"""
OpenAI Batch metadata only accepts string values.
"""
if not isinstance(metadata, dict):
return {}
sanitized_metadata: Dict[str, str] = {}
for key, value in metadata.items():
if key == "standard_logging_guardrail_information" or value is None:
continue
str_key = str(key)
if isinstance(value, str):
sanitized_metadata[str_key] = value
else:
sanitized_metadata[str_key] = safe_dumps(value)
return sanitized_metadata
def transform_retrieve_batch_request(
self,
batch_id: str,

4
litellm/llms/bedrock/chat/mantle/transformation.py
View file

@ -21,7 +21,9 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
MANTLE_ENDPOINT_TEMPLATE = "https://bedrock-mantle.{region}.api.aws/v1/messages"
MANTLE_ENDPOINT_TEMPLATE = (
"https://bedrock-mantle.{region}.api.aws/anthropic/v1/messages"
)
class AmazonMantleConfig(AmazonAnthropicClaudeConfig):

2
litellm/llms/bedrock/embed/cohere_transformation.py
View file

@ -22,7 +22,7 @@ class BedrockCohereEmbeddingConfig:
) -> dict:
for k, v in non_default_params.items():
if k == "encoding_format":
optional_params["embedding_types"] = v
optional_params["embedding_types"] = v if isinstance(v, list) else [v]
elif k == "dimensions":
optional_params["output_dimension"] = v
return optional_params

4
litellm/llms/bedrock/messages/mantle_transformation.py
View file

@ -20,7 +20,9 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
MANTLE_ENDPOINT_TEMPLATE = "https://bedrock-mantle.{region}.api.aws/v1/messages"
MANTLE_ENDPOINT_TEMPLATE = (
"https://bedrock-mantle.{region}.api.aws/anthropic/v1/messages"
)
class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):

28
litellm/llms/dashscope/common_utils.py Normal file
View file

@ -0,0 +1,28 @@
"""
Common utilities for the DashScope LLM provider.
"""
from typing import Optional
import httpx
from litellm.llms.base_llm.chat.transformation import BaseLLMException
class DashScopeError(BaseLLMException):
"""Exception class for DashScope provider errors."""
def __init__(
self,
status_code: int,
message: str,
headers: Optional[httpx.Headers] = None,
):
self.status_code = status_code
self.message = message
self.headers = headers or httpx.Headers()
super().__init__(
status_code=status_code,
message=message,
headers=dict(self.headers),
)

7
litellm/llms/dashscope/embed/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""
DashScope Embedding Module
"""
from .transformation import DashScopeEmbeddingConfig
__all__ = ["DashScopeEmbeddingConfig"]

191
litellm/llms/dashscope/embed/transformation.py Normal file
View file

@ -0,0 +1,191 @@
"""
Transformation logic from OpenAI /v1/embeddings format to DashScope's /v1/embeddings format.
Supports
- text-embedding-v4
- text-embedding-v3
Endpoint
- https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings
Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
"""
from typing import List, Optional, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
from litellm.types.utils import EmbeddingResponse, Usage
from ..common_utils import DashScopeError
DEFAULT_API_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
"""
Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
DashScope exposes an OpenAI-compatible /v1/embeddings endpoint, so the
request and response shapes are nearly identical to OpenAI's.
"""
def __init__(self) -> None:
pass
def get_supported_openai_params(self, model: str) -> List[str]:
# DashScope's compatible-mode embeddings API accepts the same params as OpenAI.
# `dimensions` / `encoding_format` are only honored by text-embedding-v3 / v4;
# earlier versions silently ignore them server-side.
return ["dimensions", "encoding_format", "user"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool = False,
) -> dict:
supported = self.get_supported_openai_params(model)
for k, v in non_default_params.items():
if v is None:
continue
if k in supported:
optional_params[k] = v
# unsupported params are dropped when drop_params=True;
# the upstream _check_valid_arg already raised UnsupportedParamsError
# for drop_params=False before this method is called.
return optional_params
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("DASHSCOPE_API_KEY")
if api_key is None:
raise ValueError(
"DashScope API key is required. Set 'DASHSCOPE_API_KEY' env var or pass api_key explicitly."
)
default_headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}",
}
return {**default_headers, **headers}
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
base = api_base or get_secret_str("DASHSCOPE_API_BASE") or DEFAULT_API_BASE
base = base.rstrip("/")
if base.endswith("/embeddings"):
return base
return f"{base}/embeddings"
def transform_embedding_request(
self,
model: str,
input: AllEmbeddingInputValues,
optional_params: dict,
headers: dict,
) -> dict:
data: dict = {
"model": model,
"input": input,
}
for key in ("dimensions", "encoding_format", "user"):
value = optional_params.get(key)
if value is not None:
data[key] = value
return data
def transform_embedding_response(
self,
model: str,
raw_response: httpx.Response,
model_response: EmbeddingResponse,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str],
request_data: dict,
optional_params: dict,
litellm_params: dict,
) -> EmbeddingResponse:
try:
response_json = raw_response.json()
except Exception as e:
raise DashScopeError(
status_code=raw_response.status_code,
message=f"Failed to parse DashScope response as JSON: {str(e)}",
)
logging_obj.post_call(
input=request_data.get("input"),
api_key=api_key,
additional_args={"complete_input_dict": request_data},
original_response=response_json,
)
if "error" in response_json:
error = response_json["error"]
message = (
error.get("message", str(error))
if isinstance(error, dict)
else str(error)
)
raise DashScopeError(
status_code=raw_response.status_code,
message=message,
)
model_response.object = "list"
model_response.data = response_json.get("data", [])
model_response.model = response_json.get("model", model)
usage = response_json.get("usage") or {}
prompt_tokens = usage.get("prompt_tokens", 0)
total_tokens = usage.get("total_tokens", prompt_tokens)
setattr(
model_response,
"usage",
Usage(
prompt_tokens=prompt_tokens,
completion_tokens=0,
total_tokens=total_tokens,
),
)
if "id" in response_json:
setattr(model_response, "id", response_json["id"])
return model_response
def get_error_class(
self,
error_message: str,
status_code: int,
headers: Union[dict, httpx.Headers],
) -> BaseLLMException:
if isinstance(headers, dict):
headers = httpx.Headers(headers)
return DashScopeError(
status_code=status_code,
message=error_message,
headers=headers,
)

7
litellm/llms/dashscope/rerank/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""
DashScope Rerank Module
"""
from .transformation import DashScopeRerankConfig
__all__ = ["DashScopeRerankConfig"]

241
litellm/llms/dashscope/rerank/transformation.py Normal file
View file

@ -0,0 +1,241 @@
"""
Transformation logic for DashScope's OpenAI-compatible /v1/reranks API.
Supports
- qwen3-rerank
(Other DashScope rerankers — gte-rerank-v2 / qwen3-vl-rerank — share the same
endpoint but have not been validated against this transformer. Behavior with
those models is undefined.)
Endpoint
- https://dashscope.aliyuncs.com/compatible-api/v1/reranks
Note: chat/embed live under `/compatible-mode/v1/`, but DashScope's rerank
route is exposed under `/compatible-api/v1/reranks` per the docs. Override
with `DASHSCOPE_API_BASE_RERANK` to point at a different host or path.
Empirically, qwen3-rerank accepts `return_documents=true` and echoes
`results[].document.text` back, even though the public docs list the flag
as supported only for gte-rerank-v2 / qwen3-vl-rerank.
Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api
"""
from typing import Any, Dict, List, Optional, Union
import httpx
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.rerank import (
OptionalRerankParams,
RerankBilledUnits,
RerankResponse,
RerankResponseMeta,
RerankTokens,
)
from ..common_utils import DashScopeError
DEFAULT_RERANK_URL = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
class DashScopeRerankConfig(BaseRerankConfig):
"""
Reference: https://help.aliyun.com/zh/model-studio/text-rerank-api
Targets DashScope's qwen3-rerank model. Request fields: model, query,
documents, top_n, return_documents. Response: results[].index,
results[].relevance_score, optionally results[].document.text (when
return_documents=true), plus a top-level usage.total_tokens counter.
"""
def __init__(self) -> None:
pass
def get_complete_url(
self,
api_base: Optional[str],
model: str,
optional_params: Optional[dict] = None,
) -> str:
if api_base is None:
api_base = get_secret_str("DASHSCOPE_API_BASE_RERANK") or DEFAULT_RERANK_URL
if api_base == DEFAULT_RERANK_URL:
return DEFAULT_RERANK_URL
cleaned = api_base.rstrip("/")
if cleaned.endswith("/reranks") or cleaned.endswith("/rerank"):
return cleaned
if cleaned.endswith("/v1"):
return f"{cleaned}/reranks"
# Unknown base: append /reranks rather than silently ignoring the caller's api_base.
return f"{cleaned}/reranks"
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
optional_params: Optional[dict] = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("DASHSCOPE_API_KEY")
if api_key is None:
raise ValueError(
"DashScope API key is required. Set 'DASHSCOPE_API_KEY' env var or pass api_key explicitly."
)
default_headers = {
"Authorization": f"Bearer {api_key}",
"accept": "application/json",
"content-type": "application/json",
}
return {**default_headers, **headers}
def get_supported_cohere_rerank_params(self, model: str) -> list:
return ["query", "documents", "top_n", "return_documents"]
def map_cohere_rerank_params(
self,
non_default_params: Optional[dict],
model: str,
drop_params: bool,
query: str,
documents: List[Union[str, Dict[str, Any]]],
custom_llm_provider: Optional[str] = None,
top_n: Optional[int] = None,
rank_fields: Optional[List[str]] = None,
return_documents: Optional[bool] = True,
max_chunks_per_doc: Optional[int] = None,
max_tokens_per_doc: Optional[int] = None,
) -> Dict:
# qwen3-rerank accepts query/documents/top_n/return_documents. The
# rest (rank_fields, max_*_per_doc) are silently dropped.
params: OptionalRerankParams = OptionalRerankParams(
query=query,
documents=documents,
)
if top_n is not None:
params["top_n"] = top_n
if return_documents is not None:
params["return_documents"] = return_documents
return dict(params)
def transform_rerank_request(
self,
model: str,
optional_rerank_params: Dict,
headers: dict,
litellm_params: Optional[dict] = None,
) -> dict:
if "query" not in optional_rerank_params:
raise ValueError("query is required for DashScope rerank")
if "documents" not in optional_rerank_params:
raise ValueError("documents is required for DashScope rerank")
request: Dict[str, Any] = {
"model": model,
"query": optional_rerank_params["query"],
"documents": optional_rerank_params["documents"],
}
if optional_rerank_params.get("top_n") is not None:
request["top_n"] = optional_rerank_params["top_n"]
if optional_rerank_params.get("return_documents") is not None:
request["return_documents"] = optional_rerank_params["return_documents"]
return request
def transform_rerank_response(
self,
model: str,
raw_response: httpx.Response,
model_response: RerankResponse,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str] = None,
request_data: Optional[dict] = None,
optional_params: Optional[dict] = None,
litellm_params: Optional[dict] = None,
) -> RerankResponse:
request_data = request_data or {}
optional_params = optional_params or {}
litellm_params = litellm_params or {}
try:
response_json = raw_response.json()
except Exception:
raise DashScopeError(
status_code=raw_response.status_code,
message=raw_response.text,
)
logging_obj.post_call(
input=request_data.get("query"),
api_key=api_key,
additional_args={"complete_input_dict": request_data},
original_response=response_json,
)
# DashScope error envelope: {"code": "...", "message": "...", "request_id": "..."}
if "code" in response_json and "results" not in response_json:
raise DashScopeError(
status_code=raw_response.status_code,
message=response_json.get("message", str(response_json)),
)
results = response_json.get("results")
if results is None:
raise DashScopeError(
status_code=raw_response.status_code,
message=f"No results in DashScope rerank response: {response_json}",
)
# qwen3-rerank returns:
# {"index": int, "relevance_score": float}
# plus, when return_documents=true was sent:
# "document": {"text": "..."}
# which already matches LiteLLM's RerankResponseDocument shape.
transformed_results: List[dict] = []
for r in results:
item: Dict[str, Any] = {
"index": r["index"],
"relevance_score": r["relevance_score"],
}
doc = r.get("document")
if isinstance(doc, dict):
item["document"] = doc
elif isinstance(doc, str):
# Defensive: spec says dict, but normalize string-shaped echoes.
item["document"] = {"text": doc}
transformed_results.append(item)
usage = response_json.get("usage") or {}
total_tokens = usage.get("total_tokens")
billed_units = RerankBilledUnits(total_tokens=total_tokens)
tokens = RerankTokens(input_tokens=total_tokens)
meta = RerankResponseMeta(billed_units=billed_units, tokens=tokens)
return RerankResponse(
id=response_json.get("id") or str(uuid.uuid4()),
results=transformed_results, # type: ignore
meta=meta,
)
def get_error_class(
self,
error_message: str,
status_code: int,
headers: Union[dict, httpx.Headers],
) -> BaseLLMException:
if isinstance(headers, dict):
headers = httpx.Headers(headers)
return DashScopeError(
status_code=status_code,
message=error_message,
headers=headers,
)

106
litellm/llms/deepseek/chat/transformation.py
View file

@ -2,13 +2,15 @@
Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completions`
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
@ -62,6 +64,48 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
return optional_params
def _fill_reasoning_content(
self, messages: List[AllMessageValues]
) -> List[AllMessageValues]:
"""
DeepSeek thinking mode requires `reasoning_content` to be passed back on
every assistant message in multi-turn conversations. If it is missing,
the API returns:
"The reasoning_content in the thinking mode must be passed back to the API."
For each assistant message that is missing `reasoning_content`:
1. Promote it from `provider_specific_fields["reasoning_content"]` if present
(LiteLLM stores provider-specific response fields there).
2. Otherwise inject a single space — the minimum value the API accepts.
"""
result: List[AllMessageValues] = []
for msg in messages:
if msg.get("role") == "assistant" and not msg.get("reasoning_content"):
patched = dict(cast(dict, msg))
provider_fields = patched.get("provider_specific_fields") or {}
stored = provider_fields.get("reasoning_content")
if stored:
patched["reasoning_content"] = stored
cleaned = dict(provider_fields)
cleaned.pop("reasoning_content", None)
patched["provider_specific_fields"] = cleaned
else:
litellm.verbose_logger.warning(
"DeepSeek thinking mode: assistant message is missing "
"`reasoning_content` and none was saved in "
"`provider_specific_fields`. A single-space placeholder "
"is being injected to satisfy API validation, but the "
"model will receive a blank reasoning chain for this turn, "
"which may silently degrade multi-turn response quality. "
"Preserve `reasoning_content` from the original assistant "
"response when building multi-turn conversation history."
)
patched["reasoning_content"] = " "
result.append(cast(AllMessageValues, patched))
else:
result.append(msg)
return result
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
@ -91,6 +135,66 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
messages=messages, model=model, is_async=False
)
def _thinking_mode_active(self, model: str, optional_params: dict) -> bool:
"""
Returns True only when thinking mode is actually active for this request:
- model supports reasoning (capability check)
- user explicitly passed thinking={"type": "enabled"} (opt-in check)
"""
return (
supports_reasoning(model=model, custom_llm_provider="deepseek")
and (optional_params.get("thinking") or {}).get("type") == "enabled"
)
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Ensures `reasoning_content` is forwarded on assistant messages for
multi-turn thinking-mode conversations (issue #28045).
Only runs when thinking mode is actually active - guarded by both
supports_reasoning() (model capability) and optional_params["thinking"]
(user explicitly enabled it), preventing spurious injection on models
like deepseek-v3.2 that support thinking as opt-in but not always-on.
"""
if self._thinking_mode_active(model=model, optional_params=optional_params):
messages = self._fill_reasoning_content(messages)
return super().transform_request(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
async def async_transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Async equivalent of transform_request — applies the same reasoning_content
fix for multi-turn thinking-mode conversations.
"""
if self._thinking_mode_active(model=model, optional_params=optional_params):
messages = self._fill_reasoning_content(messages)
return await super().async_transform_request(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:

133
litellm/llms/deepseek/messages/transformation.py Normal file
View file

@ -0,0 +1,133 @@
"""
DeepSeek Anthropic-compatible messages transformation config.
"""
from typing import Any, Dict, List, Optional, Tuple
import litellm
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
"""
DeepSeek exposes an Anthropic-compatible Messages API at
https://api.deepseek.com/anthropic.
It accepts the native Anthropic Messages conversation shape, including
thinking blocks in assistant history, but rejects Anthropic's explicit
custom-tool discriminator (`{"type": "custom"}`).
"""
@property
def custom_llm_provider(self) -> Optional[str]:
return "deepseek"
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
return api_key or get_secret_str("DEEPSEEK_API_KEY") or litellm.api_key
@staticmethod
def get_api_base(api_base: Optional[str] = None) -> str:
return (
api_base
or get_secret_str("DEEPSEEK_ANTHROPIC_API_BASE")
or get_secret_str("DEEPSEEK_API_BASE")
or "https://api.deepseek.com/anthropic"
)
def validate_anthropic_messages_environment(
self,
headers: dict,
model: str,
messages: List[Any],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> Tuple[dict, Optional[str]]:
dynamic_api_key = self.get_api_key(api_key=api_key)
if (
"x-api-key" not in headers
and "authorization" not in headers
and dynamic_api_key is not None
):
headers["x-api-key"] = dynamic_api_key
if "anthropic-version" not in headers:
headers["anthropic-version"] = "2023-06-01"
if "content-type" not in headers:
headers["content-type"] = "application/json"
headers = self._update_headers_with_anthropic_beta(
headers=headers,
optional_params=optional_params,
custom_llm_provider=self.custom_llm_provider or "deepseek",
)
return headers, api_base
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
base_url = self.get_api_base(api_base=api_base).rstrip("/")
if base_url.endswith("/v1/messages") and "/anthropic/" in base_url:
return base_url
if base_url.endswith("/v1/messages"):
base_url = base_url[: -len("/v1/messages")]
if base_url.endswith("/v1"):
base_url = base_url[: -len("/v1")]
if base_url.endswith("/beta"):
base_url = base_url[: -len("/beta")]
if not base_url.endswith("/anthropic") and "/anthropic/" not in base_url:
base_url = f"{base_url}/anthropic"
return f"{base_url}/v1/messages"
@staticmethod
def _sanitize_tools_for_deepseek(tools: Any) -> Any:
if not isinstance(tools, list):
return tools
sanitized_tools = []
for tool in tools:
if isinstance(tool, dict) and tool.get("type") == "custom":
sanitized_tool = dict(tool)
sanitized_tool.pop("type", None)
sanitized_tools.append(sanitized_tool)
else:
sanitized_tools.append(tool)
return sanitized_tools
def transform_anthropic_messages_request(
self,
model: str,
messages: List[Dict],
anthropic_messages_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Dict:
anthropic_messages_request = super().transform_anthropic_messages_request(
model=model,
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
if "tools" in anthropic_messages_request:
anthropic_messages_request["tools"] = self._sanitize_tools_for_deepseek(
anthropic_messages_request["tools"]
)
return anthropic_messages_request

24
litellm/llms/gemini/chat/transformation.py
View file

@ -1,5 +1,7 @@
from typing import List, Optional, cast
import litellm
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_generic_image_chunk_to_openai_image_obj,
convert_to_anthropic_image_obj,
@ -101,7 +103,10 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
return supported_params
def _transform_messages(
self, messages: List[AllMessageValues], model: Optional[str] = None
self,
messages: List[AllMessageValues],
model: Optional[str] = None,
litellm_params: Optional[dict] = None,
) -> List[ContentType]:
"""
Google AI Studio Gemini does not support HTTP/HTTPS URLs for files.
@ -141,14 +146,23 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
img_element["image_url"] = converted_image_url # type: ignore
elif element.get("type") == "file":
file_element = cast(ChatCompletionFileObject, element)
file_id = file_element["file"].get("file_id")
_file_field = file_element.get("file")
if _file_field is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=model,
llm_provider="gemini",
)
file_id = _file_field.get("file_id")
if file_id and ("http://" in file_id or "https://" in file_id):
# Convert HTTP/HTTPS file URL to base64 data
try:
base64_data = convert_url_to_base64(file_id)
file_element["file"]["file_data"] = base64_data # type: ignore
file_element["file"].pop("file_id", None) # type: ignore
_file_field["file_data"] = base64_data # type: ignore
_file_field.pop("file_id", None) # type: ignore
except Exception:
# If conversion fails, leave as is and let the API handle it
pass
return _gemini_convert_messages_with_history(messages=messages, model=model)
return _gemini_convert_messages_with_history(
messages=messages, model=model, litellm_params=litellm_params
)

8
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -287,7 +287,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
content_item["image_url"] = new_image_url_obj
elif content_item.get("type") == "file":
content_item = cast(ChatCompletionFileObject, content_item)
file_obj = content_item["file"]
file_obj = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj = ChatCompletionFileObjectFile(
**{ # type: ignore
k: v

503
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -6,13 +6,16 @@ Why separate file? Make it easy to see how transformation works
import json
import os
from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union, cast
import re
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast
from urllib.parse import quote
import httpx
from pydantic import BaseModel
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_get_image_mime_type_from_url,
)
@ -57,6 +60,45 @@ from ..common_utils import (
get_supports_system_message,
)
# Typed as Any to avoid introducing a module-load-time cyclic import to
# vertex_llm_base. The instance is lazily constructed by _get_vertex_base()
# the first time GCS metadata needs to be fetched.
_GCS_METADATA_VERTEX_BASE: Optional[Any] = None
# Shared sync client for GCS JSON API metadata reads so proxy/SSL settings
# from litellm's HTTP stack apply (see Greptile review on PR #27278).
_GCS_METADATA_HTTP_HANDLER: Optional[HTTPHandler] = None
_GEMINI_MIME_TYPE_ALIASES: Dict[str, str] = {
"image/jpg": "image/jpeg",
}
def _apply_gemini_mime_type_aliases(mime_type: str) -> str:
"""Normalize known MIME aliases only; does not consult the file-type registry.
Also strips MIME parameters (e.g. ``; charset=utf-8``) so that values
sourced from GCS object metadata (``contentType``) validate correctly.
"""
normalized = mime_type.split(";", 1)[0].strip().lower()
return _GEMINI_MIME_TYPE_ALIASES.get(normalized, normalized)
def _get_vertex_base() -> Any:
"""Lazily return the shared VertexBase instance to avoid a module-load-time cyclic import."""
global _GCS_METADATA_VERTEX_BASE
if _GCS_METADATA_VERTEX_BASE is None:
from ..vertex_llm_base import VertexBase
_GCS_METADATA_VERTEX_BASE = VertexBase()
return _GCS_METADATA_VERTEX_BASE
def _get_gcs_metadata_http_handler() -> HTTPHandler:
global _GCS_METADATA_HTTP_HANDLER
if _GCS_METADATA_HTTP_HANDLER is None:
_GCS_METADATA_HTTP_HANDLER = HTTPHandler(timeout=5.0)
return _GCS_METADATA_HTTP_HANDLER
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -171,12 +213,299 @@ def _apply_gemini_metadata(
return cast(PartType, part_dict)
def _parse_gs_uri(gs_uri: str) -> Tuple[str, str]:
if not gs_uri.startswith("gs://"):
raise ValueError(f"Invalid gs URI: {gs_uri}")
uri_without_scheme = gs_uri[5:] # drop gs://
uri_parts = uri_without_scheme.split("/", 1)
if len(uri_parts) != 2 or not uri_parts[0] or not uri_parts[1]:
raise ValueError(f"Invalid gs URI: {gs_uri}")
return uri_parts[0], uri_parts[1]
def _is_valid_gcs_bucket_name(bucket: str) -> bool:
"""
Validate bucket name against core GCS naming constraints.
"""
bucket_length = len(bucket)
max_bucket_length = 222 if "." in bucket else 63
if bucket_length < 3 or bucket_length > max_bucket_length:
return False
if "." in bucket and any(
len(label) == 0 or len(label) > 63 for label in bucket.split(".")
):
return False
if not re.fullmatch(r"[a-z0-9][a-z0-9._-]*[a-z0-9]", bucket):
return False
if ".." in bucket:
return False
if re.fullmatch(r"\d+\.\d+\.\d+\.\d+", bucket):
return False
return True
def _gs_uri_requires_content_type_metadata(url: str) -> bool:
"""
True when _process_gemini_media would call _get_gcs_object_content_type
(extension-less gs:// and no explicit format passed into that helper).
"""
if "gs://" not in url:
return False
extension_with_dot = os.path.splitext(url)[-1]
extension = extension_with_dot[1:] if extension_with_dot else ""
return len(extension) == 0
def _image_url_payload_may_need_sync_gcs_metadata_fetch(
raw_image_url: Any,
) -> bool:
"""
True when this image_url value (content-part image_url or assistant ``images[]``
entry) can trigger a blocking GCS metadata read for MIME resolution.
"""
fmt: Optional[str] = None
url: Optional[str] = None
if isinstance(raw_image_url, dict):
url = raw_image_url.get("url") # type: ignore[assignment]
if not isinstance(url, str):
return False
fmt = (
raw_image_url.get("format")
or raw_image_url.get("mime_type")
or raw_image_url.get("content_type")
)
elif isinstance(raw_image_url, str):
url = raw_image_url
else:
return False
if "gs://" not in url or fmt:
return False
return _gs_uri_requires_content_type_metadata(url)
def _openai_messages_may_need_sync_gcs_metadata_fetch(
messages: List[AllMessageValues],
) -> bool:
"""
Heuristic: True if any message part can trigger a blocking GCS JSON
metadata read inside _transform_request_body (extension-less gs:// without
explicit MIME hints). Covers user/system ``content`` parts and assistant
``images`` (same paths as ``_gemini_convert_messages_with_history``). Used
to decide whether ``async_transform_request_body`` should offload the sync
transform via ``asyncify``.
"""
for raw in messages:
msg: Any = raw
if not isinstance(msg, dict) and hasattr(msg, "model_dump"):
msg = msg.model_dump(exclude_none=False)
if not isinstance(msg, dict):
continue
images_field = msg.get("images")
if isinstance(images_field, list):
for image_item in images_field:
if not isinstance(image_item, dict):
continue
if _image_url_payload_may_need_sync_gcs_metadata_fetch(
image_item.get("image_url")
):
return True
content = msg.get("content")
if not isinstance(content, list):
continue
for item in content:
if not isinstance(item, dict):
continue
itype = item.get("type")
if itype == "image_url":
if _image_url_payload_may_need_sync_gcs_metadata_fetch(
item.get("image_url")
):
return True
elif itype == "file":
file_obj = item.get("file")
if not isinstance(file_obj, dict):
continue
fmt = (
file_obj.get("format")
or file_obj.get("mime_type")
or file_obj.get("content_type")
)
passed = file_obj.get("file_id") or file_obj.get("file_data")
if (
isinstance(passed, str)
and "gs://" in passed
and not fmt
and _gs_uri_requires_content_type_metadata(passed)
):
return True
return False
def _get_gcs_object_content_type(
image_url: str,
vertex_project: Optional[str] = None,
vertex_credentials: Optional[Any] = None,
) -> Optional[str]:
"""
Resolve content type from GCS object metadata.
Only attaches a Bearer token when the caller explicitly supplies Vertex
credentials, to avoid using the server's default Google credentials on
the Gemini API-key (Google AI Studio) path and being used as an oracle
for private GCS object metadata. Without explicit credentials we only
issue an anonymous request, which only succeeds for publicly-readable
objects.
"""
try:
bucket, object_name = _parse_gs_uri(image_url)
except ValueError:
return None
if not _is_valid_gcs_bucket_name(bucket):
return None
headers: Dict[str, str] = {}
explicit_vertex_auth_provided = (
vertex_project is not None or vertex_credentials is not None
)
if explicit_vertex_auth_provided:
try:
access_token, _ = _get_vertex_base().get_access_token(
credentials=vertex_credentials,
project_id=vertex_project,
)
headers["Authorization"] = f"Bearer {access_token}"
except Exception as e:
raise litellm.BadRequestError(
message=(
"Unable to fetch GCS metadata with provided Vertex credentials/project. "
f"Original error: {str(e)}"
),
model=None,
llm_provider="vertex_ai",
)
# Build the URL via httpx.URL with a fixed scheme/host and URL-encode both
# bucket and object so CodeQL does not flag the interpolation as a
# potential SSRF that could resolve to an arbitrary host.
encoded_bucket = quote(bucket, safe="")
encoded_object = quote(object_name, safe="")
metadata_url = httpx.URL(
scheme="https",
host="storage.googleapis.com",
path=f"/storage/v1/b/{encoded_bucket}/o/{encoded_object}",
params={"fields": "contentType"},
)
try:
response = _get_gcs_metadata_http_handler().get(
url=str(metadata_url),
headers=headers or None,
)
except httpx.RequestError as e:
if explicit_vertex_auth_provided:
raise litellm.BadRequestError(
message=(
"Unable to reach GCS JSON API for object metadata with provided "
f"Vertex credentials. {type(e).__name__}: {e}"
),
model=None,
llm_provider="vertex_ai",
) from e
return None
if response.is_error:
if explicit_vertex_auth_provided:
preview = (response.text or "")[:1024]
raise litellm.BadRequestError(
message=(
"Unable to read GCS object metadata with provided Vertex credentials. "
f"HTTP {response.status_code}. Response body (truncated): {preview!r}"
),
model=None,
llm_provider="vertex_ai",
)
return None
try:
payload = response.json()
except ValueError as e:
if explicit_vertex_auth_provided:
raise litellm.BadRequestError(
message=(
"GCS metadata response was not valid JSON when using provided "
f"Vertex credentials (HTTP {response.status_code}). Error: {e}"
),
model=None,
llm_provider="vertex_ai",
) from e
return None
if not isinstance(payload, dict):
if explicit_vertex_auth_provided:
raise litellm.BadRequestError(
message=(
"GCS metadata response was not a JSON object when using provided "
f"Vertex credentials (HTTP {response.status_code})."
),
model=None,
llm_provider="vertex_ai",
)
return None
content_type = payload.get("contentType")
if isinstance(content_type, str) and len(content_type) > 0:
return content_type
if explicit_vertex_auth_provided:
preview = (response.text or "")[:1024]
raise litellm.BadRequestError(
message=(
"GCS metadata JSON did not include a non-empty contentType field when "
f"using provided Vertex credentials (HTTP {response.status_code}). "
f"Body (truncated): {preview!r}"
),
model=None,
llm_provider="vertex_ai",
)
return None
def _normalize_and_validate_gemini_mime_type(
mime_type: str, model: Optional[str]
) -> str:
# Import lazily to avoid a module-level cyclic-import alert with
# litellm.types.files.
from litellm.types.files import get_file_extension_from_mime_type
normalized_mime_type = _apply_gemini_mime_type_aliases(mime_type)
try:
file_extension = get_file_extension_from_mime_type(normalized_mime_type)
file_type = get_file_type_from_extension(file_extension)
except ValueError:
raise litellm.BadRequestError(
message=f"File type not supported by gemini - {normalized_mime_type}",
model=model,
llm_provider="vertex_ai",
)
if not is_gemini_1_5_accepted_file_type(file_type):
raise litellm.BadRequestError(
message=f"File type not supported by gemini - {file_type}",
model=model,
llm_provider="vertex_ai",
)
return get_file_mime_type_for_file_type(file_type)
def _process_gemini_media(
image_url: str,
format: Optional[str] = None,
media_resolution_enum: Optional[Dict[str, str]] = None,
model: Optional[str] = None,
video_metadata: Optional[Dict[str, Any]] = None,
vertex_project: Optional[str] = None,
vertex_credentials: Optional[Any] = None,
) -> PartType:
"""
Given a media URL (image, audio, or video), return the appropriate PartType for Gemini
@ -193,20 +522,63 @@ def _process_gemini_media(
try:
# GCS URIs
if "gs://" in image_url:
# Figure out file type
extension_with_dot = os.path.splitext(image_url)[-1] # Ex: ".png"
extension = extension_with_dot[1:] # Ex: "png"
explicit_gcs_format = False
if not format:
file_type = get_file_type_from_extension(extension)
mime_type: Optional[str] = None
# For extension-less gs:// URIs, we cannot infer from path.
# If callers pass `format`/`mime_type`, this branch is skipped.
if extension:
file_type = get_file_type_from_extension(extension)
# Validate the file type is supported by Gemini
if not is_gemini_1_5_accepted_file_type(file_type):
raise Exception(f"File type not supported by gemini - {file_type}")
# Validate the file type is supported by Gemini
if not is_gemini_1_5_accepted_file_type(file_type):
raise litellm.BadRequestError(
message=f"File type not supported by gemini - {file_type}",
model=model,
llm_provider="vertex_ai",
)
mime_type = get_file_mime_type_for_file_type(file_type)
mime_type = get_file_mime_type_for_file_type(file_type)
else:
mime_type = _get_gcs_object_content_type(
image_url=image_url,
vertex_project=vertex_project,
vertex_credentials=vertex_credentials,
)
if mime_type is None:
raise litellm.BadRequestError(
message=(
f"Unable to determine mime type for gs URI: {image_url}. "
"This gs:// URI has no file extension and GCS metadata "
"lookup failed. Set it explicitly using image_url.format "
"(or image_url.mime_type/content_type) or "
"message.content[].file.format."
),
model=model,
llm_provider="vertex_ai",
)
else:
mime_type = format
explicit_gcs_format = True
if mime_type is None:
raise litellm.BadRequestError(
message=f"File type not supported by gemini - {image_url}",
model=model,
llm_provider="vertex_ai",
)
if explicit_gcs_format:
# Callers who pass format/mime_type explicitly for gs:// URIs
# rely on pass-through to Gemini (pre-PR behavior). Only apply
# known MIME aliases; skip litellm's file-type registry.
mime_type = _apply_gemini_mime_type_aliases(mime_type)
else:
mime_type = _normalize_and_validate_gemini_mime_type(
mime_type=mime_type,
model=model,
)
file_data = FileDataType(mime_type=mime_type, file_uri=image_url)
part: PartType = {"file_data": file_data}
return _apply_gemini_metadata(
@ -258,8 +630,6 @@ def _snake_to_camel(snake_str: str) -> str:
def _camel_to_snake(camel_str: str) -> str:
"""Convert camelCase to snake_case"""
import re
return re.sub(r"(?<!^)(?=[A-Z])", "_", camel_str).lower()
@ -311,6 +681,7 @@ def check_if_part_exists_in_parts(
def _gemini_convert_messages_with_history( # noqa: PLR0915
messages: List[AllMessageValues],
model: Optional[str] = None,
litellm_params: Optional[dict] = None,
) -> List[ContentType]:
"""
Converts given messages from OpenAI format to Gemini format
@ -326,6 +697,16 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
msg_i = 0
tool_call_responses = []
vertex_project = None
vertex_credentials = None
if litellm_params:
vertex_project = litellm_params.get("vertex_project") or litellm_params.get(
"vertex_ai_project"
)
vertex_credentials = litellm_params.get(
"vertex_credentials"
) or litellm_params.get("vertex_ai_credentials")
try:
while msg_i < len(messages):
user_content: List[PartType] = []
@ -351,20 +732,42 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
img_element = element
format: Optional[str] = None
media_resolution_enum: Optional[Dict[str, str]] = None
if isinstance(img_element["image_url"], dict):
image_url = img_element["image_url"]["url"]
format = img_element["image_url"].get("format")
detail = img_element["image_url"].get("detail")
raw_image_url = img_element.get("image_url")
if raw_image_url is None:
raise litellm.BadRequestError(
message="Invalid message content: element type is 'image_url' but 'image_url' field is missing ",
model=model,
llm_provider="vertex_ai",
)
if isinstance(raw_image_url, dict):
image_url = raw_image_url.get("url")
if image_url is None:
raise litellm.BadRequestError(
message="Invalid message content: element type is 'image_url' but 'url' field is missing inside 'image_url' ",
model=model,
llm_provider="vertex_ai",
)
# TypedDict does not declare mime_type/content_type;
# read via Dict[str, Any] for caller-provided MIME fields.
image_url_dict = cast(Dict[str, Any], raw_image_url)
format = (
image_url_dict.get("format")
or image_url_dict.get("mime_type")
or image_url_dict.get("content_type")
)
detail = image_url_dict.get("detail")
media_resolution_enum = (
_convert_detail_to_media_resolution_enum(detail)
)
else:
image_url = img_element["image_url"]
image_url = raw_image_url
_part = _process_gemini_media(
image_url=image_url,
format=format,
media_resolution_enum=media_resolution_enum,
model=model,
vertex_project=vertex_project,
vertex_credentials=vertex_credentials,
)
_parts.append(_part)
elif element["type"] == "input_audio":
@ -390,15 +793,31 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
image_url=openai_image_str,
format=audio_format_modified,
model=model,
vertex_project=vertex_project,
vertex_credentials=vertex_credentials,
)
_parts.append(_part)
elif element["type"] == "file":
file_element = cast(ChatCompletionFileObject, element)
file_id = file_element["file"].get("file_id")
format = file_element["file"].get("format")
file_data = file_element["file"].get("file_data")
detail = file_element["file"].get("detail")
video_metadata = file_element["file"].get("video_metadata")
_file_field = file_element.get("file")
if _file_field is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=model,
llm_provider="vertex_ai",
)
# TypedDict does not declare mime_type/content_type;
# read via Dict[str, Any] for caller-provided MIME fields.
file_dict = cast(Dict[str, Any], _file_field)
file_id = file_dict.get("file_id")
format = (
file_dict.get("format")
or file_dict.get("mime_type")
or file_dict.get("content_type")
)
file_data = file_dict.get("file_data")
detail = file_dict.get("detail")
video_metadata = file_dict.get("video_metadata")
passed_file = file_id or file_data
if passed_file is None:
raise Exception(
@ -417,13 +836,23 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
model=model,
media_resolution_enum=media_resolution_enum,
video_metadata=video_metadata,
vertex_project=vertex_project,
vertex_credentials=vertex_credentials,
)
_parts.append(_part)
except Exception:
raise Exception(
"Unable to determine mime type for file_id: {}, set this explicitly using message[{}].content[{}].file.format".format(
file_id, msg_i, element_idx
)
except litellm.BadRequestError:
raise
except Exception as e:
raise litellm.BadRequestError(
message=(
f"Unable to determine mime type for file: "
f"{file_id or 'provided data'}, set this explicitly "
f"using message[{msg_i}].content[{element_idx}].file.format "
f"(or file.mime_type/content_type). "
f"Original error: {str(e)}"
),
model=model,
llm_provider="vertex_ai",
)
user_content.extend(_parts)
elif _message_content is not None and isinstance(_message_content, str):
@ -528,7 +957,11 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
image_url_obj = image_item.get("image_url")
if isinstance(image_url_obj, dict):
assistant_image_url = image_url_obj.get("url")
format = image_url_obj.get("format")
format = (
image_url_obj.get("format")
or image_url_obj.get("mime_type")
or image_url_obj.get("content_type")
)
detail = image_url_obj.get("detail")
media_resolution_enum = (
_convert_detail_to_media_resolution_enum(detail)
@ -539,6 +972,8 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
format=format,
media_resolution_enum=media_resolution_enum,
model=model,
vertex_project=vertex_project,
vertex_credentials=vertex_credentials,
)
assistant_content.append(_part)
@ -713,11 +1148,11 @@ def _transform_request_body( # noqa: PLR0915
try:
if custom_llm_provider == "gemini":
content = litellm.GoogleAIStudioGeminiConfig()._transform_messages(
messages=messages, model=model
messages=messages, model=model, litellm_params=litellm_params
)
else:
content = litellm.VertexGeminiConfig()._transform_messages(
messages=messages, model=model
messages=messages, model=model, litellm_params=litellm_params
)
tools: Optional[Tools] = optional_params.pop("tools", None)
tool_choice: Optional[ToolConfig] = optional_params.pop("tool_choice", None)
@ -893,6 +1328,20 @@ async def async_transform_request_body(
vertex_auth_header=vertex_auth_header,
)
if _openai_messages_may_need_sync_gcs_metadata_fetch(messages):
# _transform_request_body may issue a sync httpx.get (up to 5s timeout)
# via _get_gcs_object_content_type to fetch GCS object metadata. Run the
# whole sync transformation on a worker thread so it does not block the
# async event loop.
return await asyncify(_transform_request_body)(
messages=messages,
model=model,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
cached_content=cached_content,
optional_params=optional_params,
)
return _transform_request_body(
messages=messages,
model=model,

39
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -2533,9 +2533,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return model_response
def _transform_messages(
self, messages: List[AllMessageValues], model: Optional[str] = None
self,
messages: List[AllMessageValues],
model: Optional[str] = None,
litellm_params: Optional[dict] = None,
) -> List[ContentType]:
return _gemini_convert_messages_with_history(messages=messages, model=model)
return _gemini_convert_messages_with_history(
messages=messages, model=model, litellm_params=litellm_params
)
def get_error_class(
self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers]
@ -3139,6 +3144,31 @@ class ModelResponseIterator:
self.cumulative_tool_call_index: int = 0
self.has_seen_tool_calls: bool = False
@staticmethod
def _check_streaming_error(chunk: dict) -> None:
"""Detect embedded errors (e.g. 429 RESOURCE_EXHAUSTED) in streaming chunks and raise VertexAIError."""
if "error" not in chunk:
return
error_data = chunk["error"]
if not isinstance(error_data, dict):
raise VertexAIError(
status_code=500,
message=f"Unexpected error format in mid-stream chunk: {error_data}",
)
raw_code = error_data.get("code", 500)
if raw_code is None:
raw_code = 500
try:
error_code = int(raw_code)
except (TypeError, ValueError):
error_code = 500
error_message = error_data.get("message", "Unknown error")
error_status = error_data.get("status", "UNKNOWN")
raise VertexAIError(
status_code=error_code,
message=f"{error_status} - {error_message}",
)
def _apply_stream_candidates(
self,
_candidates: List[Candidates],
@ -3256,6 +3286,11 @@ class ModelResponseIterator:
def chunk_parser(self, chunk: dict) -> Optional["ModelResponseStream"]:
try:
verbose_logger.debug(f"RAW GEMINI CHUNK: {chunk}")
# Detect mid-stream error chunks (e.g. 429 RESOURCE_EXHAUSTED).
# Vertex AI can return errors as HTTP 200 but with an "error" field in the SSE body.
self._check_streaming_error(chunk)
from litellm.types.utils import ModelResponseStream
processed_chunk = GenerateContentResponseBody(**chunk) # type: ignore

25
litellm/llms/vertex_ai/vertex_ai_partner_models/main.py
View file

@ -292,22 +292,15 @@ class VertexAIPartnerModels(VertexBase):
Returns:
Dict containing token count information
"""
try:
import vertexai
except Exception as e:
raise VertexAIError(
status_code=400,
message=f"""vertexai import failed please run `pip install -U "google-cloud-aiplatform>=1.38"`. Got error: {e}""",
)
if not (
hasattr(vertexai, "preview") or hasattr(vertexai.preview, "language_models")
):
raise VertexAIError(
status_code=400,
message="""Upgrade vertex ai. Run `pip install "google-cloud-aiplatform>=1.38"`""",
)
# Note: we intentionally do not import `vertexai` (the Gemini SDK shipped
# by `google-cloud-aiplatform`) on this path. Partner models such as
# Claude on Vertex use the Anthropic Messages API protocol directly via
# `:rawPredict`, and `VertexAIPartnerModelsTokenCounter` reaches that
# endpoint with an authenticated httpx client — it never touches the
# Gemini SDK. Requiring `google-cloud-aiplatform>=1.38` here turned a
# SDK-free Anthropic-protocol call into a hard dependency on the Gemini
# SDK (see #28084), breaking `/v1/messages/count_tokens` for Claude-on-
# Vertex on any LiteLLM install without that extra. Stay SDK-free.
try:
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import (
VertexAIPartnerModelsTokenCounter,

27
litellm/main.py
View file

@ -5720,6 +5720,33 @@ def embedding( # noqa: PLR0915
aembedding=aembedding,
headers=headers,
)
elif custom_llm_provider == "dashscope":
dashscope_key = (
api_key or litellm.api_key or get_secret_str("DASHSCOPE_API_KEY")
)
if dashscope_key is None:
raise ValueError(
"Missing API key for DashScope. Set DASHSCOPE_API_KEY environment variable or pass api_key parameter."
)
if extra_headers is not None and isinstance(extra_headers, dict):
headers = extra_headers
else:
headers = {}
response = base_llm_http_handler.embedding(
model=model,
input=input,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
logging_obj=logging,
api_base=api_base,
optional_params=optional_params,
litellm_params={},
model_response=EmbeddingResponse(),
api_key=dashscope_key,
client=client,
aembedding=aembedding,
headers=headers,
)
elif custom_llm_provider == "ovhcloud":
api_key = api_key or litellm.api_key or get_secret_str("OVHCLOUD_API_KEY")
api_base = (

374
litellm/model_prices_and_context_window_backup.json
View file

@ -2112,6 +2112,380 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"azure_ai/gpt-5.4": {
"cache_read_input_token_cost": 2.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_priority": 3e-05,
"output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-2026-03-05": {
"cache_read_input_token_cost": 2.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_priority": 3e-05,
"output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-pro": {
"cache_read_input_token_cost": 3e-06,
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
"cache_read_input_token_cost_priority": 6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_272k_tokens": 6e-05,
"input_cost_per_token_priority": 6e-05,
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00018,
"output_cost_per_token_above_272k_tokens": 0.00027,
"output_cost_per_token_priority": 0.00036,
"output_cost_per_token_above_272k_tokens_priority": 0.00054,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-pro",
"supported_endpoints": [
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-pro-2026-03-05": {
"cache_read_input_token_cost": 3e-06,
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
"cache_read_input_token_cost_priority": 6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_272k_tokens": 6e-05,
"input_cost_per_token_priority": 6e-05,
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00018,
"output_cost_per_token_above_272k_tokens": 0.00027,
"output_cost_per_token_priority": 0.00036,
"output_cost_per_token_above_272k_tokens_priority": 0.00054,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-pro",
"supported_endpoints": [
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-mini": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_above_272k_tokens": 1.5e-07,
"cache_read_input_token_cost_priority": 1.5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 3e-07,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_above_272k_tokens": 1.5e-06,
"input_cost_per_token_priority": 1.5e-06,
"input_cost_per_token_above_272k_tokens_priority": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"output_cost_per_token_above_272k_tokens": 6.75e-06,
"output_cost_per_token_priority": 9e-06,
"output_cost_per_token_above_272k_tokens_priority": 1.35e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-mini",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.4-mini-2026-03-17": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_above_272k_tokens": 1.5e-07,
"cache_read_input_token_cost_priority": 1.5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 3e-07,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_above_272k_tokens": 1.5e-06,
"input_cost_per_token_priority": 1.5e-06,
"input_cost_per_token_above_272k_tokens_priority": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"output_cost_per_token_above_272k_tokens": 6.75e-06,
"output_cost_per_token_priority": 9e-06,
"output_cost_per_token_above_272k_tokens_priority": 1.35e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-mini",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.4-nano": {
"cache_read_input_token_cost": 2e-08,
"cache_read_input_token_cost_above_272k_tokens": 4e-08,
"cache_read_input_token_cost_priority": 4e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-08,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
"input_cost_per_token_priority": 4e-07,
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"output_cost_per_token_above_272k_tokens": 1.875e-06,
"output_cost_per_token_priority": 2.5e-06,
"output_cost_per_token_above_272k_tokens_priority": 3.75e-06,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-nano",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.4-nano-2026-03-17": {
"cache_read_input_token_cost": 2e-08,
"cache_read_input_token_cost_above_272k_tokens": 4e-08,
"cache_read_input_token_cost_priority": 4e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-08,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
"input_cost_per_token_priority": 4e-07,
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"output_cost_per_token_above_272k_tokens": 1.875e-06,
"output_cost_per_token_priority": 2.5e-06,
"output_cost_per_token_above_272k_tokens_priority": 3.75e-06,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-nano",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/model_router": {
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 0,

2
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -332,8 +332,6 @@ class MCPRequestHandler:
# non-bool must not silently enable the bypass.
if getattr(server, "delegate_auth_to_upstream", False) is not True:
return False
if not getattr(server, "available_on_public_internet", True):
return False
# Never delegate for M2M (client_credentials) servers: LiteLLM
# fetches the upstream token automatically using stored credentials,
# so allowing anonymous bypass would let any external caller invoke

56
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -145,6 +145,30 @@ def _warn_on_server_name_fields(
_warn("server_name", server_name)
def _warn_internal_delegate_pkce_if_applicable(
server: MCPServer, *, source: str
) -> None:
"""Surface internal + upstream PKCE delegate in logs for operators."""
if server.auth_type != MCPAuth.oauth2:
return
if getattr(server, "delegate_auth_to_upstream", False) is not True:
return
if getattr(server, "available_on_public_internet", True):
return
if server.has_client_credentials:
return
label = get_server_prefix(server)
verbose_logger.warning(
"MCP server %r (id=%s, source=%s): internal-only (available_on_public_internet=false) "
"with delegate_auth_to_upstream=true. Anonymous callers can reach the upstream OAuth2 "
"/authorize flow and complete PKCE without a LiteLLM API key session; ensure the "
"upstream IdP and network enforce your access policy.",
label,
server.server_id,
source,
)
def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]:
"""
Deserialize optional JSON mappings stored in the database.
@ -297,32 +321,6 @@ class MCPServerManager:
)()
name_for_prefix = get_server_prefix(temp_server)
# Use alias for name if present, else server_name
alias = server_config.get("alias", None)
# Apply mcp_aliases mapping if provided
if mcp_aliases and alias is None:
# Check if this server_name has an alias in mcp_aliases
for alias_name, target_server_name in mcp_aliases.items():
if (
target_server_name == server_name
and alias_name not in used_aliases
):
alias = alias_name
used_aliases.add(alias_name)
verbose_logger.debug(
f"Mapped alias '{alias_name}' to server '{server_name}'"
)
break
# Create a temporary server object to use with get_server_prefix utility
temp_server = type(
"TempServer",
(),
{"alias": alias, "server_name": server_name, "server_id": None},
)()
name_for_prefix = get_server_prefix(temp_server)
server_url = server_config.get("url", None) or ""
# Generate stable server ID based on parameters
server_id = self._generate_stable_server_id(
@ -425,6 +423,7 @@ class MCPServerManager:
),
)
self._assign_unique_short_prefix(new_server)
_warn_internal_delegate_pkce_if_applicable(new_server, source="config")
self.config_mcp_servers[server_id] = new_server
# Check if this is an OpenAPI-based server
@ -834,6 +833,7 @@ class MCPServerManager:
)
or "urn:ietf:params:oauth:token-type:access_token",
)
_warn_internal_delegate_pkce_if_applicable(new_server, source="database")
return new_server
async def _maybe_register_openapi_tools(
@ -995,9 +995,6 @@ class MCPServerManager:
# unauthenticated caller would get LiteLLM to proxy tool
# calls using its stored client_credentials.
and not server.has_client_credentials
# Internal-only servers must not be reachable from public
# internet callers who happen to carry an upstream token.
and getattr(server, "available_on_public_internet", True)
]
combined_servers.update(delegate_server_ids)
@ -3566,6 +3563,7 @@ class MCPServerManager:
registration_url=server.registration_url,
allow_all_keys=server.allow_all_keys,
available_on_public_internet=server.available_on_public_internet,
delegate_auth_to_upstream=server.delegate_auth_to_upstream,
is_byok=server.is_byok,
byok_description=server.byok_description,
byok_api_key_help_url=server.byok_api_key_help_url,

62
litellm/proxy/_experimental/mcp_server/oauth_utils.py
View file

@ -30,23 +30,43 @@ _DEFAULT_PORTS = {"http": 80, "https": 443}
_TRUSTED_REDIRECT_ORIGINS_ENV = "MCP_TRUSTED_REDIRECT_ORIGINS"
_warned_invalid_proxy_base_url: Optional[str] = None
def _resolve_proxy_base_url_env() -> Optional[str]:
global _warned_invalid_proxy_base_url
configured = os.environ.get("PROXY_BASE_URL", "").strip()
if not configured:
return None
parsed = urlparse(configured)
if parsed.scheme in ("http", "https") and parsed.netloc:
normalized = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", ""))
return normalized.rstrip("/")
if _warned_invalid_proxy_base_url != configured:
verbose_logger.warning(
"PROXY_BASE_URL=%r is not a valid http(s) URL (missing scheme "
"or host) and will be ignored for MCP OAuth origin resolution. "
"Set it to a full URL like https://litellm.example.com.",
configured,
)
_warned_invalid_proxy_base_url = configured
return None
def get_request_base_url(request: Request) -> str:
"""
Get the base URL for the request, considering X-Forwarded-* headers.
X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured
when the request comes from a configured trusted proxy
(``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``).
Otherwise the request's literal ``base_url`` is returned, so an
untrusted caller cannot poison OAuth-discovery / redirect_uri values
by injecting headers.
Args:
request: FastAPI Request object
Returns:
The reconstructed base URL (e.g., "https://proxy.example.com")
Resolution order: ``PROXY_BASE_URL`` env var, then X-Forwarded-* when
the caller is a trusted proxy (``use_x_forwarded_for`` enabled AND
caller in ``mcp_trusted_proxy_ranges``), otherwise the request's
literal ``base_url``. Untrusted callers cannot poison OAuth-discovery
/ redirect_uri values by injecting headers.
"""
configured = _resolve_proxy_base_url_env()
if configured:
return configured
base_url = str(request.base_url).rstrip("/")
parsed = urlparse(base_url)
@ -284,4 +304,22 @@ def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
if _matches_trusted_origin_entry(redirect_netloc, entry):
return
verbose_logger.warning(
"MCP OAuth: rejecting redirect_uri %r as invalid_request. "
"Computed proxy base=%r (PROXY_BASE_URL=%r). "
"Inbound headers: X-Forwarded-Proto=%r X-Forwarded-Host=%r "
"X-Forwarded-Port=%r Host=%r. "
"Trusted-redirect-origins env=%r. "
"If this should be accepted, either align ingress X-Forwarded-* "
"with the browser URL, set PROXY_BASE_URL to your public origin, "
"or add the redirect_uri host to MCP_TRUSTED_REDIRECT_ORIGINS.",
redirect_uri,
proxy_base,
os.environ.get("PROXY_BASE_URL"),
request.headers.get("X-Forwarded-Proto"),
request.headers.get("X-Forwarded-Host"),
request.headers.get("X-Forwarded-Port"),
request.headers.get("Host"),
os.environ.get(_TRUSTED_REDIRECT_ORIGINS_ENV),
)
raise HTTPException(status_code=400, detail="invalid_request")

2
litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

6
litellm/proxy/_experimental/out/__next.__PAGE__.txt
View file

@ -1,10 +1,10 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/6188170a32c9a3c3.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/0279e5299e9f6e98.js","/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ca7a3fdb635fb7dc.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/934dbc43f8c1abde.js","/litellm-asset-prefix/_next/static/chunks/d6ab357d1bbb53f0.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4a0ccb5ed3d0c33f.js"],"default"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/6188170a32c9a3c3.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/e1a670efcb966aaa.js","/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ca7a3fdb635fb7dc.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/934dbc43f8c1abde.js","/litellm-asset-prefix/_next/static/chunks/d6ab357d1bbb53f0.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4a0ccb5ed3d0c33f.js"],"default"]
1a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
1b:"$Sreact.suspense"
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
@ -16,7 +16,7 @@ b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/61
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","async":true}]
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","async":true}]
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/0279e5299e9f6e98.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/e1a670efcb966aaa.js","async":true}]
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","async":true}]
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}]
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}]

6
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -3,4 +3,4 @@
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -5,4 +5,4 @@
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/__next._tree.txt
View file

@ -2,4 +2,4 @@
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}

0
litellm/proxy/_experimental/out/_next/static/bCuM4c6QggtFAyOTSKpXz/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/LpD6ruZoEpvYpT5IvMEoa/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/bCuM4c6QggtFAyOTSKpXz/_clientMiddlewareManifest.json → litellm/proxy/_experimental/out/_next/static/LpD6ruZoEpvYpT5IvMEoa/_clientMiddlewareManifest.json
View file

0
litellm/proxy/_experimental/out/_next/static/bCuM4c6QggtFAyOTSKpXz/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/LpD6ruZoEpvYpT5IvMEoa/_ssgManifest.js
View file

16
litellm/proxy/_experimental/out/_next/static/chunks/0279e5299e9f6e98.js → litellm/proxy/_experimental/out/_next/static/chunks/e1a670efcb966aaa.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_not-found.html
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_not-found.txt
View file

@ -10,7 +10,7 @@ b:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/li
d:I[168027,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"P":null,"b":"bCuM4c6QggtFAyOTSKpXz","c":["","_not-found"],"q":"","i":false,"f":[[["",{"children":["/_not-found",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],null,["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L9",null,{"children":"$La"}],["$","div",null,{"hidden":true,"children":["$","$Lb",null,{"children":["$","$7",null,{"name":"Next.Metadata","children":"$Lc"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$d","$undefined"],"S":true}
0:{"P":null,"b":"LpD6ruZoEpvYpT5IvMEoa","c":["","_not-found"],"q":"","i":false,"f":[[["",{"children":["/_not-found",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],null,["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L9",null,{"children":"$La"}],["$","div",null,{"hidden":true,"children":["$","$Lb",null,{"children":["$","$7",null,{"name":"Next.Metadata","children":"$Lc"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$d","$undefined"],"S":true}
a:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]
e:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
8:null

2
litellm/proxy/_experimental/out/_not-found/__next._full.txt
View file

@ -10,7 +10,7 @@ b:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/li
d:I[168027,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"P":null,"b":"bCuM4c6QggtFAyOTSKpXz","c":["","_not-found"],"q":"","i":false,"f":[[["",{"children":["/_not-found",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],null,["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L9",null,{"children":"$La"}],["$","div",null,{"hidden":true,"children":["$","$Lb",null,{"children":["$","$7",null,{"name":"Next.Metadata","children":"$Lc"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$d","$undefined"],"S":true}
0:{"P":null,"b":"LpD6ruZoEpvYpT5IvMEoa","c":["","_not-found"],"q":"","i":false,"f":[[["",{"children":["/_not-found",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],null,["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L9",null,{"children":"$La"}],["$","div",null,{"hidden":true,"children":["$","$Lb",null,{"children":["$","$7",null,{"name":"Next.Metadata","children":"$Lc"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$d","$undefined"],"S":true}
a:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]
e:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
8:null

2
litellm/proxy/_experimental/out/_not-found/__next._head.txt
View file

@ -3,4 +3,4 @@
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","h",{"children":[["$","meta",null,{"name":"robots","content":"noindex"}],["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/_not-found/__next._index.txt
View file

@ -5,4 +5,4 @@
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/_not-found/__next._not-found.__PAGE__.txt
View file

@ -1,5 +1,5 @@
1:"$Sreact.fragment"
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
3:"$Sreact.suspense"
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],null,["$","$L2",null,{"children":["$","$3",null,{"name":"Next.MetadataOutlet","children":"$@4"}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],null,["$","$L2",null,{"children":["$","$3",null,{"name":"Next.MetadataOutlet","children":"$@4"}]}]]}],"loading":null,"isPartial":false}
4:null

2
litellm/proxy/_experimental/out/_not-found/__next._not-found.txt
View file

@ -1,4 +1,4 @@
1:"$Sreact.fragment"
2:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
3:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","template":["$","$L3",null,{}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","template":["$","$L3",null,{}]}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/_not-found/__next._tree.txt
View file

@ -1,3 +1,3 @@
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"/_not-found","paramType":null,"paramKey":"/_not-found","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"/_not-found","paramType":null,"paramKey":"/_not-found","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":false}},"isRootLayout":true},"staleTime":300}

2
litellm/proxy/_experimental/out/api-reference.html
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/api-reference.txt
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.api-reference.__PAGE__.txt
View file

@ -3,7 +3,7 @@
3:I[191905,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","/litellm-asset-prefix/_next/static/chunks/0377ae18aae60c57.js","/litellm-asset-prefix/_next/static/chunks/d7798a4e148be3fe.js","/litellm-asset-prefix/_next/static/chunks/1461020743acb21c.js","/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","/litellm-asset-prefix/_next/static/chunks/eea976cf4a05fc92.js","/litellm-asset-prefix/_next/static/chunks/120b47ad353e1fc9.js","/litellm-asset-prefix/_next/static/chunks/738c339383c3b4b6.js","/litellm-asset-prefix/_next/static/chunks/3356ae3643d24081.js","/litellm-asset-prefix/_next/static/chunks/2a5f4a7388e54210.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/6c4c97f1ea6e7d77.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
7:"$Sreact.suspense"
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/2a5f4a7388e54210.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/6c4c97f1ea6e7d77.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/2a5f4a7388e54210.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/6c4c97f1ea6e7d77.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false}
4:{}
5:{}
8:null

2
litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.api-reference.txt
View file

@ -1,4 +1,4 @@
1:"$Sreact.fragment"
2:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
3:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","template":["$","$L3",null,{}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","template":["$","$L3",null,{}]}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.txt
View file

@ -3,5 +3,5 @@
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","/litellm-asset-prefix/_next/static/chunks/0377ae18aae60c57.js","/litellm-asset-prefix/_next/static/chunks/d7798a4e148be3fe.js","/litellm-asset-prefix/_next/static/chunks/1461020743acb21c.js","/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","/litellm-asset-prefix/_next/static/chunks/eea976cf4a05fc92.js","/litellm-asset-prefix/_next/static/chunks/120b47ad353e1fc9.js","/litellm-asset-prefix/_next/static/chunks/738c339383c3b4b6.js","/litellm-asset-prefix/_next/static/chunks/3356ae3643d24081.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
0:{"buildId":"bCuM4c6QggtFAyOTSKpXz","rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0377ae18aae60c57.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/d7798a4e148be3fe.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/1461020743acb21c.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/eea976cf4a05fc92.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/120b47ad353e1fc9.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/738c339383c3b4b6.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3356ae3643d24081.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0377ae18aae60c57.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/d7798a4e148be3fe.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/1461020743acb21c.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/eea976cf4a05fc92.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/120b47ad353e1fc9.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/738c339383c3b4b6.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3356ae3643d24081.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"loading":null,"isPartial":false}
6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

2
litellm/proxy/_experimental/out/api-reference/__next._full.txt
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more