scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-05 02:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 49

chore: merge latest litellm_internal_staging

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-07-19 01:36:41 +00:00
commit 1573b18844
1171 changed files with 32370 additions and 11835 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -15,6 +15,9 @@ litellm/rust_bridge/_native*.so
litellm/rust_bridge/_native*.pyd
litellm-rust/target/
# Python package build output
dist/
bun.lockb
**/.DS_Store
.aider*

28
Dockerfile
View file

@ -86,7 +86,9 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra semantic-router \
--python python3
RUN prisma generate --schema=./schema.prisma
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@ -100,7 +102,11 @@ USER root
RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
WORKDIR /app
ENV PATH="/app/.venv/bin:${PATH}"
ENV PATH="/app/.venv/bin:${PATH}" \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
PRISMA_OFFLINE_MODE=true
# Copy only what runtime needs. The application is installed inside the venv;
# the rest of the builder's /app is source and build metadata that must not
@ -115,16 +121,18 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
# enterprise.enterprise_hooks from it)
COPY --from=builder /app/enterprise /app/enterprise
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
# Prisma binaries live in $HOME/.cache (default prisma-python location),
# which is /root/.cache here. Copy only the Prisma subdirs — copying the
# whole /root/.cache drags in the uv build cache (~660 MB, includes a
# setuptools wheel that surfaces as a CVE finding even though it's not
# on the runtime sys.path).
COPY --from=builder /root/.cache/prisma /root/.cache/prisma
COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
# Prisma CLI + engines are baked under /opt/prisma, a fixed path every
# runtime uid can read and that no cache volume mount shadows. The paths are
# pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and recorded into the
# generated client at build time, so `prisma migrate deploy` on a fresh
# database needs no npm and no network access (#33650, #24554).
COPY --from=builder /opt/prisma /opt/prisma
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
find /app/.venv -type d -path "*/tornado/test" -delete && \
chmod -R a+rX /opt/prisma && \
test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
EXPOSE 4000/tcp

BIN
dist/litellm-1.79.1.tar.gz vendored
View file

Binary file not shown.

30
docker/Dockerfile.database
View file

@ -84,7 +84,9 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra semantic-router \
--python python3
RUN prisma generate --schema=./schema.prisma
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@ -97,7 +99,11 @@ USER root
RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
WORKDIR /app
ENV PATH="/app/.venv/bin:${PATH}"
ENV PATH="/app/.venv/bin:${PATH}" \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
PRISMA_OFFLINE_MODE=true
# Copy only what runtime needs. The application is installed inside the venv;
# the rest of the builder's /app is source and build metadata that must not
@ -112,16 +118,20 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
# enterprise.enterprise_hooks from it)
COPY --from=builder /app/enterprise /app/enterprise
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
# Prisma binaries live in $HOME/.cache (default prisma-python location),
# which is /root/.cache here. Copy them from the builder so they survive
# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
# Only the Prisma subdirs: the whole /root/.cache drags in the uv build cache.
COPY --from=builder /root/.cache/prisma /root/.cache/prisma
COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
# Prisma CLI + engines are baked under /opt/prisma, a fixed path every
# runtime uid can read and that no cache volume mount shadows (unlike
# /app/.cache or $HOME/.cache in readOnlyRootFilesystem + emptyDir setups).
# The paths are pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and
# recorded into the generated client at build time, so `prisma migrate
# deploy` on a fresh database needs no npm and no network access
# (#33650, #24554).
COPY --from=builder /opt/prisma /opt/prisma
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
find /app/.venv -type d -path "*/tornado/test" -delete && \
chmod -R a+rX /opt/prisma && \
test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
EXPOSE 4000/tcp

17
litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_compression_saved_tokens/migration.sql Normal file
View file

@ -0,0 +1,17 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;

9
litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_mcp_server_oauth_client_table/migration.sql Normal file
View file

@ -0,0 +1,9 @@
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_MCPServerOAuthClient" (
"server_id" TEXT NOT NULL,
"credentials" JSONB,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "LiteLLM_MCPServerOAuthClient_pkey" PRIMARY KEY ("server_id")
);

23
litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql Normal file
View file

@ -0,0 +1,23 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

25
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -399,6 +399,13 @@ model LiteLLM_MCPUserEnvVars {
@@index([server_id])
}
model LiteLLM_MCPServerOAuthClient {
server_id String @id
credentials Json?
created_at DateTime @default(now()) @map("created_at")
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
}
// Generate Tokens for Proxy
model LiteLLM_VerificationToken {
token String @id
@ -734,6 +741,9 @@ model LiteLLM_DailyUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -765,6 +775,9 @@ model LiteLLM_DailyOrganizationSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -796,6 +809,9 @@ model LiteLLM_DailyEndUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -826,6 +842,9 @@ model LiteLLM_DailyAgentSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -856,6 +875,9 @@ model LiteLLM_DailyTeamSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -888,6 +910,9 @@ model LiteLLM_DailyTagSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.78"
version = "0.4.79"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.78"
version = "0.4.79"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

19
litellm-rust/ADDING_A_PROVIDER.md
View file

@ -6,4 +6,23 @@ Three layers, same for every route (see `ocr` and `realtime` as references):
2. **Provider config (pure)** — `crates/providers/src/<provider>/<route>/transformation.rs`: implement that trait as a `const <PROVIDER>_<ROUTE>_CONFIG`, mirroring the Python provider tree. Add parity unit tests.
3. **HTTP / transport (the host)** — `crates/providers/src/<route>.rs` (e.g. `ocr.rs`, `realtime.rs`): the callable fn (`run_ocr`, `realtime`). It resolves the key, builds the auth header, builds URL + transforms via the config, then does the network call. This is the only layer allowed to do I/O.
## Coding standards
Before writing new logic, look for an existing base to extend. When a change is
“the same behavior for one more provider/endpoint/integration”, the codebase
almost always already has a shared abstraction for it (for example, provider
`BaseConfig` transformation classes in `litellm/llms/base_llm/`, shared
helpers in `litellm_core_utils/`, typed request/response models, or factory
functions). Find it first with a search, then add the new variant by inheriting
from or composing that base, overriding only what genuinely differs (model
name, parameter mapping, or auth).
Never copy an existing implementation and edit it in place, and never hand-roll
a parallel version of logic a base already provides. If you catch yourself
writing a second copy of a pattern that exists twice already, stop and extract a
base instead: put the shared shape in one place and make both call sites thin
variants of it. The test for a good abstraction is that adding the next provider
is a few declarative lines, not a new file of duplicated flow. Only diverge from
the base when behavior is genuinely different, and say so explicitly in the PR.
**Calling:** the host invokes the route fn — the Python bridge calls `run_ocr`; the `ai-gateway` server calls `realtime`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.

9
litellm-rust/AGENTS.md
View file

@ -15,3 +15,12 @@ Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-
Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these.
Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional.
## Style
All Rust in `litellm-rust/` follows the official Rust Style Guide:
https://doc.rust-lang.org/style-guide/
`rustfmt` implements its formatting by default, so run `cargo fmt` before committing; CI gates every PR on `cargo fmt --check`. Do not hand-format against rustfmt or add a `rustfmt.toml` that diverges from the default style.
Beyond formatting, follow the guide's naming and idiom conventions rustfmt cannot auto-apply: `snake_case` items/functions/modules, `UpperCamelCase` types/traits/variants, `SCREAMING_SNAKE_CASE` constants/statics (acronyms as one word, e.g. `HttpClient`), and the import grouping and item ordering it prescribes. See CLAUDE.md for the detailed version.

39
litellm-rust/CLAUDE.md
View file

@ -2,6 +2,25 @@
This file defines the rules for Rust work in LiteLLM.
## Provider Coding Standards
Before writing new logic, look for an existing base to extend. When a change is
“the same behavior for one more provider/endpoint/integration”, the codebase
almost always already has a shared abstraction for it (for example, provider
`BaseConfig` transformation classes in `litellm/llms/base_llm/`, shared
helpers in `litellm_core_utils/`, typed request/response models, or factory
functions). Find it first with a search, then add the new variant by inheriting
from or composing that base, overriding only what genuinely differs (model
name, parameter mapping, or auth).
Never copy an existing implementation and edit it in place, and never hand-roll
a parallel version of logic a base already provides. If you catch yourself
writing a second copy of a pattern that exists twice already, stop and extract a
base instead: put the shared shape in one place and make both call sites thin
variants of it. The test for a good abstraction is that adding the next provider
is a few declarative lines, not a new file of duplicated flow. Only diverge from
the base when behavior is genuinely different, and say so explicitly in the PR.
## Crates (exactly three — see AGENTS.md)
`litellm-core` describes work; `litellm-ai-gateway` executes it; `litellm-python-bridge`
@ -77,6 +96,26 @@ such as `ai-gateway`, router hosts, or standalone servers:
- Avoid `expect`/`unwrap` in server startup and request paths unless the panic is
impossible by construction and documented.
## Rust Style Guide
All Rust in `litellm-rust/` follows the official Rust Style Guide:
https://doc.rust-lang.org/style-guide/
`rustfmt` implements the guide's formatting rules by default, so the mechanical
side is enforced for you: run `cargo fmt` before committing and CI gates every
PR on `cargo fmt --check` (see Checks). Do not hand-format against rustfmt or add
a `rustfmt.toml` that diverges from the default style; the default style *is* the
guide.
The guide also covers conventions rustfmt cannot auto-apply; follow these too:
- Naming: `snake_case` for items, functions, and modules; `UpperCamelCase` for
types, traits, and enum variants; `SCREAMING_SNAKE_CASE` for constants and
statics; acronyms count as one word (`HttpClient`, not `HTTPClient`).
- Ordering and grouping the guide prescribes: imports grouped std / external /
crate-local, derives before other attributes, and consistent item order.
- Idioms the guide recommends over the formatter fighting you (e.g. prefer
restructuring an over-long expression rather than forcing an awkward wrap).
## Constants
Magic numbers and fixed strings go in a crate-level `constants.rs`, never

94
litellm-rust/Cargo.lock generated
View file

@ -19,12 +19,6 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "autocfg"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "axum"
version = "0.7.9"
@ -233,21 +227,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "futures"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.32"
@ -264,17 +243,6 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
[[package]]
name = "futures-executor"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.32"
@ -310,7 +278,6 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
@ -608,15 +575,6 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "indoc"
version = "2.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
dependencies = [
"rustversion",
]
[[package]]
name = "ipnet"
version = "2.12.0"
@ -718,15 +676,6 @@ version = "2.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
[[package]]
name = "memoffset"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.17"
@ -803,29 +752,26 @@ dependencies = [
[[package]]
name = "pyo3"
version = "0.23.5"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872"
checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c"
dependencies = [
"cfg-if",
"indoc",
"libc",
"memoffset",
"once_cell",
"portable-atomic",
"pyo3-build-config",
"pyo3-ffi",
"pyo3-macros",
"unindent",
]
[[package]]
name = "pyo3-async-runtimes"
version = "0.23.0"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "977dc837525cfd22919ba6a831413854beb7c99a256c03bf8624ad707e45810e"
checksum = "b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046"
dependencies = [
"futures",
"futures-channel",
"futures-util",
"once_cell",
"pin-project-lite",
"pyo3",
@ -834,19 +780,18 @@ dependencies = [
[[package]]
name = "pyo3-build-config"
version = "0.23.5"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb"
checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
dependencies = [
"once_cell",
"target-lexicon",
]
[[package]]
name = "pyo3-ffi"
version = "0.23.5"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d"
checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b"
dependencies = [
"libc",
"pyo3-build-config",
@ -854,9 +799,9 @@ dependencies = [
[[package]]
name = "pyo3-macros"
version = "0.23.5"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da"
checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
@ -866,13 +811,12 @@ dependencies = [
[[package]]
name = "pyo3-macros-backend"
version = "0.23.5"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028"
checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362"
dependencies = [
"heck",
"proc-macro2",
"pyo3-build-config",
"quote",
"syn",
]
@ -1321,9 +1265,9 @@ dependencies = [
[[package]]
name = "target-lexicon"
version = "0.12.16"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
[[package]]
name = "thiserror"
@ -1559,12 +1503,6 @@ version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unindent"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
[[package]]
name = "untrusted"
version = "0.9.0"

4
litellm-rust/Cargo.toml
View file

@ -15,8 +15,8 @@ repository = "https://github.com/BerriAI/litellm"
litellm-core = { path = "crates/core" }
litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false }
axum = "0.7"
pyo3 = "0.23.5"
pyo3-async-runtimes = { version = "0.23.0", features = ["tokio-runtime"] }
pyo3 = "0.29.0"
pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
rand = "0.8"
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] }
serde = { version = "1.0", features = ["derive"] }

53
litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md Normal file
View file

@ -0,0 +1,53 @@
# Provider coding standards (litellm-rust)
Rules for adding or changing an LLM provider/route in `litellm-rust`. OCR (`MISTRAL_OCR_CONFIG`) is the reference; `messages` (`ANTHROPIC_MESSAGES_CONFIG`) is the next port.
## Provider resolution
1. Always resolve the provider/model first with `get_custom_llm_provider` (`core/src/routing_utils/provider.rs`). Nothing downstream may branch on a raw model string.
2. Model/provider is resolved once, in `prepare.rs`, and passed down as typed fields. Don't re-resolve or re-parse it in transforms or handlers.
## Transforms and the base config
3. Every route defines a base config trait with `transform_request` + `transform_response` (+ `complete_url`, `supported_params`), living in `core/src/<route>/transformation.rs` (e.g. `AnthropicMessagesProviderConfig`, mirroring `OcrProviderConfig`).
4. Each provider implements that trait as a `const <PROVIDER>_<ROUTE>_CONFIG` in `core/src/providers/<provider>/<route>/transformation.rs`, mirroring the Python provider tree.
5. Individual configs implement only the request/response transforms. Shared behavior (param filtering, defaults) stays as trait default methods so future providers inherit existing logic instead of reimplementing it.
6. Prefer composition: a provider that extends another reuses the base trait's defaults or wraps another config; don't copy transform bodies between providers.
## Boundaries
7. Layers never cross: `core` = pure transforms/types (no network, env, secrets, auth, logging, global mutable state); `ai-gateway` = all I/O, auth headers, HTTP/SSE, lifecycle hooks; `python-bridge` = thin PyO3 adapter.
8. Generic/route files contain zero provider-specific branches. A provider is one module under `core/src/providers/<provider>/<route>/`; a route is a module, never a new crate.
9. Route entry point stays thin: `<route>()` -> `prepare_*` -> `CallLifecycle::run_request`, which owns the pre_call -> during_call -> provider call -> success/failure order and phase timing. Handlers validate and delegate; no business logic in them.
10. Constants (URLs, env-var names, API versions, error messages) live in a crate `constants.rs`, never inline. Env reads happen only at the host/config layer, with the `DEFAULT_*` fallback defined in `constants.rs`.
## Types and errors
11. Typed contracts only: no bare `serde_json::Value` / `String` / `Vec<String>` as a transform input or output. Parse wire bytes into typed structs/enums at the host edge; a `type` discriminator is a typed field, not a raw string.
12. Model failures as values: return typed `CoreError`, don't panic. No `unwrap`/`expect`/`panic!` on user or provider input.
13. No mutation: build values in one shot (comprehensions/iterators, `collect`), prefer immutable bindings and owned typed structs over seeding-and-mutating.
14. Early returns over deep nesting; small focused files over god modules.
15. Preserve Python output shape intentionally. If a field is always serialized as `null` for parity, keep it and pin it with a test.
## Safety and data minimization
16. Never log request/response bodies, base64 payloads, document contents, or secrets. Truncate and bound any upstream body before it crosses a host boundary.
17. Treat empty/whitespace credentials, URLs, and config values as absent at the host resolution layer.
18. Host I/O sets connect + request timeouts (no unbounded waits), reuses a shared HTTP client, and prefers rustls TLS.
## Tests and rollout
19. Every provider transform ships tests for: supported-param filtering, request body shape, response normalization, missing/null fields, bad input, and `*_match_python` fixture parity.
20. Lifecycle/hook tests cover hook order, success + failure callback payloads, pre-call guardrail blocking before any provider I/O, during-call body mutation, and provider-error mapping.
21. Rust paths stay off by default and behind Python parity tests (disabled / enabled-equals-Python / bridge-unavailable fallback) until parity is proven.
## Checks before push
22. Run, and keep green:
```bash
cd litellm-rust
cargo fmt --check
cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings
cargo test --workspace
```

12
litellm-rust/crates/ai-gateway/src/constants.rs
View file

@ -28,3 +28,15 @@ pub(crate) const DEFAULT_FLUSH_INTERVAL_MS: u64 = 500;
/// Provider attributed to realtime sessions in the logging payload.
#[cfg(feature = "server")]
pub(crate) const DEFAULT_PROVIDER: &str = "openai";
/// Full-request timeout ceiling for Anthropic Messages provider calls, in
/// seconds. Mirrors the Python Anthropic Messages default. The per-request
/// timeout from `litellm_params` still overrides this on the request builder.
pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600;
/// Connect timeout for Anthropic Messages provider calls, in seconds.
pub(crate) const MESSAGES_CONNECT_TIMEOUT_SECS: u64 = 10;
/// Max characters of an upstream error body echoed across the host boundary
/// before truncation, so provider bodies are bounded and data-minimized.
pub(crate) const MESSAGES_ERROR_BODY_MAX_CHARS: usize = 256;

1
litellm-rust/crates/ai-gateway/src/io/messages.rs Normal file
View file

@ -0,0 +1 @@
pub use crate::messages::{messages, MessagesRequest};

1
litellm-rust/crates/ai-gateway/src/io/mod.rs
View file

@ -1,3 +1,4 @@
pub mod messages;
pub mod ocr;
pub mod realtime;
pub mod realtime_pool;

1
litellm-rust/crates/ai-gateway/src/lib.rs
View file

@ -12,6 +12,7 @@
//! for the load-time config reader.
pub mod io;
pub mod messages;
pub mod ocr;
/// GIL-activity tracking. Pure (atomics only); shared by the `server` routes and

15
litellm-rust/crates/ai-gateway/src/messages/client.rs Normal file
View file

@ -0,0 +1,15 @@
use std::sync::OnceLock;
use std::time::Duration;
use crate::constants::{MESSAGES_CONNECT_TIMEOUT_SECS, MESSAGES_TIMEOUT_SECS};
pub(super) fn http_client() -> &'static reqwest::Client {
static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
CLIENT.get_or_init(|| {
reqwest::Client::builder()
.timeout(Duration::from_secs(MESSAGES_TIMEOUT_SECS))
.connect_timeout(Duration::from_secs(MESSAGES_CONNECT_TIMEOUT_SECS))
.build()
.unwrap_or_else(|_| reqwest::Client::new())
})
}

50
litellm-rust/crates/ai-gateway/src/messages/common_utils.rs Normal file
View file

@ -0,0 +1,50 @@
use litellm_core::error::{json_type_name, CoreError};
use litellm_core::messages::transformation::AnthropicMessagesProviderConfig;
use litellm_core::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG;
use litellm_core::CoreResult;
use serde_json::{Map, Value};
use crate::constants::MESSAGES_ERROR_BODY_MAX_CHARS;
pub(super) fn truncate_error_body(body: &str) -> String {
if body.chars().count() <= MESSAGES_ERROR_BODY_MAX_CHARS {
return body.to_string();
}
let truncated: String = body.chars().take(MESSAGES_ERROR_BODY_MAX_CHARS).collect();
format!("{truncated}... (truncated)")
}
pub(super) fn messages_provider_config(
provider: &str,
) -> Option<&'static dyn AnthropicMessagesProviderConfig> {
match provider {
"azure_ai" => Some(&AZURE_ANTHROPIC_MESSAGES_CONFIG),
_ => None,
}
}
pub(super) fn string_headers(
extra_headers: Option<Map<String, Value>>,
) -> CoreResult<Vec<(String, String)>> {
extra_headers
.unwrap_or_default()
.into_iter()
.map(|(key, value)| {
value
.as_str()
.map(|value| (key.clone(), value.to_string()))
.ok_or_else(|| {
CoreError::InvalidRequest(format!(
"messages extra_headers.{key} must be a string, got {}",
json_type_name(&value)
))
})
})
.collect()
}
pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool {
headers
.iter()
.any(|(key, _)| key.eq_ignore_ascii_case(name))
}

47
litellm-rust/crates/ai-gateway/src/messages/handler.rs Normal file
View file

@ -0,0 +1,47 @@
use litellm_core::error::CoreError;
use litellm_core::CoreResult;
use serde_json::Value;
use super::client::http_client;
use super::common_utils::truncate_error_body;
use super::types::ProviderMessagesRequest;
pub(super) async fn execute_messages_provider_call(
request: ProviderMessagesRequest,
) -> CoreResult<Value> {
let mut request_builder = http_client().post(&request.url).json(&request.body);
for (key, value) in &request.upstream_headers {
request_builder = request_builder.header(key, value);
}
if let Some(duration) = request.timeout {
request_builder = request_builder.timeout(duration);
}
let response = request_builder
.send()
.await
.map_err(|err| CoreError::Network(err.to_string()))?;
let status = response.status();
let text = response
.text()
.await
.map_err(|err| CoreError::Network(err.to_string()))?;
if !status.is_success() {
return Err(CoreError::Http {
status: status.as_u16(),
body: truncate_error_body(&text),
});
}
let response = serde_json::from_str(&text).map_err(|err| {
CoreError::InvalidResponse(format!("invalid messages response JSON: {err}"))
})?;
let transformed = request
.config
.transform_response(&request.model, response)?;
serde_json::to_value(transformed).map_err(|err| {
CoreError::InvalidResponse(format!("failed to serialize messages response: {err}"))
})
}

21
litellm-rust/crates/ai-gateway/src/messages/mod.rs Normal file
View file

@ -0,0 +1,21 @@
use litellm_core::CoreResult;
use serde_json::Value;
mod client;
mod common_utils;
mod handler;
mod prepare;
mod types;
pub use types::MessagesRequest;
use handler::execute_messages_provider_call;
use prepare::prepare_messages_call;
pub async fn messages(request: MessagesRequest<'_>) -> CoreResult<Value> {
let prepared = prepare_messages_call(request)?;
execute_messages_provider_call(prepared).await
}
#[cfg(test)]
mod tests;

72
litellm-rust/crates/ai-gateway/src/messages/prepare.rs Normal file
View file

@ -0,0 +1,72 @@
use litellm_core::messages::transformation::MessagesAuthStrategy;
use litellm_core::routing_utils::provider::{get_custom_llm_provider, CustomLlmProvider};
use litellm_core::CoreError;
use litellm_core::CoreResult;
use super::common_utils::{has_header, messages_provider_config, string_headers};
use super::types::{MessagesRequest, ProviderMessagesRequest};
pub(super) fn prepare_messages_call(
request: MessagesRequest<'_>,
) -> CoreResult<ProviderMessagesRequest> {
let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider)
.or_else(|| {
request
.custom_llm_provider
.map(|provider| CustomLlmProvider {
model: request.model,
custom_llm_provider: provider,
})
})
.ok_or_else(|| {
CoreError::InvalidProvider(
"unable to resolve custom_llm_provider for messages request".to_string(),
)
})?;
let model = provider_info.model.to_string();
let provider = provider_info.custom_llm_provider;
let config = messages_provider_config(provider)
.ok_or_else(|| CoreError::InvalidProvider(provider.to_string()))?;
let env_lookup = |key: &str| std::env::var(key).ok();
let mut headers = string_headers(request.extra_headers)?;
let auth_strategy = config.auth_strategy();
if !has_header(&headers, auth_strategy.header_name()) {
let api_key = config.resolve_api_key(request.api_key, &env_lookup)?;
let auth_header = match auth_strategy {
MessagesAuthStrategy::Bearer => {
("authorization".to_string(), format!("Bearer {api_key}"))
}
MessagesAuthStrategy::Header(name) => (name.to_string(), api_key),
};
headers.push(auth_header);
}
for (name, value) in config.default_headers() {
if !has_header(&headers, name) {
headers.push((name.to_string(), value.to_string()));
}
}
let url = config.complete_url(request.api_base, &model, &env_lookup)?;
let typed_request = serde_json::from_value(request.body).map_err(|err| {
CoreError::InvalidRequest(format!("invalid Anthropic messages request: {err}"))
})?;
let transformed = config.transform_request(typed_request)?;
let body = serde_json::to_value(transformed).map_err(|err| {
CoreError::InvalidRequest(format!(
"failed to serialize Anthropic messages request: {err}"
))
})?;
Ok(ProviderMessagesRequest {
model,
config,
url,
body,
upstream_headers: headers,
timeout: request.timeout,
})
}

259
litellm-rust/crates/ai-gateway/src/messages/tests.rs Normal file
View file

@ -0,0 +1,259 @@
use std::time::Duration;
use litellm_core::error::CoreError;
use serde_json::{json, Map, Value};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use super::common_utils::{
has_header, messages_provider_config, string_headers, truncate_error_body,
};
use super::{messages, MessagesRequest};
async fn read_http_request(socket: &mut TcpStream) -> String {
let mut request = Vec::new();
let mut buffer = [0_u8; 1024];
let header_end = loop {
let n = socket.read(&mut buffer).await.expect("reads request");
if n == 0 {
break request.len();
}
request.extend_from_slice(&buffer[..n]);
if let Some(position) = request.windows(4).position(|window| window == b"\r\n\r\n") {
break position + 4;
}
};
let headers = String::from_utf8_lossy(&request[..header_end]);
let content_length = headers
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().ok())
.flatten()
})
.unwrap_or(0);
while request.len().saturating_sub(header_end) < content_length {
let n = socket.read(&mut buffer).await.expect("reads body");
if n == 0 {
break;
}
request.extend_from_slice(&buffer[..n]);
}
String::from_utf8(request).expect("request is utf8")
}
fn write_response(body: &str) -> String {
format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body
)
}
#[test]
fn provider_config_only_resolves_azure_ai() {
assert!(messages_provider_config("azure_ai").is_some());
assert!(messages_provider_config("anthropic").is_none());
assert!(messages_provider_config("openai").is_none());
}
#[test]
fn truncate_error_body_caps_long_payloads() {
let body = "x".repeat(400);
let truncated = truncate_error_body(&body);
assert!(truncated.ends_with("... (truncated)"));
let prefix_chars = truncated
.strip_suffix("... (truncated)")
.expect("truncated marker present")
.chars()
.count();
assert_eq!(prefix_chars, 256);
}
#[test]
fn string_headers_rejects_non_string_values() {
let headers = json!({"x-count": 3}).as_object().unwrap().clone();
let err = string_headers(Some(headers)).expect_err("non-string header rejected");
assert!(matches!(err, CoreError::InvalidRequest(_)));
}
#[test]
fn has_header_is_case_insensitive() {
let headers = vec![("X-Api-Key".to_string(), "secret".to_string())];
assert!(has_header(&headers, "x-api-key"));
assert!(!has_header(&headers, "authorization"));
}
#[tokio::test]
async fn messages_round_trip_builds_azure_request_and_passes_response_through() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
let addr = listener.local_addr().expect("addr");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("accepts request");
let request = read_http_request(&mut socket).await;
let response_body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-sonnet-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":2}}"#;
socket
.write_all(write_response(response_body).as_bytes())
.await
.expect("writes response");
request
});
let response = messages(MessagesRequest {
model: "claude-sonnet-4-5",
body: json!({
"model": "claude-sonnet-4-5",
"max_tokens": 1024,
"messages": [{
"role": "user",
"content": [{
"type": "text",
"text": "hi",
"cache_control": {"type": "ephemeral", "scope": "global"}
}]
}]
}),
api_key: Some("sk-azure"),
api_base: Some(&format!("http://{addr}")),
custom_llm_provider: Some("azure_ai"),
extra_headers: None,
timeout: Some(Duration::from_secs(5)),
})
.await
.expect("messages request succeeds");
assert_eq!(response["content"][0]["text"], "hi");
assert_eq!(response["stop_reason"], "end_turn");
let request = server.await.expect("server task completes");
let (head, body) = request.split_once("\r\n\r\n").expect("has body");
assert!(head.starts_with("POST /anthropic/v1/messages "), "{head}");
let head_lower = head.to_ascii_lowercase();
assert!(head_lower.contains("x-api-key: sk-azure"), "{head}");
assert!(
head_lower.contains("anthropic-version: 2023-06-01"),
"{head}"
);
assert!(
head_lower.contains("content-type: application/json"),
"{head}"
);
let sent_body: Value = serde_json::from_str(body).expect("body is json");
assert_eq!(
sent_body["messages"][0]["content"][0]["cache_control"],
json!({"type": "ephemeral"})
);
}
#[tokio::test]
async fn messages_does_not_duplicate_auth_when_x_api_key_supplied() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
let addr = listener.local_addr().expect("addr");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("accepts request");
let request = read_http_request(&mut socket).await;
let response_body =
r#"{"id":"msg_2","type":"message","role":"assistant","content":[],"model":"m"}"#;
socket
.write_all(write_response(response_body).as_bytes())
.await
.expect("writes response");
request
});
let mut headers = Map::new();
headers.insert(
"x-api-key".to_string(),
Value::String("from-python".to_string()),
);
headers.insert(
"anthropic-beta".to_string(),
Value::String("token-efficient-tools-2025-02-19".to_string()),
);
messages(MessagesRequest {
model: "claude-sonnet-4-5",
body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
api_key: Some("rust-fallback-key"),
api_base: Some(&format!("http://{addr}")),
custom_llm_provider: Some("azure_ai"),
extra_headers: Some(headers),
timeout: Some(Duration::from_secs(5)),
})
.await
.expect("messages request succeeds");
let request = server.await.expect("server task completes");
let head = request
.split_once("\r\n\r\n")
.expect("has body")
.0
.to_ascii_lowercase();
let api_key_count = head
.lines()
.filter(|line| line.starts_with("x-api-key:"))
.count();
assert_eq!(api_key_count, 1, "{head}");
assert!(head.contains("x-api-key: from-python"), "{head}");
assert!(
head.contains("anthropic-beta: token-efficient-tools-2025-02-19"),
"{head}"
);
assert!(!head.contains("rust-fallback-key"), "{head}");
}
#[tokio::test]
async fn messages_maps_provider_error_status_to_http_error() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
let addr = listener.local_addr().expect("addr");
tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("accepts request");
let _ = read_http_request(&mut socket).await;
let body = "unauthorized";
let response = format!(
"HTTP/1.1 401 Unauthorized\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body
);
socket
.write_all(response.as_bytes())
.await
.expect("writes response");
});
let err = messages(MessagesRequest {
model: "claude-sonnet-4-5",
body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
api_key: Some("sk-azure"),
api_base: Some(&format!("http://{addr}")),
custom_llm_provider: Some("azure_ai"),
extra_headers: None,
timeout: Some(Duration::from_secs(5)),
})
.await
.expect_err("provider error propagates");
assert!(matches!(err, CoreError::Http { status: 401, .. }));
}
#[tokio::test]
async fn messages_rejects_unsupported_provider() {
let err = messages(MessagesRequest {
model: "claude-3-5-sonnet",
body: json!({"model": "claude-3-5-sonnet", "max_tokens": 8, "messages": []}),
api_key: Some("sk"),
api_base: Some("http://127.0.0.1:1"),
custom_llm_provider: Some("anthropic"),
extra_headers: None,
timeout: Some(Duration::from_millis(50)),
})
.await
.expect_err("unsupported provider errors");
assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "anthropic"));
}

23
litellm-rust/crates/ai-gateway/src/messages/types.rs Normal file
View file

@ -0,0 +1,23 @@
use std::time::Duration;
use litellm_core::messages::transformation::AnthropicMessagesProviderConfig;
use serde_json::{Map, Value};
pub struct MessagesRequest<'a> {
pub model: &'a str,
pub body: Value,
pub api_key: Option<&'a str>,
pub api_base: Option<&'a str>,
pub custom_llm_provider: Option<&'a str>,
pub extra_headers: Option<Map<String, Value>>,
pub timeout: Option<Duration>,
}
pub(crate) struct ProviderMessagesRequest {
pub(crate) model: String,
pub(crate) config: &'static dyn AnthropicMessagesProviderConfig,
pub(crate) url: String,
pub(crate) body: Value,
pub(crate) upstream_headers: Vec<(String, String)>,
pub(crate) timeout: Option<Duration>,
}

2
litellm-rust/crates/ai-gateway/src/python/config.rs
View file

@ -17,7 +17,7 @@ use crate::gil;
/// Load the router's `model_list` from `config_path` via the Python reader.
pub fn load_router_from_config(config_path: &str) -> CoreResult<Router> {
gil::record_acquisition();
Python::with_gil(|py| {
Python::attach(|py| {
let model_list = py
.import("litellm.proxy.read_model_list")
.and_then(|module| module.getattr("read_model_list"))

4
litellm-rust/crates/core/src/error.rs
View file

@ -19,9 +19,9 @@ pub enum CoreError {
InvalidRequest(String),
#[error("{0}")]
Auth(String),
#[error("OCR request failed with status {status}: {body}")]
#[error("upstream request failed with status {status}: {body}")]
Http { status: u16, body: String },
#[error("OCR network error: {0}")]
#[error("upstream network error: {0}")]
Network(String),
#[error("routing error: {0}")]
Routing(String),

1
litellm-rust/crates/core/src/lib.rs
View file

@ -1,5 +1,6 @@
pub mod call_lifecycle;
pub mod error;
pub mod messages;
pub mod ocr;
pub mod providers;
pub mod realtime;

2
litellm-rust/crates/core/src/messages/mod.rs Normal file
View file

@ -0,0 +1,2 @@
pub mod transformation;
pub mod types;

59
litellm-rust/crates/core/src/messages/transformation.rs Normal file
View file

@ -0,0 +1,59 @@
use crate::error::CoreResult;
use super::types::{AnthropicMessagesRequest, AnthropicMessagesResponse};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MessagesAuthStrategy {
Bearer,
Header(&'static str),
}
impl MessagesAuthStrategy {
pub fn header_name(self) -> &'static str {
match self {
Self::Bearer => "authorization",
Self::Header(header_name) => header_name,
}
}
}
pub trait AnthropicMessagesProviderConfig: Sync {
fn complete_url(
&self,
api_base: Option<&str>,
model: &str,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> CoreResult<String>;
fn resolve_api_key(
&self,
api_key: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> CoreResult<String>;
fn auth_strategy(&self) -> MessagesAuthStrategy {
MessagesAuthStrategy::Header("x-api-key")
}
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
&[
("anthropic-version", "2023-06-01"),
("content-type", "application/json"),
]
}
fn transform_request(
&self,
request: AnthropicMessagesRequest,
) -> CoreResult<AnthropicMessagesRequest> {
Ok(request)
}
fn transform_response(
&self,
_model: &str,
response: AnthropicMessagesResponse,
) -> CoreResult<AnthropicMessagesResponse> {
Ok(response)
}
}

110
litellm-rust/crates/core/src/messages/types.rs Normal file
View file

@ -0,0 +1,110 @@
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum SystemPrompt {
Text(String),
Blocks(Vec<ContentBlock>),
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum MessageContent {
Text(String),
Blocks(Vec<ContentBlock>),
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct ContentBlock {
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct CacheControl {
#[serde(rename = "type", skip_serializing_if = "Option::is_none")]
pub cache_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ttl: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub scope: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AnthropicMessage {
pub role: String,
pub content: MessageContent,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AnthropicMessagesRequest {
pub model: String,
pub messages: Vec<AnthropicMessage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub system: Option<SystemPrompt>,
#[serde(skip_serializing_if = "Option::is_none")]
pub metadata: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stop_sequences: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stream: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub temperature: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub top_p: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub top_k: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tools: Option<Vec<Value>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_choice: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub thinking: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub service_tier: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub container: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mcp_servers: Option<Vec<Value>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub context_management: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_format: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_config: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub speed: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub inference_geo: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AnthropicMessagesResponse {
pub id: String,
#[serde(rename = "type")]
pub message_type: String,
pub role: String,
pub model: String,
pub content: Vec<Value>,
// Anthropic always includes stop_reason / stop_sequence, null until the turn
// ends; serialize them even when None so callers see the same shape as Python.
pub stop_reason: Option<String>,
pub stop_sequence: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub usage: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub container: Option<Value>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

1
litellm-rust/crates/core/src/providers/anthropic/messages/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod transformation;

142
litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs Normal file
View file

@ -0,0 +1,142 @@
use crate::error::{CoreError, CoreResult};
use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy};
const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY";
const ANTHROPIC_API_BASE_ENV: &str = "ANTHROPIC_API_BASE";
const DEFAULT_ANTHROPIC_API_BASE: &str = "https://api.anthropic.com";
const MESSAGES_PATH_SUFFIX: &str = "/v1/messages";
pub struct AnthropicMessagesConfig;
pub const ANTHROPIC_MESSAGES_CONFIG: AnthropicMessagesConfig = AnthropicMessagesConfig;
pub fn non_empty(value: Option<&str>) -> Option<&str> {
value.map(str::trim).filter(|value| !value.is_empty())
}
pub fn resolve_anthropic_api_key(
api_key: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> CoreResult<String> {
non_empty(api_key)
.map(str::to_string)
.or_else(|| env_lookup(ANTHROPIC_API_KEY_ENV).filter(|value| !value.trim().is_empty()))
.ok_or_else(|| {
CoreError::Auth(
"Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY \
environment variable"
.to_string(),
)
})
}
pub fn complete_anthropic_url(
api_base: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> String {
let api_base = non_empty(api_base)
.map(str::to_string)
.or_else(|| env_lookup(ANTHROPIC_API_BASE_ENV).filter(|value| !value.trim().is_empty()))
.unwrap_or_else(|| DEFAULT_ANTHROPIC_API_BASE.to_string());
let api_base = api_base.trim_end_matches('/');
if api_base.ends_with(MESSAGES_PATH_SUFFIX) {
return api_base.to_string();
}
format!("{api_base}{MESSAGES_PATH_SUFFIX}")
}
impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig {
fn complete_url(
&self,
api_base: Option<&str>,
_model: &str,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> CoreResult<String> {
Ok(complete_anthropic_url(api_base, env_lookup))
}
fn resolve_api_key(
&self,
api_key: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> CoreResult<String> {
resolve_anthropic_api_key(api_key, env_lookup)
}
fn auth_strategy(&self) -> MessagesAuthStrategy {
MessagesAuthStrategy::Header("x-api-key")
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn url_defaults_to_public_anthropic_endpoint() {
assert_eq!(
complete_anthropic_url(None, &|_| None),
"https://api.anthropic.com/v1/messages"
);
}
#[test]
fn url_appends_messages_suffix_to_custom_base() {
assert_eq!(
complete_anthropic_url(Some("https://proxy.internal"), &|_| None),
"https://proxy.internal/v1/messages"
);
}
#[test]
fn url_leaves_complete_messages_endpoint_untouched() {
assert_eq!(
complete_anthropic_url(Some("https://proxy.internal/v1/messages"), &|_| None),
"https://proxy.internal/v1/messages"
);
}
#[test]
fn url_falls_back_to_env_base() {
let with_env = |key: &str| {
(key == ANTHROPIC_API_BASE_ENV).then(|| "https://env.anthropic".to_string())
};
assert_eq!(
complete_anthropic_url(Some(" "), &with_env),
"https://env.anthropic/v1/messages"
);
}
#[test]
fn api_key_prefers_param_then_env_then_errors() {
assert_eq!(
resolve_anthropic_api_key(Some("sk-param"), &|_| None).unwrap(),
"sk-param"
);
let with_env = |key: &str| (key == ANTHROPIC_API_KEY_ENV).then(|| "sk-env".to_string());
assert_eq!(
resolve_anthropic_api_key(Some(" "), &with_env).unwrap(),
"sk-env"
);
assert!(matches!(
resolve_anthropic_api_key(None, &|_| None).expect_err("missing key"),
CoreError::Auth(_)
));
}
#[test]
fn auth_strategy_and_default_headers_match_anthropic() {
assert_eq!(
ANTHROPIC_MESSAGES_CONFIG.auth_strategy().header_name(),
"x-api-key"
);
assert_eq!(
ANTHROPIC_MESSAGES_CONFIG.default_headers(),
&[
("anthropic-version", "2023-06-01"),
("content-type", "application/json"),
]
);
}
}

1
litellm-rust/crates/core/src/providers/anthropic/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod messages;

1
litellm-rust/crates/core/src/providers/azure_ai/messages/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod transformation;

512
litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs Normal file
View file

@ -0,0 +1,512 @@
use crate::error::{CoreError, CoreResult};
use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy};
use crate::messages::types::{
AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock,
MessageContent, SystemPrompt,
};
use crate::providers::anthropic::messages::transformation::{
non_empty, AnthropicMessagesConfig, ANTHROPIC_MESSAGES_CONFIG,
};
use serde_json::{Map, Value};
const AZURE_API_KEY_ENV: &str = "AZURE_API_KEY";
const AZURE_API_BASE_ENV: &str = "AZURE_API_BASE";
const ANTHROPIC_PATH_SEGMENT: &str = "/anthropic";
const MESSAGES_PATH_SUFFIX: &str = "/v1/messages";
const SYSTEM_ROLE: &str = "system";
const TEXT_BLOCK_TYPE: &str = "text";
pub struct AzureAnthropicMessagesConfig {
anthropic: AnthropicMessagesConfig,
}
pub const AZURE_ANTHROPIC_MESSAGES_CONFIG: AzureAnthropicMessagesConfig =
AzureAnthropicMessagesConfig {
anthropic: ANTHROPIC_MESSAGES_CONFIG,
};
pub fn resolve_azure_api_key(
api_key: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> CoreResult<String> {
non_empty(api_key)
.map(str::to_string)
.or_else(|| env_lookup(AZURE_API_KEY_ENV).filter(|value| !value.trim().is_empty()))
.ok_or_else(|| {
CoreError::Auth(
"Missing Azure API Key - Set `api_key` or the AZURE_API_KEY environment variable"
.to_string(),
)
})
}
pub fn complete_azure_anthropic_url(
api_base: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> CoreResult<String> {
let api_base = non_empty(api_base)
.map(str::to_string)
.or_else(|| env_lookup(AZURE_API_BASE_ENV).filter(|value| !value.trim().is_empty()))
.ok_or_else(|| {
CoreError::Auth(
"Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. \
Expected format: https://<resource-name>.services.ai.azure.com/anthropic"
.to_string(),
)
})?;
let api_base = api_base.trim_end_matches('/');
if api_base.ends_with(MESSAGES_PATH_SUFFIX) {
return Ok(api_base.to_string());
}
let with_anthropic = match api_base.split_once(ANTHROPIC_PATH_SEGMENT) {
Some((prefix, _)) => format!("{prefix}{ANTHROPIC_PATH_SEGMENT}"),
None => format!("{api_base}{ANTHROPIC_PATH_SEGMENT}"),
};
Ok(format!("{with_anthropic}{MESSAGES_PATH_SUFFIX}"))
}
fn strip_scope_from_block(block: &mut ContentBlock) {
if let Some(cache_control) = block.cache_control.as_mut() {
cache_control.scope = None;
}
}
fn strip_scope_from_system(system: &mut SystemPrompt) {
if let SystemPrompt::Blocks(blocks) = system {
blocks.iter_mut().for_each(strip_scope_from_block);
}
}
fn strip_scope_from_message(message: &mut AnthropicMessage) {
if let MessageContent::Blocks(blocks) = &mut message.content {
blocks.iter_mut().for_each(strip_scope_from_block);
}
}
fn text_content_block(text: String) -> ContentBlock {
let extra = Map::from_iter([
(
"type".to_string(),
Value::String(TEXT_BLOCK_TYPE.to_string()),
),
("text".to_string(), Value::String(text)),
]);
ContentBlock {
cache_control: None,
extra,
}
}
fn content_into_blocks(content: MessageContent) -> Vec<ContentBlock> {
match content {
MessageContent::Text(text) => vec![text_content_block(text)],
MessageContent::Blocks(blocks) => blocks,
}
}
fn system_into_blocks(system: Option<SystemPrompt>) -> Vec<ContentBlock> {
match system {
None => Vec::new(),
Some(SystemPrompt::Text(text)) => vec![text_content_block(text)],
Some(SystemPrompt::Blocks(blocks)) => blocks,
}
}
fn fold_system_role_messages(request: AnthropicMessagesRequest) -> AnthropicMessagesRequest {
if !request.messages.iter().any(|msg| msg.role == SYSTEM_ROLE) {
return request;
}
let (system_messages, chat_messages): (Vec<AnthropicMessage>, Vec<AnthropicMessage>) = request
.messages
.into_iter()
.partition(|msg| msg.role == SYSTEM_ROLE);
let folded_system: Vec<ContentBlock> = system_into_blocks(request.system)
.into_iter()
.chain(
system_messages
.into_iter()
.flat_map(|msg| content_into_blocks(msg.content)),
)
.collect();
AnthropicMessagesRequest {
messages: chat_messages,
system: (!folded_system.is_empty()).then_some(SystemPrompt::Blocks(folded_system)),
..request
}
}
impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig {
fn complete_url(
&self,
api_base: Option<&str>,
_model: &str,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> CoreResult<String> {
complete_azure_anthropic_url(api_base, env_lookup)
}
fn resolve_api_key(
&self,
api_key: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> CoreResult<String> {
resolve_azure_api_key(api_key, env_lookup)
}
fn auth_strategy(&self) -> MessagesAuthStrategy {
self.anthropic.auth_strategy()
}
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
self.anthropic.default_headers()
}
fn transform_request(
&self,
request: AnthropicMessagesRequest,
) -> CoreResult<AnthropicMessagesRequest> {
let mut request = fold_system_role_messages(request);
if let Some(system) = request.system.as_mut() {
strip_scope_from_system(system);
}
request
.messages
.iter_mut()
.for_each(strip_scope_from_message);
self.anthropic.transform_request(request)
}
fn transform_response(
&self,
model: &str,
response: AnthropicMessagesResponse,
) -> CoreResult<AnthropicMessagesResponse> {
self.anthropic.transform_response(model, response)
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn request_from(value: serde_json::Value) -> AnthropicMessagesRequest {
serde_json::from_value(value).expect("valid request")
}
fn to_value(request: AnthropicMessagesRequest) -> serde_json::Value {
serde_json::to_value(request).expect("serializable request")
}
#[test]
fn url_appends_anthropic_and_messages_suffix() {
let url =
complete_azure_anthropic_url(Some("https://resource.services.ai.azure.com"), &|_| None)
.expect("url builds");
assert_eq!(
url,
"https://resource.services.ai.azure.com/anthropic/v1/messages"
);
}
#[test]
fn url_keeps_existing_anthropic_segment() {
let url = complete_azure_anthropic_url(
Some("https://resource.services.ai.azure.com/anthropic"),
&|_| None,
)
.expect("url builds");
assert_eq!(
url,
"https://resource.services.ai.azure.com/anthropic/v1/messages"
);
}
#[test]
fn url_leaves_complete_messages_endpoint_untouched() {
for base in [
"https://resource.services.ai.azure.com/anthropic/v1/messages",
"https://resource.services.ai.azure.com/v1/messages",
] {
assert_eq!(
complete_azure_anthropic_url(Some(base), &|_| None).expect("url builds"),
base
);
}
}
#[test]
fn url_trims_trailing_slash_and_truncates_after_anthropic() {
let url = complete_azure_anthropic_url(
Some("https://resource.services.ai.azure.com/anthropic/extra/"),
&|_| None,
)
.expect("url builds");
assert_eq!(
url,
"https://resource.services.ai.azure.com/anthropic/v1/messages"
);
}
#[test]
fn url_falls_back_to_env_then_errors_when_absent() {
let with_env = |key: &str| {
(key == AZURE_API_BASE_ENV).then(|| "https://env.services.ai.azure.com".to_string())
};
assert_eq!(
complete_azure_anthropic_url(None, &with_env).expect("url builds"),
"https://env.services.ai.azure.com/anthropic/v1/messages"
);
let err = complete_azure_anthropic_url(Some(" "), &|_| None).expect_err("missing base");
assert!(matches!(err, CoreError::Auth(_)));
}
#[test]
fn resolve_api_key_prefers_param_then_env() {
assert_eq!(
resolve_azure_api_key(Some("sk-param"), &|_| None).unwrap(),
"sk-param"
);
let with_env = |key: &str| (key == AZURE_API_KEY_ENV).then(|| "sk-env".to_string());
assert_eq!(
resolve_azure_api_key(Some(" "), &with_env).unwrap(),
"sk-env"
);
assert!(matches!(
resolve_azure_api_key(None, &|_| None).expect_err("missing key"),
CoreError::Auth(_)
));
}
#[test]
fn auth_strategy_is_x_api_key() {
assert_eq!(
AZURE_ANTHROPIC_MESSAGES_CONFIG
.auth_strategy()
.header_name(),
"x-api-key"
);
}
#[test]
fn default_headers_match_python() {
assert_eq!(
AZURE_ANTHROPIC_MESSAGES_CONFIG.default_headers(),
&[
("anthropic-version", "2023-06-01"),
("content-type", "application/json"),
]
);
}
#[test]
fn transform_request_strips_scope_from_system_and_messages() {
let request = request_from(json!({
"model": "claude-sonnet-4-5",
"max_tokens": 1024,
"system": [
{
"type": "text",
"text": "sys",
"cache_control": {"type": "ephemeral", "ttl": "1h", "scope": "global"}
}
],
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "hi",
"cache_control": {"type": "ephemeral", "scope": "global"}
},
{"type": "text", "text": "no cache control"}
]
}
]
}));
let transformed = to_value(
AZURE_ANTHROPIC_MESSAGES_CONFIG
.transform_request(request)
.expect("request transforms"),
);
assert_eq!(
transformed["system"][0]["cache_control"],
json!({"type": "ephemeral", "ttl": "1h"})
);
assert_eq!(
transformed["messages"][0]["content"][0]["cache_control"],
json!({"type": "ephemeral"})
);
assert_eq!(
transformed["messages"][0]["content"][1],
json!({"type": "text", "text": "no cache control"})
);
}
#[test]
fn transform_request_is_idempotent_and_preserves_string_system() {
let request = request_from(json!({
"model": "claude-sonnet-4-5",
"max_tokens": 16,
"system": "plain string system",
"messages": [{"role": "user", "content": "hi"}]
}));
let once = AZURE_ANTHROPIC_MESSAGES_CONFIG
.transform_request(request)
.expect("request transforms");
let twice = AZURE_ANTHROPIC_MESSAGES_CONFIG
.transform_request(once.clone())
.expect("request transforms");
assert_eq!(once, twice);
assert_eq!(to_value(once)["system"], json!("plain string system"));
}
#[test]
fn transform_request_preserves_all_supported_params() {
let body = json!({
"model": "claude-sonnet-4-5",
"max_tokens": 256,
"messages": [{"role": "user", "content": "hi"}],
"system": "be terse",
"metadata": {"user_id": "u1"},
"stop_sequences": ["STOP"],
"stream": false,
"temperature": 0.4,
"top_p": 0.9,
"top_k": 40,
"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],
"tool_choice": {"type": "auto"},
"thinking": {"type": "enabled", "budget_tokens": 1024},
"service_tier": "auto",
"container": {"id": "c1"},
"mcp_servers": [{"type": "url", "url": "https://mcp.example", "name": "x"}],
"context_management": {"edits": []},
"output_format": {"type": "json_schema"},
"output_config": {"effort": "high"},
"speed": "fast",
"inference_geo": "us",
"litellm_metadata": {"trace": "abc"}
});
let transformed = to_value(
AZURE_ANTHROPIC_MESSAGES_CONFIG
.transform_request(request_from(body.clone()))
.expect("request transforms"),
);
assert_eq!(transformed, body);
}
#[test]
fn transform_request_folds_system_role_message_into_top_level_system() {
let request = request_from(json!({
"model": "claude-sonnet-4-5",
"max_tokens": 256,
"system": [{"type": "text", "text": "base system"}],
"messages": [
{"role": "user", "content": "fix the bug"},
{"role": "system", "content": "Available agent types: claude"}
]
}));
let transformed = to_value(
AZURE_ANTHROPIC_MESSAGES_CONFIG
.transform_request(request)
.expect("request transforms"),
);
assert_eq!(
transformed["messages"],
json!([{"role": "user", "content": "fix the bug"}])
);
assert_eq!(
transformed["system"],
json!([
{"type": "text", "text": "base system"},
{"type": "text", "text": "Available agent types: claude"}
])
);
}
#[test]
fn transform_request_folds_system_role_when_no_top_level_system() {
let request = request_from(json!({
"model": "claude-sonnet-4-5",
"max_tokens": 256,
"messages": [
{"role": "user", "content": [{"type": "text", "text": "hi"}]},
{"role": "system", "content": [{"type": "text", "text": "sys block"}]}
]
}));
let transformed = to_value(
AZURE_ANTHROPIC_MESSAGES_CONFIG
.transform_request(request)
.expect("request transforms"),
);
assert_eq!(
transformed["messages"],
json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}])
);
assert_eq!(
transformed["system"],
json!([{"type": "text", "text": "sys block"}])
);
}
#[test]
fn transform_request_leaves_requests_without_system_role_untouched() {
let body = json!({
"model": "claude-sonnet-4-5",
"max_tokens": 256,
"system": "be terse",
"messages": [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "hello"}
]
});
let transformed = to_value(
AZURE_ANTHROPIC_MESSAGES_CONFIG
.transform_request(request_from(body.clone()))
.expect("request transforms"),
);
assert_eq!(transformed, body);
}
#[test]
fn transform_request_rejects_non_object_body() {
let err = serde_json::from_value::<AnthropicMessagesRequest>(json!("bad"))
.expect_err("non-object body should error");
assert!(err.is_data());
}
#[test]
fn transform_response_passes_through() {
let response: AnthropicMessagesResponse = serde_json::from_value(json!({
"id": "msg_1",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "hello"}],
"model": "claude-sonnet-4-5",
"stop_reason": "end_turn",
"stop_sequence": null,
"usage": {"input_tokens": 1, "output_tokens": 2}
}))
.expect("valid response");
let transformed = AZURE_ANTHROPIC_MESSAGES_CONFIG
.transform_response("claude-sonnet-4-5", response)
.expect("response transforms");
let value = serde_json::to_value(transformed).expect("serializable");
assert_eq!(value["stop_reason"], json!("end_turn"));
assert_eq!(value["stop_sequence"], json!(null));
assert_eq!(value["content"][0]["text"], json!("hello"));
}
}

1
litellm-rust/crates/core/src/providers/azure_ai/mod.rs
View file

@ -1 +1,2 @@
pub mod messages;
pub mod ocr;

1
litellm-rust/crates/core/src/providers/mod.rs
View file

@ -1,3 +1,4 @@
pub mod anthropic;
pub mod azure_ai;
pub mod mistral;
pub mod openai;

7
litellm-rust/crates/core/tests/workspace_crate_allowlist.rs
View file

@ -62,12 +62,17 @@ fn parse_members(manifest: &str) -> BTreeSet<String> {
members
}
/// The immediate subdirectory names under `crates/`.
/// The crate subdirectory names under `crates/`.
///
/// A directory counts as a crate only when it holds a `Cargo.toml`; non-crate
/// directories (e.g. docs like `CODING_STANDARDS/`) are ignored so they can live
/// under `crates/` without tripping the crate-proliferation guard.
fn crate_dirs(root: &Path) -> BTreeSet<String> {
fs::read_dir(root.join("crates"))
.expect("crates/ directory should exist")
.filter_map(Result::ok)
.filter(|entry| entry.file_type().map(|ty| ty.is_dir()).unwrap_or(false))
.filter(|entry| entry.path().join("Cargo.toml").is_file())
.map(|entry| entry.file_name().to_string_lossy().into_owned())
.collect()
}

4
litellm-rust/crates/python-bridge/src/gil.rs
View file

@ -2,7 +2,7 @@
//!
//! A single chokepoint for releasing the GIL around blocking work. Every
//! blocking call in the bridge goes through [`release_gil`] instead of calling
//! `Python::allow_threads` directly, so the release count stays accurate and we
//! `Python::detach` directly, so the release count stays accurate and we
//! have one place to extend later (timing histograms, per-call labels, etc.).
use std::sync::atomic::{AtomicU64, Ordering};
@ -23,7 +23,7 @@ where
T: Send,
{
GIL_RELEASES.fetch_add(1, Ordering::Relaxed);
py.allow_threads(f)
py.detach(f)
}
/// Total GIL releases performed by the bridge so far.

91
litellm-rust/crates/python-bridge/src/lib.rs
View file

@ -1,5 +1,6 @@
use std::time::Duration;
use litellm_ai_gateway::io::messages::{messages as run_messages, MessagesRequest};
use litellm_ai_gateway::io::ocr::{ocr as run_ocr, OcrRequest};
use litellm_core::error::CoreError;
use pyo3::exceptions::{PyRuntimeError, PyValueError};
@ -167,7 +168,93 @@ fn aocr(
.await
.map_err(core_error_to_pyerr)?;
Python::with_gil(|py| json_to_py(py, value))
Python::attach(|py| json_to_py(py, value))
})
}
type MarshaledMessagesInputs = (Value, Option<Map<String, Value>>, Option<Duration>);
fn marshal_messages_inputs(
py: Python<'_>,
body: Py<PyAny>,
extra_headers: Option<Py<PyAny>>,
timeout_seconds: Option<f64>,
) -> PyResult<MarshaledMessagesInputs> {
let body = py_to_json(py, body.bind(py))?;
if !body.is_object() {
return Err(PyValueError::new_err("body must be a dict"));
}
let extra_headers = match extra_headers {
Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?),
None => None,
};
Ok((body, extra_headers, optional_timeout(timeout_seconds)))
}
#[pyfunction]
#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))]
#[allow(clippy::too_many_arguments)]
fn messages(
py: Python<'_>,
model: String,
body: Py<PyAny>,
api_key: Option<String>,
api_base: Option<String>,
custom_llm_provider: Option<String>,
extra_headers: Option<Py<PyAny>>,
timeout_seconds: Option<f64>,
) -> PyResult<Py<PyAny>> {
let (body, extra_headers, timeout) =
marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?;
let result = gil::release_gil(py, || {
pyo3_async_runtimes::tokio::get_runtime().block_on(run_messages(MessagesRequest {
model: &model,
body,
api_key: api_key.as_deref(),
api_base: api_base.as_deref(),
custom_llm_provider: custom_llm_provider.as_deref(),
extra_headers,
timeout,
}))
});
match result {
Ok(value) => json_to_py(py, value),
Err(err) => Err(core_error_to_pyerr(err)),
}
}
#[pyfunction]
#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))]
#[allow(clippy::too_many_arguments)]
fn amessages(
py: Python<'_>,
model: String,
body: Py<PyAny>,
api_key: Option<String>,
api_base: Option<String>,
custom_llm_provider: Option<String>,
extra_headers: Option<Py<PyAny>>,
timeout_seconds: Option<f64>,
) -> PyResult<Bound<'_, PyAny>> {
let (body, extra_headers, timeout) =
marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?;
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let value = run_messages(MessagesRequest {
model: &model,
body,
api_key: api_key.as_deref(),
api_base: api_base.as_deref(),
custom_llm_provider: custom_llm_provider.as_deref(),
extra_headers,
timeout,
})
.await
.map_err(core_error_to_pyerr)?;
Python::attach(|py| json_to_py(py, value))
})
}
@ -182,6 +269,8 @@ fn gil_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add_function(wrap_pyfunction!(ocr, module)?)?;
module.add_function(wrap_pyfunction!(aocr, module)?)?;
module.add_function(wrap_pyfunction!(messages, module)?)?;
module.add_function(wrap_pyfunction!(amessages, module)?)?;
module.add_function(wrap_pyfunction!(gil_stats, module)?)?;
Ok(())
}

5
litellm/__init__.py
View file

@ -316,6 +316,11 @@ disable_token_counter: bool = False
disable_add_transform_inline_image_block: bool = False
disable_add_user_agent_to_request_tags: bool = False
disable_anthropic_gemini_context_caching_transform: bool = False
enable_anthropic_prompt_caching: bool = os.getenv("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", "false").lower() == "true"
_anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL")
anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (
"1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None
)
disable_vertex_batch_output_transformation: bool = False
extra_spend_tag_headers: Optional[List[str]] = None
in_memory_llm_clients_cache: "LLMClientCache"

23
litellm/constants.py
View file

@ -2,7 +2,7 @@ import os
import sys
from typing import List, Literal, Optional
from litellm.litellm_core_utils.env_utils import get_env_int
from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_or_none
DEFAULT_HEALTH_CHECK_PROMPT = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
AZURE_DEFAULT_RESPONSES_API_VERSION = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
@ -269,9 +269,18 @@ TOOL_POLICY_CACHE_TTL_SECONDS = int(os.getenv("TOOL_POLICY_CACHE_TTL_SECONDS", 6
MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8)))
MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int(os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000))
###############################################################################################
MINIMUM_PROMPT_CACHE_TOKEN_COUNT = int(
os.getenv("MINIMUM_PROMPT_CACHE_TOKEN_COUNT", 1024)
) # minimum number of tokens to cache a prompt by Anthropic
# Providers will not cache a prefix below a minimum size. That minimum is per-model, not global:
# Anthropic's ranges from 512 to 4096 depending on the model, and can differ per platform for the
# same model. The real minimum is resolved from `prompt_cache_min_tokens` in the model cost map;
# this value is only the fallback for models the cost map has no entry for, and doubles as a global
# escape hatch when `MINIMUM_PROMPT_CACHE_TOKEN_COUNT` is explicitly set.
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE: int | None = get_env_int_or_none("MINIMUM_PROMPT_CACHE_TOKEN_COUNT")
DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT = 1024
MINIMUM_PROMPT_CACHE_TOKEN_COUNT = (
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE
if MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE is not None
else DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT
)
DEFAULT_TRIM_RATIO = float(
os.getenv("DEFAULT_TRIM_RATIO", 0.75)
) # default ratio of tokens to trim from the end of a prompt
@ -1513,6 +1522,12 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
"cost_discount_config",
"cost_margin_config",
"budget_exceeded_throttle_percentage",
# Every field editable from the Admin UI (proxy_server._GENERAL_SETTINGS_UI_LITELLM_FIELDS)
# must be listed here so a DB write from one worker overrides the live litellm attribute on
# the others when config reloads; otherwise peer workers stay on their startup value.
# test_general_settings_ui_fields_are_db_overridable enforces that pairing.
"enable_anthropic_prompt_caching",
"anthropic_prompt_caching_ttl",
]
SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))

12
litellm/google_genai/main.py
View file

@ -17,6 +17,7 @@ from litellm.llms.base_llm.google_genai.transformation import (
)
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CallTypes
from litellm.utils import ProviderConfigManager, client
if TYPE_CHECKING:
@ -39,6 +40,11 @@ base_llm_http_handler = BaseLLMHTTPHandler()
#################################################
def _mark_async_entrypoint(logging_obj: LiteLLMLoggingObj | None, marker: str, is_async: bool) -> None:
if logging_obj is not None:
logging_obj.model_call_details.setdefault("litellm_params", {})[marker] = is_async
class GenerateContentSetupResult(BaseModel):
"""Internal Type - Result of setting up a generate content call"""
@ -315,6 +321,8 @@ def generate_content(
try:
_is_async = kwargs.pop("agenerate_content", False)
_mark_async_entrypoint(kwargs.get("litellm_logging_obj"), CallTypes.agenerate_content.value, _is_async)
# Handle generationConfig parameter from kwargs for backward compatibility
if "generationConfig" in kwargs and config is None:
config = kwargs.pop("generationConfig")
@ -403,6 +411,8 @@ async def agenerate_content_stream(
try:
kwargs["agenerate_content_stream"] = True
_mark_async_entrypoint(kwargs.get("litellm_logging_obj"), CallTypes.agenerate_content_stream.value, True)
# Handle generationConfig parameter from kwargs for backward compatibility
if "generationConfig" in kwargs and config is None:
config = kwargs.pop("generationConfig")
@ -497,6 +507,8 @@ def generate_content_stream(
# Remove any async-related flags since this is the sync function
_is_async = kwargs.pop("agenerate_content_stream", False)
_mark_async_entrypoint(kwargs.get("litellm_logging_obj"), CallTypes.agenerate_content_stream.value, _is_async)
# Handle generationConfig parameter from kwargs for backward compatibility
if "generationConfig" in kwargs and config is None:
config = kwargs.pop("generationConfig")

132
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -296,18 +296,148 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return processed_messages, processed_system, remaining_points
@staticmethod
def _default_control() -> ChatCompletionCachedContent:
"""Build the cache_control block for auto-injected breakpoints.
Defaults to Anthropic's 5-minute ephemeral cache; honors the optional
``litellm.anthropic_prompt_caching_ttl`` override ("5m" or "1h").
"""
import litellm
ttl = litellm.anthropic_prompt_caching_ttl
if ttl == "5m" or ttl == "1h":
return ChatCompletionCachedContent(type="ephemeral", ttl=ttl)
return ChatCompletionCachedContent(type="ephemeral")
@staticmethod
def _request_has_cache_control(
messages: list[AllMessageValues],
system: str | list | None,
tools: list | None = None,
) -> bool:
"""Return True if the request already carries any client-supplied cache_control.
When the client (e.g. Claude Code) already marks its own breakpoints we
stand down entirely rather than add more, per the auto-caching contract.
Tools count: they are a breakpoint the client can mark, they count toward
the provider's four-block limit, and caching only the tool definitions is
a common pattern, so injecting alongside them can exceed the cap.
"""
if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages):
return True
if isinstance(system, list):
if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system):
return True
if tools is not None:
return any(isinstance(tool, dict) and tool.get("cache_control") is not None for tool in tools)
return False
@staticmethod
def get_default_injection_points(
messages: list[AllMessageValues],
system: str | list | None,
model: str,
custom_llm_provider: str | None,
tools: list | None = None,
) -> list[CacheControlInjectionPoint]:
"""Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on.
Caches the system prompt and the trailing turn, so the stable prefix
(system + tools + history) is reused while the breakpoint advances with
the conversation. Returns [] (stand down) when the flag is off, the
provider does not consume cache_control breakpoints (only anthropic /
bedrock do), the model lacks prompt-caching support, or the request
already carries client-supplied cache_control.
"""
import litellm
if litellm.enable_anthropic_prompt_caching is not True:
return []
provider = custom_llm_provider
if provider is None:
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_llm_provider,
)
try:
_, provider, _, _ = get_llm_provider(model=model)
except Exception: # noqa: BLE001 # unroutable model must never block the call, just skip auto-caching
return []
if provider not in ("anthropic", "bedrock"):
return []
from litellm.utils import supports_prompt_caching
if not supports_prompt_caching(model=model, custom_llm_provider=provider):
return []
if AnthropicCacheControlHook._request_has_cache_control(messages, system, tools):
return []
control = AnthropicCacheControlHook._default_control()
points: list[CacheControlInjectionPoint] = [
CacheControlMessageInjectionPoint(location="message", role="system", index=None, control=control),
CacheControlMessageInjectionPoint(location="message", role=None, index=-1, control=control),
]
return points
@staticmethod
def maybe_seed_default_injection_points(
non_default_params: dict[str, Any],
messages: list[AllMessageValues],
model: str,
custom_llm_provider: str | None,
tools: list | None = None,
) -> None:
"""For /chat/completions: add default injection points to the request params.
No-op when injection points are already configured (explicit config wins).
Seeding the param lets the existing prompt-management gate and the
AnthropicCacheControlHook run unchanged.
"""
if non_default_params.get("cache_control_injection_points"):
return
points = AnthropicCacheControlHook.get_default_injection_points(
messages=messages,
system=None,
model=model,
custom_llm_provider=custom_llm_provider,
tools=tools,
)
if points:
non_default_params["cache_control_injection_points"] = points
@staticmethod
def maybe_inject_cache_control(
messages: List[Dict],
system: str | list | None,
kwargs: Dict[str, Any],
model: str | None = None,
custom_llm_provider: str | None = None,
tools: list[dict] | None = None,
) -> Tuple[List[Dict], str | list | None]:
"""Extract cache_control_injection_points from kwargs and apply if present.
When none are configured but ``litellm.enable_anthropic_prompt_caching``
is on, synthesize default breakpoints for the native /v1/messages path.
Pops the key from kwargs; if remaining (non-message) points exist they
are written back so downstream transforms can handle them.
"""
injection_points = kwargs.pop("cache_control_injection_points", None)
configured = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list
list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None)
)
injection_points: list[CacheControlInjectionPoint] = configured or []
if not injection_points and model is not None:
injection_points = AnthropicCacheControlHook.get_default_injection_points(
messages=cast(list[AllMessageValues], messages), # cast-ok: Anthropic-shaped dicts from v1/messages
system=system,
tools=tools,
model=model,
custom_llm_provider=custom_llm_provider,
)
if not injection_points:
return messages, system

42
litellm/integrations/compression_interception/handler.py
View file

@ -14,6 +14,7 @@ from litellm.compression import compress
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.integrations.compression_interception import (
CompressionInterceptionConfig,
CompressionSavingsMetadata,
)
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
@ -25,6 +26,41 @@ LITELLM_CONTENT_RETRIEVE_TOOL_NAME = "litellm_content_retrieve"
_CACHE_TTL_SECONDS = 15 * 60
def _compression_savings_from_counts(
original_tokens: object, compressed_tokens: object
) -> CompressionSavingsMetadata | None:
if isinstance(original_tokens, bool) or not isinstance(original_tokens, int):
return None
if isinstance(compressed_tokens, bool) or not isinstance(compressed_tokens, int):
return None
if compressed_tokens < 0 or original_tokens < compressed_tokens:
return None
return CompressionSavingsMetadata(
tokens_before=original_tokens,
tokens_after=compressed_tokens,
tokens_saved=original_tokens - compressed_tokens,
source="compression_interception",
)
def _record_compression_savings(kwargs: dict[str, object], savings: CompressionSavingsMetadata) -> None:
"""
Attach savings to the request's litellm metadata so they land in the
SpendLog row's metadata JSON under ``compression_savings``.
``/v1/messages`` requests carry proxy metadata under ``litellm_metadata``
(the ``metadata`` key is Anthropic's own API field). The existing dict is
updated in place because the proxy and the logging object hold references
to the same object; replacing it would orphan writes made through those
references.
"""
existing = kwargs.get("litellm_metadata")
if isinstance(existing, dict):
existing["compression_savings"] = savings
return
kwargs["litellm_metadata"] = {"compression_savings": savings}
class CompressionInterceptionLogger(CustomLogger):
"""
CustomLogger that implements transparent prompt compression + retrieval loops.
@ -130,6 +166,12 @@ class CompressionInterceptionLogger(CustomLogger):
call_id = str(uuid.uuid4())
kwargs["litellm_call_id"] = call_id
self._compression_cache_by_call_id[call_id] = (cache, time.time())
savings = _compression_savings_from_counts(
original_tokens=compressed.get("original_tokens"),
compressed_tokens=compressed.get("compressed_tokens"),
)
if savings is not None:
_record_compression_savings(kwargs=kwargs, savings=savings)
verbose_logger.debug(
"CompressionInterception: compressed request [call_id=%s original=%d compressed=%d cached_keys=%d]",
call_id,

50
litellm/integrations/opik/utils.py
View file

@ -1,40 +1,38 @@
import configparser
import os
import time
import uuid
from typing import Any, Dict, Final, List, Optional, Tuple
CONFIG_FILE_PATH_DEFAULT: Final[str] = "~/.opik.config"
def create_uuid7():
ns = time.time_ns()
last = [0, 0, 0, 0]
def create_uuid7() -> str:
"""Generate an RFC 9562 conformant UUIDv7 string.
# Simple uuid7 implementation
sixteen_secs = 16_000_000_000
t1, rest1 = divmod(ns, sixteen_secs)
t2, rest2 = divmod(rest1 << 16, sixteen_secs)
t3, _ = divmod(rest2 << 12, sixteen_secs)
t3 |= 7 << 12 # Put uuid version in top 4 bits, which are 0 in t3
The top 48 bits encode the Unix timestamp in milliseconds. Opik's backend
validates this embedded timestamp on ingestion (it must fall within a window
around "now"), so the encoding has to be correct or trace/span batches are
rejected with HTTP 400. Implemented with the standard library only, so no
extra dependency is added to litellm. See ``opik.id_helpers`` for the
reference implementation.
"""
unix_ts_ms = int(time.time() * 1000)
# The next two bytes are an int (t4) with two bits for
# the variant 2 and a 14 bit sequence counter which increments
# if the time is unchanged.
if t1 == last[0] and t2 == last[1] and t3 == last[2]:
# Stop the seq counter wrapping past 0x3FFF.
# This won't happen in practice, but if it does,
# uuids after the 16383rd with that same timestamp
# will not longer be correctly ordered but
# are still unique due to the 6 random bytes.
if last[3] < 0x3FFF:
last[3] += 1
else:
last[:] = (t1, t2, t3, 0)
t4 = (2 << 14) | last[3] # Put variant 0b10 in top two bits
# Fill the 16-byte buffer with random data, then overwrite the structured
# parts (timestamp, version, variant) defined by the UUIDv7 layout.
uuid_bytes = bytearray(os.urandom(16))
# Six random bytes for the lower part of the uuid
rand = os.urandom(6)
return f"{t1:>08x}-{t2:>04x}-{t3:>04x}-{t4:>04x}-{rand.hex()}"
# First 48 bits (6 bytes): Unix timestamp in milliseconds.
uuid_bytes[0:6] = unix_ts_ms.to_bytes(6, byteorder="big")
# Version 7 in the top 4 bits of byte 6.
uuid_bytes[6] = 0x70 | (uuid_bytes[6] & 0x0F)
# Variant 0b10 in the top 2 bits of byte 8.
uuid_bytes[8] = 0x80 | (uuid_bytes[8] & 0x3F)
return str(uuid.UUID(bytes=bytes(uuid_bytes)))
def _read_opik_config_file() -> Dict[str, str]:

55
litellm/integrations/otel/emitter.py
View file

@ -72,6 +72,42 @@ def _stamp_litellm_error_attributes(span: Span, error: SpanError) -> None:
span.set_attribute(LiteLLMError.LLM_PROVIDER, error.llm_provider)
def stamp_error(
span: Span,
error: SpanError,
*,
record_event: bool = True,
set_status: bool = True,
) -> tuple[str, str] | None:
"""Stamp the full v2 error attribute set on ``span`` and return the resolved
``(error_type, message)`` pair, or ``None`` when the error carries neither a
type nor a message.
Shared by the LLM-call span (``finish_span``) and the proxy-level failure
spans (the FastAPI SERVER span and the ``auth`` phase span) so every v2 error
span carries identical keys. The semconv ``exception`` event rides alongside
the attributes so backends that map unknown string attrs to a truncated
``keyword`` (e.g. Elasticsearch's 1024-char ``ignore_above``) still see the
full untruncated message on the recognized event field. ``record_event`` and
``set_status`` are opt-outs for callers whose span lifecycle (``use_span``) or
owner (the FastAPI instrumentor) already records the event or the status.
"""
if not (error.error_type or error.message):
return None
error_type = error.error_type or "error"
message = error.message or error.error_type or "error"
_stamp_otel_error_attributes(span, error_type, message)
_stamp_litellm_error_attributes(span, error)
if set_status:
span.set_status(Status(StatusCode.ERROR, message))
if record_event:
span.add_event(
ExceptionEvent.NAME,
{ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},
)
return error_type, message
class SpanEmitter:
def __init__(
self,
@ -249,21 +285,10 @@ class SpanEmitter:
)
else None
)
if error and (error.error_type or error.message):
error_type = error.error_type or "error"
message = error.message or error.error_type or "error"
_stamp_otel_error_attributes(span, error_type, message)
_stamp_litellm_error_attributes(span, error)
span.set_status(Status(StatusCode.ERROR, message))
# Also emit the semconv ``exception`` event so backends that
# dynamic-map unknown string span attrs to ``keyword`` (e.g.
# Elasticsearch with a 1024-char ``ignore_above``) still see the
# full untruncated message on the recognized event field.
span.add_event(
ExceptionEvent.NAME,
{ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},
)
if self._event_recorder is not None and role is SpanRole.LLM_CALL:
if error:
stamped = stamp_error(span, error)
if stamped is not None and self._event_recorder is not None and role is SpanRole.LLM_CALL:
error_type, message = stamped
self._event_recorder.record_operation_exception(
span_context=span.get_span_context(),
error_type=error_type,

79
litellm/integrations/otel/logger.py
View file

@ -24,7 +24,7 @@ from litellm.integrations.otel.plumbing.context import (
set_request_baggage,
set_request_root_span,
)
from litellm.integrations.otel.emitter import SpanEmitter
from litellm.integrations.otel.emitter import SpanEmitter, stamp_error
from litellm.integrations.otel.mappers import resolve_mappers
from litellm.integrations.otel.model.metadata import (
LLMCallEvent,
@ -60,6 +60,7 @@ from litellm.integrations.otel.model.utils import to_ns
if TYPE_CHECKING:
from litellm.integrations.otel.model.destination import OtelDestination
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import (
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
@ -67,6 +68,33 @@ if TYPE_CHECKING:
LITELLM_TRACER_NAME = "litellm"
def _span_error_from_exception(
exception: "Exception | None",
*,
status_code: int | None = None,
traceback_str: str | None = None,
) -> SpanError:
"""A ``SpanError`` for a proxy-level failure that never produced a
``StandardLoggingPayload`` (auth / validation / malformed-body rejections),
mirroring ``_parse_error``'s field mapping so it stamps the same v2 keys a
failed LLM call does. ``status_code`` pins ``error.code`` to the real response
status, matching v1's SERVER-span behavior."""
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
info = StandardLoggingPayloadSetup.get_error_information(
original_exception=exception,
traceback_str=traceback_str,
)
return SpanError(
error_type=info.get("error_class") or info.get("error_code") or None,
message=info.get("error_message") or None,
code=str(status_code) if status_code is not None else (info.get("error_code") or None),
stack_trace=info.get("traceback") or None,
llm_provider=info.get("llm_provider") or None,
)
# Any callback whose class belongs to one of these modules is "the OTel
# callback" for proxy-global-registration purposes.
_OTEL_MODULES = (
@ -645,7 +673,12 @@ class OpenTelemetryV2(CustomLogger):
def start_phase_span(self, name: str) -> "Iterator[Span]":
span = self._emitter.start_span(SpanRole.SERVICE, name)
with use_span(span, end_on_exit=True):
yield span
try:
yield span
except Exception as exc:
if is_recordable_span(span):
stamp_error(span, _span_error_from_exception(exc), record_event=False, set_status=False)
raise
async def async_pre_call_hook(
self,
@ -660,6 +693,48 @@ class OpenTelemetryV2(CustomLogger):
)
return data
def record_error_attributes_on_span(
self,
span: "Span | None",
exception: "Exception | None",
status_code: int,
) -> None:
"""Stamp the v2 error.* attributes on the FastAPI-owned SERVER span for a
failure that dies before any LLM-call span exists (malformed body, auth /
validation rejection). Called from the proxy's global exception handler via
``_close_dangling_otel_server_span``. The instrumentor still owns the span's
status and lifecycle, so this only decorates it — never sets status, never
ends it — and emits no exception event, matching v1's SERVER-span behavior
and avoiding a duplicate of the event ``async_post_call_failure_hook`` or
the ``auth`` phase span already records."""
if span is None or not is_recordable_span(span):
return
stamp_error(
span,
_span_error_from_exception(exception, status_code=status_code),
record_event=False,
set_status=False,
)
async def async_post_call_failure_hook(
self,
request_data: dict,
original_exception: Exception,
user_api_key_dict: "UserAPIKeyAuth",
traceback_str: "str | None" = None,
) -> None:
"""Stamp error.* on the request's root SERVER span for a proxy-level
failure that never reached an LLM call (empty body rejected in the
endpoint, auth failure), so the failed request carries the same error keys
a failed LLM call does. v1's ``OpenTelemetry`` implemented this same hook;
v2 lost it when it stopped subclassing ``OpenTelemetry``, which is the
LIT-4179 regression for pre-call failures."""
span = request_root_span() or user_api_key_dict.parent_otel_span
if span is None or not is_recordable_span(span):
return None
stamp_error(span, _span_error_from_exception(original_exception, traceback_str=traceback_str))
return None
def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None:
# Emitted by the guardrail-recording code the moment a guardrail finishes,
# not from a post-call hook — that hook does not fire on every path (a

16
litellm/litellm_core_utils/env_utils.py
View file

@ -19,3 +19,19 @@ def get_env_int(env_var: str, default: int) -> int:
return int(raw)
except (ValueError, TypeError):
return default
def get_env_int_or_none(env_var: str) -> int | None:
"""Parse an environment variable as an integer, returning None when it is unset or unusable.
Use this instead of `get_env_int` when callers must distinguish "explicitly configured"
from "left at the default", for example when an override should take precedence over a
value resolved from somewhere else.
"""
raw = os.getenv(env_var)
if raw is None:
return None
try:
return int(raw.strip())
except (ValueError, TypeError):
return None

6
litellm/litellm_core_utils/litellm_logging.py
View file

@ -1453,6 +1453,9 @@ class Logging(LiteLLMLoggingBaseClass):
response_cost = litellm.response_cost_calculator(**response_cost_calculator_kwargs)
verbose_logger.debug(f"response_cost: {response_cost}")
additional_response_cost: object = self.model_call_details.get("additional_response_cost")
if isinstance(additional_response_cost, (int, float)) and additional_response_cost > 0:
return (response_cost or 0.0) + additional_response_cost
return response_cost
except Exception as e: # error calculating cost
debug_info = StandardLoggingModelCostFailureDebugInformation(
@ -1531,6 +1534,9 @@ class Logging(LiteLLMLoggingBaseClass):
and litellm_params.get(CallTypes.aimage_generation.value, False) is not True
and litellm_params.get(CallTypes.atranscription.value, False) is not True
and litellm_params.get(CallTypes.allm_passthrough_route.value, False) is not True
and litellm_params.get(CallTypes.aanthropic_messages.value, False) is not True
and litellm_params.get(CallTypes.agenerate_content.value, False) is not True
and litellm_params.get(CallTypes.agenerate_content_stream.value, False) is not True
)
def _is_assembled_stream_success(self, result=None) -> bool:

9
litellm/llms/anthropic/common_utils.py
View file

@ -906,16 +906,17 @@ def strip_advisor_blocks_from_messages(messages: List[Any], replace_with_text: b
def is_anthropic_invalid_thinking_signature_error(error_text: str) -> bool:
"""
Detect Anthropic 400 when encrypted thinking signatures in history do not match
the current deployment (e.g. user rotated API key or switched model endpoint).
Detect Anthropic 400 errors caused by missing or invalid thinking signatures.
Example API message:
Known error formats:
{"message":"messages.2.content.0.thinking.signature.str: Input should be a valid string"}
messages.N.content.M.thinking.signature.str: Input should be a valid string
messages.N.content.M: Invalid `signature` in `thinking` block
"""
if not error_text:
return False
lower = error_text.lower()
return "invalid" in lower and "signature" in lower and "thinking" in lower and "block" in lower
return "thinking" in lower and "signature" in lower and ("invalid" in lower or "valid string" in lower)
def strip_thinking_blocks_from_anthropic_messages(messages: List[Any]) -> List[Any]:

12
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -36,6 +36,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CallTypes
from litellm.utils import ProviderConfigManager, client
from ..utils import is_reasoning_auto_summary_enabled
@ -236,7 +237,9 @@ async def anthropic_messages(
AnthropicCacheControlHook,
)
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs)
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
)
original_stream = stream or kwargs.get("_websearch_interception_converted_stream", False)
@ -425,7 +428,9 @@ def anthropic_messages_handler(
AnthropicCacheControlHook,
)
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs)
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
)
metadata = validate_anthropic_api_metadata(metadata)
@ -463,6 +468,9 @@ def anthropic_messages_handler(
"model": original_model,
"custom_llm_provider": custom_llm_provider,
}
litellm_logging_obj.model_call_details.setdefault("litellm_params", {})[CallTypes.aanthropic_messages.value] = (
is_async
)
# Check if stream was converted for WebSearch interception
# This is set in the async wrapper above when stream=True is converted to stream=False

5
litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py
View file

@ -75,8 +75,9 @@ class AnthropicResponsesStreamWrapper:
# ---- message_start ----
if event_type == "response.created":
self._sent_message_start = True
self._chunk_queue.append(self._make_message_start())
if not self._sent_message_start:
self._sent_message_start = True
self._chunk_queue.append(self._make_message_start())
return
# ---- content_block_start for a new output message item ----

123
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -153,6 +153,9 @@ if TYPE_CHECKING:
from aiohttp import ClientSession
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
AnthropicMessagesStreamingResponse,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
@ -2091,6 +2094,37 @@ class BaseLLMHTTPHandler:
},
)
rust_messages_response = await self._maybe_rust_anthropic_messages(
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
stream=stream or False,
rust_stream_eligible=bool(stream) and not self._has_agentic_completion_hook(logging_obj),
model=model,
api_key=api_key,
api_base=api_base,
headers=headers,
request_body=request_body,
timeout=self._resolve_anthropic_messages_timeout(
litellm_params=litellm_params,
stream=stream or False,
custom_llm_provider=custom_llm_provider,
),
)
if rust_messages_response is not None:
if stream:
return self._rust_anthropic_messages_fake_stream(rust_messages_response)
return await self._finalize_anthropic_messages_response(
initial_response=rust_messages_response,
model=model,
messages=messages,
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
api_key=api_key,
kwargs=kwargs,
)
response = await self._async_post_anthropic_messages_with_http_error_retry(
async_httpx_client=async_httpx_client,
request_url=request_url,
@ -2165,6 +2199,31 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
)
return await self._finalize_anthropic_messages_response(
initial_response=initial_response,
model=model,
messages=messages,
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
api_key=api_key,
kwargs=kwargs,
)
async def _finalize_anthropic_messages_response(
self,
*,
initial_response: AnthropicMessagesResponse,
model: str,
messages: list[dict],
anthropic_messages_provider_config: BaseAnthropicMessagesConfig,
anthropic_messages_optional_request_params: dict,
logging_obj: LiteLLMLoggingObj,
custom_llm_provider: str,
api_key: str | None,
kwargs: dict,
) -> AnthropicMessagesResponse | AsyncIterator:
# Inject api_key into kwargs so follow-up calls in agentic hooks can
# authenticate. api_key is a named param here (not in kwargs), so
# _prepare_followup_kwargs would miss it otherwise.
@ -2188,6 +2247,70 @@ class BaseLLMHTTPHandler:
"anthropic_messages",
)
@staticmethod
async def _maybe_rust_anthropic_messages(
*,
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
stream: bool,
rust_stream_eligible: bool,
model: str,
api_key: str | None,
api_base: str | None,
headers: dict,
request_body: dict,
timeout: float | httpx.Timeout | None,
) -> AnthropicMessagesResponse | None:
if custom_llm_provider != "azure_ai" or litellm_params.get("rust") is not True:
return None
if stream and not rust_stream_eligible:
return None
from litellm.rust_bridge import messages as rust_messages_bridge
upstream_body = {key: value for key, value in request_body.items() if key != "stream"}
try:
rust_response = await rust_messages_bridge.amessages(
model=model,
body=upstream_body,
api_key=api_key,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
extra_headers=headers,
timeout=timeout,
)
except Exception as rust_error: # noqa: BLE001 # rollout-safety fallback: any Rust bridge failure must fall back to the Python path
verbose_logger.debug(
"Rust Anthropic messages bridge raised %s; falling back to Python path",
type(rust_error).__name__,
)
return None
if rust_response is None:
return None
response_obj = cast(AnthropicMessagesResponse, dict(rust_response))
response_obj["_hidden_params"] = {"additional_headers": {"x-litellm-rust": "true"}}
return response_obj
@staticmethod
def _rust_anthropic_messages_fake_stream(
rust_response: AnthropicMessagesResponse,
) -> "AnthropicMessagesStreamingResponse":
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
AnthropicMessagesStreamHiddenParams,
AnthropicMessagesStreamingResponse,
)
completion_stream = cast(AsyncIterator[bytes], FakeAnthropicMessagesStreamIterator(response=rust_response))
hidden_params = AnthropicMessagesStreamHiddenParams(additional_headers={"x-litellm-rust": "true"})
return AnthropicMessagesStreamingResponse(
completion_stream=completion_stream,
hidden_params=hidden_params,
)
def anthropic_messages_handler(
self,
model: str,

14
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -48,7 +48,7 @@ from ...openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
OpenAIGPTConfig,
)
from ..common_utils import FireworksAIException
from ..common_utils import FireworksAIMixin, FireworksAIException
def _extract_fireworks_hidden_params(payload: dict) -> dict:
@ -70,7 +70,7 @@ def _extract_fireworks_hidden_params(payload: dict) -> dict:
return {**top_level, **per_choice}
class FireworksAIConfig(OpenAIGPTConfig):
class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
"""
Reference: https://docs.fireworks.ai/api-reference/post-chatcompletions
@ -114,6 +114,16 @@ class FireworksAIConfig(OpenAIGPTConfig):
prompt_truncate_len: Optional[int] = None,
context_length_exceeded_behavior: Optional[Literal["error", "truncate"]] = None,
) -> None:
OpenAIGPTConfig.__init__(
self,
frequency_penalty=frequency_penalty,
max_tokens=max_tokens,
n=n,
stop=stop,
temperature=temperature,
top_p=top_p,
response_format=response_format,
)
locals_ = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:

24
litellm/llms/fireworks_ai/common_utils.py
View file

@ -12,6 +12,23 @@ class FireworksAIException(BaseLLMException):
pass
def get_fireworks_session_id(litellm_params: dict) -> str | None:
params = litellm_params
for key in ("litellm_session_id", "session_id"):
value = params.get(key)
if value:
return str(value)
metadata = params.get("metadata")
if isinstance(metadata, dict):
value = metadata.get("session_id")
if value:
return str(value)
value = params.get("litellm_trace_id")
if value:
return str(value)
return None
class FireworksAIMixin:
"""
Common Base Config functions across Fireworks AI Endpoints
@ -47,4 +64,9 @@ class FireworksAIMixin:
if api_key is None:
raise ValueError("FIREWORKS_API_KEY is not set")
return {"Authorization": "Bearer {}".format(api_key), **headers}
validated_headers = {"Authorization": "Bearer {}".format(api_key), **headers}
if not any(key.lower() == "x-session-affinity" for key in validated_headers):
session_id = get_fireworks_session_id(litellm_params)
if session_id:
validated_headers["x-session-affinity"] = session_id
return validated_headers

17
litellm/llms/fireworks_ai/cost_calculator.py
View file

@ -75,10 +75,23 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
model_info = get_model_info(model=base_model, custom_llm_provider="fireworks_ai")
## CALCULATE INPUT COST
prompt_tokens_details = usage.prompt_tokens_details
cached_tokens: int = (
prompt_tokens_details.cached_tokens
if prompt_tokens_details is not None and prompt_tokens_details.cached_tokens is not None
else 0
)
input_cost_per_token: float = model_info["input_cost_per_token"] or 0.0
cache_read_input_token_cost = model_info.get("cache_read_input_token_cost")
cache_read_cost_per_token: float = (
cache_read_input_token_cost if cache_read_input_token_cost is not None else input_cost_per_token
)
non_cached_prompt_tokens: int = max(usage.prompt_tokens - cached_tokens, 0)
prompt_cost: float = usage["prompt_tokens"] * model_info["input_cost_per_token"]
prompt_cost: float = non_cached_prompt_tokens * input_cost_per_token + cached_tokens * cache_read_cost_per_token
## CALCULATE OUTPUT COST
completion_cost = usage["completion_tokens"] * model_info["output_cost_per_token"]
output_cost_per_token: float = model_info["output_cost_per_token"] or 0.0
completion_cost: float = usage.completion_tokens * output_cost_per_token
return prompt_cost, completion_cost

47
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -1731,18 +1731,42 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"""
Check if the candidate token count is inclusive of the thinking token count
if prompttokencount + candidatesTokenCount == totalTokenCount, then the candidate token count is inclusive of the thinking token count
if promptTokenCount + candidatesTokenCount + toolUsePromptTokenCount == totalTokenCount, then the candidate token count is inclusive of the thinking token count
else the candidate token count is exclusive of the thinking token count
Addresses - https://github.com/BerriAI/litellm/pull/10141#discussion_r2052272035
"""
if usage_metadata.get("promptTokenCount", 0) + usage_metadata.get(
"candidatesTokenCount", 0
) == usage_metadata.get("totalTokenCount", 0):
return True
else:
non_thinking_tokens = (
usage_metadata.get("promptTokenCount", 0)
+ usage_metadata.get("candidatesTokenCount", 0)
+ usage_metadata.get("toolUsePromptTokenCount", 0)
)
return non_thinking_tokens == usage_metadata.get("totalTokenCount", 0)
@staticmethod
def _response_has_search_grounding(
completion_response: Union[GenerateContentResponseBody, BidiGenerateContentServerMessage],
) -> bool:
"""
Whether the response used Grounding with Google Search, detected via
groundingMetadata.webSearchQueries (an actual web search was performed).
Google bills grounding-with-Google-Search retrieved tokens separately (a per-request /
per-query search fee) and excludes them from input token billing, unlike URL context /
File Search / code execution whose tool-use tokens are charged at the input token rate.
URL context also emits groundingMetadata (with groundingChunks but no webSearchQueries),
so presence of groundingMetadata alone is not a sufficient signal.
See https://ai.google.dev/gemini-api/docs/pricing and
https://github.com/BerriAI/litellm/discussions/33198
"""
if "candidates" not in completion_response:
return False
for candidate in completion_response["candidates"] or []:
grounding_metadata, _, _, _ = VertexGeminiConfig._extract_candidate_metadata(candidate)
if VertexGeminiConfig._calculate_web_search_requests(grounding_metadata):
return True
return False
@staticmethod
def _calculate_usage(
@ -1888,12 +1912,21 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
response_tokens_details = CompletionTokensDetailsWrapper()
response_tokens_details.reasoning_tokens = reasoning_tokens
tool_use_prompt_tokens = usage_metadata.get("toolUsePromptTokenCount") or None
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cached_tokens,
audio_tokens=prompt_audio_tokens,
text_tokens=prompt_text_tokens,
image_tokens=prompt_image_tokens,
video_tokens=prompt_video_tokens,
tool_use_tokens=tool_use_prompt_tokens,
)
billable_tool_use_prompt_tokens = (
0
if VertexGeminiConfig._response_has_search_grounding(completion_response)
else (tool_use_prompt_tokens or 0)
)
completion_tokens = response_tokens or completion_response["usageMetadata"].get("candidatesTokenCount", 0)
@ -1901,7 +1934,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
completion_tokens = reasoning_tokens + completion_tokens
## GET USAGE ##
usage = Usage(
prompt_tokens=usage_metadata.get("promptTokenCount", 0),
prompt_tokens=usage_metadata.get("promptTokenCount", 0) + billable_tool_use_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=usage_metadata.get("totalTokenCount", 0),
prompt_tokens_details=prompt_tokens_details,

27
litellm/main.py
View file

@ -510,6 +510,20 @@ async def acompletion(
#########################################################
#########################################################
litellm_logging_obj = kwargs.get("litellm_logging_obj", None)
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
)
from litellm.types.llms.openai import AllMessageValues
AnthropicCacheControlHook.maybe_seed_default_injection_points(
non_default_params=kwargs,
messages=cast(list[AllMessageValues], messages), # cast-ok: acompletion types messages as a bare List
model=model,
custom_llm_provider=cast(Optional[str], custom_llm_provider), # cast-ok: read from untyped kwargs
tools=tools,
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
litellm_logging_obj.should_run_prompt_management_hooks(
prompt_id=kwargs.get("prompt_id", None),
@ -5055,6 +5069,19 @@ def completion( # type: ignore
litellm_params = {} # used to prevent unbound var errors
## PROMPT MANAGEMENT HOOKS ##
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
)
from litellm.types.llms.openai import AllMessageValues
AnthropicCacheControlHook.maybe_seed_default_injection_points(
non_default_params=non_default_params,
messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List
model=model,
custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
tools=tools,
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
litellm_logging_obj.should_run_prompt_management_hooks(
prompt_id=prompt_id, non_default_params=non_default_params

398
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

32
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -1220,11 +1220,29 @@ class MCPRequestHandler:
global_mcp_server_manager,
)
key_tools = (
key_direct_tools = (
global_mcp_server_manager.expand_tool_permissions(key_obj_perm.mcp_tool_permissions).get(server_id)
if key_obj_perm
else None
)
# Tools granted through the key's toolsets restrict this server exactly
# as direct tool permissions do; union with any direct grants so the
# tool-level check sees the key's full effective tool scope
key_toolset_ids = (key_obj_perm.mcp_toolsets or []) if key_obj_perm else []
key_toolset_tools = (
(await global_mcp_server_manager.resolve_toolset_tool_permissions(toolset_ids=key_toolset_ids)).get(
server_id
)
if key_toolset_ids
else None
)
key_tools = (
list(set(key_direct_tools or []) | set(key_toolset_tools or []))
if key_direct_tools is not None or key_toolset_tools is not None
else None
)
team_tools = (
global_mcp_server_manager.expand_tool_permissions(team_obj_perm.mcp_tool_permissions).get(server_id)
if team_obj_perm
@ -1430,8 +1448,18 @@ class MCPRequestHandler:
global_mcp_server_manager.expand_tool_permissions(key_object_permission.mcp_tool_permissions).keys()
)
# servers referenced by the key's toolset grants are part of the key's
# scope on every path (list, call, REST), subject to the same team/org
# ceilings as any other key-level grant
toolset_ids = key_object_permission.mcp_toolsets or []
toolset_servers = (
list((await global_mcp_server_manager.resolve_toolset_tool_permissions(toolset_ids=toolset_ids)).keys())
if toolset_ids
else []
)
# Combine all lists
all_servers = direct_mcp_servers + access_group_servers + tool_perm_servers
all_servers = direct_mcp_servers + access_group_servers + tool_perm_servers + toolset_servers
return list(set(all_servers))
except Exception as e:
verbose_logger.warning(f"Failed to get allowed MCP servers for key: {str(e)}")

89
litellm/proxy/_experimental/mcp_server/db.py
View file

@ -33,6 +33,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
from litellm.proxy.utils import PrismaClient
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.table_repositories import (
MCPServerOAuthClientRepository,
MCPServerRepository,
MCPUserCredentialsRepository,
)
@ -374,6 +375,12 @@ def encrypt_credentials(credentials: MCPCredentials, encryption_key: Optional[st
value=client_secret,
new_encryption_key=encryption_key,
)
client_private_key = credentials.get("client_private_key")
if client_private_key is not None:
credentials["client_private_key"] = encrypt_value_helper(
value=client_private_key,
new_encryption_key=encryption_key,
)
# AWS SigV4 credential fields
aws_access_key_id = credentials.get("aws_access_key_id")
if aws_access_key_id is not None:
@ -405,6 +412,7 @@ def decrypt_credentials(
"auth_value",
"client_id",
"client_secret",
"client_private_key",
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
@ -639,6 +647,7 @@ async def delete_mcp_server(
for model, label in (
(prisma_client.db.litellm_mcpusercredentials, "credential"),
(prisma_client.db.litellm_mcpuserenvvars, "env var"),
(prisma_client.db.litellm_mcpserveroauthclient, "OAuth client"),
):
try:
await model.delete_many(where={"server_id": server_id})
@ -823,26 +832,66 @@ async def update_mcp_server(
return updated_mcp_server
async def rotate_mcp_server_credentials_master_key(prisma_client: PrismaClient, touched_by: str, new_master_key: str):
async def get_mcp_server_oauth_client_credentials(prisma_client: PrismaClient, server_id: str) -> object | None:
"""Read the persisted (encrypted) DCR OAuth client blob for a server from the
server-scoped store, or None. Config.yaml-declared servers have no
LiteLLM_MCPServerTable row, so their dynamically registered client lives here keyed
by server_id. The returned value is the raw credentials blob for
``_get_persisted_dcr_credentials`` to parse."""
row = await MCPServerOAuthClientRepository(prisma_client).table.find_unique(where={"server_id": server_id})
if row is None:
return None
return row.credentials
async def upsert_mcp_server_oauth_client_credentials(
prisma_client: PrismaClient, server_id: str, credentials: MCPCredentials
) -> None:
"""Persist a server's dynamically registered OAuth client (RFC 7591 DCR) in the
server-scoped store keyed by server_id, independent of any LiteLLM_MCPServerTable row.
client_id/client_secret are encrypted at rest with the same salt key used for the
server row's credentials blob, so ``_apply_persisted_dcr_credentials`` decrypts them the
same way regardless of which store a server's client came from."""
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
encrypted = encrypt_credentials(credentials=dict(credentials), encryption_key=_get_salt_key())
blob = safe_dumps(encrypted)
await MCPServerOAuthClientRepository(prisma_client).table.upsert(
where={"server_id": server_id},
data={
"create": {"server_id": server_id, "credentials": blob},
"update": {"credentials": blob},
},
)
def _reencrypt_mcp_credentials_blob(credentials: object, new_master_key: str) -> str | None:
"""Decrypt an at-rest MCP credentials blob with the current key and re-encrypt it under
new_master_key, returning the serialized blob or None when there is nothing to rotate. Shared by
every table that stores an encrypted MCP credentials blob so a master-key rotation covers them
uniformly and cannot silently skip one."""
if not credentials:
return None
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps # noqa: PLC0415 # avoids circular import
creds_dict = json.loads(credentials) if isinstance(credentials, str) else dict(credentials)
decrypted = decrypt_credentials(credentials=cast(MCPCredentials, creds_dict))
encrypted = encrypt_credentials(credentials=decrypted, encryption_key=new_master_key)
return safe_dumps(encrypted)
async def rotate_mcp_server_credentials_master_key(prisma_client: PrismaClient, touched_by: str, new_master_key: str):
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps # noqa: PLC0415 # avoids circular import
mcp_servers = await MCPServerRepository(prisma_client).table.find_many()
updated = 0
for mcp_server in mcp_servers:
update_data: Dict[str, Any] = {}
credentials = mcp_server.credentials
if credentials:
# Decrypt with current key first, then re-encrypt with new key
decrypted_credentials = decrypt_credentials(
credentials=cast(MCPCredentials, dict(credentials)),
)
encrypted_credentials = encrypt_credentials(
credentials=decrypted_credentials,
encryption_key=new_master_key,
)
update_data["credentials"] = safe_dumps(encrypted_credentials)
rotated_credentials = _reencrypt_mcp_credentials_blob(mcp_server.credentials, new_master_key)
if rotated_credentials is not None:
update_data["credentials"] = rotated_credentials
rotated_env_vars = _reencrypt_global_env_var_values(mcp_server.env_vars, new_master_key)
if rotated_env_vars is not None:
@ -857,9 +906,23 @@ async def rotate_mcp_server_credentials_master_key(prisma_client: PrismaClient,
data=update_data,
)
updated += 1
oauth_clients = await MCPServerOAuthClientRepository(prisma_client).table.find_many()
oauth_updated = 0
for oauth_client in oauth_clients:
rotated_credentials = _reencrypt_mcp_credentials_blob(oauth_client.credentials, new_master_key)
if rotated_credentials is None:
continue
await MCPServerOAuthClientRepository(prisma_client).table.update(
where={"server_id": oauth_client.server_id},
data={"credentials": rotated_credentials},
)
oauth_updated += 1
verbose_proxy_logger.info(
"rotate_mcp_server_credentials_master_key: rotated %d MCP server row(s)",
"rotate_mcp_server_credentials_master_key: rotated %d MCP server row(s) and %d OAuth-client row(s)",
updated,
oauth_updated,
)

144
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -971,43 +971,93 @@ def _apply_persisted_dcr_credentials(mcp_server: MCPServer, credentials: _Persis
return True
async def _get_persisted_mcp_server_with_dcr_client_id(
mcp_server: MCPServer,
) -> Optional[tuple["LiteLLM_MCPServerTable", _PersistedDcrCredentials]]:
from litellm.proxy._experimental.mcp_server.db import get_mcp_server # noqa: PLC0415
from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415
async def _load_store_dcr_credentials(mcp_server: MCPServer) -> _PersistedDcrCredentials | None:
"""DCR client persisted in the server-scoped OAuth-client store for a config-declared server
(which has no LiteLLM_MCPServerTable row). Returns None when the store has no usable client_id
or the DB is unreachable."""
from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # avoids circular import
get_mcp_server_oauth_client_credentials,
)
from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 # avoids circular import
try:
prisma_client = get_prisma_client_or_throw("Database not connected. Cannot read MCP OAuth client registration.")
persisted_mcp_server = await get_mcp_server(
prisma_client=prisma_client,
server_id=mcp_server.server_id,
blob = await get_mcp_server_oauth_client_credentials(
prisma_client=prisma_client, server_id=mcp_server.server_id
)
except Exception as exc: # noqa: BLE001
except Exception as exc: # noqa: BLE001 # best-effort read; DB may be unreachable
verbose_logger.debug(
"register_client_with_server: failed to read persisted DCR client registration for server_id=%s: %s",
"register_client_with_server: failed to read stored DCR client for server_id=%s: %s",
mcp_server.server_id,
exc,
)
return None
if persisted_mcp_server is None:
return None
credentials = _get_persisted_dcr_credentials(persisted_mcp_server.credentials)
credentials = _get_persisted_dcr_credentials(blob)
if credentials is None or not credentials.client_id:
return None
return credentials
return persisted_mcp_server, credentials
async def hydrate_config_server_dcr_client(mcp_server: MCPServer) -> bool:
"""Overlay a config-declared server's persisted DCR client onto its in-memory object so token
refresh can authenticate. Config.yaml servers have no LiteLLM_MCPServerTable row, so their
minted client lives in the server-scoped store; without this overlay the in-memory server
carries no client_id after a restart. An explicit client_id set in config.yaml wins and is never
overwritten by a persisted store client."""
if mcp_server.client_id:
return False
credentials = await _load_store_dcr_credentials(mcp_server)
if credentials is None:
return False
return _apply_persisted_dcr_credentials(mcp_server, credentials)
async def _resolve_persisted_dcr_client(
mcp_server: MCPServer,
) -> tuple[Optional["LiteLLM_MCPServerTable"], _PersistedDcrCredentials | None]:
"""Resolve a server's persisted DCR client using the same two-level rule the write path uses, so
read and write always agree. First, whether the server HAS a LiteLLM_MCPServerTable row: a row is
always resolved to that row and the store is never consulted for a server that has a row, so a
caller-chosen server_id colliding with a config-declared server cannot inherit that config
server's client, and a row that exists but carries no usable client_id yields (row, None) rather
than a store fallback. Second, among rowless servers: a config-declared server keeps its client in
the server-scoped store, while a rowless non-config server is a throwaway temp/session server with
no persisted client. Returns (row_or_None, credentials_or_None); the row is only needed by the
reuse path to refresh the registry for a DB-declared server."""
from litellm.proxy._experimental.mcp_server.db import get_mcp_server # noqa: PLC0415 # avoids circular import
from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids circular import
global_mcp_server_manager,
)
from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 # avoids circular import
try:
prisma_client = get_prisma_client_or_throw("Database not connected. Cannot read MCP OAuth client registration.")
row = await get_mcp_server(prisma_client=prisma_client, server_id=mcp_server.server_id)
except Exception as exc: # noqa: BLE001 # best-effort read; DB may be unreachable
verbose_logger.debug(
"register_client_with_server: failed to read persisted DCR client for server_id=%s: %s",
mcp_server.server_id,
exc,
)
return None, None
if row is not None:
credentials = _get_persisted_dcr_credentials(row.credentials)
if credentials is not None and credentials.client_id:
return row, credentials
return row, None
if global_mcp_server_manager.is_config_declared_server(mcp_server.server_id):
return None, await _load_store_dcr_credentials(mcp_server)
return None, None
async def _reuse_persisted_dcr_client_if_available(
mcp_server: MCPServer, current_redirect_uri: Optional[str] = None
) -> bool:
persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server)
if persisted is None:
persisted_mcp_server, credentials = await _resolve_persisted_dcr_client(mcp_server)
if credentials is None:
return False
persisted_mcp_server, credentials = persisted
if current_redirect_uri is not None and _redirect_uri_not_registered(credentials, current_redirect_uri):
verbose_logger.debug(
"register_client_with_server: not reusing persisted DCR client for server_id=%s; its registered "
@ -1021,18 +1071,19 @@ async def _reuse_persisted_dcr_client_if_available(
if not _apply_persisted_dcr_credentials(mcp_server, credentials):
return False
from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415
global_mcp_server_manager,
)
try:
await global_mcp_server_manager.update_server(persisted_mcp_server)
except Exception as exc: # noqa: BLE001
verbose_logger.warning(
"register_client_with_server: failed to refresh persisted DCR client registration for server_id=%s: %s",
mcp_server.server_id,
exc,
if persisted_mcp_server is not None:
from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids circular import
global_mcp_server_manager,
)
try:
await global_mcp_server_manager.update_server(persisted_mcp_server)
except Exception as exc: # noqa: BLE001 # best-effort registry refresh
verbose_logger.warning(
"register_client_with_server: failed to refresh persisted DCR client registration for server_id=%s: %s",
mcp_server.server_id,
exc,
)
return bool(mcp_server.client_id)
@ -1044,10 +1095,9 @@ async def _persisted_dcr_redirect_uri_is_stale(mcp_server: MCPServer, current_re
otherwise short-circuits registration before any redirect check can run. Servers
without a persisted DCR recording (admin-configured client_id, or registered before
redirect_uris were recorded) are never reported stale."""
persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server)
if persisted is None:
_, credentials = await _resolve_persisted_dcr_client(mcp_server)
if credentials is None:
return False
_, credentials = persisted
if not _redirect_uri_not_registered(credentials, current_redirect_uri):
return False
verbose_logger.warning(
@ -1067,7 +1117,10 @@ DcrRegistrationPersistenceResult = Literal["persisted", "reused", "skipped", "fa
async def _persist_dcr_client_registration(
mcp_server: MCPServer, registration_response: object, current_redirect_uri: str
) -> DcrRegistrationPersistenceResult:
"""Persist the dynamically registered OAuth client (RFC 7591) onto the MCP server row.
"""Persist the dynamically registered OAuth client (RFC 7591) to its single home: the server's
``LiteLLM_MCPServerTable`` row when it has one, otherwise the server-scoped store when the server
is config-declared. A rowless server that is not config-declared is a throwaway temp/session
server, so its client is overlaid in memory only and not persisted.
The interactive authorization_code flow mints a ``client_id`` via Dynamic Client
Registration that discovery cannot re-derive; without persisting it the autonomous
@ -1106,16 +1159,20 @@ async def _persist_dcr_client_registration(
if await _reuse_persisted_dcr_client_if_available(mcp_server, current_redirect_uri=current_redirect_uri):
return "reused"
token_endpoint_auth_method = (
"client_secret_basic" if registration.token_endpoint_auth_method == "client_secret_basic" else None
)
credentials: MCPCredentials = {
"client_id": registration.client_id,
"client_secret": registration.client_secret,
"token_endpoint_auth_method": (
"client_secret_basic" if registration.token_endpoint_auth_method == "client_secret_basic" else None
),
"token_endpoint_auth_method": token_endpoint_auth_method,
"redirect_uris": [current_redirect_uri],
}
from litellm.proxy._experimental.mcp_server.db import update_mcp_server # noqa: PLC0415
from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # avoids circular import
update_mcp_server,
upsert_mcp_server_oauth_client_credentials,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415
global_mcp_server_manager,
)
@ -1136,7 +1193,18 @@ async def _persist_dcr_client_registration(
),
touched_by="mcp_oauth_dcr",
)
await global_mcp_server_manager.update_server(updated_row)
if updated_row is not None:
await global_mcp_server_manager.update_server(updated_row)
return "persisted"
if global_mcp_server_manager.is_config_declared_server(mcp_server.server_id):
await upsert_mcp_server_oauth_client_credentials(
prisma_client=prisma_client,
server_id=mcp_server.server_id,
credentials=credentials,
)
mcp_server.client_id = registration.client_id
mcp_server.client_secret = registration.client_secret
mcp_server.token_endpoint_auth_method = token_endpoint_auth_method
return "persisted"
except Exception as exc: # noqa: BLE001
verbose_logger.warning(

16
litellm/proxy/_experimental/mcp_server/exceptions.py
View file

@ -88,3 +88,19 @@ class MCPToolResultError(Exception):
into two identities, breaking ``isinstance`` checks against instances
created before the reload.
"""
class MCPServerListError(Exception):
"""Carrier for a classified per-server listing fault (``faults.list_outcomes.ServerListFault``).
Raised where a server fetch used to silently return an empty tool list, so each boundary can
apply its own policy: the aggregate listing absorbs it into that server's outcome, while
single-server routes relay a truthful HTTP status instead of empty-success. The fault value is
typed as ``object`` here only to avoid a circular import with the faults package; construction
sites always pass a ``ServerListFault``.
"""
def __init__(self, fault: object, server_name: str) -> None:
self.fault = fault
self.server_name = server_name
super().__init__(f"Listing tools from MCP server {server_name!r} failed")

190
litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py Normal file
View file

@ -0,0 +1,190 @@
"""Per-server outcomes for the aggregate MCP tools/list fan-out.
The aggregate listing deliberately keeps serving the healthy subset when one server fails, but a
failed server must contribute a classified outcome instead of silently shrinking the list: an empty
contribution with no signal makes a broken upstream indistinguishable from a healthy server with no
tools. Outcomes carry only machine fields (category and status code) so nothing from an upstream
body crosses the trust boundary; classification is total, so any exception out of a server fetch
becomes an outcome, never a second failure.
"""
from __future__ import annotations
from collections.abc import Iterator
from typing import Literal, NamedTuple, NoReturn, TypeAlias
import httpx
from mcp.types import Tool as MCPTool
from pydantic import BaseModel, ConfigDict
from typing_extensions import assert_never
from litellm.proxy._experimental.mcp_server.exceptions import (
MCPServerListError,
MCPUpstreamAuthError,
)
ListFaultCategory: TypeAlias = Literal[
"auth_required",
"forbidden",
"timeout",
"unreachable",
"upstream_error",
"internal",
]
class ServerListOk(BaseModel):
model_config = ConfigDict(frozen=True)
tag: Literal["ok"] = "ok"
tool_count: int
class ServerListFault(BaseModel):
"""Why a server contributed nothing to a listing: the caller must authenticate upstream
(``auth_required``/``forbidden``), the upstream did not answer (``timeout``/``unreachable``),
the upstream answered outside its contract (``upstream_error``), or the gateway itself failed
(``internal``). ``status_code`` is the upstream HTTP status when one exists."""
model_config = ConfigDict(frozen=True)
tag: ListFaultCategory
status_code: int | None = None
ServerOutcome: TypeAlias = ServerListOk | ServerListFault
SERVER_OUTCOMES_META_KEY = "litellm.ai/server_outcomes"
"""The tools/list result ``_meta`` key carrying per-server outcomes. Prefixed with the litellm.ai
domain per the MCP spec's ``_meta`` key format so it cannot collide with spec-reserved names."""
class AggregateToolListing(NamedTuple):
tools: list[MCPTool]
outcomes: dict[str, ServerOutcome]
def _iter_upstream_responses(exc: BaseException) -> Iterator[httpx.Response]:
"""Yield every ``httpx.Response`` in the exception tree (``__cause__``/``__context__``/
ExceptionGroup members) in deliberate order, mirroring how upstream failures surface through the
MCP SDK's task groups. Explicit links come first: each node's ``raise ... from`` cause, then
group members in raise order, then the incidental ``__context__`` chain, so a response raised
while handling the real failure can never shadow one on the explicit causal chain. Consumers
apply their own predicate over the stream: selecting the first response and THEN testing it
would miss a causal auth response sitting behind an unrelated earlier one."""
seen: set[int] = set()
stack = [exc]
while stack:
current = stack.pop()
if id(current) in seen:
continue
seen.add(id(current))
response = getattr(current, "response", None)
if isinstance(response, httpx.Response):
yield response
if current.__context__ is not None:
stack.append(current.__context__)
exceptions = getattr(current, "exceptions", None)
if isinstance(exceptions, tuple):
stack.extend(reversed(exceptions))
if current.__cause__ is not None:
stack.append(current.__cause__)
def _find_upstream_response(exc: BaseException) -> httpx.Response | None:
return next(_iter_upstream_responses(exc), None)
def upstream_auth_challenge(exc: BaseException) -> tuple[int, str | None] | None:
"""The first upstream 401/403 in deliberate order and its ``WWW-Authenticate`` challenge, both
read from the SAME response, so the status that picks the carrier channel and the challenge that
rides with it can never come from two different responses in the tree. Non-auth responses do not
end the scan: a causal 401 behind an unrelated 5xx must still be found, or the client never
receives the challenge it needs to re-authenticate."""
for response in _iter_upstream_responses(exc):
if response.status_code in (401, 403):
return response.status_code, response.headers.get("www-authenticate")
return None
def raise_classified_list_failure(
exc: BaseException,
server_name: str,
suppress_challenge: bool = False,
) -> NoReturn:
"""The one place a failed server fetch chooses its carrier: an upstream 401/403 travels as
``MCPUpstreamAuthError`` with the upstream's own challenge preserved (a challenge is only ever
fabricated at the HTTP edge, and only for a 401), everything else as ``MCPServerListError`` with
a classified fault. Every fetch site delegates here so the two channels cannot drift apart per
call site. ``suppress_challenge`` is for dcr_bridge servers, whose upstream challenge points
clients at the wrong protected-resource metadata and must never relay."""
auth = upstream_auth_challenge(exc)
if auth is not None:
status_code, challenge = auth
raise MCPUpstreamAuthError(
status_code=status_code,
www_authenticate=None if suppress_challenge else challenge,
server_name=server_name,
) from exc
raise MCPServerListError(classify_list_exception(exc), server_name) from exc
def classify_list_exception(exc: BaseException) -> ServerListFault:
"""Classify a per-server listing failure into exactly one outcome. Total: an exception this
function cannot recognize is the gateway's own fault (``internal``), never a re-raise."""
if isinstance(exc, MCPServerListError) and isinstance(exc.fault, ServerListFault):
return exc.fault
if isinstance(exc, MCPUpstreamAuthError):
tag = "forbidden" if exc.status_code == 403 else "auth_required"
return ServerListFault(tag=tag, status_code=exc.status_code)
if isinstance(exc, TimeoutError):
return ServerListFault(tag="timeout")
if isinstance(exc, ConnectionError):
return ServerListFault(tag="unreachable")
auth = upstream_auth_challenge(exc)
if auth is not None:
status_code, _ = auth
return ServerListFault(
tag="forbidden" if status_code == 403 else "auth_required",
status_code=status_code,
)
response = _find_upstream_response(exc)
if response is not None:
return ServerListFault(tag="upstream_error", status_code=response.status_code)
if isinstance(exc, (httpx.TimeoutException,)):
return ServerListFault(tag="timeout")
if isinstance(exc, httpx.TransportError):
return ServerListFault(tag="unreachable")
return ServerListFault(tag="internal")
def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]:
"""The client-visible form of one outcome, for the tools/list result ``_meta`` and the REST
response: category plus status code only, never upstream prose or URLs."""
match outcome.tag:
case "ok":
return {"status": "ok", "tool_count": outcome.tool_count}
case "auth_required" | "forbidden" | "timeout" | "unreachable" | "upstream_error" | "internal":
return {
"status": outcome.tag,
**({"http_status": outcome.status_code} if outcome.status_code is not None else {}),
}
case _:
assert_never(outcome.tag)
def list_fault_http_status(fault: ServerListFault) -> int:
"""The truthful HTTP status for a single-upstream listing fault per RFC 9110: the upstream's own
401/403 for auth, 504 for a timeout, 502 for an unreachable or misbehaving upstream, and 500 only
for the gateway's own failure."""
match fault.tag:
case "auth_required":
return fault.status_code or 401
case "forbidden":
return 403
case "timeout":
return 504
case "unreachable" | "upstream_error":
return 502
case "internal":
return 500
case _:
assert_never(fault.tag)

248
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -50,7 +50,15 @@ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError
from litellm.proxy._experimental.mcp_server.exceptions import (
MCPServerListError,
MCPUpstreamAuthError,
)
from litellm.proxy._experimental.mcp_server.faults.list_outcomes import (
ServerListFault,
raise_classified_list_failure,
upstream_auth_challenge,
)
from litellm.proxy._experimental.mcp_server.elicitation_handler import (
MCP_ELICITATION_AVAILABLE,
)
@ -85,6 +93,8 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchange_
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
AuthorizationCodeConfig,
CredError,
IdJagConfig,
PassthroughConfig,
ServerSpec,
TokenExchangeConfig,
@ -613,6 +623,47 @@ def _consumes_caller_authorization(server: MCPServer) -> bool:
)
_REGISTRY_DUMP_SECRET_FIELDS = frozenset(
{"authentication_token", "client_secret", "client_private_key", "aws_secret_access_key", "aws_session_token"}
)
def _redacted_registry_dump(servers: dict[str, MCPServer]) -> dict[str, dict[str, str]]:
"""A JSON-safe view of the server registry with credential fields masked, for debug logging.
The registry holds long-lived secrets as plain strings (the static token, OAuth client secret,
the ID-JAG signing key, AWS keys); dumping them verbatim hands the gateway's client identity to
anyone who can read debug logs.
"""
dumps: dict[str, dict[str, object]] = {server_id: server.model_dump() for server_id, server in servers.items()}
return {
server_id: {
field: ("**REDACTED**" if field in _REGISTRY_DUMP_SECRET_FIELDS and value is not None else str(value))
for field, value in dump.items()
}
for server_id, dump in dumps.items()
}
def _to_server_spec_fail_closed(server: MCPServer) -> Optional[ServerSpec]:
"""`to_server_spec`, except a half-configured `oauth2_id_jag` server refuses instead of deferring.
ID-JAG has no v1 arm, so deferring to v1 would let `resolve_mcp_auth` honor a caller x-mcp-*
override or fall through to the static `authentication_token`, both of which bypass the per-user
identity assertion the mode promises. That is an operator misconfiguration, not a fallback.
"""
spec = to_server_spec(server)
if spec is None and server.auth_type == MCPAuth.oauth2_id_jag:
raise_public(
CredError.of_misconfigured(
"oauth2_id_jag requires token_exchange_endpoint, id_jag_resource_token_endpoint, "
"client_id, and a client_secret or client_private_key; refusing to fall back to "
"a static credential."
)
)
return spec
def _caller_authorization_fans_out(
server: MCPServer,
scope_servers: Optional[list[MCPServer]],
@ -633,49 +684,14 @@ def _caller_authorization_fans_out(
def _extract_upstream_auth_failure(
exc: BaseException,
) -> Optional[tuple[int, Optional[str]]]:
"""Walk the exception tree looking for an HTTP 401/403 response from the
upstream MCP server.
"""The upstream 401/403 and its ``WWW-Authenticate`` header from the exception tree, or ``None``.
The MCP SDK wraps transport errors in anyio ``ExceptionGroup`` objects and
may chain through ``__cause__`` / ``__context__``. We inspect all of those
layers for an ``httpx.Response``-bearing exception (typically
``httpx.HTTPStatusError``) and extract the status code and any upstream
``WWW-Authenticate`` header.
Returns ``(status_code, www_authenticate)`` on match, else ``None``.
"""
seen: set[int] = set()
stack: list[BaseException] = [exc]
while stack:
current = stack.pop()
if id(current) in seen:
continue
seen.add(id(current))
response = getattr(current, "response", None)
if response is not None:
status_code = getattr(response, "status_code", None)
if isinstance(status_code, int) and status_code in (401, 403):
www_authenticate: Optional[str] = None
headers = getattr(response, "headers", None)
if headers is not None:
try:
www_authenticate = headers.get("www-authenticate")
except Exception:
www_authenticate = None
return status_code, www_authenticate
# anyio / PEP 654 ExceptionGroup
sub_exceptions = getattr(current, "exceptions", None)
if sub_exceptions:
stack.extend(sub_exceptions)
if current.__cause__ is not None:
stack.append(current.__cause__)
if current.__context__ is not None and current.__context__ is not current.__cause__:
stack.append(current.__context__)
return None
Delegates to the shared traversal in ``faults.list_outcomes`` so every consumer (tool listing,
tool calls, the connect-time probe) selects the same response with the same deliberate order:
explicit ``raise ... from`` causes first, ExceptionGroup members in raise order, the incidental
``__context__`` chain last. A response raised while handling the real failure can therefore never
shadow the causal one."""
return upstream_auth_challenge(exc)
def _warn_on_server_name_fields(
@ -1127,6 +1143,14 @@ class MCPServerManager:
"""
return self.config_mcp_servers | self.registry
def is_config_declared_server(self, server_id: str) -> bool:
"""True when server_id was declared in config.yaml (present in the in-memory config map).
Config servers are rowless and persistent, so their DCR client belongs in the server-scoped
store; a rowless server that is NOT config-declared is a throwaway temp/session server whose
client must not be persisted. This never overrides the row-existence check: a server that has
a LiteLLM_MCPServerTable row is always resolved to that row first."""
return server_id in self.config_mcp_servers
async def load_servers_from_config(
self,
mcp_servers_config: dict[str, Any],
@ -1345,6 +1369,12 @@ class MCPServerManager:
"subject_token_type",
DEFAULT_SUBJECT_TOKEN_TYPE,
),
# ID-JAG fields
id_jag_resource_token_endpoint=server_config.get("id_jag_resource_token_endpoint", None),
id_jag_resource=server_config.get("id_jag_resource", None),
client_private_key=server_config.get("client_private_key", None),
client_private_key_id=server_config.get("client_private_key_id", None),
client_assertion_signing_alg=server_config.get("client_assertion_signing_alg", "RS256"),
token_exchange_profile=server_config.get("token_exchange_profile", "rfc8693"),
allow_sampling=bool(server_config.get("allow_sampling", False)),
allow_elicitation=bool(server_config.get("allow_elicitation", False)),
@ -1365,10 +1395,36 @@ class MCPServerManager:
base_url=server_config.get("url", ""),
)
verbose_logger.debug(f"Loaded MCP Servers: {json.dumps(self.config_mcp_servers, indent=4, default=str)}")
verbose_logger.debug(
f"Loaded MCP Servers: {json.dumps(_redacted_registry_dump(self.config_mcp_servers), indent=4)}"
)
await self._hydrate_config_servers_dcr_clients()
self.initialize_tool_name_to_mcp_server_name_mapping()
async def _hydrate_config_servers_dcr_clients(self) -> None:
"""Overlay each config-declared server's persisted DCR client (from the server-scoped
store) onto its in-memory object so token refresh authenticates after a restart. A
best-effort no-op when the DB is unreachable at config-load time."""
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( # noqa: PLC0415 # circular import
hydrate_config_server_dcr_client,
)
for server in self.config_mcp_servers.values():
try:
if await hydrate_config_server_dcr_client(server):
verbose_logger.debug(
"hydrated persisted DCR client onto config MCP server server_id=%s",
server.server_id,
)
except Exception as exc: # noqa: BLE001 # best-effort hydration; never fail config load
verbose_logger.debug(
"load_servers_from_config: failed to hydrate DCR client for server_id=%s: %s",
server.server_id,
exc,
)
async def _register_openapi_tools(self, spec_path: str, server: MCPServer, base_url: str):
"""
Register tools from an OpenAPI specification for a given server.
@ -1792,6 +1848,21 @@ class MCPServerManager:
subject_token_type=mcp_server.subject_token_type
or (credentials_dict.get("subject_token_type") if credentials_dict else None)
or DEFAULT_SUBJECT_TOKEN_TYPE,
# ID-JAG fields — read from credentials JSON blob
id_jag_resource_token_endpoint=(
credentials_dict.get("id_jag_resource_token_endpoint") if credentials_dict else None
),
id_jag_resource=(credentials_dict.get("id_jag_resource") if credentials_dict else None),
client_private_key=self._decrypt_credential_field(
credentials_dict.get("client_private_key") if credentials_dict else None,
"client_private_key",
credentials_are_encrypted,
),
client_private_key_id=(credentials_dict.get("client_private_key_id") if credentials_dict else None),
client_assertion_signing_alg=(
credentials_dict.get("client_assertion_signing_alg") if credentials_dict else None
)
or "RS256",
token_exchange_profile=mcp_server.token_exchange_profile
or (credentials_dict.get("token_exchange_profile") if credentials_dict else None)
or "rfc8693",
@ -2668,9 +2739,10 @@ class MCPServerManager:
)
if not conflicts:
return auth, extra_headers
if isinstance(spec.config, (TokenExchangeConfig, AuthorizationCodeConfig)):
if isinstance(spec.config, (TokenExchangeConfig, AuthorizationCodeConfig, IdJagConfig)):
# The resolver owns the per-user credential here (token_exchange's exchanged
# token, authorization_code's stored token). It is authoritative: a guardrail such
# token, authorization_code's stored token, id_jag's minted assertion). It is
# authoritative: a guardrail such
# as MCPJWTSigner, static_headers, or any other injected Authorization must NOT
# shadow it (otherwise the upstream gets e.g. the signer's JWT instead of the
# exchanged token and rejects it). Drop the conflicting header so the resolved
@ -2761,20 +2833,23 @@ class MCPServerManager:
Configured MCP client instance.
"""
transport = server.transport or MCPTransport.sse
spec = None if transport == MCPTransport.stdio else to_server_spec(server)
spec = None if transport == MCPTransport.stdio else _to_server_spec_fail_closed(server)
provider = cred_provider or self._cred_provider
# A caller-supplied per-request override (mcp_auth_header / x-mcp-*) defers to the v1 path
# so it wins - except for the modes the v2 resolver owns per-caller (authorization_code's
# stored token, token_exchange's RFC 8693 minted token, and the passthrough modes'
# forwarded caller token). A caller must not be able to substitute another user's stored
# credential, nor silently disable the OBO exchange and forward an arbitrary bearer
# upstream, so we keep the v2 spec and ignore the override for these; the REST tools
# preview supplies its not-yet-persisted token through the resolver (cred_provider),
# never this path.
# stored token, token_exchange's RFC 8693 minted token, id_jag's minted assertion, and the
# passthrough modes' forwarded caller token). A caller must not be able to substitute another
# user's stored credential, nor silently disable the OBO / ID-JAG exchange and forward an
# arbitrary bearer upstream, so we keep the v2 spec and ignore the override for these; the
# REST tools preview supplies its not-yet-persisted token through the resolver
# (cred_provider), never this path.
if (
spec is not None
and mcp_auth_header
and not isinstance(spec.config, (AuthorizationCodeConfig, PassthroughConfig, TokenExchangeConfig))
and not isinstance(
spec.config,
(AuthorizationCodeConfig, IdJagConfig, PassthroughConfig, TokenExchangeConfig),
)
):
spec = None
auth_value = (
@ -3047,10 +3122,12 @@ class MCPServerManager:
server_name=server.name,
) from e
verbose_logger.warning(f"Failed to get tools from server {server.name}: {str(e)}")
return []
raise MCPServerListError(ServerListFault(tag="internal", status_code=e.status_code), server.name) from e
except MCPServerListError:
raise
except Exception as e:
verbose_logger.warning(f"Failed to get tools from server {server.name}: {str(e)}")
return []
raise_classified_list_failure(e, server.name, suppress_challenge=server.is_dcr_bridge)
async def get_prompts_from_server(
self,
@ -3682,16 +3759,17 @@ class MCPServerManager:
Uses anyio.fail_after() instead of asyncio.wait_for() to avoid conflicts
with the MCP SDK's anyio TaskGroup. See GitHub issue #20715 for details.
An upstream HTTP 401 is converted into :class:`MCPUpstreamAuthError`
instead of being swallowed to an empty tool list, regardless of the
server's auth_type. Callers route it by surface: the single-server HTTP
routes turn it into a 401 + ``WWW-Authenticate`` challenge so standards-
compliant MCP clients trigger the upstream OAuth flow, while the
multi-server ``/mcp`` aggregator absorbs it to an empty list so one
unauthenticated server doesn't fail the whole listing. Only a 401
(missing/invalid credential) drives the re-auth challenge; a 403
(authenticated but forbidden, e.g. insufficient scope) is not a re-auth
signal and, like other non-auth errors, returns an empty list.
Failures never return an empty tool list. An upstream 401 or 403 raises
:class:`MCPUpstreamAuthError` carrying the upstream's own
``WWW-Authenticate`` challenge when one was sent (a challenge is only
ever fabricated at the HTTP edge, and only for a 401: a 403 means the
caller is authenticated but not allowed, so prompting re-auth would be
wrong, while an upstream-sent 403 challenge is the RFC 6750
insufficient_scope step-up and relays verbatim). Every other failure
raises :class:`MCPServerListError` with a classified fault. Each
boundary then applies its own policy: single-server routes relay the
truthful status, the multi-server aggregator absorbs the failure into
that server's listing outcome.
Args:
client: MCP client instance
@ -3705,27 +3783,18 @@ class MCPServerManager:
tools = await client.list_tools(raise_on_error=True)
verbose_logger.debug(f"Tools from {server_name}: {tools}")
return tools
except TimeoutError:
except TimeoutError as e:
verbose_logger.warning(f"Timeout while listing tools from {server_name}")
return []
except asyncio.CancelledError:
raise MCPServerListError(ServerListFault(tag="timeout"), server_name) from e
except asyncio.CancelledError as e:
verbose_logger.warning(f"Task cancelled while listing tools from {server_name}")
return []
raise MCPServerListError(ServerListFault(tag="internal"), server_name) from e
except ConnectionError as e:
verbose_logger.warning(f"Connection error while listing tools from {server_name}: {str(e)}")
return []
raise MCPServerListError(ServerListFault(tag="unreachable"), server_name) from e
except Exception as e:
auth_info = _extract_upstream_auth_failure(e)
if auth_info is not None and auth_info[0] == 401:
_, www_authenticate = auth_info
verbose_logger.info(f"Upstream auth failure from MCP server {server_name}: HTTP 401")
raise MCPUpstreamAuthError(
status_code=401,
www_authenticate=www_authenticate,
server_name=server_name,
) from e
verbose_logger.warning(f"Error listing tools from {server_name}: {str(e)}")
return []
raise_classified_list_failure(e, server_name)
_SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024
@ -4309,10 +4378,13 @@ class MCPServerManager:
if server_auth_header is None:
server_auth_header = mcp_auth_header
# Extract subject token for OAuth2 Token Exchange (OBO) flow
# Extract subject token for OAuth2 Token Exchange (OBO) and ID-JAG flows
subject_token: Optional[str] = None
extra_headers: Optional[dict[str, str]] = None
if mcp_server.auth_type == MCPAuth.oauth2_token_exchange:
if mcp_server.auth_type in (
MCPAuth.oauth2_token_exchange,
MCPAuth.oauth2_id_jag,
):
subject_token = self._extract_bearer_token(oauth2_headers, raw_headers)
elif mcp_server.auth_type == MCPAuth.oauth2:
if mcp_server.has_client_credentials:
@ -4414,10 +4486,10 @@ class MCPServerManager:
arguments=arguments,
)
if mcp_server.auth_type == MCPAuth.oauth2_token_exchange and subject_token:
# OBO: the exchanged token may have been revoked/rotated upstream since it was cached, so
# an upstream 401 gets one re-mint + retry. Gated to this mode; all others keep the plain
# single call below.
if mcp_server.auth_type in (MCPAuth.oauth2_token_exchange, MCPAuth.oauth2_id_jag) and subject_token:
# OBO / ID-JAG: the exchanged token may have been revoked/rotated upstream since it was
# cached, so an upstream 401 gets one invalidate + re-mint + retry. Gated to these modes;
# all others keep the plain single call below.
async def _obo_call_tool_limited():
async with self._limit_outbound_concurrency(mcp_server):
return await self._obo_call_tool_with_retry(
@ -4968,6 +5040,8 @@ class MCPServerManager:
verbose_logger.debug("MCP registry refreshed (%s servers in registry)", len(registered_registry))
await self._hydrate_config_servers_dcr_clients()
def get_mcp_servers_from_ids(self, server_ids: list[str]) -> list[MCPServer]:
servers = []
registry = self.get_registry()

8
litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py
View file

@ -31,10 +31,14 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
AwsCredentialSource,
AwsSigV4Config,
Byok,
ClientAuth,
ClientCredentialsConfig,
ClientSecretAuth,
CredError,
IdJagConfig,
NoneConfig,
PassthroughConfig,
PrivateKeyJwtAuth,
ServerSpec,
SharedKey,
StaticKeys,
@ -59,6 +63,10 @@ __all__ = [
"AuthorizationCodeConfig",
"ClientCredentialsConfig",
"TokenExchangeConfig",
"IdJagConfig",
"ClientAuth",
"PrivateKeyJwtAuth",
"ClientSecretAuth",
"ApiKeyConfig",
"ApiKeySource",
"SharedKey",

61
litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py
View file

@ -21,9 +21,13 @@ from typing_extensions import assert_never
from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
ApiKeyConfig,
AuthorizationCodeConfig,
ClientAuth,
ClientSecretAuth,
CredError,
IdJagConfig,
NoneConfig,
PassthroughConfig,
PrivateKeyJwtAuth,
ServerSpec,
SharedKey,
Subject,
@ -35,6 +39,9 @@ if TYPE_CHECKING:
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.mcp_server.mcp_server_manager import MCPServer
_TOKEN_EXCHANGE_SUBJECT_TOKEN_DEFAULT = "urn:ietf:params:oauth:token-type:access_token"
_ID_JAG_SUBJECT_TOKEN_DEFAULT = "urn:ietf:params:oauth:token-type:id_token"
def to_subject(user_api_key_auth: Optional[UserAPIKeyAuth], subject_token: Optional[str]) -> Subject:
"""Map v1's authenticated principal onto the resolver's Subject.
@ -96,6 +103,8 @@ def to_server_spec(server: MCPServer) -> Optional[ServerSpec]:
)
# client_credentials (M2M) and delegate/passthrough oauth2 stay on v1
return None
case MCPAuth.oauth2_id_jag:
return _id_jag_spec(server, resource)
case MCPAuth.true_passthrough | MCPAuth.oauth_delegate:
return ServerSpec(server_id=server.server_id, resource=resource, config=PassthroughConfig())
case MCPAuth.oauth2_token_exchange:
@ -167,6 +176,58 @@ def _shared_key_spec(
)
def _id_jag_spec(server: MCPServer, resource: str) -> Optional[ServerSpec]:
"""Build an ID-JAG spec from the v1 server's raw fields, or defer (None) if half-configured.
The enum already routes here, but a server missing an endpoint, ``client_id``, or any client-auth
secret would make ``IdJagConfig`` raise at construction; returning None instead defers to v1 so a
partially configured server does not 500. ``token_exchange_endpoint`` is leg 1 (the IdP org AS);
leg 2 is ``id_jag_resource_token_endpoint`` (the upstream resource AS).
"""
org_token_endpoint = server.token_exchange_endpoint
resource_token_endpoint = server.id_jag_resource_token_endpoint
client_id = server.client_id
client_auth = _id_jag_client_auth(server)
if not org_token_endpoint or not resource_token_endpoint or not client_id or client_auth is None:
return None
return ServerSpec(
server_id=server.server_id,
resource=resource,
config=IdJagConfig(
org_token_endpoint=org_token_endpoint,
resource_token_endpoint=resource_token_endpoint,
client_id=client_id,
client_auth=client_auth,
subject_token_type=_id_jag_subject_token_type(server),
audience=server.audience,
resource=server.id_jag_resource,
scopes=tuple(server.scopes or ()),
),
)
def _id_jag_client_auth(server: MCPServer) -> Optional[ClientAuth]:
"""Private-key JWT when a key is configured, else client_secret, else None (defer to v1)."""
if server.client_private_key:
return PrivateKeyJwtAuth(
private_key=SecretStr(server.client_private_key),
key_id=server.client_private_key_id,
signing_alg=server.client_assertion_signing_alg,
)
if server.client_secret:
return ClientSecretAuth(client_secret=SecretStr(server.client_secret))
return None
def _id_jag_subject_token_type(server: MCPServer) -> str:
"""ID-JAG asserts the user's id_token, so the token-exchange access_token default maps to id_token;
an explicitly configured value (e.g. a SAML2 assertion type) is honored verbatim."""
configured = server.subject_token_type
if configured and configured != _TOKEN_EXCHANGE_SUBJECT_TOKEN_DEFAULT:
return configured
return _ID_JAG_SUBJECT_TOKEN_DEFAULT
def raise_public(error: CredError) -> NoReturn:
"""Map a resolver CredError onto the proxy's public HTTP contract. The one edge that raises."""
match error.tag:

116
litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py
View file

@ -16,6 +16,8 @@ follow-up PR with their seam. Pure v2: no imports from v1.
from __future__ import annotations
import hashlib
import httpx
from typing_extensions import assert_never
@ -33,6 +35,11 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
Ok,
Result,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.token_endpoint import (
ExchangedToken,
ExchangedTokenCache,
TokenEndpointClient,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import (
TokenExchanger,
)
@ -42,16 +49,24 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
AuthSpecKind,
AwsSigV4Config,
Byok,
ClientAuth,
ClientCredentialsConfig,
ClientSecretAuth,
CredError,
IdJagConfig,
NoneConfig,
PassthroughConfig,
PrivateKeyJwtAuth,
ServerSpec,
SharedKey,
Subject,
TokenExchangeConfig,
)
_TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
_JWT_BEARER_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:jwt-bearer"
_ID_JAG_REQUESTED_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:id-jag"
class _NullOAuthTokenStore:
"""Fail-closed default: with no token store wired, every user reads as not authorized."""
@ -87,9 +102,13 @@ class UpstreamCredentialProvider:
self,
oauth_token_store: OAuthTokenStore | None = None,
token_exchanger: TokenExchanger | None = None,
token_endpoint: TokenEndpointClient | None = None,
exchanged_tokens: ExchangedTokenCache | None = None,
) -> None:
self._oauth_token_store: OAuthTokenStore = oauth_token_store or _NullOAuthTokenStore()
self._token_exchanger: TokenExchanger = token_exchanger or _NullTokenExchanger()
self._token_endpoint: TokenEndpointClient = token_endpoint or TokenEndpointClient()
self._exchanged_tokens: ExchangedTokenCache = exchanged_tokens or ExchangedTokenCache()
async def resolve_credentials(self, subject: Subject, server: ServerSpec) -> Result[httpx.Auth, CredError]:
match server.config:
@ -103,6 +122,8 @@ class UpstreamCredentialProvider:
return _not_implemented(AuthSpecKind.client_credentials)
case TokenExchangeConfig() as config:
return await self._token_exchange(subject, server, config)
case IdJagConfig() as config:
return await self._id_jag(subject, server, config)
case AuthorizationCodeConfig():
return await self._authorization_code(subject, server)
case AwsSigV4Config():
@ -141,6 +162,53 @@ class UpstreamCredentialProvider:
return Error(CredError.of_not_implemented("api_key BYOK source not implemented yet"))
assert_never(config.key_source)
async def _id_jag(self, subject: Subject, server: ServerSpec, config: IdJagConfig) -> Result[httpx.Auth, CredError]:
if subject.inbound_token is None:
return Error(
CredError.of_precondition_required(
"ID-JAG requires a caller identity token; it asserts the calling "
"user's identity upstream and cannot use a static credential."
)
)
token = subject.inbound_token.get_secret_value()
cache_key = _id_jag_cache_key(token, server.server_id, config)
async def _exchange() -> Result[ExchangedToken, CredError]:
leg1_params = {
"grant_type": _TOKEN_EXCHANGE_GRANT_TYPE,
"requested_token_type": _ID_JAG_REQUESTED_TOKEN_TYPE,
"subject_token": token,
"subject_token_type": config.subject_token_type,
**({"audience": config.audience} if config.audience else {}),
**({"resource": config.resource} if config.resource else {}),
**({"scope": " ".join(config.scopes)} if config.scopes else {}),
}
match await self._token_endpoint.fetch(
config.org_token_endpoint,
config.client_id,
leg1_params,
config.client_auth,
):
case Error(err):
return Error(err)
case Ok(id_jag):
leg2_params = {
"grant_type": _JWT_BEARER_GRANT_TYPE,
"assertion": id_jag.access_token,
}
return await self._token_endpoint.fetch(
config.resource_token_endpoint,
config.client_id,
leg2_params,
config.client_auth,
)
match await self._exchanged_tokens.get_or_compute(cache_key, _exchange):
case Ok(access_token):
return Ok(StaticHeaderAuth(f"Bearer {access_token}"))
case Error(err):
return Error(err)
async def _authorization_code(self, subject: Subject, server: ServerSpec) -> Result[StaticHeaderAuth, CredError]:
token = await self._authz_token(subject, server)
if token is None:
@ -176,13 +244,19 @@ class UpstreamCredentialProvider:
"""Drop any cached credential the resolver owns for this `(subject, server)`.
Used after an upstream rejects the injected credential, so the next resolve re-mints rather
than serving the same rejected token until TTL. Only `token_exchange` holds a re-mintable
cached credential here; other modes are a no-op.
than serving the same rejected token until TTL. `token_exchange` and `id_jag` hold a
re-mintable cached credential here; other modes are a no-op.
"""
if isinstance(server.config, TokenExchangeConfig) and subject.inbound_token is not None:
if subject.inbound_token is None:
return
if isinstance(server.config, TokenExchangeConfig):
await self._token_exchanger.invalidate(
subject.inbound_token.get_secret_value(), server, server.config, tenant_id=subject.tenant_id
)
if isinstance(server.config, IdJagConfig):
self._exchanged_tokens.invalidate(
_id_jag_cache_key(subject.inbound_token.get_secret_value(), server.server_id, server.config)
)
async def _authz_token(self, subject: Subject, server: ServerSpec) -> OAuthToken | None:
"""The user's authorization_code token, or None when absent or the store is unreachable.
@ -196,5 +270,41 @@ class UpstreamCredentialProvider:
return None
def _id_jag_cache_key(subject_token: str, server_id: str, config: IdJagConfig) -> str:
"""Bind the cached leg-2 bearer to the caller token, the server, AND the config that minted it.
Every exchange parameter derives from the config (endpoints, audience, resource, scopes, client
auth), so a server update that changes any of them must change the key; otherwise the old bearer,
authorized under the old policy, keeps being served until its TTL. Everything is hashed, so no
secret is held in the key.
"""
material = "\x00".join(
(
subject_token,
server_id,
config.org_token_endpoint,
config.resource_token_endpoint,
config.client_id,
_client_auth_fingerprint(config.client_auth),
config.subject_token_type,
config.audience or "",
config.resource or "",
" ".join(config.scopes),
)
)
return hashlib.sha256(material.encode()).hexdigest()
def _client_auth_fingerprint(client_auth: ClientAuth) -> str:
match client_auth:
case PrivateKeyJwtAuth() as auth:
return "\x00".join(
("private_key_jwt", auth.private_key.get_secret_value(), auth.key_id or "", auth.signing_alg)
)
case ClientSecretAuth() as auth:
return "\x00".join(("client_secret", auth.client_secret.get_secret_value()))
assert_never(client_auth)
def _not_implemented(kind: AuthSpecKind) -> Result[httpx.Auth, CredError]:
return Error(CredError.of_not_implemented(f"{kind.value}: resolver arm not implemented yet"))

225
litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py Normal file
View file

@ -0,0 +1,225 @@
"""An authenticated OAuth token-endpoint call plus a short-lived-token cache.
`TokenEndpointClient.fetch` POSTs one grant to a token endpoint, authenticating the gateway as
an OAuth client via `client_auth` (RFC 7523 private-key JWT, or `client_secret_post`), and returns
the minted token or a typed `CredError`. `ExchangedTokenCache` memoizes the final token string per
opaque cache key with per-key single-flight, so concurrent callers share one round-trip and a hit
skips the endpoint entirely.
Pure v2: no imports from the v1 MCP auth handlers. The multi-leg flows that compose these (ID-JAG,
and later token_exchange / client_credentials) live in the resolver arms; this collaborator owns
only the single authenticated call and the cache.
"""
from __future__ import annotations
import asyncio
import json
import time
import uuid
import weakref
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
import httpx
import jwt
from pydantic import BaseModel, ValidationError
from typing_extensions import assert_never
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import (
MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
)
from litellm.exceptions import Timeout
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # litellm http handler is untyped
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
Error,
Ok,
Result,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
ClientAuth,
ClientSecretAuth,
CredError,
PrivateKeyJwtAuth,
)
from litellm.types.llms.custom_http import httpxSpecialProvider
CLIENT_ASSERTION_TYPE = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"
CLIENT_ASSERTION_LIFETIME_SECONDS = 60
@dataclass(frozen=True, slots=True)
class ExchangedToken:
access_token: str
expires_in: int | None
class _TokenEndpointResponse(BaseModel):
access_token: str
expires_in: int | None = None
class TokenEndpointClient:
"""One authenticated POST to an OAuth token endpoint, returning the minted token as a value."""
async def fetch(
self,
endpoint: str,
client_id: str,
grant_params: Mapping[str, str],
client_auth: ClientAuth,
) -> Result[ExchangedToken, CredError]:
try:
data = {**grant_params, **_client_auth_params(endpoint, client_id, client_auth)}
except (ValueError, TypeError, NotImplementedError, jwt.PyJWTError):
verbose_proxy_logger.warning("MCP token endpoint %s: could not sign the client assertion", endpoint)
return Error(
CredError.of_misconfigured(
"token exchange failed: could not sign the client assertion; "
"check client_private_key and client_assertion_signing_alg"
)
)
try:
raw = await _post_form(endpoint, data)
except httpx.HTTPStatusError as exc:
verbose_proxy_logger.warning(
"MCP token endpoint %s failed with status %s", endpoint, exc.response.status_code
)
return Error(
CredError.of_upstream_unavailable(f"token exchange failed with status {exc.response.status_code}")
)
except (httpx.RequestError, Timeout) as exc:
verbose_proxy_logger.warning("MCP token endpoint %s unreachable: %s", endpoint, type(exc).__name__)
return Error(
CredError.of_upstream_unavailable(
f"token exchange failed: token endpoint unreachable ({type(exc).__name__})"
)
)
except json.JSONDecodeError:
verbose_proxy_logger.warning("MCP token endpoint %s returned a non-JSON response", endpoint)
return Error(
CredError.of_upstream_unavailable("token exchange failed: token endpoint returned a non-JSON response")
)
if raw is None:
verbose_proxy_logger.warning("MCP token endpoint %s returned no response", endpoint)
return Error(CredError.of_upstream_unavailable("token exchange failed: no response from token endpoint"))
try:
parsed = _TokenEndpointResponse.model_validate(raw)
except ValidationError:
verbose_proxy_logger.warning("MCP token endpoint %s response missing access_token", endpoint)
return Error(
CredError.of_upstream_unavailable("token exchange failed: token endpoint response missing access_token")
)
return Ok(ExchangedToken(access_token=parsed.access_token, expires_in=parsed.expires_in))
class ExchangedTokenCache:
"""Memoizes the final token string per key, single-flighting concurrent misses on one lock."""
def __init__(self) -> None:
self._cache = InMemoryCache(
max_size_in_memory=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
)
self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = weakref.WeakValueDictionary()
async def get_or_compute(
self,
cache_key: str,
compute: Callable[[], Awaitable[Result[ExchangedToken, CredError]]],
) -> Result[str, CredError]:
cached = self._get(cache_key)
if cached is not None:
return Ok(cached)
async with self._lock(cache_key):
cached = self._get(cache_key)
if cached is not None:
return Ok(cached)
match await compute():
case Ok(token):
self._cache.set_cache( # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped
cache_key,
token.access_token,
ttl=_cache_ttl_seconds(token.expires_in),
)
return Ok(token.access_token)
case Error(err):
return Error(err)
def invalidate(self, cache_key: str) -> None:
"""Evict one cached token so the next `get_or_compute` re-mints (e.g. after an upstream 401)."""
self._cache.delete_cache(cache_key) # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped
def _get(self, cache_key: str) -> str | None:
value = self._cache.get_cache(cache_key) # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType] # InMemoryCache is untyped; narrowed by isinstance below
return value if isinstance(value, str) else None
def _lock(self, cache_key: str) -> asyncio.Lock:
lock = self._locks.get(cache_key)
if lock is None:
lock = asyncio.Lock()
self._locks[cache_key] = lock
return lock
def _cache_ttl_seconds(expires_in: int | None) -> int:
lifetime = expires_in if expires_in is not None else MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
return max(
lifetime - MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
)
async def _post_form(endpoint: str, data: dict[str, str]) -> object | None:
# litellm's httpx handler and httpx.Response are only partially typed; the token endpoint
# returns a JSON object that `_TokenEndpointResponse` validates, so the untyped boundary is
# contained here. A non-2xx raises `httpx.HTTPStatusError`, an unreachable endpoint raises
# `httpx.RequestError` (or litellm's `Timeout`, which the handler substitutes for
# `httpx.TimeoutException`), and a non-JSON body raises `json.JSONDecodeError`; `fetch` maps
# each to a CredError.
client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) # pyright: ignore[reportUnknownVariableType] # litellm http handler is untyped
response = await client.post(endpoint, data=data) # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType] # litellm http handler is untyped
if response is None:
return None
response.raise_for_status()
return response.json() # pyright: ignore[reportAny] # untyped JSON; validated by _TokenEndpointResponse in fetch
def _client_auth_params(endpoint: str, client_id: str, client_auth: ClientAuth) -> dict[str, str]:
match client_auth:
case PrivateKeyJwtAuth() as auth:
return {
"client_id": client_id,
"client_assertion_type": CLIENT_ASSERTION_TYPE,
"client_assertion": _client_assertion(endpoint, client_id, auth),
}
case ClientSecretAuth() as auth:
return {
"client_id": client_id,
"client_secret": auth.client_secret.get_secret_value(),
}
assert_never(client_auth)
def _client_assertion(endpoint: str, client_id: str, auth: PrivateKeyJwtAuth) -> str:
now = int(time.time())
return jwt.encode(
{
"iss": client_id,
"sub": client_id,
"aud": endpoint,
"jti": uuid.uuid4().hex,
"iat": now,
"exp": now + CLIENT_ASSERTION_LIFETIME_SECONDS,
},
auth.private_key.get_secret_value(),
algorithm=auth.signing_alg,
headers={"kid": auth.key_id} if auth.key_id else None,
)

45
litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py
View file

@ -56,6 +56,7 @@ class AuthSpecKind(str, Enum):
authorization_code = "authorization_code" # per-user 3LO; gateway-stored token
client_credentials = "client_credentials" # gateway service account (M2M)
token_exchange = "token_exchange" # RFC 8693: token endpoint + subject_token (OBO)
id_jag = "id_jag" # draft-ietf-oauth-identity-assertion-authz-grant: two-leg exchange then jwt-bearer
api_key = "api_key" # static header, any scheme (BYOK = per-user-seeded source)
passthrough = "passthrough" # client forwards an upstream-audience token
none = "none" # no upstream credential; resolve yields a no-op auth, never an error
@ -225,6 +226,49 @@ class TokenExchangeConfig(BaseModel):
scopes: tuple[str, ...] = ()
class PrivateKeyJwtAuth(BaseModel):
"""RFC 7523 private-key-JWT client authentication: the gateway signs a `client_assertion`."""
model_config = ConfigDict(frozen=True)
source: Literal["private_key_jwt"] = "private_key_jwt"
private_key: SecretStr
key_id: str | None = None
signing_alg: str = "RS256"
class ClientSecretAuth(BaseModel):
"""`client_secret_post` client authentication: the gateway posts `client_id` + `client_secret`."""
model_config = ConfigDict(frozen=True)
source: Literal["client_secret"] = "client_secret"
client_secret: SecretStr
ClientAuth = Annotated[PrivateKeyJwtAuth | ClientSecretAuth, Field(discriminator="source")]
class IdJagConfig(BaseModel):
"""draft-ietf-oauth-identity-assertion-authz-grant (Okta "AI agent token exchange").
Two legs: leg 1 is an RFC 8693 token exchange at the IdP org AS (`org_token_endpoint`) that
swaps the caller's identity token for an ID-JAG assertion; leg 2 is an RFC 7523 jwt-bearer at
the upstream resource AS (`resource_token_endpoint`) that swaps the assertion for the access
token. The gateway authenticates to both endpoints as `client_id` via `client_auth`. Required
fields are enforced at construction so a half-configured server cannot reach the arm.
"""
model_config = ConfigDict(frozen=True)
kind: Literal[AuthSpecKind.id_jag] = AuthSpecKind.id_jag
org_token_endpoint: str
resource_token_endpoint: str
client_id: str
client_auth: ClientAuth
subject_token_type: str = "urn:ietf:params:oauth:token-type:id_token"
audience: str | None = None
resource: str | None = None
scopes: tuple[str, ...] = ()
class SharedKey(BaseModel):
"""A fixed key configured on the server, identical for every caller."""
@ -323,6 +367,7 @@ AuthConfig = Annotated[
AuthorizationCodeConfig
| ClientCredentialsConfig
| TokenExchangeConfig
| IdJagConfig
| ApiKeyConfig
| PassthroughConfig
| NoneConfig

58
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -19,12 +19,20 @@ import httpx
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from litellm._logging import verbose_logger
from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError
from litellm.proxy._experimental.mcp_server.exceptions import (
MCPServerListError,
MCPUpstreamAuthError,
)
from litellm.proxy._experimental.mcp_server.faults.list_outcomes import (
classify_list_exception,
list_fault_http_status,
)
from litellm.proxy._experimental.mcp_server.ui_session_utils import (
build_effective_auth_contexts,
)
from litellm.proxy._experimental.mcp_server.utils import (
MCPMissingUserEnvVarsError,
get_server_prefix,
merge_mcp_headers,
)
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
@ -515,20 +523,19 @@ if MCP_AVAILABLE:
# enforced even when no allowlist is set (matches the SSE/HTTP path).
tools = filter_tools_by_allowed_tools(tools, server)
# Filter tools based on user_api_key_auth.object_permission.mcp_tool_permissions
# This provides per-key/team/org control over which tools can be accessed
if (
user_api_key_auth
and user_api_key_auth.object_permission
and user_api_key_auth.object_permission.mcp_tool_permissions
):
# Dict keys may be server_ids OR names/aliases; normalize so lookup
# by concrete server_id resolves name-keyed restrictions too.
allowed_tools_for_server = global_mcp_server_manager.expand_tool_permissions(
user_api_key_auth.object_permission.mcp_tool_permissions
).get(server.server_id)
if allowed_tools_for_server is not None and len(allowed_tools_for_server) > 0:
# Filter tools to only include those in the allowed list
# Filter by the key's effective tool permissions through the same
# primitive the MCP protocol path uses (direct grants, toolset grants,
# and team/agent/org ceilings), so REST listing cannot drift from it
if user_api_key_auth:
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
allowed_tools_for_server = await MCPRequestHandler.get_allowed_tools_for_server(
server_id=server.server_id,
user_api_key_auth=user_api_key_auth,
)
if allowed_tools_for_server is not None:
tools = [tool for tool in tools if _tool_name_matches(tool.name, allowed_tools_for_server)]
return _create_tool_response_objects(tools, server)
@ -627,6 +634,16 @@ if MCP_AVAILABLE:
# matching status code and WWW-Authenticate challenge; that is what
# lets standards-compliant MCP clients run the upstream OAuth flow.
raise
except MCPServerListError as e:
fault = classify_list_exception(e)
verbose_logger.info(f"Listing tools from {server.name} failed with a {fault.tag} fault")
raise HTTPException(
status_code=list_fault_http_status(fault),
detail={
"error": fault.tag,
"message": f"Failed to list tools from server {get_server_prefix(server)}",
},
) from e
except Exception as e:
verbose_logger.exception(f"Error getting tools from {server.name}: {e}")
return {
@ -838,7 +855,11 @@ if MCP_AVAILABLE:
list_tools_result.extend(tools_result)
except Exception as e:
verbose_logger.exception(f"Error getting tools from {server.name}: {e}")
errors.append(f"{server.name}: {str(e)}")
errors.append(
f"{get_server_prefix(server)}: {classify_list_exception(e).tag}"
if isinstance(e, (MCPServerListError, MCPUpstreamAuthError))
else f"{get_server_prefix(server)}: {str(e)}"
)
continue
if errors and not list_tools_result:
@ -858,7 +879,10 @@ if MCP_AVAILABLE:
request_path=request.scope.get("_original_path") or request.url.path,
)
except HTTPException as http_exc:
if http_exc.status_code == status.HTTP_404_NOT_FOUND:
if http_exc.status_code == status.HTTP_404_NOT_FOUND or server_id:
# Single-server requests relay the truthful status (a 502/504 upstream fault must
# not masquerade as a 200 empty-success body); only the multi-server aggregate
# keeps the legacy error-dict response shape below.
raise
# Internal access/IP 403s keep the legacy error-dict response shape
# so the existing contract stays intact.

243
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -348,6 +348,7 @@ if MCP_AVAILABLE:
CallToolResult,
EmbeddedResource,
ImageContent,
ListToolsResult,
Prompt,
TextContent,
)
@ -356,6 +357,14 @@ if MCP_AVAILABLE:
from litellm.proxy._experimental.mcp_server.auth.litellm_auth_handler import (
MCPAuthenticatedUser,
)
from litellm.proxy._experimental.mcp_server.faults.list_outcomes import (
SERVER_OUTCOMES_META_KEY,
AggregateToolListing,
ServerListOk,
ServerOutcome,
classify_list_exception,
outcome_wire_value,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
MCPServerManager,
_caller_authorization_fans_out,
@ -664,9 +673,12 @@ if MCP_AVAILABLE:
########################################################
@server.list_tools()
async def handle_list_tools() -> List[Tool]:
async def handle_list_tools() -> "ListToolsResult | List[Tool]":
"""
List all available tools.
List all available tools, with each server's listing outcome attached to the result's
``_meta`` (SERVER_OUTCOMES_META_KEY) so a broken upstream is distinguishable from a healthy
server with no tools. Returning a ListToolsResult (rather than a bare list) makes the MCP SDK
pass the result through unwrapped, which is what lets the ``_meta`` survive to the client.
Also captures the active session for propagation to callbacks.
"""
from mcp.server.lowlevel.server import request_ctx
@ -709,7 +721,7 @@ if MCP_AVAILABLE:
# Get mcp_servers from context variable
verbose_logger.debug("MCP list_tools - Calling _list_mcp_tools")
tools = await _list_mcp_tools(
listing = await _list_mcp_tools(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
@ -719,8 +731,15 @@ if MCP_AVAILABLE:
log_list_tools_to_spendlogs=True,
list_tools_log_source="mcp_protocol",
)
verbose_logger.info(f"MCP list_tools - Successfully returned {len(tools)} tools")
return tools
verbose_logger.info(f"MCP list_tools - Successfully returned {len(listing.tools)} tools")
if not listing.outcomes:
return listing.tools
outcome_meta = {
SERVER_OUTCOMES_META_KEY: {
key: outcome_wire_value(outcome) for key, outcome in listing.outcomes.items()
}
}
return ListToolsResult.model_validate({"tools": listing.tools, "_meta": outcome_meta})
except Exception as e:
verbose_logger.exception(f"Error in list_tools endpoint: {str(e)}")
# Return empty list instead of failing completely
@ -1746,6 +1765,13 @@ if MCP_AVAILABLE:
_mcp_gateway_initialize_instructions.reset(instructions_token)
_mcp_gateway_server_name.reset(server_name_token)
def _aggregate_server_key(server: MCPServer) -> str:
"""The client-visible key for a server in listing outcomes and spend metadata: the same
display prefix (alias, or the short prefix when that mode is enabled) the caller already
sees on the tool names. Canonical internal server names never key a caller-readable
surface; when the display naming deliberately hides them, the outcome keys must too."""
return get_server_prefix(server) or "unknown"
async def _get_tools_from_mcp_servers(
user_api_key_auth: Optional[UserAPIKeyAuth],
mcp_auth_header: Optional[str],
@ -1758,7 +1784,7 @@ if MCP_AVAILABLE:
litellm_trace_id: Optional[str] = None,
request_tags: Optional[list[str]] = None,
client_ip: Optional[str] = None,
) -> List[MCPTool]:
) -> AggregateToolListing:
"""
Helper method to fetch tools from MCP servers based on server filtering criteria.
@ -1770,10 +1796,11 @@ if MCP_AVAILABLE:
oauth2_headers: Optional dict of oauth2 headers
Returns:
List[MCPTool]: Combined list of tools from filtered servers
AggregateToolListing: Combined tools from filtered servers plus each server's
classified listing outcome
"""
if not MCP_AVAILABLE:
return []
return AggregateToolListing(tools=[], outcomes={})
list_tools_start_time = datetime.now()
litellm_logging_obj: Optional[LiteLLMLoggingObj] = None
@ -1858,10 +1885,12 @@ if MCP_AVAILABLE:
async def _fetch_and_filter_server_tools(
server: MCPServer,
) -> List[MCPTool]:
"""Fetch and filter tools from a single server with error handling."""
) -> "tuple[List[MCPTool], ServerOutcome]":
"""Fetch and filter tools from a single server, classifying any failure into that
server's outcome so the aggregate can keep serving the healthy subset without a
broken server masquerading as an empty one."""
if server is None:
return []
return [], ServerListOk(tool_count=0)
server_auth_header, extra_headers = _prepare_mcp_server_headers(
server=server,
@ -1931,8 +1960,8 @@ if MCP_AVAILABLE:
verbose_logger.debug(
f"Successfully fetched {len(tools)} tools from server {server.name}, {len(filtered_tools)} after filtering"
)
return filtered_tools
except MCPUpstreamAuthError:
return filtered_tools, ServerListOk(tool_count=len(filtered_tools))
except MCPUpstreamAuthError as e:
# Absorb so one unauthenticated server does not empty every other server's
# tools. Surfacing the upstream 401 to the client as a re-auth challenge is
# intentionally not done here: raising from this list handler cannot produce a
@ -1940,31 +1969,30 @@ if MCP_AVAILABLE:
# error). Single-server routes surface it via the request-scope preemptive
# check in _raise_preemptive_401_for_unauthenticated_servers instead.
verbose_logger.debug(f"MCP list_tools: omitting {server.name}; it needs upstream auth")
return []
return [], classify_list_exception(e)
except Exception as e:
verbose_logger.exception(f"Error getting tools from server {server.name}: {str(e)}")
return []
return [], classify_list_exception(e)
# Fetch tools from all servers in parallel
tasks = [_fetch_and_filter_server_tools(server) for server in allowed_mcp_servers]
results = await asyncio.gather(*tasks)
# Flatten results into single list
all_tools: List[MCPTool] = [tool for tools in results for tool in tools]
all_tools: List[MCPTool] = [tool for tools, _ in results for tool in tools]
server_outcomes: Dict[str, ServerOutcome] = {
_aggregate_server_key(server): outcome
for server, (_, outcome) in zip(allowed_mcp_servers, results)
if server is not None
}
# If logging is enabled, enrich spend_logs_metadata with counts
if litellm_logging_obj:
per_server_tool_counts: Dict[str, int] = {}
for server, server_tools in zip(allowed_mcp_servers, results):
if server is None:
continue
server_key = (
getattr(server, "server_name", None)
or getattr(server, "alias", None)
or getattr(server, "name", None)
or "unknown"
)
per_server_tool_counts[str(server_key)] = len(server_tools)
per_server_tool_counts: Dict[str, int] = {
_aggregate_server_key(server): len(server_tools)
for server, (server_tools, _) in zip(allowed_mcp_servers, results)
if server is not None
}
metadata_dict = litellm_logging_obj.model_call_details.get("metadata")
if isinstance(metadata_dict, dict):
@ -1975,6 +2003,9 @@ if MCP_AVAILABLE:
spend_meta["allowed_server_count"] = len(allowed_mcp_servers)
spend_meta["tool_count_total"] = len(all_tools)
spend_meta["per_server_tool_counts"] = per_server_tool_counts
spend_meta["per_server_list_outcomes"] = {
key: outcome_wire_value(outcome) for key, outcome in server_outcomes.items()
}
end_time = datetime.now()
try:
@ -1995,7 +2026,7 @@ if MCP_AVAILABLE:
verbose_logger.info(f"Successfully fetched {len(all_tools)} tools total from all MCP servers")
return all_tools
return AggregateToolListing(tools=all_tools, outcomes=server_outcomes)
except Exception as e:
# Only fire failure hook if logging was requested for this list-tools execution
if log_list_tools_to_spendlogs and user_api_key_auth is not None:
@ -2218,43 +2249,6 @@ if MCP_AVAILABLE:
server = global_mcp_server_manager.get_mcp_server_by_id(server_id)
return [t for t in tools if strip_known_server_prefix(t.name, server) in allowed_tool_names]
async def _merge_toolset_permissions(
user_api_key_auth: Optional[UserAPIKeyAuth],
) -> Optional[UserAPIKeyAuth]:
"""
Resolve mcp_toolsets on the key's object_permission into tool-level permissions
and merge them (union) into object_permission.mcp_tool_permissions.
Returns the (possibly mutated copy of) user_api_key_auth.
"""
if user_api_key_auth is None:
return None
op = user_api_key_auth.object_permission
if op is None:
return user_api_key_auth
toolset_ids = getattr(op, "mcp_toolsets", None) or []
if not toolset_ids:
return user_api_key_auth
toolset_perms = await global_mcp_server_manager.resolve_toolset_tool_permissions(toolset_ids=toolset_ids)
if not toolset_perms:
return user_api_key_auth
# Merge toolset_perms into existing mcp_tool_permissions (union)
existing = dict(op.mcp_tool_permissions or {})
for server_id, tool_names in toolset_perms.items():
existing_tools = existing.get(server_id, [])
merged = list(set(existing_tools) | set(tool_names))
existing[server_id] = merged
# Build updated object_permission with merged tool permissions and server IDs.
# Union the toolset's server IDs into mcp_servers so downstream server-level
# filtering doesn't silently drop servers that the toolset references but that
# aren't already in the key's explicit mcp_servers list.
merged_servers = list(set(op.mcp_servers or []) | set(existing.keys()))
updated_op = op.model_copy(update={"mcp_servers": merged_servers, "mcp_tool_permissions": existing})
return user_api_key_auth.model_copy(update={"object_permission": updated_op})
async def _list_mcp_tools(
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
@ -2265,7 +2259,7 @@ if MCP_AVAILABLE:
log_list_tools_to_spendlogs: bool = False,
list_tools_log_source: Optional[str] = None,
client_ip: Optional[str] = None,
) -> List[MCPTool]:
) -> AggregateToolListing:
"""
List all available MCP tools.
@ -2277,19 +2271,14 @@ if MCP_AVAILABLE:
client_ip: Client IP for IP-based server access control
Returns:
List[MCPTool]: Combined list of tools from all accessible servers
AggregateToolListing: Combined tools from all accessible servers plus each server's
classified listing outcome
"""
if not MCP_AVAILABLE:
return []
return AggregateToolListing(tools=[], outcomes={})
# Resolve toolset permissions and merge into the key's object_permission
# so that the existing filter_tools_by_key_team_permissions logic picks them up.
user_api_key_auth = await _merge_toolset_permissions(user_api_key_auth)
# Get tools from managed MCP servers with error handling
managed_tools = []
try:
managed_tools = await _get_tools_from_mcp_servers(
listing = await _get_tools_from_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
@ -2300,12 +2289,12 @@ if MCP_AVAILABLE:
list_tools_log_source=list_tools_log_source,
client_ip=client_ip,
)
verbose_logger.debug(f"Successfully fetched {len(managed_tools)} tools from managed MCP servers")
verbose_logger.debug(f"Successfully fetched {len(listing.tools)} tools from managed MCP servers")
return listing
except Exception as e:
verbose_logger.exception(f"Error getting tools from managed MCP servers: {str(e)}")
# Continue with empty managed tools list instead of failing completely
return managed_tools
# Continue with an empty listing instead of failing completely
return AggregateToolListing(tools=[], outcomes={})
async def _list_mcp_prompts(
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
@ -3582,48 +3571,70 @@ if MCP_AVAILABLE:
# preemptive challenge and let downstream authorization
# return 403.
continue
if server and server.auth_type == MCPAuth.oauth2 and not oauth2_headers:
# For per-user OAuth servers, only skip the pre-emptive 401 when
# a stored token actually exists for this user+server pair.
# If no stored token exists, fail fast with 401 so clients can
# kick off PKCE/interactive OAuth flow immediately.
if server.needs_user_oauth_token:
if getattr(server, "delegate_auth_to_upstream", False) is True:
# Delegate-auth servers run upstream PKCE: challenge with
# the proxied resource_metadata (RFC 9728), not the
# gateway authorization_uri below which would authorize
# against the gateway instead of the upstream IdP.
www_authenticate = _get_passthrough_www_authenticate(
scope=scope,
server_name=server_name,
)
raise HTTPException(
status_code=401,
detail="Unauthorized",
headers={"www-authenticate": www_authenticate},
)
# The v2 resolver owns the existence check, so every authorization_code
# resolution (egress and this discovery challenge) runs through it.
if server and server.auth_type == MCPAuth.oauth2:
# The challenge decision is per oauth2 sub-mode, not per header:
# gateway-managed modes (M2M and interactive authorization_code)
# never receive a client-supplied upstream token, so a bearer in
# Authorization is a LiteLLM key (surfaced here as oauth2_headers)
# and must not suppress the challenge. Only the delegate mode
# treats a present bearer as the upstream token. The sub-mode is
# resolved the same way egress resolves it, via
# effective_oauth2_flow: an unstamped (null oauth2_flow) row with
# the M2M shape resolves to client_credentials, so the bare
# has_client_credentials column is never trusted here.
if MCPServerManager.effective_oauth2_flow(server) == "client_credentials":
# M2M: the gateway mints its own token at egress from the
# stored client credentials, so there is nothing to challenge.
continue
if getattr(server, "delegate_auth_to_upstream", False) is not True:
# Gateway-managed interactive (authorization_code): the only
# thing that authorizes egress is a stored per-user token, so
# challenge whenever one is absent, regardless of any bearer.
# The v2 resolver owns the existence check, so every
# authorization_code resolution (egress and this discovery
# challenge) runs through it.
if await global_mcp_server_manager.has_user_oauth_token(server, user_api_key_auth):
continue
request = StarletteRequest(scope)
base_url = get_request_base_url(request)
_path = scope.get("_original_path") or scope.get("path", "") or ""
request = StarletteRequest(scope)
base_url = get_request_base_url(request)
_path = scope.get("_original_path") or scope.get("path", "") or ""
# Pick the well-known AS-metadata form that matches the inbound route
# so strict RFC 9728 §3.2 clients can resolve it correctly.
if _path.startswith(f"/mcp/{server_name}"):
_as_url = f"{base_url}/.well-known/oauth-authorization-server/mcp/{server_name}"
else:
_as_url = f"{base_url}/.well-known/oauth-authorization-server/{server_name}"
authorization_uri = f'Bearer authorization_uri="{_as_url}"'
# Pick the well-known AS-metadata form that matches the inbound route
# so strict RFC 9728 §3.2 clients can resolve it correctly.
if _path.startswith(f"/mcp/{server_name}"):
_as_url = f"{base_url}/.well-known/oauth-authorization-server/mcp/{server_name}"
else:
_as_url = f"{base_url}/.well-known/oauth-authorization-server/{server_name}"
authorization_uri = f'Bearer authorization_uri="{_as_url}"'
raise HTTPException(
status_code=401,
detail="Unauthorized",
headers={"www-authenticate": authorization_uri},
)
raise HTTPException(
status_code=401,
detail="Unauthorized",
headers={"www-authenticate": authorization_uri},
)
if not oauth2_headers:
# Delegate-auth servers run upstream PKCE: a present bearer is
# the upstream token, so only challenge when it is absent, with
# the proxied resource_metadata (RFC 9728), not the gateway
# authorization_uri above which would authorize against the
# gateway instead of the upstream IdP.
www_authenticate = _get_passthrough_www_authenticate(
scope=scope,
server_name=server_name,
)
raise HTTPException(
status_code=401,
detail="Unauthorized",
headers={"www-authenticate": www_authenticate},
)
# Delegate server with a bearer present: it is the upstream token,
# so admit the session and move to the next target. Every oauth2
# sub-mode is terminal here (continue or raise) so no oauth2 server
# reaches the token_exchange / pass-through blocks below.
continue
# token_exchange (OBO): the caller supplied no subject token. Challenge at connect
# (transport level, where WWW-Authenticate survives) with the RFC 9728 resource_metadata

3
litellm/proxy/_experimental/mcp_server/tool_search.py
View file

@ -91,7 +91,7 @@ async def handle_mcp_tool_search(
from litellm.proxy._experimental.mcp_server.server import _list_mcp_tools
mcp_tools = await _list_mcp_tools(
mcp_listing = await _list_mcp_tools(
user_api_key_auth=user_api_key_dict,
mcp_servers=mcp_servers,
client_ip=client_ip,
@ -100,6 +100,7 @@ async def handle_mcp_tool_search(
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
mcp_tools = mcp_listing.tools
tools = [
{
"name": t.name,

2
litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/404/index.html
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt
View file

@ -1,9 +1,9 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","/litellm-asset-prefix/_next/static/chunks/0kc37~1yrtr2p.js","/litellm-asset-prefix/_next/static/chunks/04s-iyzsr4cq~.js","/litellm-asset-prefix/_next/static/chunks/16zj68af4snfa.js","/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","/litellm-asset-prefix/_next/static/chunks/0tmaomqtwbi33.js","/litellm-asset-prefix/_next/static/chunks/02-2~p5k.ielz.js","/litellm-asset-prefix/_next/static/chunks/0op63kdo3uwng.js","/litellm-asset-prefix/_next/static/chunks/112_-0alpxot8.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/12yfh0_n50ojz.js","/litellm-asset-prefix/_next/static/chunks/06w8_.601z7_i.js","/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/18187o3gb9vc5.js","/litellm-asset-prefix/_next/static/chunks/0._ir~nvcseg7.js","/litellm-asset-prefix/_next/static/chunks/0vffq7buvlg04.js","/litellm-asset-prefix/_next/static/chunks/122djf0bncn-8.js","/litellm-asset-prefix/_next/static/chunks/0axk76owb7jv..js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0sqw622fcvsv4.js","/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/14g~hmf3h_efw.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0cmepm.jkel-i.js","/litellm-asset-prefix/_next/static/chunks/0onea0n77pqw1.js","/litellm-asset-prefix/_next/static/chunks/02-u6qtmsnqn0.js","/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","/litellm-asset-prefix/_next/static/chunks/05q6y.kb.q2s..js","/litellm-asset-prefix/_next/static/chunks/0rehsq9xe1kde.js","/litellm-asset-prefix/_next/static/chunks/0nk7-_~gcxbz0.js","/litellm-asset-prefix/_next/static/chunks/0~r95y0t-0dlp.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"OutletBoundary"]
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","/litellm-asset-prefix/_next/static/chunks/0mqbd99.ej13v.js","/litellm-asset-prefix/_next/static/chunks/16ufy1iyybswo.js","/litellm-asset-prefix/_next/static/chunks/0ayum-x.hkww~.js","/litellm-asset-prefix/_next/static/chunks/15jl-1gcakfwa.js","/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","/litellm-asset-prefix/_next/static/chunks/0u.r3vzo30ofk.js","/litellm-asset-prefix/_next/static/chunks/0g4qcx-c9gsxn.js","/litellm-asset-prefix/_next/static/chunks/14a-un1blorp~.js","/litellm-asset-prefix/_next/static/chunks/02_q4881cz6h~.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0wdlbe750tuzr.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","/litellm-asset-prefix/_next/static/chunks/03zkt5iyjiqcz.js","/litellm-asset-prefix/_next/static/chunks/0l02mpo6za6ie.js","/litellm-asset-prefix/_next/static/chunks/1667t2pcy0iqm.js","/litellm-asset-prefix/_next/static/chunks/0-_m4km7b1~oe.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0sqw622fcvsv4.js","/litellm-asset-prefix/_next/static/chunks/0t8el_ijoskx..js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0a.ljputcx8g5.js","/litellm-asset-prefix/_next/static/chunks/02-u6qtmsnqn0.js","/litellm-asset-prefix/_next/static/chunks/076.vm.7w-x2..js","/litellm-asset-prefix/_next/static/chunks/069dx5~5osue0.js","/litellm-asset-prefix/_next/static/chunks/0onea0n77pqw1.js","/litellm-asset-prefix/_next/static/chunks/16jfov1k2wrj0.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","/litellm-asset-prefix/_next/static/chunks/01reddhq423_f.js","/litellm-asset-prefix/_next/static/chunks/0h80lrrstjswl.js","/litellm-asset-prefix/_next/static/chunks/0cmepm.jkel-i.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/04jv9e6~9vi.l.js","/litellm-asset-prefix/_next/static/chunks/0d_sm.._5mw-p.js","/litellm-asset-prefix/_next/static/chunks/0elr0ye86.44-.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"OutletBoundary"]
7:"$Sreact.suspense"
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0vffq7buvlg04.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/122djf0bncn-8.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0axk76owb7jv..js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0sqw622fcvsv4.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/14g~hmf3h_efw.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0cmepm.jkel-i.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0onea0n77pqw1.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/02-u6qtmsnqn0.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/05q6y.kb.q2s..js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0rehsq9xe1kde.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0nk7-_~gcxbz0.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/0~r95y0t-0dlp.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1667t2pcy0iqm.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-_m4km7b1~oe.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0sqw622fcvsv4.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0t8el_ijoskx..js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0a.ljputcx8g5.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/02-u6qtmsnqn0.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/076.vm.7w-x2..js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/069dx5~5osue0.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0onea0n77pqw1.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/16jfov1k2wrj0.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/01reddhq423_f.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0h80lrrstjswl.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0cmepm.jkel-i.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/04jv9e6~9vi.l.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/0d_sm.._5mw-p.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/0elr0ye86.44-.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"DSHomUr6Sq46Bm2WLdUas"}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
8:null

10
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt
View file

@ -1,7 +1,7 @@
1:"$Sreact.fragment"
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","/litellm-asset-prefix/_next/static/chunks/0kc37~1yrtr2p.js","/litellm-asset-prefix/_next/static/chunks/04s-iyzsr4cq~.js","/litellm-asset-prefix/_next/static/chunks/16zj68af4snfa.js","/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","/litellm-asset-prefix/_next/static/chunks/0tmaomqtwbi33.js","/litellm-asset-prefix/_next/static/chunks/02-2~p5k.ielz.js","/litellm-asset-prefix/_next/static/chunks/0op63kdo3uwng.js","/litellm-asset-prefix/_next/static/chunks/112_-0alpxot8.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/12yfh0_n50ojz.js","/litellm-asset-prefix/_next/static/chunks/06w8_.601z7_i.js","/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/18187o3gb9vc5.js","/litellm-asset-prefix/_next/static/chunks/0._ir~nvcseg7.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0kc37~1yrtr2p.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/04s-iyzsr4cq~.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/16zj68af4snfa.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0tmaomqtwbi33.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/02-2~p5k.ielz.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0op63kdo3uwng.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/112_-0alpxot8.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/12yfh0_n50ojz.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/06w8_.601z7_i.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/18187o3gb9vc5.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0._ir~nvcseg7.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","/litellm-asset-prefix/_next/static/chunks/0mqbd99.ej13v.js","/litellm-asset-prefix/_next/static/chunks/16ufy1iyybswo.js","/litellm-asset-prefix/_next/static/chunks/0ayum-x.hkww~.js","/litellm-asset-prefix/_next/static/chunks/15jl-1gcakfwa.js","/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","/litellm-asset-prefix/_next/static/chunks/0u.r3vzo30ofk.js","/litellm-asset-prefix/_next/static/chunks/0g4qcx-c9gsxn.js","/litellm-asset-prefix/_next/static/chunks/14a-un1blorp~.js","/litellm-asset-prefix/_next/static/chunks/02_q4881cz6h~.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0wdlbe750tuzr.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","/litellm-asset-prefix/_next/static/chunks/03zkt5iyjiqcz.js","/litellm-asset-prefix/_next/static/chunks/0l02mpo6za6ie.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0mqbd99.ej13v.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/16ufy1iyybswo.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0ayum-x.hkww~.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/15jl-1gcakfwa.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0u.r3vzo30ofk.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0g4qcx-c9gsxn.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/14a-un1blorp~.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/02_q4881cz6h~.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0wdlbe750tuzr.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/03zkt5iyjiqcz.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0l02mpo6za6ie.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"DSHomUr6Sq46Bm2WLdUas"}
6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

36
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -1,6 +1,6 @@
1:"$Sreact.fragment"
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"MetadataBoundary"]
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"DSHomUr6Sq46Bm2WLdUas"}

14
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -1,9 +1,9 @@
1:"$Sreact.fragment"
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
4:I[557951,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"AuthProvider"]
5:I[339756,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
6:I[837457,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
4:I[557951,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"AuthProvider"]
5:I[339756,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
6:I[837457,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/15j3hwz2dxrik.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/15j3hwz2dxrik.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
:HL["/litellm-asset-prefix/_next/static/chunks/0aapv6n5bztwf.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0aapv6n5bztwf.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"DSHomUr6Sq46Bm2WLdUas"}

4
litellm/proxy/_experimental/out/__next._tree.txt
View file

@ -1,4 +1,4 @@
:HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/15j3hwz2dxrik.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/0aapv6n5bztwf.css","style"]
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"DSHomUr6Sq46Bm2WLdUas"}

0
litellm/proxy/_experimental/out/_next/static/N7WCdfNd30Hp6HEF5tFIL/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/DSHomUr6Sq46Bm2WLdUas/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/N7WCdfNd30Hp6HEF5tFIL/_clientMiddlewareManifest.js → litellm/proxy/_experimental/out/_next/static/DSHomUr6Sq46Bm2WLdUas/_clientMiddlewareManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/N7WCdfNd30Hp6HEF5tFIL/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/DSHomUr6Sq46Bm2WLdUas/_ssgManifest.js
View file

1
litellm/proxy/_experimental/out/_next/static/chunks/0-_m4km7b1~oe.js Normal file
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/_next/static/chunks/0-ahu72ndvhwn.js Normal file
View file

File diff suppressed because one or more lines are too long

31
litellm/proxy/_experimental/out/_next/static/chunks/0-hrh_uw98wb_.js Normal file
View file

@ -0,0 +1,31 @@
(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,742732,(e,r,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0}),Object.defineProperty(t,"HeadManagerContext",{enumerable:!0,get:function(){return n}});let n=e.r(555682)._(e.r(271645)).default.createContext({})},18576,(e,r,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n={WarningIcon:function(){return d},errorStyles:function(){return l},errorThemeCss:function(){return a}};for(var o in n)Object.defineProperty(t,o,{enumerable:!0,get:n[o]});e.r(555682);let i=e.r(843476);e.r(271645);let l={container:{fontFamily:'system-ui,"Segoe UI",Roboto,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji"',height:"100vh",display:"flex",alignItems:"center",justifyContent:"center"},card:{marginTop:"-32px",maxWidth:"325px",padding:"32px 28px",textAlign:"left"},icon:{marginBottom:"24px"},title:{fontSize:"24px",fontWeight:500,letterSpacing:"-0.02em",lineHeight:"32px",margin:"0 0 12px 0",color:"var(--next-error-title)"},message:{fontSize:"14px",fontWeight:400,lineHeight:"21px",margin:"0 0 20px 0",color:"var(--next-error-message)"},form:{margin:0},buttonGroup:{display:"flex",gap:"8px",alignItems:"center"},button:{display:"inline-flex",alignItems:"center",justifyContent:"center",height:"32px",padding:"0 12px",fontSize:"14px",fontWeight:500,lineHeight:"20px",borderRadius:"6px",cursor:"pointer",color:"var(--next-error-btn-text)",background:"var(--next-error-btn-bg)",border:"var(--next-error-btn-border)"},buttonSecondary:{display:"inline-flex",alignItems:"center",justifyContent:"center",height:"32px",padding:"0 12px",fontSize:"14px",fontWeight:500,lineHeight:"20px",borderRadius:"6px",cursor:"pointer",color:"var(--next-error-btn-secondary-text)",background:"var(--next-error-btn-secondary-bg)",border:"var(--next-error-btn-secondary-border)"},digestFooter:{position:"fixed",bottom:"32px",left:"0",right:"0",textAlign:"center",fontFamily:'ui-monospace,SFMono-Regular,"SF Mono",Menlo,Consolas,monospace',fontSize:"12px",lineHeight:"18px",fontWeight:400,margin:"0",color:"var(--next-error-digest)"}},a=`
:root {
--next-error-bg: #fff;
--next-error-text: #171717;
--next-error-title: #171717;
--next-error-message: #171717;
--next-error-digest: #666666;
--next-error-btn-text: #fff;
--next-error-btn-bg: #171717;
--next-error-btn-border: none;
--next-error-btn-secondary-text: #171717;
--next-error-btn-secondary-bg: transparent;
--next-error-btn-secondary-border: 1px solid rgba(0,0,0,0.08);
}
@media (prefers-color-scheme: dark) {
:root {
--next-error-bg: #0a0a0a;
--next-error-text: #ededed;
--next-error-title: #ededed;
--next-error-message: #ededed;
--next-error-digest: #a0a0a0;
--next-error-btn-text: #0a0a0a;
--next-error-btn-bg: #ededed;
--next-error-btn-border: none;
--next-error-btn-secondary-text: #ededed;
--next-error-btn-secondary-bg: transparent;
--next-error-btn-secondary-border: 1px solid rgba(255,255,255,0.14);
}
}
body { margin: 0; color: var(--next-error-text); background: var(--next-error-bg); }
`.replace(/\n\s*/g,"");function d(){return(0,i.jsx)("svg",{width:"32",height:"32",viewBox:"-0.2 -1.5 32 32",fill:"none",style:l.icon,children:(0,i.jsx)("path",{d:"M16.9328 0C18.0839 0.000116771 19.1334 0.658832 19.634 1.69531L31.4299 26.1309C32.0708 27.4588 31.1036 28.9999 29.6291 29H2.00215C0.527541 29 -0.439628 27.4588 0.201371 26.1309L11.9973 1.69531C12.4979 0.658823 13.5474 7.75066e-05 14.6984 0H16.9328ZM3.59493 26H28.0363L16.9328 3H14.6984L3.59493 26ZM15.8156 19C16.9202 19.0001 17.8156 19.8955 17.8156 21C17.8156 22.1045 16.9202 22.9999 15.8156 23C14.7111 23 13.8156 22.1046 13.8156 21C13.8156 19.8954 14.7111 19 15.8156 19ZM17.3156 16.5H14.3156V8.5H17.3156V16.5Z",fill:"var(--next-error-title)"})})}("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),r.exports=t.default)},168027,(e,r,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0}),Object.defineProperty(t,"default",{enumerable:!0,get:function(){return l}}),e.r(555682);let n=e.r(843476);e.r(271645);let o=e.r(912354),i=e.r(18576),l=function({error:e}){let r=e?.digest,t=!!r;return(0,o.handleISRError)({error:e}),(0,n.jsxs)("html",{id:"__next_error__",children:[(0,n.jsx)("head",{children:(0,n.jsx)("style",{dangerouslySetInnerHTML:{__html:i.errorThemeCss}})}),(0,n.jsxs)("body",{children:[(0,n.jsx)("div",{style:i.errorStyles.container,children:(0,n.jsxs)("div",{style:i.errorStyles.card,children:[(0,n.jsx)(i.WarningIcon,{}),(0,n.jsx)("h1",{style:i.errorStyles.title,children:"This page couldn’t load"}),(0,n.jsx)("p",{style:i.errorStyles.message,children:t?"A server error occurred. Reload to try again.":"Reload to try again, or go back."}),(0,n.jsxs)("div",{style:i.errorStyles.buttonGroup,children:[(0,n.jsx)("form",{style:i.errorStyles.form,children:(0,n.jsx)("button",{type:"submit",style:i.errorStyles.button,children:"Reload"})}),!t&&(0,n.jsx)("button",{type:"button",style:i.errorStyles.buttonSecondary,onClick:()=>{window.history.length>1?window.history.back():window.location.href="/"},children:"Back"})]})]})}),r&&(0,n.jsxs)("p",{style:i.errorStyles.digestFooter,children:["ERROR ",r]})]})]})};("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),r.exports=t.default)}]);

48
litellm/proxy/_experimental/out/_next/static/chunks/0-px9-g~2oyp5.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0-~nw1zmks9_4.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0.3q2b74j~ty5.js Normal file
View file

File diff suppressed because one or more lines are too long

3
litellm/proxy/_experimental/out/_next/static/chunks/0._ir~nvcseg7.js
View file

File diff suppressed because one or more lines are too long

10
litellm/proxy/_experimental/out/_next/static/chunks/0.bx44y-6~tug.js
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/_next/static/chunks/0.cm9osit06~i.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/0.mwuwep0859t.js Normal file
View file

File diff suppressed because one or more lines are too long

10
litellm/proxy/_experimental/out/_next/static/chunks/0.p~s6ih~c~xe.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/003w1n3_ylv_2.js Normal file
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/_next/static/chunks/00ccjtnk99zr7.js Normal file
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more