chore: merge OCR cutover into input parity

Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com>

Dockerfile

@@ -86,7 +86,9 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra semantic-router \
     --python python3
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -100,7 +102,11 @@ USER root
 RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
 
 WORKDIR /app
-ENV PATH="/app/.venv/bin:${PATH}"
+ENV PATH="/app/.venv/bin:${PATH}" \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
+    PRISMA_OFFLINE_MODE=true
 
 # Copy only what runtime needs. The application is installed inside the venv;
 # the rest of the builder's /app is source and build metadata that must not
@@ -115,16 +121,18 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
 # enterprise.enterprise_hooks from it)
 COPY --from=builder /app/enterprise /app/enterprise
 COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
-# Prisma binaries live in $HOME/.cache (default prisma-python location),
-# which is /root/.cache here. Copy only the Prisma subdirs — copying the
-# whole /root/.cache drags in the uv build cache (~660 MB, includes a
-# setuptools wheel that surfaces as a CVE finding even though it's not
-# on the runtime sys.path).
-COPY --from=builder /root/.cache/prisma /root/.cache/prisma
-COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
+# Prisma CLI + engines are baked under /opt/prisma, a fixed path every
+# runtime uid can read and that no cache volume mount shadows. The paths are
+# pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and recorded into the
+# generated client at build time, so `prisma migrate deploy` on a fresh
+# database needs no npm and no network access (#33650, #24554).
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
 
 EXPOSE 4000/tcp
 

basedpyright-code-budget.json

@@ -24,7 +24,7 @@
     "limit": 42
   },
   "reportExplicitAny": {
-    "limit": 10397
+    "limit": 10389
   },
   "reportFunctionMemberAccess": {
     "limit": 11

docker/Dockerfile.database

@@ -84,7 +84,9 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra semantic-router \
     --python python3
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -97,7 +99,11 @@ USER root
 RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
 
 WORKDIR /app
-ENV PATH="/app/.venv/bin:${PATH}"
+ENV PATH="/app/.venv/bin:${PATH}" \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
+    PRISMA_OFFLINE_MODE=true
 
 # Copy only what runtime needs. The application is installed inside the venv;
 # the rest of the builder's /app is source and build metadata that must not
@@ -112,16 +118,20 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
 # enterprise.enterprise_hooks from it)
 COPY --from=builder /app/enterprise /app/enterprise
 COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
-# Prisma binaries live in $HOME/.cache (default prisma-python location),
-# which is /root/.cache here. Copy them from the builder so they survive
-# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
-# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
-# Only the Prisma subdirs: the whole /root/.cache drags in the uv build cache.
-COPY --from=builder /root/.cache/prisma /root/.cache/prisma
-COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
+# Prisma CLI + engines are baked under /opt/prisma, a fixed path every
+# runtime uid can read and that no cache volume mount shadows (unlike
+# /app/.cache or $HOME/.cache in readOnlyRootFilesystem + emptyDir setups).
+# The paths are pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and
+# recorded into the generated client at build time, so `prisma migrate
+# deploy` on a fresh database needs no npm and no network access
+# (#33650, #24554).
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
 
 EXPOSE 4000/tcp
 

litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_compression_saved_tokens/migration.sql

@@ -0,0 +1,17 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_mcp_server_oauth_client_table/migration.sql

@@ -0,0 +1,9 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_MCPServerOAuthClient" (
+    "server_id" TEXT NOT NULL,
+    "credentials" JSONB,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_MCPServerOAuthClient_pkey" PRIMARY KEY ("server_id")
+);

litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql

@@ -0,0 +1,23 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -396,6 +396,13 @@ model LiteLLM_MCPUserEnvVars {
   @@index([server_id])
 }
 
+model LiteLLM_MCPServerOAuthClient {
+  server_id   String   @id
+  credentials Json?
+  created_at  DateTime @default(now()) @map("created_at")
+  updated_at  DateTime @default(now()) @updatedAt @map("updated_at")
+}
+
 // Generate Tokens for Proxy
 model LiteLLM_VerificationToken {
     token      String   @id
@@ -729,6 +736,9 @@ model LiteLLM_DailyUserSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -760,6 +770,9 @@ model LiteLLM_DailyOrganizationSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -791,6 +804,9 @@ model LiteLLM_DailyEndUserSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -821,6 +837,9 @@ model LiteLLM_DailyAgentSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -851,6 +870,9 @@ model LiteLLM_DailyTeamSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -883,6 +905,9 @@ model LiteLLM_DailyTagSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.78"
+version = "0.4.79"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.78"
+version = "0.4.79"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/ADDING_A_PROVIDER.md

@@ -6,4 +6,23 @@ Three layers, same for every route (see `ocr` and `realtime` as references):
 2. **Provider config (pure)** — `crates/providers/src/<provider>/<route>/transformation.rs`: implement that trait as a `const <PROVIDER>_<ROUTE>_CONFIG`, mirroring the Python provider tree. Add parity unit tests.
 3. **HTTP / transport (the host)** — `crates/providers/src/<route>.rs` (e.g. `ocr.rs`, `realtime.rs`): the callable fn (`run_ocr`, `realtime`). It resolves the key, builds the auth header, builds URL + transforms via the config, then does the network call. This is the only layer allowed to do I/O.
 
+## Coding standards
+
+Before writing new logic, look for an existing base to extend. When a change is
+“the same behavior for one more provider/endpoint/integration”, the codebase
+almost always already has a shared abstraction for it (for example, provider
+`BaseConfig` transformation classes in `litellm/llms/base_llm/`, shared
+helpers in `litellm_core_utils/`, typed request/response models, or factory
+functions). Find it first with a search, then add the new variant by inheriting
+from or composing that base, overriding only what genuinely differs (model
+name, parameter mapping, or auth).
+
+Never copy an existing implementation and edit it in place, and never hand-roll
+a parallel version of logic a base already provides. If you catch yourself
+writing a second copy of a pattern that exists twice already, stop and extract a
+base instead: put the shared shape in one place and make both call sites thin
+variants of it. The test for a good abstraction is that adding the next provider
+is a few declarative lines, not a new file of duplicated flow. Only diverge from
+the base when behavior is genuinely different, and say so explicitly in the PR.
+
 **Calling:** the host invokes the route fn — the Python bridge calls `run_ocr`; the `ai-gateway` server calls `realtime`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.

litellm-rust/AGENTS.md

@@ -15,3 +15,12 @@ Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-
 Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these.
 
 Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional.
+
+## Style
+
+All Rust in `litellm-rust/` follows the official Rust Style Guide:
+https://doc.rust-lang.org/style-guide/
+
+`rustfmt` implements its formatting by default, so run `cargo fmt` before committing; CI gates every PR on `cargo fmt --check`. Do not hand-format against rustfmt or add a `rustfmt.toml` that diverges from the default style.
+
+Beyond formatting, follow the guide's naming and idiom conventions rustfmt cannot auto-apply: `snake_case` items/functions/modules, `UpperCamelCase` types/traits/variants, `SCREAMING_SNAKE_CASE` constants/statics (acronyms as one word, e.g. `HttpClient`), and the import grouping and item ordering it prescribes. See CLAUDE.md for the detailed version.

litellm-rust/CLAUDE.md

@@ -2,6 +2,25 @@
 
 This file defines the rules for Rust work in LiteLLM.
 
+## Provider Coding Standards
+
+Before writing new logic, look for an existing base to extend. When a change is
+“the same behavior for one more provider/endpoint/integration”, the codebase
+almost always already has a shared abstraction for it (for example, provider
+`BaseConfig` transformation classes in `litellm/llms/base_llm/`, shared
+helpers in `litellm_core_utils/`, typed request/response models, or factory
+functions). Find it first with a search, then add the new variant by inheriting
+from or composing that base, overriding only what genuinely differs (model
+name, parameter mapping, or auth).
+
+Never copy an existing implementation and edit it in place, and never hand-roll
+a parallel version of logic a base already provides. If you catch yourself
+writing a second copy of a pattern that exists twice already, stop and extract a
+base instead: put the shared shape in one place and make both call sites thin
+variants of it. The test for a good abstraction is that adding the next provider
+is a few declarative lines, not a new file of duplicated flow. Only diverge from
+the base when behavior is genuinely different, and say so explicitly in the PR.
+
 ## Crates (exactly three — see AGENTS.md)
 
 `litellm-core` describes work; `litellm-ai-gateway` executes it; `litellm-python-bridge`
@@ -20,6 +39,11 @@ Route-level Rust structure mirrors LiteLLM's Python responsibilities:
 - Network execution lives in the host crate `ai-gateway` (`ai-gateway/src/io/`),
   never inside `core`.
 
+Call-hook and lifecycle instrumentation, including phase timing, usage
+accumulation, and callback payload construction, always lives in `core`.
+Hosts feed observed events into core and dispatch the completed payloads through
+their I/O logger; hosts must not own callback orchestration.
+
 Allowed in `core`:
 - Pure request transforms
 - Pure response transforms
@@ -77,6 +101,26 @@ such as `ai-gateway`, router hosts, or standalone servers:
 - Avoid `expect`/`unwrap` in server startup and request paths unless the panic is
   impossible by construction and documented.
 
+## Rust Style Guide
+
+All Rust in `litellm-rust/` follows the official Rust Style Guide:
+https://doc.rust-lang.org/style-guide/
+
+`rustfmt` implements the guide's formatting rules by default, so the mechanical
+side is enforced for you: run `cargo fmt` before committing and CI gates every
+PR on `cargo fmt --check` (see Checks). Do not hand-format against rustfmt or add
+a `rustfmt.toml` that diverges from the default style; the default style *is* the
+guide.
+
+The guide also covers conventions rustfmt cannot auto-apply; follow these too:
+- Naming: `snake_case` for items, functions, and modules; `UpperCamelCase` for
+  types, traits, and enum variants; `SCREAMING_SNAKE_CASE` for constants and
+  statics; acronyms count as one word (`HttpClient`, not `HTTPClient`).
+- Ordering and grouping the guide prescribes: imports grouped std / external /
+  crate-local, derives before other attributes, and consistent item order.
+- Idioms the guide recommends over the formatter fighting you (e.g. prefer
+  restructuring an over-long expression rather than forcing an awkward wrap).
+
 ## Constants
 
 Magic numbers and fixed strings go in a crate-level `constants.rs`, never

litellm-rust/Cargo.lock

@@ -1012,6 +1012,7 @@ dependencies = [
  "subtle",
  "tokio",
  "tokio-tungstenite",
+ "tower",
 ]
 
 [[package]]

litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md

@@ -0,0 +1,53 @@
+# Provider coding standards (litellm-rust)
+
+Rules for adding or changing an LLM provider/route in `litellm-rust`. OCR (`MISTRAL_OCR_CONFIG`) is the reference; `messages` (`ANTHROPIC_MESSAGES_CONFIG`) is the next port.
+
+## Provider resolution
+
+1. Always resolve the provider/model first with `get_custom_llm_provider` (`core/src/routing_utils/provider.rs`). Nothing downstream may branch on a raw model string.
+2. Model/provider is resolved once, in `prepare.rs`, and passed down as typed fields. Don't re-resolve or re-parse it in transforms or handlers.
+
+## Transforms and the base config
+
+3. Every route defines a base config trait with `transform_request` + `transform_response` (+ `complete_url`, `supported_params`), living in `core/src/<route>/transformation.rs` (e.g. `AnthropicMessagesProviderConfig`, mirroring `OcrProviderConfig`).
+4. Each provider implements that trait as a `const <PROVIDER>_<ROUTE>_CONFIG` in `core/src/providers/<provider>/<route>/transformation.rs`, mirroring the Python provider tree.
+5. Individual configs implement only the request/response transforms. Shared behavior (param filtering, defaults) stays as trait default methods so future providers inherit existing logic instead of reimplementing it.
+6. Prefer composition: a provider that extends another reuses the base trait's defaults or wraps another config; don't copy transform bodies between providers.
+
+## Boundaries
+
+7. Layers never cross: `core` = pure transforms/types (no network, env, secrets, auth, logging, global mutable state); `ai-gateway` = all I/O, auth headers, HTTP/SSE, lifecycle hooks; `python-bridge` = thin PyO3 adapter.
+8. Generic/route files contain zero provider-specific branches. A provider is one module under `core/src/providers/<provider>/<route>/`; a route is a module, never a new crate.
+9. Route entry point stays thin: `<route>()` -> `prepare_*` -> `CallLifecycle::run_request`, which owns the pre_call -> during_call -> provider call -> success/failure order and phase timing. Handlers validate and delegate; no business logic in them.
+10. Constants (URLs, env-var names, API versions, error messages) live in a crate `constants.rs`, never inline. Env reads happen only at the host/config layer, with the `DEFAULT_*` fallback defined in `constants.rs`.
+
+## Types and errors
+
+11. Typed contracts only: no bare `serde_json::Value` / `String` / `Vec<String>` as a transform input or output. Parse wire bytes into typed structs/enums at the host edge; a `type` discriminator is a typed field, not a raw string.
+12. Model failures as values: return typed `CoreError`, don't panic. No `unwrap`/`expect`/`panic!` on user or provider input.
+13. No mutation: build values in one shot (comprehensions/iterators, `collect`), prefer immutable bindings and owned typed structs over seeding-and-mutating.
+14. Early returns over deep nesting; small focused files over god modules.
+15. Preserve Python output shape intentionally. If a field is always serialized as `null` for parity, keep it and pin it with a test.
+
+## Safety and data minimization
+
+16. Never log request/response bodies, base64 payloads, document contents, or secrets. Truncate and bound any upstream body before it crosses a host boundary.
+17. Treat empty/whitespace credentials, URLs, and config values as absent at the host resolution layer.
+18. Host I/O sets connect + request timeouts (no unbounded waits), reuses a shared HTTP client, and prefers rustls TLS.
+
+## Tests and rollout
+
+19. Every provider transform ships tests for: supported-param filtering, request body shape, response normalization, missing/null fields, bad input, and `*_match_python` fixture parity.
+20. Lifecycle/hook tests cover hook order, success + failure callback payloads, pre-call guardrail blocking before any provider I/O, during-call body mutation, and provider-error mapping.
+21. Rust paths stay off by default and behind Python parity tests (disabled / enabled-equals-Python / bridge-unavailable fallback) until parity is proven.
+
+## Checks before push
+
+22. Run, and keep green:
+    ```bash
+    cd litellm-rust
+    cargo fmt --check
+    cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
+    cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings
+    cargo test --workspace
+    ```

litellm-rust/crates/ai-gateway/Cargo.toml

@@ -43,3 +43,4 @@ python-config = ["dep:pyo3"]
 
 [dev-dependencies]
 futures-channel = "0.3"
+tower = { version = "0.5.3", features = ["util"] }

litellm-rust/crates/ai-gateway/src/constants.rs

@@ -28,7 +28,6 @@ pub(crate) const DEFAULT_FLUSH_INTERVAL_MS: u64 = 500;
 /// Provider attributed to realtime sessions in the logging payload.
 #[cfg(feature = "server")]
 pub(crate) const DEFAULT_PROVIDER: &str = "openai";
-
 pub(crate) const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
 
 pub(crate) const VERTEXAI_CREDENTIALS_ENV: &str = "VERTEXAI_CREDENTIALS";
@@ -46,3 +45,31 @@ pub(crate) const AZURE_DOCUMENT_INTELLIGENCE_POLL_TIMEOUT_SECS: u64 = 120;
 pub(crate) const DEFAULT_MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: f64 = 50.0;
 
 pub(crate) const MAX_SAFE_FETCH_REDIRECTS: usize = 10;
+
+/// Full-request timeout ceiling for Anthropic Messages provider calls, in
+/// seconds. Mirrors the Python Anthropic Messages default. The per-request
+/// timeout from `litellm_params` still overrides this on the request builder.
+pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600;
+
+/// Connect timeout for Anthropic Messages provider calls, in seconds.
+pub(crate) const MESSAGES_CONNECT_TIMEOUT_SECS: u64 = 10;
+
+/// Max characters of an upstream error body echoed across the host boundary
+/// before truncation, so provider bodies are bounded and data-minimized.
+pub(crate) const MESSAGES_ERROR_BODY_MAX_CHARS: usize = 256;
+
+pub(crate) const DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS: u64 = 10;
+pub(crate) const DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS: u64 = 300;
+
+/// HTTP path for the non-streaming Anthropic Messages route.
+#[cfg(feature = "server")]
+pub(crate) const MESSAGES_ROUTE_PATH: &str = "/v1/messages";
+
+/// Provider name used by the Anthropic Messages route when a deployment's
+/// provider model does not carry an explicit provider prefix.
+pub(crate) const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic";
+
+/// Request headers owned by the gateway and never forwarded upstream.
+#[cfg(feature = "server")]
+pub(crate) const MESSAGES_HEADERS_NOT_FORWARDED: &[&str] =
+    &["authorization", "connection", "content-length", "host"];

litellm-rust/crates/ai-gateway/src/io/messages.rs

@@ -0,0 +1 @@
+pub use crate::messages::{messages, MessagesRequest};

litellm-rust/crates/ai-gateway/src/io/mod.rs

@@ -1,4 +1,6 @@
+pub mod messages;
 pub mod ocr;
 pub mod realtime;
 pub mod realtime_pool;
+pub mod responses_ws;
 pub mod vertex_ai;

litellm-rust/crates/ai-gateway/src/io/responses_ws.rs

@@ -0,0 +1,548 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+
+use futures_util::stream::{SplitSink, SplitStream};
+use futures_util::{Sink, SinkExt, Stream, StreamExt};
+use litellm_core::providers::openai::responses::transformation::OPENAI_RESPONSES_WS_CONFIG;
+use litellm_core::responses::types::ResponsesWsEvent;
+use litellm_core::responses::websocket::ResponsesWebSocketProviderConfig;
+use litellm_core::{CoreError, CoreResult};
+use tokio::net::TcpStream;
+use tokio::sync::Mutex;
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::http::header::{HeaderName, AUTHORIZATION};
+use tokio_tungstenite::tungstenite::http::HeaderValue;
+use tokio_tungstenite::tungstenite::Message;
+use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream};
+
+use crate::constants::{
+    DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS, DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS,
+};
+
+const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY";
+const MISSING_KEY_MESSAGE: &str =
+    "Missing OpenAI API Key - a Responses WebSocket call is being made but no key was passed via params or the OPENAI_API_KEY environment variable";
+
+pub type ResponsesUpstreamWs = WebSocketStream<MaybeTlsStream<TcpStream>>;
+type UpstreamTx = SplitSink<ResponsesUpstreamWs, Message>;
+type UpstreamRx = SplitStream<ResponsesUpstreamWs>;
+
+#[derive(Clone)]
+pub struct ResponsesWebSocketConnection {
+    socket: Arc<Mutex<Option<ResponsesUpstreamWs>>>,
+}
+
+impl ResponsesWebSocketConnection {
+    pub async fn connect_url(
+        url: &str,
+        headers: &HashMap<String, String>,
+        timeout: Option<Duration>,
+    ) -> CoreResult<Self> {
+        let mut request = url
+            .into_client_request()
+            .map_err(|error| CoreError::Network(error.to_string()))?;
+        for (name, value) in headers {
+            let header_name = name
+                .parse::<HeaderName>()
+                .map_err(|error| CoreError::InvalidRequest(error.to_string()))?;
+            let header_value = HeaderValue::from_str(value)
+                .map_err(|error| CoreError::InvalidRequest(error.to_string()))?;
+            request.headers_mut().insert(header_name, header_value);
+        }
+        let connect = connect_async(request);
+        let result = match timeout {
+            Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| {
+                CoreError::Network("Responses WebSocket connection timed out".to_string())
+            })?,
+            None => connect.await,
+        };
+        let (socket, _) = result.map_err(|error| match error {
+            tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http {
+                status: response.status().as_u16(),
+                body: String::new(),
+            },
+            other => CoreError::Network(other.to_string()),
+        })?;
+        Ok(Self {
+            socket: Arc::new(Mutex::new(Some(socket))),
+        })
+    }
+
+    pub async fn send_text(&self, text: String) -> CoreResult<()> {
+        let mut socket = self.socket.lock().await;
+        let Some(socket) = socket.as_mut() else {
+            return Err(CoreError::Network(
+                "Responses WebSocket is closed".to_string(),
+            ));
+        };
+        socket
+            .send(Message::Text(text))
+            .await
+            .map_err(|error| CoreError::Network(error.to_string()))
+    }
+
+    pub async fn recv_text(&self) -> CoreResult<Option<String>> {
+        let mut socket = self.socket.lock().await;
+        let Some(socket) = socket.as_mut() else {
+            return Ok(None);
+        };
+        match socket.next().await {
+            Some(Ok(Message::Text(text))) => Ok(Some(text)),
+            Some(Ok(Message::Binary(bytes))) => String::from_utf8(bytes.to_vec())
+                .map(Some)
+                .map_err(|error| CoreError::InvalidResponse(error.to_string())),
+            Some(Ok(Message::Close(_))) | None => Ok(None),
+            Some(Ok(_)) => Ok(None),
+            Some(Err(error)) => Err(CoreError::Network(error.to_string())),
+        }
+    }
+
+    pub async fn close(&self) -> CoreResult<()> {
+        let mut socket = self.socket.lock().await;
+        if let Some(socket) = socket.as_mut() {
+            socket
+                .close(None)
+                .await
+                .map_err(|error| CoreError::Network(error.to_string()))?;
+        }
+        *socket = None;
+        Ok(())
+    }
+}
+
+pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult<String> {
+    api_key
+        .map(str::trim)
+        .filter(|value| !value.is_empty())
+        .map(str::to_string)
+        .or_else(|| {
+            std::env::var(OPENAI_API_KEY_ENV)
+                .ok()
+                .filter(|value| !value.trim().is_empty())
+        })
+        .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string()))
+}
+
+async fn dial_upstream(
+    model: &str,
+    api_key: &str,
+    api_base: Option<&str>,
+) -> CoreResult<ResponsesUpstreamWs> {
+    let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model);
+    let mut request = url
+        .as_str()
+        .into_client_request()
+        .map_err(|error| CoreError::Network(error.to_string()))?;
+    request.headers_mut().insert(
+        AUTHORIZATION,
+        HeaderValue::from_str(&format!("Bearer {api_key}"))
+            .map_err(|error| CoreError::Auth(error.to_string()))?,
+    );
+    let result = tokio::time::timeout(
+        Duration::from_secs(DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS),
+        connect_async(request),
+    )
+    .await
+    .map_err(|_| CoreError::Network("Responses WebSocket connection timed out".to_string()))?;
+    result
+        .map(|(socket, _)| socket)
+        .map_err(|error| match error {
+            tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http {
+                status: response.status().as_u16(),
+                body: String::new(),
+            },
+            other => CoreError::Network(other.to_string()),
+        })
+}
+
+pub struct ResponsesWebSocketStreaming;
+
+impl ResponsesWebSocketStreaming {
+    pub async fn bidirectional_forward<In, Out>(
+        model: &str,
+        upstream_tx: UpstreamTx,
+        upstream_rx: UpstreamRx,
+        idle_timeout: Option<Duration>,
+        observe: impl FnMut(&ResponsesWsEvent) + Send,
+        client_in: In,
+        client_out: Out,
+    ) -> CoreResult<()>
+    where
+        In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
+        Out: Sink<ResponsesWsEvent> + Unpin + Send,
+        Out::Error: std::fmt::Display,
+    {
+        splice(
+            model,
+            upstream_tx,
+            upstream_rx,
+            idle_timeout,
+            observe,
+            client_in,
+            client_out,
+        )
+        .await
+    }
+}
+
+pub(crate) async fn splice<In, Out>(
+    model: &str,
+    mut upstream_tx: UpstreamTx,
+    mut upstream_rx: UpstreamRx,
+    idle_timeout: Option<Duration>,
+    mut observe: impl FnMut(&ResponsesWsEvent) + Send,
+    mut client_in: In,
+    mut client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
+    Out: Sink<ResponsesWsEvent> + Unpin + Send,
+    Out::Error: std::fmt::Display,
+{
+    let idle =
+        idle_timeout.unwrap_or_else(|| Duration::from_secs(DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS));
+    loop {
+        tokio::select! {
+            event = client_in.next() => {
+                let Some(event) = event else { break };
+                for outbound in OPENAI_RESPONSES_WS_CONFIG
+                    .transform_ws_request(&event, model)?
+                    .events
+                {
+                    let payload = serde_json::to_string(&outbound)
+                        .map_err(|error| CoreError::InvalidResponse(error.to_string()))?;
+                    upstream_tx.send(Message::Text(payload))
+                        .await
+                        .map_err(|error| CoreError::Network(error.to_string()))?;
+                }
+            }
+            message = upstream_rx.next() => {
+                let Some(message) = message else { break };
+                match message.map_err(|error| CoreError::Network(error.to_string()))? {
+                    Message::Text(text) => {
+                        let event = serde_json::from_str::<ResponsesWsEvent>(&text)
+                            .map_err(|error| CoreError::InvalidResponse(error.to_string()))?;
+                        observe(&event);
+                        for outbound in OPENAI_RESPONSES_WS_CONFIG
+                            .transform_ws_response(&event, model)?
+                            .events
+                        {
+                            client_out.send(outbound)
+                                .await
+                                .map_err(|error| CoreError::Network(error.to_string()))?;
+                        }
+                    }
+                    Message::Close(_) => break,
+                    _ => {}
+                }
+            }
+            _ = tokio::time::sleep(idle) => break,
+        }
+    }
+    Ok(())
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn async_responses_websocket<In, Out>(
+    model: &str,
+    api_key: Option<&str>,
+    api_base: Option<&str>,
+    first_frame: Option<ResponsesWsEvent>,
+    idle_timeout: Option<Duration>,
+    mut observe: impl FnMut(&ResponsesWsEvent) + Send,
+    client_in: In,
+    client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
+    Out: Sink<ResponsesWsEvent> + Unpin + Send,
+    Out::Error: std::fmt::Display,
+{
+    let key = resolve_api_key(api_key)?;
+    let upstream = dial_upstream(model, &key, api_base).await?;
+    let (mut upstream_tx, upstream_rx) = upstream.split();
+    if let Some(first_frame) = first_frame {
+        for outbound in OPENAI_RESPONSES_WS_CONFIG
+            .transform_ws_request(&first_frame, model)?
+            .events
+        {
+            let payload = serde_json::to_string(&outbound)
+                .map_err(|error| CoreError::InvalidResponse(error.to_string()))?;
+            upstream_tx
+                .send(Message::Text(payload))
+                .await
+                .map_err(|error| CoreError::Network(error.to_string()))?;
+        }
+    }
+    ResponsesWebSocketStreaming::bidirectional_forward(
+        model,
+        upstream_tx,
+        upstream_rx,
+        idle_timeout,
+        &mut observe,
+        client_in,
+        client_out,
+    )
+    .await
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn responses_ws<In, Out>(
+    model: &str,
+    api_key: Option<&str>,
+    api_base: Option<&str>,
+    first_frame: Option<ResponsesWsEvent>,
+    idle_timeout: Option<Duration>,
+    observe: impl FnMut(&ResponsesWsEvent) + Send,
+    client_in: In,
+    client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
+    Out: Sink<ResponsesWsEvent> + Unpin + Send,
+    Out::Error: std::fmt::Display,
+{
+    async_responses_websocket(
+        model,
+        api_key,
+        api_base,
+        first_frame,
+        idle_timeout,
+        observe,
+        client_in,
+        client_out,
+    )
+    .await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures_channel::mpsc;
+    use futures_util::{SinkExt, StreamExt};
+    use litellm_core::responses::types::ResponsesWsEventType;
+    use serde_json::json;
+    use tokio::io::AsyncWriteExt;
+    use tokio::net::TcpListener;
+    use tokio_tungstenite::accept_async;
+
+    async fn websocket_base() -> (String, tokio::task::JoinHandle<()>) {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
+        let address = listener.local_addr().expect("local address");
+        let task = tokio::spawn(async move {
+            let (stream, _) = listener.accept().await.expect("accept");
+            let mut socket = accept_async(stream).await.expect("websocket handshake");
+            while let Some(Ok(Message::Text(text))) = socket.next().await {
+                let request: serde_json::Value = serde_json::from_str(&text).expect("request json");
+                let model = request
+                    .get("model")
+                    .and_then(serde_json::Value::as_str)
+                    .or_else(|| {
+                        request
+                            .get("response")
+                            .and_then(serde_json::Value::as_object)
+                            .and_then(|response| {
+                                response.get("model").and_then(serde_json::Value::as_str)
+                            })
+                    })
+                    .expect("enforced model");
+                socket
+                    .send(Message::Text(
+                        json!({
+                            "type": "response.created",
+                            "response": {
+                                "id": format!("resp-{model}"),
+                                "model": model,
+                                "extra": "preserved"
+                            }
+                        })
+                        .to_string(),
+                    ))
+                    .await
+                    .expect("created event");
+                socket
+                    .send(Message::Text(
+                        json!({
+                            "type": "response.completed",
+                            "response": {
+                                "id": format!("resp-{model}"),
+                                "model": model,
+                                "usage": {
+                                    "input_tokens": 1,
+                                    "output_tokens": 2,
+                                    "total_tokens": 3
+                                }
+                            }
+                        })
+                        .to_string(),
+                    ))
+                    .await
+                    .expect("completed event");
+            }
+        });
+        (format!("http://{address}"), task)
+    }
+
+    fn event(value: serde_json::Value) -> ResponsesWsEvent {
+        serde_json::from_value(value).expect("event")
+    }
+
+    #[test]
+    fn explicit_nonblank_key_wins() {
+        assert_eq!(
+            resolve_api_key(Some(" explicit ")).expect("key"),
+            "explicit"
+        );
+    }
+
+    #[test]
+    fn blank_key_is_not_accepted_without_environment_key() {
+        if std::env::var(OPENAI_API_KEY_ENV).is_err() {
+            assert!(resolve_api_key(Some("  ")).is_err());
+        }
+    }
+
+    #[tokio::test]
+    async fn forwards_events_sequentially_and_enforces_model() {
+        let (api_base, server) = websocket_base().await;
+        let (client_tx, client_rx) = mpsc::unbounded();
+        let (output_tx, mut output_rx) = mpsc::unbounded();
+        let (observed_tx, observed_rx) = mpsc::unbounded();
+        client_tx
+            .unbounded_send(event(json!({
+                "type": "response.create",
+                "model": "wrong"
+            })))
+            .expect("first request");
+        client_tx
+            .unbounded_send(event(json!({
+                "type": "response.create",
+                "response": {"model": "also-wrong"}
+            })))
+            .expect("second request");
+
+        let task = tokio::spawn(async move {
+            responses_ws(
+                "authorized-model",
+                Some("test-key"),
+                Some(&api_base),
+                None,
+                Some(Duration::from_secs(1)),
+                move |event| {
+                    observed_tx
+                        .unbounded_send(event.clone())
+                        .expect("observe event");
+                },
+                client_rx,
+                output_tx,
+            )
+            .await
+        });
+
+        let first = output_rx.next().await.expect("first output");
+        let second = output_rx.next().await.expect("second output");
+        let third = output_rx.next().await.expect("third output");
+        let fourth = output_rx.next().await.expect("fourth output");
+        drop(client_tx);
+        task.await.expect("splice task").expect("successful splice");
+        server.await.expect("server task");
+
+        assert_eq!(first.event_type, ResponsesWsEventType::ResponseCreated);
+        assert_eq!(first.model(), Some("authorized-model"));
+        assert_eq!(first.data["response"]["extra"], "preserved");
+        assert_eq!(second.event_type, ResponsesWsEventType::ResponseCompleted);
+        assert_eq!(third.event_type, ResponsesWsEventType::ResponseCreated);
+        assert_eq!(fourth.event_type, ResponsesWsEventType::ResponseCompleted);
+        let observed: Vec<_> = observed_rx.collect().await;
+        assert_eq!(observed.len(), 4);
+        assert!(observed
+            .iter()
+            .all(|event| event.event_type != ResponsesWsEventType::ResponseCreate));
+    }
+
+    #[tokio::test]
+    async fn idle_timeout_ends_without_upstream_events() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
+        let address = listener.local_addr().expect("address");
+        let server = tokio::spawn(async move {
+            let (stream, _) = listener.accept().await.expect("accept");
+            let _socket = accept_async(stream).await.expect("handshake");
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        });
+        let (_client_tx, client_rx) = mpsc::unbounded::<ResponsesWsEvent>();
+        let (output_tx, mut output_rx) = mpsc::unbounded();
+        let result = responses_ws(
+            "model",
+            Some("key"),
+            Some(&format!("http://{address}")),
+            None,
+            Some(Duration::from_millis(20)),
+            |_| {},
+            client_rx,
+            output_tx,
+        )
+        .await;
+        assert!(result.is_ok());
+        assert!(output_rx.next().await.is_none());
+        server.abort();
+    }
+
+    #[tokio::test]
+    async fn dial_http_status_is_preserved() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
+        let address = listener.local_addr().expect("address");
+        let server = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("accept");
+            stream
+                .write_all(b"HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\n\r\n")
+                .await
+                .expect("response");
+        });
+        let (_client_tx, client_rx) = mpsc::unbounded::<ResponsesWsEvent>();
+        let (output_tx, _output_rx) = mpsc::unbounded();
+        let error = responses_ws(
+            "model",
+            Some("key"),
+            Some(&format!("http://{address}")),
+            None,
+            Some(Duration::from_millis(20)),
+            |_| {},
+            client_rx,
+            output_tx,
+        )
+        .await
+        .expect_err("status error");
+        assert!(matches!(error, CoreError::Http { status: 401, .. }));
+        server.await.expect("server task");
+    }
+
+    #[tokio::test]
+    async fn dial_http_500_status_is_preserved() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
+        let address = listener.local_addr().expect("address");
+        let server = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("accept");
+            stream
+                .write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n")
+                .await
+                .expect("response");
+        });
+        let (_client_tx, client_rx) = mpsc::unbounded::<ResponsesWsEvent>();
+        let (output_tx, _output_rx) = mpsc::unbounded();
+        let error = responses_ws(
+            "model",
+            Some("key"),
+            Some(&format!("http://{address}")),
+            None,
+            Some(Duration::from_millis(20)),
+            |_| {},
+            client_rx,
+            output_tx,
+        )
+        .await
+        .expect_err("status error");
+        assert!(matches!(error, CoreError::Http { status: 500, .. }));
+        server.await.expect("server task");
+    }
+}

litellm-rust/crates/ai-gateway/src/lib.rs

@@ -13,6 +13,7 @@
 
 pub(crate) mod config;
 pub mod io;
+pub mod messages;
 pub mod ocr;
 
 /// GIL-activity tracking. Pure (atomics only); shared by the `server` routes and
@@ -26,9 +27,6 @@ pub mod routes;
 #[cfg(feature = "server")]
 pub mod state;
 
-// Realtime request logging. Only the server serves realtime, so these are
-// `server`-gated; `io::realtime` exposes the generic `observe` hook while the
-// collector and callback fan-out live here.
 mod constants;
 pub mod integrations;
 #[cfg(feature = "server")]

litellm-rust/crates/ai-gateway/src/messages/client.rs

@@ -0,0 +1,15 @@
+use std::sync::OnceLock;
+use std::time::Duration;
+
+use crate::constants::{MESSAGES_CONNECT_TIMEOUT_SECS, MESSAGES_TIMEOUT_SECS};
+
+pub(super) fn http_client() -> &'static reqwest::Client {
+    static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
+    CLIENT.get_or_init(|| {
+        reqwest::Client::builder()
+            .timeout(Duration::from_secs(MESSAGES_TIMEOUT_SECS))
+            .connect_timeout(Duration::from_secs(MESSAGES_CONNECT_TIMEOUT_SECS))
+            .build()
+            .unwrap_or_else(|_| reqwest::Client::new())
+    })
+}

litellm-rust/crates/ai-gateway/src/messages/common_utils.rs

@@ -0,0 +1,52 @@
+use litellm_core::error::{json_type_name, CoreError};
+use litellm_core::messages::transformation::AnthropicMessagesProviderConfig;
+use litellm_core::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG;
+use litellm_core::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG;
+use litellm_core::CoreResult;
+use serde_json::{Map, Value};
+
+use crate::constants::MESSAGES_ERROR_BODY_MAX_CHARS;
+
+pub(super) fn truncate_error_body(body: &str) -> String {
+    if body.chars().count() <= MESSAGES_ERROR_BODY_MAX_CHARS {
+        return body.to_string();
+    }
+    let truncated: String = body.chars().take(MESSAGES_ERROR_BODY_MAX_CHARS).collect();
+    format!("{truncated}... (truncated)")
+}
+
+pub(super) fn messages_provider_config(
+    provider: &str,
+) -> Option<&'static dyn AnthropicMessagesProviderConfig> {
+    match provider {
+        "anthropic" => Some(&ANTHROPIC_MESSAGES_CONFIG),
+        "azure_ai" => Some(&AZURE_ANTHROPIC_MESSAGES_CONFIG),
+        _ => None,
+    }
+}
+
+pub(super) fn string_headers(
+    extra_headers: Option<Map<String, Value>>,
+) -> CoreResult<Vec<(String, String)>> {
+    extra_headers
+        .unwrap_or_default()
+        .into_iter()
+        .map(|(key, value)| {
+            value
+                .as_str()
+                .map(|value| (key.clone(), value.to_string()))
+                .ok_or_else(|| {
+                    CoreError::InvalidRequest(format!(
+                        "messages extra_headers.{key} must be a string, got {}",
+                        json_type_name(&value)
+                    ))
+                })
+        })
+        .collect()
+}
+
+pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool {
+    headers
+        .iter()
+        .any(|(key, _)| key.eq_ignore_ascii_case(name))
+}

litellm-rust/crates/ai-gateway/src/messages/handler.rs

@@ -0,0 +1,83 @@
+use litellm_core::error::CoreError;
+use litellm_core::CoreResult;
+use serde_json::Value;
+
+use super::client::http_client;
+use super::common_utils::truncate_error_body;
+use super::types::ProviderMessagesRequest;
+use crate::constants::ANTHROPIC_MESSAGES_PROVIDER;
+
+pub(super) async fn execute_messages_provider_call(
+    request: ProviderMessagesRequest,
+) -> CoreResult<Value> {
+    let mut request_builder = http_client().post(&request.url).json(&request.body);
+    for (key, value) in &request.upstream_headers {
+        request_builder = request_builder.header(key, value);
+    }
+    if let Some(duration) = request.timeout {
+        request_builder = request_builder.timeout(duration);
+    }
+
+    let response = request_builder
+        .send()
+        .await
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+
+    let status = response.status();
+    let text = response
+        .text()
+        .await
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+
+    if !status.is_success() {
+        return Err(CoreError::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&text),
+        });
+    }
+
+    let response = serde_json::from_str(&text).map_err(|err| {
+        CoreError::InvalidResponse(format!("invalid messages response JSON: {err}"))
+    })?;
+    let transformed = request
+        .config
+        .transform_response(&request.model, response)?;
+    serde_json::to_value(transformed).map_err(|err| {
+        CoreError::InvalidResponse(format!("failed to serialize messages response: {err}"))
+    })
+}
+
+pub(super) async fn execute_messages_provider_stream(
+    request: ProviderMessagesRequest,
+) -> CoreResult<reqwest::Response> {
+    if request.provider != ANTHROPIC_MESSAGES_PROVIDER {
+        return Err(CoreError::InvalidRequest(
+            "streaming messages is not supported for this provider".to_string(),
+        ));
+    }
+
+    let mut request_builder = http_client().post(&request.url).json(&request.body);
+    for (key, value) in &request.upstream_headers {
+        request_builder = request_builder.header(key, value);
+    }
+    if let Some(duration) = request.timeout {
+        request_builder = request_builder.timeout(duration);
+    }
+
+    let response = request_builder
+        .send()
+        .await
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+    let status = response.status();
+    if !status.is_success() {
+        let text = response
+            .text()
+            .await
+            .map_err(|err| CoreError::Network(err.to_string()))?;
+        return Err(CoreError::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&text),
+        });
+    }
+    Ok(response)
+}

litellm-rust/crates/ai-gateway/src/messages/mod.rs

@@ -0,0 +1,49 @@
+use litellm_core::CoreResult;
+use serde_json::Value;
+
+mod client;
+mod common_utils;
+mod handler;
+mod prepare;
+mod types;
+
+pub use types::MessagesRequest;
+
+use handler::{execute_messages_provider_call, execute_messages_provider_stream};
+use prepare::prepare_messages_call;
+
+pub async fn messages(request: MessagesRequest<'_>) -> CoreResult<Value> {
+    match execute_messages(request, false).await? {
+        MessagesResponse::Json(body) => Ok(body),
+        MessagesResponse::Stream(response) => {
+            drop(response);
+            Err(litellm_core::CoreError::InvalidResponse(
+                "non-streaming messages execution returned a stream".to_string(),
+            ))
+        }
+    }
+}
+
+pub(crate) enum MessagesResponse {
+    Json(Value),
+    Stream(reqwest::Response),
+}
+
+pub(crate) async fn execute_messages(
+    request: MessagesRequest<'_>,
+    stream: bool,
+) -> CoreResult<MessagesResponse> {
+    let prepared = prepare_messages_call(request)?;
+    if stream {
+        execute_messages_provider_stream(prepared)
+            .await
+            .map(MessagesResponse::Stream)
+    } else {
+        execute_messages_provider_call(prepared)
+            .await
+            .map(MessagesResponse::Json)
+    }
+}
+
+#[cfg(test)]
+mod tests;

litellm-rust/crates/ai-gateway/src/messages/prepare.rs

@@ -0,0 +1,73 @@
+use litellm_core::messages::transformation::MessagesAuthStrategy;
+use litellm_core::routing_utils::provider::{get_custom_llm_provider, CustomLlmProvider};
+use litellm_core::CoreError;
+use litellm_core::CoreResult;
+
+use super::common_utils::{has_header, messages_provider_config, string_headers};
+use super::types::{MessagesRequest, ProviderMessagesRequest};
+
+pub(super) fn prepare_messages_call(
+    request: MessagesRequest<'_>,
+) -> CoreResult<ProviderMessagesRequest> {
+    let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider)
+        .or_else(|| {
+            request
+                .custom_llm_provider
+                .map(|provider| CustomLlmProvider {
+                    model: request.model,
+                    custom_llm_provider: provider,
+                })
+        })
+        .ok_or_else(|| {
+            CoreError::InvalidProvider(
+                "unable to resolve custom_llm_provider for messages request".to_string(),
+            )
+        })?;
+    let model = provider_info.model.to_string();
+    let provider = provider_info.custom_llm_provider;
+
+    let config = messages_provider_config(provider)
+        .ok_or_else(|| CoreError::InvalidProvider(provider.to_string()))?;
+    let env_lookup = |key: &str| std::env::var(key).ok();
+
+    let mut headers = string_headers(request.extra_headers)?;
+
+    let auth_strategy = config.auth_strategy();
+    if !has_header(&headers, auth_strategy.header_name()) {
+        let api_key = config.resolve_api_key(request.api_key, &env_lookup)?;
+        let auth_header = match auth_strategy {
+            MessagesAuthStrategy::Bearer => {
+                ("authorization".to_string(), format!("Bearer {api_key}"))
+            }
+            MessagesAuthStrategy::Header(name) => (name.to_string(), api_key),
+        };
+        headers.push(auth_header);
+    }
+
+    for (name, value) in config.default_headers() {
+        if !has_header(&headers, name) {
+            headers.push((name.to_string(), value.to_string()));
+        }
+    }
+
+    let url = config.complete_url(request.api_base, &model, &env_lookup)?;
+    let typed_request = serde_json::from_value(request.body).map_err(|err| {
+        CoreError::InvalidRequest(format!("invalid Anthropic messages request: {err}"))
+    })?;
+    let transformed = config.transform_request(typed_request)?;
+    let body = serde_json::to_value(transformed).map_err(|err| {
+        CoreError::InvalidRequest(format!(
+            "failed to serialize Anthropic messages request: {err}"
+        ))
+    })?;
+
+    Ok(ProviderMessagesRequest {
+        provider: provider.to_string(),
+        model,
+        config,
+        url,
+        body,
+        upstream_headers: headers,
+        timeout: request.timeout,
+    })
+}

litellm-rust/crates/ai-gateway/src/messages/tests.rs

@@ -0,0 +1,305 @@
+use std::time::Duration;
+
+use litellm_core::error::CoreError;
+use serde_json::{json, Map, Value};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::net::{TcpListener, TcpStream};
+
+use super::common_utils::{
+    has_header, messages_provider_config, string_headers, truncate_error_body,
+};
+use super::{messages, MessagesRequest};
+
+async fn read_http_request(socket: &mut TcpStream) -> String {
+    let mut request = Vec::new();
+    let mut buffer = [0_u8; 1024];
+    let header_end = loop {
+        let n = socket.read(&mut buffer).await.expect("reads request");
+        if n == 0 {
+            break request.len();
+        }
+        request.extend_from_slice(&buffer[..n]);
+        if let Some(position) = request.windows(4).position(|window| window == b"\r\n\r\n") {
+            break position + 4;
+        }
+    };
+    let headers = String::from_utf8_lossy(&request[..header_end]);
+    let content_length = headers
+        .lines()
+        .find_map(|line| {
+            let (name, value) = line.split_once(':')?;
+            name.eq_ignore_ascii_case("content-length")
+                .then(|| value.trim().parse::<usize>().ok())
+                .flatten()
+        })
+        .unwrap_or(0);
+    while request.len().saturating_sub(header_end) < content_length {
+        let n = socket.read(&mut buffer).await.expect("reads body");
+        if n == 0 {
+            break;
+        }
+        request.extend_from_slice(&buffer[..n]);
+    }
+    String::from_utf8(request).expect("request is utf8")
+}
+
+fn write_response(body: &str) -> String {
+    format!(
+        "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+        body.len(),
+        body
+    )
+}
+
+#[test]
+fn provider_config_resolves_anthropic_and_azure_ai() {
+    assert!(messages_provider_config("anthropic").is_some());
+    assert!(messages_provider_config("azure_ai").is_some());
+    assert!(messages_provider_config("openai").is_none());
+}
+
+#[test]
+fn truncate_error_body_caps_long_payloads() {
+    let body = "x".repeat(400);
+    let truncated = truncate_error_body(&body);
+    assert!(truncated.ends_with("... (truncated)"));
+    let prefix_chars = truncated
+        .strip_suffix("... (truncated)")
+        .expect("truncated marker present")
+        .chars()
+        .count();
+    assert_eq!(prefix_chars, 256);
+}
+
+#[test]
+fn string_headers_rejects_non_string_values() {
+    let headers = json!({"x-count": 3}).as_object().unwrap().clone();
+    let err = string_headers(Some(headers)).expect_err("non-string header rejected");
+    assert!(matches!(err, CoreError::InvalidRequest(_)));
+}
+
+#[test]
+fn has_header_is_case_insensitive() {
+    let headers = vec![("X-Api-Key".to_string(), "secret".to_string())];
+    assert!(has_header(&headers, "x-api-key"));
+    assert!(!has_header(&headers, "authorization"));
+}
+
+#[tokio::test]
+async fn messages_round_trip_builds_azure_request_and_passes_response_through() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-sonnet-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":2}}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let response = messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 1024,
+            "messages": [{
+                "role": "user",
+                "content": [{
+                    "type": "text",
+                    "text": "hi",
+                    "cache_control": {"type": "ephemeral", "scope": "global"}
+                }]
+            }]
+        }),
+        api_key: Some("sk-azure"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("messages request succeeds");
+
+    assert_eq!(response["content"][0]["text"], "hi");
+    assert_eq!(response["stop_reason"], "end_turn");
+
+    let request = server.await.expect("server task completes");
+    let (head, body) = request.split_once("\r\n\r\n").expect("has body");
+    assert!(head.starts_with("POST /anthropic/v1/messages "), "{head}");
+    let head_lower = head.to_ascii_lowercase();
+    assert!(head_lower.contains("x-api-key: sk-azure"), "{head}");
+    assert!(
+        head_lower.contains("anthropic-version: 2023-06-01"),
+        "{head}"
+    );
+    assert!(
+        head_lower.contains("content-type: application/json"),
+        "{head}"
+    );
+
+    let sent_body: Value = serde_json::from_str(body).expect("body is json");
+    assert_eq!(
+        sent_body["messages"][0]["content"][0]["cache_control"],
+        json!({"type": "ephemeral"})
+    );
+}
+
+#[tokio::test]
+async fn messages_round_trip_builds_native_anthropic_request() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-sonnet-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":2}}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let response = messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 1024,
+            "messages": [{"role": "user", "content": "hi"}]
+        }),
+        api_key: Some("sk-ant"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("anthropic"),
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("messages request succeeds");
+
+    assert_eq!(response["content"][0]["text"], "hi");
+    assert_eq!(response["stop_reason"], "end_turn");
+
+    let request = server.await.expect("server task completes");
+    let (head, _) = request.split_once("\r\n\r\n").expect("has body");
+    assert!(head.starts_with("POST /v1/messages "), "{head}");
+    let head_lower = head.to_ascii_lowercase();
+    assert!(head_lower.contains("x-api-key: sk-ant"), "{head}");
+    assert!(
+        head_lower.contains("anthropic-version: 2023-06-01"),
+        "{head}"
+    );
+}
+
+#[tokio::test]
+async fn messages_does_not_duplicate_auth_when_x_api_key_supplied() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body =
+            r#"{"id":"msg_2","type":"message","role":"assistant","content":[],"model":"m"}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let mut headers = Map::new();
+    headers.insert(
+        "x-api-key".to_string(),
+        Value::String("from-python".to_string()),
+    );
+    headers.insert(
+        "anthropic-beta".to_string(),
+        Value::String("token-efficient-tools-2025-02-19".to_string()),
+    );
+
+    messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
+        api_key: Some("rust-fallback-key"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: Some(headers),
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("messages request succeeds");
+
+    let request = server.await.expect("server task completes");
+    let head = request
+        .split_once("\r\n\r\n")
+        .expect("has body")
+        .0
+        .to_ascii_lowercase();
+    let api_key_count = head
+        .lines()
+        .filter(|line| line.starts_with("x-api-key:"))
+        .count();
+    assert_eq!(api_key_count, 1, "{head}");
+    assert!(head.contains("x-api-key: from-python"), "{head}");
+    assert!(
+        head.contains("anthropic-beta: token-efficient-tools-2025-02-19"),
+        "{head}"
+    );
+    assert!(!head.contains("rust-fallback-key"), "{head}");
+}
+
+#[tokio::test]
+async fn messages_maps_provider_error_status_to_http_error() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let _ = read_http_request(&mut socket).await;
+        let body = "unauthorized";
+        let response = format!(
+            "HTTP/1.1 401 Unauthorized\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+            body.len(),
+            body
+        );
+        socket
+            .write_all(response.as_bytes())
+            .await
+            .expect("writes response");
+    });
+
+    let err = messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
+        api_key: Some("sk-azure"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect_err("provider error propagates");
+
+    assert!(matches!(err, CoreError::Http { status: 401, .. }));
+}
+
+#[tokio::test]
+async fn messages_rejects_unsupported_provider() {
+    let err = messages(MessagesRequest {
+        model: "claude-3-5-sonnet",
+        body: json!({"model": "claude-3-5-sonnet", "max_tokens": 8, "messages": []}),
+        api_key: Some("sk"),
+        api_base: Some("http://127.0.0.1:1"),
+        custom_llm_provider: Some("openai"),
+        extra_headers: None,
+        timeout: Some(Duration::from_millis(50)),
+    })
+    .await
+    .expect_err("unsupported provider errors");
+
+    assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "openai"));
+}

litellm-rust/crates/ai-gateway/src/messages/types.rs

@@ -0,0 +1,24 @@
+use std::time::Duration;
+
+use litellm_core::messages::transformation::AnthropicMessagesProviderConfig;
+use serde_json::{Map, Value};
+
+pub struct MessagesRequest<'a> {
+    pub model: &'a str,
+    pub body: Value,
+    pub api_key: Option<&'a str>,
+    pub api_base: Option<&'a str>,
+    pub custom_llm_provider: Option<&'a str>,
+    pub extra_headers: Option<Map<String, Value>>,
+    pub timeout: Option<Duration>,
+}
+
+pub(crate) struct ProviderMessagesRequest {
+    pub(crate) provider: String,
+    pub(crate) model: String,
+    pub(crate) config: &'static dyn AnthropicMessagesProviderConfig,
+    pub(crate) url: String,
+    pub(crate) body: Value,
+    pub(crate) upstream_headers: Vec<(String, String)>,
+    pub(crate) timeout: Option<Duration>,
+}

litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs

@@ -0,0 +1,514 @@
+//! `POST /v1/messages`, the Anthropic Messages HTTP surface.
+
+mod service;
+
+use axum::body::Body;
+use axum::extract::{Json, State};
+use axum::http::header::{HeaderMap, HeaderValue, CACHE_CONTROL, CONTENT_TYPE};
+use axum::http::StatusCode;
+use axum::response::{IntoResponse, Response};
+use axum::routing::post;
+use axum::Router;
+use litellm_core::CoreError;
+use serde_json::{Map, Value};
+
+use crate::auth::RequireMasterKey;
+use crate::constants::{MESSAGES_HEADERS_NOT_FORWARDED, MESSAGES_ROUTE_PATH};
+use crate::state::AppState;
+
+/// This route's contribution to the app router.
+pub fn router() -> Router<AppState> {
+    Router::new().route(MESSAGES_ROUTE_PATH, post(handle))
+}
+
+async fn handle(
+    _auth: RequireMasterKey,
+    State(state): State<AppState>,
+    headers: HeaderMap,
+    Json(body): Json<Value>,
+) -> Result<Response, MessagesRouteError> {
+    let extra_headers = forwarded_headers(&headers)?;
+    match service::run(&state.router, body, extra_headers)
+        .await
+        .map_err(MessagesRouteError::from)?
+    {
+        service::MessagesResponse::Json(body) => Ok(Json(body).into_response()),
+        service::MessagesResponse::Stream(upstream) => stream_response(upstream),
+    }
+}
+
+fn stream_response(upstream: reqwest::Response) -> Result<Response, MessagesRouteError> {
+    let content_type = upstream
+        .headers()
+        .get(CONTENT_TYPE)
+        .cloned()
+        .unwrap_or_else(|| HeaderValue::from_static("text/event-stream"));
+    let mut response = Response::builder()
+        .status(
+            StatusCode::from_u16(upstream.status().as_u16()).map_err(|error| {
+                MessagesRouteError(CoreError::InvalidResponse(format!(
+                    "invalid upstream response status: {error}"
+                )))
+            })?,
+        )
+        .header(CONTENT_TYPE, content_type);
+    if let Some(value) = upstream.headers().get(CACHE_CONTROL) {
+        response = response.header(CACHE_CONTROL, value);
+    }
+    response
+        .body(Body::from_stream(upstream.bytes_stream()))
+        .map_err(|error| {
+            MessagesRouteError(CoreError::InvalidResponse(format!(
+                "failed to build streaming response: {error}"
+            )))
+        })
+}
+
+fn forwarded_headers(headers: &HeaderMap) -> Result<Option<Map<String, Value>>, CoreError> {
+    let forwarded = headers
+        .iter()
+        .filter(|(name, _)| {
+            !MESSAGES_HEADERS_NOT_FORWARDED
+                .iter()
+                .any(|excluded| name.as_str().eq_ignore_ascii_case(excluded))
+        })
+        .map(|(name, value)| {
+            let value = value.to_str().map_err(|_| {
+                CoreError::InvalidRequest(format!("invalid value for header {}", name.as_str()))
+            })?;
+            Ok((name.to_string(), Value::String(value.to_string())))
+        })
+        .collect::<Result<Map<_, _>, CoreError>>()?;
+    Ok((!forwarded.is_empty()).then_some(forwarded))
+}
+
+#[derive(Debug)]
+struct MessagesRouteError(CoreError);
+
+impl From<CoreError> for MessagesRouteError {
+    fn from(error: CoreError) -> Self {
+        Self(error)
+    }
+}
+
+impl IntoResponse for MessagesRouteError {
+    fn into_response(self) -> Response {
+        let (status, message) = match self.0 {
+            CoreError::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message),
+            CoreError::InvalidProvider(_) | CoreError::Routing(_) => (
+                StatusCode::NOT_FOUND,
+                "no messages deployment is configured for this model".to_string(),
+            ),
+            CoreError::Auth(_) => (
+                StatusCode::BAD_GATEWAY,
+                "messages provider authentication failed".to_string(),
+            ),
+            CoreError::Http { .. }
+            | CoreError::Network(_)
+            | CoreError::Timeout
+            | CoreError::InvalidResponse(_)
+            | CoreError::InvalidType { .. }
+            | CoreError::MissingField(_) => (
+                StatusCode::BAD_GATEWAY,
+                "messages provider request failed".to_string(),
+            ),
+        };
+        (
+            status,
+            Json(serde_json::json!({"error": {"message": message}})),
+        )
+            .into_response()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use axum::body::Body;
+    use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE};
+    use axum::http::Request;
+    use axum::http::StatusCode;
+    use litellm_core::router::{Deployment, LiteLLMParams, Router as ModelRouter};
+    use serde_json::json;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
+    use tower::ServiceExt;
+
+    use super::super::app;
+    use crate::io::realtime_pool::RealtimePool;
+    use crate::state::AppState;
+
+    fn state(model: &str, api_base: String, master_key: Option<&str>) -> AppState {
+        state_with_provider(model, model, api_base, master_key)
+    }
+
+    fn state_with_provider(
+        model_alias: &str,
+        provider_model: &str,
+        api_base: String,
+        master_key: Option<&str>,
+    ) -> AppState {
+        AppState {
+            router: Arc::new(ModelRouter::new(vec![Deployment {
+                model_name: model_alias.to_string(),
+                litellm_params: LiteLLMParams {
+                    model: format!("anthropic/{provider_model}"),
+                    api_key: Some("upstream-key".to_string()),
+                    api_base: Some(api_base),
+                },
+            }])),
+            master_key: master_key.map(Arc::from),
+            loggers: Arc::new(Vec::new()),
+            realtime_pool: RealtimePool::disabled(),
+        }
+    }
+
+    async fn upstream(listener: TcpListener) -> (String, tokio::task::JoinHandle<String>) {
+        let address = listener.local_addr().expect("listener has address");
+        let server = tokio::spawn(async move {
+            let (mut socket, _) = listener.accept().await.expect("accepts request");
+            let mut request = Vec::new();
+            let mut buffer = [0_u8; 4096];
+            loop {
+                let read = socket.read(&mut buffer).await.expect("reads request");
+                request.extend_from_slice(&buffer[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            let request = String::from_utf8(request).expect("request is utf8");
+            let content_length = request
+                .lines()
+                .find_map(|line| {
+                    let (name, value) = line.split_once(':')?;
+                    name.eq_ignore_ascii_case("content-length")
+                        .then(|| value.trim().parse::<usize>().ok())
+                        .flatten()
+                })
+                .unwrap_or(0);
+            let header_end = request.find("\r\n\r\n").expect("request has headers") + 4;
+            let mut full_request = request.into_bytes();
+            while full_request.len().saturating_sub(header_end) < content_length {
+                let read = socket.read(&mut buffer).await.expect("reads body");
+                full_request.extend_from_slice(&buffer[..read]);
+            }
+            let request = String::from_utf8(full_request).expect("request is utf8");
+            let body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-test"}"#;
+            let response = format!(
+                "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+                body.len(),
+                body
+            );
+            socket
+                .write_all(response.as_bytes())
+                .await
+                .expect("writes response");
+            request
+        });
+        (format!("http://{address}"), server)
+    }
+
+    async fn streaming_upstream(
+        listener: TcpListener,
+        status: u16,
+        content_type: &'static str,
+        body: &'static str,
+    ) -> (String, tokio::task::JoinHandle<String>) {
+        let address = listener.local_addr().expect("listener has address");
+        let server = tokio::spawn(async move {
+            let (mut socket, _) = listener.accept().await.expect("accepts request");
+            let mut request = Vec::new();
+            let mut buffer = [0_u8; 4096];
+            loop {
+                let read = socket.read(&mut buffer).await.expect("reads request");
+                request.extend_from_slice(&buffer[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            let request_text = String::from_utf8(request).expect("request is utf8");
+            let content_length = request_text
+                .lines()
+                .find_map(|line| {
+                    let (name, value) = line.split_once(':')?;
+                    name.eq_ignore_ascii_case("content-length")
+                        .then(|| value.trim().parse::<usize>().ok())
+                        .flatten()
+                })
+                .unwrap_or(0);
+            let header_end = request_text.find("\r\n\r\n").expect("request has headers") + 4;
+            let mut full_request = request_text.into_bytes();
+            while full_request.len().saturating_sub(header_end) < content_length {
+                let read = socket.read(&mut buffer).await.expect("reads body");
+                full_request.extend_from_slice(&buffer[..read]);
+            }
+            let response = format!(
+                "HTTP/1.1 {status} OK\r\ncontent-type: {content_type}\r\ncache-control: no-cache\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
+                body.len()
+            );
+            socket
+                .write_all(response.as_bytes())
+                .await
+                .expect("writes response");
+            String::from_utf8(full_request).expect("request is utf8")
+        });
+        (format!("http://{address}"), server)
+    }
+
+    #[tokio::test]
+    async fn route_constructs_anthropic_upstream_request() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+        let (api_base, server) = upstream(listener).await;
+        let app = app(state("claude-test", api_base, Some("master-key")));
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .method("POST")
+                    .uri("/v1/messages")
+                    .header("authorization", "Bearer master-key")
+                    .header("x-api-key", "request-upstream-key")
+                    .header("anthropic-beta", "beta-feature")
+                    .header("content-type", "application/json")
+                    .body(Body::from(
+                        json!({
+                            "model": "claude-test",
+                            "max_tokens": 16,
+                            "messages": [{"role": "user", "content": "hello"}]
+                        })
+                        .to_string(),
+                    ))
+                    .expect("request builds"),
+            )
+            .await
+            .expect("route responds");
+        assert_eq!(response.status(), StatusCode::OK);
+        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+            .await
+            .expect("response body reads");
+        assert_eq!(
+            serde_json::from_slice::<serde_json::Value>(&body).expect("json")["id"],
+            "msg_1"
+        );
+        let upstream_request = server.await.expect("upstream task completes");
+        let (head, body) = upstream_request
+            .split_once("\r\n\r\n")
+            .expect("upstream request has body");
+        let head = head.to_ascii_lowercase();
+        assert!(head.contains("x-api-key: request-upstream-key"));
+        assert!(head.contains("anthropic-beta: beta-feature"));
+        assert!(!head.contains("authorization: bearer master-key"));
+        let body: serde_json::Value = serde_json::from_str(body).expect("upstream body is json");
+        assert_eq!(body["model"], "claude-test");
+        assert_eq!(body["messages"][0]["content"], "hello");
+    }
+
+    #[tokio::test]
+    async fn route_substitutes_model_alias_with_provider_model_upstream() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+        let (api_base, server) = upstream(listener).await;
+        let app = app(state_with_provider(
+            "production",
+            "claude-sonnet-4-5",
+            api_base,
+            Some("master-key"),
+        ));
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .method("POST")
+                    .uri("/v1/messages")
+                    .header("authorization", "Bearer master-key")
+                    .header("content-type", "application/json")
+                    .body(Body::from(
+                        json!({
+                            "model": "production",
+                            "max_tokens": 16,
+                            "messages": [{"role": "user", "content": "hello"}]
+                        })
+                        .to_string(),
+                    ))
+                    .expect("request builds"),
+            )
+            .await
+            .expect("route responds");
+        assert_eq!(response.status(), StatusCode::OK);
+        let upstream_request = server.await.expect("upstream task completes");
+        let (_, upstream_body) = upstream_request
+            .split_once("\r\n\r\n")
+            .expect("upstream request has body");
+        let upstream_body: serde_json::Value =
+            serde_json::from_str(upstream_body).expect("upstream body is json");
+        assert_eq!(upstream_body["model"], "claude-sonnet-4-5");
+        assert_ne!(upstream_body["model"], "production");
+    }
+
+    #[tokio::test]
+    async fn route_streams_anthropic_events_without_buffering_or_reordering() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+        let events = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\"}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
+        let (api_base, server) =
+            streaming_upstream(listener, 200, "text/event-stream", events).await;
+        let app = app(state("claude-test", api_base, Some("master-key")));
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .method("POST")
+                    .uri("/v1/messages")
+                    .header("authorization", "Bearer master-key")
+                    .header("content-type", "application/json")
+                    .body(Body::from(
+                        json!({
+                            "model": "claude-test",
+                            "max_tokens": 16,
+                            "stream": true,
+                            "messages": [{"role": "user", "content": "hello"}]
+                        })
+                        .to_string(),
+                    ))
+                    .expect("request builds"),
+            )
+            .await
+            .expect("route responds");
+        assert_eq!(response.status(), StatusCode::OK);
+        assert_eq!(
+            response
+                .headers()
+                .get(CONTENT_TYPE)
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "text/event-stream"
+        );
+        assert_eq!(
+            response
+                .headers()
+                .get(CACHE_CONTROL)
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "no-cache"
+        );
+        let response_body = axum::body::to_bytes(response.into_body(), usize::MAX)
+            .await
+            .expect("response body reads");
+        assert_eq!(response_body, events.as_bytes());
+        let upstream_request = server.await.expect("upstream task completes");
+        let (_, upstream_body) = upstream_request
+            .split_once("\r\n\r\n")
+            .expect("upstream request has body");
+        assert_eq!(
+            serde_json::from_str::<serde_json::Value>(upstream_body)
+                .expect("upstream body is json")["stream"],
+            true
+        );
+    }
+
+    #[tokio::test]
+    async fn route_maps_streaming_upstream_errors_before_starting_response() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+        let (api_base, server) = streaming_upstream(
+            listener,
+            429,
+            "application/json",
+            r#"{"error":"rate limited"}"#,
+        )
+        .await;
+        let app = app(state("claude-test", api_base, Some("master-key")));
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .method("POST")
+                    .uri("/v1/messages")
+                    .header("authorization", "Bearer master-key")
+                    .header("content-type", "application/json")
+                    .body(Body::from(
+                        json!({
+                            "model": "claude-test",
+                            "max_tokens": 16,
+                            "stream": true,
+                            "messages": [{"role": "user", "content": "hello"}]
+                        })
+                        .to_string(),
+                    ))
+                    .expect("request builds"),
+            )
+            .await
+            .expect("route responds");
+        assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
+        let response_body = axum::body::to_bytes(response.into_body(), usize::MAX)
+            .await
+            .expect("response body reads");
+        assert_eq!(
+            serde_json::from_slice::<serde_json::Value>(&response_body).expect("error is json")
+                ["error"]["message"],
+            "messages provider request failed"
+        );
+        server.await.expect("upstream task completes");
+    }
+
+    #[tokio::test]
+    async fn route_rejects_missing_master_key() {
+        let app = app(state(
+            "claude-test",
+            "http://127.0.0.1:1".to_string(),
+            Some("master-key"),
+        ));
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .method("POST")
+                    .uri("/v1/messages")
+                    .header("content-type", "application/json")
+                    .body(Body::from("{}"))
+                    .expect("request builds"),
+            )
+            .await
+            .expect("route responds");
+        assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
+    }
+
+    #[tokio::test]
+    async fn route_rejects_invalid_master_key() {
+        let app = app(state(
+            "claude-test",
+            "http://127.0.0.1:1".to_string(),
+            Some("master-key"),
+        ));
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .method("POST")
+                    .uri("/v1/messages")
+                    .header("authorization", "Bearer wrong-key")
+                    .header("content-type", "application/json")
+                    .body(Body::from("{}"))
+                    .expect("request builds"),
+            )
+            .await
+            .expect("route responds");
+        assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
+    }
+
+    #[tokio::test]
+    async fn route_rejects_malformed_json_without_panicking() {
+        let app = app(state(
+            "claude-test",
+            "http://127.0.0.1:1".to_string(),
+            Some("master-key"),
+        ));
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .method("POST")
+                    .uri("/v1/messages")
+                    .header("authorization", "Bearer master-key")
+                    .header("content-type", "application/json")
+                    .body(Body::from("{not-json"))
+                    .expect("request builds"),
+            )
+            .await
+            .expect("route responds");
+        assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+    }
+}

litellm-rust/crates/ai-gateway/src/routes/messages/service.rs

@@ -0,0 +1,64 @@
+use std::sync::Arc;
+
+use litellm_core::router::Router;
+use litellm_core::{CoreError, CoreResult};
+use serde_json::{Map, Value};
+
+use crate::constants::ANTHROPIC_MESSAGES_PROVIDER;
+use crate::messages::{execute_messages, MessagesRequest};
+
+pub(crate) enum MessagesResponse {
+    Json(Value),
+    Stream(reqwest::Response),
+}
+
+pub async fn run(
+    router: &Arc<Router>,
+    body: Value,
+    extra_headers: Option<Map<String, Value>>,
+) -> CoreResult<MessagesResponse> {
+    let model = body
+        .get("model")
+        .and_then(Value::as_str)
+        .map(str::trim)
+        .filter(|model| !model.is_empty())
+        .ok_or_else(|| CoreError::InvalidRequest("messages body requires a model".to_string()))?;
+    let deployment = router.get_available_deployment(model).ok_or_else(|| {
+        CoreError::Routing(format!("no deployment available for model '{model}'"))
+    })?;
+    let provider_model = deployment.litellm_params.model.as_str();
+    let upstream_model = provider_model
+        .split_once('/')
+        .map_or(provider_model, |(_, model)| model);
+    let custom_llm_provider = if provider_model.contains('/') {
+        None
+    } else {
+        Some(ANTHROPIC_MESSAGES_PROVIDER)
+    };
+    let mut body = body;
+    body.as_object_mut()
+        .ok_or_else(|| CoreError::InvalidRequest("messages body must be an object".to_string()))?
+        .insert(
+            "model".to_string(),
+            Value::String(upstream_model.to_string()),
+        );
+
+    let request = MessagesRequest {
+        model: provider_model,
+        body,
+        api_key: deployment.litellm_params.api_key.as_deref(),
+        api_base: deployment.litellm_params.api_base.as_deref(),
+        custom_llm_provider,
+        extra_headers,
+        timeout: None,
+    };
+    let stream = request.body.get("stream").and_then(Value::as_bool) == Some(true);
+    execute_messages(request, stream)
+        .await
+        .map(|response| match response {
+            crate::messages::MessagesResponse::Json(body) => MessagesResponse::Json(body),
+            crate::messages::MessagesResponse::Stream(upstream) => {
+                MessagesResponse::Stream(upstream)
+            }
+        })
+}

litellm-rust/crates/ai-gateway/src/routes/mod.rs

@@ -7,7 +7,9 @@
 
 pub mod gil;
 pub mod health;
+pub mod messages;
 pub mod realtime;
+pub mod responses;
 
 use axum::Router;
 
@@ -18,6 +20,8 @@ pub fn app(state: AppState) -> Router {
     Router::new()
         .merge(health::router())
         .merge(gil::router())
+        .merge(messages::router())
         .merge(realtime::router())
+        .merge(responses::router())
         .with_state(state)
 }

litellm-rust/crates/ai-gateway/src/routes/responses/mod.rs

@@ -0,0 +1,348 @@
+mod service;
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade};
+use axum::extract::{Query, State};
+use axum::http::StatusCode;
+use axum::response::Response;
+use axum::routing::get;
+use axum::Router;
+use futures_util::{Sink, SinkExt, StreamExt};
+use litellm_core::responses::types::{ResponsesErrorFrame, ResponsesWsEvent, ResponsesWsEventType};
+use litellm_core::router::Router as ModelRouter;
+use serde::Deserialize;
+
+use crate::auth::RequireMasterKey;
+use crate::integrations::custom_logger::CustomLogger;
+use crate::integrations::types::RequestMetadata;
+use crate::state::AppState;
+
+static CALL_SEQ: AtomicU64 = AtomicU64::new(0);
+
+fn new_call_id() -> String {
+    let nanos = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_nanos())
+        .unwrap_or(0);
+    let sequence = CALL_SEQ.fetch_add(1, Ordering::Relaxed);
+    format!("respws-{nanos:x}-{sequence:x}")
+}
+
+pub fn router() -> Router<AppState> {
+    Router::new()
+        .route("/v1/responses", get(handle))
+        .route("/responses", get(handle))
+}
+
+#[derive(Debug, Deserialize)]
+struct ResponsesQuery {
+    model: Option<String>,
+}
+
+async fn handle(
+    _auth: RequireMasterKey,
+    ws: WebSocketUpgrade,
+    State(state): State<AppState>,
+    Query(query): Query<ResponsesQuery>,
+) -> Result<Response, (StatusCode, String)> {
+    if let Some(model) = query.model.as_deref() {
+        validate_model(&state.router, model)?;
+    }
+    let router = state.router.clone();
+    let loggers = state.loggers.clone();
+    let master_key = state.master_key.clone();
+    Ok(ws.on_upgrade(move |socket| bridge(socket, router, loggers, master_key, query.model)))
+}
+
+fn validate_model(router: &ModelRouter, model: &str) -> Result<(), (StatusCode, String)> {
+    if model.trim().is_empty() {
+        return Err((
+            StatusCode::BAD_REQUEST,
+            "missing 'model' query param".to_string(),
+        ));
+    }
+    let Some(deployment) = router.get_available_deployment(model) else {
+        return Err((
+            StatusCode::NOT_FOUND,
+            format!("no deployment for model '{model}'"),
+        ));
+    };
+    if deployment.litellm_params.model.contains('/')
+        && !deployment.litellm_params.model.starts_with("openai/")
+    {
+        return Err((
+            StatusCode::BAD_REQUEST,
+            "Responses WebSocket route supports OpenAI deployments only".to_string(),
+        ));
+    }
+    Ok(())
+}
+
+async fn send_error_and_close<S>(sink: &mut S, message: String)
+where
+    S: futures_util::Sink<Message> + Unpin,
+    S::Error: std::fmt::Display,
+{
+    if let Ok(payload) = serde_json::to_string(&ResponsesErrorFrame::invalid_request(message)) {
+        let _ = sink.send(Message::Text(payload)).await;
+    }
+    let _ = sink
+        .send(Message::Close(Some(axum::extract::ws::CloseFrame {
+            code: 1008,
+            reason: "Pre-call error".into(),
+        })))
+        .await;
+    let _ = sink.close().await;
+}
+
+struct ResponseClientSink {
+    sink: futures_util::stream::SplitSink<WebSocket, Message>,
+}
+
+impl Sink<ResponsesWsEvent> for ResponseClientSink {
+    type Error = axum::Error;
+
+    fn poll_ready(
+        mut self: std::pin::Pin<&mut Self>,
+        context: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        std::pin::Pin::new(&mut self.sink).poll_ready(context)
+    }
+
+    fn start_send(
+        mut self: std::pin::Pin<&mut Self>,
+        item: ResponsesWsEvent,
+    ) -> Result<(), Self::Error> {
+        let payload = serde_json::to_string(&item).map_err(axum::Error::new)?;
+        std::pin::Pin::new(&mut self.sink).start_send(Message::Text(payload))
+    }
+
+    fn poll_flush(
+        mut self: std::pin::Pin<&mut Self>,
+        context: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        std::pin::Pin::new(&mut self.sink).poll_flush(context)
+    }
+
+    fn poll_close(
+        mut self: std::pin::Pin<&mut Self>,
+        context: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        std::pin::Pin::new(&mut self.sink).poll_close(context)
+    }
+}
+
+impl ResponseClientSink {
+    async fn close_with_code(&mut self, code: u16, reason: &'static str) {
+        let _ = self
+            .sink
+            .send(Message::Close(Some(axum::extract::ws::CloseFrame {
+                code,
+                reason: reason.into(),
+            })))
+            .await;
+        let _ = self.sink.close().await;
+    }
+}
+
+async fn bridge(
+    socket: WebSocket,
+    router: Arc<ModelRouter>,
+    loggers: Arc<Vec<Arc<dyn CustomLogger>>>,
+    master_key: Option<Arc<str>>,
+    requested_model: Option<String>,
+) {
+    let (mut ws_sink, ws_stream) = socket.split();
+    let (model, first_frame, stream) = if let Some(model) = requested_model {
+        (model, None, ws_stream)
+    } else {
+        let mut stream = ws_stream;
+        let first = match stream.next().await {
+            Some(Ok(Message::Text(text))) => {
+                match serde_json::from_str::<ResponsesWsEvent>(&text) {
+                    Ok(event) => event,
+                    Err(_) => {
+                        send_error_and_close(
+                            &mut ws_sink,
+                            "Invalid JSON in response.create event".to_string(),
+                        )
+                        .await;
+                        return;
+                    }
+                }
+            }
+            _ => {
+                send_error_and_close(&mut ws_sink, "Missing response.create event".to_string())
+                    .await;
+                return;
+            }
+        };
+        let Some(model) = first.model().filter(|value| !value.trim().is_empty()) else {
+            send_error_and_close(
+                &mut ws_sink,
+                "Missing model in response.create event".to_string(),
+            )
+            .await;
+            return;
+        };
+        if first.event_type != ResponsesWsEventType::ResponseCreate {
+            send_error_and_close(
+                &mut ws_sink,
+                "First frame must be a response.create event".to_string(),
+            )
+            .await;
+            return;
+        }
+        (model.to_string(), Some(first), stream)
+    };
+    if let Err((status, message)) = validate_model(&router, &model) {
+        let _ = status;
+        let _ = message;
+        send_error_and_close(&mut ws_sink, "Unknown model deployment".to_string()).await;
+        return;
+    }
+
+    let call_id = new_call_id();
+    let metadata = RequestMetadata {
+        user_api_key_hash: master_key.as_deref().map(crate::auth::hash_token),
+        ..RequestMetadata::default()
+    };
+    let client_in = Box::pin(stream.filter_map(|message| async move {
+        match message {
+            Ok(Message::Text(text)) => serde_json::from_str::<ResponsesWsEvent>(&text).ok(),
+            _ => None,
+        }
+    }));
+    let mut client_out = ResponseClientSink { sink: ws_sink };
+    let result = service::run(
+        &router,
+        &model,
+        first_frame,
+        None,
+        loggers,
+        call_id,
+        metadata,
+        client_in,
+        &mut client_out,
+    )
+    .await;
+    if result.is_err() {
+        client_out
+            .close_with_code(1011, "Internal server error")
+            .await;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::io::realtime_pool::RealtimePool;
+    use crate::state::AppState;
+    use axum::body::Body;
+    use axum::http::Request;
+    use litellm_core::router::Router as ModelRouter;
+    use serde_json::json;
+    use std::pin::Pin;
+    use std::sync::Arc;
+    use std::task::{Context, Poll};
+    use tower::ServiceExt;
+
+    struct RecordingSink {
+        messages: Vec<Message>,
+    }
+
+    impl Sink<Message> for RecordingSink {
+        type Error = std::convert::Infallible;
+
+        fn poll_ready(
+            self: Pin<&mut Self>,
+            _context: &mut Context<'_>,
+        ) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn start_send(mut self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> {
+            self.messages.push(item);
+            Ok(())
+        }
+
+        fn poll_flush(
+            self: Pin<&mut Self>,
+            _context: &mut Context<'_>,
+        ) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_close(
+            self: Pin<&mut Self>,
+            _context: &mut Context<'_>,
+        ) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[tokio::test]
+    async fn pre_call_error_matches_python_frame_and_close() {
+        let mut sink = RecordingSink {
+            messages: Vec::new(),
+        };
+        send_error_and_close(&mut sink, "missing model".to_string()).await;
+        let Message::Text(payload) = &sink.messages[0] else {
+            panic!("expected error text frame");
+        };
+        assert_eq!(
+            serde_json::from_str::<serde_json::Value>(payload).expect("error json"),
+            json!({
+                "type": "error",
+                "error": {
+                    "type": "invalid_request_error",
+                    "message": "missing model"
+                }
+            })
+        );
+        assert_eq!(
+            sink.messages[1],
+            Message::Close(Some(axum::extract::ws::CloseFrame {
+                code: 1008,
+                reason: "Pre-call error".into(),
+            }))
+        );
+    }
+
+    fn state() -> AppState {
+        AppState {
+            router: Arc::new(ModelRouter::default()),
+            master_key: Some(Arc::from("master-key")),
+            loggers: Arc::new(Vec::new()),
+            realtime_pool: RealtimePool::disabled(),
+        }
+    }
+
+    #[tokio::test]
+    async fn auth_rejects_responses_upgrade_before_handler() {
+        let request = Request::builder()
+            .uri("/responses?model=known")
+            .body(Body::empty())
+            .expect("request");
+        let response = router()
+            .with_state(state())
+            .oneshot(request)
+            .await
+            .expect("response");
+        assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
+    }
+
+    #[test]
+    fn unknown_query_model_is_rejected_before_upgrade() {
+        assert_eq!(
+            validate_model(&ModelRouter::default(), "unknown").expect_err("unknown model"),
+            (
+                StatusCode::NOT_FOUND,
+                "no deployment for model 'unknown'".to_string()
+            )
+        );
+    }
+}

litellm-rust/crates/ai-gateway/src/routes/responses/service.rs

@@ -0,0 +1,156 @@
+use std::sync::Arc;
+use std::time::Duration;
+
+use futures_util::{Sink, Stream};
+use litellm_core::call_lifecycle::{CallLifecycle, CallLifecycleContext};
+use litellm_core::responses::instrumentation::{
+    ResponsesWsCallbackPayload, ResponsesWsInstrumentation, ResponsesWsLogOutcome,
+    ResponsesWsMetadata,
+};
+use litellm_core::responses::types::ResponsesWsEvent;
+use litellm_core::{CoreError, CoreResult};
+
+use crate::integrations::custom_logger::{
+    CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails,
+};
+use crate::integrations::types::RequestMetadata;
+
+#[allow(clippy::too_many_arguments)]
+pub async fn run<In, Out>(
+    router: &litellm_core::router::Router,
+    model: &str,
+    first_frame: Option<ResponsesWsEvent>,
+    idle_timeout: Option<Duration>,
+    loggers: Arc<Vec<Arc<dyn CustomLogger>>>,
+    call_id: String,
+    metadata: RequestMetadata,
+    client_in: In,
+    client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
+    Out: Sink<ResponsesWsEvent> + Unpin + Send,
+    Out::Error: std::fmt::Display,
+{
+    let deployment = router.get_available_deployment(model).ok_or_else(|| {
+        CoreError::Routing(format!("no deployment available for model '{model}'"))
+    })?;
+    let params = &deployment.litellm_params;
+    let provider_model = params
+        .model
+        .strip_prefix("openai/")
+        .unwrap_or(&params.model);
+    if params.model.contains('/') && !params.model.starts_with("openai/") {
+        return Err(CoreError::InvalidProvider(
+            "Responses WebSocket route supports OpenAI deployments only".to_string(),
+        ));
+    }
+    let instrumentation = Arc::new(ResponsesWsInstrumentation::new(
+        call_id.clone(),
+        model,
+        ResponsesWsMetadata {
+            user_api_key_hash: metadata.user_api_key_hash,
+            user_api_key_user_id: metadata.user_api_key_user_id,
+            user_api_key_team_id: metadata.user_api_key_team_id,
+        },
+    ));
+    let observer_instrumentation = Arc::clone(&instrumentation);
+    let context = CallLifecycleContext::new("responses_websocket", model, "openai", call_id);
+    let result = CallLifecycle::default()
+        .run(context, (), instrumentation.as_ref(), |_| async move {
+            crate::io::responses_ws::async_responses_websocket(
+                provider_model,
+                params.api_key.as_deref(),
+                params.api_base.as_deref(),
+                first_frame,
+                idle_timeout,
+                move |event| {
+                    observer_instrumentation.observe(event);
+                },
+                client_in,
+                client_out,
+            )
+            .await
+        })
+        .await;
+    let outcome = instrumentation.take_or_build_outcome(result.is_ok());
+    dispatch_outcome(loggers, outcome).await;
+    result
+}
+
+async fn dispatch_outcome(
+    loggers: Arc<Vec<Arc<dyn CustomLogger>>>,
+    outcome: ResponsesWsLogOutcome,
+) {
+    let runner = CustomLoggerRunner::new(loggers.as_ref().clone());
+    match outcome {
+        ResponsesWsLogOutcome::Success { payload, callback } => {
+            let (details, response, start_time, end_time) = logging_values(payload, callback, None);
+            let _ = runner
+                .async_log_success_event(
+                    &details,
+                    &response,
+                    CallbackTiming::new(start_time, end_time),
+                )
+                .await;
+        }
+        ResponsesWsLogOutcome::Failure {
+            payload,
+            callback,
+            error_message,
+            error_kind,
+        } => {
+            let error = LoggingError {
+                message: error_message,
+                kind: error_kind,
+            };
+            let (details, response, start_time, end_time) =
+                logging_values(payload, callback, Some(error));
+            let _ = runner
+                .async_log_failure_event(
+                    &details,
+                    Some(&response),
+                    CallbackTiming::new(start_time, end_time),
+                )
+                .await;
+        }
+    }
+}
+
+fn logging_values(
+    payload: litellm_core::responses::instrumentation::ResponsesWsLogPayload,
+    callback: ResponsesWsCallbackPayload,
+    error: Option<LoggingError>,
+) -> (ModelCallDetails, CallbackValue, f64, f64) {
+    let start_time = payload.start_time;
+    let end_time = payload.end_time;
+    let callback = CallbackValue::new(callback.object, callback.value);
+    let details = ModelCallDetails::from_standard_logging_payload(
+        crate::integrations::types::StandardLoggingPayload {
+            id: payload.id,
+            litellm_call_id: payload.litellm_call_id,
+            call_type: payload.call_type,
+            model: payload.model,
+            custom_llm_provider: payload.custom_llm_provider,
+            response_cost: payload.response_cost,
+            prompt_tokens: payload.usage.prompt_tokens,
+            completion_tokens: payload.usage.completion_tokens,
+            total_tokens: payload.usage.total_tokens,
+            start_time: payload.start_time,
+            end_time: payload.end_time,
+            stream: payload.stream,
+            metadata: crate::integrations::types::StandardLoggingMetadata {
+                user_api_key_hash: payload.metadata.user_api_key_hash,
+                user_api_key_user_id: payload.metadata.user_api_key_user_id,
+                user_api_key_team_id: payload.metadata.user_api_key_team_id,
+                ..Default::default()
+            },
+            messages: None,
+        },
+    );
+    let details = match error {
+        Some(error) => details.with_failure_error(error),
+        None => details,
+    };
+    (details, callback, start_time, end_time)
+}

litellm-rust/crates/core/src/constants.rs

@@ -11,3 +11,6 @@ pub(crate) const MIME_IMAGE_JPEG: &str = "image/jpeg";
 pub(crate) const MIME_IMAGE_PNG: &str = "image/png";
 pub(crate) const MIME_IMAGE_TIFF: &str = "image/tiff";
 pub(crate) const MIME_IMAGE_WEBP: &str = "image/webp";
+pub const OPENAI_DEFAULT_API_BASE: &str = "https://api.openai.com";
+pub const OPENAI_RESPONSES_DEFAULT_API_BASE: &str = "https://api.openai.com/v1";
+pub const OPENAI_RESPONSES_PATH: &str = "/responses";

litellm-rust/crates/core/src/error.rs

@@ -19,11 +19,11 @@ pub enum CoreError {
     InvalidRequest(String),
     #[error("{0}")]
     Auth(String),
-    #[error("OCR request failed with status {status}: {body}")]
+    #[error("upstream request failed with status {status}: {body}")]
     Http { status: u16, body: String },
     #[error("OCR request timed out")]
     Timeout,
-    #[error("OCR network error: {0}")]
+    #[error("upstream network error: {0}")]
     Network(String),
     #[error("routing error: {0}")]
     Routing(String),

litellm-rust/crates/core/src/lib.rs

@@ -1,10 +1,12 @@
 pub mod cache;
 pub mod call_lifecycle;
-pub(crate) mod constants;
+pub mod constants;
 pub mod error;
+pub mod messages;
 pub mod ocr;
 pub mod providers;
 pub mod realtime;
+pub mod responses;
 pub mod router;
 pub mod routing_utils;
 

litellm-rust/crates/core/src/messages/mod.rs

@@ -0,0 +1,2 @@
+pub mod transformation;
+pub mod types;

litellm-rust/crates/core/src/messages/transformation.rs

@@ -0,0 +1,59 @@
+use crate::error::CoreResult;
+
+use super::types::{AnthropicMessagesRequest, AnthropicMessagesResponse};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum MessagesAuthStrategy {
+    Bearer,
+    Header(&'static str),
+}
+
+impl MessagesAuthStrategy {
+    pub fn header_name(self) -> &'static str {
+        match self {
+            Self::Bearer => "authorization",
+            Self::Header(header_name) => header_name,
+        }
+    }
+}
+
+pub trait AnthropicMessagesProviderConfig: Sync {
+    fn complete_url(
+        &self,
+        api_base: Option<&str>,
+        model: &str,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<String>;
+
+    fn resolve_api_key(
+        &self,
+        api_key: Option<&str>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<String>;
+
+    fn auth_strategy(&self) -> MessagesAuthStrategy {
+        MessagesAuthStrategy::Header("x-api-key")
+    }
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        &[
+            ("anthropic-version", "2023-06-01"),
+            ("content-type", "application/json"),
+        ]
+    }
+
+    fn transform_request(
+        &self,
+        request: AnthropicMessagesRequest,
+    ) -> CoreResult<AnthropicMessagesRequest> {
+        Ok(request)
+    }
+
+    fn transform_response(
+        &self,
+        _model: &str,
+        response: AnthropicMessagesResponse,
+    ) -> CoreResult<AnthropicMessagesResponse> {
+        Ok(response)
+    }
+}

litellm-rust/crates/core/src/messages/types.rs

@@ -0,0 +1,110 @@
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum SystemPrompt {
+    Text(String),
+    Blocks(Vec<ContentBlock>),
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum MessageContent {
+    Text(String),
+    Blocks(Vec<ContentBlock>),
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+pub struct ContentBlock {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_control: Option<CacheControl>,
+    #[serde(flatten)]
+    pub extra: Map<String, Value>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+pub struct CacheControl {
+    #[serde(rename = "type", skip_serializing_if = "Option::is_none")]
+    pub cache_type: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ttl: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub scope: Option<String>,
+    #[serde(flatten)]
+    pub extra: Map<String, Value>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct AnthropicMessage {
+    pub role: String,
+    pub content: MessageContent,
+    #[serde(flatten)]
+    pub extra: Map<String, Value>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct AnthropicMessagesRequest {
+    pub model: String,
+    pub messages: Vec<AnthropicMessage>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub max_tokens: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub system: Option<SystemPrompt>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub metadata: Option<Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub stop_sequences: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub stream: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub temperature: Option<f64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub top_p: Option<f64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub top_k: Option<i64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tools: Option<Vec<Value>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tool_choice: Option<Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub thinking: Option<Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub service_tier: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub container: Option<Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mcp_servers: Option<Vec<Value>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub context_management: Option<Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub output_format: Option<Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub output_config: Option<Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub speed: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub inference_geo: Option<String>,
+    #[serde(flatten)]
+    pub extra: Map<String, Value>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct AnthropicMessagesResponse {
+    pub id: String,
+    #[serde(rename = "type")]
+    pub message_type: String,
+    pub role: String,
+    pub model: String,
+    pub content: Vec<Value>,
+    // Anthropic always includes stop_reason / stop_sequence, null until the turn
+    // ends; serialize them even when None so callers see the same shape as Python.
+    pub stop_reason: Option<String>,
+    pub stop_sequence: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub usage: Option<Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub container: Option<Value>,
+    #[serde(flatten)]
+    pub extra: Map<String, Value>,
+}

litellm-rust/crates/core/src/providers/anthropic/messages/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs

@@ -0,0 +1,142 @@
+use crate::error::{CoreError, CoreResult};
+use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy};
+
+const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY";
+const ANTHROPIC_API_BASE_ENV: &str = "ANTHROPIC_API_BASE";
+const DEFAULT_ANTHROPIC_API_BASE: &str = "https://api.anthropic.com";
+const MESSAGES_PATH_SUFFIX: &str = "/v1/messages";
+
+pub struct AnthropicMessagesConfig;
+
+pub const ANTHROPIC_MESSAGES_CONFIG: AnthropicMessagesConfig = AnthropicMessagesConfig;
+
+pub fn non_empty(value: Option<&str>) -> Option<&str> {
+    value.map(str::trim).filter(|value| !value.is_empty())
+}
+
+pub fn resolve_anthropic_api_key(
+    api_key: Option<&str>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> CoreResult<String> {
+    non_empty(api_key)
+        .map(str::to_string)
+        .or_else(|| env_lookup(ANTHROPIC_API_KEY_ENV).filter(|value| !value.trim().is_empty()))
+        .ok_or_else(|| {
+            CoreError::Auth(
+                "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY \
+                 environment variable"
+                    .to_string(),
+            )
+        })
+}
+
+pub fn complete_anthropic_url(
+    api_base: Option<&str>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> String {
+    let api_base = non_empty(api_base)
+        .map(str::to_string)
+        .or_else(|| env_lookup(ANTHROPIC_API_BASE_ENV).filter(|value| !value.trim().is_empty()))
+        .unwrap_or_else(|| DEFAULT_ANTHROPIC_API_BASE.to_string());
+
+    let api_base = api_base.trim_end_matches('/');
+    if api_base.ends_with(MESSAGES_PATH_SUFFIX) {
+        return api_base.to_string();
+    }
+    format!("{api_base}{MESSAGES_PATH_SUFFIX}")
+}
+
+impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig {
+    fn complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<String> {
+        Ok(complete_anthropic_url(api_base, env_lookup))
+    }
+
+    fn resolve_api_key(
+        &self,
+        api_key: Option<&str>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<String> {
+        resolve_anthropic_api_key(api_key, env_lookup)
+    }
+
+    fn auth_strategy(&self) -> MessagesAuthStrategy {
+        MessagesAuthStrategy::Header("x-api-key")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn url_defaults_to_public_anthropic_endpoint() {
+        assert_eq!(
+            complete_anthropic_url(None, &|_| None),
+            "https://api.anthropic.com/v1/messages"
+        );
+    }
+
+    #[test]
+    fn url_appends_messages_suffix_to_custom_base() {
+        assert_eq!(
+            complete_anthropic_url(Some("https://proxy.internal"), &|_| None),
+            "https://proxy.internal/v1/messages"
+        );
+    }
+
+    #[test]
+    fn url_leaves_complete_messages_endpoint_untouched() {
+        assert_eq!(
+            complete_anthropic_url(Some("https://proxy.internal/v1/messages"), &|_| None),
+            "https://proxy.internal/v1/messages"
+        );
+    }
+
+    #[test]
+    fn url_falls_back_to_env_base() {
+        let with_env = |key: &str| {
+            (key == ANTHROPIC_API_BASE_ENV).then(|| "https://env.anthropic".to_string())
+        };
+        assert_eq!(
+            complete_anthropic_url(Some("  "), &with_env),
+            "https://env.anthropic/v1/messages"
+        );
+    }
+
+    #[test]
+    fn api_key_prefers_param_then_env_then_errors() {
+        assert_eq!(
+            resolve_anthropic_api_key(Some("sk-param"), &|_| None).unwrap(),
+            "sk-param"
+        );
+        let with_env = |key: &str| (key == ANTHROPIC_API_KEY_ENV).then(|| "sk-env".to_string());
+        assert_eq!(
+            resolve_anthropic_api_key(Some("  "), &with_env).unwrap(),
+            "sk-env"
+        );
+        assert!(matches!(
+            resolve_anthropic_api_key(None, &|_| None).expect_err("missing key"),
+            CoreError::Auth(_)
+        ));
+    }
+
+    #[test]
+    fn auth_strategy_and_default_headers_match_anthropic() {
+        assert_eq!(
+            ANTHROPIC_MESSAGES_CONFIG.auth_strategy().header_name(),
+            "x-api-key"
+        );
+        assert_eq!(
+            ANTHROPIC_MESSAGES_CONFIG.default_headers(),
+            &[
+                ("anthropic-version", "2023-06-01"),
+                ("content-type", "application/json"),
+            ]
+        );
+    }
+}

litellm-rust/crates/core/src/providers/anthropic/mod.rs

@@ -0,0 +1 @@
+pub mod messages;

litellm-rust/crates/core/src/providers/azure_ai/messages/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs

@@ -0,0 +1,512 @@
+use crate::error::{CoreError, CoreResult};
+use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy};
+use crate::messages::types::{
+    AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock,
+    MessageContent, SystemPrompt,
+};
+use crate::providers::anthropic::messages::transformation::{
+    non_empty, AnthropicMessagesConfig, ANTHROPIC_MESSAGES_CONFIG,
+};
+use serde_json::{Map, Value};
+
+const AZURE_API_KEY_ENV: &str = "AZURE_API_KEY";
+const AZURE_API_BASE_ENV: &str = "AZURE_API_BASE";
+const ANTHROPIC_PATH_SEGMENT: &str = "/anthropic";
+const MESSAGES_PATH_SUFFIX: &str = "/v1/messages";
+const SYSTEM_ROLE: &str = "system";
+const TEXT_BLOCK_TYPE: &str = "text";
+
+pub struct AzureAnthropicMessagesConfig {
+    anthropic: AnthropicMessagesConfig,
+}
+
+pub const AZURE_ANTHROPIC_MESSAGES_CONFIG: AzureAnthropicMessagesConfig =
+    AzureAnthropicMessagesConfig {
+        anthropic: ANTHROPIC_MESSAGES_CONFIG,
+    };
+
+pub fn resolve_azure_api_key(
+    api_key: Option<&str>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> CoreResult<String> {
+    non_empty(api_key)
+        .map(str::to_string)
+        .or_else(|| env_lookup(AZURE_API_KEY_ENV).filter(|value| !value.trim().is_empty()))
+        .ok_or_else(|| {
+            CoreError::Auth(
+                "Missing Azure API Key - Set `api_key` or the AZURE_API_KEY environment variable"
+                    .to_string(),
+            )
+        })
+}
+
+pub fn complete_azure_anthropic_url(
+    api_base: Option<&str>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> CoreResult<String> {
+    let api_base = non_empty(api_base)
+        .map(str::to_string)
+        .or_else(|| env_lookup(AZURE_API_BASE_ENV).filter(|value| !value.trim().is_empty()))
+        .ok_or_else(|| {
+            CoreError::Auth(
+                "Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. \
+                 Expected format: https://<resource-name>.services.ai.azure.com/anthropic"
+                    .to_string(),
+            )
+        })?;
+
+    let api_base = api_base.trim_end_matches('/');
+
+    if api_base.ends_with(MESSAGES_PATH_SUFFIX) {
+        return Ok(api_base.to_string());
+    }
+
+    let with_anthropic = match api_base.split_once(ANTHROPIC_PATH_SEGMENT) {
+        Some((prefix, _)) => format!("{prefix}{ANTHROPIC_PATH_SEGMENT}"),
+        None => format!("{api_base}{ANTHROPIC_PATH_SEGMENT}"),
+    };
+    Ok(format!("{with_anthropic}{MESSAGES_PATH_SUFFIX}"))
+}
+
+fn strip_scope_from_block(block: &mut ContentBlock) {
+    if let Some(cache_control) = block.cache_control.as_mut() {
+        cache_control.scope = None;
+    }
+}
+
+fn strip_scope_from_system(system: &mut SystemPrompt) {
+    if let SystemPrompt::Blocks(blocks) = system {
+        blocks.iter_mut().for_each(strip_scope_from_block);
+    }
+}
+
+fn strip_scope_from_message(message: &mut AnthropicMessage) {
+    if let MessageContent::Blocks(blocks) = &mut message.content {
+        blocks.iter_mut().for_each(strip_scope_from_block);
+    }
+}
+
+fn text_content_block(text: String) -> ContentBlock {
+    let extra = Map::from_iter([
+        (
+            "type".to_string(),
+            Value::String(TEXT_BLOCK_TYPE.to_string()),
+        ),
+        ("text".to_string(), Value::String(text)),
+    ]);
+    ContentBlock {
+        cache_control: None,
+        extra,
+    }
+}
+
+fn content_into_blocks(content: MessageContent) -> Vec<ContentBlock> {
+    match content {
+        MessageContent::Text(text) => vec![text_content_block(text)],
+        MessageContent::Blocks(blocks) => blocks,
+    }
+}
+
+fn system_into_blocks(system: Option<SystemPrompt>) -> Vec<ContentBlock> {
+    match system {
+        None => Vec::new(),
+        Some(SystemPrompt::Text(text)) => vec![text_content_block(text)],
+        Some(SystemPrompt::Blocks(blocks)) => blocks,
+    }
+}
+
+fn fold_system_role_messages(request: AnthropicMessagesRequest) -> AnthropicMessagesRequest {
+    if !request.messages.iter().any(|msg| msg.role == SYSTEM_ROLE) {
+        return request;
+    }
+
+    let (system_messages, chat_messages): (Vec<AnthropicMessage>, Vec<AnthropicMessage>) = request
+        .messages
+        .into_iter()
+        .partition(|msg| msg.role == SYSTEM_ROLE);
+
+    let folded_system: Vec<ContentBlock> = system_into_blocks(request.system)
+        .into_iter()
+        .chain(
+            system_messages
+                .into_iter()
+                .flat_map(|msg| content_into_blocks(msg.content)),
+        )
+        .collect();
+
+    AnthropicMessagesRequest {
+        messages: chat_messages,
+        system: (!folded_system.is_empty()).then_some(SystemPrompt::Blocks(folded_system)),
+        ..request
+    }
+}
+
+impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig {
+    fn complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<String> {
+        complete_azure_anthropic_url(api_base, env_lookup)
+    }
+
+    fn resolve_api_key(
+        &self,
+        api_key: Option<&str>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<String> {
+        resolve_azure_api_key(api_key, env_lookup)
+    }
+
+    fn auth_strategy(&self) -> MessagesAuthStrategy {
+        self.anthropic.auth_strategy()
+    }
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        self.anthropic.default_headers()
+    }
+
+    fn transform_request(
+        &self,
+        request: AnthropicMessagesRequest,
+    ) -> CoreResult<AnthropicMessagesRequest> {
+        let mut request = fold_system_role_messages(request);
+        if let Some(system) = request.system.as_mut() {
+            strip_scope_from_system(system);
+        }
+        request
+            .messages
+            .iter_mut()
+            .for_each(strip_scope_from_message);
+        self.anthropic.transform_request(request)
+    }
+
+    fn transform_response(
+        &self,
+        model: &str,
+        response: AnthropicMessagesResponse,
+    ) -> CoreResult<AnthropicMessagesResponse> {
+        self.anthropic.transform_response(model, response)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    fn request_from(value: serde_json::Value) -> AnthropicMessagesRequest {
+        serde_json::from_value(value).expect("valid request")
+    }
+
+    fn to_value(request: AnthropicMessagesRequest) -> serde_json::Value {
+        serde_json::to_value(request).expect("serializable request")
+    }
+
+    #[test]
+    fn url_appends_anthropic_and_messages_suffix() {
+        let url =
+            complete_azure_anthropic_url(Some("https://resource.services.ai.azure.com"), &|_| None)
+                .expect("url builds");
+        assert_eq!(
+            url,
+            "https://resource.services.ai.azure.com/anthropic/v1/messages"
+        );
+    }
+
+    #[test]
+    fn url_keeps_existing_anthropic_segment() {
+        let url = complete_azure_anthropic_url(
+            Some("https://resource.services.ai.azure.com/anthropic"),
+            &|_| None,
+        )
+        .expect("url builds");
+        assert_eq!(
+            url,
+            "https://resource.services.ai.azure.com/anthropic/v1/messages"
+        );
+    }
+
+    #[test]
+    fn url_leaves_complete_messages_endpoint_untouched() {
+        for base in [
+            "https://resource.services.ai.azure.com/anthropic/v1/messages",
+            "https://resource.services.ai.azure.com/v1/messages",
+        ] {
+            assert_eq!(
+                complete_azure_anthropic_url(Some(base), &|_| None).expect("url builds"),
+                base
+            );
+        }
+    }
+
+    #[test]
+    fn url_trims_trailing_slash_and_truncates_after_anthropic() {
+        let url = complete_azure_anthropic_url(
+            Some("https://resource.services.ai.azure.com/anthropic/extra/"),
+            &|_| None,
+        )
+        .expect("url builds");
+        assert_eq!(
+            url,
+            "https://resource.services.ai.azure.com/anthropic/v1/messages"
+        );
+    }
+
+    #[test]
+    fn url_falls_back_to_env_then_errors_when_absent() {
+        let with_env = |key: &str| {
+            (key == AZURE_API_BASE_ENV).then(|| "https://env.services.ai.azure.com".to_string())
+        };
+        assert_eq!(
+            complete_azure_anthropic_url(None, &with_env).expect("url builds"),
+            "https://env.services.ai.azure.com/anthropic/v1/messages"
+        );
+        let err = complete_azure_anthropic_url(Some("  "), &|_| None).expect_err("missing base");
+        assert!(matches!(err, CoreError::Auth(_)));
+    }
+
+    #[test]
+    fn resolve_api_key_prefers_param_then_env() {
+        assert_eq!(
+            resolve_azure_api_key(Some("sk-param"), &|_| None).unwrap(),
+            "sk-param"
+        );
+        let with_env = |key: &str| (key == AZURE_API_KEY_ENV).then(|| "sk-env".to_string());
+        assert_eq!(
+            resolve_azure_api_key(Some("  "), &with_env).unwrap(),
+            "sk-env"
+        );
+        assert!(matches!(
+            resolve_azure_api_key(None, &|_| None).expect_err("missing key"),
+            CoreError::Auth(_)
+        ));
+    }
+
+    #[test]
+    fn auth_strategy_is_x_api_key() {
+        assert_eq!(
+            AZURE_ANTHROPIC_MESSAGES_CONFIG
+                .auth_strategy()
+                .header_name(),
+            "x-api-key"
+        );
+    }
+
+    #[test]
+    fn default_headers_match_python() {
+        assert_eq!(
+            AZURE_ANTHROPIC_MESSAGES_CONFIG.default_headers(),
+            &[
+                ("anthropic-version", "2023-06-01"),
+                ("content-type", "application/json"),
+            ]
+        );
+    }
+
+    #[test]
+    fn transform_request_strips_scope_from_system_and_messages() {
+        let request = request_from(json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 1024,
+            "system": [
+                {
+                    "type": "text",
+                    "text": "sys",
+                    "cache_control": {"type": "ephemeral", "ttl": "1h", "scope": "global"}
+                }
+            ],
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": "hi",
+                            "cache_control": {"type": "ephemeral", "scope": "global"}
+                        },
+                        {"type": "text", "text": "no cache control"}
+                    ]
+                }
+            ]
+        }));
+
+        let transformed = to_value(
+            AZURE_ANTHROPIC_MESSAGES_CONFIG
+                .transform_request(request)
+                .expect("request transforms"),
+        );
+
+        assert_eq!(
+            transformed["system"][0]["cache_control"],
+            json!({"type": "ephemeral", "ttl": "1h"})
+        );
+        assert_eq!(
+            transformed["messages"][0]["content"][0]["cache_control"],
+            json!({"type": "ephemeral"})
+        );
+        assert_eq!(
+            transformed["messages"][0]["content"][1],
+            json!({"type": "text", "text": "no cache control"})
+        );
+    }
+
+    #[test]
+    fn transform_request_is_idempotent_and_preserves_string_system() {
+        let request = request_from(json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 16,
+            "system": "plain string system",
+            "messages": [{"role": "user", "content": "hi"}]
+        }));
+        let once = AZURE_ANTHROPIC_MESSAGES_CONFIG
+            .transform_request(request)
+            .expect("request transforms");
+        let twice = AZURE_ANTHROPIC_MESSAGES_CONFIG
+            .transform_request(once.clone())
+            .expect("request transforms");
+        assert_eq!(once, twice);
+        assert_eq!(to_value(once)["system"], json!("plain string system"));
+    }
+
+    #[test]
+    fn transform_request_preserves_all_supported_params() {
+        let body = json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 256,
+            "messages": [{"role": "user", "content": "hi"}],
+            "system": "be terse",
+            "metadata": {"user_id": "u1"},
+            "stop_sequences": ["STOP"],
+            "stream": false,
+            "temperature": 0.4,
+            "top_p": 0.9,
+            "top_k": 40,
+            "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],
+            "tool_choice": {"type": "auto"},
+            "thinking": {"type": "enabled", "budget_tokens": 1024},
+            "service_tier": "auto",
+            "container": {"id": "c1"},
+            "mcp_servers": [{"type": "url", "url": "https://mcp.example", "name": "x"}],
+            "context_management": {"edits": []},
+            "output_format": {"type": "json_schema"},
+            "output_config": {"effort": "high"},
+            "speed": "fast",
+            "inference_geo": "us",
+            "litellm_metadata": {"trace": "abc"}
+        });
+        let transformed = to_value(
+            AZURE_ANTHROPIC_MESSAGES_CONFIG
+                .transform_request(request_from(body.clone()))
+                .expect("request transforms"),
+        );
+        assert_eq!(transformed, body);
+    }
+
+    #[test]
+    fn transform_request_folds_system_role_message_into_top_level_system() {
+        let request = request_from(json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 256,
+            "system": [{"type": "text", "text": "base system"}],
+            "messages": [
+                {"role": "user", "content": "fix the bug"},
+                {"role": "system", "content": "Available agent types: claude"}
+            ]
+        }));
+
+        let transformed = to_value(
+            AZURE_ANTHROPIC_MESSAGES_CONFIG
+                .transform_request(request)
+                .expect("request transforms"),
+        );
+
+        assert_eq!(
+            transformed["messages"],
+            json!([{"role": "user", "content": "fix the bug"}])
+        );
+        assert_eq!(
+            transformed["system"],
+            json!([
+                {"type": "text", "text": "base system"},
+                {"type": "text", "text": "Available agent types: claude"}
+            ])
+        );
+    }
+
+    #[test]
+    fn transform_request_folds_system_role_when_no_top_level_system() {
+        let request = request_from(json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 256,
+            "messages": [
+                {"role": "user", "content": [{"type": "text", "text": "hi"}]},
+                {"role": "system", "content": [{"type": "text", "text": "sys block"}]}
+            ]
+        }));
+
+        let transformed = to_value(
+            AZURE_ANTHROPIC_MESSAGES_CONFIG
+                .transform_request(request)
+                .expect("request transforms"),
+        );
+
+        assert_eq!(
+            transformed["messages"],
+            json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}])
+        );
+        assert_eq!(
+            transformed["system"],
+            json!([{"type": "text", "text": "sys block"}])
+        );
+    }
+
+    #[test]
+    fn transform_request_leaves_requests_without_system_role_untouched() {
+        let body = json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 256,
+            "system": "be terse",
+            "messages": [
+                {"role": "user", "content": "hi"},
+                {"role": "assistant", "content": "hello"}
+            ]
+        });
+        let transformed = to_value(
+            AZURE_ANTHROPIC_MESSAGES_CONFIG
+                .transform_request(request_from(body.clone()))
+                .expect("request transforms"),
+        );
+        assert_eq!(transformed, body);
+    }
+
+    #[test]
+    fn transform_request_rejects_non_object_body() {
+        let err = serde_json::from_value::<AnthropicMessagesRequest>(json!("bad"))
+            .expect_err("non-object body should error");
+        assert!(err.is_data());
+    }
+
+    #[test]
+    fn transform_response_passes_through() {
+        let response: AnthropicMessagesResponse = serde_json::from_value(json!({
+            "id": "msg_1",
+            "type": "message",
+            "role": "assistant",
+            "content": [{"type": "text", "text": "hello"}],
+            "model": "claude-sonnet-4-5",
+            "stop_reason": "end_turn",
+            "stop_sequence": null,
+            "usage": {"input_tokens": 1, "output_tokens": 2}
+        }))
+        .expect("valid response");
+        let transformed = AZURE_ANTHROPIC_MESSAGES_CONFIG
+            .transform_response("claude-sonnet-4-5", response)
+            .expect("response transforms");
+        let value = serde_json::to_value(transformed).expect("serializable");
+        assert_eq!(value["stop_reason"], json!("end_turn"));
+        assert_eq!(value["stop_sequence"], json!(null));
+        assert_eq!(value["content"][0]["text"], json!("hello"));
+    }
+}

litellm-rust/crates/core/src/providers/azure_ai/mod.rs

@@ -1 +1,2 @@
+pub mod messages;
 pub mod ocr;

litellm-rust/crates/core/src/providers/mod.rs

@@ -1,3 +1,4 @@
+pub mod anthropic;
 pub mod azure_ai;
 pub mod mistral;
 pub mod openai;

litellm-rust/crates/core/src/providers/openai/mod.rs

@@ -1 +1,2 @@
 pub mod realtime;
+pub mod responses;

litellm-rust/crates/core/src/providers/openai/responses/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/core/src/providers/openai/responses/transformation.rs

@@ -0,0 +1,48 @@
+use crate::responses::types::{ResponsesWsEvent, ResponsesWsTransformResult};
+use crate::responses::websocket::{enforce_model, ResponsesWebSocketProviderConfig};
+use crate::CoreResult;
+
+pub struct OpenAIResponsesWsConfig;
+
+pub const OPENAI_RESPONSES_WS_CONFIG: OpenAIResponsesWsConfig = OpenAIResponsesWsConfig;
+
+impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig {
+    fn supports_native_websocket(&self) -> bool {
+        true
+    }
+
+    fn transform_ws_request(
+        &self,
+        event: &ResponsesWsEvent,
+        model: &str,
+    ) -> CoreResult<ResponsesWsTransformResult> {
+        Ok(ResponsesWsTransformResult::passthrough(enforce_model(
+            event, model,
+        )))
+    }
+
+    fn transform_ws_response(
+        &self,
+        event: &ResponsesWsEvent,
+        _model: &str,
+    ) -> CoreResult<ResponsesWsTransformResult> {
+        Ok(ResponsesWsTransformResult::passthrough(event.clone()))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn openai_config_is_native_and_enforces_model() {
+        let event: ResponsesWsEvent =
+            serde_json::from_value(serde_json::json!({"type":"response.create"}))
+                .expect("valid event");
+        let result = OPENAI_RESPONSES_WS_CONFIG
+            .transform_ws_request(&event, "gpt-5")
+            .expect("valid transform");
+        assert_eq!(result.events[0].model(), Some("gpt-5"));
+        assert!(OPENAI_RESPONSES_WS_CONFIG.supports_native_websocket());
+    }
+}

litellm-rust/crates/core/src/responses/instrumentation.rs

@@ -0,0 +1,365 @@
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Mutex;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use serde_json::Value;
+
+use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming};
+use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType};
+use crate::{CoreError, CoreResult};
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ResponsesWsUsage {
+    pub prompt_tokens: u64,
+    pub completion_tokens: u64,
+    pub total_tokens: u64,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ResponsesWsMetadata {
+    pub user_api_key_hash: Option<String>,
+    pub user_api_key_user_id: Option<String>,
+    pub user_api_key_team_id: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct ResponsesWsLogPayload {
+    pub id: String,
+    pub litellm_call_id: String,
+    pub call_type: String,
+    pub model: String,
+    pub custom_llm_provider: String,
+    pub response_cost: f64,
+    pub usage: ResponsesWsUsage,
+    pub start_time: f64,
+    pub end_time: f64,
+    pub stream: bool,
+    pub metadata: ResponsesWsMetadata,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum ResponsesWsLogOutcome {
+    Success {
+        payload: ResponsesWsLogPayload,
+        callback: ResponsesWsCallbackPayload,
+    },
+    Failure {
+        payload: ResponsesWsLogPayload,
+        callback: ResponsesWsCallbackPayload,
+        error_message: String,
+        error_kind: String,
+    },
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct ResponsesWsCallbackPayload {
+    pub object: String,
+    pub value: Value,
+}
+
+struct InstrumentationState {
+    litellm_call_id: String,
+    id: String,
+    model: String,
+    usage: ResponsesWsUsage,
+    start_time: f64,
+    end_time: f64,
+    metadata: ResponsesWsMetadata,
+    outcome: Option<ResponsesWsLogOutcome>,
+}
+
+pub struct ResponsesWsInstrumentation {
+    state: Mutex<InstrumentationState>,
+}
+
+impl ResponsesWsInstrumentation {
+    pub fn new(
+        litellm_call_id: impl Into<String>,
+        model: impl Into<String>,
+        metadata: ResponsesWsMetadata,
+    ) -> Self {
+        let litellm_call_id = litellm_call_id.into();
+        let now = epoch_seconds();
+        Self {
+            state: Mutex::new(InstrumentationState {
+                id: litellm_call_id.clone(),
+                litellm_call_id,
+                model: model.into(),
+                usage: ResponsesWsUsage::default(),
+                start_time: now,
+                end_time: now,
+                metadata,
+                outcome: None,
+            }),
+        }
+    }
+
+    pub fn observe(&self, event: &ResponsesWsEvent) {
+        if !matches!(
+            event.event_type,
+            ResponsesWsEventType::ResponseCreated
+                | ResponsesWsEventType::ResponseCompleted
+                | ResponsesWsEventType::ResponseFailed
+                | ResponsesWsEventType::ResponseIncomplete
+                | ResponsesWsEventType::Error
+        ) {
+            return;
+        }
+        let Ok(mut state) = self.state.lock() else {
+            return;
+        };
+        let Some(response) = event.data.get("response").and_then(Value::as_object) else {
+            return;
+        };
+        if let Some(id) = response
+            .get("id")
+            .and_then(Value::as_str)
+            .filter(|value| !value.is_empty())
+        {
+            state.id = id.to_string();
+            state.litellm_call_id = id.to_string();
+        }
+        if let Some(model) = response
+            .get("model")
+            .and_then(Value::as_str)
+            .filter(|value| !value.is_empty())
+        {
+            state.model = model.to_string();
+        }
+        let Some(usage) = response.get("usage").and_then(Value::as_object) else {
+            return;
+        };
+        if let Some(input) = usage.get("input_tokens").and_then(Value::as_u64) {
+            state.usage.prompt_tokens += input;
+        }
+        if let Some(output) = usage.get("output_tokens").and_then(Value::as_u64) {
+            state.usage.completion_tokens += output;
+        }
+        state.usage.total_tokens += usage
+            .get("total_tokens")
+            .and_then(Value::as_u64)
+            .unwrap_or_else(|| {
+                usage
+                    .get("input_tokens")
+                    .and_then(Value::as_u64)
+                    .unwrap_or(0)
+                    + usage
+                        .get("output_tokens")
+                        .and_then(Value::as_u64)
+                        .unwrap_or(0)
+            });
+    }
+
+    pub fn success_outcome(&self) -> ResponsesWsLogOutcome {
+        let mut state = self
+            .state
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        state.end_time = epoch_seconds();
+        ResponsesWsLogOutcome::Success {
+            payload: build_payload(&state),
+            callback: ResponsesWsCallbackPayload {
+                object: "responses_websocket".to_string(),
+                value: Value::Null,
+            },
+        }
+    }
+
+    pub fn failure_outcome(&self) -> ResponsesWsLogOutcome {
+        let mut state = self
+            .state
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        state.end_time = epoch_seconds();
+        ResponsesWsLogOutcome::Failure {
+            payload: build_payload(&state),
+            callback: ResponsesWsCallbackPayload {
+                object: "error".to_string(),
+                value: serde_json::json!({
+                    "message": "Responses WebSocket session ended in failure",
+                    "kind": "ResponsesWebSocketError",
+                }),
+            },
+            error_message: "Responses WebSocket session ended in failure".to_string(),
+            error_kind: "ResponsesWebSocketError".to_string(),
+        }
+    }
+
+    pub fn take_outcome(&self) -> Option<ResponsesWsLogOutcome> {
+        self.state
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
+            .outcome
+            .take()
+    }
+
+    pub fn take_or_build_outcome(&self, success: bool) -> ResponsesWsLogOutcome {
+        self.take_outcome().unwrap_or_else(|| {
+            if success {
+                self.success_outcome()
+            } else {
+                self.failure_outcome()
+            }
+        })
+    }
+}
+
+type LifecycleFuture<'a, T> = Pin<Box<dyn Future<Output = CoreResult<T>> + Send + 'a>>;
+
+impl CallLifecycleHooks<(), (), ()> for ResponsesWsInstrumentation {
+    type PreCallFuture<'a> = LifecycleFuture<'a, ()>;
+    type DuringCallFuture<'a> = LifecycleFuture<'a, ()>;
+    type SuccessFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
+    type FailureFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
+
+    fn async_pre_call_hook<'a>(
+        &'a self,
+        _context: &'a CallLifecycleContext,
+        request: (),
+    ) -> Self::PreCallFuture<'a> {
+        Box::pin(async move { Ok(request) })
+    }
+
+    fn async_during_call_hook<'a>(
+        &'a self,
+        _context: &'a CallLifecycleContext,
+        request: (),
+    ) -> Self::DuringCallFuture<'a> {
+        Box::pin(async move { Ok(request) })
+    }
+
+    fn async_log_success_event<'a>(
+        &'a self,
+        _context: &'a CallLifecycleContext,
+        _response: &'a (),
+        _timing: &'a CallLifecycleTiming,
+    ) -> Self::SuccessFuture<'a> {
+        Box::pin(async move {
+            let outcome = self.success_outcome();
+            if let Ok(mut state) = self.state.lock() {
+                state.outcome = Some(outcome);
+            }
+        })
+    }
+
+    fn async_log_failure_event<'a>(
+        &'a self,
+        _context: &'a CallLifecycleContext,
+        _error: &'a CoreError,
+        _timing: &'a CallLifecycleTiming,
+    ) -> Self::FailureFuture<'a> {
+        Box::pin(async move {
+            let outcome = self.failure_outcome();
+            if let Ok(mut state) = self.state.lock() {
+                state.outcome = Some(outcome);
+            }
+        })
+    }
+}
+
+fn build_payload(state: &InstrumentationState) -> ResponsesWsLogPayload {
+    ResponsesWsLogPayload {
+        id: state.id.clone(),
+        litellm_call_id: state.litellm_call_id.clone(),
+        call_type: "responses_websocket".to_string(),
+        model: state.model.clone(),
+        custom_llm_provider: "openai".to_string(),
+        response_cost: 0.0,
+        usage: state.usage.clone(),
+        start_time: state.start_time,
+        end_time: state.end_time,
+        stream: true,
+        metadata: state.metadata.clone(),
+    }
+}
+
+fn epoch_seconds() -> f64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs_f64())
+        .unwrap_or(0.0)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn event(value: Value) -> ResponsesWsEvent {
+        serde_json::from_value(value).expect("valid Responses WebSocket event")
+    }
+
+    #[test]
+    fn accumulates_upstream_usage_and_identity() {
+        let instrumentation =
+            ResponsesWsInstrumentation::new("call-1", "gpt-5", ResponsesWsMetadata::default());
+        instrumentation.observe(&event(serde_json::json!({
+            "type": "response.completed",
+            "response": {
+                "id": "resp-1",
+                "model": "gpt-5-mini",
+                "usage": {
+                    "input_tokens": 3,
+                    "output_tokens": 5,
+                    "total_tokens": 8
+                }
+            }
+        })));
+
+        let ResponsesWsLogOutcome::Success { payload, .. } = instrumentation.success_outcome()
+        else {
+            panic!("expected success outcome");
+        };
+        assert_eq!(payload.id, "resp-1");
+        assert_eq!(payload.model, "gpt-5-mini");
+        assert_eq!(payload.usage.prompt_tokens, 3);
+        assert_eq!(payload.usage.completion_tokens, 5);
+        assert_eq!(payload.usage.total_tokens, 8);
+        assert!(payload.end_time >= payload.start_time);
+    }
+
+    #[test]
+    fn builds_failure_payload_without_dispatching_callbacks() {
+        let instrumentation =
+            ResponsesWsInstrumentation::new("call-1", "gpt-5", ResponsesWsMetadata::default());
+        assert!(matches!(
+            instrumentation.failure_outcome(),
+            ResponsesWsLogOutcome::Failure { .. }
+        ));
+    }
+
+    #[tokio::test]
+    async fn lifecycle_records_success_outcome_for_provider_completion() {
+        let instrumentation =
+            ResponsesWsInstrumentation::new("call-1", "gpt-5", ResponsesWsMetadata::default());
+        let result = crate::call_lifecycle::CallLifecycle::default()
+            .run(
+                crate::call_lifecycle::CallLifecycleContext::new(
+                    "responses_websocket",
+                    "gpt-5",
+                    "openai",
+                    "call-1",
+                ),
+                (),
+                &instrumentation,
+                |_| async { Ok::<(), CoreError>(()) },
+            )
+            .await;
+
+        assert!(result.is_ok());
+        assert!(matches!(
+            instrumentation.take_outcome(),
+            Some(ResponsesWsLogOutcome::Success { .. })
+        ));
+    }
+
+    #[test]
+    fn builds_outcome_when_lifecycle_did_not_record_one() {
+        let instrumentation =
+            ResponsesWsInstrumentation::new("call-1", "gpt-5", ResponsesWsMetadata::default());
+        assert!(matches!(
+            instrumentation.take_or_build_outcome(true),
+            ResponsesWsLogOutcome::Success { .. }
+        ));
+    }
+}

litellm-rust/crates/core/src/responses/mod.rs

@@ -0,0 +1,3 @@
+pub mod instrumentation;
+pub mod types;
+pub mod websocket;

litellm-rust/crates/core/src/responses/types.rs

@@ -0,0 +1,166 @@
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::{Map, Value};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ResponsesWsEventType {
+    ResponseCreate,
+    ResponseCreated,
+    ResponseCompleted,
+    ResponseFailed,
+    ResponseIncomplete,
+    Error,
+    Other(String),
+}
+
+impl ResponsesWsEventType {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::ResponseCreate => "response.create",
+            Self::ResponseCreated => "response.created",
+            Self::ResponseCompleted => "response.completed",
+            Self::ResponseFailed => "response.failed",
+            Self::ResponseIncomplete => "response.incomplete",
+            Self::Error => "error",
+            Self::Other(value) => value,
+        }
+    }
+}
+
+impl Serialize for ResponsesWsEventType {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(self.as_str())
+    }
+}
+
+impl<'de> Deserialize<'de> for ResponsesWsEventType {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let value = String::deserialize(deserializer)?;
+        Ok(match value.as_str() {
+            "response.create" => Self::ResponseCreate,
+            "response.created" => Self::ResponseCreated,
+            "response.completed" => Self::ResponseCompleted,
+            "response.failed" => Self::ResponseFailed,
+            "response.incomplete" => Self::ResponseIncomplete,
+            "error" => Self::Error,
+            _ => Self::Other(value),
+        })
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct ResponsesWsEvent {
+    #[serde(rename = "type")]
+    pub event_type: ResponsesWsEventType,
+    #[serde(flatten)]
+    pub data: Map<String, Value>,
+}
+
+impl ResponsesWsEvent {
+    pub fn model(&self) -> Option<&str> {
+        let model = self.data.get("model").and_then(Value::as_str);
+        if model.is_some() {
+            return model;
+        }
+        self.data
+            .get("response")
+            .and_then(Value::as_object)
+            .and_then(|response| response.get("model"))
+            .and_then(Value::as_str)
+    }
+
+    pub fn is_response_create(&self) -> bool {
+        self.event_type == ResponsesWsEventType::ResponseCreate
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct ResponsesWsTransformResult {
+    pub events: Vec<ResponsesWsEvent>,
+}
+
+impl ResponsesWsTransformResult {
+    pub fn passthrough(event: ResponsesWsEvent) -> Self {
+        Self {
+            events: vec![event],
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ResponsesErrorFrame {
+    #[serde(rename = "type")]
+    pub frame_type: &'static str,
+    pub error: ResponsesErrorBody,
+}
+
+impl ResponsesErrorFrame {
+    pub fn invalid_request(message: impl Into<String>) -> Self {
+        Self {
+            frame_type: "error",
+            error: ResponsesErrorBody {
+                error_type: "invalid_request_error",
+                message: message.into(),
+            },
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ResponsesErrorBody {
+    #[serde(rename = "type")]
+    pub error_type: &'static str,
+    pub message: String,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn event_type_round_trips_known_and_unknown_values() {
+        let known: ResponsesWsEventType =
+            serde_json::from_str("\"response.completed\"").expect("valid event type");
+        assert_eq!(known, ResponsesWsEventType::ResponseCompleted);
+        let unknown: ResponsesWsEventType =
+            serde_json::from_str("\"response.output_text.delta\"").expect("valid event type");
+        assert_eq!(
+            unknown,
+            ResponsesWsEventType::Other("response.output_text.delta".to_string())
+        );
+    }
+
+    #[test]
+    fn error_frame_matches_proxy_shape() {
+        let frame = ResponsesErrorFrame::invalid_request("missing model");
+        assert_eq!(
+            serde_json::to_value(frame).expect("serializable"),
+            serde_json::json!({
+                "type": "error",
+                "error": {
+                    "type": "invalid_request_error",
+                    "message": "missing model"
+                }
+            })
+        );
+    }
+
+    #[test]
+    fn model_reads_flat_and_nested_create_shapes() {
+        let flat: ResponsesWsEvent =
+            serde_json::from_value(serde_json::json!({"type":"response.create","model":"gpt-5"}))
+                .expect("valid event");
+        let nested: ResponsesWsEvent = serde_json::from_value(serde_json::json!({
+            "type":"response.create",
+            "response":{"model":"gpt-5-mini"}
+        }))
+        .expect("valid event");
+        assert_eq!(flat.model(), Some("gpt-5"));
+        assert_eq!(nested.model(), Some("gpt-5-mini"));
+    }
+}

litellm-rust/crates/core/src/responses/websocket.rs

@@ -0,0 +1,188 @@
+use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH};
+use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult};
+use crate::CoreResult;
+
+pub trait ResponsesWebSocketProviderConfig: Sync {
+    fn supports_native_websocket(&self) -> bool {
+        false
+    }
+
+    fn model_in_websocket_url(&self) -> bool {
+        true
+    }
+
+    fn complete_websocket_url(&self, api_base: Option<&str>, model: &str) -> String {
+        complete_websocket_url(api_base, model, self.model_in_websocket_url())
+    }
+
+    fn transform_ws_request(
+        &self,
+        event: &ResponsesWsEvent,
+        model: &str,
+    ) -> CoreResult<ResponsesWsTransformResult>;
+
+    fn transform_ws_response(
+        &self,
+        event: &ResponsesWsEvent,
+        model: &str,
+    ) -> CoreResult<ResponsesWsTransformResult>;
+}
+
+pub fn complete_websocket_url(
+    api_base: Option<&str>,
+    model: &str,
+    model_in_websocket_url: bool,
+) -> String {
+    let base = api_base
+        .map(str::trim)
+        .filter(|value| !value.is_empty())
+        .unwrap_or(OPENAI_RESPONSES_DEFAULT_API_BASE);
+    let (base_without_query, query) = base
+        .split_once('?')
+        .map_or((base, None), |(value, query)| (value, Some(query)));
+    let response_url = format!(
+        "{}{}",
+        base_without_query.trim_end_matches('/'),
+        OPENAI_RESPONSES_PATH
+    );
+    let scheme_flipped = if let Some(rest) = response_url.strip_prefix("https://") {
+        format!("wss://{rest}")
+    } else if let Some(rest) = response_url.strip_prefix("http://") {
+        format!("ws://{rest}")
+    } else {
+        response_url
+    };
+    let url = query.map_or(scheme_flipped.clone(), |value| {
+        format!("{scheme_flipped}?{value}")
+    });
+    if !model_in_websocket_url
+        || query.is_some_and(|value| {
+            value
+                .split('&')
+                .any(|part| part.split('=').next() == Some("model"))
+        })
+    {
+        return url;
+    }
+    format!(
+        "{url}{}model={}",
+        if query.is_some() { "&" } else { "?" },
+        percent_encode(model)
+    )
+}
+
+fn percent_encode(value: &str) -> String {
+    value
+        .bytes()
+        .map(|byte| {
+            if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
+                format!("{}", byte as char)
+            } else {
+                format!("%{byte:02X}")
+            }
+        })
+        .collect()
+}
+
+pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent {
+    if !event.is_response_create() {
+        return event.clone();
+    }
+    let mut enforced = event.clone();
+    let has_flat_model = enforced.data.contains_key("model");
+    if let Some(response) = enforced
+        .data
+        .get_mut("response")
+        .and_then(serde_json::Value::as_object_mut)
+    {
+        response.insert(
+            "model".to_string(),
+            serde_json::Value::String(model.to_string()),
+        );
+        if has_flat_model {
+            enforced.data.insert(
+                "model".to_string(),
+                serde_json::Value::String(model.to_string()),
+            );
+        }
+    } else {
+        enforced.data.insert(
+            "model".to_string(),
+            serde_json::Value::String(model.to_string()),
+        );
+    }
+    enforced
+}
+
+pub fn is_terminal_event(event_type: &ResponsesWsEventType) -> bool {
+    matches!(
+        event_type,
+        ResponsesWsEventType::ResponseCreated
+            | ResponsesWsEventType::ResponseCompleted
+            | ResponsesWsEventType::ResponseFailed
+            | ResponsesWsEventType::ResponseIncomplete
+            | ResponsesWsEventType::Error
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn event(value: serde_json::Value) -> ResponsesWsEvent {
+        serde_json::from_value(value).expect("valid event")
+    }
+
+    #[test]
+    fn url_construction_matches_python_defaults_and_query_behavior() {
+        assert_eq!(
+            complete_websocket_url(None, "gpt-5", true),
+            "wss://api.openai.com/v1/responses?model=gpt-5"
+        );
+        assert_eq!(
+            complete_websocket_url(Some("http://localhost:8080/"), "gpt 5", true),
+            "ws://localhost:8080/responses?model=gpt%205"
+        );
+        assert_eq!(
+            complete_websocket_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true),
+            "wss://example.test/v1/responses?foo=bar&model=gpt-5"
+        );
+        assert_eq!(
+            complete_websocket_url(Some("https://example.test?model=existing"), "gpt-5", true),
+            "wss://example.test/responses?model=existing"
+        );
+    }
+
+    #[test]
+    fn enforce_model_overrides_flat_and_nested_values() {
+        let flat = enforce_model(
+            &event(serde_json::json!({"type":"response.create","model":"wrong"})),
+            "gpt-5",
+        );
+        assert_eq!(flat.model(), Some("gpt-5"));
+        let nested = enforce_model(
+            &event(serde_json::json!({
+                "type":"response.create",
+                "model":"wrong",
+                "response":{"model":"also-wrong"}
+            })),
+            "gpt-5",
+        );
+        assert_eq!(nested.model(), Some("gpt-5"));
+        assert_eq!(
+            nested
+                .data
+                .get("response")
+                .and_then(|value| value.get("model")),
+            Some(&serde_json::json!("gpt-5"))
+        );
+        let nested_without_flat = enforce_model(
+            &event(serde_json::json!({
+                "type":"response.create",
+                "response":{"model":"also-wrong"}
+            })),
+            "gpt-5",
+        );
+        assert!(!nested_without_flat.data.contains_key("model"));
+    }
+}

litellm-rust/crates/core/tests/workspace_crate_allowlist.rs

@@ -62,12 +62,17 @@ fn parse_members(manifest: &str) -> BTreeSet<String> {
     members
 }
 
-/// The immediate subdirectory names under `crates/`.
+/// The crate subdirectory names under `crates/`.
+///
+/// A directory counts as a crate only when it holds a `Cargo.toml`; non-crate
+/// directories (e.g. docs like `CODING_STANDARDS/`) are ignored so they can live
+/// under `crates/` without tripping the crate-proliferation guard.
 fn crate_dirs(root: &Path) -> BTreeSet<String> {
     fs::read_dir(root.join("crates"))
         .expect("crates/ directory should exist")
         .filter_map(Result::ok)
         .filter(|entry| entry.file_type().map(|ty| ty.is_dir()).unwrap_or(false))
+        .filter(|entry| entry.path().join("Cargo.toml").is_file())
         .map(|entry| entry.file_name().to_string_lossy().into_owned())
         .collect()
 }

litellm-rust/crates/python-bridge/src/lib.rs

@@ -1,8 +1,11 @@
+use std::collections::HashMap;
 use std::time::Duration;
 
+use litellm_ai_gateway::io::messages::{messages as run_messages, MessagesRequest};
 use litellm_ai_gateway::io::ocr::{ocr as run_ocr, OcrRequest};
+use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection;
 use litellm_core::error::CoreError;
-use pyo3::exceptions::PyValueError;
+use pyo3::exceptions::{PyRuntimeError, PyValueError};
 use pyo3::prelude::*;
 use pyo3::types::{PyAny, PyDict};
 use serde_json::{Map, Value};
@@ -29,12 +32,23 @@ fn json_to_py(py: Python<'_>, value: Value) -> PyResult<Py<PyAny>> {
     Ok(json.call_method1("loads", (encoded,))?.unbind())
 }
 
-fn core_error_to_pyerr(py: Python<'_>, err: CoreError) -> PyErr {
+fn ocr_error_to_pyerr(py: Python<'_>, err: CoreError) -> PyErr {
     let status_code = err.public_status_code();
     let message = err.public_message();
     build_rust_ocr_error(py, &message, status_code).unwrap_or_else(|import_err| import_err)
 }
 
+fn core_error_to_pyerr(err: CoreError) -> PyErr {
+    match err {
+        CoreError::Auth(message) => PyValueError::new_err(message),
+        CoreError::InvalidProvider(_)
+        | CoreError::InvalidRequest(_)
+        | CoreError::InvalidType { .. }
+        | CoreError::MissingField(_) => PyValueError::new_err(err.to_string()),
+        other => PyRuntimeError::new_err(other.to_string()),
+    }
+}
+
 fn build_rust_ocr_error(
     py: Python<'_>,
     message: &str,
@@ -71,6 +85,76 @@ fn optional_timeout(timeout_seconds: Option<f64>) -> Option<Duration> {
     })
 }
 
+fn marshal_headers(
+    py: Python<'_>,
+    headers: Option<Py<PyAny>>,
+) -> PyResult<HashMap<String, String>> {
+    let value = match headers {
+        Some(headers) => py_to_json(py, headers.bind(py))?,
+        None => Value::Object(Map::new()),
+    };
+    let Value::Object(headers) = value else {
+        return Err(PyValueError::new_err("headers must be a dict"));
+    };
+    headers
+        .into_iter()
+        .map(|(name, value)| {
+            value
+                .as_str()
+                .map(|value| (name, value.to_string()))
+                .ok_or_else(|| PyValueError::new_err("header values must be strings"))
+        })
+        .collect()
+}
+
+#[pyclass]
+struct ResponsesWebSocketConnection {
+    inner: RustResponsesWebSocketConnection,
+}
+
+#[pymethods]
+impl ResponsesWebSocketConnection {
+    #[classmethod]
+    #[pyo3(signature = (url, headers=None, timeout_seconds=None))]
+    fn connect<'py>(
+        _cls: &Bound<'py, pyo3::types::PyType>,
+        py: Python<'py>,
+        url: String,
+        headers: Option<Py<PyAny>>,
+        timeout_seconds: Option<f64>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let headers = marshal_headers(py, headers)?;
+        let timeout = optional_timeout(timeout_seconds);
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout)
+                .await
+                .map_err(core_error_to_pyerr)?;
+            Python::attach(|py| Py::new(py, ResponsesWebSocketConnection { inner }))
+        })
+    }
+
+    fn send_text<'py>(&self, py: Python<'py>, text: String) -> PyResult<Bound<'py, PyAny>> {
+        let inner = self.inner.clone();
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            inner.send_text(text).await.map_err(core_error_to_pyerr)
+        })
+    }
+
+    fn recv_text<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        let inner = self.inner.clone();
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            inner.recv_text().await.map_err(core_error_to_pyerr)
+        })
+    }
+
+    fn close<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        let inner = self.inner.clone();
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            inner.close().await.map_err(core_error_to_pyerr)
+        })
+    }
+}
+
 fn marshal_inputs(
     py: Python<'_>,
     document: Py<PyAny>,
@@ -130,7 +214,7 @@ fn ocr(
 
     match result {
         Ok(value) => json_to_py(py, value),
-        Err(err) => Err(core_error_to_pyerr(py, err)),
+        Err(err) => Err(ocr_error_to_pyerr(py, err)),
     }
 }
 
@@ -172,7 +256,93 @@ fn aocr(
             litellm_call_id: None,
         })
         .await
-        .map_err(|err| Python::attach(|py| core_error_to_pyerr(py, err)))?;
+        .map_err(|err| Python::attach(|py| ocr_error_to_pyerr(py, err)))?;
+
+        Python::attach(|py| json_to_py(py, value))
+    })
+}
+
+type MarshaledMessagesInputs = (Value, Option<Map<String, Value>>, Option<Duration>);
+
+fn marshal_messages_inputs(
+    py: Python<'_>,
+    body: Py<PyAny>,
+    extra_headers: Option<Py<PyAny>>,
+    timeout_seconds: Option<f64>,
+) -> PyResult<MarshaledMessagesInputs> {
+    let body = py_to_json(py, body.bind(py))?;
+    if !body.is_object() {
+        return Err(PyValueError::new_err("body must be a dict"));
+    }
+    let extra_headers = match extra_headers {
+        Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?),
+        None => None,
+    };
+    Ok((body, extra_headers, optional_timeout(timeout_seconds)))
+}
+
+#[pyfunction]
+#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))]
+#[allow(clippy::too_many_arguments)]
+fn messages(
+    py: Python<'_>,
+    model: String,
+    body: Py<PyAny>,
+    api_key: Option<String>,
+    api_base: Option<String>,
+    custom_llm_provider: Option<String>,
+    extra_headers: Option<Py<PyAny>>,
+    timeout_seconds: Option<f64>,
+) -> PyResult<Py<PyAny>> {
+    let (body, extra_headers, timeout) =
+        marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?;
+
+    let result = gil::release_gil(py, || {
+        pyo3_async_runtimes::tokio::get_runtime().block_on(run_messages(MessagesRequest {
+            model: &model,
+            body,
+            api_key: api_key.as_deref(),
+            api_base: api_base.as_deref(),
+            custom_llm_provider: custom_llm_provider.as_deref(),
+            extra_headers,
+            timeout,
+        }))
+    });
+
+    match result {
+        Ok(value) => json_to_py(py, value),
+        Err(err) => Err(core_error_to_pyerr(err)),
+    }
+}
+
+#[pyfunction]
+#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))]
+#[allow(clippy::too_many_arguments)]
+fn amessages(
+    py: Python<'_>,
+    model: String,
+    body: Py<PyAny>,
+    api_key: Option<String>,
+    api_base: Option<String>,
+    custom_llm_provider: Option<String>,
+    extra_headers: Option<Py<PyAny>>,
+    timeout_seconds: Option<f64>,
+) -> PyResult<Bound<'_, PyAny>> {
+    let (body, extra_headers, timeout) =
+        marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?;
+
+    pyo3_async_runtimes::tokio::future_into_py(py, async move {
+        let value = run_messages(MessagesRequest {
+            model: &model,
+            body,
+            api_key: api_key.as_deref(),
+            api_base: api_base.as_deref(),
+            custom_llm_provider: custom_llm_provider.as_deref(),
+            extra_headers,
+            timeout,
+        })
+        .await
+        .map_err(core_error_to_pyerr)?;
 
         Python::attach(|py| json_to_py(py, value))
     })
@@ -189,6 +359,9 @@ fn gil_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
 fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(ocr, module)?)?;
     module.add_function(wrap_pyfunction!(aocr, module)?)?;
+    module.add_function(wrap_pyfunction!(messages, module)?)?;
+    module.add_function(wrap_pyfunction!(amessages, module)?)?;
+    module.add_class::<ResponsesWebSocketConnection>()?;
     module.add_function(wrap_pyfunction!(gil_stats, module)?)?;
     Ok(())
 }

litellm/constants.py

@@ -1517,6 +1517,12 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
     "cost_discount_config",
     "cost_margin_config",
     "budget_exceeded_throttle_percentage",
+    # Every field editable from the Admin UI (proxy_server._GENERAL_SETTINGS_UI_LITELLM_FIELDS)
+    # must be listed here so a DB write from one worker overrides the live litellm attribute on
+    # the others when config reloads; otherwise peer workers stay on their startup value.
+    # test_general_settings_ui_fields_are_db_overridable enforces that pairing.
+    "enable_anthropic_prompt_caching",
+    "anthropic_prompt_caching_ttl",
 ]
 SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"]
 DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))

litellm/integrations/compression_interception/handler.py

@@ -14,6 +14,7 @@ from litellm.compression import compress
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.types.integrations.compression_interception import (
     CompressionInterceptionConfig,
+    CompressionSavingsMetadata,
 )
 from litellm.types.integrations.custom_logger import (
     AgenticLoopPlan,
@@ -25,6 +26,41 @@ LITELLM_CONTENT_RETRIEVE_TOOL_NAME = "litellm_content_retrieve"
 _CACHE_TTL_SECONDS = 15 * 60
 
 
+def _compression_savings_from_counts(
+    original_tokens: object, compressed_tokens: object
+) -> CompressionSavingsMetadata | None:
+    if isinstance(original_tokens, bool) or not isinstance(original_tokens, int):
+        return None
+    if isinstance(compressed_tokens, bool) or not isinstance(compressed_tokens, int):
+        return None
+    if compressed_tokens < 0 or original_tokens < compressed_tokens:
+        return None
+    return CompressionSavingsMetadata(
+        tokens_before=original_tokens,
+        tokens_after=compressed_tokens,
+        tokens_saved=original_tokens - compressed_tokens,
+        source="compression_interception",
+    )
+
+
+def _record_compression_savings(kwargs: dict[str, object], savings: CompressionSavingsMetadata) -> None:
+    """
+    Attach savings to the request's litellm metadata so they land in the
+    SpendLog row's metadata JSON under ``compression_savings``.
+
+    ``/v1/messages`` requests carry proxy metadata under ``litellm_metadata``
+    (the ``metadata`` key is Anthropic's own API field). The existing dict is
+    updated in place because the proxy and the logging object hold references
+    to the same object; replacing it would orphan writes made through those
+    references.
+    """
+    existing = kwargs.get("litellm_metadata")
+    if isinstance(existing, dict):
+        existing["compression_savings"] = savings
+        return
+    kwargs["litellm_metadata"] = {"compression_savings": savings}
+
+
 class CompressionInterceptionLogger(CustomLogger):
     """
     CustomLogger that implements transparent prompt compression + retrieval loops.
@@ -130,6 +166,12 @@ class CompressionInterceptionLogger(CustomLogger):
                 call_id = str(uuid.uuid4())
                 kwargs["litellm_call_id"] = call_id
             self._compression_cache_by_call_id[call_id] = (cache, time.time())
+            savings = _compression_savings_from_counts(
+                original_tokens=compressed.get("original_tokens"),
+                compressed_tokens=compressed.get("compressed_tokens"),
+            )
+            if savings is not None:
+                _record_compression_savings(kwargs=kwargs, savings=savings)
             verbose_logger.debug(
                 "CompressionInterception: compressed request [call_id=%s original=%d compressed=%d cached_keys=%d]",
                 call_id,

litellm/integrations/otel/emitter.py

@@ -72,6 +72,42 @@ def _stamp_litellm_error_attributes(span: Span, error: SpanError) -> None:
         span.set_attribute(LiteLLMError.LLM_PROVIDER, error.llm_provider)
 
 
+def stamp_error(
+    span: Span,
+    error: SpanError,
+    *,
+    record_event: bool = True,
+    set_status: bool = True,
+) -> tuple[str, str] | None:
+    """Stamp the full v2 error attribute set on ``span`` and return the resolved
+    ``(error_type, message)`` pair, or ``None`` when the error carries neither a
+    type nor a message.
+
+    Shared by the LLM-call span (``finish_span``) and the proxy-level failure
+    spans (the FastAPI SERVER span and the ``auth`` phase span) so every v2 error
+    span carries identical keys. The semconv ``exception`` event rides alongside
+    the attributes so backends that map unknown string attrs to a truncated
+    ``keyword`` (e.g. Elasticsearch's 1024-char ``ignore_above``) still see the
+    full untruncated message on the recognized event field. ``record_event`` and
+    ``set_status`` are opt-outs for callers whose span lifecycle (``use_span``) or
+    owner (the FastAPI instrumentor) already records the event or the status.
+    """
+    if not (error.error_type or error.message):
+        return None
+    error_type = error.error_type or "error"
+    message = error.message or error.error_type or "error"
+    _stamp_otel_error_attributes(span, error_type, message)
+    _stamp_litellm_error_attributes(span, error)
+    if set_status:
+        span.set_status(Status(StatusCode.ERROR, message))
+    if record_event:
+        span.add_event(
+            ExceptionEvent.NAME,
+            {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},
+        )
+    return error_type, message
+
+
 class SpanEmitter:
     def __init__(
         self,
@@ -212,21 +248,10 @@ class SpanEmitter:
             )
             else None
         )
-        if error and (error.error_type or error.message):
-            error_type = error.error_type or "error"
-            message = error.message or error.error_type or "error"
-            _stamp_otel_error_attributes(span, error_type, message)
-            _stamp_litellm_error_attributes(span, error)
-            span.set_status(Status(StatusCode.ERROR, message))
-            # Also emit the semconv ``exception`` event so backends that
-            # dynamic-map unknown string span attrs to ``keyword`` (e.g.
-            # Elasticsearch with a 1024-char ``ignore_above``) still see the
-            # full untruncated message on the recognized event field.
-            span.add_event(
-                ExceptionEvent.NAME,
-                {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},
-            )
-            if self._event_recorder is not None and role is SpanRole.LLM_CALL:
+        if error:
+            stamped = stamp_error(span, error)
+            if stamped is not None and self._event_recorder is not None and role is SpanRole.LLM_CALL:
+                error_type, message = stamped
                 self._event_recorder.record_operation_exception(
                     span_context=span.get_span_context(),
                     error_type=error_type,

litellm/integrations/otel/logger.py

@@ -24,7 +24,7 @@ from litellm.integrations.otel.plumbing.context import (
     set_request_baggage,
     set_request_root_span,
 )
-from litellm.integrations.otel.emitter import SpanEmitter
+from litellm.integrations.otel.emitter import SpanEmitter, stamp_error
 from litellm.integrations.otel.mappers import resolve_mappers
 from litellm.integrations.otel.model.metadata import (
     LLMCallEvent,
@@ -59,6 +59,7 @@ from litellm.integrations.otel.model.spans import SpanRole, span_role_for_servic
 from litellm.integrations.otel.model.utils import to_ns
 
 if TYPE_CHECKING:
+    from litellm.proxy._types import UserAPIKeyAuth
     from litellm.types.utils import (
         StandardLoggingGuardrailInformation,
         StandardLoggingPayload,
@@ -66,6 +67,33 @@ if TYPE_CHECKING:
 
 LITELLM_TRACER_NAME = "litellm"
 
+
+def _span_error_from_exception(
+    exception: "Exception | None",
+    *,
+    status_code: int | None = None,
+    traceback_str: str | None = None,
+) -> SpanError:
+    """A ``SpanError`` for a proxy-level failure that never produced a
+    ``StandardLoggingPayload`` (auth / validation / malformed-body rejections),
+    mirroring ``_parse_error``'s field mapping so it stamps the same v2 keys a
+    failed LLM call does. ``status_code`` pins ``error.code`` to the real response
+    status, matching v1's SERVER-span behavior."""
+    from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+
+    info = StandardLoggingPayloadSetup.get_error_information(
+        original_exception=exception,
+        traceback_str=traceback_str,
+    )
+    return SpanError(
+        error_type=info.get("error_class") or info.get("error_code") or None,
+        message=info.get("error_message") or None,
+        code=str(status_code) if status_code is not None else (info.get("error_code") or None),
+        stack_trace=info.get("traceback") or None,
+        llm_provider=info.get("llm_provider") or None,
+    )
+
+
 # Any callback whose class belongs to one of these modules is "the OTel
 # callback" for proxy-global-registration purposes.
 _OTEL_MODULES = (
@@ -558,7 +586,12 @@ class OpenTelemetryV2(CustomLogger):
     def start_phase_span(self, name: str) -> "Iterator[Span]":
         span = self._emitter.start_span(SpanRole.SERVICE, name)
         with use_span(span, end_on_exit=True):
-            yield span
+            try:
+                yield span
+            except Exception as exc:
+                if is_recordable_span(span):
+                    stamp_error(span, _span_error_from_exception(exc), record_event=False, set_status=False)
+                raise
 
     async def async_pre_call_hook(
         self,
@@ -573,6 +606,48 @@ class OpenTelemetryV2(CustomLogger):
         )
         return data
 
+    def record_error_attributes_on_span(
+        self,
+        span: "Span | None",
+        exception: "Exception | None",
+        status_code: int,
+    ) -> None:
+        """Stamp the v2 error.* attributes on the FastAPI-owned SERVER span for a
+        failure that dies before any LLM-call span exists (malformed body, auth /
+        validation rejection). Called from the proxy's global exception handler via
+        ``_close_dangling_otel_server_span``. The instrumentor still owns the span's
+        status and lifecycle, so this only decorates it — never sets status, never
+        ends it — and emits no exception event, matching v1's SERVER-span behavior
+        and avoiding a duplicate of the event ``async_post_call_failure_hook`` or
+        the ``auth`` phase span already records."""
+        if span is None or not is_recordable_span(span):
+            return
+        stamp_error(
+            span,
+            _span_error_from_exception(exception, status_code=status_code),
+            record_event=False,
+            set_status=False,
+        )
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict,
+        original_exception: Exception,
+        user_api_key_dict: "UserAPIKeyAuth",
+        traceback_str: "str | None" = None,
+    ) -> None:
+        """Stamp error.* on the request's root SERVER span for a proxy-level
+        failure that never reached an LLM call (empty body rejected in the
+        endpoint, auth failure), so the failed request carries the same error keys
+        a failed LLM call does. v1's ``OpenTelemetry`` implemented this same hook;
+        v2 lost it when it stopped subclassing ``OpenTelemetry``, which is the
+        LIT-4179 regression for pre-call failures."""
+        span = request_root_span() or user_api_key_dict.parent_otel_span
+        if span is None or not is_recordable_span(span):
+            return None
+        stamp_error(span, _span_error_from_exception(original_exception, traceback_str=traceback_str))
+        return None
+
     def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None:
         # Emitted by the guardrail-recording code the moment a guardrail finishes,
         # not from a post-call hook — that hook does not fire on every path (a

litellm/llms/custom_httpx/llm_http_handler.py

@@ -1,6 +1,8 @@
 import asyncio
 import json
+import os
 import ssl
+from contextlib import asynccontextmanager
 from functools import lru_cache
 from typing import (
     TYPE_CHECKING,
@@ -146,12 +148,23 @@ from litellm.utils import (
     async_pre_call_deployment_hook,
 )
 
+
+def _rust_responses_websocket_enabled(
+    custom_llm_provider: str | None,
+    litellm_params: GenericLiteLLMParams,
+) -> bool:
+    return custom_llm_provider == "openai" and litellm_params.get("rust") is True
+
+
 from .http_handler import get_shared_realtime_ssl_context
 
 if TYPE_CHECKING:
     from aiohttp import ClientSession
 
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+    from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+        AnthropicMessagesStreamingResponse,
+    )
     from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
     from litellm.types.llms.openai_evals import (
         CancelEvalResponse,
@@ -1807,6 +1820,37 @@ class BaseLLMHTTPHandler:
             },
         )
 
+        rust_messages_response = await self._maybe_rust_anthropic_messages(
+            custom_llm_provider=custom_llm_provider,
+            litellm_params=litellm_params,
+            stream=stream or False,
+            rust_stream_eligible=bool(stream) and not self._has_agentic_completion_hook(logging_obj),
+            model=model,
+            api_key=api_key,
+            api_base=api_base,
+            headers=headers,
+            request_body=request_body,
+            timeout=self._resolve_anthropic_messages_timeout(
+                litellm_params=litellm_params,
+                stream=stream or False,
+                custom_llm_provider=custom_llm_provider,
+            ),
+        )
+        if rust_messages_response is not None:
+            if stream:
+                return self._rust_anthropic_messages_fake_stream(rust_messages_response)
+            return await self._finalize_anthropic_messages_response(
+                initial_response=rust_messages_response,
+                model=model,
+                messages=messages,
+                anthropic_messages_provider_config=anthropic_messages_provider_config,
+                anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+                api_key=api_key,
+                kwargs=kwargs,
+            )
+
         response = await self._async_post_anthropic_messages_with_http_error_retry(
             async_httpx_client=async_httpx_client,
             request_url=request_url,
@@ -1881,6 +1925,31 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
             )
 
+        return await self._finalize_anthropic_messages_response(
+            initial_response=initial_response,
+            model=model,
+            messages=messages,
+            anthropic_messages_provider_config=anthropic_messages_provider_config,
+            anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+            logging_obj=logging_obj,
+            custom_llm_provider=custom_llm_provider,
+            api_key=api_key,
+            kwargs=kwargs,
+        )
+
+    async def _finalize_anthropic_messages_response(
+        self,
+        *,
+        initial_response: AnthropicMessagesResponse,
+        model: str,
+        messages: list[dict],
+        anthropic_messages_provider_config: BaseAnthropicMessagesConfig,
+        anthropic_messages_optional_request_params: dict,
+        logging_obj: LiteLLMLoggingObj,
+        custom_llm_provider: str,
+        api_key: str | None,
+        kwargs: dict,
+    ) -> AnthropicMessagesResponse | AsyncIterator:
         # Inject api_key into kwargs so follow-up calls in agentic hooks can
         # authenticate. api_key is a named param here (not in kwargs), so
         # _prepare_followup_kwargs would miss it otherwise.
@@ -1904,6 +1973,76 @@ class BaseLLMHTTPHandler:
             "anthropic_messages",
         )
 
+    @staticmethod
+    def _rust_env_enabled() -> bool:
+        return os.getenv("LITELLM_RUST", "").strip().lower() in {"1", "true", "yes", "on"}
+
+    @staticmethod
+    async def _maybe_rust_anthropic_messages(
+        *,
+        custom_llm_provider: str,
+        litellm_params: GenericLiteLLMParams,
+        stream: bool,
+        rust_stream_eligible: bool,
+        model: str,
+        api_key: str | None,
+        api_base: str | None,
+        headers: dict,
+        request_body: dict,
+        timeout: float | httpx.Timeout | None,
+    ) -> AnthropicMessagesResponse | None:
+        if custom_llm_provider not in ("azure_ai", "anthropic"):
+            return None
+        if litellm_params.get("rust") is not True and not BaseLLMHTTPHandler._rust_env_enabled():
+            return None
+        if stream and not rust_stream_eligible:
+            return None
+
+        from litellm.rust_bridge import messages as rust_messages_bridge
+
+        upstream_body = {key: value for key, value in request_body.items() if key != "stream"}
+        try:
+            rust_response = await rust_messages_bridge.amessages(
+                model=model,
+                body=upstream_body,
+                api_key=api_key,
+                api_base=api_base,
+                custom_llm_provider=custom_llm_provider,
+                extra_headers=headers,
+                timeout=timeout,
+            )
+        except Exception as rust_error:  # noqa: BLE001  # rollout-safety fallback: any Rust bridge failure must fall back to the Python path
+            verbose_logger.debug(
+                "Rust Anthropic messages bridge raised %s; falling back to Python path",
+                type(rust_error).__name__,
+            )
+            return None
+        if rust_response is None:
+            return None
+
+        response_obj = cast(AnthropicMessagesResponse, dict(rust_response))
+        response_obj["_hidden_params"] = {"additional_headers": {"x-litellm-rust": "true"}}
+        return response_obj
+
+    @staticmethod
+    def _rust_anthropic_messages_fake_stream(
+        rust_response: AnthropicMessagesResponse,
+    ) -> "AnthropicMessagesStreamingResponse":
+        from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
+            FakeAnthropicMessagesStreamIterator,
+        )
+        from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+            AnthropicMessagesStreamHiddenParams,
+            AnthropicMessagesStreamingResponse,
+        )
+
+        completion_stream = cast(AsyncIterator[bytes], FakeAnthropicMessagesStreamIterator(response=rust_response))
+        hidden_params = AnthropicMessagesStreamHiddenParams(additional_headers={"x-litellm-rust": "true"})
+        return AnthropicMessagesStreamingResponse(
+            completion_stream=completion_stream,
+            hidden_params=hidden_params,
+        )
+
     def anthropic_messages_handler(
         self,
         model: str,
@@ -5806,12 +5945,29 @@ class BaseLLMHTTPHandler:
                 },
             )
 
-            async with websockets.connect(  # type: ignore
-                ws_url,
-                additional_headers=headers,
-                max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
-                ssl=ssl_context,
-            ) as backend_ws:
+            @asynccontextmanager
+            async def _backend_connection():
+                if _rust_responses_websocket_enabled(custom_llm_provider, litellm_params):
+                    from litellm.rust_bridge import responses_websocket as rust_responses_websocket
+
+                    rust_backend = await rust_responses_websocket.connect(
+                        url=ws_url,
+                        headers={str(key): str(value) for key, value in headers.items()},
+                        timeout=timeout,
+                    )
+                    if rust_backend is not None:
+                        yield rust_backend
+                        return
+
+                async with websockets.connect(  # type: ignore
+                    ws_url,
+                    additional_headers=headers,
+                    max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
+                    ssl=ssl_context,
+                ) as backend:
+                    yield backend
+
+            async with _backend_connection() as backend_ws:
                 _request_data: Dict[str, Any] = {}
                 if litellm_metadata:
                     _request_data["litellm_metadata"] = litellm_metadata

litellm/llms/fireworks_ai/chat/transformation.py

@@ -48,7 +48,7 @@ from ...openai.chat.gpt_transformation import (
     OpenAIChatCompletionStreamingHandler,
     OpenAIGPTConfig,
 )
-from ..common_utils import FireworksAIException
+from ..common_utils import FireworksAIMixin, FireworksAIException
 
 
 def _extract_fireworks_hidden_params(payload: dict) -> dict:
@@ -70,7 +70,7 @@ def _extract_fireworks_hidden_params(payload: dict) -> dict:
     return {**top_level, **per_choice}
 
 
-class FireworksAIConfig(OpenAIGPTConfig):
+class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
     """
     Reference: https://docs.fireworks.ai/api-reference/post-chatcompletions
 
@@ -114,6 +114,16 @@ class FireworksAIConfig(OpenAIGPTConfig):
         prompt_truncate_len: Optional[int] = None,
         context_length_exceeded_behavior: Optional[Literal["error", "truncate"]] = None,
     ) -> None:
+        OpenAIGPTConfig.__init__(
+            self,
+            frequency_penalty=frequency_penalty,
+            max_tokens=max_tokens,
+            n=n,
+            stop=stop,
+            temperature=temperature,
+            top_p=top_p,
+            response_format=response_format,
+        )
         locals_ = locals().copy()
         for key, value in locals_.items():
             if key != "self" and value is not None:

litellm/llms/fireworks_ai/common_utils.py

@@ -12,6 +12,23 @@ class FireworksAIException(BaseLLMException):
     pass
 
 
+def get_fireworks_session_id(litellm_params: dict) -> str | None:
+    params = litellm_params
+    for key in ("litellm_session_id", "session_id"):
+        value = params.get(key)
+        if value:
+            return str(value)
+    metadata = params.get("metadata")
+    if isinstance(metadata, dict):
+        value = metadata.get("session_id")
+        if value:
+            return str(value)
+    value = params.get("litellm_trace_id")
+    if value:
+        return str(value)
+    return None
+
+
 class FireworksAIMixin:
     """
     Common Base Config functions across Fireworks AI Endpoints
@@ -47,4 +64,9 @@ class FireworksAIMixin:
         if api_key is None:
             raise ValueError("FIREWORKS_API_KEY is not set")
 
-        return {"Authorization": "Bearer {}".format(api_key), **headers}
+        validated_headers = {"Authorization": "Bearer {}".format(api_key), **headers}
+        if not any(key.lower() == "x-session-affinity" for key in validated_headers):
+            session_id = get_fireworks_session_id(litellm_params)
+            if session_id:
+                validated_headers["x-session-affinity"] = session_id
+        return validated_headers

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -1744,6 +1744,30 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         )
         return non_thinking_tokens == usage_metadata.get("totalTokenCount", 0)
 
+    @staticmethod
+    def _response_has_search_grounding(
+        completion_response: Union[GenerateContentResponseBody, BidiGenerateContentServerMessage],
+    ) -> bool:
+        """
+        Whether the response used Grounding with Google Search, detected via
+        groundingMetadata.webSearchQueries (an actual web search was performed).
+
+        Google bills grounding-with-Google-Search retrieved tokens separately (a per-request /
+        per-query search fee) and excludes them from input token billing, unlike URL context /
+        File Search / code execution whose tool-use tokens are charged at the input token rate.
+        URL context also emits groundingMetadata (with groundingChunks but no webSearchQueries),
+        so presence of groundingMetadata alone is not a sufficient signal.
+        See https://ai.google.dev/gemini-api/docs/pricing and
+        https://github.com/BerriAI/litellm/discussions/33198
+        """
+        if "candidates" not in completion_response:
+            return False
+        for candidate in completion_response["candidates"] or []:
+            grounding_metadata, _, _, _ = VertexGeminiConfig._extract_candidate_metadata(candidate)
+            if VertexGeminiConfig._calculate_web_search_requests(grounding_metadata):
+                return True
+        return False
+
     @staticmethod
     def _calculate_usage(
         completion_response: Union[GenerateContentResponseBody, BidiGenerateContentServerMessage],
@@ -1899,12 +1923,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             tool_use_tokens=tool_use_prompt_tokens,
         )
 
+        billable_tool_use_prompt_tokens = (
+            0
+            if VertexGeminiConfig._response_has_search_grounding(completion_response)
+            else (tool_use_prompt_tokens or 0)
+        )
+
         completion_tokens = response_tokens or completion_response["usageMetadata"].get("candidatesTokenCount", 0)
         if not VertexGeminiConfig.is_candidate_token_count_inclusive(usage_metadata) and reasoning_tokens:
             completion_tokens = reasoning_tokens + completion_tokens
         ## GET USAGE ##
         usage = Usage(
-            prompt_tokens=usage_metadata.get("promptTokenCount", 0) + (tool_use_prompt_tokens or 0),
+            prompt_tokens=usage_metadata.get("promptTokenCount", 0) + billable_tool_use_prompt_tokens,
             completion_tokens=completion_tokens,
             total_tokens=usage_metadata.get("totalTokenCount", 0),
             prompt_tokens_details=prompt_tokens_details,

litellm/proxy/_experimental/mcp_server/db.py

@@ -33,6 +33,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
 from litellm.proxy.utils import PrismaClient
 from litellm.repositories.object_permission_repository import ObjectPermissionRepository
 from litellm.repositories.table_repositories import (
+    MCPServerOAuthClientRepository,
     MCPServerRepository,
     MCPUserCredentialsRepository,
 )
@@ -374,6 +375,12 @@ def encrypt_credentials(credentials: MCPCredentials, encryption_key: Optional[st
             value=client_secret,
             new_encryption_key=encryption_key,
         )
+    client_private_key = credentials.get("client_private_key")
+    if client_private_key is not None:
+        credentials["client_private_key"] = encrypt_value_helper(
+            value=client_private_key,
+            new_encryption_key=encryption_key,
+        )
     # AWS SigV4 credential fields
     aws_access_key_id = credentials.get("aws_access_key_id")
     if aws_access_key_id is not None:
@@ -405,6 +412,7 @@ def decrypt_credentials(
         "auth_value",
         "client_id",
         "client_secret",
+        "client_private_key",
         "aws_access_key_id",
         "aws_secret_access_key",
         "aws_session_token",
@@ -639,6 +647,7 @@ async def delete_mcp_server(
         for model, label in (
             (prisma_client.db.litellm_mcpusercredentials, "credential"),
             (prisma_client.db.litellm_mcpuserenvvars, "env var"),
+            (prisma_client.db.litellm_mcpserveroauthclient, "OAuth client"),
         ):
             try:
                 await model.delete_many(where={"server_id": server_id})
@@ -823,26 +832,66 @@ async def update_mcp_server(
     return updated_mcp_server
 
 
-async def rotate_mcp_server_credentials_master_key(prisma_client: PrismaClient, touched_by: str, new_master_key: str):
+async def get_mcp_server_oauth_client_credentials(prisma_client: PrismaClient, server_id: str) -> object | None:
+    """Read the persisted (encrypted) DCR OAuth client blob for a server from the
+    server-scoped store, or None. Config.yaml-declared servers have no
+    LiteLLM_MCPServerTable row, so their dynamically registered client lives here keyed
+    by server_id. The returned value is the raw credentials blob for
+    ``_get_persisted_dcr_credentials`` to parse."""
+    row = await MCPServerOAuthClientRepository(prisma_client).table.find_unique(where={"server_id": server_id})
+    if row is None:
+        return None
+    return row.credentials
+
+
+async def upsert_mcp_server_oauth_client_credentials(
+    prisma_client: PrismaClient, server_id: str, credentials: MCPCredentials
+) -> None:
+    """Persist a server's dynamically registered OAuth client (RFC 7591 DCR) in the
+    server-scoped store keyed by server_id, independent of any LiteLLM_MCPServerTable row.
+    client_id/client_secret are encrypted at rest with the same salt key used for the
+    server row's credentials blob, so ``_apply_persisted_dcr_credentials`` decrypts them the
+    same way regardless of which store a server's client came from."""
     from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 
+    encrypted = encrypt_credentials(credentials=dict(credentials), encryption_key=_get_salt_key())
+    blob = safe_dumps(encrypted)
+    await MCPServerOAuthClientRepository(prisma_client).table.upsert(
+        where={"server_id": server_id},
+        data={
+            "create": {"server_id": server_id, "credentials": blob},
+            "update": {"credentials": blob},
+        },
+    )
+
+
+def _reencrypt_mcp_credentials_blob(credentials: object, new_master_key: str) -> str | None:
+    """Decrypt an at-rest MCP credentials blob with the current key and re-encrypt it under
+    new_master_key, returning the serialized blob or None when there is nothing to rotate. Shared by
+    every table that stores an encrypted MCP credentials blob so a master-key rotation covers them
+    uniformly and cannot silently skip one."""
+    if not credentials:
+        return None
+    from litellm.litellm_core_utils.safe_json_dumps import safe_dumps  # noqa: PLC0415  # avoids circular import
+
+    creds_dict = json.loads(credentials) if isinstance(credentials, str) else dict(credentials)
+    decrypted = decrypt_credentials(credentials=cast(MCPCredentials, creds_dict))
+    encrypted = encrypt_credentials(credentials=decrypted, encryption_key=new_master_key)
+    return safe_dumps(encrypted)
+
+
+async def rotate_mcp_server_credentials_master_key(prisma_client: PrismaClient, touched_by: str, new_master_key: str):
+    from litellm.litellm_core_utils.safe_json_dumps import safe_dumps  # noqa: PLC0415  # avoids circular import
+
     mcp_servers = await MCPServerRepository(prisma_client).table.find_many()
 
     updated = 0
     for mcp_server in mcp_servers:
         update_data: Dict[str, Any] = {}
 
-        credentials = mcp_server.credentials
-        if credentials:
-            # Decrypt with current key first, then re-encrypt with new key
-            decrypted_credentials = decrypt_credentials(
-                credentials=cast(MCPCredentials, dict(credentials)),
-            )
-            encrypted_credentials = encrypt_credentials(
-                credentials=decrypted_credentials,
-                encryption_key=new_master_key,
-            )
-            update_data["credentials"] = safe_dumps(encrypted_credentials)
+        rotated_credentials = _reencrypt_mcp_credentials_blob(mcp_server.credentials, new_master_key)
+        if rotated_credentials is not None:
+            update_data["credentials"] = rotated_credentials
 
         rotated_env_vars = _reencrypt_global_env_var_values(mcp_server.env_vars, new_master_key)
         if rotated_env_vars is not None:
@@ -857,9 +906,23 @@ async def rotate_mcp_server_credentials_master_key(prisma_client: PrismaClient,
             data=update_data,
         )
         updated += 1
+
+    oauth_clients = await MCPServerOAuthClientRepository(prisma_client).table.find_many()
+    oauth_updated = 0
+    for oauth_client in oauth_clients:
+        rotated_credentials = _reencrypt_mcp_credentials_blob(oauth_client.credentials, new_master_key)
+        if rotated_credentials is None:
+            continue
+        await MCPServerOAuthClientRepository(prisma_client).table.update(
+            where={"server_id": oauth_client.server_id},
+            data={"credentials": rotated_credentials},
+        )
+        oauth_updated += 1
+
     verbose_proxy_logger.info(
-        "rotate_mcp_server_credentials_master_key: rotated %d MCP server row(s)",
+        "rotate_mcp_server_credentials_master_key: rotated %d MCP server row(s) and %d OAuth-client row(s)",
         updated,
+        oauth_updated,
     )
 
 

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -971,43 +971,93 @@ def _apply_persisted_dcr_credentials(mcp_server: MCPServer, credentials: _Persis
     return True
 
 
-async def _get_persisted_mcp_server_with_dcr_client_id(
-    mcp_server: MCPServer,
-) -> Optional[tuple["LiteLLM_MCPServerTable", _PersistedDcrCredentials]]:
-    from litellm.proxy._experimental.mcp_server.db import get_mcp_server  # noqa: PLC0415
-    from litellm.proxy.utils import get_prisma_client_or_throw  # noqa: PLC0415
+async def _load_store_dcr_credentials(mcp_server: MCPServer) -> _PersistedDcrCredentials | None:
+    """DCR client persisted in the server-scoped OAuth-client store for a config-declared server
+    (which has no LiteLLM_MCPServerTable row). Returns None when the store has no usable client_id
+    or the DB is unreachable."""
+    from litellm.proxy._experimental.mcp_server.db import (  # noqa: PLC0415  # avoids circular import
+        get_mcp_server_oauth_client_credentials,
+    )
+    from litellm.proxy.utils import get_prisma_client_or_throw  # noqa: PLC0415  # avoids circular import
 
     try:
         prisma_client = get_prisma_client_or_throw("Database not connected. Cannot read MCP OAuth client registration.")
-        persisted_mcp_server = await get_mcp_server(
-            prisma_client=prisma_client,
-            server_id=mcp_server.server_id,
+        blob = await get_mcp_server_oauth_client_credentials(
+            prisma_client=prisma_client, server_id=mcp_server.server_id
         )
-    except Exception as exc:  # noqa: BLE001
+    except Exception as exc:  # noqa: BLE001  # best-effort read; DB may be unreachable
         verbose_logger.debug(
-            "register_client_with_server: failed to read persisted DCR client registration for server_id=%s: %s",
+            "register_client_with_server: failed to read stored DCR client for server_id=%s: %s",
             mcp_server.server_id,
             exc,
         )
         return None
 
-    if persisted_mcp_server is None:
-        return None
-
-    credentials = _get_persisted_dcr_credentials(persisted_mcp_server.credentials)
+    credentials = _get_persisted_dcr_credentials(blob)
     if credentials is None or not credentials.client_id:
         return None
+    return credentials
 
-    return persisted_mcp_server, credentials
+
+async def hydrate_config_server_dcr_client(mcp_server: MCPServer) -> bool:
+    """Overlay a config-declared server's persisted DCR client onto its in-memory object so token
+    refresh can authenticate. Config.yaml servers have no LiteLLM_MCPServerTable row, so their
+    minted client lives in the server-scoped store; without this overlay the in-memory server
+    carries no client_id after a restart. An explicit client_id set in config.yaml wins and is never
+    overwritten by a persisted store client."""
+    if mcp_server.client_id:
+        return False
+    credentials = await _load_store_dcr_credentials(mcp_server)
+    if credentials is None:
+        return False
+    return _apply_persisted_dcr_credentials(mcp_server, credentials)
+
+
+async def _resolve_persisted_dcr_client(
+    mcp_server: MCPServer,
+) -> tuple[Optional["LiteLLM_MCPServerTable"], _PersistedDcrCredentials | None]:
+    """Resolve a server's persisted DCR client using the same two-level rule the write path uses, so
+    read and write always agree. First, whether the server HAS a LiteLLM_MCPServerTable row: a row is
+    always resolved to that row and the store is never consulted for a server that has a row, so a
+    caller-chosen server_id colliding with a config-declared server cannot inherit that config
+    server's client, and a row that exists but carries no usable client_id yields (row, None) rather
+    than a store fallback. Second, among rowless servers: a config-declared server keeps its client in
+    the server-scoped store, while a rowless non-config server is a throwaway temp/session server with
+    no persisted client. Returns (row_or_None, credentials_or_None); the row is only needed by the
+    reuse path to refresh the registry for a DB-declared server."""
+    from litellm.proxy._experimental.mcp_server.db import get_mcp_server  # noqa: PLC0415  # avoids circular import
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: PLC0415  # avoids circular import
+        global_mcp_server_manager,
+    )
+    from litellm.proxy.utils import get_prisma_client_or_throw  # noqa: PLC0415  # avoids circular import
+
+    try:
+        prisma_client = get_prisma_client_or_throw("Database not connected. Cannot read MCP OAuth client registration.")
+        row = await get_mcp_server(prisma_client=prisma_client, server_id=mcp_server.server_id)
+    except Exception as exc:  # noqa: BLE001  # best-effort read; DB may be unreachable
+        verbose_logger.debug(
+            "register_client_with_server: failed to read persisted DCR client for server_id=%s: %s",
+            mcp_server.server_id,
+            exc,
+        )
+        return None, None
+
+    if row is not None:
+        credentials = _get_persisted_dcr_credentials(row.credentials)
+        if credentials is not None and credentials.client_id:
+            return row, credentials
+        return row, None
+    if global_mcp_server_manager.is_config_declared_server(mcp_server.server_id):
+        return None, await _load_store_dcr_credentials(mcp_server)
+    return None, None
 
 
 async def _reuse_persisted_dcr_client_if_available(
     mcp_server: MCPServer, current_redirect_uri: Optional[str] = None
 ) -> bool:
-    persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server)
-    if persisted is None:
+    persisted_mcp_server, credentials = await _resolve_persisted_dcr_client(mcp_server)
+    if credentials is None:
         return False
-    persisted_mcp_server, credentials = persisted
     if current_redirect_uri is not None and _redirect_uri_not_registered(credentials, current_redirect_uri):
         verbose_logger.debug(
             "register_client_with_server: not reusing persisted DCR client for server_id=%s; its registered "
@@ -1021,18 +1071,19 @@ async def _reuse_persisted_dcr_client_if_available(
     if not _apply_persisted_dcr_credentials(mcp_server, credentials):
         return False
 
-    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: PLC0415
-        global_mcp_server_manager,
-    )
-
-    try:
-        await global_mcp_server_manager.update_server(persisted_mcp_server)
-    except Exception as exc:  # noqa: BLE001
-        verbose_logger.warning(
-            "register_client_with_server: failed to refresh persisted DCR client registration for server_id=%s: %s",
-            mcp_server.server_id,
-            exc,
+    if persisted_mcp_server is not None:
+        from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: PLC0415  # avoids circular import
+            global_mcp_server_manager,
         )
+
+        try:
+            await global_mcp_server_manager.update_server(persisted_mcp_server)
+        except Exception as exc:  # noqa: BLE001  # best-effort registry refresh
+            verbose_logger.warning(
+                "register_client_with_server: failed to refresh persisted DCR client registration for server_id=%s: %s",
+                mcp_server.server_id,
+                exc,
+            )
     return bool(mcp_server.client_id)
 
 
@@ -1044,10 +1095,9 @@ async def _persisted_dcr_redirect_uri_is_stale(mcp_server: MCPServer, current_re
     otherwise short-circuits registration before any redirect check can run. Servers
     without a persisted DCR recording (admin-configured client_id, or registered before
     redirect_uris were recorded) are never reported stale."""
-    persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server)
-    if persisted is None:
+    _, credentials = await _resolve_persisted_dcr_client(mcp_server)
+    if credentials is None:
         return False
-    _, credentials = persisted
     if not _redirect_uri_not_registered(credentials, current_redirect_uri):
         return False
     verbose_logger.warning(
@@ -1067,7 +1117,10 @@ DcrRegistrationPersistenceResult = Literal["persisted", "reused", "skipped", "fa
 async def _persist_dcr_client_registration(
     mcp_server: MCPServer, registration_response: object, current_redirect_uri: str
 ) -> DcrRegistrationPersistenceResult:
-    """Persist the dynamically registered OAuth client (RFC 7591) onto the MCP server row.
+    """Persist the dynamically registered OAuth client (RFC 7591) to its single home: the server's
+    ``LiteLLM_MCPServerTable`` row when it has one, otherwise the server-scoped store when the server
+    is config-declared. A rowless server that is not config-declared is a throwaway temp/session
+    server, so its client is overlaid in memory only and not persisted.
 
     The interactive authorization_code flow mints a ``client_id`` via Dynamic Client
     Registration that discovery cannot re-derive; without persisting it the autonomous
@@ -1106,16 +1159,20 @@ async def _persist_dcr_client_registration(
     if await _reuse_persisted_dcr_client_if_available(mcp_server, current_redirect_uri=current_redirect_uri):
         return "reused"
 
+    token_endpoint_auth_method = (
+        "client_secret_basic" if registration.token_endpoint_auth_method == "client_secret_basic" else None
+    )
     credentials: MCPCredentials = {
         "client_id": registration.client_id,
         "client_secret": registration.client_secret,
-        "token_endpoint_auth_method": (
-            "client_secret_basic" if registration.token_endpoint_auth_method == "client_secret_basic" else None
-        ),
+        "token_endpoint_auth_method": token_endpoint_auth_method,
         "redirect_uris": [current_redirect_uri],
     }
 
-    from litellm.proxy._experimental.mcp_server.db import update_mcp_server  # noqa: PLC0415
+    from litellm.proxy._experimental.mcp_server.db import (  # noqa: PLC0415  # avoids circular import
+        update_mcp_server,
+        upsert_mcp_server_oauth_client_credentials,
+    )
     from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: PLC0415
         global_mcp_server_manager,
     )
@@ -1136,7 +1193,18 @@ async def _persist_dcr_client_registration(
             ),
             touched_by="mcp_oauth_dcr",
         )
-        await global_mcp_server_manager.update_server(updated_row)
+        if updated_row is not None:
+            await global_mcp_server_manager.update_server(updated_row)
+            return "persisted"
+        if global_mcp_server_manager.is_config_declared_server(mcp_server.server_id):
+            await upsert_mcp_server_oauth_client_credentials(
+                prisma_client=prisma_client,
+                server_id=mcp_server.server_id,
+                credentials=credentials,
+            )
+        mcp_server.client_id = registration.client_id
+        mcp_server.client_secret = registration.client_secret
+        mcp_server.token_endpoint_auth_method = token_endpoint_auth_method
         return "persisted"
     except Exception as exc:  # noqa: BLE001
         verbose_logger.warning(

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -93,6 +93,8 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchange_
 )
 from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
     AuthorizationCodeConfig,
+    CredError,
+    IdJagConfig,
     PassthroughConfig,
     ServerSpec,
     TokenExchangeConfig,
@@ -621,6 +623,47 @@ def _consumes_caller_authorization(server: MCPServer) -> bool:
     )
 
 
+_REGISTRY_DUMP_SECRET_FIELDS = frozenset(
+    {"authentication_token", "client_secret", "client_private_key", "aws_secret_access_key", "aws_session_token"}
+)
+
+
+def _redacted_registry_dump(servers: dict[str, MCPServer]) -> dict[str, dict[str, str]]:
+    """A JSON-safe view of the server registry with credential fields masked, for debug logging.
+
+    The registry holds long-lived secrets as plain strings (the static token, OAuth client secret,
+    the ID-JAG signing key, AWS keys); dumping them verbatim hands the gateway's client identity to
+    anyone who can read debug logs.
+    """
+    dumps: dict[str, dict[str, object]] = {server_id: server.model_dump() for server_id, server in servers.items()}
+    return {
+        server_id: {
+            field: ("**REDACTED**" if field in _REGISTRY_DUMP_SECRET_FIELDS and value is not None else str(value))
+            for field, value in dump.items()
+        }
+        for server_id, dump in dumps.items()
+    }
+
+
+def _to_server_spec_fail_closed(server: MCPServer) -> Optional[ServerSpec]:
+    """`to_server_spec`, except a half-configured `oauth2_id_jag` server refuses instead of deferring.
+
+    ID-JAG has no v1 arm, so deferring to v1 would let `resolve_mcp_auth` honor a caller x-mcp-*
+    override or fall through to the static `authentication_token`, both of which bypass the per-user
+    identity assertion the mode promises. That is an operator misconfiguration, not a fallback.
+    """
+    spec = to_server_spec(server)
+    if spec is None and server.auth_type == MCPAuth.oauth2_id_jag:
+        raise_public(
+            CredError.of_misconfigured(
+                "oauth2_id_jag requires token_exchange_endpoint, id_jag_resource_token_endpoint, "
+                "client_id, and a client_secret or client_private_key; refusing to fall back to "
+                "a static credential."
+            )
+        )
+    return spec
+
+
 def _caller_authorization_fans_out(
     server: MCPServer,
     scope_servers: Optional[list[MCPServer]],
@@ -1100,6 +1143,14 @@ class MCPServerManager:
         """
         return self.config_mcp_servers | self.registry
 
+    def is_config_declared_server(self, server_id: str) -> bool:
+        """True when server_id was declared in config.yaml (present in the in-memory config map).
+        Config servers are rowless and persistent, so their DCR client belongs in the server-scoped
+        store; a rowless server that is NOT config-declared is a throwaway temp/session server whose
+        client must not be persisted. This never overrides the row-existence check: a server that has
+        a LiteLLM_MCPServerTable row is always resolved to that row first."""
+        return server_id in self.config_mcp_servers
+
     async def load_servers_from_config(
         self,
         mcp_servers_config: dict[str, Any],
@@ -1318,6 +1369,12 @@ class MCPServerManager:
                     "subject_token_type",
                     DEFAULT_SUBJECT_TOKEN_TYPE,
                 ),
+                # ID-JAG fields
+                id_jag_resource_token_endpoint=server_config.get("id_jag_resource_token_endpoint", None),
+                id_jag_resource=server_config.get("id_jag_resource", None),
+                client_private_key=server_config.get("client_private_key", None),
+                client_private_key_id=server_config.get("client_private_key_id", None),
+                client_assertion_signing_alg=server_config.get("client_assertion_signing_alg", "RS256"),
                 token_exchange_profile=server_config.get("token_exchange_profile", "rfc8693"),
                 allow_sampling=bool(server_config.get("allow_sampling", False)),
                 allow_elicitation=bool(server_config.get("allow_elicitation", False)),
@@ -1338,10 +1395,36 @@ class MCPServerManager:
                     base_url=server_config.get("url", ""),
                 )
 
-        verbose_logger.debug(f"Loaded MCP Servers: {json.dumps(self.config_mcp_servers, indent=4, default=str)}")
+        verbose_logger.debug(
+            f"Loaded MCP Servers: {json.dumps(_redacted_registry_dump(self.config_mcp_servers), indent=4)}"
+        )
+
+        await self._hydrate_config_servers_dcr_clients()
 
         self.initialize_tool_name_to_mcp_server_name_mapping()
 
+    async def _hydrate_config_servers_dcr_clients(self) -> None:
+        """Overlay each config-declared server's persisted DCR client (from the server-scoped
+        store) onto its in-memory object so token refresh authenticates after a restart. A
+        best-effort no-op when the DB is unreachable at config-load time."""
+        from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (  # noqa: PLC0415  # circular import
+            hydrate_config_server_dcr_client,
+        )
+
+        for server in self.config_mcp_servers.values():
+            try:
+                if await hydrate_config_server_dcr_client(server):
+                    verbose_logger.debug(
+                        "hydrated persisted DCR client onto config MCP server server_id=%s",
+                        server.server_id,
+                    )
+            except Exception as exc:  # noqa: BLE001  # best-effort hydration; never fail config load
+                verbose_logger.debug(
+                    "load_servers_from_config: failed to hydrate DCR client for server_id=%s: %s",
+                    server.server_id,
+                    exc,
+                )
+
     async def _register_openapi_tools(self, spec_path: str, server: MCPServer, base_url: str):
         """
         Register tools from an OpenAPI specification for a given server.
@@ -1765,6 +1848,21 @@ class MCPServerManager:
             subject_token_type=mcp_server.subject_token_type
             or (credentials_dict.get("subject_token_type") if credentials_dict else None)
             or DEFAULT_SUBJECT_TOKEN_TYPE,
+            # ID-JAG fields — read from credentials JSON blob
+            id_jag_resource_token_endpoint=(
+                credentials_dict.get("id_jag_resource_token_endpoint") if credentials_dict else None
+            ),
+            id_jag_resource=(credentials_dict.get("id_jag_resource") if credentials_dict else None),
+            client_private_key=self._decrypt_credential_field(
+                credentials_dict.get("client_private_key") if credentials_dict else None,
+                "client_private_key",
+                credentials_are_encrypted,
+            ),
+            client_private_key_id=(credentials_dict.get("client_private_key_id") if credentials_dict else None),
+            client_assertion_signing_alg=(
+                credentials_dict.get("client_assertion_signing_alg") if credentials_dict else None
+            )
+            or "RS256",
             token_exchange_profile=mcp_server.token_exchange_profile
             or (credentials_dict.get("token_exchange_profile") if credentials_dict else None)
             or "rfc8693",
@@ -2641,9 +2739,10 @@ class MCPServerManager:
                 )
                 if not conflicts:
                     return auth, extra_headers
-                if isinstance(spec.config, (TokenExchangeConfig, AuthorizationCodeConfig)):
+                if isinstance(spec.config, (TokenExchangeConfig, AuthorizationCodeConfig, IdJagConfig)):
                     # The resolver owns the per-user credential here (token_exchange's exchanged
-                    # token, authorization_code's stored token). It is authoritative: a guardrail such
+                    # token, authorization_code's stored token, id_jag's minted assertion). It is
+                    # authoritative: a guardrail such
                     # as MCPJWTSigner, static_headers, or any other injected Authorization must NOT
                     # shadow it (otherwise the upstream gets e.g. the signer's JWT instead of the
                     # exchanged token and rejects it). Drop the conflicting header so the resolved
@@ -2734,20 +2833,23 @@ class MCPServerManager:
             Configured MCP client instance.
         """
         transport = server.transport or MCPTransport.sse
-        spec = None if transport == MCPTransport.stdio else to_server_spec(server)
+        spec = None if transport == MCPTransport.stdio else _to_server_spec_fail_closed(server)
         provider = cred_provider or self._cred_provider
         # A caller-supplied per-request override (mcp_auth_header / x-mcp-*) defers to the v1 path
         # so it wins - except for the modes the v2 resolver owns per-caller (authorization_code's
-        # stored token, token_exchange's RFC 8693 minted token, and the passthrough modes'
-        # forwarded caller token). A caller must not be able to substitute another user's stored
-        # credential, nor silently disable the OBO exchange and forward an arbitrary bearer
-        # upstream, so we keep the v2 spec and ignore the override for these; the REST tools
-        # preview supplies its not-yet-persisted token through the resolver (cred_provider),
-        # never this path.
+        # stored token, token_exchange's RFC 8693 minted token, id_jag's minted assertion, and the
+        # passthrough modes' forwarded caller token). A caller must not be able to substitute another
+        # user's stored credential, nor silently disable the OBO / ID-JAG exchange and forward an
+        # arbitrary bearer upstream, so we keep the v2 spec and ignore the override for these; the
+        # REST tools preview supplies its not-yet-persisted token through the resolver
+        # (cred_provider), never this path.
         if (
             spec is not None
             and mcp_auth_header
-            and not isinstance(spec.config, (AuthorizationCodeConfig, PassthroughConfig, TokenExchangeConfig))
+            and not isinstance(
+                spec.config,
+                (AuthorizationCodeConfig, IdJagConfig, PassthroughConfig, TokenExchangeConfig),
+            )
         ):
             spec = None
         auth_value = (
@@ -4276,10 +4378,13 @@ class MCPServerManager:
         if server_auth_header is None:
             server_auth_header = mcp_auth_header
 
-        # Extract subject token for OAuth2 Token Exchange (OBO) flow
+        # Extract subject token for OAuth2 Token Exchange (OBO) and ID-JAG flows
         subject_token: Optional[str] = None
         extra_headers: Optional[dict[str, str]] = None
-        if mcp_server.auth_type == MCPAuth.oauth2_token_exchange:
+        if mcp_server.auth_type in (
+            MCPAuth.oauth2_token_exchange,
+            MCPAuth.oauth2_id_jag,
+        ):
             subject_token = self._extract_bearer_token(oauth2_headers, raw_headers)
         elif mcp_server.auth_type == MCPAuth.oauth2:
             if mcp_server.has_client_credentials:
@@ -4381,10 +4486,10 @@ class MCPServerManager:
             arguments=arguments,
         )
 
-        if mcp_server.auth_type == MCPAuth.oauth2_token_exchange and subject_token:
-            # OBO: the exchanged token may have been revoked/rotated upstream since it was cached, so
-            # an upstream 401 gets one re-mint + retry. Gated to this mode; all others keep the plain
-            # single call below.
+        if mcp_server.auth_type in (MCPAuth.oauth2_token_exchange, MCPAuth.oauth2_id_jag) and subject_token:
+            # OBO / ID-JAG: the exchanged token may have been revoked/rotated upstream since it was
+            # cached, so an upstream 401 gets one invalidate + re-mint + retry. Gated to these modes;
+            # all others keep the plain single call below.
             async def _obo_call_tool_limited():
                 async with self._limit_outbound_concurrency(mcp_server):
                     return await self._obo_call_tool_with_retry(
@@ -4935,6 +5040,8 @@ class MCPServerManager:
 
         verbose_logger.debug("MCP registry refreshed (%s servers in registry)", len(registered_registry))
 
+        await self._hydrate_config_servers_dcr_clients()
+
     def get_mcp_servers_from_ids(self, server_ids: list[str]) -> list[MCPServer]:
         servers = []
         registry = self.get_registry()

litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py

@@ -31,10 +31,14 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
     AwsCredentialSource,
     AwsSigV4Config,
     Byok,
+    ClientAuth,
     ClientCredentialsConfig,
+    ClientSecretAuth,
     CredError,
+    IdJagConfig,
     NoneConfig,
     PassthroughConfig,
+    PrivateKeyJwtAuth,
     ServerSpec,
     SharedKey,
     StaticKeys,
@@ -59,6 +63,10 @@ __all__ = [
     "AuthorizationCodeConfig",
     "ClientCredentialsConfig",
     "TokenExchangeConfig",
+    "IdJagConfig",
+    "ClientAuth",
+    "PrivateKeyJwtAuth",
+    "ClientSecretAuth",
     "ApiKeyConfig",
     "ApiKeySource",
     "SharedKey",

litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py

@@ -21,9 +21,13 @@ from typing_extensions import assert_never
 from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
     ApiKeyConfig,
     AuthorizationCodeConfig,
+    ClientAuth,
+    ClientSecretAuth,
     CredError,
+    IdJagConfig,
     NoneConfig,
     PassthroughConfig,
+    PrivateKeyJwtAuth,
     ServerSpec,
     SharedKey,
     Subject,
@@ -35,6 +39,9 @@ if TYPE_CHECKING:
     from litellm.proxy._types import UserAPIKeyAuth
     from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
+_TOKEN_EXCHANGE_SUBJECT_TOKEN_DEFAULT = "urn:ietf:params:oauth:token-type:access_token"
+_ID_JAG_SUBJECT_TOKEN_DEFAULT = "urn:ietf:params:oauth:token-type:id_token"
+
 
 def to_subject(user_api_key_auth: Optional[UserAPIKeyAuth], subject_token: Optional[str]) -> Subject:
     """Map v1's authenticated principal onto the resolver's Subject.
@@ -96,6 +103,8 @@ def to_server_spec(server: MCPServer) -> Optional[ServerSpec]:
                 )
             # client_credentials (M2M) and delegate/passthrough oauth2 stay on v1
             return None
+        case MCPAuth.oauth2_id_jag:
+            return _id_jag_spec(server, resource)
         case MCPAuth.true_passthrough | MCPAuth.oauth_delegate:
             return ServerSpec(server_id=server.server_id, resource=resource, config=PassthroughConfig())
         case MCPAuth.oauth2_token_exchange:
@@ -167,6 +176,58 @@ def _shared_key_spec(
     )
 
 
+def _id_jag_spec(server: MCPServer, resource: str) -> Optional[ServerSpec]:
+    """Build an ID-JAG spec from the v1 server's raw fields, or defer (None) if half-configured.
+
+    The enum already routes here, but a server missing an endpoint, ``client_id``, or any client-auth
+    secret would make ``IdJagConfig`` raise at construction; returning None instead defers to v1 so a
+    partially configured server does not 500. ``token_exchange_endpoint`` is leg 1 (the IdP org AS);
+    leg 2 is ``id_jag_resource_token_endpoint`` (the upstream resource AS).
+    """
+    org_token_endpoint = server.token_exchange_endpoint
+    resource_token_endpoint = server.id_jag_resource_token_endpoint
+    client_id = server.client_id
+    client_auth = _id_jag_client_auth(server)
+    if not org_token_endpoint or not resource_token_endpoint or not client_id or client_auth is None:
+        return None
+    return ServerSpec(
+        server_id=server.server_id,
+        resource=resource,
+        config=IdJagConfig(
+            org_token_endpoint=org_token_endpoint,
+            resource_token_endpoint=resource_token_endpoint,
+            client_id=client_id,
+            client_auth=client_auth,
+            subject_token_type=_id_jag_subject_token_type(server),
+            audience=server.audience,
+            resource=server.id_jag_resource,
+            scopes=tuple(server.scopes or ()),
+        ),
+    )
+
+
+def _id_jag_client_auth(server: MCPServer) -> Optional[ClientAuth]:
+    """Private-key JWT when a key is configured, else client_secret, else None (defer to v1)."""
+    if server.client_private_key:
+        return PrivateKeyJwtAuth(
+            private_key=SecretStr(server.client_private_key),
+            key_id=server.client_private_key_id,
+            signing_alg=server.client_assertion_signing_alg,
+        )
+    if server.client_secret:
+        return ClientSecretAuth(client_secret=SecretStr(server.client_secret))
+    return None
+
+
+def _id_jag_subject_token_type(server: MCPServer) -> str:
+    """ID-JAG asserts the user's id_token, so the token-exchange access_token default maps to id_token;
+    an explicitly configured value (e.g. a SAML2 assertion type) is honored verbatim."""
+    configured = server.subject_token_type
+    if configured and configured != _TOKEN_EXCHANGE_SUBJECT_TOKEN_DEFAULT:
+        return configured
+    return _ID_JAG_SUBJECT_TOKEN_DEFAULT
+
+
 def raise_public(error: CredError) -> NoReturn:
     """Map a resolver CredError onto the proxy's public HTTP contract. The one edge that raises."""
     match error.tag:

litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py

@@ -16,6 +16,8 @@ follow-up PR with their seam. Pure v2: no imports from v1.
 
 from __future__ import annotations
 
+import hashlib
+
 import httpx
 from typing_extensions import assert_never
 
@@ -33,6 +35,11 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
     Ok,
     Result,
 )
+from litellm.proxy._experimental.mcp_server.outbound_credentials.token_endpoint import (
+    ExchangedToken,
+    ExchangedTokenCache,
+    TokenEndpointClient,
+)
 from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import (
     TokenExchanger,
 )
@@ -42,16 +49,24 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
     AuthSpecKind,
     AwsSigV4Config,
     Byok,
+    ClientAuth,
     ClientCredentialsConfig,
+    ClientSecretAuth,
     CredError,
+    IdJagConfig,
     NoneConfig,
     PassthroughConfig,
+    PrivateKeyJwtAuth,
     ServerSpec,
     SharedKey,
     Subject,
     TokenExchangeConfig,
 )
 
+_TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
+_JWT_BEARER_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:jwt-bearer"
+_ID_JAG_REQUESTED_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:id-jag"
+
 
 class _NullOAuthTokenStore:
     """Fail-closed default: with no token store wired, every user reads as not authorized."""
@@ -87,9 +102,13 @@ class UpstreamCredentialProvider:
         self,
         oauth_token_store: OAuthTokenStore | None = None,
         token_exchanger: TokenExchanger | None = None,
+        token_endpoint: TokenEndpointClient | None = None,
+        exchanged_tokens: ExchangedTokenCache | None = None,
     ) -> None:
         self._oauth_token_store: OAuthTokenStore = oauth_token_store or _NullOAuthTokenStore()
         self._token_exchanger: TokenExchanger = token_exchanger or _NullTokenExchanger()
+        self._token_endpoint: TokenEndpointClient = token_endpoint or TokenEndpointClient()
+        self._exchanged_tokens: ExchangedTokenCache = exchanged_tokens or ExchangedTokenCache()
 
     async def resolve_credentials(self, subject: Subject, server: ServerSpec) -> Result[httpx.Auth, CredError]:
         match server.config:
@@ -103,6 +122,8 @@ class UpstreamCredentialProvider:
                 return _not_implemented(AuthSpecKind.client_credentials)
             case TokenExchangeConfig() as config:
                 return await self._token_exchange(subject, server, config)
+            case IdJagConfig() as config:
+                return await self._id_jag(subject, server, config)
             case AuthorizationCodeConfig():
                 return await self._authorization_code(subject, server)
             case AwsSigV4Config():
@@ -141,6 +162,53 @@ class UpstreamCredentialProvider:
                 return Error(CredError.of_not_implemented("api_key BYOK source not implemented yet"))
         assert_never(config.key_source)
 
+    async def _id_jag(self, subject: Subject, server: ServerSpec, config: IdJagConfig) -> Result[httpx.Auth, CredError]:
+        if subject.inbound_token is None:
+            return Error(
+                CredError.of_precondition_required(
+                    "ID-JAG requires a caller identity token; it asserts the calling "
+                    "user's identity upstream and cannot use a static credential."
+                )
+            )
+        token = subject.inbound_token.get_secret_value()
+        cache_key = _id_jag_cache_key(token, server.server_id, config)
+
+        async def _exchange() -> Result[ExchangedToken, CredError]:
+            leg1_params = {
+                "grant_type": _TOKEN_EXCHANGE_GRANT_TYPE,
+                "requested_token_type": _ID_JAG_REQUESTED_TOKEN_TYPE,
+                "subject_token": token,
+                "subject_token_type": config.subject_token_type,
+                **({"audience": config.audience} if config.audience else {}),
+                **({"resource": config.resource} if config.resource else {}),
+                **({"scope": " ".join(config.scopes)} if config.scopes else {}),
+            }
+            match await self._token_endpoint.fetch(
+                config.org_token_endpoint,
+                config.client_id,
+                leg1_params,
+                config.client_auth,
+            ):
+                case Error(err):
+                    return Error(err)
+                case Ok(id_jag):
+                    leg2_params = {
+                        "grant_type": _JWT_BEARER_GRANT_TYPE,
+                        "assertion": id_jag.access_token,
+                    }
+                    return await self._token_endpoint.fetch(
+                        config.resource_token_endpoint,
+                        config.client_id,
+                        leg2_params,
+                        config.client_auth,
+                    )
+
+        match await self._exchanged_tokens.get_or_compute(cache_key, _exchange):
+            case Ok(access_token):
+                return Ok(StaticHeaderAuth(f"Bearer {access_token}"))
+            case Error(err):
+                return Error(err)
+
     async def _authorization_code(self, subject: Subject, server: ServerSpec) -> Result[StaticHeaderAuth, CredError]:
         token = await self._authz_token(subject, server)
         if token is None:
@@ -176,13 +244,19 @@ class UpstreamCredentialProvider:
         """Drop any cached credential the resolver owns for this `(subject, server)`.
 
         Used after an upstream rejects the injected credential, so the next resolve re-mints rather
-        than serving the same rejected token until TTL. Only `token_exchange` holds a re-mintable
-        cached credential here; other modes are a no-op.
+        than serving the same rejected token until TTL. `token_exchange` and `id_jag` hold a
+        re-mintable cached credential here; other modes are a no-op.
         """
-        if isinstance(server.config, TokenExchangeConfig) and subject.inbound_token is not None:
+        if subject.inbound_token is None:
+            return
+        if isinstance(server.config, TokenExchangeConfig):
             await self._token_exchanger.invalidate(
                 subject.inbound_token.get_secret_value(), server, server.config, tenant_id=subject.tenant_id
             )
+        if isinstance(server.config, IdJagConfig):
+            self._exchanged_tokens.invalidate(
+                _id_jag_cache_key(subject.inbound_token.get_secret_value(), server.server_id, server.config)
+            )
 
     async def _authz_token(self, subject: Subject, server: ServerSpec) -> OAuthToken | None:
         """The user's authorization_code token, or None when absent or the store is unreachable.
@@ -196,5 +270,41 @@ class UpstreamCredentialProvider:
             return None
 
 
+def _id_jag_cache_key(subject_token: str, server_id: str, config: IdJagConfig) -> str:
+    """Bind the cached leg-2 bearer to the caller token, the server, AND the config that minted it.
+
+    Every exchange parameter derives from the config (endpoints, audience, resource, scopes, client
+    auth), so a server update that changes any of them must change the key; otherwise the old bearer,
+    authorized under the old policy, keeps being served until its TTL. Everything is hashed, so no
+    secret is held in the key.
+    """
+    material = "\x00".join(
+        (
+            subject_token,
+            server_id,
+            config.org_token_endpoint,
+            config.resource_token_endpoint,
+            config.client_id,
+            _client_auth_fingerprint(config.client_auth),
+            config.subject_token_type,
+            config.audience or "",
+            config.resource or "",
+            " ".join(config.scopes),
+        )
+    )
+    return hashlib.sha256(material.encode()).hexdigest()
+
+
+def _client_auth_fingerprint(client_auth: ClientAuth) -> str:
+    match client_auth:
+        case PrivateKeyJwtAuth() as auth:
+            return "\x00".join(
+                ("private_key_jwt", auth.private_key.get_secret_value(), auth.key_id or "", auth.signing_alg)
+            )
+        case ClientSecretAuth() as auth:
+            return "\x00".join(("client_secret", auth.client_secret.get_secret_value()))
+    assert_never(client_auth)
+
+
 def _not_implemented(kind: AuthSpecKind) -> Result[httpx.Auth, CredError]:
     return Error(CredError.of_not_implemented(f"{kind.value}: resolver arm not implemented yet"))

litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py

@@ -0,0 +1,225 @@
+"""An authenticated OAuth token-endpoint call plus a short-lived-token cache.
+
+`TokenEndpointClient.fetch` POSTs one grant to a token endpoint, authenticating the gateway as
+an OAuth client via `client_auth` (RFC 7523 private-key JWT, or `client_secret_post`), and returns
+the minted token or a typed `CredError`. `ExchangedTokenCache` memoizes the final token string per
+opaque cache key with per-key single-flight, so concurrent callers share one round-trip and a hit
+skips the endpoint entirely.
+
+Pure v2: no imports from the v1 MCP auth handlers. The multi-leg flows that compose these (ID-JAG,
+and later token_exchange / client_credentials) live in the resolver arms; this collaborator owns
+only the single authenticated call and the cache.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+import uuid
+import weakref
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import dataclass
+
+import httpx
+import jwt
+from pydantic import BaseModel, ValidationError
+from typing_extensions import assert_never
+
+from litellm._logging import verbose_proxy_logger
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.constants import (
+    MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
+    MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
+    MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
+    MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
+)
+from litellm.exceptions import Timeout
+from litellm.llms.custom_httpx.http_handler import (
+    get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]  # litellm http handler is untyped
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
+    Error,
+    Ok,
+    Result,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
+    ClientAuth,
+    ClientSecretAuth,
+    CredError,
+    PrivateKeyJwtAuth,
+)
+from litellm.types.llms.custom_http import httpxSpecialProvider
+
+CLIENT_ASSERTION_TYPE = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"
+CLIENT_ASSERTION_LIFETIME_SECONDS = 60
+
+
+@dataclass(frozen=True, slots=True)
+class ExchangedToken:
+    access_token: str
+    expires_in: int | None
+
+
+class _TokenEndpointResponse(BaseModel):
+    access_token: str
+    expires_in: int | None = None
+
+
+class TokenEndpointClient:
+    """One authenticated POST to an OAuth token endpoint, returning the minted token as a value."""
+
+    async def fetch(
+        self,
+        endpoint: str,
+        client_id: str,
+        grant_params: Mapping[str, str],
+        client_auth: ClientAuth,
+    ) -> Result[ExchangedToken, CredError]:
+        try:
+            data = {**grant_params, **_client_auth_params(endpoint, client_id, client_auth)}
+        except (ValueError, TypeError, NotImplementedError, jwt.PyJWTError):
+            verbose_proxy_logger.warning("MCP token endpoint %s: could not sign the client assertion", endpoint)
+            return Error(
+                CredError.of_misconfigured(
+                    "token exchange failed: could not sign the client assertion; "
+                    "check client_private_key and client_assertion_signing_alg"
+                )
+            )
+        try:
+            raw = await _post_form(endpoint, data)
+        except httpx.HTTPStatusError as exc:
+            verbose_proxy_logger.warning(
+                "MCP token endpoint %s failed with status %s", endpoint, exc.response.status_code
+            )
+            return Error(
+                CredError.of_upstream_unavailable(f"token exchange failed with status {exc.response.status_code}")
+            )
+        except (httpx.RequestError, Timeout) as exc:
+            verbose_proxy_logger.warning("MCP token endpoint %s unreachable: %s", endpoint, type(exc).__name__)
+            return Error(
+                CredError.of_upstream_unavailable(
+                    f"token exchange failed: token endpoint unreachable ({type(exc).__name__})"
+                )
+            )
+        except json.JSONDecodeError:
+            verbose_proxy_logger.warning("MCP token endpoint %s returned a non-JSON response", endpoint)
+            return Error(
+                CredError.of_upstream_unavailable("token exchange failed: token endpoint returned a non-JSON response")
+            )
+        if raw is None:
+            verbose_proxy_logger.warning("MCP token endpoint %s returned no response", endpoint)
+            return Error(CredError.of_upstream_unavailable("token exchange failed: no response from token endpoint"))
+        try:
+            parsed = _TokenEndpointResponse.model_validate(raw)
+        except ValidationError:
+            verbose_proxy_logger.warning("MCP token endpoint %s response missing access_token", endpoint)
+            return Error(
+                CredError.of_upstream_unavailable("token exchange failed: token endpoint response missing access_token")
+            )
+        return Ok(ExchangedToken(access_token=parsed.access_token, expires_in=parsed.expires_in))
+
+
+class ExchangedTokenCache:
+    """Memoizes the final token string per key, single-flighting concurrent misses on one lock."""
+
+    def __init__(self) -> None:
+        self._cache = InMemoryCache(
+            max_size_in_memory=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
+            default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
+        )
+        self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = weakref.WeakValueDictionary()
+
+    async def get_or_compute(
+        self,
+        cache_key: str,
+        compute: Callable[[], Awaitable[Result[ExchangedToken, CredError]]],
+    ) -> Result[str, CredError]:
+        cached = self._get(cache_key)
+        if cached is not None:
+            return Ok(cached)
+        async with self._lock(cache_key):
+            cached = self._get(cache_key)
+            if cached is not None:
+                return Ok(cached)
+            match await compute():
+                case Ok(token):
+                    self._cache.set_cache(  # pyright: ignore[reportUnknownMemberType]  # InMemoryCache is untyped
+                        cache_key,
+                        token.access_token,
+                        ttl=_cache_ttl_seconds(token.expires_in),
+                    )
+                    return Ok(token.access_token)
+                case Error(err):
+                    return Error(err)
+
+    def invalidate(self, cache_key: str) -> None:
+        """Evict one cached token so the next `get_or_compute` re-mints (e.g. after an upstream 401)."""
+        self._cache.delete_cache(cache_key)  # pyright: ignore[reportUnknownMemberType]  # InMemoryCache is untyped
+
+    def _get(self, cache_key: str) -> str | None:
+        value = self._cache.get_cache(cache_key)  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]  # InMemoryCache is untyped; narrowed by isinstance below
+        return value if isinstance(value, str) else None
+
+    def _lock(self, cache_key: str) -> asyncio.Lock:
+        lock = self._locks.get(cache_key)
+        if lock is None:
+            lock = asyncio.Lock()
+            self._locks[cache_key] = lock
+        return lock
+
+
+def _cache_ttl_seconds(expires_in: int | None) -> int:
+    lifetime = expires_in if expires_in is not None else MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
+    return max(
+        lifetime - MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
+        MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
+    )
+
+
+async def _post_form(endpoint: str, data: dict[str, str]) -> object | None:
+    # litellm's httpx handler and httpx.Response are only partially typed; the token endpoint
+    # returns a JSON object that `_TokenEndpointResponse` validates, so the untyped boundary is
+    # contained here. A non-2xx raises `httpx.HTTPStatusError`, an unreachable endpoint raises
+    # `httpx.RequestError` (or litellm's `Timeout`, which the handler substitutes for
+    # `httpx.TimeoutException`), and a non-JSON body raises `json.JSONDecodeError`; `fetch` maps
+    # each to a CredError.
+    client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)  # pyright: ignore[reportUnknownVariableType]  # litellm http handler is untyped
+    response = await client.post(endpoint, data=data)  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]  # litellm http handler is untyped
+    if response is None:
+        return None
+    response.raise_for_status()
+    return response.json()  # pyright: ignore[reportAny]  # untyped JSON; validated by _TokenEndpointResponse in fetch
+
+
+def _client_auth_params(endpoint: str, client_id: str, client_auth: ClientAuth) -> dict[str, str]:
+    match client_auth:
+        case PrivateKeyJwtAuth() as auth:
+            return {
+                "client_id": client_id,
+                "client_assertion_type": CLIENT_ASSERTION_TYPE,
+                "client_assertion": _client_assertion(endpoint, client_id, auth),
+            }
+        case ClientSecretAuth() as auth:
+            return {
+                "client_id": client_id,
+                "client_secret": auth.client_secret.get_secret_value(),
+            }
+    assert_never(client_auth)
+
+
+def _client_assertion(endpoint: str, client_id: str, auth: PrivateKeyJwtAuth) -> str:
+    now = int(time.time())
+    return jwt.encode(
+        {
+            "iss": client_id,
+            "sub": client_id,
+            "aud": endpoint,
+            "jti": uuid.uuid4().hex,
+            "iat": now,
+            "exp": now + CLIENT_ASSERTION_LIFETIME_SECONDS,
+        },
+        auth.private_key.get_secret_value(),
+        algorithm=auth.signing_alg,
+        headers={"kid": auth.key_id} if auth.key_id else None,
+    )

litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py

@@ -56,6 +56,7 @@ class AuthSpecKind(str, Enum):
     authorization_code = "authorization_code"  # per-user 3LO; gateway-stored token
     client_credentials = "client_credentials"  # gateway service account (M2M)
     token_exchange = "token_exchange"  # RFC 8693: token endpoint + subject_token (OBO)
+    id_jag = "id_jag"  # draft-ietf-oauth-identity-assertion-authz-grant: two-leg exchange then jwt-bearer
     api_key = "api_key"  # static header, any scheme (BYOK = per-user-seeded source)
     passthrough = "passthrough"  # client forwards an upstream-audience token
     none = "none"  # no upstream credential; resolve yields a no-op auth, never an error
@@ -225,6 +226,49 @@ class TokenExchangeConfig(BaseModel):
     scopes: tuple[str, ...] = ()
 
 
+class PrivateKeyJwtAuth(BaseModel):
+    """RFC 7523 private-key-JWT client authentication: the gateway signs a `client_assertion`."""
+
+    model_config = ConfigDict(frozen=True)
+    source: Literal["private_key_jwt"] = "private_key_jwt"
+    private_key: SecretStr
+    key_id: str | None = None
+    signing_alg: str = "RS256"
+
+
+class ClientSecretAuth(BaseModel):
+    """`client_secret_post` client authentication: the gateway posts `client_id` + `client_secret`."""
+
+    model_config = ConfigDict(frozen=True)
+    source: Literal["client_secret"] = "client_secret"
+    client_secret: SecretStr
+
+
+ClientAuth = Annotated[PrivateKeyJwtAuth | ClientSecretAuth, Field(discriminator="source")]
+
+
+class IdJagConfig(BaseModel):
+    """draft-ietf-oauth-identity-assertion-authz-grant (Okta "AI agent token exchange").
+
+    Two legs: leg 1 is an RFC 8693 token exchange at the IdP org AS (`org_token_endpoint`) that
+    swaps the caller's identity token for an ID-JAG assertion; leg 2 is an RFC 7523 jwt-bearer at
+    the upstream resource AS (`resource_token_endpoint`) that swaps the assertion for the access
+    token. The gateway authenticates to both endpoints as `client_id` via `client_auth`. Required
+    fields are enforced at construction so a half-configured server cannot reach the arm.
+    """
+
+    model_config = ConfigDict(frozen=True)
+    kind: Literal[AuthSpecKind.id_jag] = AuthSpecKind.id_jag
+    org_token_endpoint: str
+    resource_token_endpoint: str
+    client_id: str
+    client_auth: ClientAuth
+    subject_token_type: str = "urn:ietf:params:oauth:token-type:id_token"
+    audience: str | None = None
+    resource: str | None = None
+    scopes: tuple[str, ...] = ()
+
+
 class SharedKey(BaseModel):
     """A fixed key configured on the server, identical for every caller."""
 
@@ -323,6 +367,7 @@ AuthConfig = Annotated[
     AuthorizationCodeConfig
     | ClientCredentialsConfig
     | TokenExchangeConfig
+    | IdJagConfig
     | ApiKeyConfig
     | PassthroughConfig
     | NoneConfig

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt

@@ -1,9 +1,9 @@
 1:"$Sreact.fragment"
-2:I[347257,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ClientPageRoot"]
-3:I[871135,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","/litellm-asset-prefix/_next/static/chunks/0kc37~1yrtr2p.js","/litellm-asset-prefix/_next/static/chunks/04s-iyzsr4cq~.js","/litellm-asset-prefix/_next/static/chunks/16zj68af4snfa.js","/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","/litellm-asset-prefix/_next/static/chunks/0tmaomqtwbi33.js","/litellm-asset-prefix/_next/static/chunks/02-2~p5k.ielz.js","/litellm-asset-prefix/_next/static/chunks/0op63kdo3uwng.js","/litellm-asset-prefix/_next/static/chunks/112_-0alpxot8.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/12yfh0_n50ojz.js","/litellm-asset-prefix/_next/static/chunks/06w8_.601z7_i.js","/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/18187o3gb9vc5.js","/litellm-asset-prefix/_next/static/chunks/0._ir~nvcseg7.js","/litellm-asset-prefix/_next/static/chunks/0vffq7buvlg04.js","/litellm-asset-prefix/_next/static/chunks/122djf0bncn-8.js","/litellm-asset-prefix/_next/static/chunks/0axk76owb7jv..js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0sqw622fcvsv4.js","/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/14g~hmf3h_efw.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0cmepm.jkel-i.js","/litellm-asset-prefix/_next/static/chunks/0onea0n77pqw1.js","/litellm-asset-prefix/_next/static/chunks/02-u6qtmsnqn0.js","/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","/litellm-asset-prefix/_next/static/chunks/05q6y.kb.q2s..js","/litellm-asset-prefix/_next/static/chunks/0rehsq9xe1kde.js","/litellm-asset-prefix/_next/static/chunks/0nk7-_~gcxbz0.js","/litellm-asset-prefix/_next/static/chunks/0~r95y0t-0dlp.js"],"default"]
-6:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"OutletBoundary"]
+2:I[347257,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ClientPageRoot"]
+3:I[871135,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","/litellm-asset-prefix/_next/static/chunks/0mqbd99.ej13v.js","/litellm-asset-prefix/_next/static/chunks/16ufy1iyybswo.js","/litellm-asset-prefix/_next/static/chunks/0ayum-x.hkww~.js","/litellm-asset-prefix/_next/static/chunks/15jl-1gcakfwa.js","/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","/litellm-asset-prefix/_next/static/chunks/0u.r3vzo30ofk.js","/litellm-asset-prefix/_next/static/chunks/0g4qcx-c9gsxn.js","/litellm-asset-prefix/_next/static/chunks/14a-un1blorp~.js","/litellm-asset-prefix/_next/static/chunks/02_q4881cz6h~.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0wdlbe750tuzr.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","/litellm-asset-prefix/_next/static/chunks/03zkt5iyjiqcz.js","/litellm-asset-prefix/_next/static/chunks/0l02mpo6za6ie.js","/litellm-asset-prefix/_next/static/chunks/1667t2pcy0iqm.js","/litellm-asset-prefix/_next/static/chunks/0-_m4km7b1~oe.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0sqw622fcvsv4.js","/litellm-asset-prefix/_next/static/chunks/0t8el_ijoskx..js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0a.ljputcx8g5.js","/litellm-asset-prefix/_next/static/chunks/02-u6qtmsnqn0.js","/litellm-asset-prefix/_next/static/chunks/076.vm.7w-x2..js","/litellm-asset-prefix/_next/static/chunks/069dx5~5osue0.js","/litellm-asset-prefix/_next/static/chunks/0onea0n77pqw1.js","/litellm-asset-prefix/_next/static/chunks/16jfov1k2wrj0.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","/litellm-asset-prefix/_next/static/chunks/01reddhq423_f.js","/litellm-asset-prefix/_next/static/chunks/0h80lrrstjswl.js","/litellm-asset-prefix/_next/static/chunks/0cmepm.jkel-i.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/04jv9e6~9vi.l.js","/litellm-asset-prefix/_next/static/chunks/0d_sm.._5mw-p.js","/litellm-asset-prefix/_next/static/chunks/0elr0ye86.44-.js"],"default"]
+6:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"OutletBoundary"]
 7:"$Sreact.suspense"
-0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0vffq7buvlg04.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/122djf0bncn-8.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0axk76owb7jv..js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0sqw622fcvsv4.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/14g~hmf3h_efw.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0cmepm.jkel-i.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0onea0n77pqw1.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/02-u6qtmsnqn0.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/05q6y.kb.q2s..js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0rehsq9xe1kde.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0nk7-_~gcxbz0.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/0~r95y0t-0dlp.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
+0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1667t2pcy0iqm.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-_m4km7b1~oe.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0sqw622fcvsv4.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0t8el_ijoskx..js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0a.ljputcx8g5.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/02-u6qtmsnqn0.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/076.vm.7w-x2..js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/069dx5~5osue0.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0onea0n77pqw1.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/16jfov1k2wrj0.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/01reddhq423_f.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0h80lrrstjswl.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0cmepm.jkel-i.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/04jv9e6~9vi.l.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/0d_sm.._5mw-p.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/0elr0ye86.44-.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"DSHomUr6Sq46Bm2WLdUas"}
 4:{}
 5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
 8:null

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt

@@ -1,7 +1,7 @@
 1:"$Sreact.fragment"
-2:I[92825,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ClientSegmentRoot"]
-3:I[216370,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","/litellm-asset-prefix/_next/static/chunks/0kc37~1yrtr2p.js","/litellm-asset-prefix/_next/static/chunks/04s-iyzsr4cq~.js","/litellm-asset-prefix/_next/static/chunks/16zj68af4snfa.js","/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","/litellm-asset-prefix/_next/static/chunks/0tmaomqtwbi33.js","/litellm-asset-prefix/_next/static/chunks/02-2~p5k.ielz.js","/litellm-asset-prefix/_next/static/chunks/0op63kdo3uwng.js","/litellm-asset-prefix/_next/static/chunks/112_-0alpxot8.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/12yfh0_n50ojz.js","/litellm-asset-prefix/_next/static/chunks/06w8_.601z7_i.js","/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/18187o3gb9vc5.js","/litellm-asset-prefix/_next/static/chunks/0._ir~nvcseg7.js"],"default"]
-4:I[339756,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
-5:I[837457,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
-0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0kc37~1yrtr2p.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/04s-iyzsr4cq~.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/16zj68af4snfa.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0tmaomqtwbi33.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/02-2~p5k.ielz.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0op63kdo3uwng.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/112_-0alpxot8.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/12yfh0_n50ojz.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/06w8_.601z7_i.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/18187o3gb9vc5.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0._ir~nvcseg7.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
+2:I[92825,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ClientSegmentRoot"]
+3:I[216370,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","/litellm-asset-prefix/_next/static/chunks/0mqbd99.ej13v.js","/litellm-asset-prefix/_next/static/chunks/16ufy1iyybswo.js","/litellm-asset-prefix/_next/static/chunks/0ayum-x.hkww~.js","/litellm-asset-prefix/_next/static/chunks/15jl-1gcakfwa.js","/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","/litellm-asset-prefix/_next/static/chunks/0u.r3vzo30ofk.js","/litellm-asset-prefix/_next/static/chunks/0g4qcx-c9gsxn.js","/litellm-asset-prefix/_next/static/chunks/14a-un1blorp~.js","/litellm-asset-prefix/_next/static/chunks/02_q4881cz6h~.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0wdlbe750tuzr.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","/litellm-asset-prefix/_next/static/chunks/03zkt5iyjiqcz.js","/litellm-asset-prefix/_next/static/chunks/0l02mpo6za6ie.js"],"default"]
+4:I[339756,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
+5:I[837457,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
+0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0mqbd99.ej13v.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/16ufy1iyybswo.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0ayum-x.hkww~.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/15jl-1gcakfwa.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0u.r3vzo30ofk.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0g4qcx-c9gsxn.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/14a-un1blorp~.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/02_q4881cz6h~.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0wdlbe750tuzr.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/03zkt5iyjiqcz.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0l02mpo6za6ie.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"DSHomUr6Sq46Bm2WLdUas"}
 6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

litellm/proxy/_experimental/out/__next._head.txt

@@ -1,6 +1,6 @@
 1:"$Sreact.fragment"
-2:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ViewportBoundary"]
-3:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"MetadataBoundary"]
+2:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ViewportBoundary"]
+3:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"MetadataBoundary"]
 4:"$Sreact.suspense"
-5:I[27201,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"IconMark"]
-0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
+5:I[27201,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"IconMark"]
+0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"DSHomUr6Sq46Bm2WLdUas"}

litellm/proxy/_experimental/out/__next._index.txt

@@ -1,9 +1,9 @@
 1:"$Sreact.fragment"
-2:I[867271,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
-3:I[71195,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
-4:I[557951,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"AuthProvider"]
-5:I[339756,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
-6:I[837457,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
+2:I[867271,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
+3:I[71195,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
+4:I[557951,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"AuthProvider"]
+5:I[339756,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
+6:I[837457,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
 :HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/15j3hwz2dxrik.css","style"]
-0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/15j3hwz2dxrik.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
+:HL["/litellm-asset-prefix/_next/static/chunks/0aapv6n5bztwf.css","style"]
+0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0aapv6n5bztwf.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0l6q6-77u4dy~.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"DSHomUr6Sq46Bm2WLdUas"}

litellm/proxy/_experimental/out/__next._tree.txt

@@ -1,4 +1,4 @@
 :HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/15j3hwz2dxrik.css","style"]
+:HL["/litellm-asset-prefix/_next/static/chunks/0aapv6n5bztwf.css","style"]
 :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
-0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
+0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"DSHomUr6Sq46Bm2WLdUas"}

litellm/proxy/_experimental/out/_next/static/chunks/0-hrh_uw98wb_.js

@@ -0,0 +1,31 @@
+(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,742732,(e,r,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0}),Object.defineProperty(t,"HeadManagerContext",{enumerable:!0,get:function(){return n}});let n=e.r(555682)._(e.r(271645)).default.createContext({})},18576,(e,r,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n={WarningIcon:function(){return d},errorStyles:function(){return l},errorThemeCss:function(){return a}};for(var o in n)Object.defineProperty(t,o,{enumerable:!0,get:n[o]});e.r(555682);let i=e.r(843476);e.r(271645);let l={container:{fontFamily:'system-ui,"Segoe UI",Roboto,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji"',height:"100vh",display:"flex",alignItems:"center",justifyContent:"center"},card:{marginTop:"-32px",maxWidth:"325px",padding:"32px 28px",textAlign:"left"},icon:{marginBottom:"24px"},title:{fontSize:"24px",fontWeight:500,letterSpacing:"-0.02em",lineHeight:"32px",margin:"0 0 12px 0",color:"var(--next-error-title)"},message:{fontSize:"14px",fontWeight:400,lineHeight:"21px",margin:"0 0 20px 0",color:"var(--next-error-message)"},form:{margin:0},buttonGroup:{display:"flex",gap:"8px",alignItems:"center"},button:{display:"inline-flex",alignItems:"center",justifyContent:"center",height:"32px",padding:"0 12px",fontSize:"14px",fontWeight:500,lineHeight:"20px",borderRadius:"6px",cursor:"pointer",color:"var(--next-error-btn-text)",background:"var(--next-error-btn-bg)",border:"var(--next-error-btn-border)"},buttonSecondary:{display:"inline-flex",alignItems:"center",justifyContent:"center",height:"32px",padding:"0 12px",fontSize:"14px",fontWeight:500,lineHeight:"20px",borderRadius:"6px",cursor:"pointer",color:"var(--next-error-btn-secondary-text)",background:"var(--next-error-btn-secondary-bg)",border:"var(--next-error-btn-secondary-border)"},digestFooter:{position:"fixed",bottom:"32px",left:"0",right:"0",textAlign:"center",fontFamily:'ui-monospace,SFMono-Regular,"SF Mono",Menlo,Consolas,monospace',fontSize:"12px",lineHeight:"18px",fontWeight:400,margin:"0",color:"var(--next-error-digest)"}},a=`
+:root {
+  --next-error-bg: #fff;
+  --next-error-text: #171717;
+  --next-error-title: #171717;
+  --next-error-message: #171717;
+  --next-error-digest: #666666;
+  --next-error-btn-text: #fff;
+  --next-error-btn-bg: #171717;
+  --next-error-btn-border: none;
+  --next-error-btn-secondary-text: #171717;
+  --next-error-btn-secondary-bg: transparent;
+  --next-error-btn-secondary-border: 1px solid rgba(0,0,0,0.08);
+}
+@media (prefers-color-scheme: dark) {
+  :root {
+    --next-error-bg: #0a0a0a;
+    --next-error-text: #ededed;
+    --next-error-title: #ededed;
+    --next-error-message: #ededed;
+    --next-error-digest: #a0a0a0;
+    --next-error-btn-text: #0a0a0a;
+    --next-error-btn-bg: #ededed;
+    --next-error-btn-border: none;
+    --next-error-btn-secondary-text: #ededed;
+    --next-error-btn-secondary-bg: transparent;
+    --next-error-btn-secondary-border: 1px solid rgba(255,255,255,0.14);
+  }
+}
+body { margin: 0; color: var(--next-error-text); background: var(--next-error-bg); }
+`.replace(/\n\s*/g,"");function d(){return(0,i.jsx)("svg",{width:"32",height:"32",viewBox:"-0.2 -1.5 32 32",fill:"none",style:l.icon,children:(0,i.jsx)("path",{d:"M16.9328 0C18.0839 0.000116771 19.1334 0.658832 19.634 1.69531L31.4299 26.1309C32.0708 27.4588 31.1036 28.9999 29.6291 29H2.00215C0.527541 29 -0.439628 27.4588 0.201371 26.1309L11.9973 1.69531C12.4979 0.658823 13.5474 7.75066e-05 14.6984 0H16.9328ZM3.59493 26H28.0363L16.9328 3H14.6984L3.59493 26ZM15.8156 19C16.9202 19.0001 17.8156 19.8955 17.8156 21C17.8156 22.1045 16.9202 22.9999 15.8156 23C14.7111 23 13.8156 22.1046 13.8156 21C13.8156 19.8954 14.7111 19 15.8156 19ZM17.3156 16.5H14.3156V8.5H17.3156V16.5Z",fill:"var(--next-error-title)"})})}("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),r.exports=t.default)},168027,(e,r,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0}),Object.defineProperty(t,"default",{enumerable:!0,get:function(){return l}}),e.r(555682);let n=e.r(843476);e.r(271645);let o=e.r(912354),i=e.r(18576),l=function({error:e}){let r=e?.digest,t=!!r;return(0,o.handleISRError)({error:e}),(0,n.jsxs)("html",{id:"__next_error__",children:[(0,n.jsx)("head",{children:(0,n.jsx)("style",{dangerouslySetInnerHTML:{__html:i.errorThemeCss}})}),(0,n.jsxs)("body",{children:[(0,n.jsx)("div",{style:i.errorStyles.container,children:(0,n.jsxs)("div",{style:i.errorStyles.card,children:[(0,n.jsx)(i.WarningIcon,{}),(0,n.jsx)("h1",{style:i.errorStyles.title,children:"This page couldn’t load"}),(0,n.jsx)("p",{style:i.errorStyles.message,children:t?"A server error occurred. Reload to try again.":"Reload to try again, or go back."}),(0,n.jsxs)("div",{style:i.errorStyles.buttonGroup,children:[(0,n.jsx)("form",{style:i.errorStyles.form,children:(0,n.jsx)("button",{type:"submit",style:i.errorStyles.button,children:"Reload"})}),!t&&(0,n.jsx)("button",{type:"button",style:i.errorStyles.buttonSecondary,onClick:()=>{window.history.length>1?window.history.back():window.location.href="/"},children:"Back"})]})]})}),r&&(0,n.jsxs)("p",{style:i.errorStyles.digestFooter,children:["ERROR ",r]})]})]})};("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),r.exports=t.default)}]);