Merge remote-tracking branch 'origin/main' into fix-session-limits

.circleci/config.yml

@@ -141,7 +141,7 @@ commands:
             node --version
             npm --version
   install_rust:
-    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
+    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself. Also restores the dev-profile cargo cache that save_cargo_target writes on main, minus the workspace crates' fingerprints so those always rebuild from the checked-out source."
     steps:
       - run:
           name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
@@ -167,9 +167,29 @@ commands:
             /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
             rm -f /tmp/rustup-init
             echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
+            echo 'export CARGO_INCREMENTAL=0' >> "$BASH_ENV"
             export PATH="$HOME/.cargo/bin:$PATH"
             rustc --version
             cargo --version
+            { rustc -vV; cc --version; cat /etc/os-release; } > /tmp/cargo-build-env
+      - restore_cache:
+          keys:
+            - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+            - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-
+      - run:
+          name: Force a rebuild of the workspace crates restored from the cargo cache
+          command: rm -rf litellm-rust/target/debug/.fingerprint/litellm-*
+  save_cargo_target:
+    steps:
+      - when:
+          condition:
+            equal: [main, << pipeline.git.branch >>]
+          steps:
+            - save_cache:
+                key: v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+                paths:
+                  - ~/.cargo/registry
+                  - ~/project/litellm-rust/target/debug
   start_postgres:
     description: "Start a postgres-db container on port 5432 and wait until it accepts connections."
     parameters:
@@ -281,51 +301,11 @@ commands:
             # `uv sync --package litellm-enterprise` here — that overwrites the
             # shared .venv and strips out dev/test deps (pytest, prisma, etc.).
             uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
-  setup_litellm_test_deps:
+  install_windows_toolchain:
     steps:
-      - checkout
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - restore_cache:
-          keys:
-            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
       - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - save_cache:
-          paths:
-            - ~/.cache/uv
-          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
-
-jobs:
-  # Add Windows testing job
-  using_litellm_on_windows:
-    executor:
-      name: win/default
-      shell: powershell.exe
-    working_directory: ~/project
-    environment:
-      UV_PYTHON: "3.11"
-      CARGO_HTTP_MULTIPLEXING: "false"
-      CARGO_NET_RETRY: "5"
-    steps:
-      - checkout
-      - run:
-          name: Install Python
-          command: |
-            choco install python --version=3.11.0 -y --no-progress --force
-            refreshenv
-            python --version
-          environment:
-            CHOCOLATEY_CONFIRM_ALL: "true"
-      - run:
-          name: Install Dependencies
+          name: Install Rust and uv
           no_output_timeout: 30m
-          environment:
-            UV_HTTP_TIMEOUT: "300"
           command: |
             $rustupInit = Join-Path $env:TEMP "rustup-init.exe"
             $rustupVersion = "1.28.2"
@@ -365,6 +345,55 @@ jobs:
             if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) {
               Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
             }
+  setup_litellm_test_deps:
+    steps:
+      - checkout
+      - setup_google_dns
+      - install_uv
+      - install_rust
+      - restore_cache:
+          keys:
+            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            uv sync --frozen --all-groups --all-extras --python 3.12
+      - setup_litellm_enterprise_pip
+      - save_cache:
+          paths:
+            - ~/.cache/uv
+          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
+      - save_cargo_target
+
+jobs:
+  # Add Windows testing job
+  using_litellm_on_windows:
+    executor:
+      name: win/default
+      shell: powershell.exe
+    working_directory: ~/project
+    environment:
+      UV_PYTHON: "3.11"
+      CARGO_HTTP_MULTIPLEXING: "false"
+      CARGO_NET_RETRY: "5"
+    steps:
+      - checkout
+      - run:
+          name: Install Python
+          command: |
+            choco install python --version=3.11.0 -y --no-progress --force
+            refreshenv
+            python --version
+          environment:
+            CHOCOLATEY_CONFIRM_ALL: "true"
+      - install_windows_toolchain
+      - run:
+          name: Install Dependencies
+          no_output_timeout: 30m
+          environment:
+            UV_HTTP_TIMEOUT: "300"
+          command: |
+            $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
             for ($attempt = 1; $attempt -le 5; $attempt++) {
               Write-Host "uv sync attempt $attempt/5"
               uv sync --frozen --group dev --python 3.11
@@ -380,17 +409,68 @@ jobs:
           name: Run Windows-specific test
           command: |
             uv run --no-sync python -m pytest tests/windows_tests/ -v
+
+  windows_release_wheel:
+    executor:
+      name: win/default
+      shell: powershell.exe
+      size: xlarge
+    working_directory: ~/project
+    environment:
+      UV_PYTHON: "3.11"
+      CARGO_HTTP_MULTIPLEXING: "false"
+      CARGO_NET_RETRY: "5"
+    steps:
+      - checkout
       - run:
-          name: Guard against MAX_PATH-busting packaged wheel paths
+          name: Skip job when no windows-release-relevant files changed
+          shell: bash.exe
+          command: bash .circleci/scripts/path_filter.sh windows-release
+      - run:
+          name: Install Python
+          command: |
+            choco install python --version=3.11.0 -y --no-progress --force
+            refreshenv
+            python --version
+          environment:
+            CHOCOLATEY_CONFIRM_ALL: "true"
+      - install_windows_toolchain
+      - run:
+          name: Record the Rust build environment for the release cargo cache key
+          command: |
+            & "$HOME\.cargo\bin\rustc.exe" -vV | Out-File -Encoding ascii .cargo-build-env
+      - restore_cache:
+          keys:
+            - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+            - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-
+      - run:
+          name: Force a rebuild of the workspace crates restored from the cargo cache
+          command: |
+            $fingerprints = "litellm-rust/target/release/.fingerprint"
+            if (Test-Path $fingerprints) {
+              Get-ChildItem -Path $fingerprints -Filter "litellm-*" | Remove-Item -Recurse -Force
+            }
+      - run:
+          name: Build the release wheel and install it under a worst-case MAX_PATH prefix
           no_output_timeout: 30m
           environment:
             UV_HTTP_TIMEOUT: "300"
           command: |
             $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
-            cargo --version
-            Get-ChildItem -Path "litellm\rust_bridge" -Filter "_native*" -File -ErrorAction SilentlyContinue | Remove-Item -Force
             uv build --wheel --out-dir dist
-            uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py
+            if ($LASTEXITCODE -ne 0) {
+              exit $LASTEXITCODE
+            }
+            python tests/windows_tests/check_windows_wheel_install.py
+      - when:
+          condition:
+            equal: [main, << pipeline.git.branch >>]
+          steps:
+            - save_cache:
+                key: v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
+                paths:
+                  - ~/.cargo/registry
+                  - ~/project/litellm-rust/target/release
 
   base_sdk_install:
     docker:
@@ -418,6 +498,10 @@ jobs:
             uv venv /tmp/base-sdk --python 3.12
             VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl
             /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py
+      - run:
+          name: Guard against MAX_PATH-busting packaged wheel paths
+          command: |
+            python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only
 
   local_testing_part1:
     docker:
@@ -446,6 +530,7 @@ jobs:
           paths:
             - ~/.cache/uv
           key: v1-uv-cache-{{ checksum "uv.lock" }}
+      - save_cargo_target
       - run:
           name: Run prisma ./docker/entrypoint.sh
           command: |
@@ -3120,10 +3205,14 @@ jobs:
         type: enum
         enum: [standard, replica]
         default: standard
+      parallelism:
+        type: integer
+        default: 1
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
+    parallelism: << parameters.parallelism >>
     steps:
       - setup_litellm_test_deps
       - when:
@@ -3249,6 +3338,7 @@ jobs:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
+    parallelism: 4
     steps:
       - setup_litellm_test_deps
       - run:
@@ -3258,10 +3348,11 @@ jobs:
           name: Run unit tests
           command: |
             mkdir -p test-results/unit
-            mapfile -t files < <(find tests/unit -name 'test_*.py' | sort)
-            if [ "${#files[@]}" -eq 0 ]; then echo "tests/unit holds no test_*.py files; nothing to run"; exit 0; fi
+            shard="$(find tests/unit -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)"
+            if [ -z "${shard}" ]; then echo "shard ${CIRCLE_NODE_INDEX} received no tests/unit files; nothing to run"; exit 0; fi
+            mapfile -t files < <(printf '%s\n' "${shard}")
             set +e
-            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --junitxml=test-results/unit/junit.xml
+            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short -o junit_family=xunit1 --junitxml=test-results/unit/junit.xml
             status=$?
             set -e
             if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
@@ -3328,7 +3419,11 @@ workflows:
           name: integration-<< matrix.suite >>
           matrix:
             parameters:
-              suite: [management, accounting, database, providers, extensions, mcp, sdk, cost, browser]
+              suite: [management, accounting, database, providers, mcp, sdk, cost, browser]
+      - integration_contracts:
+          name: integration-extensions
+          suite: extensions
+          parallelism: 4
       - integration_contracts:
           name: integration-<< matrix.suite >>-replica
           matrix:
@@ -3343,6 +3438,7 @@ workflows:
             equal: ["", << pipeline.parameters.routing_parity_base >>]
     jobs:
       - using_litellm_on_windows
+      - windows_release_wheel
       - unit
       - provider_replay_harness
       - base_sdk_install

.circleci/scripts/classify_changes.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies>}"
+category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release>}"
 
 has_client=false
 has_backend=false
@@ -9,6 +9,7 @@ has_ci=false
 has_provider_harness=false
 has_cost_map=false
 has_mcp_dependencies=false
+has_windows_release=false
 outside_cost_map_set=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
@@ -22,6 +23,10 @@ while IFS= read -r file || [ -n "$file" ]; do
     tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
       has_provider_harness=true ;;
   esac
+  case "$file" in
+    litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*)
+      has_windows_release=true ;;
+  esac
   case "$file" in
     ui/* | tests/e2e/ui/*) has_client=true ;;
     docs/* | *.md | *.mdx) : ;;
@@ -46,6 +51,9 @@ case "$category" in
   provider-harness)
     [ "$has_provider_harness" = true ] && echo run || echo skip
     ;;
+  windows-release)
+    [ "$has_windows_release" = true ] && echo run || echo skip
+    ;;
   backend)
     [ "$has_backend" = true ] && echo run || echo skip
     ;;

.circleci/scripts/run_integration.sh

@@ -212,6 +212,15 @@ if [ "$suite" = browser ]; then
   exit 0
 fi
 
+node_files=()
+if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then
+  split="$(.venv/bin/python tests/integration/run.py "$suite" --list \
+    | circleci tests split --split-by=timings --timings-type=filename)"
+  read -r -a node_files <<< "$(printf '%s' "$split" | tr '\n' ' ')"
+  test "${#node_files[@]}" -gt 0
+  printf '%s\n' "${node_files[@]}" > "$results/node-files.txt"
+fi
+
 env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   INTEGRATION_RUN_ID="$integration_identity" \
   DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
@@ -225,7 +234,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \
   INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \
   INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
-  .venv/bin/python tests/integration/run.py "$suite" --results "$results"
+  .venv/bin/python tests/integration/run.py "$suite" --results "$results" "${node_files[@]}"
 
 if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
   for covered_pid in "$proxy_pid" "$peer_pid"; do

.circleci/scripts/unit_selection.sh

@@ -46,6 +46,7 @@ legacy_paths() {
       echo tests/unit/google_genai
       echo tests/unit/router_strategy
       echo tests/unit/router_utils
+      echo tests/unit/proxy/common_utils/test_cache_aware_routing.py
       echo tests/unit/enterprise/enterprise_callbacks/send_emails
       echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
       echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py

.github/pull_request_template.md

@@ -54,7 +54,7 @@ After: the same request comes back with real token counts, so the dashboard show
 
 ## Affected release
 
-<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1", and add the `backport-stable` label so the fix is cherry-picked onto the rc line before the stable is tagged. Drop the section otherwise -->
+<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1". Add the `backport-stable` label only when the regression is a P0, meaning its Linear ticket is Urgent (a security hole however narrow, data loss, or a crash or outage for every user on that version), because every labeled PR must be cherry-picked onto the baking rc line before the stable can be tagged; every other regression fix ships in the next rc unlabeled. Drop the section otherwise -->
 
 ## Linear ticket
 

.github/scripts/assert_ci_coverage.py

@@ -516,7 +516,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
         str(path.relative_to(repo_root))
         for folders in groups.values()
         for folder in folders
-        for path in (integration_root / folder).glob("test_*.py")
+        for path in (integration_root / folder).rglob("test_*.py")
     )
     browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json"
     browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else ()

.github/workflows/test-code-quality.yml

@@ -146,6 +146,9 @@ jobs:
       - name: check_migrations_no_data_rewrites
         run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
 
+      - name: check_unbounded_in_lists (fails on findings not in the baseline)
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

ARCHITECTURE.md

@@ -336,7 +336,7 @@ Each translation is isolated in its own file, making it easy to test and modify
 | `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
 | `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
 | `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
-| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
+| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/pass_through/messages/transformation.py` |
 | `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
 | `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
 | Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |

README.md

@@ -362,6 +362,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
 | [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) |  |  |  |  | ✅ |  |  |  |  |  |
 | [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

cookbook/litellm_proxy_server/mcp/README.md

@@ -0,0 +1,37 @@
+# Publish MCP servers in the AI Hub
+
+Set `litellm_settings.public_mcp_servers` to the concrete IDs of the servers you want listed in the public AI Hub. Pin `server_id` in each configuration entry so the publication list stays stable across deployments
+
+```yaml
+mcp_servers:
+  documentation:
+    server_id: documentation-mcp
+    url: https://mcp.example.com/mcp
+    transport: http
+    available_on_public_internet: true
+
+litellm_settings:
+  public_mcp_hub_strict_whitelist: true
+  public_mcp_servers:
+    - documentation-mcp
+```
+
+Use `documentation-mcp`, the `server_id`, in the publication list. The configuration key `documentation`, display names, and aliases are not publication IDs. Database-created servers use the ID returned by `/v1/mcp/server`
+
+The dashboard's **AI Hub > MCP Hub > Manage MCP Hub Visibility** dialog edits this same list. Its YAML example includes the selected server IDs. With database-backed configuration (`store_model_in_db: true`), a value declared in YAML is owned by that file: edit the file and reload, or remove that key from YAML to let the dashboard manage it in the database. File-backed deployments can save the list directly to their configuration file
+
+To remove all explicit entries, save an empty selection in the dialog or configure:
+
+```yaml
+litellm_settings:
+  public_mcp_hub_strict_whitelist: true
+  public_mcp_servers: []
+```
+
+## Hub listing and network access
+
+The **Hub listing** column in AI Hub identifies servers that appear in `/public/mcp_hub`. The dashboard derives this status from the current registry and publication settings. Setting `mcp_info.is_public` on a server does not publish it; that response field is derived metadata. `mcp_info.is_public_explicit` identifies registered servers included in the explicit publication list
+
+Gateway cards and server details show **All Networks** when `available_on_public_internet` is enabled or the server is explicitly published in `public_mcp_servers`. They show **Internal Only** when both are false. The per-server flag defaults to `true`; explicit publication overrides a disabled flag for compatibility. Older proxies that omit the metadata needed to determine access show **Unknown**. These labels describe allowed client IPs; authentication and tool permissions still apply
+
+The default `public_mcp_hub_strict_whitelist: true` lists only registered servers in `public_mcp_servers`. Legacy mode (`false`) additionally lists registered servers with `available_on_public_internet: true`. In legacy mode, clearing the explicit publication list leaves these automatically listed servers visible. Enable strict mode when the publication list should fully determine hub visibility

litellm-rust/crates/core/src/chat_completions/common_utils.rs

@@ -3,6 +3,7 @@ use litellm_llms::{
     anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG,
     base_llm::chat::transformation::BaseConfig,
     bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
+    openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG,
 };
 use serde_json::{Map, Value};
 
@@ -14,6 +15,7 @@ pub(super) fn chat_completions_provider_config(provider: &str) -> Option<&'stati
     match provider {
         "anthropic" => Some(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG),
         "bedrock" => Some(&BEDROCK_CHAT_COMPLETIONS_CONFIG),
+        "openai_like" => Some(&OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG),
         _ => None,
     }
 }

litellm-rust/crates/llms/src/lib.rs

@@ -7,6 +7,7 @@ pub mod cohere;
 mod error;
 pub mod mistral;
 pub mod openai;
+pub mod openai_like;
 pub mod reducto;
 pub mod vertex_ai;
 

litellm-rust/crates/llms/src/openai_like/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/openai_like/chat/transformation.rs

@@ -0,0 +1,270 @@
+//! `litellm/llms/openai_like/chat/transformation.py`: the chat config every
+//! OpenAI-compatible endpoint shares. The body is already OpenAI-shaped, so
+//! parameters pass through verbatim; the port keeps Python's two deviations,
+//! the `max_completion_tokens` -> `max_tokens` rename and the usage
+//! `*_tokens` null-to-zero sanitize.
+
+use litellm_auth::{CredentialPlacement, SecretValue};
+use litellm_core_utils::core_helpers::unix_now;
+use litellm_types::{
+    llms::openai::ChatMessage,
+    utils::{ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse},
+};
+use serde_json::{Map, Value, json};
+
+use crate::{
+    Error,
+    base_llm::{
+        auth::AuthScheme,
+        chat::transformation::{
+            BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData,
+            ValidatedEnvironment,
+        },
+    },
+    openai_like::common_utils::{complete_openai_like_url, openai_compatible_provider_info},
+};
+
+/// OpenAI parameter names the Rust path can place verbatim in the request body.
+/// Tool parameters are absent on purpose: the message gate already declines
+/// tool-call content, and a `tools` request that did get through would produce
+/// a tool-call response this port cannot normalize yet, so it declines before
+/// the call instead of after it.
+const SUPPORTED_PARAMS: &[(&str, &str)] = &[
+    ("frequency_penalty", "frequency_penalty"),
+    ("logit_bias", "logit_bias"),
+    ("logprobs", "logprobs"),
+    ("top_logprobs", "top_logprobs"),
+    ("max_tokens", "max_tokens"),
+    ("max_completion_tokens", "max_completion_tokens"),
+    ("modalities", "modalities"),
+    ("prediction", "prediction"),
+    ("n", "n"),
+    ("presence_penalty", "presence_penalty"),
+    ("seed", "seed"),
+    ("stop", "stop"),
+    ("stream_options", "stream_options"),
+    ("temperature", "temperature"),
+    ("top_p", "top_p"),
+    ("audio", "audio"),
+    ("web_search_options", "web_search_options"),
+    ("service_tier", "service_tier"),
+    ("safety_identifier", "safety_identifier"),
+    ("prompt_cache_key", "prompt_cache_key"),
+    ("prompt_cache_retention", "prompt_cache_retention"),
+    ("store", "store"),
+    ("response_format", "response_format"),
+];
+
+/// Call configuration the caller may pass that never enters the request body.
+const CONFIG_PARAMS: &[&str] = &["custom_endpoint", "extra_headers", "max_retries"];
+
+pub struct OpenAILikeChatConfig;
+
+pub const OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG: OpenAILikeChatConfig = OpenAILikeChatConfig;
+
+impl BaseConfig for OpenAILikeChatConfig {
+    fn supported_openai_param_mappings(&self) -> &'static [(&'static str, &'static str)] {
+        SUPPORTED_PARAMS
+    }
+
+    fn get_complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        optional_params: &Map<String, Value>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        let custom_endpoint = optional_params
+            .get("custom_endpoint")
+            .and_then(Value::as_bool)
+            .unwrap_or(false);
+        complete_openai_like_url(api_base, custom_endpoint, env_lookup)
+    }
+
+    fn transform_request(
+        &self,
+        model: &str,
+        messages: Vec<ChatMessage>,
+        optional_params: Map<String, Value>,
+    ) -> Result<ProviderChatRequestData, Error> {
+        let mut params = Map::from_iter(
+            optional_params
+                .into_iter()
+                .filter(|(key, _)| !CONFIG_PARAMS.contains(&key.as_str())),
+        );
+        // Most OpenAI-compatible endpoints take `max_tokens`, not
+        // `max_completion_tokens`, so Python's `map_openai_params` renames it
+        // and lets it overwrite a `max_tokens` the caller also sent.
+        if let Some(limit) = params.remove("max_completion_tokens") {
+            params.insert("max_tokens".to_string(), limit);
+        }
+        let body = Map::from_iter(
+            [
+                ("model".to_string(), json!(model)),
+                ("messages".to_string(), json!(messages)),
+            ]
+            .into_iter()
+            .chain(params),
+        );
+        Ok(ProviderChatRequestData {
+            body: Value::Object(body),
+            stream_shape: Default::default(),
+        })
+    }
+
+    fn transform_response(
+        &self,
+        model: &str,
+        response: ProviderChatResponseData,
+    ) -> Result<ChatCompletionsResponse, Error> {
+        let mut body = response.body;
+        sanitize_usage(&mut body);
+        let body = body
+            .as_object()
+            .ok_or_else(|| Error::InvalidResponse("chat response is not an object".into()))?;
+
+        let choices = body
+            .get("choices")
+            .and_then(Value::as_array)
+            .ok_or(Error::MissingField("choices"))?
+            .iter()
+            .enumerate()
+            .map(|(position, choice)| normalize_choice(position, choice))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        let usage = body.get("usage").and_then(Value::as_object);
+        let field = |name: &str| {
+            usage
+                .and_then(|usage| usage.get(name))
+                .and_then(Value::as_u64)
+                .unwrap_or(0)
+        };
+        let details = usage.and_then(|usage| usage.get("prompt_tokens_details"));
+
+        Ok(ChatCompletionsResponse {
+            created: body
+                .get("created")
+                .and_then(Value::as_u64)
+                .unwrap_or_else(unix_now),
+            model: body
+                .get("model")
+                .and_then(Value::as_str)
+                .unwrap_or(model)
+                .to_string(),
+            choices,
+            usage: litellm_types::utils::ChatCompletionsUsage {
+                prompt_tokens: field("prompt_tokens"),
+                completion_tokens: field("completion_tokens"),
+                total_tokens: field("total_tokens"),
+                prompt_tokens_details: litellm_types::utils::PromptTokensDetails {
+                    cached_tokens: details
+                        .and_then(|d| d.get("cached_tokens"))
+                        .and_then(Value::as_u64)
+                        .unwrap_or(0),
+                    cache_creation_tokens: details
+                        .and_then(|d| d.get("cache_creation_tokens"))
+                        .and_then(Value::as_u64)
+                        .unwrap_or(0),
+                    text_tokens: details
+                        .and_then(|d| d.get("text_tokens"))
+                        .and_then(Value::as_u64)
+                        .unwrap_or(0),
+                },
+            },
+        })
+    }
+
+    /// `OpenAILikeBase._validate_environment`: a forwarded `authorization` is
+    /// the whole credential, and any other call authenticates with the
+    /// resolved key as a bearer. The key resolves to `""` when neither the
+    /// deployment nor `OPENAI_LIKE_API_KEY` sets one, because vllm-compatible
+    /// endpoints take no key; Python still sends `Bearer ` in that case.
+    fn validate_environment(
+        &self,
+        headers: Headers,
+        api_key: Option<&str>,
+        _model: &str,
+        _optional_params: &Map<String, Value>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<ValidatedEnvironment, Error> {
+        if headers
+            .iter()
+            .any(|(name, _)| name.eq_ignore_ascii_case("authorization"))
+        {
+            return Ok(ValidatedEnvironment {
+                headers,
+                auth: AuthScheme::Forwarded,
+            });
+        }
+        let (_, key) = openai_compatible_provider_info(None, api_key, env_lookup);
+        Ok(ValidatedEnvironment {
+            headers,
+            auth: AuthScheme::Credential {
+                placement: CredentialPlacement::Bearer,
+                secret: SecretValue::new(key.unwrap_or_default()),
+            },
+        })
+    }
+
+    fn config_params(&self) -> &'static [&'static str] {
+        CONFIG_PARAMS
+    }
+}
+
+/// `OpenAILikeChatConfig._sanitize_usage_obj`: a provider that reports a null
+/// `*_tokens` entry breaks OpenAI clients, so nulls become 0. Python scrubs
+/// every top-level usage key ending in `_tokens`.
+fn sanitize_usage(body: &mut Value) {
+    if let Some(usage) = body.get_mut("usage").and_then(Value::as_object_mut) {
+        for (key, value) in usage.iter_mut() {
+            if key.ends_with("_tokens") && value.is_null() {
+                *value = json!(0);
+            }
+        }
+    }
+}
+
+fn normalize_choice(position: usize, choice: &Value) -> Result<ChatCompletionsChoice, Error> {
+    let message = choice
+        .get("message")
+        .and_then(Value::as_object)
+        .ok_or(Error::MissingField("message"))?;
+    if message
+        .get("tool_calls")
+        .and_then(Value::as_array)
+        .is_some_and(|calls| !calls.is_empty())
+    {
+        // Python rewrites the lone tool call into content only under
+        // `json_mode`, a request flag `transform_response` cannot see, and the
+        // normalized type cannot carry tool calls at all. Declining is
+        // terminal at this point, but passing back an empty assistant turn
+        // would fabricate the reply.
+        return Err(Error::Unsupported("tool call response"));
+    }
+    if message.get("refusal").is_some_and(|value| !value.is_null()) {
+        return Err(Error::Unsupported("refusal response"));
+    }
+    let content = message.get("content");
+    if content.is_some_and(|value| !value.is_null() && !value.is_string()) {
+        return Err(Error::Unsupported("non-text response content"));
+    }
+    Ok(ChatCompletionsChoice {
+        index: choice
+            .get("index")
+            .and_then(Value::as_u64)
+            .unwrap_or(position as u64),
+        message: ChatCompletionsChoiceMessage {
+            role: message
+                .get("role")
+                .and_then(Value::as_str)
+                .unwrap_or("assistant")
+                .to_string(),
+            content: content.and_then(Value::as_str).map(str::to_string),
+        },
+        finish_reason: choice
+            .get("finish_reason")
+            .and_then(Value::as_str)
+            .unwrap_or("")
+            .to_string(),
+    })
+}

litellm-rust/crates/llms/src/openai_like/common_utils.rs

@@ -0,0 +1,58 @@
+//! Shared OpenAI-like credential and endpoint resolution, mirroring
+//! `litellm/llms/openai_like/common_utils.py`.
+
+use crate::Error;
+
+/// `OpenAILikeChatConfig._get_openai_compatible_provider_info`: the deployment's
+/// `api_base` wins over `OPENAI_LIKE_API_BASE`, and the deployment key over
+/// `OPENAI_LIKE_API_KEY`, with an empty key allowed because vllm-compatible
+/// endpoints do not require one.
+pub fn openai_compatible_provider_info(
+    api_base: Option<&str>,
+    api_key: Option<&str>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> (Option<String>, Option<String>) {
+    let api_base = api_base
+        .map(str::to_string)
+        .or_else(|| env_lookup("OPENAI_LIKE_API_BASE"));
+    let api_key = api_key
+        .map(str::to_string)
+        .or_else(|| env_lookup("OPENAI_LIKE_API_KEY"))
+        .or(Some(String::new()));
+    (api_base, api_key)
+}
+
+/// `OpenAILikeBase._validate_environment` requires an api base and, when the
+/// caller gave no `custom_endpoint`, appends the route suffix. A caller-supplied
+/// `custom_endpoint` base is used as is.
+pub fn complete_openai_like_url(
+    api_base: Option<&str>,
+    custom_endpoint: bool,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> Result<String, Error> {
+    let (api_base, _) = openai_compatible_provider_info(api_base, None, env_lookup);
+    let api_base = api_base.ok_or_else(|| {
+        Error::InvalidRequest(
+            "Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({LLM_PROVIDER}_API_KEY) or via params"
+                .to_string(),
+        )
+    })?;
+    if custom_endpoint {
+        return Ok(api_base);
+    }
+    Ok(format!(
+        "{}/chat/completions",
+        api_base.trim_end_matches('/')
+    ))
+}
+
+/// The api key the call resolves to. `None` means neither the deployment nor the
+/// environment supplied one, which is valid for endpoints that take no key.
+pub fn resolve_openai_like_api_key(
+    api_key: Option<&str>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> Option<String> {
+    openai_compatible_provider_info(None, api_key, env_lookup)
+        .1
+        .filter(|key| !key.is_empty())
+}

litellm-rust/crates/llms/src/openai_like/mod.rs

@@ -0,0 +1,2 @@
+pub mod chat;
+pub mod common_utils;

litellm-rust/crates/llms/tests/openai_like_chat_transformation.rs

@@ -0,0 +1,343 @@
+use litellm_llms::{
+    Error,
+    base_llm::{
+        auth::AuthScheme,
+        chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported},
+    },
+    openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG,
+};
+use litellm_types::{llms::openai::ChatMessage, utils::ChatCompletionsResponse};
+use rstest::rstest;
+use serde_json::{Map, Value, json};
+
+fn messages(value: Value) -> Vec<ChatMessage> {
+    serde_json::from_value(value).expect("valid messages")
+}
+
+fn params(value: Value) -> Map<String, Value> {
+    match value {
+        Value::Object(map) => map,
+        other => panic!("params must be an object, got {other}"),
+    }
+}
+
+fn no_env(_: &str) -> Option<String> {
+    None
+}
+
+fn env_with<'a>(name: &'a str, value: &'a str) -> impl Fn(&str) -> Option<String> + 'a {
+    move |key| (key == name).then(|| value.to_string())
+}
+
+fn transform(model: &str, msgs: Value, opts: Value) -> Value {
+    OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG
+        .transform_request(model, messages(msgs), params(opts))
+        .expect("request transforms")
+        .body
+}
+
+fn transform_response(body: Value) -> Result<ChatCompletionsResponse, Error> {
+    OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG
+        .transform_response("some-model", ProviderChatResponseData { body })
+}
+
+fn reason(msgs: Value, opts: Value) -> Option<Unsupported> {
+    OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), &params(opts))
+}
+
+#[rstest]
+fn builds_the_openai_shaped_body() {
+    let body = transform(
+        "my-model",
+        json!([
+            {"role": "system", "content": "be terse"},
+            {"role": "user", "content": "hi"},
+        ]),
+        json!({"temperature": 0.5, "max_tokens": 8}),
+    );
+    assert_eq!(body["model"], json!("my-model"));
+    assert_eq!(
+        body["messages"],
+        json!([
+            {"role": "system", "content": "be terse"},
+            {"role": "user", "content": "hi"},
+        ])
+    );
+    assert_eq!(body["temperature"], json!(0.5));
+    assert_eq!(body["max_tokens"], json!(8));
+}
+
+#[rstest]
+fn renames_max_completion_tokens_to_max_tokens() {
+    // `OpenAILikeChatConfig.map_openai_params`: most OpenAI-compatible providers
+    // support `max_tokens`, not `max_completion_tokens`.
+    let body = transform(
+        "my-model",
+        json!([{"role": "user", "content": "hi"}]),
+        json!({"max_completion_tokens": 12}),
+    );
+    assert_eq!(body["max_tokens"], json!(12));
+    assert!(body.get("max_completion_tokens").is_none());
+}
+
+#[rstest]
+fn max_completion_tokens_wins_when_both_limits_are_sent() {
+    // Python assigns `max_tokens = max_completion_tokens` after copying the
+    // params, so the renamed value outranks a caller-supplied `max_tokens`.
+    let body = transform(
+        "my-model",
+        json!([{"role": "user", "content": "hi"}]),
+        json!({"max_tokens": 8, "max_completion_tokens": 12}),
+    );
+    assert_eq!(body["max_tokens"], json!(12));
+    assert!(body.get("max_completion_tokens").is_none());
+}
+
+#[rstest]
+fn call_configuration_never_enters_the_body() {
+    let body = transform(
+        "my-model",
+        json!([{"role": "user", "content": "hi"}]),
+        json!({"custom_endpoint": true, "extra_headers": {"x": "y"}, "max_retries": 2}),
+    );
+    assert_eq!(
+        body.as_object().unwrap().keys().collect::<Vec<_>>(),
+        vec!["model", "messages"]
+    );
+}
+
+#[rstest]
+#[case::appends_the_chat_completions_suffix("https://vllm.example.com/v1", json!({}), "https://vllm.example.com/v1/chat/completions")]
+#[case::trims_a_trailing_slash("https://vllm.example.com/v1/", json!({}), "https://vllm.example.com/v1/chat/completions")]
+#[case::a_custom_endpoint_is_used_as_is("https://vllm.example.com/v1/chat/completions", json!({"custom_endpoint": true}), "https://vllm.example.com/v1/chat/completions")]
+fn complete_url(#[case] api_base: &str, #[case] opts: Value, #[case] expected: &str) {
+    assert_eq!(
+        OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG
+            .get_complete_url(Some(api_base), "my-model", &params(opts), &no_env)
+            .expect("url resolves"),
+        expected
+    );
+}
+
+#[rstest]
+fn api_base_falls_back_to_the_environment() {
+    assert_eq!(
+        OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG
+            .get_complete_url(
+                None,
+                "my-model",
+                &params(json!({})),
+                &env_with("OPENAI_LIKE_API_BASE", "https://env.example.com/v1"),
+            )
+            .expect("url resolves"),
+        "https://env.example.com/v1/chat/completions"
+    );
+}
+
+#[rstest]
+fn a_missing_api_base_is_an_error() {
+    assert!(matches!(
+        OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG
+            .get_complete_url(None, "my-model", &params(json!({})), &no_env),
+        Err(Error::InvalidRequest(message)) if message.starts_with("Missing API Base")
+    ));
+}
+
+#[rstest]
+fn the_resolved_key_authenticates_as_a_bearer() {
+    let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG
+        .validate_environment(
+            vec![],
+            Some("sk-test"),
+            "my-model",
+            &params(json!({})),
+            &no_env,
+        )
+        .expect("validates");
+    assert!(matches!(
+        validated.auth,
+        AuthScheme::Credential {
+            placement: litellm_auth::CredentialPlacement::Bearer,
+            ..
+        }
+    ));
+}
+
+#[rstest]
+fn the_key_falls_back_to_the_environment() {
+    let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG
+        .validate_environment(
+            vec![],
+            None,
+            "my-model",
+            &params(json!({})),
+            &env_with("OPENAI_LIKE_API_KEY", "sk-env"),
+        )
+        .expect("validates");
+    let AuthScheme::Credential { secret, .. } = validated.auth else {
+        panic!("expected a bearer credential");
+    };
+    assert_eq!(secret.expose(), "sk-env");
+}
+
+#[rstest]
+fn a_forwarded_authorization_is_the_whole_credential() {
+    // Python adds `Bearer <key>` only when the caller did not already send
+    // `Authorization`, so the forwarded header wins over the deployment key.
+    let headers = vec![("Authorization".to_string(), "Bearer caller".to_string())];
+    let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG
+        .validate_environment(
+            headers,
+            Some("sk-test"),
+            "my-model",
+            &params(json!({})),
+            &no_env,
+        )
+        .expect("validates");
+    assert!(matches!(validated.auth, AuthScheme::Forwarded));
+}
+
+#[rstest]
+fn keyless_calls_still_validate_for_endpoints_that_take_no_key() {
+    // vllm-compatible endpoints require no api key; Python resolves `""` and
+    // sends `Bearer `, so validation must not fail on the missing key.
+    let validated = OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG
+        .validate_environment(vec![], None, "my-model", &params(json!({})), &no_env)
+        .expect("validates");
+    let AuthScheme::Credential { secret, .. } = validated.auth else {
+        panic!("expected a bearer credential");
+    };
+    assert_eq!(secret.expose(), "");
+}
+
+#[rstest]
+fn normalizes_an_openai_response() {
+    let response = transform_response(json!({
+        "created": 1_700_000_000,
+        "model": "served-model-name",
+        "choices": [{
+            "index": 0,
+            "message": {"role": "assistant", "content": "hello"},
+            "finish_reason": "stop",
+        }],
+        "usage": {"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8},
+    }))
+    .expect("response normalizes");
+    assert_eq!(response.created, 1_700_000_000);
+    assert_eq!(response.model, "served-model-name");
+    assert_eq!(
+        response.choices[0].message.content.as_deref(),
+        Some("hello")
+    );
+    assert_eq!(response.choices[0].finish_reason, "stop");
+    assert_eq!(response.usage.prompt_tokens, 3);
+    assert_eq!(response.usage.completion_tokens, 5);
+    assert_eq!(response.usage.total_tokens, 8);
+}
+
+#[rstest]
+fn null_token_fields_in_usage_become_zero() {
+    // `_sanitize_usage_obj`: providers that return null token values break
+    // OpenAI clients, so the response is scrubbed at the source.
+    let response = transform_response(json!({
+        "model": "m",
+        "choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
+        "usage": {"prompt_tokens": 3, "completion_tokens": null, "total_tokens": null},
+    }))
+    .expect("response normalizes");
+    assert_eq!(response.usage.completion_tokens, 0);
+    assert_eq!(response.usage.total_tokens, 0);
+    assert_eq!(response.usage.prompt_tokens, 3);
+}
+
+#[rstest]
+fn a_tool_call_response_declines_instead_of_dropping_the_calls() {
+    // The `json_mode` rewrite needs a request flag the route does not carry, so
+    // a tool-call answer falls back to Python rather than losing the calls.
+    assert_eq!(
+        transform_response(json!({
+            "model": "m",
+            "choices": [{
+                "message": {
+                    "role": "assistant",
+                    "content": null,
+                    "tool_calls": [{
+                        "id": "call_1",
+                        "type": "function",
+                        "function": {"name": "f", "arguments": "{}"},
+                    }],
+                },
+                "finish_reason": "tool_calls",
+            }],
+        })),
+        Err(Error::Unsupported("tool call response"))
+    );
+}
+
+#[rstest]
+fn a_refusal_declines_instead_of_returning_an_empty_reply() {
+    assert_eq!(
+        transform_response(json!({
+            "model": "m",
+            "choices": [{
+                "message": {"role": "assistant", "content": null, "refusal": "cannot help"},
+                "finish_reason": "stop",
+            }],
+        })),
+        Err(Error::Unsupported("refusal response"))
+    );
+}
+
+#[rstest]
+fn a_non_text_response_content_declines() {
+    assert_eq!(
+        transform_response(json!({
+            "model": "m",
+            "choices": [{
+                "message": {"role": "assistant", "content": [{"type": "text", "text": "hi"}]},
+                "finish_reason": "stop",
+            }],
+        })),
+        Err(Error::Unsupported("non-text response content"))
+    );
+}
+
+#[rstest]
+#[case::streaming(json!({"stream": true}), "streaming")]
+#[case::unrecognized_param(json!({"some_provider_knob": 1}), "unrecognized request parameter")]
+fn declines(#[case] opts: Value, #[case] expected: &'static str) {
+    assert_eq!(
+        reason(json!([{"role": "user", "content": "hi"}]), opts),
+        Some(Unsupported(expected))
+    );
+}
+
+#[rstest]
+fn accepts_standard_openai_params() {
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": "hi"}]),
+            json!({
+                "temperature": 0.2,
+                "top_p": 0.9,
+                "max_tokens": 16,
+                "response_format": {"type": "json_object"},
+                "custom_endpoint": true,
+            }),
+        ),
+        None
+    );
+}
+
+#[rstest]
+fn tool_parameters_decline_before_the_call() {
+    // A `tools` request would come back with tool calls this port cannot
+    // normalize, so it declines at the gate instead of after the call.
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"tools": [{"type": "function", "function": {"name": "f"}}]}),
+        ),
+        Some(Unsupported("unrecognized request parameter"))
+    );
+}

litellm-rust/crates/model-catalog/src/model_info.rs

@@ -104,6 +104,9 @@ pub struct ModelInfo {
     /// Rate applied once the prompt exceeds the token threshold in the field name.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
+    /// Balanced service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_read_input_token_cost_balanced: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cache_read_input_token_cost_batches: Option<f64>,
     /// Flex service-tier rate for the same-named base field.
@@ -211,6 +214,9 @@ pub struct ModelInfo {
     /// Rate applied once the prompt exceeds the token threshold in the field name.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub input_cost_per_token_above_512k_tokens: Option<f64>,
+    /// Balanced service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub input_cost_per_token_balanced: Option<f64>,
     /// USD per prompt token via the provider's batch API.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub input_cost_per_token_batches: Option<f64>,
@@ -357,6 +363,9 @@ pub struct ModelInfo {
     /// Rate applied once the prompt exceeds the token threshold in the field name.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub output_cost_per_token_above_512k_tokens: Option<f64>,
+    /// Balanced service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub output_cost_per_token_balanced: Option<f64>,
     /// USD per generated token via the provider's batch API.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub output_cost_per_token_batches: Option<f64>,

litellm/__init__.py

@@ -172,6 +172,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
     "levo",
     "compression_interception",
     "newrelic",
+    "signoz",
 ]
 cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
 logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
@@ -1691,7 +1692,7 @@ if TYPE_CHECKING:
         SagemakerNovaConfig as SagemakerNovaConfig,
     )
     from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig
-    from .llms.anthropic.experimental_pass_through.messages.transformation import (
+    from .llms.anthropic.pass_through.messages.transformation import (
         AnthropicMessagesConfig as AnthropicMessagesConfig,
     )
     from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (

litellm/_lazy_imports_registry.py

@@ -742,7 +742,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     ),
     "CohereChatConfig": (".llms.cohere.chat.transformation", "CohereChatConfig"),
     "AnthropicMessagesConfig": (
-        ".llms.anthropic.experimental_pass_through.messages.transformation",
+        ".llms.anthropic.pass_through.messages.transformation",
         "AnthropicMessagesConfig",
     ),
     "BedrockClaudePlatformMessagesConfig": (

litellm/caching/caching.py

@@ -25,7 +25,7 @@ from litellm._logging import verbose_logger
 from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
 from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
 from litellm.types.caching import *
-from litellm.types.utils import EmbeddingResponse, all_litellm_params
+from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
 
 from .azure_blob_cache import AzureBlobCache
 from .base_cache import BaseCache
@@ -377,7 +377,6 @@ class Cache:
             return preset_cache_key
 
         combined_kwargs: Final = ModelParamHelper._get_all_llm_api_params()
-        litellm_param_kwargs: Final = all_litellm_params
         is_semantic_cache: Final = self._is_semantic_cache()
         scope_excluded_params: Final = self._SEMANTIC_CACHE_SCOPE_EXCLUDED_PARAMS if is_semantic_cache else frozenset()
         for param in kwargs:
@@ -387,7 +386,7 @@ class Cache:
                 param_value: str | None = self._get_param_value(param, kwargs)
                 if param_value is not None:
                     cache_key += f"{param}: {param_value}"
-            elif param not in litellm_param_kwargs:  # check if user passed in optional param - e.g. top_k
+            elif not is_litellm_owned_kwarg(param):
                 if litellm.enable_caching_on_provider_specific_optional_params is True:  # feature flagged for now
                     if kwargs[param] is None:
                         continue  # ignore None params

litellm/caching/caching_handler.py

@@ -52,7 +52,7 @@ from litellm.types.utils import (
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-    from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+    from litellm.llms.anthropic.pass_through.messages.response_cache import (
         AnthropicMessagesStreamCacheWriter,
     )
     from litellm.types.utils import PromptTokensDetailsWrapper
@@ -127,7 +127,7 @@ def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> boo
     spend and callback records. A plain (non-stream) replay logs here, since nothing
     else will.
     """
-    from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+    from litellm.llms.anthropic.pass_through.messages.response_cache import (
         CachedAnthropicMessagesStreamIterator,
     )
     from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
@@ -930,7 +930,7 @@ class LLMCachingHandler:
         elif (
             call_type == CallTypes.anthropic_messages.value or call_type == CallTypes.aanthropic_messages.value
         ) and isinstance(cached_result, dict):
-            from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+            from litellm.llms.anthropic.pass_through.messages.response_cache import (
                 convert_cached_anthropic_messages_result,
             )
 
@@ -1150,7 +1150,7 @@ class LLMCachingHandler:
             return result
         if not isinstance(result, AsyncIterator):
             return result
-        from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+        from litellm.llms.anthropic.pass_through.messages.response_cache import (
             AnthropicMessagesStreamCacheWriter,
         )
 

litellm/constants.py

@@ -49,6 +49,7 @@ DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SE
 DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10))
 DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512))
 DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = int(os.getenv("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", "16"))
+DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY: Final = get_env_int("DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY", 200)
 # https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
 MAX_S3_OBJECT_KEY_BYTES: Final = 1024
 S3_BOUNDED_OBJECT_KEY_HEAD_BYTES: Final = 64
@@ -945,6 +946,7 @@ openai_compatible_endpoints: Final[list] = [
     "https://api.libertai.io/v1",
     "https://pinstripes.io/v1",
     "https://api.meta.ai/v1",
+    "https://api.sailresearch.com/v1",
     "https://api.cognition.ai/v1",
     "https://api.scx.ai/v1",
     "https://gigachat.devices.sberbank.ru/api/v1",
@@ -1020,6 +1022,7 @@ openai_compatible_providers: Final[list] = [
     "meta",  # Meta Model API (Muse Spark) - JSON-configured provider
     "cognition",
     "scx-ai",
+    "sail",
 ]
 
 OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))
@@ -1607,6 +1610,8 @@ ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS: Final = {
 # e.g., 'x-pass-anthropic-beta: value' becomes 'anthropic-beta: value'
 # Works for all LLM pass-through endpoints (Vertex AI, Anthropic, Bedrock, etc.)
 PASS_THROUGH_HEADER_PREFIX: Final = "x-pass-"
+INTERNAL_KWARG_PREFIX: Final = "_litellm_"
+CONTROL_OPTIONS_KEY: Final = f"{INTERNAL_KWARG_PREFIX}control"
 
 AZURE_SPEECH_CUSTOM_LLM_PROVIDER: Final = "azure_speech"
 AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX: Final = "/azure_speech"

litellm/images/main.py

@@ -25,7 +25,7 @@ from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from litellm.llms.custom_llm import CustomLLM
-from litellm.utils import exception_type, get_litellm_params
+from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params
 
 #################### Initialize provider clients ####################
 llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
@@ -52,7 +52,6 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import (
     LITELLM_IMAGE_VARIATION_PROVIDERS,
     LlmProviders,
-    all_litellm_params,
 )
 from litellm.utils import (
     ImageResponse,
@@ -249,11 +248,7 @@ def image_generation(
             "size",
             "style",
         ]
-        litellm_params: Final = all_litellm_params
-        default_params: Final = openai_params + litellm_params
-        non_default_params: Final = {
-            k: v for k, v in kwargs.items() if k not in default_params
-        }  # model-specific params - pass them straight to the model/provider
+        non_default_params: Final = filter_out_litellm_params(kwargs, excluding=openai_params)
 
         image_generation_config: BaseImageGenerationConfig | None = None
         if custom_llm_provider is not None and custom_llm_provider in LlmProviders._member_map_.values():
@@ -757,11 +752,7 @@ def image_edit(
             "style",
             "async_call",
         ]
-        litellm_params_list: Final = all_litellm_params
-        default_params: Final = openai_params + litellm_params_list
-        non_default_params: Final = {
-            k: v for k, v in kwargs.items() if k not in default_params
-        }  # model-specific params - pass them straight to the model/provider
+        non_default_params: Final = filter_out_litellm_params(kwargs, excluding=openai_params)
         litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
         litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
         model_info: Final = kwargs.get("model_info", None)

litellm/integrations/adaptive_concurrency.py

@@ -0,0 +1,78 @@
+"""
+Adaptive in-flight concurrency limiter (AIMD, Vector ARC style).
+
+Grows the limit additively after `limit` consecutive clean completions and
+halves it only on an explicit throttle signal (429, 503, SlowDown, or a
+transport error out of the PUT). With floor == ceiling it degenerates to a
+fixed-width semaphore.
+"""
+
+import asyncio
+from collections import deque
+from contextlib import suppress
+from dataclasses import dataclass
+from typing import Final
+
+
+@dataclass(frozen=True, slots=True)
+class PutSample:
+    throttled: bool
+
+
+class AdaptiveConcurrencyLimiter:
+    """AIMD in-flight limiter used as `async with limiter:`."""
+
+    def __init__(self, initial: int, floor: int, ceiling: int) -> None:
+        if not 1 <= floor <= ceiling:
+            raise ValueError(f"adaptive limiter bounds must satisfy 1 <= floor <= ceiling, got {floor}..{ceiling}")
+        self._limit: int = min(max(initial, floor), ceiling)
+        self._floor: Final[int] = floor
+        self._ceiling: Final[int] = ceiling
+        self._clean_streak: int = 0
+        self._in_flight: int = 0
+        self._waiters: deque[asyncio.Future[None]] = deque()  # mutable-ok: waiters queue up behind a full limit
+
+    @property
+    def limit(self) -> int:
+        return self._limit
+
+    async def __aenter__(self) -> "AdaptiveConcurrencyLimiter":
+        if self._in_flight < self._limit:
+            self._in_flight += 1
+            return self
+        waiter: Final = asyncio.get_running_loop().create_future()
+        self._waiters.append(waiter)
+        try:
+            await waiter
+        except asyncio.CancelledError:
+            if waiter.done() and not waiter.cancelled():
+                self._in_flight -= 1
+                self._grant()
+            else:
+                with suppress(ValueError):
+                    self._waiters.remove(waiter)
+            raise
+        return self
+
+    def _grant(self) -> None:
+        while self._in_flight < self._limit and self._waiters:
+            waiter = self._waiters.popleft()
+            if waiter.done():
+                continue
+            self._in_flight += 1
+            waiter.set_result(None)
+
+    async def __aexit__(self, *_: object) -> None:
+        self._in_flight -= 1
+        self._grant()
+
+    def record(self, sample: PutSample) -> None:
+        if sample.throttled:
+            self._limit = max(self._floor, self._limit // 2)
+            self._clean_streak = 0
+            return
+        self._clean_streak += 1
+        if self._clean_streak >= self._limit and self._limit < self._ceiling:
+            self._limit += 1
+            self._clean_streak = 0
+            self._grant()

litellm/integrations/anthropic_cache_control_hook.py

@@ -695,6 +695,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         tools: list | None = None,
         cache_control: object = None,
         request_kwargs: object = None,
+        on_messages_route: bool = False,
     ) -> bool:
         """Return True if the request already carries any client-supplied cache_control.
 
@@ -704,10 +705,14 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         envelope. Configured injection points are an explicit instruction and are
         applied alongside the client's marks, bounded by the provider cap.
         """
-        return (
-            AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
-            + AnthropicCacheControlHook.count_external_cache_breakpoints(tools, cache_control, request_kwargs)
-        ) > 0
+        external_breakpoints: Final = (
+            AnthropicCacheControlHook.count_external_cache_breakpoints_on_messages_route(
+                tools, cache_control, request_kwargs
+            )
+            if on_messages_route
+            else AnthropicCacheControlHook.count_external_cache_breakpoints(tools, cache_control, request_kwargs)
+        )
+        return AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) + external_breakpoints > 0
 
     @staticmethod
     def get_default_injection_points(
@@ -719,6 +724,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         enable_prompt_caching: bool | None = None,
         cache_control: object = None,
         request_kwargs: object = None,
+        on_messages_route: bool = False,
     ) -> list[CacheControlInjectionPoint]:
         """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on.
 
@@ -739,7 +745,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if not supports_anthropic_cache_control(model, custom_llm_provider):
             return []
 
-        if AnthropicCacheControlHook._request_has_cache_control(messages, system, tools, cache_control, request_kwargs):
+        if AnthropicCacheControlHook._request_has_cache_control(
+            messages, system, tools, cache_control, request_kwargs, on_messages_route
+        ):
             return []
 
         if is_claude_code_one_shot_subagent_request(
@@ -968,6 +976,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 enable_prompt_caching=enable_prompt_caching,
                 cache_control=cache_control,
                 request_kwargs=kwargs,
+                on_messages_route=True,
             )
             if model is not None
             else ()

litellm/integrations/callback_configs.json

@@ -502,6 +502,27 @@
     },
     "description": "S3 Bucket (AWS) Logging Integration"
   },
+  {
+    "id": "signoz",
+    "displayName": "SigNoz",
+    "logo": "signoz.svg",
+    "supports_key_team_logging": true,
+    "dynamic_params": {
+      "signoz_ingestion_endpoint": {
+        "type": "text",
+        "ui_name": "SigNoz Ingestion Endpoint",
+        "description": "Ingestion endpoint for this team, e.g. https://ingest.us.signoz.cloud:443 for SigNoz Cloud or your own collector. Leave blank to use the proxy's configured endpoint. Regions: https://signoz.io/docs/ingestion/signoz-cloud/overview/",
+        "required": false
+      },
+      "signoz_ingestion_key": {
+        "type": "password",
+        "ui_name": "SigNoz Ingestion Key (optional)",
+        "description": "Ingestion key for this team, so its traces land in its own SigNoz account. Not needed for self-hosted SigNoz. Keys: https://signoz.io/docs/ingestion/signoz-cloud/keys/",
+        "required": false
+      }
+    },
+    "description": "SigNoz Logging Integration. Setup: https://signoz.io/docs/litellm-observability/"
+  },
   {
     "id": "sqs",
     "displayName": "SQS",

litellm/integrations/custom_guardrail.py

@@ -3,7 +3,7 @@ import copy
 import hashlib
 import os
 import secrets
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from datetime import datetime
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, get_args
@@ -37,6 +37,7 @@ from litellm.types.utils import (
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
     from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
+    from litellm.proxy._types import UserAPIKeyAuth
 dc: Final = DualCache()
 
 
@@ -106,6 +107,33 @@ def is_guardrail_intervention(e: Exception) -> bool:
     return is_fastapi_http_exception(e, _GUARDRAIL_BLOCK_STATUS_CODES)
 
 
+def _user_api_key_auth_from_request(request_data: Mapping[str, object]) -> "UserAPIKeyAuth":
+    from litellm.proxy._types import UserAPIKeyAuth
+
+    metadata: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data))
+    stamped: Final[Mapping[str, object]] = metadata if isinstance(metadata, dict) else {}
+
+    def stamped_str(field: str) -> str | None:
+        value: Final = stamped.get(field)
+        return value if isinstance(value, str) else None
+
+    return UserAPIKeyAuth(
+        user_id=stamped_str("user_api_key_user_id"),
+        team_id=stamped_str("user_api_key_team_id"),
+        end_user_id=stamped_str("user_api_key_end_user_id"),
+        api_key=stamped_str("user_api_key_hash"),
+        request_route=stamped_str("user_api_key_request_route"),
+    )
+
+
+def _unified_hook_fields(guardrail: "CustomGuardrail", request_data: Mapping[str, object]) -> Mapping[str, object]:
+    metadata_bucket: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data))
+    return {
+        "guardrail_to_apply": guardrail,
+        **({"litellm_metadata": metadata_bucket} if isinstance(metadata_bucket, dict) else {}),
+    }
+
+
 def _strict_guardrail_modes_enabled() -> bool:
     """Whether guardrail-mode validation raises (default) or logs a warning.
 
@@ -789,8 +817,6 @@ class CustomGuardrail(CustomLogger):
         return unified_guardrail
 
     async def async_pre_call_deployment_hook(self, kwargs: dict[str, Any], call_type: CallTypes | None) -> dict | None:
-        from litellm.proxy._types import UserAPIKeyAuth
-
         # should run guardrail
         litellm_guardrails: Final = kwargs.get("guardrails")
         if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
@@ -808,13 +834,7 @@ class CustomGuardrail(CustomLogger):
             if target is not self:
                 kwargs["guardrail_to_apply"] = self
             result: Final = await target.async_pre_call_hook(
-                user_api_key_dict=UserAPIKeyAuth(
-                    user_id=kwargs.get("user_api_key_user_id"),
-                    team_id=kwargs.get("user_api_key_team_id"),
-                    end_user_id=kwargs.get("user_api_key_end_user_id"),
-                    api_key=kwargs.get("user_api_key_hash"),
-                    request_route=kwargs.get("user_api_key_request_route"),
-                ),
+                user_api_key_dict=_user_api_key_auth_from_request(kwargs),
                 cache=dc,
                 data=kwargs,
                 call_type="completion" if call_type == CallTypes.completion else "acompletion",
@@ -827,6 +847,52 @@ class CustomGuardrail(CustomLogger):
 
         return kwargs
 
+    async def async_pre_call_hook_on_messages(
+        self,
+        request_data: Mapping[str, object],
+        messages: Sequence[AllMessageValues],
+    ) -> tuple[AllMessageValues, ...]:
+        from litellm.proxy.guardrails.exception_utils import (
+            enrich_http_exception_with_guardrail_context,
+            pre_call_rejection,
+        )
+
+        target: Final = self._deployment_hook_target()
+        scan_request: Final[dict[str, object]] = {  # mutable-ok: async_pre_call_hook writes into the dict it is handed
+            **{key: value for key, value in request_data.items() if key not in _PRE_CALL_CONTENT_KEYS},
+            "messages": list(messages),
+            **({} if target is self else _unified_hook_fields(self, request_data)),
+        }
+        try:
+            result: Final = await target.async_pre_call_hook(
+                user_api_key_dict=_user_api_key_auth_from_request(scan_request),
+                cache=dc,
+                data=scan_request,
+                call_type="acompletion",
+            )
+        except SensitiveDataRouteException as e:
+            unroutable: Final = pre_call_rejection(
+                f"{e.guardrail_name or self.guardrail_name} asked to reroute the request to {e.route_to_model} "
+                "over retrieved content; a request cannot be rerouted after retrieval, so it was blocked",
+                self.guardrail_name,
+            )
+            enrich_http_exception_with_guardrail_context(unroutable, self)
+            raise unroutable from e
+        except Exception as e:
+            enrich_http_exception_with_guardrail_context(e, self)
+            raise
+        if result is None:
+            return tuple(messages)
+        if isinstance(result, dict):
+            scanned: Final = result.get("messages")
+            return tuple(scanned) if isinstance(scanned, list) else tuple(messages)
+        if isinstance(result, str):
+            rejection: Final = pre_call_rejection(result, self.guardrail_name)
+            enrich_http_exception_with_guardrail_context(rejection, self)
+            raise rejection
+        enrich_http_exception_with_guardrail_context(result, self)
+        raise result
+
     async def async_post_call_success_deployment_hook(
         self,
         request_data: dict,
@@ -836,8 +902,6 @@ class CustomGuardrail(CustomLogger):
         """
         Allow modifying / reviewing the response just after it's received from the deployment.
         """
-        from litellm.proxy._types import UserAPIKeyAuth
-
         # should run guardrail
         litellm_guardrails: Final = request_data.get("guardrails")
         if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
@@ -851,13 +915,7 @@ class CustomGuardrail(CustomLogger):
             if target is not self:
                 request_data["guardrail_to_apply"] = self  # rebind-ok: dispatch consumes this key
             result: Final = await target.async_post_call_success_hook(
-                user_api_key_dict=UserAPIKeyAuth(
-                    user_id=request_data.get("user_api_key_user_id"),
-                    team_id=request_data.get("user_api_key_team_id"),
-                    end_user_id=request_data.get("user_api_key_end_user_id"),
-                    api_key=request_data.get("user_api_key_hash"),
-                    request_route=request_data.get("user_api_key_request_route"),
-                ),
+                user_api_key_dict=_user_api_key_auth_from_request(request_data),
                 data=request_data,
                 response=response,
             )

litellm/integrations/langtrace.py

@@ -10,6 +10,14 @@ if TYPE_CHECKING:
 else:
     Span = Any
 
+LANGTRACE_DEFAULT_HOST: Final = "https://app.langtrace.ai"
+LANGTRACE_TRACE_PATH: Final = "/api/trace"
+
+
+def langtrace_trace_endpoint(api_host: str | None) -> str:
+    host: Final = (api_host or LANGTRACE_DEFAULT_HOST).rstrip("/")
+    return host if host.endswith(LANGTRACE_TRACE_PATH) else host + LANGTRACE_TRACE_PATH
+
 
 class LangtraceAttributes:
     """

litellm/integrations/opentelemetry.py

@@ -15,6 +15,7 @@ from litellm.integrations._types.open_inference import (
     SpanAttributes,
 )
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.langtrace import LANGTRACE_TRACE_PATH
 from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
     OTEL_SEMCONV_STABILITY_OPT_IN_ENV,
     OTELGenAISemconvMixin,
@@ -3334,6 +3335,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         if signal_type == "traces" and "/v2/trace/otlp" in endpoint:
             return endpoint
 
+        if signal_type == "traces" and self.callback_name == "langtrace" and endpoint.endswith(LANGTRACE_TRACE_PATH):
+            return endpoint
+
         # Check if endpoint already ends with the correct signal path
         target_path: Final = f"/v1/{signal_type}"
         if endpoint.endswith(target_path):

litellm/integrations/otel/model/config.py

@@ -41,6 +41,7 @@ class ExporterOwner(str, Enum):
     LEVO = "levo"
     AGENTOPS = "agentops"
     NEWRELIC = "newrelic"
+    SIGNOZ = "signoz"
 
 
 class _OTelV2Flag(BaseSettings):

litellm/integrations/otel/presets/__init__.py

@@ -30,6 +30,11 @@ from litellm.integrations.otel.presets.phoenix import (
     phoenix_preset,
     phoenix_project_headers,
 )
+from litellm.integrations.otel.presets.signoz import (
+    signoz_dynamic_endpoint,
+    signoz_dynamic_headers,
+    signoz_preset,
+)
 from litellm.integrations.otel.presets.weave import weave_dynamic_headers, weave_preset
 from litellm.types.utils import StandardCallbackDynamicParams
 
@@ -44,6 +49,7 @@ PRESET_BY_CALLBACK: Final[Mapping[str, Preset]] = MappingProxyType(
         "langtrace": langtrace_preset,
         "levo": levo_preset,
         "newrelic": newrelic_preset,
+        "signoz": signoz_preset,
         "weave_otel": weave_preset,
     }
 )
@@ -58,6 +64,7 @@ DYNAMIC_HEADERS_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynami
             "arize": arize_dynamic_headers,
             "langfuse_otel": langfuse_dynamic_headers,
             "newrelic": newrelic_dynamic_headers,
+            "signoz": signoz_dynamic_headers,
             "weave_otel": weave_dynamic_headers,
         }
     )
@@ -71,6 +78,7 @@ DYNAMIC_ENDPOINT_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynam
     MappingProxyType(
         {
             "newrelic": newrelic_dynamic_endpoint,
+            "signoz": signoz_dynamic_endpoint,
         }
     )
 )
@@ -153,5 +161,6 @@ __all__ = [
     "newrelic_preset",
     "phoenix_preset",
     "project_routing_headers",
+    "signoz_preset",
     "weave_preset",
 ]

litellm/integrations/otel/presets/signoz.py

@@ -0,0 +1,95 @@
+from functools import lru_cache
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.integrations.otel.model.config import (
+    ExporterOwner,
+    ExporterSpec,
+    OpenTelemetryV2Config,
+)
+from litellm.integrations.otel.presets.utils import ensure_mappers
+from litellm.litellm_core_utils.url_utils import is_url_destination_allowed_by_host
+from litellm.types.utils import StandardCallbackDynamicParams
+
+SIGNOZ_INGESTION_ENDPOINT_ENV: Final = "SIGNOZ_INGESTION_ENDPOINT"
+
+
+class _SigNozSettings(BaseSettings):
+    model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
+
+    endpoint: str | None = Field(default=None, validation_alias=SIGNOZ_INGESTION_ENDPOINT_ENV)
+    ingestion_key: str | None = Field(default=None, validation_alias="SIGNOZ_INGESTION_KEY")
+
+
+def signoz_preset(
+    *,
+    config_overrides: OpenTelemetryV2Config | None = None,
+    allow_missing_credentials: bool = False,
+) -> OpenTelemetryV2Config:
+    settings: Final = _SigNozSettings()
+    base: Final = config_overrides or OpenTelemetryV2Config()
+    key: Final = settings.ingestion_key
+    spec: Final = ExporterSpec(
+        kind="otlp_http",
+        endpoint=settings.endpoint,
+        headers=(f"signoz-ingestion-key={key}" if key else None),
+        owner=ExporterOwner.SIGNOZ,
+        requires_headers=bool(key),
+    )
+    return base.model_copy(
+        update=MappingProxyType(
+            {
+                "exporters": (*base.exporters, spec),
+                "mapper_names": ensure_mappers(base.mapper_names, "genai"),
+            }
+        )
+    )
+
+
+@lru_cache(maxsize=128)
+def _warn_host_not_allowlisted(endpoint: str) -> None:
+    verbose_logger.warning(
+        "SigNoz: not exporting to key/team endpoint '%s'. Add its host to "
+        "litellm_settings.provider_url_destination_allowed_hosts to permit it",
+        endpoint,
+    )
+
+
+@lru_cache(maxsize=128)
+def _warn_endpoint_without_key(endpoint: str) -> None:
+    verbose_logger.warning(
+        "SigNoz: not exporting to key/team endpoint '%s'. Set signoz_ingestion_key alongside it; "
+        "a keyless collector needs the global callback",
+        endpoint,
+    )
+
+
+def _tenant_endpoint_is_unusable(params: StandardCallbackDynamicParams) -> bool:
+    return bool(params.get("signoz_ingestion_endpoint")) and signoz_dynamic_endpoint(params) is None
+
+
+def signoz_dynamic_endpoint(params: StandardCallbackDynamicParams) -> str | None:
+    endpoint: Final = params.get("signoz_ingestion_endpoint")
+    if not endpoint or not endpoint.startswith(("http://", "https://")):
+        return None
+    if not params.get("signoz_ingestion_key"):
+        _warn_endpoint_without_key(endpoint)
+        return None
+    if not is_url_destination_allowed_by_host(endpoint, litellm.provider_url_destination_allowed_hosts):
+        _warn_host_not_allowlisted(endpoint)
+        return None
+    return endpoint
+
+
+def signoz_dynamic_headers(
+    params: StandardCallbackDynamicParams,
+) -> dict[str, str]:  # mutable-ok: DYNAMIC_HEADERS_BY_CALLBACK returns a dict
+    key: Final = params.get("signoz_ingestion_key")
+    if _tenant_endpoint_is_unusable(params) or not key:
+        return {}  # mutable-ok: same registry contract
+    return {"signoz-ingestion-key": key}  # mutable-ok: same registry contract

litellm/integrations/s3.py

@@ -36,22 +36,86 @@ def resolve_s3_log_prompts_only(configured: object, environ: Mapping[str, str] |
         return True
 
 
-def resolve_s3_max_concurrent_uploads(configured: object, fallback: int) -> int:
+def _resolve_positive_int(setting: str, configured: object, fallback: int, *, reject_bool: bool) -> int:
     if configured is None or configured == "":
         return fallback
+    if reject_bool and isinstance(configured, bool):
+        verbose_logger.warning(
+            "s3 logging: %s=%r is a boolean, not an integer, using %s", setting, configured, fallback
+        )
+        return fallback
+    try:
+        bound: Final = _UPLOAD_BOUND.validate_python(configured.strip() if isinstance(configured, str) else configured)
+    except ValidationError:
+        verbose_logger.warning("s3 logging: %s=%r is not an integer, using %s", setting, configured, fallback)
+        return fallback
+    if bound < 1:
+        verbose_logger.warning("s3 logging: %s=%r must be at least 1, using %s", setting, configured, fallback)
+        return fallback
+    return bound
+
+
+def resolve_s3_max_concurrent_uploads(configured: object, fallback: int) -> int:
+    return _resolve_positive_int("s3_max_concurrent_uploads", configured, fallback, reject_bool=False)
+
+
+def resolve_s3_max_queue_size(configured: object, fallback: int) -> int:
+    return _resolve_positive_int("s3_max_queue_size", configured, fallback, reject_bool=True)
+
+
+def resolve_s3_max_retry_age_seconds(configured: object, fallback: int | None) -> int | None:
+    if configured is None or configured == "":
+        return None
+    if isinstance(configured, bool):
+        verbose_logger.warning(
+            "s3 logging: s3_max_retry_age_seconds=%r is a boolean, not an integer, falling back to %r",
+            configured,
+            fallback,
+        )
+        return fallback
     try:
         bound: Final = _UPLOAD_BOUND.validate_python(configured.strip() if isinstance(configured, str) else configured)
     except ValidationError:
         verbose_logger.warning(
-            "s3 logging: s3_max_concurrent_uploads=%r is not an integer, using %s", configured, fallback
+            "s3 logging: s3_max_retry_age_seconds=%r is not an integer, falling back to %r", configured, fallback
         )
         return fallback
-    if bound < 1:
+    if bound < 0:
         verbose_logger.warning(
-            "s3 logging: s3_max_concurrent_uploads=%r must be at least 1, using %s", configured, fallback
+            "s3 logging: s3_max_retry_age_seconds=%r must be at least 0, falling back to %r", configured, fallback
         )
         return fallback
-    return bound
+    return bound or None
+
+
+def resolve_s3_max_adaptive_concurrency(configured: object, fallback: int) -> int:
+    return _resolve_positive_int("s3_max_adaptive_concurrency", configured, fallback, reject_bool=True)
+
+
+def resolve_s3_drop_on_terminal_error(configured: object) -> bool:
+    if configured is None or configured == "":
+        return True
+    try:
+        return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
+    except ValidationError:
+        verbose_logger.warning(
+            "s3 logging: s3_drop_on_terminal_error=%r is not a boolean, dropping terminal-failed uploads",
+            configured,
+        )
+        return True
+
+
+def resolve_s3_adaptive_concurrency(configured: object) -> bool:
+    if configured is None or configured == "":
+        return False
+    try:
+        return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
+    except ValidationError:
+        verbose_logger.warning(
+            "s3 logging: s3_adaptive_concurrency=%r is not a boolean, keeping the fixed upload width",
+            configured,
+        )
+        return False
 
 
 def resolve_s3_batch_file_upload(configured: object) -> bool:

litellm/integrations/s3_v2.py

@@ -3,14 +3,19 @@ s3 Bucket Logging Integration
 
 async_log_success_event: Processes the event, stores it in memory for DEFAULT_S3_FLUSH_INTERVAL_SECONDS seconds or until DEFAULT_S3_BATCH_SIZE and then flushes to s3
 async_log_failure_event: Processes the event, stores it in memory for DEFAULT_S3_FLUSH_INTERVAL_SECONDS seconds or until DEFAULT_S3_BATCH_SIZE and then flushes to s3
-NOTE 1: S3 does not provide a BATCH PUT API endpoint; by default each element is uploaded concurrently (bounded by s3_max_concurrent_uploads), or with s3_batch_file_upload the whole flush is written as one .jsonl file
+NOTE 1: S3 does not provide a BATCH PUT API endpoint; by default each element is uploaded concurrently with the fixed s3_max_concurrent_uploads bound (or an adaptive bound when s3_adaptive_concurrency is on, backing off only on throttling), or with s3_batch_file_upload the whole flush is written as one .jsonl file
 """
 
 import asyncio
+import contextvars
+import logging
+import re
 import time
-from collections.abc import Mapping
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import dataclass
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Final, cast
+from functools import partial
+from typing import TYPE_CHECKING, Final, Literal, cast
 from urllib.parse import quote
 from uuid import uuid4
 
@@ -21,15 +26,22 @@ from litellm._logging import print_verbose, verbose_logger
 from litellm.constants import (
     DEFAULT_S3_BATCH_SIZE,
     DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
+    DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
     DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
 )
+from litellm.integrations.adaptive_concurrency import AdaptiveConcurrencyLimiter, PutSample
 from litellm.integrations.s3 import (
     get_s3_object_download_filename,
     get_s3_object_key,
     prompts_only_payload,
+    resolve_s3_adaptive_concurrency,
     resolve_s3_batch_file_upload,
+    resolve_s3_drop_on_terminal_error,
     resolve_s3_log_prompts_only,
+    resolve_s3_max_adaptive_concurrency,
     resolve_s3_max_concurrent_uploads,
+    resolve_s3_max_queue_size,
+    resolve_s3_max_retry_age_seconds,
     resolve_sse_params,
 )
 from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
@@ -50,6 +62,42 @@ if TYPE_CHECKING:
     from botocore.credentials import Credentials
 
 
+UploadOutcome = Literal["delivered", "retry", "dropped"]
+
+_TERMINAL_ERROR_CODES: Final = frozenset(
+    {
+        "EntityTooLarge",
+        "InvalidArgument",
+        "MalformedXML",
+        "InvalidDigest",
+        "KeyTooLongError",
+        "BadDigest",
+        "InvalidRequest",
+    }
+)
+_BODY_CODED_STATUSES: Final = frozenset({400, 403})
+_RETRYABLE_STATUSES: Final = frozenset({403, 500, 503})
+_S3_ERROR_CODE: Final = re.compile(r"<Code>([^<]+)</Code>")
+
+
+@dataclass(frozen=True, slots=True)
+class _PreparedPut:
+    json_string: str
+    headers: Mapping[str, str]
+
+
+def _s3_error_code(response: httpx.Response) -> str | None:
+    text: Final = response.text
+    match: Final = _S3_ERROR_CODE.search(text) if isinstance(text, str) else None
+    return match.group(1) if match else None
+
+
+def _is_terminal(response: httpx.Response) -> bool:
+    """True only for object-specific, unrecoverable rejections (400/403 with a terminal XML code).
+    Unknown codes, empty or non-XML bodies, and every other status fail safe toward retry."""
+    return response.status_code in _BODY_CODED_STATUSES and _s3_error_code(response) in _TERMINAL_ERROR_CODES
+
+
 def _s3_key_parent(s3_object_key: str) -> str:
     return s3_object_key.rsplit("/", 1)[0] if "/" in s3_object_key else ""
 
@@ -58,11 +106,19 @@ class S3BatchUploadError(Exception):
     def __init__(self, failed: int, total: int) -> None:
         self.failed = failed
         self.total = total
-        super().__init__(f"{failed} of {total} S3 uploads failed; events kept in queue for the next flush")
+        super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush")
+
+
+_in_flush: Final[contextvars.ContextVar[bool]] = contextvars.ContextVar("s3_v2_in_flush", default=False)
 
 
 class S3Logger(CustomBatchLogger, BaseAWSLLM):
     preserve_events_added_during_flush = True
+    _flush_retries: int = 0
+    _requeued_count: int = 0
+    _upload_limiter: asyncio.Semaphore | AdaptiveConcurrencyLimiter | None = None
+    s3_drop_on_terminal_error: bool = True
+    s3_max_retry_age_seconds: int | None = 3600
 
     def __init__(
         self,
@@ -92,6 +148,11 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         s3_sse_kms_key_id: str | None = None,
         s3_log_prompts_only: bool | None = None,
         s3_max_concurrent_uploads: int = DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
+        s3_max_queue_size: int | None = None,
+        s3_max_retry_age_seconds: int | None = 3600,
+        s3_drop_on_terminal_error: bool = True,
+        s3_adaptive_concurrency: bool = False,
+        s3_max_adaptive_concurrency: int | None = None,
         s3_batch_file_upload: bool = False,
         s3_callback_params_override: dict | None = None,
         **kwargs,
@@ -135,9 +196,22 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 s3_sse_kms_key_id=s3_sse_kms_key_id,
                 s3_log_prompts_only=s3_log_prompts_only,
                 s3_max_concurrent_uploads=s3_max_concurrent_uploads,
+                s3_max_queue_size=s3_max_queue_size,
+                s3_max_retry_age_seconds=s3_max_retry_age_seconds,
+                s3_drop_on_terminal_error=s3_drop_on_terminal_error,
+                s3_adaptive_concurrency=s3_adaptive_concurrency,
+                s3_max_adaptive_concurrency=s3_max_adaptive_concurrency,
                 s3_batch_file_upload=s3_batch_file_upload,
             )
-            self._upload_semaphore = asyncio.Semaphore(self.s3_max_concurrent_uploads)
+            self._upload_limiter = (
+                AdaptiveConcurrencyLimiter(
+                    initial=self.s3_max_concurrent_uploads,
+                    floor=self.s3_max_concurrent_uploads,
+                    ceiling=max(self.s3_max_concurrent_uploads, self.s3_max_adaptive_concurrency),
+                )
+                if self.s3_adaptive_concurrency
+                else asyncio.Semaphore(self.s3_max_concurrent_uploads)
+            )
             verbose_logger.debug("s3 logger using endpoint url %s", s3_endpoint_url)
 
             # IMPORTANT
@@ -158,8 +232,12 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 flush_lock=self.flush_lock,
                 flush_interval=s3_flush_interval,
                 batch_size=s3_batch_size,
+                max_queue_size=self.s3_max_queue_size,
             )
             self.log_queue: list[s3BatchLoggingElement] = []
+            self._requeued_count = 0
+            self._flush_retries = 0
+            self._flush_dropped: dict[int, s3BatchLoggingElement] = {}
 
             # Call BaseAWSLLM's __init__
             BaseAWSLLM.__init__(self)
@@ -194,6 +272,11 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         s3_sse_kms_key_id: str | None = None,
         s3_log_prompts_only: bool | None = None,
         s3_max_concurrent_uploads: int = DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
+        s3_max_queue_size: int | None = None,
+        s3_max_retry_age_seconds: int | None = 3600,
+        s3_drop_on_terminal_error: bool = True,
+        s3_adaptive_concurrency: bool = False,
+        s3_max_adaptive_concurrency: int | None = None,
         s3_batch_file_upload: bool = False,
         params_source: dict | None = None,
     ):
@@ -259,6 +342,37 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
         )
 
+        configured_queue_size: Final = params.get("s3_max_queue_size")
+        constructor_queue_size: Final = resolve_s3_max_queue_size(
+            s3_max_queue_size, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE
+        )
+        self.s3_max_queue_size = resolve_s3_max_queue_size(configured_queue_size, constructor_queue_size)
+
+        configured_retry_age: Final = params.get("s3_max_retry_age_seconds")
+        constructor_retry_age: Final = resolve_s3_max_retry_age_seconds(s3_max_retry_age_seconds, 3600)
+        self.s3_max_retry_age_seconds = (
+            constructor_retry_age
+            if configured_retry_age is None or configured_retry_age == ""
+            else resolve_s3_max_retry_age_seconds(configured_retry_age, constructor_retry_age)
+        )
+
+        configured_drop: Final = params.get("s3_drop_on_terminal_error")
+        self.s3_drop_on_terminal_error = resolve_s3_drop_on_terminal_error(
+            configured_drop if configured_drop is not None else s3_drop_on_terminal_error
+        )
+
+        self.s3_adaptive_concurrency = s3_adaptive_concurrency or resolve_s3_adaptive_concurrency(
+            params.get("s3_adaptive_concurrency")
+        )
+
+        configured_adaptive_ceiling: Final = params.get("s3_max_adaptive_concurrency")
+        self.s3_max_adaptive_concurrency = resolve_s3_max_adaptive_concurrency(
+            s3_max_adaptive_concurrency
+            if configured_adaptive_ceiling is None or configured_adaptive_ceiling == ""
+            else configured_adaptive_ceiling,
+            DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
+        )
+
         self.s3_batch_file_upload = s3_batch_file_upload or resolve_s3_batch_file_upload(
             params.get("s3_batch_file_upload")
         )
@@ -310,6 +424,35 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         }
         return {key: value for key, value in candidates.items() if value}
 
+    def _prepare_put(self, batch_logging_element: s3BatchLoggingElement) -> _PreparedPut:
+        try:
+            import base64
+            import hashlib
+        except ImportError:
+            raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
+
+        json_string: Final = (
+            batch_logging_element.body
+            if batch_logging_element.body is not None
+            else safe_dumps(batch_logging_element.payload)
+        )
+        content_hash: Final = hashlib.sha256(json_string.encode("utf-8")).hexdigest()
+        content_md5: Final = base64.b64encode(
+            hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()
+        ).decode()
+        return _PreparedPut(
+            json_string=json_string,
+            headers={
+                "Content-Type": batch_logging_element.content_type,
+                "Content-MD5": content_md5,
+                "x-amz-content-sha256": content_hash,
+                "Content-Language": "en",
+                "Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"',
+                "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
+                **self._sse_headers(),
+            },
+        )
+
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         await self._async_log_event_base(
             kwargs=kwargs,
@@ -384,12 +527,21 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             verbose_logger.exception("s3 Layer Error - %s", e)
             self.handle_callback_failure(callback_name="S3Logger")
 
-    async def async_upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement) -> bool:
-        try:
-            import base64
-            import hashlib
-        except ImportError:
-            raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
+    @property
+    def _upload_semaphore(self) -> asyncio.Semaphore | AdaptiveConcurrencyLimiter:
+        limiter: Final = self._upload_limiter
+        if limiter is None:
+            raise AttributeError("_upload_semaphore")
+        return limiter
+
+    @_upload_semaphore.setter
+    def _upload_semaphore(self, value: asyncio.Semaphore | AdaptiveConcurrencyLimiter) -> None:
+        self._upload_limiter = value
+
+    async def async_upload_data_to_s3(
+        self,
+        batch_logging_element: s3BatchLoggingElement,
+    ) -> bool:
         try:
             from litellm.litellm_core_utils.asyncify import asyncify
 
@@ -400,31 +552,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
 
             url: Final = self._build_object_url(batch_logging_element.s3_object_key)
 
-            # Convert JSON to string
-            json_string: Final = (
-                batch_logging_element.body
-                if batch_logging_element.body is not None
-                else safe_dumps(batch_logging_element.payload)
-            )
-
-            # Calculate SHA256 hash of the content
-            content_hash: Final = hashlib.sha256(json_string.encode("utf-8")).hexdigest()
-            content_md5: Final = base64.b64encode(
-                hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()
-            ).decode()
-
-            # Prepare the request
-            headers: Final = {
-                "Content-Type": batch_logging_element.content_type,
-                "Content-MD5": content_md5,
-                "x-amz-content-sha256": content_hash,
-                "Content-Language": "en",
-                "Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"',
-                "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
-                **self._sse_headers(),
-            }
-
-            async def signed_put() -> httpx.Response:
+            async def signed_put(prepared: _PreparedPut) -> httpx.Response:
                 credentials: Final = await asyncified_get_credentials(
                     aws_access_key_id=self.s3_aws_access_key_id,
                     aws_secret_access_key=self.s3_aws_secret_access_key,
@@ -436,18 +564,26 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                     aws_web_identity_token=self.s3_aws_web_identity_token,
                     aws_sts_endpoint=self.s3_aws_sts_endpoint,
                 )
-                signed_headers: Final = await run_aws_signing(self._sign_put, credentials, url, json_string, headers)
+                signed_headers: Final = await run_aws_signing(
+                    self._sign_put, credentials, url, prepared.json_string, prepared.headers
+                )
                 try:
-                    return await self.async_httpx_client.put(url, data=json_string, headers=signed_headers)
+                    return await self.async_httpx_client.put(url, data=prepared.json_string, headers=signed_headers)
                 except httpx.HTTPStatusError as error:
                     return error.response
 
             max_retries: Final = 3
+            prepared: Final = self._prepare_put(batch_logging_element)
             for attempt in range(max_retries):
-                response = await signed_put()
-                if response.status_code in (403, 500, 503) and attempt < max_retries - 1:
+                response = await self._recorded_put(partial(signed_put, prepared))
+                if (
+                    response.status_code in _RETRYABLE_STATUSES
+                    and not (self.s3_drop_on_terminal_error and _is_terminal(response))
+                    and attempt < max_retries - 1
+                ):
                     wait_time = 2**attempt  # 1s, 2s
-                    verbose_logger.warning(
+                    verbose_logger.log(
+                        logging.DEBUG if _in_flush.get() else logging.WARNING,
                         "S3 upload returned %s, retrying in %ss (attempt %s/%s) key=%s",
                         response.status_code,
                         wait_time,
@@ -455,6 +591,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                         max_retries,
                         batch_logging_element.s3_object_key,
                     )
+                    self._flush_retries += 1
                     await asyncio.sleep(wait_time)
                     continue
                 response.raise_for_status()
@@ -462,6 +599,13 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         except Exception as e:
             verbose_logger.exception("Error uploading to s3: %s", e)
             self.handle_callback_failure(callback_name="S3Logger")
+            if isinstance(e, httpx.HTTPStatusError) and self.s3_drop_on_terminal_error and _is_terminal(e.response):
+                verbose_logger.warning(
+                    "s3 logging: dropping object %s after terminal status %s",
+                    batch_logging_element.s3_object_key,
+                    e.response.status_code,
+                )
+                self._flush_dropped[id(batch_logging_element)] = batch_logging_element
             return False
         return True
 
@@ -483,11 +627,64 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         #  see custom_batch_logger.py which triggers the flush
         #########################################################
         uploads: Final = self._batch_file_elements(batch) if self._batch_file_mode_active() else batch
-        results: Final = await asyncio.gather(*(self._upload_bounded(element) for element in uploads))
-        failed: Final = tuple(element for element, ok in zip(uploads, results, strict=True) if not ok)
-        if not failed:
+        self._flush_retries = 0
+        self._flush_dropped = {}  # mutable-ok: per-flush drop marks read back by _upload_bounded
+        stale: Final = min(self._requeued_count, len(uploads)) if len(uploads) == len(batch) else 0
+        order: Final = (*range(stale, len(uploads)), *range(stale))
+        ordered: Final = await asyncio.gather(*(self._upload_outcome(uploads[i]) for i in order))
+        outcomes: Final = dict(zip(order, ordered, strict=True))
+        results: Final = tuple(outcomes[i] for i in range(len(uploads)))
+        if self._flush_retries:
+            verbose_logger.warning(
+                "s3 logging: %s in-call retries across %s uploads this flush",
+                self._flush_retries,
+                len(uploads),
+            )
+        delivered: Final = sum(1 for outcome in results if outcome == "delivered")
+        bucket_wide: Final = delivered == 0
+        failed: Final = tuple(
+            (element, outcome) for element, outcome in zip(uploads, results, strict=True) if outcome != "delivered"
+        )
+        now: Final = time.monotonic()
+        requeued: Final = (
+            tuple(element for element, _ in failed)
+            if bucket_wide
+            else tuple(
+                element
+                if element.retrying_since is not None or self.s3_max_retry_age_seconds is None
+                else element.model_copy(update={"retrying_since": now})
+                for element, outcome in failed
+                if outcome != "dropped"
+                and not (
+                    self.s3_max_retry_age_seconds is not None
+                    and element.retrying_since is not None
+                    and now - element.retrying_since > self.s3_max_retry_age_seconds
+                )
+            )
+        )
+        dropped: Final = len(failed) - len(requeued)
+        if dropped:
+            verbose_logger.warning(
+                "s3 logging: %s uploads dropped (terminal or retrying longer than s3_max_retry_age_seconds=%s)",
+                dropped,
+                self.s3_max_retry_age_seconds,
+            )
+        if not requeued:
+            self._requeued_count = 0
             return
-        self.log_queue = [*failed, *self.log_queue[len(batch) :]]
+        arrivals: Final = self.log_queue[len(batch) :]
+        overflow: Final = max(0, len(requeued) + len(arrivals) - self.max_queue_size)
+        if overflow:
+            verbose_logger.warning(
+                "s3 logging: queue exceeded max_queue_size=%s after a failed flush, dropped %s oldest events",
+                self.max_queue_size,
+                overflow,
+            )
+        self.log_queue = [  # mutable-ok: log_queue is the flush buffer shared with custom_batch_logger
+            *requeued,
+            *arrivals,
+        ][overflow:]
+        self._requeued_count = max(0, len(requeued) - overflow)
         raise S3BatchUploadError(failed=len(failed), total=len(uploads))
 
     def _batch_file_mode_active(self) -> bool:
@@ -502,8 +699,37 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         return True
 
     async def _upload_bounded(self, element: s3BatchLoggingElement) -> bool:
-        async with self._upload_semaphore:
-            return await self.async_upload_data_to_s3(element)
+        token: Final = _in_flush.set(True)
+        try:
+            async with self._upload_semaphore:
+                return await self.async_upload_data_to_s3(element)
+        finally:
+            _in_flush.reset(token)
+
+    async def _upload_outcome(self, element: s3BatchLoggingElement) -> UploadOutcome:
+        delivered: Final = await self._upload_bounded(element)
+        if delivered:
+            return "delivered"
+        if id(element) in self._flush_dropped:
+            return "dropped"
+        return "retry"
+
+    async def _recorded_put(self, signed_put: Callable[[], Awaitable[httpx.Response]]) -> httpx.Response:
+        limiter: Final = self._upload_limiter
+        adaptive: Final = limiter if isinstance(limiter, AdaptiveConcurrencyLimiter) else None
+        try:
+            response: Final = await signed_put()
+        except Exception:
+            if adaptive is not None:
+                adaptive.record(PutSample(throttled=True))
+            raise
+        if adaptive is not None:
+            adaptive.record(
+                PutSample(
+                    throttled=response.status_code in (429, 503) or _s3_error_code(response) == "SlowDown",
+                )
+            )
+        return response
 
     def _batch_file_elements(self, batch: tuple[s3BatchLoggingElement, ...]) -> tuple[s3BatchLoggingElement, ...]:
         now: Final = datetime.now(timezone.utc)
@@ -527,6 +753,9 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             content_type="application/x-ndjson",
             s3_object_key=f"{parent}/{batch_name}.jsonl" if parent else f"{batch_name}.jsonl",
             s3_object_download_filename=f"{batch_name}.jsonl",
+            retrying_since=min(
+                (element.retrying_since for element in elements if element.retrying_since is not None), default=None
+            ),
         )
 
     def create_s3_batch_logging_element(
@@ -596,58 +825,35 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         )
 
     def upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement):
-        try:
-            import base64
-            import hashlib
-        except ImportError:
-            raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
         try:
             verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key)
 
             url: Final = self._build_object_url(batch_logging_element.s3_object_key)
 
-            # Convert JSON to string
-            json_string: Final = (
-                batch_logging_element.body
-                if batch_logging_element.body is not None
-                else safe_dumps(batch_logging_element.payload)
-            )
-
-            # Calculate SHA256 hash of the content
-            content_hash: Final = hashlib.sha256(json_string.encode("utf-8")).hexdigest()
-            content_md5: Final = base64.b64encode(
-                hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()
-            ).decode()
-
-            # Prepare the request
-            headers: Final = {
-                "Content-Type": batch_logging_element.content_type,
-                "Content-MD5": content_md5,
-                "x-amz-content-sha256": content_hash,
-                "Content-Language": "en",
-                "Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"',
-                "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
-                **self._sse_headers(),
-            }
+            prepared: Final = self._prepare_put(batch_logging_element)
 
             httpx_client: Final = _get_httpx_client(
                 params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None)
             )
 
-            def signed_put() -> httpx.Response:
+            def signed_put(prepared_put: _PreparedPut) -> httpx.Response:
                 credentials: Final = self.get_credentials(
                     aws_access_key_id=self.s3_aws_access_key_id,
                     aws_secret_access_key=self.s3_aws_secret_access_key,
                     aws_session_token=self.s3_aws_session_token,
                     aws_region_name=self.s3_region_name,
                 )
-                signed_headers: Final = self._sign_put(credentials, url, json_string, headers)
-                return httpx_client.put(url, data=json_string, headers=signed_headers)
+                signed_headers: Final = self._sign_put(credentials, url, prepared_put.json_string, prepared_put.headers)
+                return httpx_client.put(url, data=prepared_put.json_string, headers=signed_headers)
 
             max_retries: Final = 3
             for attempt in range(max_retries):
-                response = signed_put()
-                if response.status_code in (403, 500, 503) and attempt < max_retries - 1:
+                response = signed_put(prepared)
+                if (
+                    response.status_code in _RETRYABLE_STATUSES
+                    and not (self.s3_drop_on_terminal_error and _is_terminal(response))
+                    and attempt < max_retries - 1
+                ):
                     wait_time = 2**attempt  # 1s, 2s
                     verbose_logger.warning(
                         "S3 upload returned %s, retrying in %ss (attempt %s/%s) key=%s",
@@ -664,6 +870,12 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         except Exception as e:
             verbose_logger.exception("Error uploading to s3: %s", e)
             self.handle_callback_failure(callback_name="S3Logger")
+            if isinstance(e, httpx.HTTPStatusError) and self.s3_drop_on_terminal_error and _is_terminal(e.response):
+                verbose_logger.warning(
+                    "s3 logging: dropping object %s after terminal status %s",
+                    batch_logging_element.s3_object_key,
+                    e.response.status_code,
+                )
 
     async def _download_object_from_s3(self, s3_object_key: str) -> dict | None:
         """

litellm/integrations/shadow_eval_logger.py

@@ -111,7 +111,7 @@ def _chat_request_from_anthropic_messages(
     because the logged optional_params switch dialect per provider path (the bridge's
     inner completion rewrites them to chat shape mid-flight); the adapter translates
     them alongside the messages, and sampling params copy through untranslated."""
-    from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
+    from litellm.llms.anthropic.pass_through.adapters.transformation import (
         LiteLLMAnthropicMessagesAdapter,
     )
 

litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py

@@ -2,11 +2,13 @@
 Vector Store Pre-Call Hook
 
 This hook is called before making an LLM request when a vector store is configured.
-It searches the vector store for relevant context and appends it to the messages.
+It searches the vector store for relevant context, runs the request's pre-call guardrails
+over that context, and appends it to the messages.
 """
 
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from dataclasses import dataclass
+from itertools import chain
 from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_args
 
 from pydantic import TypeAdapter, ValidationError
@@ -16,7 +18,9 @@ import litellm
 import litellm.vector_stores
 from litellm._logging import verbose_logger
 from litellm.exceptions import VectorStoreSearchError
+from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionUserMessage,
@@ -42,6 +46,24 @@ else:
 SEARCH_FAILURES_FIELD: Final = "vector_store_search_failures"
 _DEFAULT_FAILURE_MODE: Final[VectorStoreSearchFailureMode] = "annotate"
 _FAILURE_MODE_ADAPTER: Final = TypeAdapter(VectorStoreSearchFailureMode)
+_STR_KEYED_ADAPTER: Final = TypeAdapter(dict[str, object])
+_GUARDRAIL_KEYS_THE_PROXY_MERGES_INTO_METADATA: Final = frozenset(
+    {"guardrails", "guardrail_config", "policies", "include_guardrail_response"}
+)
+
+
+def _scan_request(model: str, non_default_params: Mapping[str, object]) -> Mapping[str, object]:
+    try:
+        proxy_request: Final = _STR_KEYED_ADAPTER.validate_python(non_default_params.get("proxy_server_request"))
+        client_body: Final = _STR_KEYED_ADAPTER.validate_python(proxy_request.get("body"))
+    except ValidationError:
+        return {**non_default_params, "model": model}
+    proxy_request_params: Final = {**client_body, **non_default_params}
+    return {
+        key: value
+        for key, value in proxy_request_params.items()
+        if key not in _GUARDRAIL_KEYS_THE_PROXY_MERGES_INTO_METADATA
+    }
 
 
 class ProxyRuntime(Protocol):
@@ -82,7 +104,7 @@ SearchOutcome = SearchSucceeded | SearchFailed
 
 @dataclass(frozen=True, slots=True)
 class VectorStoreAugmentation:
-    messages: tuple[AllMessageValues, ...]
+    context_messages: tuple[AllMessageValues, ...]
     search_results: tuple[VectorStoreSearchResponse, ...]
     failures: tuple[VectorStoreSearchFailure, ...]
 
@@ -95,7 +117,8 @@ class VectorStorePreCallHook(CustomLogger):
     When a vector store is configured, this hook:
     1. Extracts the query from the last user message
     2. Calls litellm.vector_stores.search() to get relevant context
-    3. Appends the search results as context to the messages
+    3. Runs the request's pre-call guardrails over each store's context message
+    4. Appends the (possibly masked) context to the messages, or raises the guardrail's block
     """
 
     def __init__(self, proxy_runtime: ProxyRuntime | None = None):
@@ -170,7 +193,50 @@ class VectorStorePreCallHook(CustomLogger):
                 case _:
                     assert_never(failure_mode)
 
-        return model, list(augmentation.messages), non_default_params
+        scanned_context: Final = await self._scanned_context_messages(
+            model=model,
+            non_default_params=non_default_params,
+            context_messages=augmentation.context_messages,
+        )
+        return (
+            model,
+            self._messages_with_context(messages=messages, context_messages=scanned_context),
+            non_default_params,
+        )
+
+    async def _scanned_context_messages(
+        self,
+        model: str,
+        non_default_params: Mapping[str, object],
+        context_messages: Sequence[AllMessageValues],
+    ) -> tuple[AllMessageValues, ...]:
+        request_data: Final = _scan_request(model, non_default_params)
+        guardrails: Final = tuple(
+            callback
+            for callback in litellm.callbacks
+            if isinstance(callback, CustomGuardrail)
+            and callback.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.pre_call)
+        )
+        if not guardrails:
+            return tuple(context_messages)
+        scanned: Final = [
+            await self._scan_through(guardrails=guardrails, request_data=request_data, messages=(context_message,))
+            for context_message in context_messages
+        ]
+        return tuple(chain.from_iterable(scanned))
+
+    async def _scan_through(
+        self,
+        guardrails: Sequence[CustomGuardrail],
+        request_data: Mapping[str, object],
+        messages: Sequence[AllMessageValues],
+    ) -> tuple[AllMessageValues, ...]:
+        if not guardrails:
+            return tuple(messages)
+        scanned: Final = await guardrails[0].async_pre_call_hook_on_messages(
+            request_data=request_data, messages=messages
+        )
+        return await self._scan_through(guardrails=guardrails[1:], request_data=request_data, messages=scanned)
 
     async def _augment_messages(
         self,
@@ -234,7 +300,7 @@ class VectorStorePreCallHook(CustomLogger):
         failures: Final = tuple(outcome.failure for outcome in outcomes if isinstance(outcome, SearchFailed))
 
         return VectorStoreAugmentation(
-            messages=self._messages_with_context(messages=messages, search_results=search_results),
+            context_messages=self._context_messages(search_results),
             search_results=search_results,
             failures=failures,
         )
@@ -309,19 +375,21 @@ class VectorStorePreCallHook(CustomLogger):
 
         return None
 
-    def _messages_with_context(
-        self,
-        messages: Sequence[AllMessageValues],
-        search_results: Sequence[VectorStoreSearchResponse],
-    ) -> tuple[AllMessageValues, ...]:
-        context_messages: Final = tuple(
+    def _context_messages(self, search_results: Sequence[VectorStoreSearchResponse]) -> tuple[AllMessageValues, ...]:
+        return tuple(
             context_message
             for search_response in search_results
             if (context_message := self._context_message(search_response)) is not None
         )
+
+    def _messages_with_context(
+        self,
+        messages: Sequence[AllMessageValues],
+        context_messages: Sequence[AllMessageValues],
+    ) -> list[AllMessageValues]:
         if not context_messages:
-            return tuple(messages)
-        return (*messages[:-1], *context_messages, *messages[-1:])
+            return list(messages)
+        return [*messages[:-1], *context_messages, *messages[-1:]]
 
     def _context_message(self, search_response: VectorStoreSearchResponse) -> AllMessageValues | None:
         """Build the context message for one vector store's results, or None when it returned nothing usable."""

litellm/integrations/websearch_interception/ARCHITECTURE.md

@@ -70,7 +70,7 @@ Claude Code (Anthropic's official CLI) sends web search requests using Anthropic
 
 Native tools are converted to LiteLLM standard format **before** sending to the provider:
 
-1. **Conversion Point** (`litellm/llms/anthropic/experimental_pass_through/messages/handler.py`):
+1. **Conversion Point** (`litellm/llms/anthropic/pass_through/messages/handler.py`):
    - In `anthropic_messages()` function (lines 60-127)
    - Runs BEFORE the API request is made
    - Detects native web search tools using `is_web_search_tool()`

litellm/litellm_core_utils/custom_logger_registry.py

@@ -89,6 +89,7 @@ class CustomLoggerRegistry:
         "langtrace": OpenTelemetry,
         "weave_otel": OpenTelemetry,
         "levo": OpenTelemetry,
+        "signoz": OpenTelemetry,
         "mlflow": MlflowLogger,
         "langfuse": LangfusePromptManagement,
         "otel": OpenTelemetry,

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -2346,6 +2346,12 @@ def _map_exception_by_status(
             )
 
 
+def _is_guardrail_block(original_exception: Exception) -> bool:
+    from litellm.integrations.custom_guardrail import is_guardrail_intervention
+
+    return is_guardrail_intervention(original_exception)
+
+
 def exception_type(
     model,
     original_exception,
@@ -2356,6 +2362,8 @@ def exception_type(
     """Maps an LLM Provider Exception to OpenAI Exception Format"""
     if any(isinstance(original_exception, exc_type) for exc_type in litellm.LITELLM_EXCEPTION_TYPES):
         return original_exception
+    if _is_guardrail_block(original_exception):
+        return original_exception
     exception_mapping_worked = False
     exception_provider = custom_llm_provider
     mappable_exception: Final[_ProviderHTTPException] = cast("_ProviderHTTPException", original_exception)

litellm/litellm_core_utils/get_litellm_params.py

@@ -1,9 +1,15 @@
+import reprlib
 from collections.abc import Mapping, MutableMapping
+from dataclasses import dataclass, fields
 from types import MappingProxyType
 from typing import Final
 
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.constants import CONTROL_OPTIONS_KEY
 from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm.llms.openai.data_residency import infer_openai_data_residency
+from litellm.types.litellm_params import MAX_CONTROL_INT_DIGITS, ControlOptions
 from litellm.types.router import CustomPricingLiteLLMParams
 
 AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
@@ -70,6 +76,51 @@ OPTIONAL_KWARGS_KEYS: Final = (
 # Backward-compatible alias for existing imports/tests.
 _OPTIONAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS
 
+_CONTROL_OPTIONS: Final = TypeAdapter(ControlOptions)
+_CONTROL_OPTION_NAMES: Final = tuple(field.name for field in fields(ControlOptions))
+_MAX_SHOWN_INT_BITS: Final = 64
+_EXPECTED: Final = f"expected a positive integer of at most {MAX_CONTROL_INT_DIGITS} digits"
+
+
+class _BoundedRepr(reprlib.Repr):
+    def repr_int(self, x: int, level: int) -> str:
+        if x.bit_length() > _MAX_SHOWN_INT_BITS:
+            return f"<int of {x.bit_length()} bits>"
+        return super().repr_int(x, level)
+
+
+_BOUNDED_REPR: Final = _BoundedRepr()
+
+
+@dataclass(frozen=True, slots=True)
+class InvalidControlOption:
+    param: str
+    message: str
+
+
+def parse_control_options(kwargs: Mapping[str, object]) -> ControlOptions | InvalidControlOption:
+    given: Final = {  # mutable-ok: TypeAdapter.validate_python takes a dict
+        name: kwargs[name] for name in _CONTROL_OPTION_NAMES if name in kwargs
+    }
+    try:
+        return _CONTROL_OPTIONS.validate_python(given)
+    except ValidationError as e:
+        param: Final = str(e.errors(include_url=False)[0]["loc"][0])
+        return InvalidControlOption(
+            param=param, message=f"Invalid {param}={_BOUNDED_REPR.repr(given[param])}: {_EXPECTED}"
+        )
+
+
+def stored_control_options(litellm_params: Mapping[str, object]) -> ControlOptions:
+    control: Final = litellm_params.get(CONTROL_OPTIONS_KEY)
+    return control if isinstance(control, ControlOptions) else ControlOptions()
+
+
+def with_control_options(litellm_params: Mapping[str, object], control: ControlOptions) -> dict[str, object]:
+    if control == ControlOptions():
+        return dict(litellm_params)  # mutable-ok: completion() hands litellm_params to provider code typed as dict
+    return {**litellm_params, CONTROL_OPTIONS_KEY: control}  # mutable-ok: same dict contract as above
+
 
 def _get_base_model_from_litellm_call_metadata(
     metadata: dict | None,
@@ -130,7 +181,6 @@ def get_litellm_params(
     api_version: str | None = None,
     max_retries: int | None = None,
     litellm_request_debug: bool | None = None,
-    stream_chunk_size: int | None = None,
     **kwargs,
 ) -> dict:
     _litellm_metadata_dict: Final = litellm_metadata if isinstance(litellm_metadata, dict) else None
@@ -193,7 +243,6 @@ def get_litellm_params(
         "max_retries": max_retries,
         "use_litellm_proxy": use_litellm_proxy,
         "litellm_request_debug": litellm_request_debug,
-        "stream_chunk_size": stream_chunk_size,
     }
 
     # Sparse extraction: only add kwargs keys that are actually present

litellm/litellm_core_utils/initialize_dynamic_callback_params.py

@@ -113,6 +113,8 @@ _supported_callback_params: Final[tuple[str, ...]] = (
     "dd_agent_port",
     "newrelic_api_key",
     "newrelic_region",
+    "signoz_ingestion_endpoint",
+    "signoz_ingestion_key",
     "turn_off_message_logging",
 )
 
@@ -126,6 +128,8 @@ _request_blocked_callback_params: Final = frozenset(
         "dd_agent_port",
         "newrelic_api_key",
         "newrelic_region",
+        "signoz_ingestion_endpoint",
+        "signoz_ingestion_key",
     }
 )
 
@@ -138,6 +142,8 @@ _trusted_overlay_callback_params: Final = frozenset(
     {
         "newrelic_api_key",
         "newrelic_region",
+        "signoz_ingestion_endpoint",
+        "signoz_ingestion_key",
     }
 )
 

litellm/litellm_core_utils/litellm_logging.py

@@ -60,6 +60,7 @@ from litellm.integrations.arize.arize import ArizeLogger
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.deepeval.deepeval import DeepEvalLogger
+from litellm.integrations.langtrace import langtrace_trace_endpoint
 from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
 from litellm.litellm_core_utils.classifier_logging import (
@@ -4920,9 +4921,9 @@ def _init_custom_logger_compatible_class(
 
             otel_config = OpenTelemetryConfig(
                 exporter="otlp_http",
-                endpoint="https://langtrace.ai/api/trace",
+                endpoint=langtrace_trace_endpoint(os.getenv("LANGTRACE_API_HOST")),
+                headers=f"x-api-key={os.environ['LANGTRACE_API_KEY']}",
             )
-            os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
             for callback in _in_memory_loggers:
                 if isinstance(callback, OpenTelemetry) and callback.callback_name == "langtrace":
                     return callback
@@ -4930,6 +4931,38 @@ def _init_custom_logger_compatible_class(
             _in_memory_loggers.append(_otel_logger)
             return _otel_logger
 
+        elif logging_integration == "signoz":
+            from litellm.integrations.otel.presets.signoz import (
+                SIGNOZ_INGESTION_ENDPOINT_ENV,
+            )
+
+            _signoz_endpoint: Final = os.getenv(SIGNOZ_INGESTION_ENDPOINT_ENV)
+            if not _signoz_endpoint:
+                raise ValueError(f"{SIGNOZ_INGESTION_ENDPOINT_ENV} not found in environment variables")
+
+            _signoz_v2: Final = _maybe_construct_otel_v2("signoz", _in_memory_loggers)
+            if _signoz_v2 is not None:
+                return _signoz_v2
+
+            from litellm.integrations.opentelemetry import (
+                OpenTelemetry,
+                OpenTelemetryConfig,
+            )
+
+            _signoz_base: Final = _signoz_endpoint.rstrip("/")
+            _signoz_key: Final = os.getenv("SIGNOZ_INGESTION_KEY")
+            _signoz_config: Final = OpenTelemetryConfig(
+                exporter="otlp_http",
+                endpoint=(_signoz_base if _signoz_base.endswith("/v1/traces") else f"{_signoz_base}/v1/traces"),
+                headers=(f"signoz-ingestion-key={_signoz_key}" if _signoz_key else None),
+            )
+            for callback in _in_memory_loggers:
+                if isinstance(callback, OpenTelemetry) and callback.callback_name == "signoz":
+                    return callback
+            _signoz_logger: Final = OpenTelemetry(config=_signoz_config, callback_name="signoz")
+            _in_memory_loggers.append(_signoz_logger)
+            return _signoz_logger
+
         elif logging_integration == "mlflow":
             for callback in _in_memory_loggers:
                 if isinstance(callback, MlflowLogger):

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -70,6 +70,7 @@ _SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple(
 _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
     {
         ServiceTier.FLEX.value: ServiceTier.FLEX.value,
+        ServiceTier.BALANCED.value: ServiceTier.BALANCED.value,
         ServiceTier.PRIORITY.value: ServiceTier.PRIORITY.value,
         ServiceTier.FAST.value: ServiceTier.PRIORITY.value,
         ServiceTier.ULTRAFAST.value: ServiceTier.ULTRAFAST.value,
@@ -252,7 +253,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str:
 
     Args:
         base_key: The base cost key (e.g., "input_cost_per_token")
-        service_tier: The service tier ("flex", "priority", "fast", "ultrafast", or None for standard)
+        service_tier: The service tier ("flex", "balanced", "priority", "fast", "ultrafast", or None for standard)
 
     Returns:
         str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1992,11 +1992,14 @@ def is_encrypted_reasoning_block(block: object) -> bool:
 def is_unsignable_thinking_block(block: object) -> bool:
     """A thinking block Anthropic cannot accept on input.
 
-    Anthropic verifies the thinking signature cryptographically, so a block whose
-    signature is null, empty, or missing (e.g. from an open-source reasoning model)
-    is rejected with a 400 and must be dropped rather than blanked or repaired, and
-    so is a block whose signature or data carries another provider's encrypted
-    reasoning. A `redacted_thinking` block Anthropic minted is always kept.
+    Anthropic verifies the signature cryptographically, so a block with a null,
+    empty, or missing signature (e.g. from an open-source reasoning model) is
+    rejected with a 400, and so is a block whose signature or data carries
+    another provider's encrypted reasoning. It also rejects a `thinking` block
+    whose text is empty or whitespace-only ("each thinking block must contain
+    thinking"), regardless of signature, e.g. when a `thinking_blocks` history
+    item from a non-Anthropic reasoning provider is replayed through this path.
+    `redacted_thinking` blocks carry no signature and are always kept.
     """
     if is_encrypted_reasoning_block(block):
         return True
@@ -2006,7 +2009,10 @@ def is_unsignable_thinking_block(block: object) -> bool:
     if mapping.get("type") != "thinking":
         return False
     signature: Final = mapping.get("signature")
-    return not (isinstance(signature, str) and len(signature) > 0)
+    if not (isinstance(signature, str) and len(signature) > 0):
+        return True
+    thinking_text: Final = mapping.get("thinking")
+    return not (isinstance(thinking_text, str) and len(thinking_text.strip()) > 0)
 
 
 def strip_encrypted_reasoning_from_messages(messages: object) -> None:

litellm/litellm_core_utils/streaming_handler.py

@@ -1504,8 +1504,11 @@ class CustomStreamWrapper:
 
                 self.tool_call = True
 
-            if hasattr(chunk, "usage") and chunk.usage is not None:
-                model_response.usage = chunk.usage
+            chunk_usage: Final = getattr(chunk, "usage", None)
+            if isinstance(chunk_usage, Usage):
+                model_response.usage = chunk_usage
+            elif isinstance(chunk_usage, BaseModel):
+                model_response.usage = Usage(**chunk_usage.model_dump())
 
             ## RETURN ARG
             result: Final = self.return_processed_chunk_logic(

litellm/llms/anthropic/cache_aware_routing.py

@@ -0,0 +1,236 @@
+from __future__ import annotations
+
+import time
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final
+from urllib.parse import urlparse
+
+from pydantic import BaseModel, JsonValue, TypeAdapter
+
+import litellm
+from litellm._internal_context import current_billing_time, pinned_billing_time
+from litellm.caching.dual_cache import DualCache
+from litellm.llms.anthropic.prompt_cache_prediction import (
+    NativePredictionTarget,
+    PromptPrefix,
+    TokenCounter,
+    UnsupportedPredictionTarget,
+    cache_scope,
+    count_prompt_tokens,
+    parse_prompt,
+    resolve_prediction_target,
+    supported_prediction_headers,
+)
+from litellm.proxy.common_utils.prompt_cache_pricing import price_cache_tokens
+from litellm.proxy.hooks.prompt_cache_prediction import lookup
+from litellm.types.management_endpoints.prompt_cache_prediction import (
+    CacheCostScenario,
+    CacheEvidence,
+    CachePredictionArm,
+    CacheTokenBuckets,
+)
+from litellm.types.router import Deployment
+from litellm.utils import get_prompt_cache_min_tokens
+
+__all__: Final = ("AnthropicCacheRouting", "TokenCounter", "predict_arm")
+
+_JSON: Final = TypeAdapter(Mapping[str, JsonValue])
+_NATIVE_OPTIONS: Final = frozenset(
+    (
+        "max_tokens",
+        "system",
+        "tools",
+        "tool_choice",
+        "thinking",
+        "output_config",
+        "cache_control",
+        "speed",
+        "service_tier",
+        "temperature",
+        "top_p",
+        "top_k",
+        "stop_sequences",
+        "stream",
+    )
+)
+
+
+class _ModelLimits(BaseModel):
+    max_input_tokens: int | None = None
+    max_output_tokens: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class AnthropicCacheRouting:
+    body: Mapping[str, JsonValue]
+    prefix: PromptPrefix
+    requested_output_limit: int
+
+    @staticmethod
+    def request_body(
+        url: str,
+        headers: Mapping[str, str],
+        body: Mapping[str, JsonValue],
+        request_kwargs: Mapping[str, object],
+        messages: Sequence[Mapping[str, object]] | None,
+    ) -> Mapping[str, JsonValue] | None:
+        if not urlparse(url).path.endswith("/v1/messages") or not supported_prediction_headers(headers):
+            return None
+        return _JSON.validate_python(
+            MappingProxyType(
+                {
+                    **body,
+                    **MappingProxyType({key: request_kwargs[key] for key in _NATIVE_OPTIONS if key in request_kwargs}),
+                    "messages": messages,
+                }
+            )
+        )
+
+    @classmethod
+    def from_body(cls, body: Mapping[str, JsonValue]) -> AnthropicCacheRouting | None:
+        prefix: Final = parse_prompt(body)
+        limit: Final = body.get("max_tokens")
+        if prefix is None or not isinstance(limit, int) or isinstance(limit, bool) or limit <= 0:
+            return None
+        return cls(body, prefix, limit)
+
+    @staticmethod
+    def supports(deployment: Deployment) -> bool:
+        return isinstance(resolve_prediction_target(deployment.litellm_params), NativePredictionTarget)
+
+    async def is_warm(self, deployment: Deployment, caller: str, cache: DualCache, now: float) -> bool:
+        target: Final = resolve_prediction_target(deployment.litellm_params)
+        if not isinstance(target, NativePredictionTarget):
+            return False
+        scope: Final = cache_scope(caller, deployment.model_info.id or "", target.api_key, target.model)
+        observation: Final = await lookup(cache, scope, self.prefix, now=now)
+        return observation is not None and observation.expires_at > now
+
+    @staticmethod
+    def fits(deployment: Deployment, input_tokens: int, output_tokens: int) -> bool:
+        target: Final = resolve_prediction_target(deployment.litellm_params)
+        if not isinstance(target, NativePredictionTarget):
+            return False
+        limits: Final = _ModelLimits.model_validate(
+            MappingProxyType(
+                {
+                    **litellm.get_model_info(target.model, custom_llm_provider="anthropic"),
+                    **deployment.model_info.model_dump(exclude_none=True),
+                }
+            )
+        )
+        return (
+            limits.max_input_tokens is not None
+            and input_tokens + output_tokens <= limits.max_input_tokens
+            and limits.max_output_tokens is not None
+            and output_tokens <= limits.max_output_tokens
+        )
+
+    async def predict(
+        self,
+        deployment: Deployment,
+        caller: str,
+        cache: DualCache,
+        counter: TokenCounter,
+        now: float | None,
+    ) -> CachePredictionArm:
+        return await predict_arm(deployment, self.body, self.prefix, caller, cache, counter, now=now)
+
+    @staticmethod
+    def cost(arm: CachePredictionArm, output_tokens: int) -> float | None:
+        return (
+            price_cache_tokens(arm.model or "", arm.deployment_id, arm.estimate.tokens, output_tokens)
+            if arm.estimate is not None
+            else None
+        )
+
+    @staticmethod
+    async def count_tokens(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
+        return await count_prompt_tokens(model, api_key, body)
+
+
+def _buckets(prefix_tokens: int, suffix_tokens: int, read_tokens: int, ttl_seconds: int) -> CacheTokenBuckets:
+    return CacheTokenBuckets(
+        uncached_input_tokens=suffix_tokens,
+        cache_read_input_tokens=read_tokens,
+        cache_creation_5m_input_tokens=prefix_tokens - read_tokens if ttl_seconds == 300 else 0,
+        cache_creation_1h_input_tokens=prefix_tokens - read_tokens if ttl_seconds == 3600 else 0,
+    )
+
+
+def _scenario(model: str, deployment_id: str, tokens: CacheTokenBuckets) -> CacheCostScenario | None:
+    cost: Final = price_cache_tokens(model=model, deployment_id=deployment_id, tokens=tokens)
+    return CacheCostScenario(tokens=tokens, input_cost=cost) if cost is not None else None
+
+
+async def predict_arm(
+    deployment: Deployment,
+    body: Mapping[str, JsonValue],
+    prefix: PromptPrefix,
+    caller_key_hash: str,
+    cache: DualCache,
+    token_counter: TokenCounter,
+    now: float | None = None,
+) -> CachePredictionArm:
+    deployment_id: Final = deployment.model_info.id or ""
+    params: Final = deployment.litellm_params
+    unknown: Final = CachePredictionArm(deployment_id=deployment_id, model=params.model)
+    if deployment.model_info.blocked:
+        return unknown.model_copy(update=MappingProxyType({"reason": "unsupported_deployment_configuration"}))
+    target: Final = resolve_prediction_target(params)
+    if isinstance(target, UnsupportedPredictionTarget):
+        return unknown.model_copy(update=MappingProxyType({"reason": target.reason}))
+    model: Final = target.model
+    api_key: Final = target.api_key
+    total_count: Final = await token_counter(model, api_key, body)
+    prefix_count: Final = await token_counter(model, api_key, prefix.prefix_body)
+    if total_count is None or prefix_count is None or total_count < prefix_count:
+        return unknown.model_copy(update=MappingProxyType({"reason": "token_count_unavailable"}))
+    scope: Final = cache_scope(caller_key_hash, deployment_id, api_key, model)
+    checked_at: Final = time.time() if now is None else now
+    observation: Final = await lookup(cache, scope, prefix, now=checked_at)
+    exact: Final = observation is not None and observation.fingerprint == prefix.fingerprint
+    cacheable: Final = observation.cached_tokens if exact and observation is not None else prefix_count
+    if cacheable > total_count or (observation is not None and observation.cached_tokens > cacheable):
+        return unknown.model_copy(update=MappingProxyType({"reason": "inconsistent_prefix_token_count"}))
+    suffix: Final = total_count - cacheable
+    evidence: Final = (
+        CacheEvidence(observed_at=observation.observed_at, expires_at=observation.expires_at)
+        if observation is not None
+        else None
+    )
+    if cacheable < get_prompt_cache_min_tokens(params.model):
+        disabled: Final = _scenario(model, deployment_id, CacheTokenBuckets(uncached_input_tokens=total_count))
+        if disabled is None:
+            return unknown.model_copy(update=MappingProxyType({"reason": "pricing_unavailable"}))
+        return CachePredictionArm(
+            deployment_id=deployment_id,
+            model=model,
+            cache_state="disabled",
+            reason="below_cache_minimum",
+            estimate=disabled,
+            cold=disabled,
+            warm=disabled,
+            token_count_source="anthropic_count_tokens",
+        )
+    fresh: Final = observation is not None and observation.expires_at > checked_at
+    read: Final = observation.cached_tokens if fresh and observation is not None else 0
+    with pinned_billing_time(current_billing_time()):
+        cold: Final = _scenario(model, deployment_id, _buckets(cacheable, suffix, 0, prefix.ttl_seconds))
+        warm: Final = _scenario(model, deployment_id, _buckets(cacheable, suffix, cacheable, prefix.ttl_seconds))
+        estimate: Final = _scenario(model, deployment_id, _buckets(cacheable, suffix, read, prefix.ttl_seconds))
+    if cold is None or warm is None or estimate is None:
+        return unknown.model_copy(update=MappingProxyType({"reason": "pricing_unavailable"}))
+    return CachePredictionArm(
+        deployment_id=deployment_id,
+        model=model,
+        cache_state="warm" if fresh and exact else "partial" if fresh else "stale" if observation else "unknown",
+        reason=None if fresh else "observation_expired" if observation else "no_compatible_observation",
+        estimate=estimate,
+        cold=cold,
+        warm=warm,
+        evidence=evidence,
+        token_count_source="anthropic_count_tokens",
+    )

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -25,7 +25,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
 
 from litellm._logging import verbose_proxy_logger
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
-from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
+from litellm.llms.anthropic.pass_through.adapters.transformation import (
     LiteLLMAnthropicMessagesAdapter,
     is_provider_native_tool_dict,
 )
@@ -365,7 +365,7 @@ class AnthropicMessagesHandler(BaseTranslation):
     def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]:
         import uuid
 
-        from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
+        from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
             FakeAnthropicMessagesStreamIterator,
         )
         from litellm.llms.base_llm.guardrail_translation.utils import (

litellm/llms/anthropic/pass_through/adapters/handler.py

@@ -11,15 +11,15 @@ from typing_extensions import TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.asyncify import run_async_function
-from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
+from litellm.llms.anthropic.pass_through.adapters.transformation import (
     AnthropicAdapter,
 )
-from litellm.llms.anthropic.experimental_pass_through.context_management import (
+from litellm.llms.anthropic.pass_through.context_management import (
     AnthropicContextManagementError,
     PolyfillResult,
     apply_context_management,
 )
-from litellm.llms.anthropic.experimental_pass_through.utils import (
+from litellm.llms.anthropic.pass_through.utils import (
     is_reasoning_auto_summary_enabled,
     litellm_logging_obj_from_kwargs,
     local_model_name,
@@ -102,7 +102,7 @@ async def _prepare_context_managed_request(
     user_api_key_auth: "UserAPIKeyAuth | None" = None,
 ) -> PolyfillResult | None:
     """Apply client compaction history, then optional context_management polyfill."""
-    from litellm.llms.anthropic.experimental_pass_through.context_management.editors.compact import (
+    from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
         apply_client_compaction_block_history,
     )
 
@@ -179,7 +179,7 @@ def _polyfill_will_run(
     if edits is None:
         return False
 
-    from litellm.llms.anthropic.experimental_pass_through.context_management.constants import (
+    from litellm.llms.anthropic.pass_through.context_management.constants import (
         COMPACT_EDIT_TYPE,
     )
 
@@ -205,7 +205,7 @@ def _spec_has_non_compact_edits(
     if edits is None:
         return False
 
-    from litellm.llms.anthropic.experimental_pass_through.context_management.constants import (
+    from litellm.llms.anthropic.pass_through.context_management.constants import (
         COMPACT_EDIT_TYPE,
     )
 
@@ -240,7 +240,7 @@ def _normalize_spec_edits(
     if _context_management_explicitly_dropped(additional_drop_params):
         return None
 
-    from litellm.llms.anthropic.experimental_pass_through.context_management.dispatcher import (
+    from litellm.llms.anthropic.pass_through.context_management.dispatcher import (
         _normalize_spec,
     )
 
@@ -437,7 +437,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         Handles both string ("max") and dict ({"effort": "max", "summary": ...})
         formats. Uses model registry to check supports_xhigh/supports_minimal.
         """
-        from litellm.llms.anthropic.experimental_pass_through.utils import (
+        from litellm.llms.anthropic.pass_through.utils import (
             normalize_reasoning_effort_value,
         )
 

litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py

@@ -118,7 +118,7 @@ class _CombinedChunkSplitter:
     @staticmethod
     def _is_combined(chunk: "ModelResponseStream") -> bool:
         """True if ``chunk`` carries response content AND a finish_reason."""
-        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+        from litellm.llms.anthropic.pass_through.messages.utils import (
             openai_chat_refusal_text,
         )
 
@@ -1029,7 +1029,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         delta: Final = processed_chunk["delta"]
         if delta.get("stop_reason") == "max_tokens":
             return processed_chunk
-        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+        from litellm.llms.anthropic.pass_through.messages.utils import (
             refusal_stop_details,
         )
 
@@ -1083,7 +1083,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     @staticmethod
     def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
         from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
-        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+        from litellm.llms.anthropic.pass_through.messages.utils import (
             openai_chat_refusal_text,
         )
 
@@ -1120,7 +1120,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         - Different content types in the response
         - Specific markers in the content
         """
-        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+        from litellm.llms.anthropic.pass_through.messages.utils import (
             openai_chat_refusal_text,
         )
 

litellm/llms/anthropic/pass_through/adapters/transformation.py

@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
 from pydantic import JsonValue, TypeAdapter
 
 import litellm
-from litellm.llms.anthropic.experimental_pass_through.utils import (
+from litellm.llms.anthropic.pass_through.utils import (
     is_reasoning_auto_summary_enabled,
     prompt_cache_key_from_user_id,
 )
@@ -134,14 +134,14 @@ from litellm.llms.anthropic.common_utils import (
     normalize_anthropic_tool_use_id,
     strip_encrypted_reasoning_blocks_from_anthropic_messages,
 )
-from litellm.llms.anthropic.experimental_pass_through.context_management import (
+from litellm.llms.anthropic.pass_through.context_management import (
     PolyfillResult,
 )
-from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import (
+from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import (
     convert_mid_conversation_system_turns,
     is_system_role_message,
 )
-from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+from litellm.llms.anthropic.pass_through.messages.utils import (
     openai_chat_refusal_text,
     refusal_stop_details,
 )

litellm/llms/anthropic/pass_through/context_management/editors/compact.py

@@ -30,7 +30,11 @@ from litellm.types.llms.anthropic import (
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
     from litellm.proxy._types import UserAPIKeyAuth
-    from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitDescriptor, RateLimitResponse
+    from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+        RateLimitDescriptor,
+        RateLimitDescriptorRateLimitObject,
+        RateLimitResponse,
+    )
     from litellm.router import Router
     from litellm.types.llms.anthropic import (
         AllAnthropicPassThroughMessageValues,
@@ -149,6 +153,10 @@ class _CreateOrgRateLimitDescriptors(Protocol):
     ) -> "Sequence[RateLimitDescriptor]": ...
 
 
+class _GetProxyHook(Protocol):
+    def __call__(self, hook: str) -> object: ...
+
+
 class _ShouldRateLimit(Protocol):
     def __call__(
         self,
@@ -492,6 +500,24 @@ async def _check_summary_model_budget(
     return True
 
 
+def _without_parallel_request_gauges(
+    descriptors: "Sequence[RateLimitDescriptor]",
+) -> "tuple[RateLimitDescriptor, ...]":
+    return tuple(_without_parallel_request_gauge(descriptor) for descriptor in descriptors)
+
+
+def _without_parallel_request_gauge(descriptor: "RateLimitDescriptor") -> "RateLimitDescriptor":
+    rate_limit: Final = descriptor.get("rate_limit")
+    if rate_limit is None or rate_limit.get("max_parallel_requests") is None:
+        return descriptor
+    windowed_limits: Final[RateLimitDescriptorRateLimitObject] = {
+        "requests_per_unit": rate_limit.get("requests_per_unit"),
+        "tokens_per_unit": rate_limit.get("tokens_per_unit"),
+        "window_size": rate_limit.get("window_size"),
+    }
+    return {**descriptor, "rate_limit": windowed_limits}
+
+
 async def _check_summary_model_rate_limit(
     user_api_key_auth: Optional["UserAPIKeyAuth"],
     summary_model: str,
@@ -508,21 +534,28 @@ async def _check_summary_model_rate_limit(
     ``read_only`` mode so no counter is reserved or incremented — the summary
     call's actual usage is still charged exactly once by the limiter's
     post-call success hook (via the propagated ``litellm_metadata``).
+    ``max_parallel_requests`` gauges are left out of the check: the summary
+    call runs inside the caller's already admitted request, whose own slot
+    would otherwise count against it.
 
     Returns True (allow) outside the proxy, when the active limiter does not
     expose the read-only descriptor check (legacy limiter), or when the
-    descriptor set cannot be built — the only deny signal is a definitive
-    ``OVER_LIMIT`` response, so an internal error here forwards the request
-    uncompacted rather than blocking every summary.
+    descriptor set cannot be built — the deny signals are a definitive
+    ``OVER_LIMIT`` response and the limiter's own fail-closed rejection
+    (``RateLimitUnverifiableError``, raised when ``fail_closed_rate_limit_enforcement``
+    is on and the counters could not be verified), so any other internal error here
+    forwards the request uncompacted rather than blocking every summary.
     """
     if user_api_key_auth is None:
         return True
     try:
+        from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitUnverifiableError
         from litellm.proxy.proxy_server import proxy_logging_obj
     except Exception:
         return True
 
-    limiter: Final[object] = getattr(proxy_logging_obj, "max_parallel_request_limiter", None)
+    get_proxy_hook: Final[_GetProxyHook | None] = getattr(proxy_logging_obj, "get_proxy_hook", None)
+    limiter: Final[object] = get_proxy_hook("parallel_request_limiter") if get_proxy_hook is not None else None
     should_rate_limit_check: Final[_ShouldRateLimit | None] = getattr(limiter, "should_rate_limit", None)
     create_descriptors: Final[_CreateRateLimitDescriptors | None] = getattr(
         limiter, "_create_rate_limit_descriptors", None
@@ -566,7 +599,9 @@ async def _check_summary_model_rate_limit(
             requested_model=summary_model,
             descriptors=base_descriptors,
         )
-        descriptors: Final = (*base_descriptors, *create_org_descriptors(user_api_key_auth, summary_model))
+        descriptors: Final = _without_parallel_request_gauges(
+            (*base_descriptors, *create_org_descriptors(user_api_key_auth, summary_model))
+        )
         if not descriptors:
             return True
         parent_otel_span: Final[object] = getattr(user_api_key_auth, "parent_otel_span", None)
@@ -575,6 +610,13 @@ async def _check_summary_model_rate_limit(
             parent_otel_span=parent_otel_span,
             read_only=True,
         )
+    except RateLimitUnverifiableError as e:
+        verbose_logger.warning(
+            "compact_20260112: rate-limit counters for summary_model=%s could not be verified; denying: %s",
+            summary_model,
+            e.detail,
+        )
+        return False
     except Exception as e:
         verbose_logger.warning(
             "compact_20260112: unexpected error during rate-limit check for summary_model=%s; allowing: %s",
@@ -756,7 +798,7 @@ def _count_effective_tokens(
     threshold check matches the downstream ``input_tokens`` metric.
     """
     # Local import to avoid pulling the adapter at module load time.
-    from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
+    from litellm.llms.anthropic.pass_through.adapters.transformation import (
         LiteLLMAnthropicMessagesAdapter,
     )
 
@@ -913,7 +955,7 @@ def _build_summary_messages(
     system prompt); the conversation history is translated to OpenAI shape;
     the summarization prompt is appended as a final user turn.
     """
-    from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
+    from litellm.llms.anthropic.pass_through.adapters.transformation import (
         LiteLLMAnthropicMessagesAdapter,
     )
 

litellm/llms/anthropic/pass_through/messages/agentic_streaming_iterator.py

@@ -336,7 +336,7 @@ class AgenticAnthropicStreamingIterator:
             await task
 
     async def aclose(self) -> None:
-        from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+        from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
             aclose_if_supported,
         )
 
@@ -379,7 +379,7 @@ class AgenticAnthropicStreamingIterator:
             if hasattr(result, "__aiter__"):
                 self._follow_up_iterator = result.__aiter__()
             elif isinstance(result, dict):
-                from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
+                from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
                     FakeAnthropicMessagesStreamIterator,
                 )
                 from litellm.types.llms.anthropic_messages.anthropic_response import (

litellm/llms/anthropic/pass_through/messages/handler.py

@@ -215,7 +215,7 @@ async def _try_websearch_short_circuit(
         if response is not None:
             anthropic_response = cast(AnthropicMessagesResponse, response)
             if stream:
-                from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
+                from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
                     FakeAnthropicMessagesStreamIterator,
                 )
 
@@ -531,7 +531,7 @@ def anthropic_messages_handler(
     # reference the provider cannot resolve. Popped from kwargs so it never reaches the provider.
     skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False)
     if not skip_mcp_handler and tools:
-        from litellm.llms.anthropic.experimental_pass_through.messages.mcp_handler import (
+        from litellm.llms.anthropic.pass_through.messages.mcp_handler import (
             anthropic_messages_with_mcp,
         )
         from litellm.responses.mcp.litellm_proxy_mcp_handler import (

litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py

@@ -67,7 +67,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
         custom_llm_provider: str | None,
         **kwargs,
     ) -> AnthropicMessagesResponse | AsyncIterator:
-        from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
+        from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
             FakeAnthropicMessagesStreamIterator,
         )
 

litellm/llms/anthropic/pass_through/messages/mcp_handler.py

@@ -180,7 +180,7 @@ async def anthropic_messages_with_mcp(
         )
 
     if stream:
-        from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
+        from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
             FakeAnthropicMessagesStreamIterator,
         )
 

litellm/llms/anthropic/pass_through/messages/response_cache.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Final
 import litellm
 from litellm._logging import verbose_logger
 from litellm.caching.caching_handler import create_cache_write_task
-from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
     AnthropicMessagesStreamingResponse,
     BaseAnthropicMessagesStreamingIterator,
     _is_message_stop_chunk,

litellm/llms/anthropic/pass_through/messages/streaming_iterator.py

@@ -16,7 +16,7 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
-from litellm.llms.anthropic.experimental_pass_through.messages.utils import INCOMPLETE_STREAM_ERROR_MESSAGE
+from litellm.llms.anthropic.pass_through.messages.utils import INCOMPLETE_STREAM_ERROR_MESSAGE
 from litellm.proxy.pass_through_endpoints.success_handler import (
     PassThroughEndpointLogging,
 )

litellm/llms/anthropic/pass_through/messages/transformation.py

@@ -590,7 +590,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         litellm_logging_obj: LiteLLMLoggingObj,
     ) -> AsyncIterator:
         """Helper function to handle Anthropic streaming responses using the existing logging handlers"""
-        from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+        from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
             BaseAnthropicMessagesStreamingIterator,
         )
 

litellm/llms/anthropic/pass_through/responses_adapters/handler.py

@@ -109,7 +109,7 @@ def _build_responses_kwargs(
     if isinstance(reasoning, dict):
         effort: Final[object] = reasoning.get("effort")
         if isinstance(effort, str):
-            from litellm.llms.anthropic.experimental_pass_through.utils import (
+            from litellm.llms.anthropic.pass_through.utils import (
                 normalize_reasoning_effort_value,
             )
 

litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py

@@ -15,7 +15,7 @@ from litellm.exceptions import MidStreamFallbackError
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     encrypted_reasoning_signature,
 )
-from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+from litellm.llms.anthropic.pass_through.messages.utils import (
     INCOMPLETE_STREAM_ERROR_MESSAGE,
     refusal_stop_details,
     responses_output_refusal_text,

litellm/llms/anthropic/pass_through/responses_adapters/transformation.py

@@ -20,11 +20,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking_budget,
 )
-from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+from litellm.llms.anthropic.pass_through.messages.utils import (
     refusal_stop_details,
     responses_output_refusal_text,
 )
-from litellm.llms.anthropic.experimental_pass_through.utils import (
+from litellm.llms.anthropic.pass_through.utils import (
     is_reasoning_auto_summary_enabled,
     prompt_cache_key_from_user_id,
 )
@@ -69,7 +69,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
         if raw_usage is None:
             return AnthropicUsage(input_tokens=0, output_tokens=0)
 
-        from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
+        from litellm.llms.anthropic.pass_through.adapters.transformation import (
             LiteLLMAnthropicMessagesAdapter,
         )
         from litellm.responses.utils import ResponseAPILoggingUtils

litellm/llms/anthropic/prompt_cache_prediction.py

@@ -16,7 +16,7 @@ import litellm
 from litellm.llms.anthropic.common_utils import AnthropicModelInfo, is_anthropic_oauth_key
 from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
 from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES
-from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+from litellm.llms.anthropic.pass_through.messages.transformation import (
     DEFAULT_ANTHROPIC_API_VERSION,
     AnthropicMessagesConfig,
 )

litellm/llms/azure_ai/anthropic/messages_transformation.py

@@ -4,7 +4,7 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig
 
 from typing import Any, Final
 
-from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+from litellm.llms.anthropic.pass_through.messages.transformation import (
     AnthropicMessagesConfig,
 )
 from litellm.llms.azure.common_utils import BaseAzureLLM

litellm/llms/base_llm/chat/transformation.py

@@ -4,7 +4,7 @@ Common base config for all LLM providers
 
 import types
 from abc import ABC, abstractmethod
-from collections.abc import AsyncIterator, Iterator
+from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final, Union
 
 import httpx
@@ -255,6 +255,15 @@ class BaseConfig(ABC):
     ) -> dict:
         pass
 
+    def transform_extra_body(
+        self,
+        extra_body: Mapping[str, object],
+        request: Mapping[str, object],
+        model: str,
+        litellm_params: Mapping[str, object],
+    ) -> Mapping[str, object]:
+        return extra_body
+
     def sign_request(
         self,
         headers: dict,
@@ -384,6 +393,8 @@ class BaseConfig(ABC):
         client: AsyncHTTPHandler | None = None,
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
+        *,
+        litellm_params: Mapping[str, object],
     ) -> "CustomStreamWrapper":
         raise NotImplementedError
 
@@ -399,6 +410,8 @@ class BaseConfig(ABC):
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
+        *,
+        litellm_params: Mapping[str, object],
     ) -> "CustomStreamWrapper":
         raise NotImplementedError
 

litellm/llms/base_llm/responses/transformation.py

@@ -1,5 +1,6 @@
 import types
 from abc import ABC, abstractmethod
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final, cast
 
 import httpx
@@ -364,6 +365,15 @@ class BaseResponsesAPIConfig(ABC):
                 out.append(item)
         return cast(ResponseInputParam, out)
 
+    def transform_extra_body(
+        self,
+        extra_body: Mapping[str, object],
+        request: Mapping[str, object],
+        model: str,
+        litellm_params: GenericLiteLLMParams,
+    ) -> Mapping[str, object]:
+        return extra_body
+
     @staticmethod
     def normalize_responses_api_request_dict(data: dict[str, Any]) -> dict[str, Any]:
         """Apply provider-agnostic fixes to an outbound Responses API request dict."""

litellm/llms/bedrock/chat/agentcore/transformation.py

@@ -5,7 +5,7 @@ https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgen
 """
 
 import json
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Mapping
 from typing import TYPE_CHECKING, Any, Final, Optional, Union
 from urllib.parse import quote
 
@@ -643,6 +643,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         client: Union[HTTPHandler, "AsyncHTTPHandler"] | None = None,
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
+        *,
+        litellm_params: Mapping[str, object],
     ) -> "CustomStreamWrapper":
         """
         Simplified sync streaming - returns a generator that yields ModelResponse chunks.
@@ -862,6 +864,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         client: Optional["AsyncHTTPHandler"] = None,
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
+        *,
+        litellm_params: Mapping[str, object],
     ) -> "CustomStreamWrapper":
         """
         Simplified async streaming - returns an async generator that yields ModelResponse chunks.

litellm/llms/bedrock/chat/converse_handler.py

@@ -7,6 +7,7 @@ import litellm
 from litellm.anthropic_beta_headers_manager import (
     update_headers_with_filtered_beta,
 )
+from litellm.litellm_core_utils.get_litellm_params import stored_control_options
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -18,7 +19,7 @@ from litellm.types.utils import ModelResponse
 from litellm.utils import CustomStreamWrapper
 
 from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token, pop_aws_auth_params, run_aws_signing
-from ..common_utils import BedrockError, _get_all_bedrock_regions, error_response_text, stream_chunk_size_from
+from ..common_utils import BedrockError, _get_all_bedrock_regions, error_response_text
 from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
 
 
@@ -280,7 +281,7 @@ class BedrockConverseLLM(BaseAWSLLM):
     ):
         ## SETUP ##
         stream: Final = optional_params.pop("stream", None)
-        stream_chunk_size: Final = stream_chunk_size_from(litellm_params) if stream is True else None
+        stream_chunk_size: Final = stored_control_options(litellm_params).stream_chunk_size if stream is True else None
         unencoded_model_id: Final = optional_params.pop("model_id", None)
         fake_stream = optional_params.pop("fake_stream", False)
         json_mode: Final = optional_params.get("json_mode", False)

litellm/llms/bedrock/chat/converse_transformation.py

@@ -1831,7 +1831,7 @@ class AmazonConverseConfig(BaseConfig):
         anthropic_beta_list: list,
     ) -> None:
         """Keep only compact_20260112 edits for Bedrock; add beta header or drop field."""
-        from litellm.llms.anthropic.experimental_pass_through.context_management.constants import (
+        from litellm.llms.anthropic.pass_through.context_management.constants import (
             COMPACT_EDIT_TYPE,
         )
         from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py

@@ -225,7 +225,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
 
         anthropic_request.pop("model", None)
         anthropic_request.pop("stream", None)
-        anthropic_request.pop("stream_chunk_size", None)
         apply_bedrock_invoke_structured_output(
             model=model,
             request_body=anthropic_request,

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -1,6 +1,7 @@
 import copy
 import json
 import time
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final, cast, get_args
 
 import httpx
@@ -9,6 +10,7 @@ from pydantic import TypeAdapter, ValidationError
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.litellm_core_utils.get_litellm_params import stored_control_options
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
 from litellm.litellm_core_utils.prompt_templates.factory import (
     cohere_message_pt,
@@ -18,7 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
 )
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.bedrock.chat.invoke_handler import make_call, make_sync_call
-from litellm.llms.bedrock.common_utils import BedrockError, stream_chunk_size_from
+from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.llms.bedrock.request_metadata import (
     bedrock_request_metadata_headers,
     merge_bedrock_invoke_headers,
@@ -180,7 +182,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
     ) -> dict:
         ## SETUP ##
         stream: Final = optional_params.pop("stream", None)
-        optional_params.pop("stream_chunk_size", None)
         custom_prompt_dict: Final[dict] = litellm_params.pop("custom_prompt_dict", None) or {}
         hf_model_name: Final = litellm_params.get("hf_model_name", None)
 
@@ -452,8 +453,10 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         client: AsyncHTTPHandler | None = None,
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
+        *,
+        litellm_params: Mapping[str, object],
     ) -> CustomStreamWrapper:
-        chunk_size: Final = stream_chunk_size_from(logging_obj.litellm_params)
+        chunk_size: Final = stored_control_options(litellm_params).stream_chunk_size
         completion_stream, response_headers = await make_call(
             client=client,
             api_base=api_base,
@@ -489,11 +492,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
+        *,
+        litellm_params: Mapping[str, object],
     ) -> CustomStreamWrapper:
         sync_client: Final = (
             _get_httpx_client(params={}) if client is None or isinstance(client, AsyncHTTPHandler) else client
         )
-        chunk_size: Final = stream_chunk_size_from(logging_obj.litellm_params)
+        chunk_size: Final = stored_control_options(litellm_params).stream_chunk_size
         completion_stream, response_headers = make_sync_call(
             client=sync_client,
             api_base=api_base,

litellm/llms/bedrock/claude_platform/messages_transformation.py

@@ -1,7 +1,7 @@
 from typing import Any, Final
 
 import litellm
-from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+from litellm.llms.anthropic.pass_through.messages.transformation import (
     DEFAULT_ANTHROPIC_API_VERSION,
     AnthropicMessagesConfig,
 )

litellm/llms/bedrock/common_utils.py

@@ -18,7 +18,7 @@ if TYPE_CHECKING:
     from litellm.types.llms.bedrock import BedrockCreateBatchRequest
 
 import httpx
-from pydantic import ConfigDict, TypeAdapter, ValidationError
+from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm import verbose_logger
@@ -86,15 +86,6 @@ class BedrockError(BaseLLMException):
 
 
 _BEDROCK_AWS_AUTH_PARAMETER_KEYS: Final[tuple[str, ...]] = (*AWS_AUTH_PARAM_KEYS, "aws_region_name")
-_STREAM_CHUNK_SIZE_VALIDATOR: Final[TypeAdapter[int | None]] = TypeAdapter(int | None, config=ConfigDict(strict=True))
-
-
-def stream_chunk_size_from(litellm_params: Mapping[str, object]) -> int | None:
-    raw: Final = litellm_params.get("stream_chunk_size")
-    try:
-        return _STREAM_CHUNK_SIZE_VALIDATOR.validate_python(raw)
-    except ValidationError as e:
-        raise BedrockError(status_code=400, message=f"Invalid stream_chunk_size={raw!r}. Expected int. Error: {e}")
 
 
 def merge_bedrock_aws_request_params(

litellm/llms/bedrock/files/transformation.py

@@ -58,7 +58,7 @@ from litellm.types.llms.openai import (
     OpenAIFileObject,
     PathLike,
 )
-from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums, all_litellm_params
+from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums, is_litellm_owned_kwarg
 from litellm.utils import get_llm_provider, get_optional_params
 
 from ..base_aws_llm import BaseAWSLLM
@@ -907,7 +907,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
                     {
                         k: v
                         for k, v in optional_params.items()
-                        if k not in all_litellm_params or k in _LITELLM_PARAMS_THE_MAPPER_TAKES
+                        if not is_litellm_owned_kwarg(k) or k in _LITELLM_PARAMS_THE_MAPPER_TAKES
                     }
                 ),
             )

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -18,7 +18,7 @@ from litellm.llms.anthropic.chat.transformation import (
     AnthropicConfig,
 )
 from litellm.llms.anthropic.common_utils import AnthropicModelInfo
-from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+from litellm.llms.anthropic.pass_through.messages.transformation import (
     AnthropicMessagesConfig,
 )
 from litellm.llms.base_llm.anthropic_messages.transformation import (
@@ -798,7 +798,7 @@ class AmazonAnthropicClaudeMessagesConfig(
         merge them from ``message_start`` so logging/cost sees a consistent usage
         object (fixes negative input costs: LIT-2411).
         """
-        from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+        from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
             BaseAnthropicMessagesStreamingIterator,
         )
 

litellm/llms/bedrock/messages/mantle_transformation.py

@@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Any, Final
 import httpx
 from pydantic import TypeAdapter
 
-from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+from litellm.llms.anthropic.pass_through.messages.transformation import (
     DEFAULT_ANTHROPIC_API_VERSION,
     AnthropicMessagesConfig,
 )

litellm/llms/bytez/chat/transformation.py

@@ -1,6 +1,7 @@
 import json
 import time
 import traceback
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
@@ -258,6 +259,8 @@ class BytezChatConfig(BaseConfig):
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
+        *,
+        litellm_params: Mapping[str, object],
     ) -> "BytezCustomStreamWrapper":
         if client is None or isinstance(client, AsyncHTTPHandler):
             client = _get_httpx_client(params={})
@@ -300,6 +303,8 @@ class BytezChatConfig(BaseConfig):
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
+        *,
+        litellm_params: Mapping[str, object],
     ) -> "BytezCustomStreamWrapper":
         if client is None or isinstance(client, HTTPHandler):
             client = get_async_httpx_client(llm_provider=LlmProviders.BYTEZ, params={})

litellm/llms/custom_httpx/llm_http_handler.py

@@ -204,7 +204,7 @@ if TYPE_CHECKING:
     from litellm.integrations.custom_logger import CustomLogger
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
-    from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
+    from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
         FakeAnthropicMessagesStreamIterator,
     )
     from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
@@ -649,7 +649,16 @@ class BaseLLMHTTPHandler:
         def sign_and_log(
             transformed: dict[str, object],  # mutable-ok: async_completion takes dict
         ) -> tuple[dict[str, object], dict[str, object], bytes | None]:  # mutable-ok: async_completion takes dict
-            data: Final = {**transformed, **extra_body} if extra_body is not None else transformed
+            data: Final = (
+                {
+                    **transformed,
+                    **provider_config.transform_extra_body(
+                        extra_body=extra_body, request=transformed, model=model, litellm_params=litellm_params
+                    ),
+                }
+                if extra_body is not None
+                else transformed
+            )
             signed: Final = cast(  # cast-ok: sign_request is declared as a bare dict
                 "tuple[dict[str, object], bytes | None]",
                 provider_config.sign_request(
@@ -781,6 +790,7 @@ class BaseLLMHTTPHandler:
                     messages=messages,
                     client=client,
                     json_mode=json_mode,
+                    litellm_params=litellm_params,
                 )
             completion_stream, headers = self.make_sync_call(
                 provider_config=provider_config,
@@ -944,6 +954,7 @@ class BaseLLMHTTPHandler:
                 client=client,
                 json_mode=json_mode,
                 signed_json_body=signed_json_body,
+                litellm_params=litellm_params,
             )
 
         completion_stream, _response_headers = await self.make_async_call_stream_helper(
@@ -2115,7 +2126,7 @@ class BaseLLMHTTPHandler:
 
         initial_response: AsyncIterator | AnthropicMessagesResponse
         if stream:
-            from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+            from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
                 AnthropicMessagesStreamingResponse,
                 anthropic_messages_stream_hidden_params,
             )
@@ -2139,7 +2150,7 @@ class BaseLLMHTTPHandler:
                     hidden_params=stream_hidden_params,
                 )
 
-            from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
+            from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import (
                 AgenticAnthropicStreamingIterator,
             )
 
@@ -2421,7 +2432,11 @@ class BaseLLMHTTPHandler:
         data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
 
         if extra_body:
-            data.update(extra_body)
+            data.update(
+                responses_api_provider_config.transform_extra_body(
+                    extra_body=extra_body, request=data, model=model, litellm_params=litellm_params
+                )
+            )
         stream = bool(stream or data.get("stream"))
 
         # Preserve the OpenAI-style request context (not sent to the provider) for streaming
@@ -2609,7 +2624,11 @@ class BaseLLMHTTPHandler:
         data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
 
         if extra_body:
-            data.update(extra_body)
+            data.update(
+                responses_api_provider_config.transform_extra_body(
+                    extra_body=extra_body, request=data, model=model, litellm_params=litellm_params
+                )
+            )
         stream = bool(stream or data.get("stream"))
 
         # Preserve the OpenAI-style request context (not sent to the provider) for streaming
@@ -5523,7 +5542,7 @@ class BaseLLMHTTPHandler:
             from typing import cast
 
             from litellm._logging import verbose_logger
-            from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
+            from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
                 FakeAnthropicMessagesStreamIterator,
             )
             from litellm.types.llms.anthropic_messages.anthropic_response import (

litellm/llms/deepseek/messages/transformation.py

@@ -5,7 +5,7 @@ DeepSeek Anthropic-compatible messages transformation config.
 from typing import Any, Final
 
 import litellm
-from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+from litellm.llms.anthropic.pass_through.messages.transformation import (
     AnthropicMessagesConfig,
 )
 from litellm.secret_managers.main import get_secret_str

litellm/llms/elevenlabs/text_to_speech/transformation.py

@@ -18,7 +18,7 @@ from litellm.llms.base_llm.text_to_speech.transformation import (
     TextToSpeechRequestData,
 )
 from litellm.secret_managers.main import get_secret_str
-from litellm.types.utils import all_litellm_params
+from litellm.types.utils import is_litellm_owned_kwarg
 
 from ..common_utils import ElevenLabsException
 
@@ -241,7 +241,7 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig):
                 continue
             mapped_params[key] = value
 
-        reserved_kwarg_keys: Final = set(all_litellm_params) | {
+        reserved_kwarg_keys: Final = {
             self.ELEVENLABS_QUERY_PARAMS_KEY,
             self.ELEVENLABS_VOICE_ID_KEY,
             "voice",
@@ -260,7 +260,7 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig):
                 mapped_params[key] = value
 
         for key in list(kwargs.keys()):
-            if key in reserved_kwarg_keys:
+            if key in reserved_kwarg_keys or is_litellm_owned_kwarg(key):
                 continue
             value = kwargs[key]
             if value is None:

litellm/llms/github_copilot/messages/transformation.py

@@ -1,7 +1,7 @@
 from typing import Any, Final
 
 from litellm.exceptions import AuthenticationError
-from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+from litellm.llms.anthropic.pass_through.messages.transformation import (
     AnthropicMessagesConfig,
 )
 

litellm/llms/langgraph/chat/transformation.py

@@ -9,6 +9,7 @@ Non-streaming endpoint: POST /runs/wait
 """
 
 import json
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final, Optional, Union, cast
 
 import httpx
@@ -285,6 +286,8 @@ class LangGraphConfig(BaseConfig):
         client: Union[HTTPHandler, "AsyncHTTPHandler"] | None = None,
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
+        *,
+        litellm_params: Mapping[str, object],
     ) -> CustomStreamWrapper:
         """
         Get a CustomStreamWrapper for synchronous streaming.
@@ -344,6 +347,8 @@ class LangGraphConfig(BaseConfig):
         client: Optional["AsyncHTTPHandler"] = None,
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
+        *,
+        litellm_params: Mapping[str, object],
     ) -> CustomStreamWrapper:
         """
         Get a CustomStreamWrapper for asynchronous streaming.