chore: merge main for MCP OAuth CI compatibility

.circleci/config.yml

@@ -1,10 +1,33 @@
 version: 2.1
+parameters:
+  run_migration_tests:
+    type: boolean
+    default: false
+  migration_candidate_image:
+    type: string
+    default: ""
+  migration_source_sha:
+    type: string
+    default: ""
 orbs:
   codecov: codecov/codecov@4.0.1
   node: circleci/node@5.1.0 # Add this line to declare the node orb
   win: circleci/windows@5.0 # Add Windows orb
 
 commands:
+  checkout_migration_source:
+    steps:
+      - run:
+          name: Select the requested migration test revision
+          environment:
+            MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >>
+          command: |
+            revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}"
+            [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
+            git init
+            git remote add origin https://github.com/BerriAI/litellm.git
+            git fetch --depth 1 origin "$revision"
+            git checkout --detach FETCH_HEAD
   skip_if_unrelated_changes:
     parameters:
       category:
@@ -2871,20 +2894,41 @@ jobs:
           destination: e2e-server-root-path-playwright-report
 
   build_docker_database_image:
+    parameters:
+      migration_qualification:
+        type: boolean
+        default: false
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
     steps:
-      - checkout
-      - skip_if_unrelated_changes
+      - when:
+          condition: << parameters.migration_qualification >>
+          steps:
+            - checkout_migration_source
+      - unless:
+          condition: << parameters.migration_qualification >>
+          steps:
+            - checkout
+            - skip_if_unrelated_changes
 
       - run:
           name: Build Docker image
+          environment:
+            MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >>
           command: |
-            docker build \
-              -t litellm-docker-database:ci \
-              -f docker/Dockerfile.database .
+            if [ -n "$MIGRATION_CANDIDATE_IMAGE" ]; then
+              [[ "$MIGRATION_CANDIDATE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+@sha256:[0-9a-f]{64}$ ]] || exit 1
+              docker pull "$MIGRATION_CANDIDATE_IMAGE"
+              docker tag "$MIGRATION_CANDIDATE_IMAGE" litellm-docker-database:ci
+            else
+              docker build \
+                --label org.opencontainers.image.revision="$(git rev-parse HEAD)" \
+                -t litellm-docker-database:ci \
+                -f docker/Dockerfile.database .
+            fi
+            python3 .circleci/scripts/run_migration_tests.py record-image
 
       - run:
           name: Save Docker image to workspace root
@@ -2895,6 +2939,78 @@ jobs:
           root: .
           paths:
             - litellm-docker-database.tar.zst
+            - migration-image.json
+
+  migration_startup_tests:
+    parameters:
+      suite:
+        type: enum
+        enum: [startup, recovery, legacy]
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    environment:
+      LITELLM_MIGRATION_TESTS: "1"
+      LITELLM_MIGRATION_TEST_IMAGE: litellm-docker-database:ci
+      MIGRATION_TEST_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres
+      MIGRATION_TEST_CONTAINER_ADMIN_URL: postgresql://postgres:postgres@host.docker.internal:5432/postgres
+      MIGRATION_TEST_OUTPUT: /tmp/migration-results
+      PYTHONPATH: tests/e2e
+    steps:
+      - checkout_migration_source
+      - install_uv
+      - install_rust
+      - restore_cache:
+          keys:
+            - v1-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Install test dependencies
+          command: uv sync --frozen --all-groups --all-extras --python 3.12
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: Load the shared candidate and start PostgreSQL
+          command: |
+            zstd -d litellm-docker-database.tar.zst --stdout | docker load
+            docker run -d --name migration-postgres \
+              -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \
+              -p 5432:5432 \
+              postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "60"
+      - run:
+          name: Run migration startup regressions
+          environment:
+            MIGRATION_TEST_SUITE: << parameters.suite >>
+            MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >>
+          command: |
+            mkdir -p /tmp/migration-results
+            uv run --no-sync python .circleci/scripts/run_migration_tests.py
+          no_output_timeout: 15m
+      - store_test_results:
+          path: /tmp/migration-results/junit
+      - run:
+          name: Package migration diagnostics
+          when: always
+          command: |
+            mkdir -p /tmp/migration-artifacts
+            if [ -d /tmp/migration-results ]; then
+              tar -czf /tmp/migration-artifacts/diagnostics.tar.gz -C /tmp/migration-results .
+              if [ -f /tmp/migration-results/verdict.json ]; then
+                cp /tmp/migration-results/verdict.json /tmp/migration-artifacts/verdict.json
+              fi
+            fi
+      - store_artifacts:
+          path: /tmp/migration-artifacts
+          destination: migration-results
+      - run:
+          name: Remove migration test containers
+          when: always
+          command: |
+            docker ps -aq --filter label=litellm-migration-test=true | xargs -r docker rm -f
+            docker rm -f migration-postgres || true
 
   test_bad_database_url:
     machine:
@@ -3026,8 +3142,61 @@ jobs:
       - store_artifacts:
           path: test-results
 
+  unit:
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    steps:
+      - setup_litellm_test_deps
+      - run:
+          name: Generate Prisma client
+          command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+      - run:
+          name: Run unit tests
+          command: |
+            mkdir -p test-results/unit
+            mapfile -t files < <(find tests/unit -name 'test_*.py' | sort)
+            if [ "${#files[@]}" -eq 0 ]; then echo "tests/unit holds no test_*.py files; nothing to run"; exit 0; fi
+            set +e
+            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --junitxml=test-results/unit/junit.xml
+            status=$?
+            set -e
+            if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
+            exit "$status"
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
 workflows:
+  migration_startup:
+    when: << pipeline.parameters.run_migration_tests >>
+    jobs: &migration_jobs
+      - build_docker_database_image:
+          migration_qualification: true
+      - migration_startup_tests:
+          name: migration-startup
+          suite: startup
+          requires: [build_docker_database_image]
+      - migration_startup_tests:
+          name: migration-recovery
+          suite: recovery
+          requires: [build_docker_database_image]
+      - migration_startup_tests:
+          name: migration-legacy-and-pooling
+          suite: legacy
+          requires: [build_docker_database_image]
+  migration_startup_scheduled:
+    triggers:
+      - schedule:
+          cron: "17 0,6,12,18 * * *"
+          filters:
+            branches:
+              only: litellm_internal_staging
+    jobs: *migration_jobs
   integration:
+    unless: << pipeline.parameters.run_migration_tests >>
     jobs:
       - integration_contracts:
           name: integration-<< matrix.suite >>
@@ -3040,6 +3209,7 @@ workflows:
                 - main
                 - /litellm_.*/
   build_and_test:
+    unless: << pipeline.parameters.run_migration_tests >>
     jobs:
       - using_litellm_on_windows:
           filters: &main_branches
@@ -3047,6 +3217,8 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - unit:
+          filters: *main_branches
       - provider_replay_harness
       - base_sdk_install:
           filters: *main_branches

.circleci/scripts/run_migration_tests.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import Final
+from xml.etree import ElementTree
+
+SUITES: Final = {
+    "startup": (("test_startup.py",), 12),
+    "recovery": (("test_recovery.py",), 15),
+    "legacy": (("test_legacy.py", "test_pooling.py"), 11),
+}
+
+
+def successful_junit(path: Path, expected: int, exit_code: int) -> bool:
+    if exit_code != 0 or not path.is_file():
+        return False
+    try:
+        root: Final = ElementTree.parse(path).getroot()
+    except ElementTree.ParseError:
+        return False
+    cases: Final = tuple(root.iter("testcase"))
+    identities: Final = frozenset((case.get("classname"), case.get("name")) for case in cases)
+    return len(cases) == len(identities) == expected and all(
+        not any(case.find(tag) is not None for tag in ("failure", "error", "skipped")) for case in cases
+    )
+
+
+def output(*command: str) -> str:
+    return subprocess.check_output(command, text=True, timeout=90).strip()
+
+
+def record_image() -> None:
+    source: Final = output("git", "rev-parse", "HEAD")
+    image: Final = output("docker", "image", "inspect", "litellm-docker-database:ci", "--format", "{{.Id}}")
+    revision: Final = output(
+        "docker",
+        "image",
+        "inspect",
+        "litellm-docker-database:ci",
+        "--format",
+        '{{index .Config.Labels "org.opencontainers.image.revision"}}',
+    )
+    assert re.fullmatch(r"[0-9a-f]{40}", source), "Invalid source revision"
+    assert revision == source, "Candidate image revision differs from the tested source"
+    Path("migration-image.json").write_text(
+        json.dumps(
+            {
+                "source_sha": source,
+                "image_id": image,
+                "candidate_image": os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""),
+            }
+        )
+    )
+
+
+def main() -> int:
+    suite: Final = os.environ["MIGRATION_TEST_SUITE"]
+    files, expected = SUITES[suite]
+    metadata: Final = json.loads(Path("migration-image.json").read_text())
+    assert metadata["source_sha"] == output("git", "rev-parse", "HEAD"), "Image and test source revisions differ"
+    assert metadata["image_id"] == output(
+        "docker", "image", "inspect", os.environ["LITELLM_MIGRATION_TEST_IMAGE"], "--format", "{{.Id}}"
+    ), "Loaded image differs from the build output"
+    assert metadata["candidate_image"] == os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""), "Wrong release candidate"
+    destination: Final = Path(os.environ["MIGRATION_TEST_OUTPUT"])
+    junit: Final = destination / "junit" / "results.xml"
+    junit.parent.mkdir(parents=True, exist_ok=True)
+    result: Final = subprocess.run(
+        (
+            sys.executable,
+            "-m",
+            "pytest",
+            *(f"tests/e2e/migrations/{name}" for name in files),
+            "-vv",
+            "--tb=short",
+            "--durations=10",
+            f"--junitxml={junit}",
+            "-o",
+            "addopts=",
+            "--reruns=0",
+        ),
+        check=False,
+        timeout=1200,
+    )
+    passed: Final = successful_junit(junit, expected, result.returncode)
+    (destination / "verdict.json").write_text(
+        json.dumps(
+            {
+                **metadata,
+                "suite": suite,
+                "expected_cases": expected,
+                "passed": passed,
+                "pytest_exit_code": result.returncode,
+                "test_revision": metadata["source_sha"],
+                "workflow_id": os.environ.get("CIRCLE_WORKFLOW_ID", ""),
+                "job_number": os.environ.get("CIRCLE_BUILD_NUM", ""),
+            },
+            indent=2,
+        )
+    )
+    return 0 if passed else 1
+
+
+if __name__ == "__main__":
+    if sys.argv[1:] == ["record-image"]:
+        record_image()
+    else:
+        raise SystemExit(main())

.github/e2e-stack/assert_tests_ran.py

@@ -1,3 +1,5 @@
+import os
+import re
 import sys
 import xml.etree.ElementTree as ET
 from pathlib import Path
@@ -15,6 +17,7 @@ def main() -> int:
         _ = sys.stdout.write("::error::could not read the test execution report\n")
         return 1
     cases: Final = tuple(report.iter("testcase"))
+    expected_count: Final = os.environ.get("E2E_REQUIRED_TEST_COUNT")
     passed: Final = frozenset(
         case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
     )
@@ -35,9 +38,22 @@ def main() -> int:
         skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
         _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
         for case in cases:
-            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error")):
+            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error", "skipped")):
                 continue
-            _ = sys.stdout.write(f"  failed: {case.get('classname', '')}::{case.get('name', '')}\n")
+            outcome = "skipped" if case.find("skipped") is not None else "failed"
+            _ = sys.stdout.write(f"  {outcome}: {case.get('classname', '')}::{case.get('name', '')}\n")
+            for prop in case.findall("./properties/property"):
+                name = prop.get("name", "")
+                value = prop.get("value", "")
+                if name in ("oauth_failure_phase", "oauth_exception_type", "oauth_frame") and re.fullmatch(
+                    r"[A-Za-z0-9_.:<>-]{1,240}", value
+                ):
+                    _ = sys.stdout.write(f"    {name}: {value}\n")
+    if expected_count is not None and (
+        len(cases) != int(expected_count) or any(case.find("skipped") is not None for case in cases)
+    ):
+        _ = sys.stdout.write("::error::required test count was not met or a required case was skipped\n")
+        return 1
     if (
         selected
         and not missing

.github/e2e-stack/select_tests.py

@@ -4,7 +4,8 @@ from typing import Final
 
 SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
 UNSUPPORTED: Final = re.compile(
-    r"^tests/e2e/(ui|claude_code|load)/"
+    r"^tests/e2e/(ui|claude_code|load|migrations)/"
+    r"|^tests/e2e/mcp/test_mcp_oauth_happy_path_e2e\.py$"
     r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
     r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
     r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"

.github/template.yaml

@@ -21,7 +21,7 @@ Parameters:
   WorkerConfigParameter:
     Type: String
     Description: Sample environment variable
-    Default: '{"model": null, "alias": null, "api_base": null, "api_version": "2023-07-01-preview", "debug": false, "temperature": null, "max_tokens": null, "request_timeout": 600, "max_budget": null, "telemetry": true, "drop_params": false, "add_function_to_prompt": false, "headers": null, "save": false, "config": null, "use_queue": false}'
+    Default: '{"model": null, "alias": null, "api_base": null, "api_version": "2023-07-01-preview", "debug": false, "temperature": null, "max_tokens": null, "request_timeout": 600, "max_budget": null, "drop_params": false, "add_function_to_prompt": false, "headers": null, "save": false, "config": null, "use_queue": false}'
 
 Resources:
   MyUrlFunctionPermissions:

.github/workflows/_test-unit-base.yml

@@ -165,33 +165,41 @@ jobs:
           DIST: ${{ inputs.dist }}
           COVERAGE_CORE: sysmon
         run: |
-          if [ "${WORKERS}" = "0" ]; then
-            uv run --no-sync pytest ${TEST_PATH:?} \
-              --tb=short -vv \
-              --maxfail="${MAX_FAILURES}" \
-              --reruns "${RERUNS}" \
-              --reruns-delay 1 \
-              --timeout="${TEST_TIMEOUT_SECONDS}" \
-              --rerun-except "from pytest-timeout" \
-              --durations=20 \
-              --cov=./litellm --cov=./enterprise/litellm_enterprise \
-              --cov-report=xml:coverage.xml \
-              --cov-config=pyproject.toml
-          else
-            uv run --no-sync pytest ${TEST_PATH:?} \
-              --tb=short -vv \
-              --maxfail="${MAX_FAILURES}" \
-              -n "${WORKERS}" \
-              --reruns "${RERUNS}" \
-              --reruns-delay 1 \
-              --timeout="${TEST_TIMEOUT_SECONDS}" \
-              --rerun-except "from pytest-timeout" \
-              --dist="${DIST}" \
-              --durations=20 \
-              --cov=./litellm --cov=./enterprise/litellm_enterprise \
-              --cov-report=xml:coverage.xml \
-              --cov-config=pyproject.toml
+          found_path=false
+          for path in ${TEST_PATH}; do
+            if [ -e "${path%%::*}" ]; then
+              found_path=true
+              break
+            fi
+          done
+          if [ "$found_path" = false ]; then
+            echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run"
+            exit 0
           fi
+          xdist_args=()
+          if [ "${WORKERS}" != "0" ]; then
+            xdist_args=(-n "${WORKERS}" --dist="${DIST}")
+          fi
+          set +e
+          uv run --no-sync pytest ${TEST_PATH:?} \
+            --tb=short -vv \
+            --maxfail="${MAX_FAILURES}" \
+            "${xdist_args[@]}" \
+            --reruns "${RERUNS}" \
+            --reruns-delay 1 \
+            --timeout="${TEST_TIMEOUT_SECONDS}" \
+            --rerun-except "from pytest-timeout" \
+            --durations=20 \
+            --cov=./litellm --cov=./enterprise/litellm_enterprise \
+            --cov-report=xml:coverage.xml \
+            --cov-config=pyproject.toml
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "pytest collected no tests from ${TEST_PATH}; passing"
+            exit 0
+          fi
+          exit "$status"
 
       - name: Save coverage report
         if: always() && steps.changes.outputs.decision != 'skip'

.github/workflows/test-mcp-oauth-e2e.yml

@@ -0,0 +1,180 @@
+name: MCP OAuth happy path
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/test-mcp-oauth-e2e.yml'
+      - '.github/e2e-stack/**'
+      - 'tests/e2e/*.py'
+      - 'tests/e2e/pytest.ini'
+      - 'tests/e2e/idp_realm.json'
+      - 'tests/e2e/mcp/**'
+      - 'litellm/experimental_mcp_client/**'
+      - 'litellm/proxy/_experimental/mcp_server/**'
+      - 'litellm/proxy/auth/**'
+      - 'litellm/proxy/management_endpoints/mcp_management_endpoints.py'
+      - 'litellm/proxy/_types.py'
+      - 'litellm/types/mcp_server/mcp_server_manager.py'
+      - 'litellm/proxy/management_endpoints/*sso*.py'
+      - 'litellm/proxy/management_endpoints/sso/**'
+      - 'litellm/proxy/common_utils/encrypt_decrypt_utils.py'
+      - 'litellm/proxy/proxy_server.py'
+      - 'litellm/proxy/schema.prisma'
+      - 'ui/litellm-dashboard/src/app/connect/**'
+      - 'ui/litellm-dashboard/src/app/mcp/oauth/**'
+      - 'pyproject.toml'
+      - 'uv.lock'
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: mcp-oauth-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  oauth:
+    if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    environment: e2e-changed
+    timeout-minutes: 45
+    permissions:
+      contents: read
+      id-token: write
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: litellm
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U litellm"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_HOST: 127.0.0.1
+      DATABASE_PORT: '5432'
+      DATABASE_USER: litellm
+      DATABASE_PASSWORD: dbpassword9090
+      DATABASE_NAME: litellm
+      DATABASE_URL: postgresql://litellm:dbpassword9090@127.0.0.1:5432/litellm
+      E2E_KEYCLOAK_URL: http://127.0.0.1:8081
+      E2E_KEYCLOAK_ADMIN_USER: admin
+      E2E_KEYCLOAK_ADMIN_PASSWORD: e2e-ephemeral-idp-not-a-secret
+      E2E_FIXTURE_MODE: live
+      E2E_PROVIDER_CACHE: '0'
+      E2E_MCP_OAUTH_LIVE: '1'
+      E2E_REQUIRED_TEST_COUNT: '4'
+    steps:
+      - name: Checkout the tested source
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+
+      - name: Require and materialize the upstream login
+        env:
+          STORAGE_STATE: ${{ secrets.E2E_LINEAR_STORAGE_STATE_B64 }}
+        run: |
+          umask 077
+          python3 - <<'PY'
+          import base64
+          import json
+          import os
+          import secrets
+          from pathlib import Path
+          encoded = os.environ.get("STORAGE_STATE", "")
+          if not encoded:
+              raise SystemExit("E2E_LINEAR_STORAGE_STATE_B64 is required; capture and provision a test-account login")
+          state = json.loads(base64.b64decode(encoded, validate=True))
+          if not isinstance(state, dict) or not state.get("cookies"):
+              raise SystemExit("The captured login must contain browser cookies")
+          directory = Path(os.environ["RUNNER_TEMP"]) / "mcp-oauth-private"
+          directory.mkdir(mode=0o700)
+          path = directory / "linear-state.json"
+          path.write_text(json.dumps(state))
+          with open(os.environ["GITHUB_ENV"], "a") as output:
+              output.write(f"E2E_LINEAR_STORAGE_STATE={path}\n")
+              for name in ("LITELLM_MASTER_KEY", "LITELLM_SALT_KEY"):
+                  value = "sk-e2e-" + secrets.token_hex(24)
+                  print(f"::add-mask::{value}")
+                  output.write(f"{name}={value}\n")
+          PY
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.13'
+      - uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: '0.10.9'
+      - uses: ./.github/actions/cache-cargo-build
+      - name: Install the frozen E2E environment
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra proxy-runtime --extra extra_proxy --group ci --group proxy-dev --group e2e-dev
+          uv run --no-sync python scripts/prisma_generate_if_needed.py
+          uv run --no-sync playwright install --with-deps chromium
+
+      - name: Configure license access
+        id: aws
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+        with:
+          role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+          aws-region: us-east-1
+          role-session-name: mcp-oauth-${{ github.run_id }}
+          role-duration-seconds: 900
+          output-env-credentials: false
+          output-credentials: true
+      - name: Load the E2E license
+        env:
+          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          license="$(aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license --query SecretString --output text)"
+          test -n "${license}"
+          echo "::add-mask::${license}"
+          echo "LITELLM_LICENSE=${license}" >> "${GITHUB_ENV}"
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version-file: ui/litellm-dashboard/.nvmrc
+      - name: Build the gateway consent UI at the tested commit
+        run: |
+          cd ui/litellm-dashboard
+          ../../scripts/with_dashboard_node.sh npm ci
+          ../../scripts/with_dashboard_node.sh npm run build
+          mkdir -p ../../litellm/proxy/_experimental/out
+          cp -r out/. ../../litellm/proxy/_experimental/out/
+          find ../../litellm/proxy/_experimental/out -name '*.html' ! -name index.html | while read -r page; do
+            mkdir -p "${page%.html}"
+            mv "${page}" "${page%.html}/index.html"
+          done
+
+      - name: Prepare the isolated database and IdP
+        run: |
+          umask 077
+          bash .github/e2e-stack/start-idp.sh
+          uv run --no-sync python migrations/run.py > "${RUNNER_TEMP}/mcp-oauth-private/migrations.log" 2>&1
+
+      - name: Run every required OAuth variant without retries
+        run: |
+          umask 077
+          uv run --no-sync pytest -c tests/e2e/pytest.ini tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py \
+            --rootdir=. --reruns 0 --tb=short -o junit_family=xunit1 \
+            --junitxml="${RUNNER_TEMP}/mcp-oauth-private/results.xml" \
+            > "${RUNNER_TEMP}/mcp-oauth-private/pytest.log" 2>&1
+      - name: Report JUnit results and reject skipped or missing cases
+        if: always()
+        run: |
+          uv run --no-sync python .github/e2e-stack/assert_tests_ran.py \
+            "${RUNNER_TEMP}/mcp-oauth-private/results.xml" tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py
+      - name: Remove private login and logs
+        if: always()
+        run: |
+          docker rm -f e2e-keycloak >/dev/null 2>&1 || true
+          rm -rf "${RUNNER_TEMP}/mcp-oauth-private"

cookbook/livekit_agent_sdk/config.example.yaml

@@ -15,7 +15,6 @@ model_list:
 
 litellm_settings:
   drop_params: True
-  telemetry: False
 
 general_settings:
   master_key: sk-1234  # Change this to a secure key

cookbook/misc/config.yaml

@@ -55,7 +55,6 @@ litellm_settings:
   # budget_duration: 30d
   num_retries: 5
   request_timeout: 600
-  telemetry: False
   context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}]
 
 general_settings: 

litellm-proxy-extras/litellm_proxy_extras/migration_lock.py

@@ -0,0 +1,89 @@
+import random
+import time
+from collections.abc import Generator, Mapping
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Final
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from litellm_proxy_extras._logging import logger
+from litellm_proxy_extras.prisma_toolchain import MIGRATION_LOCK_TIMEOUT_ENV_VAR, migration_lock_timeout
+
+MIGRATION_LOCK_KEY: Final = int.from_bytes(b"llm_mig2", "big")
+
+if TYPE_CHECKING:
+    import psycopg
+
+
+def migration_environment(environment: Mapping[str, str]) -> Mapping[str, str]:
+    database_url: Final = environment.get("DATABASE_URL")
+    direct_url: Final = environment.get("DIRECT_URL")
+    if not database_url or not direct_url:
+        return environment
+    schema: Final = next((value for key, value in parse_qsl(urlsplit(database_url).query) if key == "schema"), "public")
+    direct: Final = urlsplit(direct_url)
+    parameters: Final = tuple((key, value) for key, value in parse_qsl(direct.query) if key != "schema")
+    return {
+        **environment,
+        "DATABASE_URL": urlunsplit(direct._replace(query=urlencode((*parameters, ("schema", schema))))),
+    }
+
+
+@dataclass(frozen=True, slots=True)
+class _LockResult:
+    acquired: bool
+
+
+def _try_lock(connection: "psycopg.Connection[tuple[object, ...]]", key: int = MIGRATION_LOCK_KEY) -> bool:
+    from psycopg.rows import class_row
+
+    with connection.cursor(row_factory=class_row(_LockResult)) as cursor:
+        row: Final = cursor.execute("SELECT pg_try_advisory_xact_lock(%s) AS acquired", (key,)).fetchone()
+    return row is not None and row.acquired
+
+
+@dataclass(frozen=True, slots=True)
+class MigrationCoordinator:
+    connection: "psycopg.Connection[tuple[object, ...]]"
+
+    def check_connection(self) -> None:
+        self.connection.execute("SELECT 1")
+
+    def acquire_prisma_lock(self) -> None:
+        deadline: Final = time.monotonic() + migration_lock_timeout()
+        while time.monotonic() < deadline:
+            if _try_lock(self.connection, 72707369):
+                return
+            time.sleep(min(random.uniform(0.5, 1.5), max(0.0, deadline - time.monotonic())))
+        raise RuntimeError(
+            "Timed out waiting for Prisma's lock to recover migration history. LiteLLM startup has stopped. "
+            "Another migration or a pooled database session may still hold the lock. Check the database lock holder. "
+            "When using a transaction pooler, configure DIRECT_URL to reach the same database without the pooler."
+        )
+
+
+@contextmanager
+def migration_lock(database_url: str) -> Generator[MigrationCoordinator, None, None]:
+    import psycopg
+
+    wait_seconds: Final = migration_lock_timeout()
+    deadline: Final = time.monotonic() + wait_seconds
+    try:
+        with psycopg.connect(database_url, connect_timeout=10, autocommit=True) as connection:
+            coordinator: Final = MigrationCoordinator(connection)
+            logger.info("Waiting for the v2 migration coordinator lock (up to %ss)", wait_seconds)
+            while time.monotonic() < deadline:
+                with connection.transaction():
+                    if _try_lock(connection):
+                        logger.info("Acquired the v2 migration coordinator lock")
+
+                        yield coordinator
+                        coordinator.check_connection()
+                        return
+                time.sleep(min(random.uniform(0.5, 1.5), max(0.0, deadline - time.monotonic())))
+    except psycopg.Error as exc:
+        raise RuntimeError(f"Lost or could not establish v2 migration coordination with the database: {exc}") from exc
+    raise RuntimeError(
+        f"Timed out waiting for another v2 migration resolver after {wait_seconds}s. "
+        f"Check the running migration or increase {MIGRATION_LOCK_TIMEOUT_ENV_VAR}."
+    )

litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py

@@ -0,0 +1,158 @@
+import hashlib
+import subprocess
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Final
+from uuid import uuid4
+
+from litellm_proxy_extras import prisma_toolchain
+from litellm_proxy_extras._logging import logger
+from litellm_proxy_extras.migration_lock import MigrationCoordinator
+
+if TYPE_CHECKING:
+    import psycopg
+
+
+@dataclass(frozen=True, slots=True)
+class MigrationProgress:
+    checksum: str
+    applied_steps_count: int
+    logs: str
+    id: str = ""
+    finished: bool = False
+
+    def confirms_completion(self, script: bytes) -> bool:
+        return (
+            self.applied_steps_count == 1
+            and not self.logs.strip()
+            and self.checksum == hashlib.sha256(script).hexdigest()
+        )
+
+
+def _migration_records(
+    connection: "psycopg.Connection[tuple[object, ...]]", schema: str, migration: Path
+) -> tuple[MigrationProgress, ...]:
+    from psycopg import sql
+    from psycopg.rows import class_row
+
+    with connection.cursor(row_factory=class_row(MigrationProgress)) as cursor:
+        records: Final = cursor.execute(
+            sql.SQL(
+                "SELECT id, checksum, applied_steps_count, coalesce(logs, '') AS logs, "
+                "finished_at IS NOT NULL AS finished FROM {} "
+                "WHERE migration_name = %s AND rolled_back_at IS NULL"
+            ).format(sql.Identifier(schema, "_prisma_migrations")),
+            (migration.parent.name,),
+        ).fetchall()
+    return tuple(records)
+
+
+def recover_completed_migration(coordinator: MigrationCoordinator, schema: str, migration: Path) -> bool:
+    """Finish a proven successful row without erasing its durable completion evidence.
+
+    The caller commits this checkpoint before running another Prisma command.
+    """
+    from psycopg import sql
+
+    coordinator.acquire_prisma_lock()
+    records: Final = _migration_records(coordinator.connection, schema, migration)
+    unfinished: Final = tuple(record for record in records if not record.finished)
+    script: Final = migration.read_bytes()
+    if not unfinished:
+        return any(record.checksum == hashlib.sha256(script).hexdigest() for record in records)
+    if len(unfinished) != 1 or not unfinished[0].confirms_completion(script):
+        return False
+    progress: Final = unfinished[0]
+    result: Final = coordinator.connection.execute(
+        sql.SQL(
+            "UPDATE {} SET finished_at = current_timestamp "
+            "WHERE id = %s AND checksum = %s AND applied_steps_count = 1 "
+            "AND finished_at IS NULL AND rolled_back_at IS NULL AND coalesce(logs, '') = %s"
+        ).format(sql.Identifier(schema, "_prisma_migrations")),
+        (progress.id, progress.checksum, progress.logs),
+    )
+    if result.rowcount != 1:
+        raise RuntimeError("Could not complete the confirmed migration history row; retry startup.")
+    logger.info("Completed migration %s using its successful SQL step and matching checksum", migration.parent.name)
+    return True
+
+
+def migration_files(directory: Path) -> tuple[tuple[str, str], ...]:
+    return tuple(
+        (path.parent.name, hashlib.sha256(path.read_bytes()).hexdigest())
+        for path in sorted((directory / "migrations").glob("*/migration.sql"))
+    )
+
+
+def baseline_current_schema(
+    coordinator: MigrationCoordinator,
+    schema: str,
+    migrations_dir: Path,
+    prisma_command: str,
+    prisma_env: Mapping[str, str],
+) -> None:
+    from psycopg import sql
+
+    packaged_dir: Final = Path(__file__).parent
+    migrations: Final = migration_files(migrations_dir)
+    if (
+        not migrations
+        or migrations != migration_files(packaged_dir)
+        or (migrations_dir / "schema.prisma").read_bytes() != (packaged_dir / "schema.prisma").read_bytes()
+    ):
+        raise RuntimeError("Cannot automatically baseline an existing database with custom migration history.")
+
+    coordinator.acquire_prisma_lock()
+    existing: Final = coordinator.connection.execute(
+        "SELECT to_regclass(%s)", (sql.Identifier(schema, "_prisma_migrations").as_string(coordinator.connection),)
+    ).fetchone()
+    if existing is not None and existing[0] is not None:
+        return
+    try:
+        prisma_toolchain.run_prisma(
+            (
+                prisma_command,
+                "migrate",
+                "diff",
+                "--from-schema-datasource",
+                str(migrations_dir / "schema.prisma"),
+                "--to-schema-datamodel",
+                str(migrations_dir / "schema.prisma"),
+                "--exit-code",
+            ),
+            timeout=prisma_toolchain.prisma_command_timeout(),
+            env=prisma_env,
+        )
+    except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
+        raise RuntimeError(
+            "Cannot automatically baseline this database: its schema has not been verified to match this build. "
+            "Establish the existing migration history before retrying. No schema reconciliation was performed. "
+            "If using a transaction pooler, configure DIRECT_URL to reach the same database without the pooler. "
+            f"Schema verification detail: {exc.stderr}"
+        ) from exc
+
+    coordinator.check_connection()
+    ledger: Final = sql.Identifier(schema, "_prisma_migrations")
+    coordinator.connection.execute(
+        sql.SQL(
+            "CREATE TABLE {} (id varchar(36) PRIMARY KEY NOT NULL, checksum varchar(64) NOT NULL, "
+            "finished_at timestamptz, migration_name varchar(255) NOT NULL, logs text, rolled_back_at timestamptz, "
+            "started_at timestamptz NOT NULL DEFAULT now(), applied_steps_count integer NOT NULL DEFAULT 0)"
+        ).format(ledger)
+    )
+    with coordinator.connection.cursor() as cursor:
+        cursor.executemany(
+            sql.SQL(
+                "INSERT INTO {} (id, checksum, migration_name, logs, started_at, finished_at) "
+                "VALUES (%s, %s, %s, '', current_timestamp, current_timestamp)"
+            ).format(ledger),
+            tuple((str(uuid4()), checksum, name) for name, checksum in migrations),
+        )
+    logger.warning(
+        "Legacy migration history was missing. The existing Prisma schema matches this build; "
+        "adopted %s packaged migrations as a baseline. No schema changes were applied, and "
+        "historical data backfills were not replayed or verified. Continuing startup; "
+        "review any feature-specific backfill requirements.",
+        len(migrations),
+    )

litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py

@@ -59,6 +59,7 @@ except ImportError:
 PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT"
 PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT"
 PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT"
+MIGRATION_LOCK_TIMEOUT_ENV_VAR = "LITELLM_MIGRATION_LOCK_TIMEOUT"
 NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR"
 
 DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0
@@ -106,6 +107,10 @@ def prisma_command_timeout() -> float:
     )
 
 
+def migration_lock_timeout() -> float:
+    return _timeout_from_env(MIGRATION_LOCK_TIMEOUT_ENV_VAR, 600.0)
+
+
 def prisma_bootstrap_timeout() -> float:
     """Seconds the one-time Node toolchain install may run for."""
     return _timeout_from_env(

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -6,6 +6,7 @@ import shutil
 import subprocess
 import tempfile
 import time
+from collections.abc import Callable
 from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import TYPE_CHECKING, Final, Optional
@@ -78,15 +79,10 @@ MAX_MIGRATE_DEPLOY_ATTEMPTS = 4
 
 @dataclass(frozen=True)
 class _MigrateAttemptBudget:
-    """Retries left, and the recoveries already run.
-
-    A recovery that lands something new costs nothing, so a database full of
-    objects `prisma db push` created works through them one per pass. Anything
-    that made no progress spends an attempt, so a stuck run still gives up.
-    """
+    """Independent bounds for failed attempts and Prisma lock contention."""
 
     attempts_left: int
-    recoveries: frozenset[str] = frozenset()
+    contention_seconds_left: float = 600.0
 
     @property
     def exhausted(self) -> bool:
@@ -99,10 +95,14 @@ class _MigrateAttemptBudget:
     def spend(self) -> "_MigrateAttemptBudget":
         return replace(self, attempts_left=self.attempts_left - 1)
 
-    def after_recovery(self, recovery: str) -> "_MigrateAttemptBudget":
-        if recovery in self.recoveries:
-            return self.spend()
-        return replace(self, recoveries=self.recoveries | {recovery})
+    def after_contention(self, elapsed: float) -> "_MigrateAttemptBudget":
+        remaining: Final = self.contention_seconds_left - elapsed
+        if remaining <= 0:
+            raise RuntimeError(
+                "Timed out waiting for Prisma's migration advisory lock. Check the running migration "
+                "or increase LITELLM_MIGRATION_LOCK_TIMEOUT."
+            )
+        return replace(self, contention_seconds_left=remaining)
 
 
 _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
@@ -836,12 +836,51 @@ class ProxyExtrasDBManager:
 
     @staticmethod
     def _setup_database_v2(use_migrate: bool) -> bool:
+        if not use_migrate:
+            return ProxyExtrasDBManager._run_database_v2(False)
+        from litellm_proxy_extras.migration_lock import migration_environment, migration_lock
+        from litellm_proxy_extras.migration_recovery import baseline_current_schema, recover_completed_migration
+
+        database_url: Final = os.environ.get("DATABASE_URL")
+        if not database_url:
+            raise RuntimeError("DATABASE_URL is required for v2 migrations")
+        lock_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(os.environ.get("DIRECT_URL") or database_url)
+        schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
+
+        def recover_completed(name: str) -> bool:
+            if Path(name).name != name or "\\" in name:
+                return False
+            migration: Final = Path(os.getcwd()) / "migrations" / name / "migration.sql"
+            if not migration.is_file():
+                return False
+            with migration_lock(lock_url) as coordinator:
+                return recover_completed_migration(coordinator, schema, migration)
+
+        def baseline_existing(migrations_dir: str) -> None:
+            with migration_lock(lock_url) as coordinator:
+                baseline_current_schema(
+                    coordinator,
+                    schema,
+                    Path(migrations_dir),
+                    _get_prisma_command(),
+                    migration_environment(_get_prisma_env()),
+                )
+
+        while not ProxyExtrasDBManager._run_database_v2(True, recover_completed, baseline_existing):
+            continue
+        return True
+
+    @staticmethod
+    def _run_database_v2(
+        use_migrate: bool,
+        recover_completed: Callable[[str], bool] = lambda name: False,
+        baseline_existing: "Callable[[str], None] | None" = None,
+    ) -> bool:
         """
         v2 migration resolver (opt-in via --use_v2_migration_resolver).
 
-        Runs `prisma migrate deploy` and handles standard recovery paths
-        (P3005 baseline, P3009/P3018 idempotent errors, deadlocks against a
-        concurrent migrate deploy). Critically, it does
+        Runs `prisma migrate deploy`, baselines verified existing schemas,
+        and recovers confirmed SQL completion or reported deadlocks. It does
         NOT call `_resolve_all_migrations` — the diff-and-force recovery that
         caused schema thrashing when two LiteLLM versions contended for the
         same DB during rolling deploys.
@@ -850,10 +889,9 @@ class ProxyExtrasDBManager:
         is logged as a warning, not a fatal error — users whose DBs got into
         weird shapes from the old thrashing should still be able to start.
 
-        The retry budget only counts attempts that made no progress: see
-        _MigrateAttemptBudget.
+        False requests a committed recovery checkpoint and another deploy
+        pass. True means every pending migration is complete.
         """
-        schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma"
         migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
 
         if not use_migrate:
@@ -886,14 +924,22 @@ class ProxyExtrasDBManager:
         original_dir = os.getcwd()
         os.chdir(migrations_dir)
         deploy_timeout = prisma_migrate_deploy_timeout()
-        budget = _MigrateAttemptBudget(attempts_left=MAX_MIGRATE_DEPLOY_ATTEMPTS)
+        from litellm_proxy_extras.migration_lock import migration_environment, migration_lock_timeout
+
+        migration_env: Final = migration_environment(_get_prisma_env())
+
+        budget = _MigrateAttemptBudget(
+            attempts_left=MAX_MIGRATE_DEPLOY_ATTEMPTS,
+            contention_seconds_left=migration_lock_timeout(),
+        )
         try:
             while not budget.exhausted:
+                attempt_started = time.monotonic()
                 try:
                     result = prisma_toolchain.run_prisma(
                         [_get_prisma_command(), "migrate", "deploy"],
                         timeout=deploy_timeout,
-                        env=_get_prisma_env(),
+                        env=migration_env,
                     )
                     logger.info(f"prisma migrate deploy stdout: {result.stdout}")
                     return True
@@ -909,8 +955,16 @@ class ProxyExtrasDBManager:
                     next_budget = budget.spend()
 
                 except subprocess.CalledProcessError as e:
+                    if "P3005" in (e.stderr or "") and baseline_existing is not None:
+                        baseline_existing(migrations_dir)
+                        return False
+                    failed_migration = ProxyExtrasDBManager._v2_failed_migration_name(e.stderr or "")
+                    if failed_migration and recover_completed(failed_migration):
+                        return False
                     next_budget = ProxyExtrasDBManager._budget_after_deploy_failure(
-                        e, budget, schema_path
+                        e,
+                        budget,
+                        time.monotonic() - attempt_started,
                     )
 
                 if next_budget.attempts_left < budget.attempts_left:
@@ -919,19 +973,41 @@ class ProxyExtrasDBManager:
 
             raise RuntimeError(
                 f"Database migration failed after {MAX_MIGRATE_DEPLOY_ATTEMPTS} "
-                "attempts that made no progress (timeouts, deadlock retries, or a "
-                "recovery that had already run once). Check database connectivity, "
+                "attempts that made no progress (timeouts or deadlock retries). Check database connectivity, "
                 "load, and _prisma_migrations ledger state, and raise "
                 f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out."
             )
         finally:
             os.chdir(original_dir)
 
+    @staticmethod
+    def _v2_failed_migration_name(stderr: str) -> "str | None":
+        if "P3009" in stderr:
+            match = re.search(r"`(\d+_[^`\r\n]+)`", stderr)
+            return match.group(1) if match else None
+        if "P3018" in stderr:
+            match = re.search(r"Migration name: (\d+_[^\r\n]+)", stderr)
+            return match.group(1) if match else None
+        return None
+
+    @staticmethod
+    def _v2_roll_back_migration_best_effort(migration_name: str) -> None:
+        from litellm_proxy_extras.migration_lock import migration_environment
+
+        try:
+            prisma_toolchain.run_prisma(
+                [_get_prisma_command(), "migrate", "resolve", "--rolled-back", migration_name],
+                timeout=prisma_command_timeout(),
+                env=migration_environment(_get_prisma_env()),
+            )
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+            pass
+
     @staticmethod
     def _budget_after_deploy_failure(
         error: subprocess.CalledProcessError,
         budget: "_MigrateAttemptBudget",
-        schema_path: str,
+        attempt_seconds: float = 0.0,
     ) -> "_MigrateAttemptBudget":
         """Recover from one failed `prisma migrate deploy`, and price the pass.
 
@@ -940,37 +1016,35 @@ class ProxyExtrasDBManager:
         """
         stderr = error.stderr or ""
 
-        if "P3005" in stderr and "database schema is not empty" in stderr:
-            logger.info("Schema exists but no migrations ledger — creating baseline")
-            if ProxyExtrasDBManager._create_baseline_migration(schema_path):
-                return budget.after_recovery("baseline")
-            return budget.spend()
-
         if "P3009" in stderr:
-            migration_match = re.search(r"`(\d+_\S+?)`", stderr)
-            if migration_match and ProxyExtrasDBManager._is_idempotent_error(stderr):
-                name = migration_match.group(1)
-                logger.info(
-                    f"Migration {name} failed idempotently — marking applied and retrying"
-                )
-                ProxyExtrasDBManager._mark_migration_applied(name)
-                return budget.after_recovery(f"resolved:{name}")
-            if migration_match:
-                migration_name = migration_match.group(1)
+            migration_name = ProxyExtrasDBManager._v2_failed_migration_name(stderr)
+            if migration_name:
                 ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name)
-                if ledger_logs is not None and (
-                    ledger_logs == "" or _MIGRATION_DEADLOCK_MARKER in ledger_logs
-                ):
+                if ledger_logs and _MIGRATION_DEADLOCK_MARKER in ledger_logs:
                     logger.info(
                         "Migration %s failed in a concurrent migrate deploy "
                         "deadlock race, rolling its ledger row back and retrying",
                         migration_name,
                     )
-                    ProxyExtrasDBManager._roll_back_migration_best_effort(migration_name)
+                    ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)
                     return budget.spend()
             raise RuntimeError(
-                "Database migration failed and cannot be auto-recovered. "
-                f"Manual intervention required.\n\nPrisma error:\n{stderr}"
+                "Migration completion could not be verified. LiteLLM startup has stopped.\n\n"
+                f"Prisma migration history (migration name and start time):\n{stderr}\n\n"
+                "A migration has a start record but no successful completion record. "
+                "LiteLLM cannot determine whether its SQL committed from this record alone. "
+                "Startup stopped to avoid repeating or skipping database changes.\n\n"
+                "Before resolving, stop other migration runners and inspect _prisma_migrations, "
+                "the named migration.sql from this build, database logs, and the actual database objects and data. "
+                "Use the same database and this build's schema and migration files for recovery:\n"
+                "- Only after verifying every migration change is present, run "
+                "prisma migrate resolve --applied <migration_name>, then retry startup.\n"
+                "- Only after verifying no migration changes remain (or fully undoing partial changes), run "
+                "prisma migrate resolve --rolled-back <migration_name>, then retry startup. "
+                "This command updates history; it does not undo SQL.\n"
+                "Replace <migration_name> with the reported name. If the outcome remains uncertain, "
+                "leave migration history unchanged and contact your database administrator. "
+                "Repeated restarts alone will not resolve this state."
             ) from error
 
         if "P3018" in stderr:
@@ -981,25 +1055,14 @@ class ProxyExtrasDBManager:
                     f"and retry.\n\nPrisma error:\n{stderr}"
                 ) from error
 
-            migration_match = re.search(r"Migration name: (\d+_\S+)", stderr)
-            if migration_match and ProxyExtrasDBManager._is_idempotent_error(stderr):
-                name = migration_match.group(1)
+            migration_name = ProxyExtrasDBManager._v2_failed_migration_name(stderr)
+            if migration_name and _MIGRATION_DEADLOCK_MARKER in stderr:
                 logger.info(
-                    f"Migration {name} SQL hit idempotent error — marking applied and retrying"
-                )
-                ProxyExtrasDBManager._mark_migration_applied(name)
-                return budget.after_recovery(f"resolved:{name}")
-
-            if migration_match and _MIGRATION_DEADLOCK_MARKER in stderr:
-                logger.info(
-                    "Migration %s deadlocked against a concurrent "
-                    "migrate deploy, rolling its ledger row back "
-                    "and retrying",
-                    migration_match.group(1),
-                )
-                ProxyExtrasDBManager._roll_back_migration_best_effort(
-                    migration_match.group(1)
+                    "Migration %s deadlocked against a concurrent migrate deploy, "
+                    "rolling its ledger row back and retrying",
+                    migration_name,
                 )
+                ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)
                 return budget.spend()
 
             raise RuntimeError(
@@ -1009,19 +1072,17 @@ class ProxyExtrasDBManager:
 
         if _MIGRATION_DEADLOCK_MARKER in stderr:
             logger.info(
-                "prisma migrate deploy attempt %s deadlocked against "
-                "a concurrent migrate deploy, retrying",
+                "prisma migrate deploy attempt %s deadlocked against a concurrent migrate deploy, retrying",
                 budget.attempt_number,
             )
             return budget.spend()
 
         if "P1002" in stderr and "advisory lock" in stderr:
             logger.info(
-                "prisma migrate deploy attempt %s timed out waiting for "
-                "the advisory lock a concurrent migrate deploy holds, retrying",
-                budget.attempt_number,
+                "Waiting for the advisory lock held by another Prisma migration; "
+                "contention does not spend a migration failure attempt"
             )
-            return budget.spend()
+            return budget.after_contention(attempt_seconds)
 
         raise RuntimeError(
             "Database migration failed and cannot be auto-recovered. "

litellm-proxy-extras/tests/test_setup_database_fail_fast.py

@@ -6,7 +6,8 @@ The v2 resolver is opt-in via `--use_v2_migration_resolver` / the
 """
 
 import subprocess
-from unittest.mock import patch
+from types import SimpleNamespace
+from unittest.mock import MagicMock, Mock, patch
 
 import pytest
 
@@ -31,12 +32,7 @@ def _fake_migrate_deploy_failure(returncode: int, stderr: str):
 
 def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
     """v2: a permission failure during migrate deploy raises RuntimeError."""
-    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
-    monkeypatch.setattr(
-        ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
-    )
-    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
-    (tmp_path / "schema.prisma").write_text("// stub")
+    _stub_v2_env(monkeypatch, tmp_path)
 
     stderr = (
         "Error: P3018\nMigration name: 20250326162113_baseline\n"
@@ -49,19 +45,14 @@ def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
 
 def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
     """v2: a non-idempotent migration failure raises (no silent recovery)."""
-    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
-    monkeypatch.setattr(
-        ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
-    )
-    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
-    (tmp_path / "schema.prisma").write_text("// stub")
+    _stub_v2_env(monkeypatch, tmp_path)
 
     stderr = (
         "Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n"
         'Reason: syntax error at or near "BRKN" LINE 42'
     )
     with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
-        with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
+        with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
 
@@ -135,8 +126,7 @@ def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path):
 def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_path):
     """v2: a failing `prisma db push` must raise RuntimeError, not leak
     CalledProcessError past proxy_cli.py's `except RuntimeError`."""
-    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
-    (tmp_path / "schema.prisma").write_text("// stub")
+    monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path))
 
     stderr = "db push error"
     with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
@@ -153,8 +143,7 @@ def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path):
     import psycopg
 
     monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
-    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
-    (tmp_path / "schema.prisma").write_text("// stub")
+    monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path))
 
     class _FakeConn:
         def __enter__(self):
@@ -176,70 +165,28 @@ def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path):
     ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path))
 
 
-def test_v2_resolve_specific_migration_failure_raises_runtime_error(
-    monkeypatch, tmp_path
-):
-    """If marking a migration as applied fails inside P3009 idempotent
-    recovery, the subprocess error must be re-raised as RuntimeError so
-    proxy_cli.py catches it cleanly (instead of leaking CalledProcessError)."""
-    monkeypatch.setattr(
-        ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
-    )
-    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
-    (tmp_path / "schema.prisma").write_text("// stub")
-    monkeypatch.setattr(
-        ProxyExtrasDBManager, "_roll_back_migration", lambda *a, **kw: None
-    )
-
-    # First call: migrate deploy -> P3009 idempotent error.
-    # Recovery path tries _resolve_specific_migration; that also raises.
-    def _failing_resolve(*a, **kw):
-        raise subprocess.CalledProcessError(
-            returncode=1,
-            cmd="prisma migrate resolve --applied",
-            stderr="resolve failed",
-            output="",
-        )
-
-    monkeypatch.setattr(
-        ProxyExtrasDBManager, "_resolve_specific_migration", _failing_resolve
-    )
-
-    stderr = (
-        "Error: P3009\nMigration `20260101000000_some_migration` failed\n"
-        "relation already exists"
-    )
-    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
-        with pytest.raises(
-            RuntimeError, match="Failed to mark migration .* as applied"
-        ):
+def test_v2_duplicate_object_p3009_is_not_marked_applied(monkeypatch, tmp_path):
+    _stub_v2_env(monkeypatch, tmp_path, ledger_logs="relation already exists")
+    stderr = "Error: P3009\nMigration `20260101000000_some_migration` failed\nrelation already exists"
+    with patch(
+        "litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)
+    ) as run:
+        with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
+    assert tuple(call.args[0][1:] for call in run.call_args_list if "migrate" in call.args[0]) == (
+        ["migrate", "deploy"],
+    )
 
 
 def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path):
-    """v2 must never call _resolve_all_migrations — that's the bug it fixes."""
-    monkeypatch.setattr(
-        ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
+    _stub_v2_env(monkeypatch, tmp_path)
+    run = Mock(side_effect=_succeed_after(0, ""))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", run)
+
+    assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) is True
+    assert tuple(call.args[0][1:] for call in run.call_args_list if "migrate" in call.args[0]) == (
+        ["migrate", "deploy"],
     )
-    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
-    (tmp_path / "schema.prisma").write_text("// stub")
-
-    class FakeResult:
-        stdout = "Applied migration.\n"
-        stderr = ""
-
-    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", lambda *a, **kw: FakeResult())
-
-    resolve_called = {"n": 0}
-    monkeypatch.setattr(
-        ProxyExtrasDBManager,
-        "_resolve_all_migrations",
-        lambda *a, **kw: resolve_called.__setitem__("n", resolve_called["n"] + 1),
-    )
-
-    ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
-    assert ok is True
-    assert resolve_called["n"] == 0, "v2 must not invoke the diff-and-force recovery"
 
 
 _DEADLOCK_P3018_STDERR = (
@@ -250,14 +197,34 @@ _DEADLOCK_P3018_STDERR = (
 )
 
 
-def _stub_v2_env(monkeypatch, tmp_path):
+def _stub_v2_env(monkeypatch, tmp_path, ledger_logs=""):
+    import psycopg
+
     monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
-    monkeypatch.setattr(
-        ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
-    )
-    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
-    (tmp_path / "schema.prisma").write_text("// stub")
+    monkeypatch.delenv("DIRECT_URL", raising=False)
+    monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path))
     monkeypatch.setattr("time.sleep", lambda _: None)
+    connection = MagicMock()
+    connection.__enter__.return_value = connection
+    cursor = connection.cursor.return_value.__enter__.return_value
+    cursor.execute.return_value = cursor
+    cursor.fetchone.return_value = SimpleNamespace(acquired=True)
+    cursor.fetchall.return_value = []
+    empty = MagicMock()
+    empty.fetchall.return_value = []
+    empty.fetchone.return_value = None
+    ledger = MagicMock()
+    ledger.fetchone.return_value = (ledger_logs,)
+
+    def execute(query, *args, **kwargs):
+        if "SELECT logs FROM" in str(query):
+            if ledger_logs is None:
+                raise psycopg.OperationalError("ledger is unavailable")
+            return ledger
+        return empty
+
+    connection.execute.side_effect = execute
+    monkeypatch.setattr("psycopg.connect", lambda *args, **kwargs: connection)
 
 
 def _succeed_after(failures: int, stderr: str):
@@ -272,9 +239,7 @@ def _succeed_after(failures: int, stderr: str):
             return _OkResult()
         calls["n"] += 1
         if calls["n"] <= failures:
-            raise subprocess.CalledProcessError(
-                returncode=1, cmd=args[0], stderr=stderr, output=""
-            )
+            raise subprocess.CalledProcessError(returncode=1, cmd=args[0], stderr=stderr, output="")
         return _OkResult()
 
     return _run
@@ -285,28 +250,21 @@ def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path):
     instance rolls the ledger row back and retries instead of dying."""
     _stub_v2_env(monkeypatch, tmp_path)
 
-    rolled_back = []
-    monkeypatch.setattr(
-        ProxyExtrasDBManager,
-        "_roll_back_migration",
-        lambda name: rolled_back.append(name),
-    )
-    monkeypatch.setattr(
-        ProxyExtrasDBManager,
-        "_resolve_specific_migration",
-        lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
-    )
-    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, _DEADLOCK_P3018_STDERR))
+    run = Mock(side_effect=_succeed_after(1, _DEADLOCK_P3018_STDERR))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", run)
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
-    assert rolled_back == ["20260415120000_health_check_latest_per_model_index"]
+    assert tuple(call.args[0][1:] for call in run.call_args_list if "migrate" in call.args[0]) == (
+        ["migrate", "deploy"],
+        ["migrate", "resolve", "--rolled-back", "20260415120000_health_check_latest_per_model_index"],
+        ["migrate", "deploy"],
+    )
 
 
 def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path):
     """v2: a deadlock on every attempt still fails after the retry budget."""
     _stub_v2_env(monkeypatch, tmp_path)
-    monkeypatch.setattr(ProxyExtrasDBManager, "_roll_back_migration", lambda name: None)
 
     with patch(
         "litellm_proxy_extras.prisma_toolchain.run_prisma",
@@ -319,7 +277,7 @@ def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path):
 def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_path):
     """v2: the surviving instance sees the victim's failed ledger row as P3009.
     When that row's logs show a deadlock, roll it back and retry."""
-    _stub_v2_env(monkeypatch, tmp_path)
+    _stub_v2_env(monkeypatch, tmp_path, ledger_logs="ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock")
 
     stderr = (
         "Error: P3009\n"
@@ -327,61 +285,39 @@ def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_
         "The `20260415120000_health_check_latest_per_model_index` migration "
         "started at 2026-09-01 18:46:13 UTC failed"
     )
-    monkeypatch.setattr(
-        ProxyExtrasDBManager,
-        "_failed_migration_logs",
-        lambda name: "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock",
-    )
-    rolled_back = []
-    monkeypatch.setattr(
-        ProxyExtrasDBManager,
-        "_roll_back_migration",
-        lambda name: rolled_back.append(name),
-    )
-    monkeypatch.setattr(
-        ProxyExtrasDBManager,
-        "_resolve_specific_migration",
-        lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
-    )
-    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
+    run = Mock(side_effect=_succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", run)
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
-    assert rolled_back == ["20260415120000_health_check_latest_per_model_index"]
+    assert tuple(call.args[0][1:] for call in run.call_args_list if "migrate" in call.args[0]) == (
+        ["migrate", "deploy"],
+        ["migrate", "resolve", "--rolled-back", "20260415120000_health_check_latest_per_model_index"],
+        ["migrate", "deploy"],
+    )
 
 
-def test_v2_p3009_empty_ledger_logs_rolls_back_and_retries(monkeypatch, tmp_path):
-    """v2: empty failed ledger logs mean a concurrent deploy moved it on."""
+def test_v2_p3009_empty_ledger_logs_do_not_prove_completion(monkeypatch, tmp_path):
     _stub_v2_env(monkeypatch, tmp_path)
-
     stderr = (
         "Error: P3009\n"
         "migrate found failed migrations in the target database\n"
         "The `20260415120000_health_check_latest_per_model_index` migration "
         "started at 2026-09-01 18:46:13 UTC failed"
     )
-    monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name: "")
-    rolled_back = []
-    monkeypatch.setattr(
-        ProxyExtrasDBManager,
-        "_roll_back_migration",
-        lambda name: rolled_back.append(name),
+    with patch(
+        "litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)
+    ) as run:
+        with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
+            ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
+    assert tuple(call.args[0][1:] for call in run.call_args_list if "migrate" in call.args[0]) == (
+        ["migrate", "deploy"],
     )
-    monkeypatch.setattr(
-        ProxyExtrasDBManager,
-        "_resolve_specific_migration",
-        lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
-    )
-    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
-
-    ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
-    assert ok is True
-    assert rolled_back == ["20260415120000_health_check_latest_per_model_index"]
 
 
 def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path):
     """v2: an unreadable ledger cannot establish that P3009 was a deadlock."""
-    _stub_v2_env(monkeypatch, tmp_path)
+    _stub_v2_env(monkeypatch, tmp_path, ledger_logs=None)
 
     stderr = (
         "Error: P3009\n"
@@ -389,21 +325,15 @@ def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path):
         "The `20260415120000_health_check_latest_per_model_index` migration "
         "started at 2026-09-01 18:46:13 UTC failed"
     )
-    monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name: None)
-    monkeypatch.setattr(
-        ProxyExtrasDBManager,
-        "_roll_back_migration",
-        lambda name: pytest.fail("an unreadable ledger must not trigger a retry"),
-    )
     monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
-    with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
+    with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
         ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
 
 def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path):
     """v2: a failed ledger row whose logs show a real SQL error stays fatal."""
-    _stub_v2_env(monkeypatch, tmp_path)
+    _stub_v2_env(monkeypatch, tmp_path, ledger_logs='ERROR: syntax error at or near "BRKN"')
 
     stderr = (
         "Error: P3009\n"
@@ -411,14 +341,9 @@ def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path):
         "The `20260101000000_genuinely_broken` migration started at "
         "2026-09-01 18:46:13 UTC failed"
     )
-    monkeypatch.setattr(
-        ProxyExtrasDBManager,
-        "_failed_migration_logs",
-        lambda name: 'ERROR: syntax error at or near "BRKN"',
-    )
 
     with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
-        with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
+        with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
 

litellm/__init__.py

@@ -240,7 +240,6 @@ email: Optional[str] = (
 token: Optional[str] = (
     None  # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
 )
-telemetry = True
 max_tokens: int = DEFAULT_MAX_TOKENS  # OpenAI Defaults
 drop_params = drop_params_env_flag(os.environ, verbose_logger)
 modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False))

litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py

@@ -85,7 +85,7 @@ def _filter_reserved_headers(
 
 
 def _request_scoped_runtime_session_id(
-    params: Mapping[str, Any],
+    params: Mapping[str, object],
     litellm_params: Mapping[str, Any],
 ) -> str | None:
     context_id: Final = get_session_id_from_a2a_params(params)

litellm/a2a_protocol/providers/watsonx_orchestrate/config.py

@@ -20,7 +20,7 @@ class WatsonxOrchestrateA2AConfig(BaseA2AProviderConfig):
         params: dict[str, Any],
         api_base: str | None = None,
         **kwargs: Any,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """Handle a non-streaming A2A request via WXO runs API."""
         litellm_params: Final = kwargs.get("litellm_params")
         if not litellm_params:
@@ -40,7 +40,7 @@ class WatsonxOrchestrateA2AConfig(BaseA2AProviderConfig):
         params: dict[str, Any],
         api_base: str | None = None,
         **kwargs: Any,
-    ) -> AsyncIterator[dict[str, Any]]:
+    ) -> AsyncIterator[dict[str, object]]:
         """Handle a streaming A2A request via WXO streaming runs API."""
         litellm_params: Final = kwargs.get("litellm_params")
         if not litellm_params:

litellm/a2a_protocol/utils.py

@@ -17,7 +17,7 @@ class A2ARequestUtils:
     """Utility class for A2A request/response processing."""
 
     @staticmethod
-    def extract_text_from_message(message: Any) -> str:
+    def extract_text_from_message(message: object) -> str:
         """
         Extract text content from A2A message parts.
 
@@ -142,7 +142,7 @@ class A2ARequestUtils:
         return prompt_tokens, completion_tokens, total_tokens
 
 
-def get_session_id_from_a2a_params(params: Mapping[str, Any]) -> str | None:
+def get_session_id_from_a2a_params(params: Mapping[str, object]) -> str | None:
     message: Final = params.get("message", {})
     if isinstance(message, dict):
         return message.get("contextId")
@@ -166,7 +166,7 @@ def scope_session_to_principal(session_id: str, principal: str | None) -> str:
 
 
 # Backwards compatibility aliases
-def extract_text_from_a2a_message(message: Any) -> str:
+def extract_text_from_a2a_message(message: object) -> str:
     return A2ARequestUtils.extract_text_from_message(message)
 
 

litellm/integrations/gitlab/gitlab_prompt_manager.py

@@ -200,8 +200,8 @@ class GitLabTemplateManager:
             metadata=metadata,
         )
 
-    def _parse_yaml_basic(self, yaml_str: str) -> dict[str, Any]:
-        result: Final[dict[str, Any]] = {}
+    def _parse_yaml_basic(self, yaml_str: str) -> dict[str, bool | int | float | str]:
+        result: Final[dict[str, bool | int | float | str]] = {}
         for line in yaml_str.split("\n"):
             line = line.strip()
             if ":" in line and not line.startswith("#"):

litellm/integrations/vantage/vantage_logger.py

@@ -59,7 +59,7 @@ class VantageLogger(FocusLogger):
                     raw_interval,
                 )
 
-        destination_config: Final[dict[str, Any]] = {}
+        destination_config: Final[dict[str, str]] = {}
         if resolved_api_key:
             destination_config["api_key"] = resolved_api_key
         if resolved_token:
@@ -93,7 +93,7 @@ class VantageLogger(FocusLogger):
 
         pod_lock_manager = None
         if proxy_logging_obj is not None:
-            writer: Final = getattr(proxy_logging_obj, "db_spend_update_writer", None)
+            writer: Final[object] = getattr(proxy_logging_obj, "db_spend_update_writer", None)
             if writer is not None:
                 pod_lock_manager = getattr(writer, "pod_lock_manager", None)
 

litellm/interactions/agents/http_handler.py

@@ -7,7 +7,7 @@ duplicated. BaseAgentsAPIConfig stays as pure transform code.
 """
 
 from collections.abc import Coroutine, Mapping
-from typing import Any, Final
+from typing import Final
 
 import httpx
 
@@ -38,7 +38,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         name: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         extra_body: Mapping[str, object] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
@@ -93,7 +93,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         name: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         extra_body: Mapping[str, object] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
@@ -141,7 +141,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         agents_api_config: BaseAgentsAPIConfig,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
@@ -181,7 +181,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         agents_api_config: BaseAgentsAPIConfig,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ) -> AgentListResponse:
@@ -216,7 +216,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         name: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
@@ -259,7 +259,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         name: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ) -> AgentCreateResponse:
@@ -295,7 +295,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         name: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
@@ -338,7 +338,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         name: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ) -> AgentDeleteResult:
@@ -374,7 +374,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         name: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | None = None,
         _is_async: bool = False,
@@ -417,7 +417,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         name: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: dict[str, str] | None = None,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ) -> AgentVersionsResponse:

litellm/litellm_core_utils/logging_utils.py

@@ -67,7 +67,9 @@ def _truncate_base64_in_string(value: str) -> str:
     return _DATA_URI_RE.sub(_base64_data_uri_replacer, value)
 
 
-def _truncate_base64_in_value(value: Any) -> Any:
+def _truncate_base64_in_value(
+    value: str | dict[str, object] | list[object] | None,
+) -> str | dict[str, object] | list[object] | None:
     """Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict).
 
     Uses an explicit stack instead of recursion to satisfy the project's

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -138,9 +138,13 @@ class _ToolCallDelta(TypedDict, total=False):
 
 
 class _ToolCallChoice(TypedDict, total=False):
+    index: ReadOnly[int]
     delta: ReadOnly[_ToolCallDelta]
 
 
+_ToolCallKey: TypeAlias = tuple[int, int]
+
+
 class _ToolCallChunk(TypedDict):
     choices: ReadOnly[Sequence[_ToolCallChoice]]
 
@@ -417,40 +421,41 @@ class ChunkProcessor:
     @staticmethod
     def _iter_tool_call_fragments(
         tool_call_chunks: Sequence["_ToolCallChunk"],
-    ) -> Iterator[tuple[int, str, str]]:
+    ) -> Iterator[tuple[_ToolCallKey, str, str]]:
         for chunk in tool_call_chunks:
             for choice in chunk["choices"]:
                 delta = choice.get("delta")
                 if not delta:
                     continue
-                for tool_call in delta.get("tool_calls", ()):
+                choice_index = choice.get("index", 0)
+                for tool_call in delta.get("tool_calls") or ():
                     if not tool_call:
                         continue
                     if isinstance(tool_call, dict):
-                        index = tool_call.get("index", 0)
+                        key = (choice_index, tool_call.get("index", 0))
                         function = tool_call.get("function")
                         if isinstance(function, dict):
                             if fragment_arguments := function.get("arguments"):
-                                yield index, "arguments", fragment_arguments
+                                yield key, "arguments", fragment_arguments
                         elif function_arguments := getattr(function, "arguments", None):
-                            yield index, "arguments", function_arguments
+                            yield key, "arguments", function_arguments
                         custom = tool_call.get("custom")
                         if isinstance(custom, dict) and (custom_input := custom.get("input")):
-                            yield index, "custom_input", custom_input
+                            yield key, "custom_input", custom_input
                     else:
-                        index = getattr(tool_call, "index", 0)
+                        key = (choice_index, getattr(tool_call, "index", 0))
                         function = getattr(tool_call, "function", None)
                         if object_arguments := getattr(function, "arguments", None):
-                            yield index, "arguments", object_arguments
+                            yield key, "arguments", object_arguments
                         custom = getattr(tool_call, "custom", None)
                         if object_custom_input := getattr(custom, "input", None):
-                            yield index, "custom_input", object_custom_input
+                            yield key, "custom_input", object_custom_input
 
     @staticmethod
-    def _join_fragments_by_index_and_field(
-        fragment_records: Iterator[tuple[int, str, str]],
-    ) -> Mapping[tuple[int, str], str]:
-        def group_key(record: tuple[int, str, str]) -> tuple[int, str]:
+    def _join_fragments_by_key_and_field(
+        fragment_records: Iterator[tuple[_ToolCallKey, str, str]],
+    ) -> Mapping[tuple[_ToolCallKey, str], str]:
+        def group_key(record: tuple[_ToolCallKey, str, str]) -> tuple[_ToolCallKey, str]:
             return record[0], record[1]
 
         return MappingProxyType(
@@ -468,13 +473,14 @@ class ChunkProcessor:
         tool_calls_list: list[
             ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall
         ] = []  # mutable-ok: see return type
-        tool_call_map: Final[dict[int, dict[str, Any]]] = {}  # Map to store tool calls by index
+        tool_call_map: Final[dict[_ToolCallKey, dict[str, Any]]] = {}
 
         for chunk in tool_call_chunks:
             choices = chunk["choices"]
             for choice in choices:
                 delta = choice.get("delta", {})
-                tool_calls = delta.get("tool_calls", [])
+                tool_calls = delta.get("tool_calls") or ()
+                choice_index = choice.get("index", 0)
 
                 for tool_call in tool_calls:
                     # Handle both dict and object formats
@@ -496,9 +502,9 @@ class ChunkProcessor:
 
                     # Get index (handle both dict and object)
                     if isinstance(tool_call, dict):
-                        index = tool_call.get("index", 0)
+                        index = (choice_index, tool_call.get("index", 0))
                     else:
-                        index = getattr(tool_call, "index", 0)
+                        index = (choice_index, getattr(tool_call, "index", 0))
 
                     if index not in tool_call_map:
                         tool_call_map[index] = {
@@ -573,7 +579,7 @@ class ChunkProcessor:
                         if isinstance(provider_fields, dict):
                             merged_provider_fields.update(provider_fields)
 
-        joined_fragments: Final = self._join_fragments_by_index_and_field(
+        joined_fragments: Final = self._join_fragments_by_key_and_field(
             self._iter_tool_call_fragments(tool_call_chunks)
         )
 

litellm/litellm_core_utils/url_utils.py

@@ -418,7 +418,7 @@ def _extract_redirect_url(response: httpx.Response, request_url: str) -> str:
     return str(httpx.URL(request_url).join(location))
 
 
-def safe_get(client: Any, url: str, **kwargs: Any) -> httpx.Response:
+def safe_get(client: _UrlFetcher, url: str, **kwargs: Any) -> httpx.Response:
     """
     Fetch a user-supplied URL with SSRF protection on every redirect hop.
 
@@ -461,7 +461,7 @@ def safe_get(client: Any, url: str, **kwargs: Any) -> httpx.Response:
     raise SSRFError("Too many redirects")
 
 
-async def async_safe_get(client: Any, url: str, **kwargs: Any) -> httpx.Response:
+async def async_safe_get(client: _AsyncUrlFetcher, url: str, **kwargs: Any) -> httpx.Response:
     """Async version of safe_get."""
     if not getattr(litellm, "user_url_validation", True):
         kwargs.setdefault("follow_redirects", True)

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -1457,7 +1457,9 @@ class AnthropicMessagesHandler(BaseTranslation):
         if not any(is_text_delta(event) for item in responses_so_far for event in cls._iter_sse_events(item)):
             from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
 
-            raise UndeliverableStreamRewrite(guardrail_name)
+            raise UndeliverableStreamRewrite(
+                guardrail_name, "the buffered stream carries no text_delta event to land the text rewrite on"
+            )
         replacements: Final = chain((rewritten_text,), repeat(""))
 
         def rewrite_text_delta(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
@@ -1498,7 +1500,11 @@ class AnthropicMessagesHandler(BaseTranslation):
         if len(block_indices) != len(post_guardrail_tool_calls):
             from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
 
-            raise UndeliverableStreamRewrite(guardrail_name)
+            raise UndeliverableStreamRewrite(
+                guardrail_name,
+                f"the guardrail returned {len(post_guardrail_tool_calls)} tool calls for a stream that carried "
+                f"{len(block_indices)} tool_use blocks",
+            )
         rewrites_by_block: Final = MappingProxyType(
             {
                 index: after

litellm/llms/anthropic/chat/handler.py

@@ -596,7 +596,7 @@ class ModelResponseIterator:
         self.reasoning_content_chunks: list[str] = []
 
         # Track server tool use inputs and results for code_interpreter_results
-        self._server_tool_inputs: dict[str, Any] = {}
+        self._server_tool_inputs: dict[str, object] = {}
         self.tool_results: list[dict[str, Any]] = []
         self._current_server_tool_id: str | None = None
         self._container_id: str | None = None

litellm/llms/azure/completion/handler.py

@@ -1,6 +1,7 @@
 from collections.abc import Callable
-from typing import Any, Final
+from typing import Final
 
+import httpx
 from openai import AsyncAzureOpenAI, AzureOpenAI
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -191,7 +192,7 @@ class AzureTextCompletion(BaseAzureLLM):
         model: str,
         api_base: str,
         data: dict,
-        timeout: Any,
+        timeout: float | httpx.Timeout | None,
         model_response: ModelResponse,
         logging_obj: LiteLLMLoggingObj,
         max_retries: int,
@@ -253,7 +254,7 @@ class AzureTextCompletion(BaseAzureLLM):
         api_version: str,
         data: dict,
         model: str,
-        timeout: Any,
+        timeout: float | httpx.Timeout | None,
         azure_ad_token: str | None = None,
         client=None,
         litellm_params: dict = {},
@@ -306,7 +307,7 @@ class AzureTextCompletion(BaseAzureLLM):
         api_version: str,
         data: dict,
         model: str,
-        timeout: Any,
+        timeout: float | httpx.Timeout | None,
         azure_ad_token: str | None = None,
         client=None,
         litellm_params: dict = {},

litellm/llms/azure_ai/ocr/document_intelligence/transformation.py

@@ -12,7 +12,7 @@ import asyncio
 import re
 import time
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Final
 from urllib.parse import quote
 
 import httpx
@@ -127,7 +127,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
 
     def map_ocr_params(
         self,
-        non_default_params: dict,
+        non_default_params: Mapping[str, object],
         optional_params: dict,
         model: str,
     ) -> dict:
@@ -164,7 +164,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
             raise UnsupportedParamsError(message=f"{e}", model=model, llm_provider="azure_ai") from e
 
     @staticmethod
-    def _normalize_pages_param(pages: Any) -> str:
+    def _normalize_pages_param(pages: object) -> str:
         """
         Convert a caller-provided `pages` value to Azure DI's query-string
         form. Azure expects 1-based page numbers, grammar: `^(\\d+(-\\d+)?)(,\\s*(\\d+(-\\d+)?))*$`.
@@ -412,7 +412,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
             raise ValueError("Document URL is required")
 
         # Build Azure DI request
-        data: Final[dict[str, Any]] = {}
+        data: Final[dict[str, str]] = {}
 
         # Check if it's a data URI (base64)
         if document_url.startswith("data:"):

litellm/llms/bedrock/batches/transformation.py

@@ -2,7 +2,7 @@ import os
 import re
 import time
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Final, Literal, cast
+from typing import TYPE_CHECKING, Final, Literal, cast
 
 from httpx import Headers, Response
 from pydantic import TypeAdapter, ValidationError
@@ -170,7 +170,7 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
         create_batch_data: CreateBatchRequest,
         optional_params: dict,
         litellm_params: dict,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Transform the batch creation request to Bedrock format.
 
@@ -354,7 +354,7 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
         )
 
     @staticmethod
-    def _get_openai_compatible_batch_metadata(metadata: Any) -> dict[str, str]:
+    def _get_openai_compatible_batch_metadata(metadata: object) -> dict[str, str]:
         """
         OpenAI Batch metadata only accepts string values.
         """
@@ -379,7 +379,7 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
         batch_id: str,
         optional_params: dict,
         litellm_params: dict,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Transform batch retrieval request for Bedrock.
 
@@ -523,7 +523,7 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
             )
 
         # Enrich metadata with useful Bedrock fields
-        enriched_metadata_raw: Final[dict[str, Any]] = {
+        enriched_metadata_raw: Final[dict[str, object]] = {
             "jobName": response_data.get("jobName"),
             "clientRequestToken": response_data.get("clientRequestToken"),
             "modelId": response_data.get("modelId"),

litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py

@@ -110,7 +110,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
         headers: dict,
     ) -> dict:
         input_prompt: Final = self._convert_messages_to_prompt(messages=messages)
-        request_data: Final[dict[str, Any]] = {"inputPrompt": input_prompt}
+        request_data: Final[dict[str, object]] = {"inputPrompt": input_prompt}
 
         media_source: Final = self._build_media_source(optional_params)
         if media_source is not None:

litellm/llms/bytez/chat/transformation.py

@@ -335,10 +335,10 @@ class BytezChatConfig(BaseConfig):
 
 
 class BytezCustomStreamWrapper(CustomStreamWrapper):
-    def chunk_creator(self, chunk: Any):
+    def chunk_creator(self, chunk: object):
         try:
             model_response: Final = self.model_response_creator()
-            response_obj: dict[str, Any] = {}
+            response_obj: dict[str, object] = {}
 
             response_obj = {
                 "text": chunk,
@@ -346,7 +346,7 @@ class BytezCustomStreamWrapper(CustomStreamWrapper):
                 "finish_reason": "",
             }
 
-            completion_obj: Final[dict[str, Any]] = {"content": chunk}
+            completion_obj: Final[dict[str, object]] = {"content": chunk}
 
             return self.return_processed_chunk_logic(
                 completion_obj=completion_obj,

litellm/llms/custom_httpx/aiohttp_handler.py

@@ -1,5 +1,5 @@
 import ssl
-from collections.abc import Callable
+from collections.abc import AsyncIterable, Callable, Iterable
 from typing import TYPE_CHECKING, Any, Final, cast
 
 import aiohttp
@@ -212,7 +212,7 @@ class BaseLLMAIOHTTPHandler:
         litellm_params: dict,
         stream: bool = False,
         files: dict | None = None,
-        content: Any = None,
+        content: str | bytes | Iterable[bytes] | AsyncIterable[bytes] | None = None,
         params: dict | None = None,
     ) -> httpx.Response:
         max_retry_on_unprocessable_entity_error: Final = provider_config.max_retry_on_unprocessable_entity_error

litellm/llms/deprecated_providers/aleph_alpha.py

@@ -146,7 +146,7 @@ class AlephAlphaConfig:
                 setattr(self.__class__, key, value)
 
     @classmethod
-    def get_config(cls):
+    def get_config(cls) -> dict[str, object]:
         return {
             k: v
             for k, v in cls.__dict__.items()

litellm/llms/lemonade/chat/transformation.py

@@ -170,7 +170,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig):
         model: str,
         api_base: str | None = None,
         api_key: str | None = None,
-    ) -> Any:
+    ) -> dict[str, object]:
         if model.startswith("lemonade/"):
             model = model.split("/", 1)[1]
 

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -1169,10 +1169,22 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             choice.index for response in responses_so_far for choice in response.choices
         )
         fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far)
-        if len(stream_choice_indices) != 1 or len(fragments_by_tool_call) != len(post_guardrail_tool_calls):
+        if len(stream_choice_indices) != 1:
             from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
 
-            raise UndeliverableStreamRewrite(guardrail_name)
+            raise UndeliverableStreamRewrite(
+                guardrail_name,
+                f"the stream carries {len(stream_choice_indices)} choices and tool-call rewrites are only written "
+                "back on single-choice streams",
+            )
+        if len(fragments_by_tool_call) != len(post_guardrail_tool_calls):
+            from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
+
+            raise UndeliverableStreamRewrite(
+                guardrail_name,
+                f"the guardrail returned {len(post_guardrail_tool_calls)} tool calls for a stream that carried "
+                f"{len(fragments_by_tool_call)}",
+            )
         for before, (name, arguments), fragments in zip(
             pre_guardrail_tool_calls, post_guardrail_tool_calls, fragments_by_tool_call
         ):

litellm/llms/openai/image_edit/transformation.py

@@ -66,7 +66,7 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
     def _add_image_to_files(
         self,
         files_list: list[tuple[str, Any]],
-        image: Any,
+        image: object,
         field_name: str,
     ) -> None:
         """Add an image to the files list with appropriate content type"""

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -167,6 +167,34 @@ def _tool_call_rewrite(before: _ToolCallShape, after: _ToolCallShape) -> _ToolCa
     return _ToolCallShape(name=after.name if after.name != before.name else None, arguments=after.arguments)
 
 
+def _undeliverable_tool_call_rewrite_reason(
+    call_ids: Sequence[str],
+    tool_call_item_count: int,
+    post_guardrail_tool_call_count: int,
+    unresolved_argument_event: bool,
+    rewritten_call_ids: frozenset[str],
+    event_call_ids: frozenset[str],
+) -> str | None:
+    if len(call_ids) != tool_call_item_count:
+        return (
+            f"{tool_call_item_count - len(call_ids)} of the stream's {tool_call_item_count} tool call items "
+            "carry no call_id"
+        )
+    if len(frozenset(call_ids)) != len(call_ids):
+        return "the stream's tool call items repeat a call_id"
+    if len(call_ids) != post_guardrail_tool_call_count:
+        return (
+            f"the guardrail returned {post_guardrail_tool_call_count} tool calls for the stream's "
+            f"{len(call_ids)} tool call items"
+        )
+    if unresolved_argument_event:
+        return "a tool call argument event names an item_id that no output_item event introduced"
+    missing_call_ids: Final = sorted(rewritten_call_ids - event_call_ids)
+    if missing_call_ids:
+        return f"no stream event carries the rewritten call_id {', '.join(missing_call_ids)}"
+    return None
+
+
 class ResponseOutputEnvelope(TypedDict, total=False):
     """Dict form of a Responses API response, as far as guardrail write-back reads it."""
 
@@ -999,7 +1027,11 @@ class OpenAIResponsesHandler(BaseTranslation):
         ):
             from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
 
-            raise UndeliverableStreamRewrite(guardrail_name)
+            raise UndeliverableStreamRewrite(
+                guardrail_name,
+                "the scanned text events are not all output_text deltas with an integer output_index and "
+                "content_index, so the text rewrite has nowhere to land",
+            )
         self._sync_stream_events_with_rewrites(
             stream_events=stream_events,
             rewrites_by_position=MappingProxyType(dict(zip(placeable_positions, chain((rewritten_text,), repeat(""))))),
@@ -1106,16 +1138,18 @@ class OpenAIResponsesHandler(BaseTranslation):
             call_id is None and stream_item_field(event, "type") in _TOOL_CALL_PAYLOAD_EVENT_TYPES
             for event, call_id in zip(stream_events, event_call_ids)
         )
-        if (
-            len(call_ids) != len(tool_call_items)
-            or len(frozenset(call_ids)) != len(call_ids)
-            or len(call_ids) != len(post_guardrail_tool_calls)
-            or unresolved_argument_event
-            or not rewrites_by_call_id.keys() <= frozenset(event_call_ids)
-        ):
+        undeliverable_reason: Final = _undeliverable_tool_call_rewrite_reason(
+            call_ids=call_ids,
+            tool_call_item_count=len(tool_call_items),
+            post_guardrail_tool_call_count=len(post_guardrail_tool_calls),
+            unresolved_argument_event=unresolved_argument_event,
+            rewritten_call_ids=frozenset(rewrites_by_call_id),
+            event_call_ids=frozenset(call_id for call_id in event_call_ids if call_id is not None),
+        )
+        if undeliverable_reason is not None:
             from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
 
-            raise UndeliverableStreamRewrite(guardrail_name)
+            raise UndeliverableStreamRewrite(guardrail_name, undeliverable_reason)
         for output_item, rewrite in (
             (output_item, rewrites_by_call_id[call_id])
             for output_item, call_id in zip(tool_call_items, call_ids)

litellm/llms/runwayml/text_to_speech/transformation.py

@@ -78,7 +78,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
         aspeech: bool,
         api_base: str | None,
         api_key: str | None,
-        **kwargs: Any,
+        **kwargs: object,
     ) -> Union[
         "HttpxBinaryResponseContent",
         Coroutine[object, object, "HttpxBinaryResponseContent"],

litellm/llms/vertex_ai/files/transformation.py

@@ -651,7 +651,7 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows(
 
 def _openai_batch_jsonl_entry_to_vertex_rows(
     openai_entry: dict[str, Any],
-    map_openai_to_vertex_params: Callable[[dict[str, Any]], dict[str, Any]],
+    map_openai_to_vertex_params: Callable[[dict[str, Any]], dict[str, object]],
 ) -> tuple[Mapping[str, object], ...]:
     """
     Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to.
@@ -774,7 +774,7 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream):
     def __init__(
         self,
         openai_file_content: FileTypes,
-        map_openai_to_vertex_params: Callable[[dict[str, Any]], dict[str, Any]],
+        map_openai_to_vertex_params: Callable[[dict[str, Any]], dict[str, object]],
     ) -> None:
         self._openai_file_content = openai_file_content
         self._map_openai_to_vertex_params = map_openai_to_vertex_params
@@ -948,7 +948,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
     def _map_openai_to_vertex_params(
         self,
         openai_request_body: dict[str, Any],
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         wrapper to call VertexGeminiConfig.map_openai_params
         """

litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py

@@ -3,7 +3,7 @@ Google AI Studio /batchEmbedContents Embeddings Endpoint
 """
 
 import json
-from typing import TYPE_CHECKING, Any, Final, Literal
+from typing import TYPE_CHECKING, Final, Literal
 
 import httpx
 
@@ -210,7 +210,7 @@ class GoogleBatchEmbeddings(VertexLLM):
             )
 
         ### TRANSFORMATION (sync path) ###
-        request_data: Any
+        request_data: VertexAIBatchEmbeddingsRequestBody | dict[str, object]
         if use_embed_content:
             resolved_files = {}
             if api_key:

litellm/llms/vertex_ai/vertex_ai_non_gemini.py

@@ -64,7 +64,7 @@ def _get_client_from_cache(client_cache_key: str):
     return litellm.in_memory_llm_clients_cache.get_cache(client_cache_key)
 
 
-def _set_client_in_cache(client_cache_key: str, vertex_llm_model: Any):
+def _set_client_in_cache(client_cache_key: str, vertex_llm_model: object):
     litellm.in_memory_llm_clients_cache.set_cache(
         key=client_cache_key,
         value=vertex_llm_model,

litellm/llms/watsonx/audio_transcription/transformation.py

@@ -4,7 +4,7 @@ Translates from OpenAI's `/v1/audio/transcriptions` to IBM WatsonX's `/ml/v1/aud
 WatsonX follows the OpenAI spec for audio transcription.
 """
 
-from typing import Any, Final
+from typing import Final
 
 from httpx import Response
 
@@ -124,7 +124,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran
         }
 
         # Convert TypedDict to regular dict for AudioTranscriptionRequestData
-        form_data_dict: Final[dict[str, Any]] = dict(form_data)
+        form_data_dict: Final[dict[str, object]] = dict(form_data)
 
         return AudioTranscriptionRequestData(data=form_data_dict, files=files)
 

litellm/main.py

@@ -8759,6 +8759,39 @@ def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_o
         setattr(usage, "cost", computed_cost)
 
 
+_NON_TEXT_DELTA_FIELDS: Final = (
+    "tool_calls",
+    "function_call",
+    "reasoning_content",
+    "thinking_blocks",
+    "annotations",
+    "audio",
+    "images",
+    "provider_specific_fields",
+)
+
+
+def _stream_choice_delta(choice: object) -> Mapping[str, object]:
+    delta: Final = choice.get("delta", {}) if isinstance(choice, dict) else getattr(choice, "delta", {})
+    if isinstance(delta, Mapping):
+        return delta
+    if isinstance(delta, BaseModel):
+        return delta.model_dump()
+    return {}
+
+
+def _delta_carries_more_than_text(delta: Mapping[str, object]) -> bool:
+    return any(delta.get(field) is not None for field in _NON_TEXT_DELTA_FIELDS)
+
+
+def _simple_text_part(choices: Sequence[object]) -> str | None:
+    deltas: Final = tuple(_stream_choice_delta(choice) for choice in choices)
+    if any(_delta_carries_more_than_text(delta) for delta in deltas):
+        return None
+    content: Final = deltas[0].get("content")
+    return content if isinstance(content, str) else ""
+
+
 def stream_chunk_builder(
     chunks: list,
     messages: Sequence | None = None,
@@ -8803,31 +8836,11 @@ def stream_chunk_builder(
             if not chunk.get("choices"):
                 continue
 
-            choice = chunk["choices"][0]
-            delta_obj = choice.get("delta", {}) if isinstance(choice, dict) else getattr(choice, "delta", {})
-            if isinstance(delta_obj, dict):
-                delta = delta_obj
-            elif hasattr(delta_obj, "model_dump"):
-                delta = cast(dict[str, Any], delta_obj.model_dump())
-            else:
-                delta = {}
-
-            if (
-                delta.get("tool_calls") is not None
-                or delta.get("function_call") is not None
-                or delta.get("reasoning_content") is not None
-                or delta.get("thinking_blocks") is not None
-                or delta.get("annotations") is not None
-                or delta.get("audio") is not None
-                or delta.get("images") is not None
-                or delta.get("provider_specific_fields") is not None
-            ):
+            if (part := _simple_text_part(chunk["choices"])) is None:
                 is_simple_text_stream = False
                 break
-
-            content = delta.get("content")
-            if isinstance(content, str) and content:
-                simple_content_parts.append(content)
+            if part:
+                simple_content_parts.append(part)
 
         if is_simple_text_stream:
             if simple_content_parts:
@@ -8864,9 +8877,10 @@ def stream_chunk_builder(
         tool_call_chunks: Final = [
             chunk
             for chunk in chunks
-            if chunk.get("choices")
-            and "tool_calls" in chunk["choices"][0]["delta"]
-            and chunk["choices"][0]["delta"]["tool_calls"] is not None
+            if any(
+                "tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
+                for choice in chunk.get("choices") or ()
+            )
         ]
 
         if len(tool_call_chunks) > 0:

litellm/model_prices_and_context_window_backup.json

@@ -41329,6 +41329,7 @@
     },
     "openrouter/deepseek/deepseek-v3.2": {
         "cache_read_input_token_cost": 1.345e-07,
+        "deprecation_date": "2026-09-28",
         "input_cost_per_token": 2.69e-07,
         "input_cost_per_token_cache_hit": 1.345e-07,
         "litellm_provider": "openrouter",
@@ -41350,6 +41351,7 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v3.2-exp": {
+        "deprecation_date": "2026-09-28",
         "input_cost_per_token": 2.7e-07,
         "input_cost_per_token_cache_hit": 2e-08,
         "litellm_provider": "openrouter",
@@ -43114,6 +43116,7 @@
         "output_cost_per_token": 1.2e-06,
         "cache_creation_input_token_cost": 0.0,
         "cache_read_input_token_cost": 3e-08,
+        "deprecation_date": "2026-10-08",
         "litellm_provider": "openrouter",
         "max_input_tokens": 204800,
         "max_output_tokens": 131072,
@@ -66773,9 +66776,9 @@
         "supports_web_search": true
     },
     "openrouter/z-ai/glm-5.2": {
-        "input_cost_per_token": 5.544e-07,
-        "output_cost_per_token": 1.7424e-06,
-        "cache_read_input_token_cost": 1.0296e-07,
+        "input_cost_per_token": 6.496e-07,
+        "output_cost_per_token": 2.0416e-06,
+        "cache_read_input_token_cost": 1.2064e-07,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 131072,
@@ -67135,9 +67138,9 @@
         "supports_web_search": true
     },
     "openrouter/deepseek/deepseek-v4-flash": {
-        "input_cost_per_token": 3.724e-08,
-        "output_cost_per_token": 7.448e-08,
-        "cache_read_input_token_cost": 7.448e-09,
+        "input_cost_per_token": 3.668e-08,
+        "output_cost_per_token": 7.336e-08,
+        "cache_read_input_token_cost": 7.336e-09,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
@@ -67950,6 +67953,7 @@
         "input_cost_per_token": 2.7e-07,
         "output_cost_per_token": 1e-06,
         "cache_read_input_token_cost": 1.35e-07,
+        "deprecation_date": "2026-09-28",
         "litellm_provider": "openrouter",
         "max_input_tokens": 163840,
         "max_output_tokens": 32768,
@@ -68691,6 +68695,7 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-r1-distill-llama-70b": {
+        "deprecation_date": "2026-09-28",
         "input_cost_per_token": 8e-07,
         "output_cost_per_token": 8e-07,
         "litellm_provider": "openrouter",
@@ -71964,6 +71969,7 @@
         "supports_web_search": false
     },
     "openrouter/baidu/ernie-4.5-vl-424b-a47b": {
+        "deprecation_date": "2026-10-08",
         "input_cost_per_token": 4.2e-07,
         "litellm_provider": "openrouter",
         "max_input_tokens": 123000,

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -2867,7 +2867,7 @@ class MCPServerManager:
             normalize_server_name(value) for value in (*iter_known_server_prefixes(server), server.name) if value
         )
 
-    def _server_exposes_tool(self, server: MCPServer, tool_name: str) -> bool:
+    def server_exposes_tool(self, server: MCPServer, tool_name: str) -> bool:
         owned: Final = self._owned_mapping_values(server)
         mapped_owners: Final = (
             self.tool_name_to_mcp_server_name_mapping.get(spelling)
@@ -2875,6 +2875,20 @@ class MCPServerManager:
         )
         return any(owner is not None and normalize_server_name(owner) in owned for owner in mapped_owners)
 
+    def _known_prefix_to_server(self) -> Mapping[str, MCPServer]:
+        """Every prefix form a tool name may carry, keyed to its server; a form two servers share
+        stays with the one registered first."""
+        return {
+            normalize_server_name(known_prefix): server
+            for server in reversed(tuple(self.get_registry().values()))
+            for known_prefix in iter_known_server_prefixes(server)
+        }
+
+    def server_owning_tool_name_prefix(self, tool_name: str) -> MCPServer | None:
+        prefix_to_server: Final = self._known_prefix_to_server()
+        matched: Final = match_known_server_prefix(tool_name, prefix_to_server.keys())
+        return None if matched is None else prefix_to_server.get(matched[0])
+
     def remove_server(self, mcp_server: LiteLLM_MCPServerTable):
         """
         Remove a server from the registry
@@ -6114,7 +6128,7 @@ class MCPServerManager:
         if mcp_server is None:
             raise ValueError(f"Tool {name} not found")
 
-        if resolved_by_server_name_only and not self._server_exposes_tool(mcp_server, name):
+        if resolved_by_server_name_only and not self.server_exposes_tool(mcp_server, name):
             raise ValueError(f"Tool {name} not found")
 
         return mcp_server
@@ -6475,15 +6489,7 @@ class MCPServerManager:
             MCPServer if found, None otherwise
         """
         registry_servers: Final = list(self.get_registry().values())
-
-        # Build prefix → server lookup covering every known form a tool name
-        # may take (alias / server_name / server_id / short ID).  This is what
-        # makes the short-prefix mode work without breaking historical names.
-        prefix_to_server: Final[dict[str, MCPServer]] = {}
-        for server in registry_servers:
-            for known_prefix in iter_known_server_prefixes(server):
-                normalised = normalize_server_name(known_prefix)
-                prefix_to_server.setdefault(normalised, server)
+        prefix_to_server: Final = self._known_prefix_to_server()
 
         # First try with the original tool name
         if tool_name in self.tool_name_to_mcp_server_name_mapping:
@@ -6501,7 +6507,7 @@ class MCPServerManager:
         if matched is not None:
             matched_prefix, original_tool_name = matched
             matched_server: Final = prefix_to_server.get(matched_prefix)
-            if matched_server is not None and self._server_exposes_tool(matched_server, original_tool_name):
+            if matched_server is not None and self.server_exposes_tool(matched_server, original_tool_name):
                 return matched_server
 
         return None

litellm/proxy/_experimental/mcp_server/server.py

@@ -2888,6 +2888,40 @@ if MCP_AVAILABLE:
                 headers={"WWW-Authenticate": get_byok_www_authenticate()},
             )
 
+    async def _list_tools_before_first_call(
+        server: MCPServer | None,
+        tool_name: str,
+        allowed_mcp_servers: list[MCPServer],
+        user_api_key_auth: UserAPIKeyAuth | None,
+        mcp_auth_header: str | None,
+        mcp_server_auth_headers: dict[str, dict[str, str]] | None,
+        oauth2_headers: dict[str, str] | None,
+        raw_headers: dict[str, str] | None,
+    ) -> None:
+        """List ``server`` with the caller's own credentials when it does not yet expose ``tool_name`` here.
+
+        The startup fill skips a server whose upstream wants the caller's token, and mcp 2 no
+        longer lists before an uncached tools/call, so a worker that has not served tools/list
+        for this caller would otherwise answer 404 for a tool the caller can see. Gating on the
+        requested tool, not on any prior listing, keeps callers with different upstream catalogs
+        from masking each other.
+        """
+        if server is None or global_mcp_server_manager.server_exposes_tool(server, tool_name):
+            return
+        if all(allowed.server_id != server.server_id for allowed in allowed_mcp_servers):
+            return
+        try:
+            await _get_tools_from_mcp_servers(
+                user_api_key_auth=user_api_key_auth,
+                mcp_auth_header=mcp_auth_header,
+                mcp_servers=[server.server_id],
+                mcp_server_auth_headers=mcp_server_auth_headers,
+                oauth2_headers=oauth2_headers,
+                raw_headers=raw_headers,
+            )
+        except Exception as e:  # noqa: BLE001  # best effort: resolution below answers as it did before
+            verbose_logger.debug("MCP tools/call: listing %s before its first call failed: %s", server.name, e)
+
     async def execute_mcp_tool(
         name: str,
         arguments: dict[str, object],
@@ -2948,6 +2982,27 @@ if MCP_AVAILABLE:
                     all_registry_prefixes.add(normalize_server_name(known_prefix))
             name_is_prefixed = is_tool_name_prefixed(name, known_server_prefixes=all_registry_prefixes)
 
+        first_call_target: Final = (
+            requested_server
+            if requested_server is not None and not name_is_prefixed
+            else global_mcp_server_manager.server_owning_tool_name_prefix(name)
+        )
+        first_call_tool_name: Final = (
+            name
+            if first_call_target is None or (requested_server is not None and not name_is_prefixed)
+            else strip_known_server_prefix(name, first_call_target)
+        )
+        await _list_tools_before_first_call(
+            server=first_call_target,
+            tool_name=first_call_tool_name,
+            allowed_mcp_servers=allowed_mcp_servers,
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+        )
+
         if requested_server is not None and not name_is_prefixed:
             # REST callers may pass server_id with the upstream tool name (no
             # LiteLLM prefix). The first segment is not a registered server

litellm/proxy/a2a/agent_card.py

@@ -10,7 +10,7 @@ and uses LiteLLM auth.
 import re
 from collections.abc import Mapping
 from copy import deepcopy
-from typing import Any, Final, Literal
+from typing import Final, Literal
 
 SupportedA2AVersion = Literal["0.3", "1.0"]
 
@@ -44,7 +44,7 @@ def normalize_protocol_version(version: object) -> SupportedA2AVersion | None:
     return next((supported for supported in SUPPORTED_A2A_PROTOCOL_VERSIONS if supported == major_minor), None)
 
 
-def resolve_served_protocol_version(card: Mapping[str, Any] | None) -> str:
+def resolve_served_protocol_version(card: Mapping[str, object] | None) -> str:
     """Return the validated protocol version an agent card pins, else the default."""
     normalized: Final = normalize_protocol_version(card.get("protocolVersion") if card else None)
     return normalized if normalized is not None else LITELLM_A2A_PROTOCOL_VERSION
@@ -53,7 +53,7 @@ def resolve_served_protocol_version(card: Mapping[str, Any] | None) -> str:
 # Security scheme exposed by the LiteLLM-fronted agent card. Always replaces
 # whatever upstream advertised — the client must authenticate to the proxy,
 # not the upstream agent.
-LITELLM_SECURITY_SCHEMES: Final[dict[str, dict[str, Any]]] = {
+LITELLM_SECURITY_SCHEMES: Final[dict[str, dict[str, str]]] = {
     "LiteLLMKey": {
         "type": "http",
         "scheme": "bearer",
@@ -112,7 +112,7 @@ _ALLOWED_TOP_LEVEL_KEYS: Final = {
     "url",
 }
 
-_DEFAULT_SKILLS: Final[list[dict[str, Any]]] = [
+_DEFAULT_SKILLS: Final[list[dict[str, str | list[str]]]] = [
     {
         "id": "chat",
         "name": "Chat",
@@ -129,7 +129,7 @@ _DEFAULT_MODES: Final[list[str]] = ["text"]
 _DEFAULT_AGENT_VERSION: Final = "1.0.0"
 
 
-def _filter_capabilities(upstream_capabilities: Any) -> dict[str, Any]:
+def _filter_capabilities(upstream_capabilities: object) -> dict[str, object]:
     """Return a capabilities dict containing only allowlisted, truthy keys."""
     if not isinstance(upstream_capabilities, dict):
         return {}
@@ -143,13 +143,13 @@ def _default_litellm_provider(proxy_base_url: str) -> dict[str, str]:
 
 
 def merge_agent_card(
-    upstream_card: Mapping[str, Any] | None,
+    upstream_card: Mapping[str, object] | None,
     *,
     proxy_url: str,
     proxy_base_url: str,
     name: str | None = None,
     description: str | None = None,
-) -> dict[str, Any]:
+) -> dict[str, object]:
     """
     Build the LiteLLM-fronted agent card.
 
@@ -169,7 +169,7 @@ def merge_agent_card(
         A dict suitable for serving as the proxy's agent card. Only keys in
         the v1.0 AgentCard schema (plus ``supportedInterfaces``) are emitted.
     """
-    base: Final[dict[str, Any]] = deepcopy(dict(upstream_card)) if upstream_card else {}
+    base: Final[dict[str, object]] = deepcopy(dict(upstream_card)) if upstream_card else {}
 
     # Keep the upstream ``url`` on the stored card: the runtime A2A
     # invocation path reads it from ``agent_card_params`` to know where to

litellm/proxy/client/credentials.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import Any, Final
 
 import requests
@@ -69,8 +70,8 @@ class CredentialsManagementClient:
     def create(
         self,
         credential_name: str,
-        credential_info: dict[str, Any],
-        credential_values: dict[str, Any],
+        credential_info: Mapping[str, object],
+        credential_values: Mapping[str, object],
         return_request: bool = False,
     ) -> dict[str, Any] | requests.Request:
         """

litellm/proxy/dev_config.yaml

@@ -212,7 +212,6 @@ sandbox_tools:
 
 litellm_settings:
   drop_params: True
-  telemetry: False
   code_interpreter_interception_params:
     enabled: true
     sandbox_tool_name: e2b_sandbox

litellm/proxy/example_config_yaml/oai_misc_config.yaml

@@ -38,7 +38,6 @@ litellm_settings:
   # budget_duration: 30d
   num_retries: 5
   request_timeout: 600
-  telemetry: False
   context_window_fallbacks: [{"gpt-5-mini": ["gpt-5.5"]}]
   default_team_settings: 
     - team_id: team-1

litellm/proxy/guardrails/guardrail_hooks/guardrails_ai/guardrails_ai.py

@@ -7,7 +7,7 @@
 
 import json
 import os
-from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict
+from typing import TYPE_CHECKING, Final, Literal, TypedDict
 
 from fastapi import HTTPException
 
@@ -181,7 +181,7 @@ class GuardrailsAI(CustomGuardrail):
     ):  # raise exception if invalid, return a str for the user to receive - if rejected, or return a modified dictionary for passing into litellm
         return await self.process_input(data=data, call_type=call_type)
 
-    async def async_logging_hook(self, kwargs: dict, result: Any, call_type: str) -> tuple[dict, Any]:
+    async def async_logging_hook(self, kwargs: dict, result: object, call_type: str) -> tuple[dict, object]:
         if call_type == "acompletion" or call_type == "completion":
             kwargs = await self.process_input(data=kwargs, call_type=call_type)
 

litellm/proxy/guardrails/guardrail_hooks/singulr/singulr.py

@@ -24,7 +24,7 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.guardrails import GuardrailEventHooks
-from litellm.types.llms.openai import AllMessageValues
+from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
 from litellm.types.proxy.guardrails.guardrail_hooks.base import (
     GuardrailConfigModel,
 )
@@ -36,7 +36,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.singulr import (
     ToolCall,
     ToolCallFunction,
 )
-from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs
+from litellm.types.utils import CallTypes, ChatCompletionMessageToolCall, GenericGuardrailAPIInputs
 
 _DEFAULT_API_BASE: Final = "http://localhost:8003"
 _GUARD_ENDPOINT: Final = "/api/v1/ai-gateway/litellm-v2"
@@ -339,7 +339,7 @@ class SingulrGuardrail(CustomGuardrail):
         return inputs
 
     @staticmethod
-    def _build_tool_call(tool_call: Mapping[str, Any]) -> "ToolCall | None":
+    def _build_tool_call(tool_call: ChatCompletionToolCallChunk | ChatCompletionMessageToolCall) -> "ToolCall | None":
         tool_call_id: Final = tool_call.get("id")
         fun: Final = tool_call.get("function")
         if not tool_call_id or not fun:

litellm/proxy/policy_engine/pipeline_executor.py

@@ -8,7 +8,7 @@ pass/fail actions (allow, block, next, modify_response) and data forwarding.
 import copy
 import time
 from collections.abc import Callable, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar
+from typing import TYPE_CHECKING, Final, Literal, TypeVar
 
 from pydantic import BaseModel
 
@@ -50,12 +50,16 @@ except ImportError:
 
 
 class UndeliverableStreamRewrite(Exception):
-    def __init__(self, guardrail_name: str) -> None:
-        super().__init__(
-            f"Guardrail '{guardrail_name}' rewrote the streamed response in a way this endpoint's "
-            "streaming pipeline cannot deliver"
-        )
+    def __init__(self, guardrail_name: str, reason: str) -> None:
+        super().__init__(guardrail_name, reason)
         self.guardrail_name: Final = guardrail_name
+        self.reason: Final = reason
+
+    def __str__(self) -> str:
+        return (
+            f"Guardrail '{self.guardrail_name}' rewrote the streamed response but the rewrite cannot be written "
+            f"back to the stream: {self.reason}"
+        )
 
 
 def _tool_call_shape(tool_call: object) -> tuple[object, object]:
@@ -82,8 +86,22 @@ def _rewrote(sent: tuple[object, ...] | None, returned: tuple[object, ...] | Non
     return sent is not None and returned is not None and returned != sent
 
 
-def _changed_count(sent: tuple[object, ...] | None, returned: tuple[object, ...] | None) -> bool:
-    return sent is not None and returned is not None and len(returned) != len(sent)
+def _count_change(sent: tuple[object, ...] | None, returned: tuple[object, ...] | None) -> tuple[int, int] | None:
+    if sent is None or returned is None or len(returned) == len(sent):
+        return None
+    return (len(sent), len(returned))
+
+
+def _tool_call_mismatch_reason(
+    sent: tuple[tuple[object, object], ...] | None, returned: tuple[tuple[object, object], ...] | None
+) -> str | None:
+    if sent == returned:
+        return None
+    sent_count: Final = len(sent or ())
+    returned_count: Final = len(returned or ())
+    if sent_count == returned_count:
+        return "the legacy hook changed a tool call's name or arguments, which this path cannot write back"
+    return f"the legacy hook returned {returned_count} tool calls for a stream that carried {sent_count}"
 
 
 _GuardrailMethodT = TypeVar("_GuardrailMethodT", bound=Callable[..., object])
@@ -110,7 +128,7 @@ class _StreamRewriteObserver(CustomGuardrail):
         self.inner: Final = inner
         self.rewrote_texts = False
         self.rewrote_tool_calls = False
-        self.changed_tool_call_count = False
+        self.tool_call_count_change: tuple[int, int] | None = None
 
     def structured_messages_cover_full_request(self) -> bool:
         return self.inner.structured_messages_cover_full_request()
@@ -131,11 +149,22 @@ class _StreamRewriteObserver(CustomGuardrail):
         returned_tool_shapes: Final = _tool_call_shapes(outputs.get("tool_calls"))
         self.rewrote_texts = self.rewrote_texts or _rewrote(sent_texts, _text_snapshot(outputs.get("texts")))
         self.rewrote_tool_calls = self.rewrote_tool_calls or _rewrote(sent_tool_shapes, returned_tool_shapes)
-        self.changed_tool_call_count = self.changed_tool_call_count or _changed_count(
+        self.tool_call_count_change = self.tool_call_count_change or _count_change(
             sent_tool_shapes, returned_tool_shapes
         )
         return outputs
 
+    def discard_reason(self, deliver_rewrites: bool) -> str | None:
+        if self.tool_call_count_change is not None:
+            sent, returned = self.tool_call_count_change
+            return (
+                f"the guardrail returned {returned} tool calls for a stream that carried {sent}, and a rewrite "
+                "that drops or adds a tool call cannot be written back"
+            )
+        if not deliver_rewrites and (self.rewrote_texts or self.rewrote_tool_calls):
+            return "this endpoint's streaming pipeline does not write ended-stream rewrites back yet"
+        return None
+
 
 class _ScannedTextRecorder(CustomGuardrail):
     def __init__(self, guardrail_name: str) -> None:
@@ -200,13 +229,24 @@ class _LegacyHookStreamAdapter(CustomGuardrail):
         if rewrite is None:
             return inputs
         rescanned: Final = await self._rescan(rewrite, logging_obj)
+        guardrail_name: Final = self.guardrail_name or "unknown"
         if rescanned is None:
-            raise UndeliverableStreamRewrite(self.guardrail_name or "unknown")
+            raise UndeliverableStreamRewrite(
+                guardrail_name, "the legacy hook's response could not be rescanned by this endpoint's translation"
+            )
         rewritten: Final = rescanned.get("texts")
-        if len(_scanned_texts(rewritten)) != len(_scanned_texts(inputs.get("texts"))):
-            raise UndeliverableStreamRewrite(self.guardrail_name or "unknown")
-        if _tool_call_shapes(rescanned.get("tool_calls")) != _tool_call_shapes(inputs.get("tool_calls")):
-            raise UndeliverableStreamRewrite(self.guardrail_name or "unknown")
+        returned_text_count: Final = len(_scanned_texts(rewritten))
+        sent_text_count: Final = len(_scanned_texts(inputs.get("texts")))
+        if returned_text_count != sent_text_count:
+            raise UndeliverableStreamRewrite(
+                guardrail_name,
+                f"the legacy hook returned {returned_text_count} texts for a stream that carried {sent_text_count}",
+            )
+        tool_call_mismatch: Final = _tool_call_mismatch_reason(
+            _tool_call_shapes(inputs.get("tool_calls")), _tool_call_shapes(rescanned.get("tool_calls"))
+        )
+        if tool_call_mismatch is not None:
+            raise UndeliverableStreamRewrite(guardrail_name, tool_call_mismatch)
         if not rewritten:
             return inputs
         rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": rewritten}
@@ -253,14 +293,16 @@ def _prepare_hook_input(
 
 def _release_original_chunks(
     guardrail_name: str,
+    reason: str,
     streaming_chunks: list[object],  # mutable-ok: shared buffered-stream chunks, restored in place
     originals: Sequence[object],
 ) -> None:
     streaming_chunks[:] = originals  # rebind-ok: the caller's buffer is the stream the client receives
     verbose_proxy_logger.warning(
-        "Pipeline: guardrail '%s' rewrote the streamed response in a way this endpoint's streaming "
-        "pipeline cannot deliver yet; the rewrite was discarded and the original stream released",
+        "Pipeline: guardrail '%s' rewrote the streamed response but the rewrite could not be written back to "
+        "the stream: %s. The whole rewrite, text rewrites included, was discarded and the original stream released",
         guardrail_name,
+        reason,
     )
 
 
@@ -272,11 +314,11 @@ class PipelineExecutor:
         steps: list[PipelineStep],
         mode: str,
         data: dict,
-        user_api_key_dict: Any,
+        user_api_key_dict: "UserAPIKeyAuth",
         call_type: str,
         policy_name: str,
         raw_request_snapshot: dict | None = None,  # mutable-ok: same request-payload shape as data
-        streaming_chunks: list[Any] | None = None,  # mutable-ok: shared buffered-stream chunks, read per step
+        streaming_chunks: list[object] | None = None,  # mutable-ok: shared buffered-stream chunks, read per step
         endpoint_translation: "BaseTranslation | None" = None,
     ) -> PipelineExecutionResult:
         """
@@ -433,13 +475,12 @@ class PipelineExecutor:
                     user_api_key_dict=user_api_key_dict,
                     request_data=hook_input,
                 )
-        except UndeliverableStreamRewrite:
-            _release_original_chunks(step.guardrail, streaming_chunks, originals)
+        except UndeliverableStreamRewrite as undeliverable:
+            _release_original_chunks(step.guardrail, undeliverable.reason, streaming_chunks, originals)
             return
-        if observer.changed_tool_call_count or (
-            not deliver_rewrites and (observer.rewrote_texts or observer.rewrote_tool_calls)
-        ):
-            _release_original_chunks(step.guardrail, streaming_chunks, originals)
+        discard_reason: Final = observer.discard_reason(deliver_rewrites)
+        if discard_reason is not None:
+            _release_original_chunks(step.guardrail, discard_reason, streaming_chunks, originals)
             return
         if not callback.records_own_guardrail_information:
             add_guardrail_to_applied_guardrails_header(request_data=hook_input, guardrail_name=step.guardrail)
@@ -449,10 +490,10 @@ class PipelineExecutor:
         step: PipelineStep,
         mode: str,
         data: dict,
-        user_api_key_dict: Any,
+        user_api_key_dict: "UserAPIKeyAuth",
         call_type: str,
         raw_request_snapshot: dict | None = None,  # mutable-ok: same request-payload shape as data
-        streaming_chunks: list[Any] | None = None,  # mutable-ok: shared buffered-stream chunks, read per step
+        streaming_chunks: list[object] | None = None,  # mutable-ok: shared buffered-stream chunks, read per step
         endpoint_translation: "BaseTranslation | None" = None,
     ) -> tuple[
         Literal["pass", "fail", "error"],
@@ -681,7 +722,7 @@ def _extract_error_message(e: Exception) -> str:
     if isinstance(e, ModifyResponseException):
         return str(e)
     if HTTPException is not None and isinstance(e, HTTPException):
-        detail: Final = getattr(e, "detail", None)
+        detail: Final[object] = getattr(e, "detail", None)
         if detail:
             return str(detail)
     return str(e)

litellm/proxy/proxy_cli.py

@@ -56,8 +56,6 @@ if litellm_mode == "DEV":
     load_dotenv()
 from enum import Enum
 
-telemetry: Final = None
-
 
 class LiteLLMDatabaseConnectionPool(Enum):
     database_connection_pool_limit = 10
@@ -758,9 +756,11 @@ class ProxyInitializationHelpers:
 )
 @click.option(
     "--telemetry",
-    default=True,
+    default=None,
     type=bool,
-    help="Helps us know if people are using this feature. Turn this off by doing `--telemetry False`",
+    hidden=True,
+    expose_value=False,
+    help="Deprecated no-op kept so existing start commands still parse",
 )
 @click.option(
     "--log_config",
@@ -977,7 +977,6 @@ def run_server(
     add_function_to_prompt,
     config,
     max_budget,
-    telemetry,
     test,
     local,
     num_workers,
@@ -1082,7 +1081,6 @@ def run_server(
             max_tokens=max_tokens,
             request_timeout=request_timeout,
             max_budget=max_budget,
-            telemetry=telemetry,
             drop_params=drop_params,
             add_function_to_prompt=add_function_to_prompt,
             headers=headers,

litellm/proxy/proxy_server.py

@@ -1230,12 +1230,12 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
                 general_settings,
             ) = await proxy_config.load_config(router=llm_router, config_file_path=worker_config)
         elif isinstance(worker_config, dict):
-            await initialize(**worker_config)
+            await initialize_from_worker_config(worker_config)
         else:
             # if not, assume it's a json string
             worker_config = json.loads(worker_config)
             if isinstance(worker_config, dict):
-                await initialize(**worker_config)
+                await initialize_from_worker_config(worker_config)
 
     enforce_master_key_boot_verdict(
         await with_stored_secrets_counted(
@@ -1510,6 +1510,12 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
         except Exception as e:
             verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e)
 
+    if prisma_client is not None and hasattr(prisma_client, "stop_view_setup_task"):
+        try:
+            await prisma_client.stop_view_setup_task()
+        except Exception as e:
+            verbose_proxy_logger.error("Error stopping the spend view setup task: %s", e)
+
     await _drain_spend_event_producer_on_shutdown()
 
     await flush_spend_counters_on_shutdown()
@@ -2420,7 +2426,6 @@ user_debug = False
 user_max_tokens = None
 user_request_timeout = None
 user_temperature = None
-user_telemetry = True
 user_config: Final = None
 user_headers = None
 user_config_file_path: str | None = None
@@ -8615,6 +8620,14 @@ def save_worker_config(**data):
     os.environ["WORKER_CONFIG"] = json.dumps(data)
 
 
+LEGACY_WORKER_CONFIG_KEYS: Final = frozenset({"telemetry"})
+
+
+async def initialize_from_worker_config(worker_config: Mapping[str, object]) -> None:
+    supported: Final = MappingProxyType({k: v for k, v in worker_config.items() if k not in LEGACY_WORKER_CONFIG_KEYS})
+    await initialize(**supported)
+
+
 async def initialize(
     model=None,
     alias=None,
@@ -8626,7 +8639,6 @@ async def initialize(
     max_tokens=None,
     request_timeout=600,
     max_budget=None,
-    telemetry=False,
     drop_params=True,
     add_function_to_prompt=True,
     headers=None,
@@ -8642,7 +8654,6 @@ async def initialize(
         user_user_max_tokens, \
         user_request_timeout, \
         user_temperature, \
-        user_telemetry, \
         user_headers, \
         experimental, \
         llm_model_list, \
@@ -8749,7 +8760,6 @@ async def initialize(
         dynamic_config["general"]["max_budget"] = litellm.max_budget
     if experimental:
         pass
-    user_telemetry = telemetry
 
 
 # for streaming
@@ -10808,14 +10818,7 @@ class ProxyStartupEvent:
             if hasattr(prisma_client, "db") and hasattr(prisma_client.db, "start_token_refresh_task"):
                 await prisma_client.db.start_token_refresh_task()
 
-            ## Add necessary views to proxy ##
-            asyncio.create_task(
-                prisma_client.check_view_exists()
-            )  # check if all necessary views exist. Don't block execution
-
-            asyncio.create_task(
-                prisma_client._set_spend_logs_row_count_in_proxy_state()
-            )  # set the spend logs row count in proxy state. Don't block execution
+            prisma_client.start_view_setup_task()
 
             if hasattr(prisma_client, "start_db_health_watchdog_task"):
                 await prisma_client.start_db_health_watchdog_task()

litellm/proxy/utils.py

@@ -38,6 +38,7 @@ from typing import (
     Literal,
     Optional,
     Protocol,
+    TypeAlias,
     TypeVar,
     Union,
     cast,
@@ -269,6 +270,15 @@ class _RelTuplesRow(TypedDict):
     reltuples: ReadOnly[int]
 
 
+_VIEW_SETUP_POLL_INTERVAL_SECONDS: Final = 5.0
+_VIEW_SETUP_DEADLINE_SECONDS: Final = 15 * 60.0
+_VIEW_SETUP_GATE_TABLE: Final = '"LiteLLM_SpendLogs"'
+_VIEW_SETUP_GATE_PROBE_ROWS: Final = TypeAdapter(tuple[Mapping[str, bool], ...])
+
+_ViewSetupOutcome: TypeAlias = Literal["ready", "timed_out"]
+_ViewSetupAttempt: TypeAlias = Literal["ready", "table_missing"] | Exception
+
+
 class _EndUserBatchTable(Protocol):
     def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ...
 
@@ -4297,6 +4307,7 @@ class PrismaClient:
             self.db = writer_wrapper  # Client to connect to Prisma db
         self._db_reconnect_lock = asyncio.Lock()
         self._db_health_watchdog_task: asyncio.Task | None = None
+        self._view_setup_task: asyncio.Task[_ViewSetupOutcome] | None = None
         self._db_last_reconnect_attempt_ts: float = 0.0
         self._db_reconnect_cooldown_seconds: int = max(1, int(os.getenv("PRISMA_RECONNECT_COOLDOWN_SECONDS", "15")))
         self._db_read_only_recreate_ts: float = 0.0
@@ -6343,6 +6354,71 @@ class PrismaClient:
         self._db_health_watchdog_task = None
         verbose_proxy_logger.info("Stopped Prisma DB health watchdog")
 
+    def start_view_setup_task(self) -> None:
+        if self._view_setup_task is not None:
+            return
+        self._view_setup_task = asyncio.create_task(self._run_view_setup())
+
+    async def stop_view_setup_task(self) -> None:
+        if self._view_setup_task is None:
+            return
+        self._view_setup_task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await self._view_setup_task
+        self._view_setup_task = None
+
+    async def _run_view_setup(
+        self,
+        poll_interval_seconds: float = _VIEW_SETUP_POLL_INTERVAL_SECONDS,
+        deadline_seconds: float = _VIEW_SETUP_DEADLINE_SECONDS,
+    ) -> _ViewSetupOutcome:
+        deadline: Final = time.monotonic() + deadline_seconds
+        while True:
+            if (attempt := await self._attempt_view_setup()) == "ready":
+                return "ready"
+            if time.monotonic() >= deadline:
+                self._log_view_setup_timeout(attempt, deadline_seconds)
+                return "timed_out"
+            await asyncio.sleep(poll_interval_seconds)
+
+    async def _attempt_view_setup(self) -> _ViewSetupAttempt:
+        try:
+            if not await self._view_setup_gate_table_present():
+                verbose_proxy_logger.debug(
+                    "Waiting for table %s before creating the spend views", _VIEW_SETUP_GATE_TABLE
+                )
+                return "table_missing"
+            await self._set_spend_logs_row_count_in_proxy_state()
+            await self.check_view_exists()
+            return "ready"
+        except Exception as e:
+            verbose_proxy_logger.warning("Spend view setup attempt failed, retrying until the schema settles: %s", e)
+            return e
+
+    def _log_view_setup_timeout(
+        self, last_attempt: Literal["table_missing"] | Exception, deadline_seconds: float
+    ) -> None:
+        if isinstance(last_attempt, Exception):
+            verbose_proxy_logger.error(
+                "Gave up creating the spend views after %ss; the last attempt failed with: %s. "
+                "Fix that error and restart the proxy.",
+                deadline_seconds,
+                last_attempt,
+            )
+            return
+        verbose_proxy_logger.error(
+            "Gave up creating the spend views: table %s did not appear within %ss. "
+            "Run the database migrations against this database and restart the proxy.",
+            _VIEW_SETUP_GATE_TABLE,
+            deadline_seconds,
+        )
+
+    async def _view_setup_gate_table_present(self) -> bool:
+        rows: Final = _VIEW_SETUP_GATE_PROBE_ROWS.validate_python(
+            await self.db.query_raw("SELECT to_regclass($1) IS NOT NULL AS present", _VIEW_SETUP_GATE_TABLE)
+        )
+        return rows[0]["present"]
+
     async def _db_health_watchdog_loop(self) -> None:
         while True:
             try:

litellm/proxy/wildcard_config.yaml

@@ -49,4 +49,3 @@ general_settings:
 
 litellm_settings:
   drop_params: True
-  telemetry: False

litellm/router_strategy/adaptive_router/hooks.py

@@ -86,7 +86,7 @@ def _resolve_session_key(kwargs: dict[str, Any]) -> str | None:
     return hashlib.sha256(payload.encode("utf-8")).hexdigest()
 
 
-def _last_user_content(messages: list[dict[str, Any]] | None) -> str | None:
+def _last_user_content(messages: Sequence[Mapping[str, object]] | None) -> str | None:
     if not messages:
         return None
     for msg in reversed(messages):

litellm/router_strategy/auto_router/auto_router.py

@@ -3,7 +3,7 @@ Auto-Routing Strategy that works with a Semantic Router Config
 """
 
 import asyncio
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, Optional
 
 from pydantic import BaseModel, ConfigDict
@@ -158,7 +158,7 @@ class AutoRouter(CustomLogger):
         return await asyncio.shield(build_task)
 
     @staticmethod
-    def _extract_text_from_messages(messages: list[dict[str, Any]]) -> str:
+    def _extract_text_from_messages(messages: Sequence[Mapping[str, object]]) -> str:
         """
         Extract text content from the last user message for routing.
 

model_prices_and_context_window.json

@@ -41329,6 +41329,7 @@
     },
     "openrouter/deepseek/deepseek-v3.2": {
         "cache_read_input_token_cost": 1.345e-07,
+        "deprecation_date": "2026-09-28",
         "input_cost_per_token": 2.69e-07,
         "input_cost_per_token_cache_hit": 1.345e-07,
         "litellm_provider": "openrouter",
@@ -41350,6 +41351,7 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v3.2-exp": {
+        "deprecation_date": "2026-09-28",
         "input_cost_per_token": 2.7e-07,
         "input_cost_per_token_cache_hit": 2e-08,
         "litellm_provider": "openrouter",
@@ -43114,6 +43116,7 @@
         "output_cost_per_token": 1.2e-06,
         "cache_creation_input_token_cost": 0.0,
         "cache_read_input_token_cost": 3e-08,
+        "deprecation_date": "2026-10-08",
         "litellm_provider": "openrouter",
         "max_input_tokens": 204800,
         "max_output_tokens": 131072,
@@ -66773,9 +66776,9 @@
         "supports_web_search": true
     },
     "openrouter/z-ai/glm-5.2": {
-        "input_cost_per_token": 5.544e-07,
-        "output_cost_per_token": 1.7424e-06,
-        "cache_read_input_token_cost": 1.0296e-07,
+        "input_cost_per_token": 6.496e-07,
+        "output_cost_per_token": 2.0416e-06,
+        "cache_read_input_token_cost": 1.2064e-07,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 131072,
@@ -67135,9 +67138,9 @@
         "supports_web_search": true
     },
     "openrouter/deepseek/deepseek-v4-flash": {
-        "input_cost_per_token": 3.724e-08,
-        "output_cost_per_token": 7.448e-08,
-        "cache_read_input_token_cost": 7.448e-09,
+        "input_cost_per_token": 3.668e-08,
+        "output_cost_per_token": 7.336e-08,
+        "cache_read_input_token_cost": 7.336e-09,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
@@ -67950,6 +67953,7 @@
         "input_cost_per_token": 2.7e-07,
         "output_cost_per_token": 1e-06,
         "cache_read_input_token_cost": 1.35e-07,
+        "deprecation_date": "2026-09-28",
         "litellm_provider": "openrouter",
         "max_input_tokens": 163840,
         "max_output_tokens": 32768,
@@ -68691,6 +68695,7 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-r1-distill-llama-70b": {
+        "deprecation_date": "2026-09-28",
         "input_cost_per_token": 8e-07,
         "output_cost_per_token": 8e-07,
         "litellm_provider": "openrouter",
@@ -71964,6 +71969,7 @@
         "supports_web_search": false
     },
     "openrouter/baidu/ernie-4.5-vl-424b-a47b": {
+        "deprecation_date": "2026-10-08",
         "input_cost_per_token": 4.2e-07,
         "litellm_provider": "openrouter",
         "max_input_tokens": 123000,

proxy_server_config.yaml

@@ -173,7 +173,6 @@ litellm_settings:
   # budget_duration: 30d
   num_retries: 5
   request_timeout: 600
-  telemetry: False
   context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}]
   default_team_settings: 
     - team_id: team-1

pyproject.toml

@@ -196,6 +196,7 @@ dev = [
     "mypy==1.20.1",
     "keyring==25.7.0",
     "pytest==9.0.3",
+    "pytest-socket==0.8.1",
     "tomli==2.4.1; python_version < '3.11'",
     "pytest-mock==3.15.1",
     "pytest-asyncio==1.3.0",

scripts/benchmark_anthropic_messages_perf.py

@@ -256,7 +256,6 @@ general_settings:
   master_key: {api_key}
 
 litellm_settings:
-  telemetry: false
 """,
         encoding="utf-8",
     )

scripts/benchmark_chat_completions_perf.py

@@ -228,7 +228,6 @@ general_settings:
 
 litellm_settings:
   drop_params: true
-  telemetry: false
 """,
         encoding="utf-8",
     )

scripts/budget_ratchet_check.py

@@ -7,13 +7,17 @@ driven DOWN over time. This check compares every budget file against its own
 content at the merge-base with the target branch and fails (exits 1, red) if:
 
   * a rule's `limit` went up,
-  * a rule was dropped from a budget (its ceiling effectively became infinite), or
+  * a rule was dropped from a budget (its ceiling effectively became infinite) while
+    its checker still emits it, or
   * an entire budget file was deleted.
 
 New rules and lowered/equal limits are fine. So is a rule that graduated: once a
 paired config (ruff.toml for the ruff-strict budget) selects the rule outright it
 hard-fails at the first violation, which is stricter than any ceiling the budget
 could hold, so dropping its entry tightens the guard rather than removing it.
+Likewise a retired rule: once the paired checker (check_test_quality.py for the
+test-quality budget) no longer emits a code, its entry has no ceiling left to
+loosen.
 
 This is deliberately NOT a gating check. It should turn the run red so that a
 loosening is impossible to miss in review, but it must stay OUT of the
@@ -29,11 +33,12 @@ Usage:
 from __future__ import annotations
 
 import argparse
+import importlib.util
 import json
 import subprocess
 import sys
 from pathlib import Path
-from types import MappingProxyType
+from types import MappingProxyType, ModuleType
 from typing import Final, NamedTuple
 
 if sys.version_info >= (3, 11):
@@ -49,6 +54,7 @@ DEFAULT_BUDGETS: tuple[str, ...] = (
     "test-quality-budget.json",
 )
 GRADUATION_CONFIGS = MappingProxyType({"ruff-strict-budget.json": "ruff.toml"})
+RETIREMENT_SOURCES = MappingProxyType({"test-quality-budget.json": "check_test_quality"})
 
 
 class Regression(NamedTuple):
@@ -139,20 +145,40 @@ def graduated_selectors(rel: str) -> tuple[str, ...]:
     )
 
 
+def _load_script(name: str) -> ModuleType:
+    if name in sys.modules:
+        return sys.modules[name]
+    spec: Final = importlib.util.spec_from_file_location(name, REPO_ROOT / "scripts" / f"{name}.py")
+    assert spec is not None and spec.loader is not None
+    module: Final = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def retired_rules(rel: str, base: dict[str, object]) -> frozenset[str]:
+    """Rules in the base budget that the paired checker can no longer emit, so there is no ceiling to loosen."""
+    source: Final = RETIREMENT_SOURCES.get(rel)
+    if source is None:
+        return frozenset()
+    return frozenset(_limits(base)) - _load_script(source).RULE_CODES
+
+
 def _regression_detail(
     rule: str,
     base_limits: dict[str, int],
     head_limits: dict[str, int],
     graduated: tuple[str, ...],
+    retired: frozenset[str] = frozenset(),
 ) -> str | None:
-    """Why `rule` regressed vs base, or None when it held flat, fell, or graduated.
+    """Why `rule` regressed vs base, or None when it held flat, fell, or left the budget legitimately.
 
-    A dropped rule is terminal unless it graduated; otherwise the only loosening
-    left is a raised limit.
+    A dropped rule is terminal unless it graduated or retired; otherwise the only
+    loosening left is a raised limit.
     """
     base_limit = base_limits[rule]
     if rule not in head_limits:
-        if graduated and rule.startswith(graduated):
+        if rule in retired or (graduated and rule.startswith(graduated)):
             return None
         return f"rule dropped (limit {base_limit} -> removed)"
     if head_limits[rule] > base_limit:
@@ -165,6 +191,7 @@ def regressions_for(
     base: dict | None,
     head: dict | None,
     graduated: tuple[str, ...] = (),
+    retired: frozenset[str] = frozenset(),
 ) -> list[Regression]:
     if base is None:
         return []  # new budget file: nothing to ratchet against yet
@@ -175,7 +202,7 @@ def regressions_for(
     return [
         Regression(rel, rule, detail)
         for rule in sorted(base_limits)
-        if (detail := _regression_detail(rule, base_limits, head_limits, graduated)) is not None
+        if (detail := _regression_detail(rule, base_limits, head_limits, graduated, retired)) is not None
     ]
 
 
@@ -209,7 +236,7 @@ def main() -> int:
             print(f"skip {rel}: new file (no base at {base_ref} to ratchet against)")
             continue
         checked.append(rel)
-        regressions.extend(regressions_for(rel, base, head, graduated_selectors(rel)))
+        regressions.extend(regressions_for(rel, base, head, graduated_selectors(rel), retired_rules(rel, base)))
 
     if regressions:
         print(

scripts/check_test_quality.py

@@ -48,10 +48,6 @@ TQ006   A `pytest.skip` reached only when a credential-shaped environment variab
         deliberate branch. The gate follows one local or module-level binding, which is
         the `key = os.getenv(...)` then `if not key: pytest.skip(...)` shape most of
         these use.
-TQ008   A `patch(...)` whose target is a `litellm.` internal. Patching the SDK's own
-        functions pins the test to the current wiring instead of the behaviour, and it
-        is the idiom the suite reaches for instead of faking the HTTP boundary. Mocking
-        a third-party client, a transport, or anything outside `litellm.` is untouched.
 TQ007   A module global that a conftest saves before every test and restores after it.
         The save/restore list is a hand-maintained inventory of the leaks the suite
         already knows about, so it is allowed to shrink and never to grow: a new entry
@@ -150,6 +146,10 @@ SDK_MODULE: Final = "litellm"
 SUBPROCESS_SPAWNS: Final = frozenset(("run", "Popen", "check_output", "check_call", "call"))
 INTERPRETER_ISOLATION_FLAGS: Final = frozenset(("-I", "-P"))
 
+RULE_CODES: Final = frozenset((
+    "TQ000", "TQ001", "TQ002", "TQ003", "TQ004", "TQ005", "TQ006", "TQ007", "TQ009",
+))
+
 CREDENTIAL_NAME_RE: Final = re.compile(
     r"(?:API_KEY|_KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|DATABASE_URL|ACCESS_KEY_ID)$"
 )
@@ -483,67 +483,6 @@ def iter_global_mutation_violations(path: Path, tree: ast.Module) -> Iterator[Vi
                 )
 
 
-def _is_sdk_internal(dotted: str) -> bool:
-    return dotted == SDK_MODULE or dotted.startswith(f"{SDK_MODULE}.")
-
-
-def _sdk_import_bindings(tree: ast.Module) -> Iterator[tuple[str, str]]:
-    """(local name, dotted path) for every import that binds something under `litellm`."""
-    for node in ast.walk(tree):
-        if isinstance(node, ast.Import):
-            yield from (
-                (alias.asname, alias.name) if alias.asname else (root, root)
-                for alias in node.names
-                if _is_sdk_internal(alias.name)
-                for root in (alias.name.partition(".")[0],)
-            )
-        elif isinstance(node, ast.ImportFrom) and node.module and _is_sdk_internal(node.module):
-            yield from ((alias.asname or alias.name, f"{node.module}.{alias.name}") for alias in node.names)
-
-
-def _sdk_aliases(tree: ast.Module) -> Mapping[str, str]:
-    """Local names bound to something under `litellm`, mapped to the path they stand for.
-
-    `from litellm.llms.openai.chat import handler` then `patch.object(handler.X, ...)`
-    reaches the same internal as the dotted string form and has to read the same way.
-    """
-    return MappingProxyType({name: dotted for name, dotted in _sdk_import_bindings(tree)})
-
-
-def _resolved(dotted: str, aliases: Mapping[str, str]) -> str:
-    root, _, rest = dotted.partition(".")
-    base: Final = aliases.get(root, root)
-    return f"{base}.{rest}" if rest else base
-
-
-def _patch_targets(call: ast.Call, aliases: Mapping[str, str]) -> Iterator[str]:
-    """What a patch installer is replacing: the dotted string it names, or the
-    attribute chain handed to `patch.object` / `patch.dict`, resolved through the
-    module's imports so a locally bound SDK object reads as its full path."""
-    for first in call.args[:1]:
-        if isinstance(first, ast.Constant) and isinstance(first.value, str):
-            yield first.value
-        elif dotted := _dotted_name(first):
-            yield _resolved(dotted, aliases)
-
-
-def iter_internal_patch_violations(path: Path, tree: ast.Module) -> Iterator[Violation]:
-    aliases: Final = _sdk_aliases(tree)
-    for node in ast.walk(tree):
-        if not (isinstance(node, ast.Call) and _is_patch_installer(_dotted_name(node.func))):
-            continue
-        for target in _patch_targets(node, aliases):
-            if _is_sdk_internal(target):
-                yield Violation(
-                    path,
-                    node.lineno,
-                    "TQ008",
-                    f"patches `{target}`, an SDK internal, so the test is pinned to how the code is "
-                    "wired rather than what it does; fake the HTTP boundary (respx / MockTransport) "
-                    f"or inject the collaborator (suppress: `# {SUPPRESSION_TOKEN}: <reason>`)",
-                )
-
-
 def _environ_keys(node: ast.AST) -> Iterator[str]:
     for inner in ast.walk(node):
         if isinstance(inner, ast.Call) and _dotted_name(inner.func) in ENVIRON_READERS:
@@ -784,7 +723,6 @@ def check_file(path: Path) -> tuple[Violation, ...]:
             *iter_global_mutation_violations(path, tree),
             *iter_credential_skip_violations(path, tree),
             *iter_conftest_inventory_violations(path, tree),
-            *iter_internal_patch_violations(path, tree),
             *iter_child_interpreter_violations(path, tree),
         )
         if violation.line not in skip

test-quality-budget.json

@@ -20,9 +20,6 @@
   "TQ007": {
     "limit": 117
   },
-  "TQ008": {
-    "limit": 10993
-  },
   "TQ009": {
     "limit": 59
   }

tests/AGENTS.md

@@ -0,0 +1,38 @@
+# Tests
+
+Nothing on the other side of the call: `tests/unit`. A proxy we start with an upstream we script:
+`tests/integration`. Someone else's service with real credentials: `tests/e2e`. Two fit, split it
+
+## What good looks like
+
+Red when the claim in the name is broken. Prove it: mutate the behaviour, red; restore, green. Put the
+mutation in the PR body
+
+```python
+def test_custom_price_is_reported_and_charged(gateway: Gateway) -> None:
+    with gateway.scenario() as scenario:
+        model = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
+        response = gateway.request("POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": "price control"}]})
+        assert response.status_code == 200, response.text
+        assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(20 * 0.001 + 20 * 0.002)
+```
+
+Rates in the test, expected computed by hand, one call, `response.text` in the assert
+
+Assert the whole value. Iterating `expected_body.items()` (`test_responses_api_request_body.py`) cannot
+see an extra key; that is the shape of `stream_options.include_usage` (#19777, #28553)
+
+The linter catches no-assert, mock-echo and credential skips. It cannot see an assert
+behind an `if` (a poll that ends in `pytest.fail` is fine), `except Exception` around the call
+(`test_router.py`: `except Exception as e: print(f"FAILED TEST")`), or blanket `--reruns`
+
+## Where it goes
+
+What the assertion depends on goes in the test; everything else in conftest. A rate in a fixture three
+directories up makes a failed assertion unreadable. Extend the file that already covers the behaviour
+
+## Writing it so a human can read it
+
+Name says what broke: `test_send_batched_with_valid_data` says nothing. Build, one call, assert, on one
+screen. Helpers named for what they return, `_pii_prompt(marker, email)`, not `_setup()`. Context in the
+assert message, not a comment

tests/code_coverage_tests/test_e2e_changed_gate.py

@@ -1,3 +1,4 @@
+import os
 import subprocess
 import sys
 import xml.etree.ElementTree as ET
@@ -131,6 +132,7 @@ def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]:
     (
         (("tests/e2e/logging/test_datadog_e2e.py", "litellm/router.py"), ("tests/e2e/logging/test_datadog_e2e.py",)),
         (("tests/e2e/ui/test_keys.py", "tests/e2e/claude_code/test_cli.py", "tests/e2e/load/test_burst.py"), ()),
+        (("tests/e2e/migrations/test_startup.py", "tests/e2e/migrations/test_recovery.py"), ()),
         (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()),
         (("tests/e2e/guardrails/test_presidio_masking_e2e.py",), ()),
         (("tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py",), ()),
@@ -180,6 +182,10 @@ def test_a_changed_canary_file_is_selected_once_alongside_a_harness_change() ->
     assert select_tests((CANARY[1], "tests/e2e/proxy_client.py")) == CANARY
 
 
+def test_dedicated_migration_tests_do_not_suppress_shared_harness_canaries() -> None:
+    assert select_tests(("tests/e2e/migrations/test_startup.py", "tests/e2e/conftest.py")) == CANARY
+
+
 def test_the_canary_joins_directly_selected_files_in_sorted_order() -> None:
     assert select_tests(("tests/e2e/logging/test_datadog_e2e.py", ".github/e2e-stack/up.sh")) == (
         *CANARY,
@@ -226,3 +232,61 @@ def test_an_unusable_secret_is_named_without_printing_its_value(
     assert unprintable not in result.stderr
     assert result.stdout == ""
     assert not env_path.exists()
+
+
+@pytest.mark.parametrize("phase", ("setup", "call", "teardown"))
+@pytest.mark.parametrize("required_count", ("1", "4"))
+def test_oauth_failure_diagnostics_do_not_publish_private_payloads(
+    tmp_path: Path, phase: str, required_count: str
+) -> None:
+    suite: Final = ET.Element("testsuite")
+    case: Final = ET.SubElement(suite, "testcase", file=SELECTED[0])
+    private: Final = "private-token-in-exception-message"
+    failure: Final = ET.SubElement(case, "failure", message=private)
+    failure.text = private
+    properties: Final = ET.SubElement(case, "properties")
+    for name, value in (
+        ("oauth_failure_phase", phase),
+        ("oauth_exception_type", "AssertionError"),
+        ("oauth_frame", "oauth_gateway.py:120:start"),
+        ("oauth_frame", f"injected\\n{private}"),
+        ("unrelated_property", private),
+    ):
+        _ = ET.SubElement(properties, "property", name=name, value=value)
+    report: Final = tmp_path / "report.xml"
+    ET.ElementTree(suite).write(report)
+    result: Final = subprocess.run(
+        [sys.executable, "-I", str(GATE), str(report), SELECTED[0]],
+        capture_output=True,
+        text=True,
+        env={**os.environ, "E2E_REQUIRED_TEST_COUNT": required_count},
+    )
+    assert result.returncode == 1
+    assert f"oauth_failure_phase: {phase}" in result.stdout
+    assert "oauth_exception_type: AssertionError" in result.stdout
+    assert "oauth_frame: oauth_gateway.py:120:start" in result.stdout
+    assert private not in result.stdout + result.stderr
+
+
+@pytest.mark.parametrize(
+    ("count", "skip", "expected"), ((0, False, 1), (3, False, 1), (4, False, 0), (5, False, 1), (4, True, 1))
+)
+def test_required_count_reports_cases_before_rejecting(tmp_path: Path, count: int, skip: bool, expected: int) -> None:
+    suite = ET.Element("testsuite")
+    for index in range(count):
+        case = ET.SubElement(suite, "testcase", file=SELECTED[0], classname="OAuth", name=f"variant{index}")
+        if skip and index == 0:
+            ET.SubElement(case, "skipped", message="private-skip-reason")
+    report = tmp_path / "report.xml"
+    ET.ElementTree(suite).write(report)
+    result = subprocess.run(
+        [sys.executable, "-I", str(GATE), str(report), SELECTED[0]],
+        env={**os.environ, "E2E_REQUIRED_TEST_COUNT": "4"},
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == expected
+    assert f"{count} collected, {int(skip)} skipped" in result.stdout
+    if skip:
+        assert "skipped: OAuth::variant0" in result.stdout
+    assert "private-skip-reason" not in result.stdout + result.stderr

tests/e2e/AGENTS.md

@@ -2,10 +2,36 @@
 
 Code-style rules for writing tests under `tests/e2e/`. The harness already encodes the plumbing; your job is the feature-specific behavior, not reinventing it. For what a complete test must do (the lifecycle contract, asserting both recorded state and enforced behavior) and how to run a suite, see `CONTRIBUTING.md` in this directory. Repo-wide conventions live in the root `AGENTS.md`
 
+## What good looks like
+
+Only what a real provider proves. If it holds against our scripted upstream: `tests/integration`
+
+```python
+def test_pre_call_masks_pii_on_chat_completions(self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str) -> None:
+    name = f"e2e-presidio-pre-chat-{unique_marker()}"
+    _register_presidio(client, resources, name=name)
+    email = _fake_email()
+    _assert_eventually_masked(
+        lambda: client.chat(scoped_key, MODEL, _pii_prompt(unique_marker(), email), guardrails=[name], max_tokens=128),
+        _first_content,
+        email=email,
+    )
+```
+
+Marker per run, so a leftover guardrail cannot pass it. `resources.defer(...)` at creation, so a failed
+assert still tears down. Assert what the caller receives
+
+## Where it goes
+
+By the surface a customer would name: `guardrails`, `llm_translation`, `management`. Mutation check
+deferred; it needs credentials
+
 ## Suite folders
 
 Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family or behavior area. If you add a new folder, you must add a line here describing what kind of tests belong in it, so the layout stays self-describing. `gateway/` is the exception: it holds proxy configuration only and never tests
 
+- `migrations/` - isolated Docker startup, concurrent migration, crash recovery, and legacy database compatibility. The CircleCI migration workflow enables `LITELLM_MIGRATION_TESTS=1`; these tests own their proxy containers and databases, so they do not use the shared proxy preflight or shared database cleanup
+
 - `llm_translation/` - LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown
 - `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation
 - `embeddings/` - the `/embeddings` endpoint across providers
@@ -14,7 +40,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
 - `quota_management/` - quota enforcement and accounting, one subfolder per behavior: `ratelimit/` (rpm/tpm blocks, window reset, pacing headers on live traffic), `budgets/` (budget definition, enforcement, and reset windows: key, team, tag, soft, multi-window), and `spend_tracking/` (spend logging and cost attribution on `/spend/*`)
 - `management/` - key/team/user/organization management routes: create/update/delete persistence via the info routes, team membership, and llm-only-key route denials (API surface; not Playwright)
 - `a2a/` - the A2A (agent-to-agent) surface: admin registration via `/v1/agents`, proxy-fronted card discovery at `/.well-known/agent-card.json`, and JSON-RPC `message/send` invocation, driving agents backed by the litellm completion bridge (a real provider) and asserting protocol-version normalization (0.3 vs 1.0)
-- `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion lists and executes the server's tools with the stored per-user token
+- `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions` in `test_mcp_chat_completion_oauth_e2e.py` and direct MCP protocol operations in `test_mcp_oauth_happy_path_e2e.py`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion or protocol call lists and executes the server's tools with the stored per-user token
 - `logging/` - logging-integration delivery (datadog and friends)
 - `security/` - secret handling and log-leak protection
 - `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory regression test (`test_reliability_memory_e2e.py`: a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak)
@@ -26,14 +52,14 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
 
 ## MCP suite: real Datadog only
 
-Every test under `tests/e2e/mcp/` must exercise the proxy against the real Datadog remote MCP server. Do not add a compose service, FastMCP fixture, mock upstream, or any other fake MCP host for this suite
+Every test under `tests/e2e/mcp/` must exercise the proxy against the real Datadog remote MCP server, except the two Linear OAuth tests `test_mcp_chat_completion_oauth_e2e.py` and `test_mcp_oauth_happy_path_e2e.py`. Do not add a compose service, FastMCP fixture, mock upstream, or any other fake MCP host for this suite
 
 - Register via `register_datadog_mcp` in `tests/e2e/mcp/datadog_mcp.py` (or extend that helper if you need a different `toolsets=` / `allowed_tools` slice of the same Datadog endpoint). That posts `/v1/mcp/server` with `url=datadog_mcp_url(...)` and static headers `DD-API-KEY` / `DD-APPLICATION-KEY` from the process env
 - Auth is Datadog's documented CI/header path, not a browser OAuth authorize/token dance. Hard-fail when `DD_API_KEY` or `DD_APP_KEY` is missing (`assert_dd_mcp_creds`); never skip for a missing fake upstream
 - Prefer calling real Datadog tools that prove the product path (e.g. `search_datadog_logs` for list/call and permission denials). Seed a unique marker (`e2e-datadog-mcp-*`) in a chat completion when you need a log the tool can find; dual-read with `dd_logs` from conftest when delivery matters
 - Delete the MCP server (and any keys) through `resources.defer` the same way every other suite tears down
 - If a new MCP behavior cannot be covered with Datadog's tool surface, say so in the PR and get agreement before inventing another upstream; the default is always Datadog
-- The one standing exception is `test_mcp_chat_completion_oauth_e2e.py`. Datadog authenticates with the static `DD-API-KEY` / `DD-APPLICATION-KEY` headers and exposes no authorize/token dance at all, so it cannot exercise gateway-managed OAuth or per-user token seeding in any form. That test drives a real Linear MCP server instead; it is still a real remote upstream, so the no-mock, no-fixture rule above holds unchanged
+- The two standing exceptions are `test_mcp_chat_completion_oauth_e2e.py` and `test_mcp_oauth_happy_path_e2e.py`. Datadog authenticates with the static `DD-API-KEY` / `DD-APPLICATION-KEY` headers and exposes no authorize/token dance at all, so these tests drive a real Linear MCP server instead; they are still real remote upstreams, so the no-mock, no-fixture rule above holds unchanged. The direct OAuth test also uses the existing live provider edge to inspect forwarded headers without replay, and owns a separate source-built gateway for cold restarts
 
 ## Lay the pattern down in a class
 
@@ -152,7 +178,7 @@ MCPs - endpoint features with the protocol op as the variant
 mcp.<operation>.<auth_family>.<assertion>
   operation   : list_tools | call_tool | list_resources | read_resource | list_prompts | get_prompt
   auth_family : none | api_key | bearer | oauth
-  assertion   : succeeds | denied_without_permission
+  assertion   : succeeds | denied_without_permission | persists_across_processes
   e.g.  mcp.call_tool.oauth.succeeds
 ```
 

tests/e2e/CONTRIBUTING.md

@@ -248,3 +248,56 @@ The semantic header set is `content-type`, `accept`, `anthropic-version`, `anthr
 Excluded transport and telemetry headers are `host`, `content-length`, `connection`, `accept-encoding`, `user-agent`, `traceparent`, `tracestate`, `x-request-id`, `x-client-request-id` and `x-stainless-*`. Inbound transfer-encoding is unsupported; send JSON with content-length framing. The destination represents host identity and the relay carries original body bytes. Replay does not verify credentials, SDK timeout/retry behavior, transport performance, model availability or stateful remote IDs. Live relay uses original request bytes and header values, never the stored identity
 
 Strict replay harness regression tests live in `tests/code_coverage_tests/test_provider_replay_harness.py`. The CircleCI `provider_replay_harness` job runs them alongside the existing legacy harness files with `--noconftest -o pythonpath=tests/e2e`; they need only synthetic HTTP providers and temporary fixture storage
+
+
+## MCP OAuth happy path
+
+`test_mcp_oauth_happy_path_e2e.py` runs one shared scenario with four variants:
+aggregate gateway SSO and explicitly configured per-server JWT, each directly
+against Linear and through the live provider edge. The edge forwards to real
+Linear without replay and compares the forwarded bearer to the encrypted
+canonical user/server credential. This observes the forwarding boundary, not
+Linear's internal logs. Direct variants independently exercise discovery
+
+Use the existing database preparation, Prisma generation and Keycloak setup.
+Build and stage the dashboard from the tested checkout as in the UI runner.
+Provide `DATABASE_URL`, `LITELLM_MASTER_KEY`, `LITELLM_SALT_KEY`, `LITELLM_LICENSE`,
+and the `E2E_KEYCLOAK_*` settings. Capture a test-account Linear login using
+`mcp/linear_session_capture.py` and set `E2E_LINEAR_STORAGE_STATE` to that private
+file. The test workspace must contain a team. Do not publish browser state or
+raw test/proxy output
+
+```bash
+E2E_MCP_OAUTH_LIVE=1 E2E_FIXTURE_MODE=live E2E_PROVIDER_CACHE=0 \
+  uv run --no-sync pytest tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py \
+  --rootdir=. --reruns 0
+```
+
+The test starts and restarts its own source-built proxy on a free loopback port,
+retaining its database and SSO client but no Redis or process-local cache. It
+does not restart an existing proxy or clear shared databases. Gateway login,
+consent, immediate list/call and post-restart reconnect must all succeed. The
+aggregate client never injects a gateway header; the explicitly labeled JWT
+variant configures `x-litellm-api-key` for the first consent and reconnects with
+only its gateway JWT after restart
+
+`.github/workflows/test-mcp-oauth-e2e.yml` automatically requests a run for
+same-repository pull requests changing MCP, gateway authentication/SSO, consent
+UI, dependencies or the relevant E2E harness/workflow paths. It retains manual
+`workflow_dispatch` for targeted verification. The four cases run in the
+protected `e2e-changed` environment after its normal deployment approval;
+reviewers should approve and inspect this separate OAuth check when it appears.
+Fork pull requests do not run this credentialed job; use a reviewed
+same-repository branch for their verification. The workflow's path-filtered
+check is not configured here as a globally required branch-protection check.
+Provision `E2E_LINEAR_STORAGE_STATE_B64` as a secret there and retain the existing E2E license/AWS role configuration. A missing or
+expired session fails the job; collection, deselection and skips are not passes.
+The generic changed-test job excludes this file because it requires an owned
+proxy and consent UI. No LLM call is needed
+
+Coverage remains limited to authorization-code OAuth over HTTP. M2M, OBO,
+PKCE passthrough, static/BYOK, ID-JAG, forwarding, SigV4 and stdio are outside this
+scenario; consult the registry and LIT-3559 for their existing coverage and gaps.
+LIT-4506 owns broader isolation/failure regressions. LIT-7737 retains ownership
+of dependency/Python compatibility and its matrix; this test reuses its delivered
+environment and does not change dependency constraints or compatibility gates

tests/e2e/conftest.py

@@ -17,6 +17,7 @@ import functools
 import os
 from collections.abc import Generator, Iterator
 from datetime import datetime, timezone
+from pathlib import Path
 from types import MappingProxyType
 from typing import Final
 
@@ -28,6 +29,7 @@ from e2e_config import (
     FIXTURE_DIR,
     FIXTURE_MODE_RAW,
     MANAGED_FILES_OPT_IN_ENV,
+    MCP_OAUTH_LIVE_OPT_IN_ENV,
     PROMPT_CACHING_OPT_IN_ENV,
     PROXY_BASE_URL,
     REDIS_CHAOS_OPT_IN_ENV,
@@ -56,6 +58,7 @@ OPT_IN_MARKERS: Final = MappingProxyType(
         "prompt_caching_stack": PROMPT_CACHING_OPT_IN_ENV,
         "redis_chaos": REDIS_CHAOS_OPT_IN_ENV,
         "cli_determinism": CLI_DETERMINISM_OPT_IN_ENV,
+        "mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV,
     }
 )
 
@@ -88,6 +91,9 @@ def jwt_identity(idp: Keycloak, resources: ResourceManager, proxy: ProxyClient)
 
 
 def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line(
+        "markers", "migration_startup: isolated container startup tests run by the migration CI workflow"
+    )
     config.addinivalue_line(
         "markers",
         "provider_live: requires actual provider timing, limits, state, or a response that echoes this"
@@ -132,6 +138,11 @@ def pytest_configure(config: pytest.Config) -> None:
         "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from "
         "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set",
     )
+    config.addinivalue_line(
+        "markers",
+        "mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless "
+        "E2E_MCP_OAUTH_LIVE is set",
+    )
 
 
 def pytest_sessionstart(session: pytest.Session) -> None:
@@ -177,6 +188,11 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item
         items[:] = [item for item in items if not _needs_unset_opt_in(item)]
     for item in items:
         attach_result_properties(item)
+    if os.environ.get("LITELLM_MIGRATION_TESTS") != "1":
+        deselected = [item for item in items if item.get_closest_marker("migration_startup") is not None]
+        items[:] = [item for item in items if item.get_closest_marker("migration_startup") is None]
+        if deselected:
+            deselected[0].config.hook.pytest_deselected(items=deselected)
     items.sort(key=lambda item: item.get_closest_marker("load") is not None)
 
 
@@ -211,7 +227,9 @@ def pytest_runtest_setup(item: pytest.Item) -> None:
     run even when none is up. Never skip for a missing proxy. Replay mode needs
     the proxy too: only provider-bound traffic replays from the bundle."""
     LIVE_PROVIDER_REQUIRED.set(item.get_closest_marker("provider_live") is not None)
-    if item.get_closest_marker("e2e") is None:
+    if item.get_closest_marker("e2e") is None or item.get_closest_marker("migration_startup") is not None:
+        return
+    if isinstance(item, pytest.Function) and "oauth_gateway" in item.fixturenames:
         return
     reason = _proxy_fail_reason()
     if reason is not None:
@@ -224,7 +242,7 @@ def pytest_runtest_call(item: pytest.Item) -> None:
     guard before truncating the spend-log DB. Tests under `tests/e2e/` without the
     `e2e` marker (pure unit coverage for the harness itself) never hit the proxy,
     so they must not arm the destructive DB truncate."""
-    if item.get_closest_marker("e2e") is None:
+    if item.get_closest_marker("e2e") is None or item.get_closest_marker("migration_startup") is not None:
         return
     item.session.stash[_E2E_TEST_RAN] = True
 
@@ -236,6 +254,13 @@ def pytest_runtest_makereport(
     """Stash the call-phase outcome so teardown can tell a passed test from a
     failed one without re-deriving it."""
     report = yield
+    if item.get_closest_marker("mcp_oauth_live") is not None and call.excinfo is not None:
+        # Publish code locations only, never exception messages, source text or locals.
+        item.user_properties.append(("oauth_failure_phase", report.when))
+        item.user_properties.append(("oauth_exception_type", call.excinfo.type.__name__))
+        for entry in call.excinfo.traceback:
+            item.user_properties.append(("oauth_frame", f"{Path(entry.path).name}:{entry.lineno + 1}:{entry.name}"))
+        report.user_properties = list(item.user_properties)
     if report.when == "call":
         item.stash[_CALL_PASSED] = report.passed
     return report

tests/e2e/coverage_registry/mcp.yaml

@@ -71,6 +71,14 @@
   assertions: [succeeds]
   source: "db.py user_oauth_credential lookup"
   rationale: OAuth2 token passthrough; per-user credential storage
+- id: mcp.call_tool.oauth.persists_across_processes
+  module: mcp
+  tier: P1
+  operation: call_tool
+  auth_family: oauth
+  assertions: [persists_across_processes]
+  source: "outbound_credentials/per_user_oauth_store.py V2PerUserTokenStore"
+  rationale: Stored per-user token survives a verified restart of an owned gateway with no Redis cache
 - id: mcp.list_tools.none.succeeds
   module: mcp
   tier: P1

tests/e2e/e2e_config.py

@@ -52,6 +52,7 @@ CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5")
 
 LINEAR_MCP_URL = os.environ.get("E2E_LINEAR_MCP_URL", "https://mcp.linear.app/mcp")
 LINEAR_STORAGE_STATE = os.environ.get("E2E_LINEAR_STORAGE_STATE", "")
+LINEAR_READONLY_TOOL: Final = "list_teams"  # as listed by tools/list on mcp.linear.app when PR #33787 landed
 
 # Jaeger query API of the compose stack's OTEL trace destination (the `jaeger`
 # service in docker-compose.yml maps it to host 16686). Trace-completeness tests
@@ -144,6 +145,7 @@ MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK"
 PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK"
 REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS"
 CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM"
+MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE"
 ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
 ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
 ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3"))

tests/e2e/idp.py

@@ -435,7 +435,7 @@ def _signal_process_group(process_id: int, signum: int) -> bool:
     return True
 
 
-def _stop_process_group(child: subprocess.Popen[bytes]) -> None:
+def stop_process_group(child: subprocess.Popen[bytes]) -> None:
     _signal_process_group(child.pid, signal.SIGTERM)
     deadline: Final = time.monotonic() + 5
     while _process_group_exists(child.pid):
@@ -476,7 +476,7 @@ def run_oidc_profile(proxy_url: str, command: list[str]) -> int:
             try:
                 return child.wait()
             finally:
-                _stop_process_group(child)
+                stop_process_group(child)
 
 
 if __name__ == "__main__":

tests/e2e/mcp/oauth_chat_client.py

@@ -18,7 +18,7 @@ import asyncio
 import re
 import time
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Final
 from urllib.parse import parse_qsl
 
 import httpx
@@ -26,11 +26,21 @@ import httpx2
 import pytest
 from e2e_config import PROXY_BASE_URL, REQUEST_TIMEOUT
 from e2e_http import AuthHeaders, NoBody, unwrap
+from idp import Identity
 from mcp import ClientSession
 from mcp.client.auth import OAuthClientProvider
 from mcp.client.streamable_http import streamable_http_client
 from mcp.shared.auth import AuthorizationCodeResult, OAuthClientInformationFull, OAuthClientMetadata, OAuthToken
-from models import ChatBody, ChatResponse, McpServerCreateBody, McpServerInfo
+from mcp.types import TextContent
+from models import (
+    ChatBody,
+    ChatResponse,
+    McpOauthUserCredentialStatus,
+    McpServerCreateBody,
+    McpServerInfo,
+    McpServerUserCredentialListResponse,
+    McpServerUserCredentialRow,
+)
 from proxy_client import ProxyClient
 
 if TYPE_CHECKING:
@@ -44,8 +54,8 @@ OAUTH_CLIENT_REDIRECT_URI = "http://127.0.0.1:53682/e2e/callback"
 BROWSER_CONSENT_TIMEOUT = 60.0
 
 
-def _mcp_url(alias: str) -> str:
-    return f"{PROXY_BASE_URL}/{alias}/mcp"
+def _mcp_url(alias: str, base_url: str = PROXY_BASE_URL) -> str:
+    return f"{base_url.rstrip('/')}/{alias}/mcp"
 
 
 class InMemoryTokenStorage:
@@ -69,7 +79,13 @@ class InMemoryTokenStorage:
         self._client_info = client_info
 
 
-async def _browser_follow_authorize(start_url: str, storage_state_path: str) -> tuple[str, str | None]:
+async def _browser_follow_authorize(
+    start_url: str,
+    storage_state_path: str,
+    identity: Identity | None = None,
+    server_alias: str | None = None,
+    allow_upstream_consent: bool = True,
+) -> tuple[str, str | None]:
     """Play the browser's role for a real upstream whose authorize endpoint
     serves an interactive consent page (Linear). A headless Chromium primed
     with a human's saved Linear session opens the gateway authorize URL and
@@ -85,6 +101,9 @@ async def _browser_follow_authorize(start_url: str, storage_state_path: str) ->
 
     def _note_request(request: object) -> None:
         url = getattr(request, "url", "")
+        host = httpx.URL(url).host
+        if not allow_upstream_consent and (host == "linear.app" or host.endswith(".linear.app")):
+            captured["upstream_consent"] = "seen"
         if url.startswith(OAUTH_CLIENT_REDIRECT_URI) and "url" not in captured:
             captured["url"] = url
 
@@ -96,7 +115,7 @@ async def _browser_follow_authorize(start_url: str, storage_state_path: str) ->
         context = await browser.new_context(storage_state=storage_state_path)
         await context.route(re.compile(re.escape(OAUTH_CLIENT_REDIRECT_URI) + r".*"), _swallow_redirect)
         page = await context.new_page()
-        page.on("request", _note_request)
+        context.on("request", _note_request)
         page.on("framenavigated", lambda frame: trail.append(frame.url.split("?", 1)[0]))
         await page.goto(start_url, wait_until="domcontentloaded")
         deadline = time.monotonic() + BROWSER_CONSENT_TIMEOUT
@@ -105,8 +124,29 @@ async def _browser_follow_authorize(start_url: str, storage_state_path: str) ->
                 await page.wait_for_load_state("networkidle", timeout=8000)
             except Exception:  # noqa: BLE001 - a busy consent page never idles; fall through and try to advance it
                 pass
-            if "url" in captured:
+            if "upstream_consent" in captured or "url" in captured:
                 break
+            if await page.locator("#username").count() and identity is not None:
+                await page.locator("#username").fill(identity.username)
+                await page.locator("#password").fill(identity.password)
+                await page.locator("#kc-login").click()
+                continue
+            if "/ui/connect" in page.url and server_alias is not None:
+                card = page.locator("div.cursor-pointer").filter(has=page.get_by_text(server_alias, exact=True))
+                if await card.count() != 1:
+                    await asyncio.sleep(0.5)
+                    continue
+                connect = card.get_by_text("Connect", exact=True)
+                if await connect.count():
+                    await connect.click()
+                    continue
+                if not await card.locator("svg.text-success").count():
+                    await asyncio.sleep(0.5)
+                    continue
+                finish = page.get_by_role("button", name="Finish connecting", exact=True)
+                if await finish.count() and await finish.is_enabled():
+                    await finish.click()
+                    continue
             control = page.locator(
                 'button[name="action"][value="approve"], button:has-text("Authorize"), '
                 'button:has-text("Allow"), button:has-text("@"), a:has-text("@")'
@@ -118,27 +158,44 @@ async def _browser_follow_authorize(start_url: str, storage_state_path: str) ->
         final_url = page.url
         await browser.close()
 
+    # A redirect chain can finish inside goto/networkidle before the loop checks the page.
+    assert "upstream_consent" not in captured, "cold reconnect required upstream consent"
     landing = captured.get("url")
     assert landing is not None, (
         f"consent flow never reached {OAUTH_CLIENT_REDIRECT_URI}; "
         f"final={final_url.split('?', 1)[0]!r}; trail={trail[-6:]}"
     )
     params = dict(parse_qsl(httpx.URL(landing).query.decode()))
-    assert "code" in params, f"client redirect_uri carried no code: {landing}"
+    assert "code" in params, "client redirect_uri carried no authorization code"
     return params["code"], params.get("state")
 
 
-def _oauth_provider(url: str, storage: InMemoryTokenStorage, storage_state_path: str) -> OAuthClientProvider:
+def _oauth_provider(
+    url: str,
+    storage: InMemoryTokenStorage,
+    storage_state_path: str | None,
+    identity: Identity | None = None,
+    server_alias: str | None = None,
+    allow_upstream_consent: bool = True,
+) -> OAuthClientProvider:
     """The SDK's real OAuth machinery (RFC 9728/8414 discovery, RFC 7591 DCR,
     PKCE, token exchange) with the browser leg driven by Playwright against the
     upstream's consent screen."""
     code_holder: dict[str, str | None] = {}  # mutable-ok: hand-off between the two SDK callbacks
 
-    async def redirect_handler(authorize_url: str) -> None:
-        code, state = await _browser_follow_authorize(authorize_url, storage_state_path)
+    async def _reject_redirect(_: str) -> None:
+        raise AssertionError("gateway demanded a fresh upstream consent; stored per-user token was not reused")
+
+    async def _follow_redirect(authorize_url: str) -> None:
+        assert storage_state_path is not None
+        code, state = await _browser_follow_authorize(
+            authorize_url, storage_state_path, identity, server_alias, allow_upstream_consent
+        )
         code_holder["code"] = code
         code_holder["state"] = state
 
+    redirect_handler: Final = _reject_redirect if storage_state_path is None else _follow_redirect
+
     async def callback_handler() -> AuthorizationCodeResult:
         code = code_holder.get("code")
         assert code is not None, "callback_handler ran before the authorize redirect completed"
@@ -167,24 +224,45 @@ class _HeaderInjectingTransport(httpx2.AsyncBaseTransport):
     store the upstream token for from the key on the token exchange, exactly
     like a production MCP host configured with a LiteLLM key header."""
 
-    def __init__(self, inner: httpx2.AsyncBaseTransport, headers: dict[str, str]) -> None:
+    def __init__(self, inner: httpx2.AsyncBaseTransport, headers: dict[str, str], gateway_url: str) -> None:
         self._inner = inner
         self._headers = headers
+        self._gateway_url = httpx2.URL(gateway_url)
+
+    @staticmethod
+    def _port(url: httpx2.URL) -> int | None:
+        if url.port is not None:
+            return url.port
+        return {"http": 80, "https": 443}.get(url.scheme)
 
     async def handle_async_request(self, request: httpx2.Request) -> httpx2.Response:
-        for name, value in self._headers.items():
-            if name not in request.headers:
-                request.headers[name] = value
+        same_origin: Final = (
+            request.url.scheme == self._gateway_url.scheme
+            and request.url.host == self._gateway_url.host
+            and self._port(request.url) == self._port(self._gateway_url)
+        )
+        if same_origin:
+            for name, value in self._headers.items():
+                if name not in request.headers:
+                    request.headers[name] = value
+        else:
+            for name, value in self._headers.items():
+                if request.headers.get(name) == value:
+                    del request.headers[name]
         return await self._inner.handle_async_request(request)
 
+    async def aclose(self) -> None:
+        await self._inner.aclose()
 
-def _oauth_http_client(headers: dict[str, str], auth: OAuthClientProvider) -> httpx2.AsyncClient:
+
+def _oauth_http_client(
+    headers: dict[str, str], auth: OAuthClientProvider, gateway_url: str = PROXY_BASE_URL
+) -> httpx2.AsyncClient:
     return httpx2.AsyncClient(
-        headers=headers,
         auth=auth,
         timeout=httpx2.Timeout(REQUEST_TIMEOUT),
         follow_redirects=True,
-        transport=_HeaderInjectingTransport(httpx2.AsyncHTTPTransport(), headers),
+        transport=_HeaderInjectingTransport(httpx2.AsyncHTTPTransport(), headers, gateway_url),
     )
 
 
@@ -199,6 +277,43 @@ async def _seed_via_dance(
                 return tuple(sorted(tool.name for tool in listed.tools))
 
 
+@dataclass(frozen=True, slots=True)
+class OauthToolRun:
+    tools: tuple[str, ...]
+    is_error: bool
+    text: str
+
+
+async def _list_and_call(
+    url: str,
+    headers: dict[str, str],
+    storage: InMemoryTokenStorage,
+    storage_state_path: str | None,
+    tool: str,
+    arguments: dict[str, str],
+    gateway_url: str = PROXY_BASE_URL,
+    identity: Identity | None = None,
+    server_alias: str | None = None,
+    allow_upstream_consent: bool = True,
+) -> OauthToolRun:
+    async with _oauth_http_client(
+        headers,
+        _oauth_provider(url, storage, storage_state_path, identity, server_alias, allow_upstream_consent),
+        gateway_url,
+    ) as http_client:
+        async with streamable_http_client(url, http_client=http_client) as (read, write):
+            async with ClientSession(read, write) as session:
+                await session.initialize()
+                listed: Final = await session.list_tools()
+                result: Final = await session.call_tool(tool, arguments)
+                text: Final = "".join(content.text for content in result.content if isinstance(content, TextContent))
+                return OauthToolRun(
+                    tools=tuple(sorted(tool_item.name for tool_item in listed.tools)),
+                    is_error=result.is_error,
+                    text=text,
+                )
+
+
 @dataclass(frozen=True, slots=True)
 class ChatMcpClient:
     proxy: ProxyClient
@@ -252,6 +367,53 @@ class ChatMcpClient:
             f"last error: {last_error!r}"
         )
 
+    def list_and_call(
+        self,
+        alias: str,
+        headers: dict[str, str],
+        storage: InMemoryTokenStorage,
+        storage_state_path: str | None,
+        tool: str,
+        arguments: dict[str, str],
+        base_url: str = PROXY_BASE_URL,
+        identity: Identity | None = None,
+        allow_upstream_consent: bool = True,
+    ) -> OauthToolRun:
+        return asyncio.run(
+            _list_and_call(
+                f"{base_url.rstrip('/')}/mcp" if identity is not None else _mcp_url(alias, base_url),
+                headers,
+                storage,
+                storage_state_path,
+                tool,
+                arguments,
+                base_url,
+                identity,
+                alias,
+                allow_upstream_consent,
+            )
+        )
+
+    def server_user_credentials(self, server_id: str) -> tuple[McpServerUserCredentialRow, ...]:
+        return unwrap(
+            self.proxy.transport.get(
+                f"/v1/mcp/server/{server_id}/user-credentials",
+                headers=self.proxy.transport.master,
+                params=NoBody(),
+                response_type=McpServerUserCredentialListResponse,
+            )
+        ).root
+
+    def revoke_user_token(self, server_id: str, headers: AuthHeaders) -> None:
+        _ = unwrap(
+            self.proxy.transport.delete(
+                f"/v1/mcp/server/{server_id}/oauth-user-credential",
+                headers=headers,
+                json=NoBody(),
+                response_type=McpOauthUserCredentialStatus,
+            )
+        )
+
     def chat_with_mcp(self, headers: AuthHeaders, body: ChatBody) -> ChatResponse:
         """POST /chat/completions carrying the LiteLLM key in `headers` (either
         ingress form) with an MCP server attached in `body.tools`. The gateway

tests/e2e/mcp/oauth_gateway.py

@@ -0,0 +1,198 @@
+"""An owned, source-built OAuth gateway with cold restarts and credential observations.
+
+Only this child process is restarted. Its database and SSO client survive while
+its process-local caches do not; Redis is deliberately absent from its config.
+The optional live edge measures headers without recording credentials or bodies.
+"""
+
+from __future__ import annotations
+
+import os
+import socket
+import subprocess
+import sys
+import threading
+import time
+from collections.abc import Callable, Mapping
+from contextlib import ExitStack
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Final
+
+import psycopg
+from e2e_http import NoBody
+from idp import Keycloak, stop_process_group
+from proxy_client import ProxyClient, build_proxy_client
+from psycopg.rows import class_row
+from pydantic import BaseModel, SecretStr, TypeAdapter, ValidationError
+
+INHERITED_ENV_PREFIXES: Final = ("REDIS_", "MICROSOFT_", "GOOGLE_", "GENERIC_", "PROXY_")
+
+
+class StoredOAuth(BaseModel):
+    type: str
+    access_token: SecretStr
+
+
+@dataclass(frozen=True, slots=True)
+class CredentialRow:
+    credential_b64: str = field(repr=False)
+
+
+def stored_oauth(user_id: str, server_id: str) -> StoredOAuth:
+    """Read the encrypted credential because management APIs omit the plaintext token."""
+    from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
+
+    with psycopg.Connection[CredentialRow].connect(
+        os.environ["DATABASE_URL"], row_factory=class_row(CredentialRow)
+    ) as conn:
+        row: Final = conn.execute(
+            'SELECT credential_b64 FROM "LiteLLM_MCPUserCredentials" WHERE user_id = %s AND server_id = %s',
+            (user_id, server_id),
+        ).fetchone()
+    assert row is not None, "canonical user/server has no persisted credential"
+    plaintext: Final = decrypt_value_helper(
+        row.credential_b64, "e2e_mcp_oauth", exception_type="debug", return_original_value=False
+    )
+    assert plaintext is not None, "persisted credential must decrypt with the gateway salt"
+    assert plaintext != row.credential_b64, "persisted credential must be encrypted"
+    try:
+        credential: Final = StoredOAuth.model_validate_json(plaintext)
+    except ValidationError:
+        raise AssertionError("decrypted credential is not an OAuth payload") from None
+    assert credential.type == "oauth2"
+    assert bool(credential.access_token.get_secret_value()), "stored upstream token is empty"
+    return credential
+
+
+class RpcMethod(BaseModel):
+    method: str = ""
+
+
+@dataclass(slots=True)
+class OAuthObservation:
+    gateway_token: str = field(default="", repr=False)
+    _seen: tuple[tuple[str, str, bool], ...] = field(default=(), init=False, repr=False)
+    _lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False)
+
+    def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None:
+        if body is None or not url.endswith("/mcp"):
+            return
+        try:
+            operation: Final = RpcMethod.model_validate_json(body).method
+        except ValidationError:
+            return
+        if operation not in ("tools/list", "tools/call"):
+            return
+        received: Final = headers.get("authorization", "")
+        gateway_leaked: Final = any(
+            value in (self.gateway_token, f"Bearer {self.gateway_token}") for value in headers.values()
+        )
+        with self._lock:
+            self._seen = (*self._seen, (operation, received, gateway_leaked))
+
+    def assert_forwarded(self, expected: StoredOAuth) -> None:
+        with self._lock:
+            snapshot: Final = self._seen
+            self._seen = ()
+        assert {item[0] for item in snapshot} == {"tools/list", "tools/call"}, "missing upstream observations"
+        expected_header: Final = f"Bearer {expected.access_token.get_secret_value()}"
+        assert all(item[1] == expected_header for item in snapshot), "upstream bearer did not match the stored token"
+        assert all(not item[2] for item in snapshot), "gateway bearer leaked to the upstream"
+
+
+def available_port() -> int:
+    with socket.socket() as listener:
+        listener.bind(("127.0.0.1", 0))
+        return TypeAdapter(tuple[str, int]).validate_python(listener.getsockname())[1]
+
+
+@dataclass(slots=True)
+class OAuthGateway:
+    base_url: str
+    proxy: ProxyClient
+    _environment: Mapping[str, str] = field(repr=False)
+    _command: tuple[str, ...] = field(repr=False)
+    _log_path: Path
+    _child: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False)
+
+    def start(self) -> None:
+        with self._log_path.open("ab") as log:
+            self._child = subprocess.Popen(
+                self._command,
+                env=self._environment,
+                stdout=log,
+                stderr=log,
+                start_new_session=True,
+            )
+        deadline: Final = time.monotonic() + 120
+        while time.monotonic() < deadline:
+            assert self._child.poll() is None, "owned OAuth gateway exited; inspect its private log"
+            result = self.proxy.transport.probe("/health/liveliness", params=NoBody())
+            if result.status_code == 200:
+                return
+            time.sleep(0.5)
+        raise AssertionError("owned OAuth gateway did not become ready")
+
+    def stop(self) -> None:
+        if self._child is not None:
+            stop_process_group(self._child)
+            assert self._child.poll() is not None, "old gateway process is still alive"
+
+    def restart(self) -> None:
+        assert self._child is not None
+        previous: Final = self._child.pid
+        self.stop()
+        self.start()
+        assert self._child.pid != previous, "gateway restart did not create a new process"
+
+
+def owned_gateway(idp: Keycloak, directory: Path, cleanup: ExitStack) -> OAuthGateway:
+    for name in ("DATABASE_URL", "LITELLM_LICENSE", "LITELLM_SALT_KEY", "LITELLM_MASTER_KEY"):
+        assert os.environ.get(name), f"{name} is required for the owned OAuth gateway"
+    port: Final = available_port()
+    base_url: Final = f"http://127.0.0.1:{port}"
+
+    def defer(callback: Callable[[], object]) -> None:
+        cleanup.callback(callback)
+
+    browser: Final = idp.browser_client(callback_url=f"{base_url}/sso/callback", defer=defer)
+    config: Final = directory / "oauth-gateway.yaml"
+    config.write_text(
+        "model_list: []\n"
+        "general_settings:\n"
+        "  master_key: os.environ/LITELLM_MASTER_KEY\n"
+        "  database_url: os.environ/DATABASE_URL\n"
+        "  enable_jwt_auth: true\n"
+        "  litellm_jwtauth:\n"
+        "    user_id_jwt_field: sub\n"
+        "    user_email_jwt_field: email\n"
+        "    team_ids_jwt_field: groups\n"
+        "    user_id_upsert: true\n"
+    )
+    environment: Final = {
+        **{key: value for key, value in os.environ.items() if not key.startswith(INHERITED_ENV_PREFIXES)},
+        **browser.environment(idp.discovery()),
+        "PROXY_BASE_URL": base_url,
+        "JWT_PUBLIC_KEY_URL": idp.jwks_url,
+        "JWT_ISSUER": idp.issuer,
+        "JWT_AUDIENCE": "litellm-e2e",
+        "DISABLE_SCHEMA_UPDATE": "true",
+        "STORE_MODEL_IN_DB": "True",
+        "PYTHONPATH": str(Path(__file__).resolve().parents[3]),
+    }
+    gateway: Final = OAuthGateway(
+        base_url=base_url,
+        proxy=build_proxy_client(
+            base_url=base_url,
+            control_plane_base_url=base_url,
+            replica_urls=(base_url,),
+            master_key=os.environ["LITELLM_MASTER_KEY"],
+        ),
+        _environment=environment,
+        _command=(sys.executable, "-m", "litellm.proxy.proxy_cli", "--config", str(config), "--port", str(port)),
+        _log_path=directory / "oauth-gateway.log",
+    )
+    cleanup.callback(gateway.stop)
+    gateway.start()
+    return gateway

tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py

@@ -0,0 +1,207 @@
+"""Real OAuth consent, immediate MCP operations and cold-restart persistence.
+
+Aggregate SSO uses the SDK's normal authentication. The per-server variant is
+explicitly a configured two-header client, not an Authorization-only OAuth host.
+The observed variants forward to the same real Linear upstream and compare its
+bearer at the forwarding boundary; direct variants retain unmodified discovery.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterator
+from contextlib import ExitStack
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final, Literal
+
+import pytest
+from e2e_config import LINEAR_MCP_URL, LINEAR_READONLY_TOOL, LINEAR_STORAGE_STATE, unique_marker
+from e2e_http import AuthHeaders, NoBody, get_external, unwrap
+from idp import Identity, Keycloak
+from lifecycle import ResourceManager
+from models import (
+    McpOauthCredentials,
+    McpServerCreateBody,
+    ObjectPermission,
+    TeamMemberAddBody,
+    TeamMemberEntry,
+    TeamUpdateBody,
+)
+from oauth_chat_client import ChatMcpClient, InMemoryTokenStorage, OauthToolRun, build_chat_client
+from oauth_gateway import OAuthGateway, OAuthObservation, owned_gateway, stored_oauth
+from provider_edge import LiveEdge, start_provider_edge
+from proxy_client import ProxyClient
+from pydantic import BaseModel, ValidationError
+
+pytestmark = [pytest.mark.e2e, pytest.mark.mcp_oauth_live, pytest.mark.provider_live]
+
+
+class OAuthMetadata(BaseModel):
+    authorization_endpoint: str
+    token_endpoint: str
+    registration_endpoint: str
+
+
+class LinearTeam(BaseModel):
+    id: str
+    name: str
+
+
+class LinearTeams(BaseModel):
+    teams: tuple[LinearTeam, ...]
+
+
+def assert_tool_result(run: OauthToolRun, tool: str) -> None:
+    assert tool in run.tools
+    assert run.is_error is False
+    try:
+        result: Final = LinearTeams.model_validate_json(run.text)
+    except ValidationError:
+        raise AssertionError("list_teams did not return the expected teams payload") from None
+    assert result.teams, "the test workspace must contain at least one team"
+    assert all(team.id and team.name for team in result.teams), "team results must contain identifiers and names"
+
+
+@pytest.fixture(scope="module")
+def oauth_gateway(idp: Keycloak, tmp_path_factory: pytest.TempPathFactory) -> Iterator[OAuthGateway]:
+    assert LINEAR_STORAGE_STATE and Path(LINEAR_STORAGE_STATE).is_file(), (
+        "E2E_LINEAR_STORAGE_STATE must name a captured Linear login; see mcp/linear_session_capture.py"
+    )
+    assert os.environ.get("E2E_FIXTURE_MODE", "live") == "live", "OAuth acceptance cannot use replay"
+    with ExitStack() as cleanup:
+        yield owned_gateway(idp, tmp_path_factory.mktemp("mcp-oauth"), cleanup)
+
+
+@pytest.fixture(scope="module")
+def proxy(oauth_gateway: OAuthGateway) -> ProxyClient:
+    return oauth_gateway.proxy
+
+
+@pytest.fixture(scope="module")
+def client(proxy: ProxyClient) -> ChatMcpClient:
+    return build_chat_client(proxy)
+
+
+class TestMcpOauthHappyPath:
+    @pytest.mark.covers("mcp.list_tools.oauth.succeeds")
+    @pytest.mark.covers("mcp.call_tool.oauth.succeeds")
+    @pytest.mark.covers("mcp.call_tool.oauth.persists_across_processes")
+    @pytest.mark.parametrize("route", ("aggregate_sso", "explicit_header_jwt"))
+    @pytest.mark.parametrize("observed", (False, True), ids=("direct", "observed"))
+    def test_consent_list_call_and_cold_restart(
+        self,
+        client: ChatMcpClient,
+        resources: ResourceManager,
+        jwt_identity: Identity,
+        idp: Keycloak,
+        oauth_gateway: OAuthGateway,
+        route: Literal["aggregate_sso", "explicit_header_jwt"],
+        observed: bool,
+    ) -> None:
+        alias: Final = f"e2elinear{unique_marker()}"
+        tool: Final = f"{alias}-{LINEAR_READONLY_TOOL}"
+        token: Final = idp.access_token(jwt_identity)
+        observation: Final = OAuthObservation(gateway_token=token)
+        edge: Final = (
+            start_provider_edge(
+                LiveEdge(observe_request=observation.observe),
+                mounts=MappingProxyType(
+                    {"linear": "https://mcp.linear.app", ".well-known": "https://mcp.linear.app/.well-known"}
+                ),
+            )
+            if observed
+            else None
+        )
+        if edge is not None:
+            resources.defer(edge.shutdown)
+        metadata: Final = (
+            unwrap(
+                get_external(
+                    "https://mcp.linear.app/.well-known/oauth-authorization-server",
+                    response_type=OAuthMetadata,
+                )
+            )
+            if observed
+            else None
+        )
+        created: Final = client.create_server(
+            McpServerCreateBody(
+                alias=alias,
+                server_name=alias,
+                url=f"{edge.edge.api_base('linear')}/mcp" if edge is not None else LINEAR_MCP_URL,
+                transport="http",
+                allow_all_keys=False,
+                auth_type="oauth2",
+                oauth2_flow="authorization_code",
+                per_server_oauth_discovery=route == "explicit_header_jwt",
+                authorization_url=metadata.authorization_endpoint if metadata else None,
+                token_url=metadata.token_endpoint if metadata else None,
+                registration_url=metadata.registration_endpoint if metadata else None,
+                credentials=McpOauthCredentials(upstream_resource=LINEAR_MCP_URL) if observed else None,
+            )
+        )
+        resources.defer(lambda: client.delete_server(created.server_id))
+        assert client.server_user_credentials(created.server_id) == (), (
+            "scenario must start without upstream credentials"
+        )
+        unwrap(
+            client.proxy.transport.post(
+                "/team/member_add",
+                headers=client.proxy.transport.master,
+                json=TeamMemberAddBody(
+                    team_id=jwt_identity.group, member=TeamMemberEntry(user_id=jwt_identity.user_id, role="user")
+                ),
+                response_type=NoBody,
+            )
+        )
+        client.proxy.update_team(
+            TeamUpdateBody(
+                team_id=jwt_identity.group,
+                object_permission=ObjectPermission(mcp_servers=[created.server_id]),
+            )
+        )
+        headers: Final = {"x-litellm-api-key": f"Bearer {token}"} if route == "explicit_header_jwt" else {}
+        resources.defer(
+            lambda: client.revoke_user_token(
+                created.server_id,
+                AuthHeaders(authorization=f"Bearer {idp.access_token(jwt_identity)}"),
+            )
+        )
+        identity: Final = jwt_identity if route == "aggregate_sso" else None
+        first: Final = client.list_and_call(
+            alias,
+            headers,
+            InMemoryTokenStorage(),
+            LINEAR_STORAGE_STATE,
+            tool,
+            {},
+            base_url=oauth_gateway.base_url,
+            identity=identity,
+        )
+        assert_tool_result(first, tool)
+        credentials: Final = client.server_user_credentials(created.server_id)
+        assert len(credentials) == 1
+        assert credentials[0].user_id == jwt_identity.user_id
+        assert credentials[0].credential_type == "oauth2"
+        first_stored_oauth: Final = stored_oauth(jwt_identity.user_id, created.server_id)
+        if observed:
+            observation.assert_forwarded(first_stored_oauth)
+        oauth_gateway.restart()
+        fresh_token: Final = idp.access_token(jwt_identity)
+        observation.gateway_token = fresh_token
+        second: Final = client.list_and_call(
+            alias,
+            {"Authorization": f"Bearer {fresh_token}"} if identity is None else {},
+            InMemoryTokenStorage(),
+            LINEAR_STORAGE_STATE if identity is not None else None,
+            tool,
+            {},
+            base_url=oauth_gateway.base_url,
+            identity=identity,
+            allow_upstream_consent=False,
+        )
+        assert_tool_result(second, tool)
+        second_stored_oauth: Final = stored_oauth(jwt_identity.user_id, created.server_id)
+        if observed:
+            observation.assert_forwarded(second_stored_oauth)

tests/e2e/migrations/checks.py

@@ -0,0 +1,134 @@
+import hashlib
+from contextlib import ExitStack
+from typing import Final
+from uuid import uuid4
+
+from psycopg import sql
+
+from .containers import Containers, Replica, failed, until
+from .database import GATE_KEY, Database
+from .startup_models import Migration
+
+COMPLETE_SQL: Final = "CREATE TABLE migration_effect (id int PRIMARY KEY); INSERT INTO migration_effect VALUES (1);"
+COMPLETE: Final = Migration("20990101000000_startup_test", COMPLETE_SQL)
+NEXT: Final = Migration(
+    "20990102000000_next_test",
+    "CREATE TABLE migration_next (id int PRIMARY KEY); INSERT INTO migration_next VALUES (2);",
+)
+FATAL: Final = Migration(COMPLETE.name, "DO $$ BEGIN RAISE EXCEPTION 'MIGRATION_TEST_FATAL'; END $$;")
+GATED: Final = Migration(
+    COMPLETE.name, f"SELECT pg_advisory_lock({GATE_KEY}); {COMPLETE.script} SELECT pg_advisory_unlock({GATE_KEY});"
+)
+
+
+def start_replicas(
+    stack: ExitStack, containers: Containers, database: Database, migrations: tuple[Migration, ...] = (), count: int = 3
+) -> tuple[Replica, ...]:
+    return tuple(stack.enter_context(containers.start(database, migrations)) for _ in range(count))
+
+
+def assert_completed(database: Database, migration: Migration = COMPLETE) -> None:
+    assert database.query(
+        'SELECT finished_at IS NOT NULL, rolled_back_at IS NULL, applied_steps_count FROM '
+        '_prisma_migrations WHERE migration_name = %s',
+        (migration.name,),
+    ) == ((True, True, 1),), "Expected exactly one successful SQL execution"
+    assert database.query("SELECT id FROM migration_effect") == ((1,),)
+
+
+def confirmed_history(database: Database) -> str:
+    database.execute(COMPLETE_SQL)
+    row_id: Final = str(uuid4())
+    database.execute(
+        "INSERT INTO _prisma_migrations (id, migration_name, checksum, applied_steps_count) VALUES (%s, %s, %s, 1)",
+        (row_id, COMPLETE.name, hashlib.sha256(COMPLETE.script.encode()).hexdigest()),
+    )
+    return row_id
+
+
+def assert_original_proof(database: Database, row_id: str, finished: bool) -> None:
+    assert database.query(
+        'SELECT id, applied_steps_count, finished_at IS NOT NULL, rolled_back_at IS NULL FROM '
+        '_prisma_migrations WHERE migration_name = %s',
+        (COMPLETE.name,),
+    ) == ((row_id, 1, finished, True),), "Recovery lost or replaced the original durable SQL proof"
+    assert database.query("SELECT id FROM migration_effect") == ((1,),)
+
+
+def pause_completion(database: Database) -> None:
+    database.execute(
+        sql.SQL(
+            "CREATE FUNCTION migration_pause() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN "
+            "IF NEW.migration_name = {name} AND NEW.finished_at IS NOT NULL THEN "
+            "PERFORM pg_advisory_lock({gate}); PERFORM pg_advisory_unlock({gate}); END IF; RETURN NEW; END $$; "
+            'CREATE TRIGGER migration_pause BEFORE UPDATE ON _prisma_migrations FOR EACH ROW '
+            'EXECUTE FUNCTION migration_pause()'
+        ).format(name=sql.Literal(COMPLETE.name), gate=sql.Literal(GATE_KEY))
+    )
+
+
+def interrupt_owner(
+    containers: Containers, database: Database, after_commit: bool, *, stop_database_session: bool = True
+) -> None:
+    if after_commit:
+        pause_completion(database)
+    with database.lock():
+        with containers.start(database, (COMPLETE if after_commit else GATED,)) as owner:
+            until("migration at the intended crash boundary", lambda: bool(database.blocked()))
+            assert database.exists("migration_effect") == after_commit
+            assert database.query(
+                "SELECT finished_at IS NULL FROM _prisma_migrations WHERE migration_name = %s", (COMPLETE.name,)
+            ) == ((True,),)
+            blocked: Final = database.blocked()
+            assert len(blocked) == 1
+            backend: Final = blocked[0][0]
+            assert owner.state().Running
+            owner.kill()
+            assert owner.state().ExitCode == 137
+            if stop_database_session:
+                database.query("SELECT pg_terminate_backend(%s)", (backend,))
+                until(
+                    "terminated migration backend released",
+                    lambda: not database.query("SELECT pid FROM pg_stat_activity WHERE pid = %s", (backend,)),
+                )
+                assert database.query(
+                    "SELECT finished_at IS NULL, applied_steps_count FROM _prisma_migrations WHERE migration_name = %s",
+                    (COMPLETE.name,),
+                ) == ((True, int(after_commit)),)
+                assert database.exists("migration_effect") == after_commit
+    if not stop_database_session:
+        until(
+            "database backend noticed container death",
+            lambda: not database.query("SELECT pid FROM pg_stat_activity WHERE pid = %s", (backend,)),
+            60,
+        )
+
+
+def unconfirmed(replicas: tuple[Replica, ...], database: Database) -> None:
+    failed(replicas, "Migration completion could not be verified")
+    started: Final = str(
+        database.query(
+            "SELECT to_char(started_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS') FROM "
+            '_prisma_migrations WHERE migration_name = %s',
+            (COMPLETE.name,),
+        )[0][0]
+    )
+    for replica in replicas:
+        assert_guidance(replica.logs(), started)
+
+
+def assert_guidance(log: str, started: str) -> None:
+    for detail in (
+        COMPLETE.name,
+        started,
+        "cannot determine whether its SQL committed",
+        "_prisma_migrations",
+        "migration.sql",
+        "Only after verifying every migration change is present",
+        "prisma migrate resolve --applied <migration_name>",
+        "Only after verifying no migration changes remain",
+        "prisma migrate resolve --rolled-back <migration_name>",
+        "leave migration history unchanged",
+        "Repeated restarts alone",
+    ):
+        assert detail in log, f"Missing recovery guidance: {detail}"

tests/e2e/migrations/conftest.py

@@ -0,0 +1,62 @@
+import json
+import os
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlsplit
+
+import pytest
+from _pytest.fixtures import SubRequest
+
+from .containers import Containers, docker, ready
+from .database import Database, Databases
+
+
+@pytest.fixture(scope="session")
+def migration_image(tmp_path_factory: pytest.TempPathFactory) -> str:
+    configured: Final = os.environ.get("LITELLM_MIGRATION_TEST_IMAGE")
+    assert configured, "LITELLM_MIGRATION_TEST_IMAGE must name the built candidate image"
+    image: Final = docker("image", "inspect", configured, "--format", "{{.Id}}")
+    assert image.startswith("sha256:"), "Unable to identify the candidate image"
+    output: Final = Path(os.environ.get("MIGRATION_TEST_OUTPUT", str(tmp_path_factory.getbasetemp())))
+    output.mkdir(parents=True, exist_ok=True)
+    (output / "image.json").write_text(json.dumps({"requested": configured, "image_id": image}))
+    return image
+
+
+@pytest.fixture(scope="session")
+def databases() -> Databases:
+    admin: Final = os.environ.get("MIGRATION_TEST_ADMIN_URL", "")
+    parsed: Final = urlsplit(admin)
+    assert parsed.hostname in ("127.0.0.1", "localhost"), "Use an isolated loopback PostgreSQL test cluster"
+    assert parsed.port and parsed.path and not parsed.query, "Supply the test cluster port and admin database"
+    container_admin: Final = os.environ.get(
+        "MIGRATION_TEST_CONTAINER_ADMIN_URL",
+        admin.replace("127.0.0.1", "host.docker.internal").replace("localhost", "host.docker.internal"),
+    )
+    return Databases(admin, container_admin)
+
+
+@pytest.fixture(scope="session")
+def migrated_template(
+    databases: Databases, migration_image: str, tmp_path_factory: pytest.TempPathFactory
+) -> Iterator[Database]:
+    output: Final = Path(os.environ.get("MIGRATION_TEST_OUTPUT", str(tmp_path_factory.getbasetemp()))) / "seed"
+    with databases.create() as database:
+        with Containers(migration_image, output).start(database) as replica:
+            ready((replica,), database)
+        yield database
+
+
+@pytest.fixture
+def database(databases: Databases, migrated_template: Database) -> Iterator[Database]:
+    with databases.create(migrated_template) as database:
+        yield database
+
+
+@pytest.fixture
+def containers(migration_image: str, tmp_path: Path, request: SubRequest) -> Containers:
+    configured: Final = os.environ.get("MIGRATION_TEST_OUTPUT")
+    output: Final = Path(configured) / request.node.name if configured else tmp_path
+    output.mkdir(parents=True, exist_ok=True)
+    return Containers(migration_image, output)

tests/e2e/migrations/containers.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+import hashlib
+import subprocess
+import time
+from collections.abc import Callable, Generator, Mapping
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+from uuid import uuid4
+
+from e2e_http import NoBody, Success, unwrap
+from models import KeyGenerateBody, KeyGenerateResponse, KeyInfoParams, KeyInfoResponse
+from transport import HttpTransport
+
+from .database import Database, prisma_url
+from .startup_models import ContainerState, Migration, Observation, Readiness
+
+MASTER_KEY: Final = "sk-migration-ci-fixture"
+
+
+def docker(*args: str) -> str:
+    result: Final = subprocess.run(("docker", *args), capture_output=True, text=True, timeout=90)
+    assert result.returncode == 0, f"Docker operation failed: {result.stderr}"
+    return result.stdout.strip()
+
+
+def until(description: str, condition: Callable[[], bool], seconds: float = 150) -> None:
+    deadline: Final = time.monotonic() + seconds
+    while time.monotonic() < deadline:
+        if condition():
+            return
+        time.sleep(0.25)
+    raise AssertionError(f"Timed out waiting for {description}")
+
+
+@dataclass(frozen=True, slots=True)
+class Replica:
+    name: str
+    transport: HttpTransport
+    output: Path
+
+    def state(self) -> ContainerState:
+        return ContainerState.model_validate_json(docker("inspect", "--format", "{{json .State}}", self.name))
+
+    def observe(self) -> Observation:
+        state: Final = self.state()
+        result: Final = self.transport.get(
+            "/health/readiness", headers=self.transport.master, params=NoBody(), response_type=Readiness, timeout=1
+        )
+        ready: Final = isinstance(result, Success) and result.data.status == "healthy" and result.data.db == "connected"
+        return Observation(None if state.Running else state.ExitCode, ready)
+
+    def logs(self) -> str:
+        result: Final = subprocess.run(("docker", "logs", self.name), capture_output=True, text=True, timeout=30)
+        assert result.returncode == 0, result.stderr
+        return result.stdout + result.stderr
+
+    def kill(self) -> None:
+        if self.state().Running:
+            docker("kill", self.name)
+
+    def usable(self, database: Database) -> None:
+        alias: Final = f"migration-{uuid4().hex}"
+        key: Final = unwrap(
+            self.transport.post(
+                "/key/generate",
+                headers=self.transport.master,
+                json=KeyGenerateBody(key_alias=alias),
+                response_type=KeyGenerateResponse,
+            )
+        ).key
+        info: Final = unwrap(
+            self.transport.get(
+                "/key/info",
+                headers=self.transport.master,
+                params=KeyInfoParams(key=key),
+                response_type=KeyInfoResponse,
+            )
+        )
+        assert info.info.key_alias == alias
+        assert database.query(
+            'SELECT key_alias FROM "LiteLLM_VerificationToken" WHERE token = %s',
+            (hashlib.sha256(key.encode()).hexdigest(),),
+        ) == ((alias,),)
+
+
+def ready(replicas: tuple[Replica, ...], database: Database) -> None:
+    def all_ready() -> bool:
+        observations: Final = tuple(replica.observe() for replica in replicas)
+        assert all(item.exit_code is None for item in observations), "Replica exited before readiness"
+        return all(item.ready for item in observations)
+
+    until("every replica ready", all_ready)
+    for replica in replicas:
+        replica.usable(database)
+
+
+def failed(replicas: tuple[Replica, ...], marker: str) -> None:
+    def all_stopped() -> bool:
+        observations: Final = tuple(replica.observe() for replica in replicas)
+        assert not any(item.ready for item in observations), "Failed migration exposed a ready proxy"
+        return all(item.exit_code is not None for item in observations)
+
+    until("every replica to reject startup", all_stopped)
+    for replica in replicas:
+        assert replica.state().ExitCode != 0, "Failed startup returned success"
+        assert marker in replica.logs(), f"Startup failed outside the expected migration: {marker}"
+
+
+def waiting(replicas: tuple[Replica, ...], seconds: float) -> None:
+    deadline: Final = time.monotonic() + seconds
+    while time.monotonic() < deadline:
+        assert all(item.exit_code is None and not item.ready for item in (replica.observe() for replica in replicas)), (
+            "Contending replica exited or served early"
+        )
+        time.sleep(0.25)
+
+
+@dataclass(frozen=True, slots=True)
+class Containers:
+    image: str
+    output: Path
+
+    @contextmanager
+    def start(
+        self,
+        database: Database,
+        migrations: tuple[Migration, ...] = (),
+        *,
+        v2: bool = True,
+        disabled: bool = False,
+        environment: Mapping[str, str] | None = None,
+    ) -> Generator[Replica]:
+        name: Final = f"litellm-migration-{uuid4().hex[:16]}"
+        directory: Final = self.output / name
+        directory.mkdir(parents=True)
+        for migration in migrations:
+            write_migration(directory, migration)
+        (directory / "config.yaml").write_text(
+            "model_list: []\ngeneral_settings:\n  master_key: os.environ/LITELLM_MASTER_KEY\n"
+        )
+        env: Final = {
+            "DATABASE_URL": prisma_url(database.container_url, database.schema),
+            "LITELLM_MASTER_KEY": MASTER_KEY,
+            "LITELLM_SALT_KEY": MASTER_KEY,
+            "LITELLM_LOCAL_MODEL_COST_MAP": "True",
+            "LITELLM_TELEMETRY": "False",
+            "LITELLM_LOG": "INFO",
+            "DATABASE_CONNECTION_POOL_LIMIT": "2",
+            "DEFAULT_NUM_WORKERS_LITELLM_PROXY": "1",
+            "USE_V2_MIGRATION_RESOLVER": str(v2).lower(),
+            "DISABLE_SCHEMA_UPDATE": str(disabled).lower(),
+            "LITELLM_MIGRATION_DIR": "/migration-test/prisma",
+            "LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT": "180",
+            **(environment or {}),
+        }
+        try:
+            docker(
+                "run",
+                "-d",
+                "--name",
+                name,
+                "--label",
+                "litellm-migration-test=true",
+                "--add-host",
+                "host.docker.internal:host-gateway",
+                "-p",
+                "127.0.0.1::4000",
+                "-v",
+                f"{directory}:/migration-test",
+                *(arg for key, value in env.items() for arg in ("-e", f"{key}={value}")),
+                self.image,
+                "--config",
+                "/migration-test/config.yaml",
+                "--host",
+                "0.0.0.0",
+                "--port",
+                "4000",
+            )
+            port: Final = int(docker("port", name, "4000/tcp").rsplit(":", 1)[1])
+            replica: Final = Replica(name, HttpTransport(f"http://127.0.0.1:{port}", MASTER_KEY, 15), directory)
+            yield replica
+        finally:
+            try:
+                state: Final = subprocess.run(
+                    ("docker", "inspect", "--format", "{{json .State}}", name),
+                    capture_output=True,
+                    text=True,
+                    timeout=30,
+                )
+                (directory / "state.json").write_text(state.stdout or state.stderr)
+                logs: Final = subprocess.run(("docker", "logs", name), capture_output=True, text=True, timeout=30)
+                (directory / "proxy.log").write_text(logs.stdout + logs.stderr)
+            finally:
+                subprocess.run(("docker", "rm", "-f", name), capture_output=True, text=True, timeout=30, check=True)
+
+
+def write_migration(directory: Path, migration: Migration) -> None:
+    path: Final = directory / "prisma" / "migrations" / migration.name
+    path.mkdir(parents=True)
+    (path / "migration.sql").write_text(migration.script)

tests/e2e/migrations/database.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+from collections.abc import Generator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Final, LiteralString
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+from uuid import uuid4
+
+import psycopg
+from psycopg import sql
+from pydantic import TypeAdapter
+
+Scalar = str | int | bool | None
+ROWS: Final = TypeAdapter(tuple[tuple[Scalar, ...], ...])
+GATE_KEY: Final = 39178002
+PRISMA_LOCK: Final = 72707369
+COORDINATOR_LOCK: Final = int.from_bytes(b"llm_mig2", "big")
+
+
+def connect_url(url: str, name: str) -> str:
+    return urlunsplit(urlsplit(url)._replace(path=f"/{name}", query=""))
+
+
+def prisma_url(url: str, schema: str) -> str:
+    parsed: Final = urlsplit(url)
+    query: Final = tuple((key, value) for key, value in parse_qsl(parsed.query) if key != "schema")
+    return urlunsplit(parsed._replace(query=urlencode((*query, ("schema", schema)))))
+
+
+@dataclass(frozen=True, slots=True)
+class Database:
+    name: str
+    url: str
+    container_url: str
+    schema: str = "public"
+
+    @contextmanager
+    def connection(self) -> Generator[psycopg.Connection[tuple[object, ...]]]:
+        with psycopg.connect(self.url, autocommit=True, connect_timeout=5) as connection:
+            connection.execute(sql.SQL("SET search_path TO {}").format(sql.Identifier(self.schema)))
+            connection.execute("SET statement_timeout = '15s'")
+            yield connection
+
+    def execute(self, statement: LiteralString | sql.Composed, params: tuple[Scalar, ...] = ()) -> None:
+        with self.connection() as connection:
+            connection.execute(statement, params or None)
+
+    def query(
+        self, statement: LiteralString | sql.Composed, params: tuple[Scalar, ...] = ()
+    ) -> tuple[tuple[Scalar, ...], ...]:
+        with self.connection() as connection:
+            return ROWS.validate_python(connection.execute(statement, params or None).fetchall())
+
+    def exists(self, name: str) -> bool:
+        return self.query("SELECT to_regclass(%s) IS NOT NULL", (name,)) == ((True,),)
+
+    def history(self) -> tuple[tuple[Scalar, ...], ...]:
+        if not self.exists("_prisma_migrations"):
+            return ()
+        return self.query(
+            "SELECT id, migration_name, checksum, started_at::text, finished_at::text, rolled_back_at::text, "
+            "applied_steps_count, logs FROM _prisma_migrations ORDER BY id"
+        )
+
+    def blocked(self, key: int = GATE_KEY) -> tuple[tuple[Scalar, ...], ...]:
+        return self.query(
+            "SELECT pid FROM pg_locks WHERE locktype = 'advisory' AND NOT granted "
+            "AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) "
+            "AND classid = %s AND objid = %s ORDER BY pid",
+            (key >> 32, key & 0xFFFFFFFF),
+        )
+
+    @contextmanager
+    def lock(self, key: int = GATE_KEY) -> Generator[None]:
+        with self.connection() as connection:
+            connection.execute("SELECT pg_advisory_lock(%s)", (key,))
+            try:
+                yield
+            finally:
+                connection.execute("SELECT pg_advisory_unlock(%s)", (key,))
+
+
+@dataclass(frozen=True, slots=True)
+class Databases:
+    admin_url: str
+    container_admin_url: str
+
+    @contextmanager
+    def create(self, template: Database | None = None, schema: str = "public") -> Generator[Database]:
+        name: Final = f"litellm_migration_test_{uuid4().hex[:20]}"
+        database: Final = Database(
+            name, connect_url(self.admin_url, name), connect_url(self.container_admin_url, name), schema
+        )
+        with psycopg.connect(self.admin_url, autocommit=True, connect_timeout=5) as connection:
+            connection.execute(
+                sql.SQL("CREATE DATABASE {} TEMPLATE {}").format(
+                    sql.Identifier(name), sql.Identifier(template.name if template else "template0")
+                )
+            )
+        try:
+            yield database
+        finally:
+            with psycopg.connect(self.admin_url, autocommit=True, connect_timeout=5) as connection:
+                connection.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(name)))
+
+
+@contextmanager
+def restricted_user(database: Database) -> Generator[Database]:
+    role: Final = f"migration_reader_{uuid4().hex[:16]}"
+    password: Final = "migration-test-password"
+    with database.connection() as connection:
+        connection.execute(
+            sql.SQL("CREATE ROLE {} LOGIN PASSWORD {}").format(sql.Identifier(role), sql.Literal(password))
+        )
+    try:
+        database.execute(
+            sql.SQL("GRANT USAGE ON SCHEMA {} TO {}").format(sql.Identifier(database.schema), sql.Identifier(role))
+        )
+        database.execute(
+            sql.SQL("GRANT SELECT ON ALL TABLES IN SCHEMA {} TO {}").format(
+                sql.Identifier(database.schema), sql.Identifier(role)
+            )
+        )
+        local: Final = urlsplit(database.url)
+        remote: Final = urlsplit(database.container_url)
+        yield Database(
+            database.name,
+            urlunsplit(local._replace(netloc=f"{role}:{password}@{local.hostname}:{local.port}")),
+            urlunsplit(remote._replace(netloc=f"{role}:{password}@{remote.hostname}:{remote.port}")),
+            database.schema,
+        )
+    finally:
+        database.execute(sql.SQL("DROP OWNED BY {}").format(sql.Identifier(role)))
+        database.execute(sql.SQL("DROP ROLE {}").format(sql.Identifier(role)))

tests/e2e/migrations/startup_models.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+
+from pydantic import BaseModel
+
+
+class Readiness(BaseModel):
+    status: str = ""
+    db: str = ""
+
+
+class ContainerState(BaseModel):
+    Running: bool
+    ExitCode: int
+
+
+@dataclass(frozen=True, slots=True)
+class Observation:
+    exit_code: int | None
+    ready: bool
+
+
+@dataclass(frozen=True, slots=True)
+class Migration:
+    name: str
+    script: str

tests/e2e/migrations/test_legacy.py

@@ -0,0 +1,87 @@
+from contextlib import ExitStack
+from dataclasses import replace
+from typing import Final, Literal
+
+import pytest
+
+from .checks import COMPLETE, assert_completed, confirmed_history, assert_original_proof, start_replicas
+from .containers import Containers, failed, ready
+from .database import Database, Databases
+
+pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup]
+
+
+def adopt_legacy(containers: Containers, database: Database) -> None:
+    count: Final = database.query("SELECT count(*) FROM _prisma_migrations")[0][0]
+    existing_keys: Final = database.query('SELECT token FROM "LiteLLM_VerificationToken" ORDER BY token')
+    database.execute(
+        'INSERT INTO "LiteLLM_ShadowEvalJob" (id, group_id, target_id, router_name, judge_model, '
+        "shadow_percentage, max_turns, ends_at, stopped_at) VALUES ('migration-legacy', "
+        "'migration-legacy', 'target', 'router', 'judge', 1, 1, now(), now())"
+    )
+    database.execute("DROP TABLE _prisma_migrations")
+    with ExitStack() as stack:
+        replicas: Final = start_replicas(stack, containers, database)
+        ready(replicas, database)
+        logs: Final = "\n".join(replica.logs() for replica in replicas)
+        for detail in (
+            "Legacy migration history was missing",
+            "historical data backfills were not replayed or verified",
+            "Continuing startup",
+        ):
+            assert detail in logs
+    assert database.query("SELECT count(*) FROM _prisma_migrations") == ((count,),)
+    assert database.query(
+        'SELECT count(*) FROM _prisma_migrations WHERE finished_at IS NULL OR rolled_back_at IS '
+        'NOT NULL OR applied_steps_count <> 0'
+    ) == ((0,),)
+    assert set(existing_keys).issubset(database.query('SELECT token FROM "LiteLLM_VerificationToken" ORDER BY token'))
+    assert database.query("SELECT stopped_by FROM \"LiteLLM_ShadowEvalJob\" WHERE id = 'migration-legacy'") == (
+        (None,),
+    )
+
+
+class TestLegacyMigrations:
+    def test_matching_schema_warns_and_starts(self, containers: Containers, database: Database) -> None:
+        adopt_legacy(containers, database)
+
+    @pytest.mark.parametrize("fault", ("schema_drift", "custom_migrations", "empty_ledger"))
+    def test_unrecognized_legacy_state_is_not_baselined(
+        self, containers: Containers, database: Database, fault: str
+    ) -> None:
+        if fault == "empty_ledger":
+            database.execute("TRUNCATE _prisma_migrations")
+        else:
+            database.execute("DROP TABLE _prisma_migrations")
+        if fault == "schema_drift":
+            database.execute('ALTER TABLE "LiteLLM_VerificationToken" DROP COLUMN key_alias CASCADE')
+        with containers.start(database, (COMPLETE,) if fault == "custom_migrations" else ()) as replica:
+            failed((replica,), "Cannot automatically baseline" if fault != "empty_ledger" else "migration")
+        assert not database.exists("migration_effect")
+        if database.exists("_prisma_migrations"):
+            assert database.query(
+                "SELECT count(*) FROM _prisma_migrations WHERE finished_at IS NOT NULL AND applied_steps_count <> 1"
+            ) == ((0,),)
+
+    @pytest.mark.parametrize("scenario", ("upgrade", "recovery", "legacy"))
+    def test_non_default_schema(
+        self, containers: Containers, databases: Databases, scenario: Literal["upgrade", "recovery", "legacy"]
+    ) -> None:
+        with databases.create(schema="migration tenant") as database:
+            with containers.start(database) as seed:
+                ready((seed,), database)
+            match scenario:
+                case "upgrade":
+                    with ExitStack() as stack:
+                        ready(start_replicas(stack, containers, database, (COMPLETE,)), database)
+                    assert_completed(database)
+                case "recovery":
+                    original: Final = confirmed_history(database)
+                    with ExitStack() as stack:
+                        ready(start_replicas(stack, containers, database, (COMPLETE,)), database)
+                    assert_original_proof(database, original, True)
+                case "legacy":
+                    adopt_legacy(containers, database)
+            public: Final = replace(database, schema="public")
+            assert not public.exists("_prisma_migrations")
+            assert not public.exists('"LiteLLM_VerificationToken"')

tests/e2e/migrations/test_pooling.py

@@ -0,0 +1,136 @@
+import subprocess
+from collections.abc import Generator
+from contextlib import ExitStack, contextmanager
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlsplit, urlunsplit
+from uuid import uuid4
+
+import psycopg
+import pytest
+from psycopg import sql
+
+from .checks import COMPLETE, assert_completed
+from .containers import Containers, docker, ready, until
+from .database import Database, Databases, prisma_url, restricted_user
+
+POOL_IMAGE: Final = (
+    "ghcr.io/cloudnative-pg/pgbouncer@sha256:e6ddfe22d845e603825e235dd8334b21ecd125abea2a2172478f556b8dee2bb8"
+)
+pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup]
+
+
+@contextmanager
+def application_user(database: Database) -> Generator[Database]:
+    with restricted_user(database) as application:
+        role: Final = sql.Identifier(str(urlsplit(application.url).username))
+        schema: Final = sql.Identifier(database.schema)
+        database.execute(sql.SQL("REVOKE CREATE ON SCHEMA {} FROM PUBLIC").format(schema))
+        for statement in (
+            "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA {} TO {}",
+            "GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA {} TO {}",
+            "ALTER DEFAULT PRIVILEGES IN SCHEMA {} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO {}",
+            "ALTER DEFAULT PRIVILEGES IN SCHEMA {} GRANT USAGE, SELECT ON SEQUENCES TO {}",
+        ):
+            database.execute(sql.SQL(statement).format(schema, role))
+        assert application.query("SELECT has_schema_privilege(current_user, %s, 'CREATE')", (database.schema,)) == (
+            (False,),
+        )
+        yield application
+
+
+@contextmanager
+def pool(database: Database, output: Path) -> Generator[str]:
+    name: Final = f"litellm-migration-pool-{uuid4().hex[:12]}"
+    url: Final = urlsplit(database.container_url)
+    directory: Final = output / name
+    directory.mkdir(parents=True)
+    (directory / "users.txt").write_text(f'"{url.username}" "{url.password}"\n')
+    (directory / "pgbouncer.ini").write_text(
+        f"[databases]\n* = host={url.hostname} port={url.port} user={url.username} password={url.password}\n"
+        "[pgbouncer]\nlisten_addr = 0.0.0.0\nlisten_port = 6432\nauth_type = trust\nauth_file = /pool/users.txt\n"
+        "pool_mode = transaction\ndefault_pool_size = 1\nreserve_pool_size = 0\nmax_client_conn = 100\n"
+        'max_prepared_statements = 100\nquery_wait_timeout = 8\nignore_startup_parameters = '
+        'extra_float_digits,options\n'
+    )
+    try:
+        docker(
+            "run",
+            "-d",
+            "--name",
+            name,
+            "--label",
+            "litellm-migration-test=true",
+            "--add-host",
+            "host.docker.internal:host-gateway",
+            "-p",
+            "0.0.0.0::6432",
+            "-v",
+            f"{directory}:/pool:ro",
+            "--entrypoint",
+            "/usr/bin/pgbouncer",
+            POOL_IMAGE,
+            "/pool/pgbouncer.ini",
+        )
+        port: Final = int(docker("port", name, "6432/tcp").splitlines()[0].rsplit(":", 1)[1])
+        local_url: Final = urlunsplit(url._replace(netloc=f"{url.username}:{url.password}@127.0.0.1:{port}"))
+
+        def connected() -> bool:
+            try:
+                with psycopg.connect(local_url, autocommit=True, connect_timeout=2) as connection:
+                    return connection.execute("SELECT 1").fetchone() == (1,)
+            except psycopg.Error:
+                return False
+
+        until("PgBouncer ready", connected, 30)
+        yield local_url.replace("127.0.0.1", "host.docker.internal") + "?pgbouncer=true"
+    finally:
+        try:
+            logs: Final = subprocess.run(("docker", "logs", name), text=True, capture_output=True, timeout=30)
+            (directory / "pool.log").write_text(logs.stdout + logs.stderr)
+        finally:
+            subprocess.run(("docker", "rm", "-f", name), capture_output=True, text=True, timeout=30, check=True)
+
+
+class TestMigrationPooling:
+    @pytest.mark.parametrize("scenario,replica_count", (("fresh", 3), ("upgrade", 3), ("legacy", 3), ("upgrade", 6)))
+    def test_direct_migrations_with_one_application_backend(
+        self,
+        containers: Containers,
+        databases: Databases,
+        migrated_template: Database,
+        scenario: str,
+        replica_count: int,
+    ) -> None:
+        with databases.create(None if scenario == "fresh" else migrated_template) as database:
+            if scenario == "legacy":
+                database.execute("DROP TABLE _prisma_migrations")
+            with (
+                application_user(database) as application,
+                pool(application, containers.output) as pooled_url,
+                ExitStack() as stack,
+            ):
+                replicas: Final = tuple(
+                    stack.enter_context(
+                        containers.start(
+                            database,
+                            (COMPLETE,) if scenario == "upgrade" else (),
+                            environment={
+                                "DATABASE_URL": prisma_url(pooled_url, database.schema),
+                                "DIRECT_URL": database.container_url,
+                            },
+                        )
+                    )
+                    for _ in range(replica_count)
+                )
+                ready(replicas, database)
+                if scenario == "upgrade":
+                    assert_completed(database)
+                if scenario == "legacy":
+                    assert any(
+                        "historical data backfills were not replayed or verified" in replica.logs()
+                        for replica in replicas
+                    )
+                    assert database.query("SELECT count(*) FROM _prisma_migrations WHERE applied_steps_count <> 0") == (
+                        (0,),
+                    )

tests/e2e/migrations/test_recovery.py

@@ -0,0 +1,185 @@
+from contextlib import ExitStack
+from typing import Final, Literal
+from uuid import uuid4
+
+import pytest
+
+from .checks import (
+    COMPLETE,
+    FATAL,
+    GATED,
+    NEXT,
+    assert_completed,
+    confirmed_history,
+    interrupt_owner,
+    assert_original_proof,
+    pause_completion,
+    start_replicas,
+    unconfirmed,
+)
+from .containers import Containers, failed, ready, until, waiting
+from .database import COORDINATOR_LOCK, GATE_KEY, Database
+from .startup_models import Migration
+
+pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup]
+
+
+class TestMigrationRecovery:
+    @pytest.mark.parametrize("after_commit", (False, True))
+    def test_container_owner_crash(self, containers: Containers, database: Database, after_commit: bool) -> None:
+        interrupt_owner(containers, database, after_commit, stop_database_session=False)
+        history: Final = database.history()
+        assert database.query(
+            "SELECT applied_steps_count FROM _prisma_migrations WHERE migration_name = %s", (COMPLETE.name,)
+        ) == ((int(after_commit),),)
+        with ExitStack() as stack:
+            successors: Final = start_replicas(stack, containers, database, (COMPLETE if after_commit else GATED,))
+            if after_commit:
+                ready(successors, database)
+                assert_completed(database)
+            else:
+                unconfirmed(successors, database)
+                assert database.history() == history
+
+    @pytest.mark.parametrize("after_commit", (False, True))
+    def test_owner_and_database_session_crash(
+        self, containers: Containers, database: Database, after_commit: bool
+    ) -> None:
+        interrupt_owner(containers, database, after_commit)
+        history: Final = database.history()
+        with ExitStack() as stack:
+            successors: Final = start_replicas(stack, containers, database, (COMPLETE,))
+            if after_commit:
+                ready(successors, database)
+                assert_completed(database)
+                return
+            unconfirmed(successors, database)
+            assert database.history() == history
+        with containers.start(database, (COMPLETE,)) as restarted:
+            unconfirmed((restarted,), database)
+            assert database.history() == history
+
+    @pytest.mark.parametrize("later_failure", (False, True))
+    def test_remaining_migrations_after_recovery(
+        self, containers: Containers, database: Database, later_failure: bool
+    ) -> None:
+        original: Final = confirmed_history(database)
+        next_migration: Final = Migration(
+            NEXT.name,
+            f"SELECT pg_advisory_lock({GATE_KEY}); "
+            + (FATAL.script if later_failure else NEXT.script)
+            + f" SELECT pg_advisory_unlock({GATE_KEY});",
+        )
+        with ExitStack() as stack:
+            with database.lock():
+                owner: Final = stack.enter_context(containers.start(database, (COMPLETE, next_migration)))
+
+                def pending() -> bool:
+                    observation: Final = owner.observe()
+                    assert observation.exit_code is None and not observation.ready, (
+                        "Recovered owner served before pending SQL completed"
+                    )
+                    return bool(database.blocked())
+
+                until("recovering owner reached the next migration", pending)
+                assert_original_proof(database, original, True)
+                assert not database.exists("migration_next")
+                followers: Final = start_replicas(stack, containers, database, (COMPLETE, next_migration), count=2)
+                replicas: Final = (owner, *followers)
+                waiting(replicas, 1)
+            if later_failure:
+                failed(replicas, NEXT.name)
+                assert database.query(
+                    "SELECT finished_at IS NULL, logs LIKE %s FROM _prisma_migrations WHERE migration_name = %s",
+                    ("%MIGRATION_TEST_FATAL%", NEXT.name),
+                ) == ((True, True),)
+            else:
+                ready(replicas, database)
+                assert database.query("SELECT id FROM migration_next") == ((2,),)
+            assert_original_proof(database, original, True)
+
+    def test_second_crash_during_recovery_is_atomic(self, containers: Containers, database: Database) -> None:
+        original: Final = confirmed_history(database)
+        pause_completion(database)
+        with database.lock():
+            with containers.start(database, (COMPLETE,)) as recovering:
+                until("history update blocked before commit", lambda: bool(database.blocked()))
+                assert_original_proof(database, original, False)
+                blocked: Final = database.blocked()
+                assert len(blocked) == 1
+                assert database.query("SELECT pg_terminate_backend(%s)", (blocked[0][0],)) == ((True,),)
+                failed((recovering,), "Lost or could not establish v2 migration coordination")
+                assert_original_proof(database, original, False)
+        with ExitStack() as stack:
+            ready(start_replicas(stack, containers, database, (COMPLETE,)), database)
+        assert_original_proof(database, original, True)
+
+    def test_competing_recovery_rechecks_stale_failures(self, containers: Containers, database: Database) -> None:
+        original: Final = confirmed_history(database)
+        with ExitStack() as stack:
+            with database.lock(COORDINATOR_LOCK):
+                replicas: Final = start_replicas(stack, containers, database, (COMPLETE,))
+                until(
+                    "all replicas observed the unfinished migration",
+                    lambda: all(
+                        "Waiting for the v2 migration coordinator lock" in replica.logs() for replica in replicas
+                    ),
+                )
+                assert_original_proof(database, original, False)
+            ready(replicas, database)
+        assert_original_proof(database, original, True)
+
+    @pytest.mark.parametrize(
+        "fault", ("no_steps", "extra_steps", "failure_logs", "checksum", "duplicate_history", "missing_script")
+    )
+    def test_unproven_history_is_never_repaired(
+        self,
+        containers: Containers,
+        database: Database,
+        fault: Literal["no_steps", "extra_steps", "failure_logs", "checksum", "duplicate_history", "missing_script"],
+    ) -> None:
+        confirmed_history(database)
+        match fault:
+            case "no_steps":
+                database.execute(
+                    "UPDATE _prisma_migrations SET applied_steps_count = 0 WHERE migration_name = %s", (COMPLETE.name,)
+                )
+            case "extra_steps":
+                database.execute(
+                    "UPDATE _prisma_migrations SET applied_steps_count = 2 WHERE migration_name = %s", (COMPLETE.name,)
+                )
+            case "failure_logs":
+                database.execute(
+                    "UPDATE _prisma_migrations SET logs = 'permission denied' WHERE migration_name = %s",
+                    (COMPLETE.name,),
+                )
+            case "checksum":
+                database.execute(
+                    "UPDATE _prisma_migrations SET checksum = %s WHERE migration_name = %s", ("0" * 64, COMPLETE.name)
+                )
+            case "duplicate_history":
+                database.execute(
+                    'INSERT INTO _prisma_migrations (id, migration_name, checksum, '
+                    'applied_steps_count) SELECT %s, migration_name, checksum, '
+                    'applied_steps_count FROM _prisma_migrations WHERE migration_name = %s',
+                    (str(uuid4()), COMPLETE.name),
+                )
+            case "missing_script":
+                pass
+        history: Final = database.history()
+        with containers.start(database, () if fault == "missing_script" else (COMPLETE,)) as replica:
+            unconfirmed((replica,), database)
+        assert database.history() == history
+        assert database.query("SELECT id FROM migration_effect") == ((1,),)
+
+    def test_coordinator_timeout_preserves_proof(self, containers: Containers, database: Database) -> None:
+        original: Final = confirmed_history(database)
+        with database.lock(COORDINATOR_LOCK):
+            with containers.start(
+                database, (COMPLETE,), environment={"LITELLM_MIGRATION_LOCK_TIMEOUT": "3"}
+            ) as replica:
+                failed((replica,), "Timed out waiting for another v2 migration resolver")
+                assert_original_proof(database, original, False)
+        with containers.start(database, (COMPLETE,)) as replica:
+            ready((replica,), database)
+        assert_original_proof(database, original, True)

tests/e2e/migrations/test_startup.py

@@ -0,0 +1,101 @@
+from contextlib import ExitStack
+from typing import Final
+
+import pytest
+
+from .checks import COMPLETE, FATAL, GATED, assert_completed, start_replicas
+from .containers import Containers, failed, ready, until, waiting
+from .database import PRISMA_LOCK, Database, Databases, restricted_user
+from .startup_models import Migration
+
+pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup]
+
+
+class TestMigrationStartup:
+    @pytest.mark.parametrize("replicas,v2", ((1, True), (3, True), (1, False)))
+    def test_fresh_database(self, containers: Containers, databases: Databases, replicas: int, v2: bool) -> None:
+        with databases.create() as database, ExitStack() as stack:
+            ready(tuple(stack.enter_context(containers.start(database, v2=v2)) for _ in range(replicas)), database)
+            assert database.query(
+                "SELECT count(*) FROM _prisma_migrations WHERE finished_at IS NULL AND rolled_back_at IS NULL"
+            ) == ((0,),)
+            assert database.query("SELECT count(*) > 0 FROM _prisma_migrations") == ((True,),)
+
+    def test_concurrent_upgrade(self, containers: Containers, database: Database) -> None:
+        with ExitStack() as stack:
+            ready(start_replicas(stack, containers, database, (COMPLETE,)), database)
+            assert_completed(database)
+
+    def test_waiters_survive_prolonged_contention(self, containers: Containers, database: Database) -> None:
+        with ExitStack() as stack:
+            with database.lock():
+                owner: Final = stack.enter_context(containers.start(database, (GATED,)))
+                until("owner blocked in migration SQL", lambda: bool(database.blocked()))
+                followers: Final = start_replicas(stack, containers, database, (GATED,), count=2)
+                until("both followers attempted Prisma locking", lambda: len(database.blocked(PRISMA_LOCK)) == 2)
+                waiting((owner, *followers), 120)
+            ready((owner, *followers), database)
+            assert_completed(database, GATED)
+
+    def test_lock_deadline_then_restart(self, containers: Containers, database: Database) -> None:
+        history: Final = database.history()
+        with database.lock(PRISMA_LOCK):
+            with containers.start(
+                database, (COMPLETE,), environment={"LITELLM_MIGRATION_LOCK_TIMEOUT": "12"}
+            ) as replica:
+                until("Prisma lock contention", lambda: bool(database.blocked(PRISMA_LOCK)))
+                failed((replica,), "Timed out waiting for")
+                assert database.history() == history
+                assert not database.exists("migration_effect")
+        with containers.start(database, (COMPLETE,)) as restarted:
+            ready((restarted,), database)
+            assert_completed(database)
+
+    def test_fatal_sql(self, containers: Containers, database: Database) -> None:
+        with ExitStack() as stack:
+            replicas: Final = start_replicas(stack, containers, database, (FATAL,))
+            failed(replicas, COMPLETE.name)
+            assert database.query(
+                'SELECT count(*) FROM _prisma_migrations WHERE migration_name = %s AND logs LIKE '
+                '%s AND finished_at IS NULL',
+                (COMPLETE.name, "%MIGRATION_TEST_FATAL%"),
+            ) == ((1,),)
+
+    def test_duplicate_object_does_not_hide_incomplete_sql(self, containers: Containers, database: Database) -> None:
+        database.execute(
+            "CREATE TABLE migration_existing (id int PRIMARY KEY); INSERT INTO migration_existing VALUES (42)"
+        )
+        migration: Final = Migration(
+            COMPLETE.name, "CREATE TABLE migration_existing (id int PRIMARY KEY); " + COMPLETE.script
+        )
+        with ExitStack() as stack:
+            failed(start_replicas(stack, containers, database, (migration,)), COMPLETE.name)
+            assert not database.exists("migration_effect")
+            assert database.query("SELECT id FROM migration_existing") == ((42,),)
+            assert database.query(
+                "SELECT finished_at IS NULL FROM _prisma_migrations WHERE migration_name = %s", (COMPLETE.name,)
+            ) == ((True,),)
+
+    @pytest.mark.parametrize("v2", (True, False))
+    def test_restart_preserves_history_and_data(self, containers: Containers, database: Database, v2: bool) -> None:
+        history: Final = database.history()
+        before: Final = database.query('SELECT token FROM "LiteLLM_VerificationToken" ORDER BY token')
+        for _ in range(2):
+            with containers.start(database, v2=v2) as replica:
+                ready((replica,), database)
+        assert database.history() == history
+        assert set(before).issubset(database.query('SELECT token FROM "LiteLLM_VerificationToken" ORDER BY token'))
+
+    def test_disabled_migrations(self, containers: Containers, database: Database) -> None:
+        history: Final = database.history()
+        with containers.start(database, (FATAL,), disabled=True) as replica:
+            ready((replica,), database)
+        assert database.history() == history
+
+    def test_insufficient_privileges(self, containers: Containers, database: Database) -> None:
+        history: Final = database.history()
+        with restricted_user(database) as limited:
+            with containers.start(limited, (COMPLETE,)) as replica:
+                failed((replica,), "permission denied")
+        assert database.history() == history
+        assert not database.exists("migration_effect")

tests/e2e/models.py

@@ -192,7 +192,7 @@ class ImageUrl(BaseModel):
 class TextContentPart(BaseModel):
     type: str = "text"
     text: str
-    cache_control: "CacheControl | None" = None
+    cache_control: CacheControl | None = None
 
 
 class ImageContentPart(BaseModel):
@@ -572,6 +572,10 @@ class McpInfo(BaseModel):
     logo_url: str | None = None
 
 
+class McpOauthCredentials(BaseModel):
+    upstream_resource: str
+
+
 class McpServerCreateBody(BaseModel):
     """POST /v1/mcp/server. For a gateway-managed OAuth server, `auth_type` is
     `oauth2` and `oauth2_flow` is `authorization_code`; the upstream endpoints
@@ -584,8 +588,11 @@ class McpServerCreateBody(BaseModel):
     allow_all_keys: bool = True
     auth_type: str | None = None
     oauth2_flow: Literal["client_credentials", "authorization_code"] | None = None
+    per_server_oauth_discovery: bool | None = None
     authorization_url: str | None = None
     token_url: str | None = None
+    registration_url: str | None = None
+    credentials: McpOauthCredentials | None = None
     server_name: str | None = None
     description: str | None = None
     mcp_info: McpInfo | None = None
@@ -625,6 +632,26 @@ class McpServerListResponse(RootModel[list[McpServerRow]]):
     """GET /v1/mcp/server answers with a bare array of servers."""
 
 
+class McpServerUserCredentialRow(BaseModel):
+    user_id: str
+    credential_type: Literal["oauth2", "byok"]
+    expires_at: str | None = None
+    connected_at: str | None = None
+    updated_at: str
+
+
+class McpServerUserCredentialListResponse(RootModel[tuple[McpServerUserCredentialRow, ...]]):
+    """GET /v1/mcp/server/{server_id}/user-credentials answers with a bare array."""
+
+
+class McpOauthUserCredentialStatus(BaseModel):
+    server_id: str
+    has_credential: bool
+    expires_at: str | None = None
+    is_expired: bool = False
+    connected_at: str | None = None
+
+
 class ToolsetTool(BaseModel):
     server_id: str
     tool_name: str
@@ -1172,8 +1199,9 @@ class TeamNewResponse(BaseModel):
 
 class TeamUpdateBody(BaseModel):
     team_id: str
-    team_alias: str
+    team_alias: str | None = None
     models: list[str] | None = None
+    object_permission: ObjectPermission | None = None
 
 
 class TeamInfoParams(BaseModel):

tests/e2e/provider_edge.py

@@ -46,7 +46,7 @@ import os
 import re
 import threading
 from collections import deque
-from collections.abc import Generator, Mapping, Sequence
+from collections.abc import Callable, Generator, Mapping, Sequence
 from contextlib import closing, contextmanager
 from dataclasses import dataclass, field, replace
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -538,7 +538,7 @@ class ReplayEdge:
 
 @dataclass(frozen=True, slots=True)
 class LiveEdge:
-    pass
+    observe_request: Callable[[str, Mapping[str, str], bytes | None], None] | None = None
 
 
 type EdgeBackend = RecordEdge | ReplayEdge | LiveEdge | CacheEdge
@@ -787,10 +787,13 @@ def _handle_record(
 def _handle_live(
     method: str, url: str, headers: Mapping[str, str], body: bytes | None, timeout: float,
     cache: CacheEdge | None = None, mount: str = "", test_key: str | None = None,
+    observe_request: Callable[[str, Mapping[str, str], bytes | None], None] | None = None,
 ) -> EdgeOutcome:
     forwarded: Final = {
         name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS
     }
+    if observe_request is not None:
+        observe_request(url, forwarded, body)
     head: Final = (
         forward_stream(method, url, headers=forwarded, body=body, timeout=timeout)
         if cache is None else cache.forward(mount, method, url, forwarded, body, timeout, test_key=test_key)
@@ -868,9 +871,10 @@ def handle_edge_request(
                 method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout,
                 backend, mount, test_key,
             )
-        case LiveEdge():
+        case LiveEdge(observe_request=observe_request):
             return _handle_live(
-                method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout
+                method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout,
+                observe_request=observe_request,
             )
         case RecordEdge():
             return _handle_record(

tests/e2e/proxy_client.py

@@ -89,6 +89,7 @@ from models import (
     TeamDeleteBody,
     TeamNewBody,
     TeamNewResponse,
+    TeamUpdateBody,
     ToolsetCreateBody,
     ToolsetRow,
     ToolsetUpdateBody,
@@ -871,6 +872,16 @@ class ProxyClient:
             )
         ).team_id
 
+    def update_team(self, body: TeamUpdateBody) -> None:
+        unwrap(
+            self.transport.post(
+                "/team/update",
+                headers=self.transport.master,
+                json=body,
+                response_type=NoBody,
+            )
+        )
+
     def delete_team(self, team_id: str) -> None:
         result = self.transport.post(
             "/team/delete",

tests/e2e/pytest.ini

@@ -12,3 +12,4 @@ markers =
     prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set
     cli_determinism: drives the real claude CLI for several seconds; deselected unless E2E_CLI_DETERMINISM is set
     redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set
+    mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set

tests/e2e/quota_management/spend_tracking/spend_e2e_client.py

@@ -359,6 +359,17 @@ class SpendClient:
     def probe(self, path: str, *, params: DateRangeParams) -> ProbeResult:
         return self.proxy.transport.probe(path, params=params)
 
+    def probe_until_healthy(self, path: str, *, params: DateRangeParams) -> ProbeResult:
+        outcome: Final = await_converged(
+            lambda: self.probe(path, params=params),
+            converged=lambda result: result.healthy,
+            timeout=self.proxy.poll_timeout,
+            interval=self.proxy.poll_interval,
+            now=time.monotonic,
+            sleep=time.sleep,
+        )
+        return outcome.result if isinstance(outcome, Converged) else outcome.last_result
+
     def create_user(self, *, email: str, role: UserRole, user_id: str) -> str:
         return unwrap(
             self.proxy.transport.post(

tests/e2e/quota_management/spend_tracking/test_spend_routes.py

@@ -17,9 +17,11 @@ fast: no batch-write wait, no provider calls.
 """
 
 from datetime import datetime, timedelta, timezone
+from typing import Final
 
 import pytest
 
+from e2e_http import ProbeResult
 from models import DateRangeParams
 from spend_e2e_client import SpendClient
 
@@ -72,15 +74,10 @@ SPEND_ROUTES = (
 
 _SPEND_PREFIXES = ("/spend", "/global/spend", "/global/activity")
 
-_MISSING_VIEW_SKIP = pytest.mark.skip(
-    reason=(
-        "LIT-5211: on a fresh database the proxy's startup view creation can lose the race "
-        "against schema migrations, leaving MonthlyGlobalSpend/DailyTagSpend/Last30d* views "
-        "missing and these routes 500ing until the views exist"
-    )
-)
-
-_VIEW_BACKED_ROUTES = frozenset(
+# Served from the MonthlyGlobalSpend / DailyTagSpend / Last30d* views, which the
+# proxy creates in the background once the schema migrations have landed, so on a
+# fresh database they can 500 for a while after the proxy starts serving.
+_VIEW_BACKED_ROUTES: Final = frozenset(
     (
         "/global/spend",
         "/global/spend/keys",
@@ -98,15 +95,15 @@ def _date_range() -> DateRangeParams:
     return DateRangeParams(start_date=start.isoformat(), end_date=end.isoformat())
 
 
-@pytest.mark.parametrize(
-    "route",
-    tuple(
-        pytest.param(route, marks=_MISSING_VIEW_SKIP) if route in _VIEW_BACKED_ROUTES else route
-        for route in SPEND_ROUTES
-    ),
-)
+def _probe(client: SpendClient, route: str) -> ProbeResult:
+    if route in _VIEW_BACKED_ROUTES:
+        return client.probe_until_healthy(route, params=_date_range())
+    return client.probe(route, params=_date_range())
+
+
+@pytest.mark.parametrize("route", SPEND_ROUTES)
 def test_spend_route_responsive(client: SpendClient, route: str) -> None:
-    result = client.probe(route, params=_date_range())
+    result = _probe(client, route)
     print(f"{route} -> {result.status_code}\n{result.body[:600]}")
     assert result.healthy, f"{route} -> {result.status_code}\n{result.body[:600]}"
 

tests/integration/AGENTS.md

@@ -0,0 +1,25 @@
+# tests/integration
+
+Real proxy, Postgres, Redis, scripted upstream. `README.md` has shards and CI wiring
+
+## What good looks like
+
+The root example is from here. The spend row lands async: poll, never sleep
+
+```python
+rows = eventually(
+    lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (body["id"],)),
+    lambda values: len(values) == 1,
+    seconds=70,
+)
+assert float(rows[0]["spend"]) == pytest.approx(10 * 0.001 + 5 * 0.0001 + 7 * 0.002 + 4 * 0.002)
+```
+
+`sleep(3)` fails on a slow runner and taxes every fast one. Assert the outbound body in the upstream
+handler; a leaked field is invisible from the response. `monkeypatch.setenv` is fine; patching our own
+function in a full stack is not
+
+## Where it goes
+
+By the domain a user would name: `pricing`, `spend`, `routing`. Add the node and its `covers` ids to
+`contracts.json` or collection fails. Needs no proxy, DB or Redis: `tests/unit`

tests/integration/_support/process.py

@@ -74,8 +74,6 @@ def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str],
                 str(port),
                 "--num_workers",
                 "1",
-                "--telemetry",
-                "False",
                 "--use_prisma_db_push",
                 "--enforce_prisma_migration_check",
             ],

tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py

@@ -728,12 +728,36 @@ ERROR: relation "SomeTable" already exists
 """
 
 
+@pytest.mark.parametrize(
+    "pooled,direct,expected",
+    (
+        ("postgresql://pool/db?pgbouncer=true", None, "postgresql://pool/db?pgbouncer=true"),
+        ("postgresql://pool/db?pgbouncer=true", "postgresql://writer/db", "postgresql://writer/db?schema=public"),
+        (
+            "postgresql://pool/db?schema=tenant%20one&pgbouncer=true",
+            "postgresql://writer/db?sslmode=require&schema=wrong",
+            "postgresql://writer/db?sslmode=require&schema=tenant+one",
+        ),
+    ),
+)
+def test_v2_migrations_use_the_direct_connection_with_the_runtime_schema(pooled, direct, expected):
+    from litellm_proxy_extras.migration_lock import migration_environment
+
+    environment = {"DATABASE_URL": pooled, "PRISMA_OFFLINE_MODE": "true"}
+    configured = {**environment, **({"DIRECT_URL": direct} if direct else {})}
+    migrated = migration_environment(configured)
+
+    assert migrated["DATABASE_URL"] == expected
+    assert migrated["PRISMA_OFFLINE_MODE"] == "true"
+    assert configured["DATABASE_URL"] == pooled
+
+
 class _MigrateDeployHarness:
     """Drives _setup_database_v2 with a scripted sequence of
     `prisma migrate deploy` outcomes, with every recovery command faked out so
     nothing touches a database or the packaged migrations directory."""
 
-    def __init__(self, monkeypatch, tmp_path, outcomes, repeat_last=False):
+    def __init__(self, monkeypatch, tmp_path, outcomes, repeat_last=False, confirmed_migrations=()):
         import subprocess as subprocess_module
 
         import litellm_proxy_extras.utils as utils_module
@@ -744,34 +768,20 @@ class _MigrateDeployHarness:
         self._outcomes = list(outcomes)
         self._repeat_last = repeat_last
         self._subprocess_module = subprocess_module
+        self.confirmed_migrations = set(confirmed_migrations)
 
         monkeypatch.delenv("DATABASE_URL", raising=False)
-        monkeypatch.setattr(
-            ProxyExtrasDBManager, "_get_prisma_dir", staticmethod(lambda: str(tmp_path))
-        )
-        monkeypatch.setattr(
-            ProxyExtrasDBManager,
-            "_create_baseline_migration",
-            staticmethod(self._fake_baseline),
-        )
-        monkeypatch.setattr(
-            ProxyExtrasDBManager,
-            "_roll_back_migration",
-            staticmethod(lambda name: None),
-        )
-        monkeypatch.setattr(
-            ProxyExtrasDBManager,
-            "_resolve_specific_migration",
-            staticmethod(self.resolved.append),
-        )
+        monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path))
         monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", self._fake_run)
+        monkeypatch.setattr(utils_module, "_get_prisma_env", lambda: {})
         monkeypatch.setattr(utils_module.time, "sleep", lambda seconds: None)
 
         self.baseline_succeeds = True
 
     def _fake_baseline(self, *args, **kwargs):
         self.baselines += 1
-        return self.baseline_succeeds
+        if not self.baseline_succeeds:
+            raise RuntimeError("The existing schema was not verified")
 
     def _next_outcome(self):
         if self._outcomes:
@@ -791,79 +801,126 @@ class _MigrateDeployHarness:
         raise self._subprocess_module.CalledProcessError(1, cmd, stderr=outcome)
 
     def run(self):
-        return ProxyExtrasDBManager._setup_database_v2(use_migrate=True)
+        while not ProxyExtrasDBManager._run_database_v2(
+            use_migrate=True,
+            recover_completed=self._fake_recovery,
+            baseline_existing=self._fake_baseline,
+        ):
+            continue
+        return True
+
+    def _fake_recovery(self, name):
+        if name not in self.confirmed_migrations:
+            return False
+        self.confirmed_migrations.remove(name)
+        self.resolved.append(name)
+        return True
 
 
 class TestMigrateDeployAttemptAccounting:
-    """A `prisma db push` database has a full schema and no ledger, so the v2
-    resolver baselines it and then works through every migration whose objects
-    already exist. Those recoveries make progress, so they must not spend the
-    retry budget, which is there to stop a run that is getting nowhere."""
-
-    def test_a_push_created_database_finishes_bootstrapping(
-        self, monkeypatch, tmp_path
-    ):
-        already_there = [
-            "20250329084805_new_cron_job_table",
-            "20250806095134_rename_alias_to_server_name_mcp_table",
-            "20260224203854_add_agent_object_permissions_table",
-            "20260301120000_fourth_table",
-            "20260302120000_fifth_table",
-            "20260303120000_sixth_table",
-        ]
+    def test_a_push_created_database_finishes_bootstrapping(self, monkeypatch, tmp_path):
         harness = _MigrateDeployHarness(
             monkeypatch,
             tmp_path,
-            [_P3005_STDERR]
-            + [_p3018_stderr(name) for name in already_there]
-            + ["ok"],
+            [_P3005_STDERR, "ok"],
         )
 
         assert harness.run() is True
         assert harness.baselines == 1
-        assert harness.resolved == already_there
-        assert len(harness.deploy_calls) == len(already_there) + 2
+        assert harness.resolved == []
+        assert len(harness.deploy_calls) == 2
 
-    def test_repeated_recovery_of_one_migration_still_gives_up(
-        self, monkeypatch, tmp_path
-    ):
+    def test_repeated_recovery_of_one_migration_still_gives_up(self, monkeypatch, tmp_path):
         harness = _MigrateDeployHarness(
             monkeypatch,
             tmp_path,
             [_p3018_stderr("20250329084805_new_cron_job_table")],
             repeat_last=True,
+            confirmed_migrations=("20250329084805_new_cron_job_table",),
         )
 
         with pytest.raises(RuntimeError):
             harness.run()
-        assert len(harness.deploy_calls) <= _ATTEMPT_BUDGET + 1
+        assert len(harness.deploy_calls) == 2
+        assert harness.resolved == ["20250329084805_new_cron_job_table"]
 
     def test_timeouts_still_spend_the_budget(self, monkeypatch, tmp_path):
-        harness = _MigrateDeployHarness(
-            monkeypatch, tmp_path, ["timeout"], repeat_last=True
-        )
+        harness = _MigrateDeployHarness(monkeypatch, tmp_path, ["timeout"], repeat_last=True)
 
         with pytest.raises(RuntimeError):
             harness.run()
         assert len(harness.deploy_calls) == _ATTEMPT_BUDGET
 
-    def test_a_baseline_that_never_lands_stops_after_the_budget(
-        self, monkeypatch, tmp_path
-    ):
-        harness = _MigrateDeployHarness(
-            monkeypatch, tmp_path, [_P3005_STDERR], repeat_last=True
-        )
+    def test_an_unverified_baseline_stops_without_replaying_migrations(self, monkeypatch, tmp_path):
+        harness = _MigrateDeployHarness(monkeypatch, tmp_path, [_P3005_STDERR], repeat_last=True)
         harness.baseline_succeeds = False
 
         with pytest.raises(RuntimeError):
             harness.run()
-        assert len(harness.deploy_calls) == _ATTEMPT_BUDGET
+        assert len(harness.deploy_calls) == 1
+
+    def test_lock_contention_does_not_spend_the_failure_budget(self, monkeypatch, tmp_path):
+        harness = _MigrateDeployHarness(
+            monkeypatch,
+            tmp_path,
+            ["Error: P1002\nTimed out waiting for the advisory lock"] * 6 + ["ok"],
+        )
+        assert harness.run() is True
+        assert len(harness.deploy_calls) == 7
+
+    def test_duplicate_object_error_without_completion_proof_is_fatal(self, monkeypatch, tmp_path):
+        harness = _MigrateDeployHarness(monkeypatch, tmp_path, [_p3018_stderr("20260101000000_x")])
+        with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
+            harness.run()
+        assert harness.resolved == []
+        assert len(harness.deploy_calls) == 1
+
+    @pytest.mark.parametrize("name", ("20260101000000_x", "20260101000000_migration with spaces"))
+    def test_an_interrupted_migration_with_confirmed_sql_can_finish(self, monkeypatch, tmp_path, name):
+        harness = _MigrateDeployHarness(
+            monkeypatch,
+            tmp_path,
+            [f"Error: P3009\nThe `{name}` migration failed", "ok"],
+            confirmed_migrations=(name,),
+        )
+        assert harness.run() is True
+        assert harness.resolved == [name]
+
+    def test_an_interrupted_migration_without_confirmation_stops(self, monkeypatch, tmp_path):
+        name = "20260101000000_x"
+        started = "2026-09-12 20:15:06.694553 UTC"
+        report = f"Error: P3009\nThe `{name}` migration started at {started} failed"
+        harness = _MigrateDeployHarness(
+            monkeypatch,
+            tmp_path,
+            [report],
+        )
+        with pytest.raises(RuntimeError, match="Migration completion could not be verified") as failure:
+            harness.run()
+        message = str(failure.value)
+        assert name in message
+        assert started in message
+        assert "start record but no successful completion record" in message
+        assert "cannot determine whether its SQL committed" in message
+        assert "avoid repeating or skipping database changes" in message
+        assert "_prisma_migrations" in message
+        assert "migration.sql" in message
+        assert "same database" in message
+        assert "Only after verifying every migration change is present" in message
+        assert "prisma migrate resolve --applied <migration_name>" in message
+        assert "Only after verifying no migration changes remain" in message
+        assert "prisma migrate resolve --rolled-back <migration_name>" in message
+        assert "leave migration history unchanged" in message
+        assert "Repeated restarts alone" in message
+        assert report in message
+        assert len(harness.deploy_calls) == 1
+        assert harness.resolved == []
 
     def test_an_unrecoverable_error_is_not_retried(self, monkeypatch, tmp_path):
         harness = _MigrateDeployHarness(
             monkeypatch,
             tmp_path,
-            ["Error: P3018\n\nMigration name: 20260101000000_x\n\nERROR: syntax error at or near \"SLECT\"\n"],
+            ['Error: P3018\n\nMigration name: 20260101000000_x\n\nERROR: syntax error at or near "SLECT"\n'],
             repeat_last=True,
         )
 
@@ -873,6 +930,36 @@ class TestMigrateDeployAttemptAccounting:
         assert harness.resolved == []
 
 
+@pytest.mark.parametrize(
+    "steps,logs,script,expected",
+    (
+        (1, "", b"CREATE TABLE item (id int);", True),
+        (0, "", b"CREATE TABLE item (id int);", False),
+        (0, "already exists", b"CREATE TABLE item (id int);", False),
+        (1, "permission denied", b"CREATE TABLE item (id int);", False),
+        (1, "", b"CREATE TABLE item (id text);", False),
+        (2, "", b"CREATE TABLE item (id int);", False),
+    ),
+)
+def test_migration_completion_requires_a_matching_successful_script(steps, logs, script, expected):
+    import hashlib
+
+    from litellm_proxy_extras.migration_recovery import MigrationProgress
+
+    progress = MigrationProgress(hashlib.sha256(b"CREATE TABLE item (id int);").hexdigest(), steps, logs)
+    assert progress.confirms_completion(script) is expected
+
+
+def test_prisma_lock_waiting_has_its_own_deadline():
+    from litellm_proxy_extras.utils import _MigrateAttemptBudget
+
+    budget = _MigrateAttemptBudget(attempts_left=4, contention_seconds_left=2)
+    waiting = budget.after_contention(1)
+    assert waiting.attempts_left == 4
+    with pytest.raises(RuntimeError, match="advisory lock"):
+        waiting.after_contention(2)
+
+
 class TestJWTKeyMappingCascade:
     """Regression tests for issue #33702.
 

tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py

@@ -883,7 +883,6 @@ async def test_provider_specific_fields_in_proxy_http_response(
         max_tokens=None,
         request_timeout=600,
         max_budget=None,
-        telemetry=False,
         drop_params=True,
         add_function_to_prompt=False,
         headers=None,

tests/proxy_migration_tests/test_migration_ci.py

@@ -0,0 +1,36 @@
+import importlib.util
+from pathlib import Path
+from typing import Final
+
+import pytest
+
+SCRIPT: Final = Path(__file__).resolve().parents[2] / ".circleci/scripts/run_migration_tests.py"
+SPEC: Final = importlib.util.spec_from_file_location("migration_ci", SCRIPT)
+assert SPEC is not None and SPEC.loader is not None
+MODULE: Final = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MODULE)
+
+
+@pytest.mark.parametrize(
+    "xml,expected,exit_code,passed",
+    (
+        ('<testsuites><testsuite><testcase name="a"/><testcase name="b"/></testsuite></testsuites>', 2, 0, True),
+        ('<testsuites><testsuite><testcase name="a"/></testsuite></testsuites>', 2, 0, False),
+        ("<testsuite><testcase><failure/></testcase></testsuite>", 1, 0, False),
+        ("<testsuite><testcase><error/></testcase></testsuite>", 1, 0, False),
+        ("<testsuite><testcase><skipped/></testcase></testsuite>", 1, 0, False),
+        ("<testsuite><testcase/></testsuite>", 1, 1, False),
+        ("<testsuite><testcase/></testsuite>", 1, 5, False),
+        ("<testsuite/>", 1, 0, False),
+        ("<broken", 1, 0, False),
+        (None, 1, 0, False),
+        ('<testsuite><testcase name="a"/><testcase name="a"/></testsuite>', 2, 0, False),
+    ),
+)
+def test_only_a_complete_passing_suite_can_certify_an_image(
+    tmp_path: Path, xml: str | None, expected: int, exit_code: int, passed: bool
+) -> None:
+    path: Final = tmp_path / "results.xml"
+    if xml is not None:
+        path.write_text(xml)
+    assert MODULE.successful_junit(path, expected, exit_code) is passed

tests/proxy_unit_tests/test_proxy_server.py

@@ -2382,7 +2382,7 @@ async def test_proxy_model_group_info_rerank(prisma_client):  # noqa: F811  # py
 
 @pytest.mark.asyncio
 async def test_proxy_server_prisma_setup():
-    from litellm.proxy.proxy_server import ProxyStartupEvent, proxy_state
+    from litellm.proxy.proxy_server import ProxyStartupEvent
     from litellm.proxy.utils import ProxyLogging
     from litellm.caching import DualCache
 
@@ -2393,35 +2393,28 @@ async def test_proxy_server_prisma_setup():
     ) as mock_prisma_client:
         mock_client = mock_prisma_client.return_value  # This is the mocked instance
         mock_client.connect = AsyncMock()  # Mock the connect method
-        mock_client.check_view_exists = AsyncMock()  # Mock the check_view_exists method
+        mock_client.start_view_setup_task = MagicMock()
         mock_client.health_check = AsyncMock()  # Mock the health_check method
-        mock_client._set_spend_logs_row_count_in_proxy_state = (
-            AsyncMock()
-        )  # Mock the _set_spend_logs_row_count_in_proxy_state method
         mock_client.start_db_health_watchdog_task = AsyncMock()
         # Mock the db attribute with start_token_refresh_task for RDS IAM token refresh
         mock_db = MagicMock()
         mock_db.start_token_refresh_task = AsyncMock()
         mock_client.db = mock_db
 
-        await ProxyStartupEvent._setup_prisma_client(
+        prisma_client = await ProxyStartupEvent._setup_prisma_client(
             database_url=os.getenv("DATABASE_URL"),
             proxy_logging_obj=ProxyLogging(user_api_key_cache=user_api_key_cache),
             user_api_key_cache=user_api_key_cache,
         )
 
-        # Verify our mocked methods were called
+        assert prisma_client is mock_client
         mock_client.connect.assert_called_once()
-        mock_client.check_view_exists.assert_called_once()
+        mock_client.start_view_setup_task.assert_called_once()
 
         # Note: This is REALLY IMPORTANT to check that the health check is called
         # This is how we ensure the DB is ready before proceeding
         mock_client.health_check.assert_called_once()
 
-        # check that the spend logs row count is set in proxy state
-        mock_client._set_spend_logs_row_count_in_proxy_state.assert_called_once()
-        assert proxy_state.get_proxy_state_variable("spend_logs_row_count") is not None
-
 
 @pytest.mark.asyncio
 async def test_proxy_server_prisma_setup_invalid_db(monkeypatch):