diff --git a/.circleci/config.yml b/.circleci/config.yml index 23ef423039f..a8a33335ad7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,6 +111,28 @@ commands: - wait_for_service: url: tcp://localhost:6379 timeout: "60" + start_openai_record_replay_proxy: + description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." + steps: + - run: + name: Start record/replay proxy + background: true + command: | + CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \ + RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \ + uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090 + - run: + name: Wait for record/replay proxy + command: | + for i in $(seq 1 30); do + if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then + echo "record/replay proxy is up" + exit 0 + fi + sleep 1 + done + echo "record/replay proxy did not become ready" >&2 + exit 1 setup_litellm_enterprise_pip: steps: - run: @@ -182,7 +204,14 @@ jobs: - run: name: Run Windows-specific test command: | - uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v + uv run --no-sync python -m pytest tests/windows_tests/ -v + - run: + name: Guard against MAX_PATH-busting packaged wheel paths + environment: + UV_HTTP_TIMEOUT: "300" + command: | + uv build --wheel --out-dir dist + uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py local_testing_part1: docker: @@ -242,8 +271,15 @@ jobs: - run: name: Rename the coverage files command: | - mv coverage.xml local_testing_part1_coverage.xml - mv .coverage local_testing_part1_coverage + # When CI reruns only the failed tests, a parallel node can receive + # zero tests and pytest never writes coverage. Emit empty placeholders + # so persist_to_workspace and the downstream coverage combine stay green. + if [ -f coverage.xml ]; then + mv coverage.xml local_testing_part1_coverage.xml + mv .coverage local_testing_part1_coverage + else + touch local_testing_part1_coverage.xml local_testing_part1_coverage + fi # Store test results - store_test_results: @@ -307,8 +343,15 @@ jobs: - run: name: Rename the coverage files command: | - mv coverage.xml local_testing_part2_coverage.xml - mv .coverage local_testing_part2_coverage + # When CI reruns only the failed tests, a parallel node can receive + # zero tests and pytest never writes coverage. Emit empty placeholders + # so persist_to_workspace and the downstream coverage combine stay green. + if [ -f coverage.xml ]; then + mv coverage.xml local_testing_part2_coverage.xml + mv .coverage local_testing_part2_coverage + else + touch local_testing_part2_coverage.xml local_testing_part2_coverage + fi # Store test results - store_test_results: @@ -431,6 +474,120 @@ jobs: - auth_ui_unit_tests_coverage.xml - auth_ui_unit_tests_coverage + proxy_behavior_tests: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Seed DB schema via prisma db push + command: | + uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate + - run: + name: Run proxy management behavior tests + command: | + mkdir -p test-results + uv run --no-sync python -m pytest tests/proxy_behavior \ + -v --junitxml=test-results/junit.xml --durations=10 + no_output_timeout: 15m + - store_test_results: + path: test-results + + proxy_security_tests: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Seed DB schema via prisma db push + command: | + uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate + - run: + name: Run proxy security tests + command: | + mkdir -p test-results + uv run --no-sync python -m pytest tests/proxy_security_tests \ + -v --junitxml=test-results/junit.xml --durations=10 + no_output_timeout: 15m + - store_test_results: + path: test-results + + schema_migration_check: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + # An empty database; the test applies every committed migration itself. + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate + - run: + name: Check schema.prisma is in sync with committed migrations + command: | + mkdir -p test-results + uv run --no-sync python -m pytest tests/proxy_migration_tests \ + -v --junitxml=test-results/junit.xml --durations=10 + no_output_timeout: 15m + - store_test_results: + path: test-results + litellm_router_testing: # Runs all tests with the "router" keyword docker: - *python312_image @@ -457,6 +614,11 @@ jobs: - run: name: Run tests command: | + # On a "rerun failed tests" build a parallel node can receive no + # tests, so the test command never creates test-results. Pre-create it + # so store_test_results doesn't fail the node on a missing path. + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ @@ -1485,6 +1647,7 @@ jobs: command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker tag litellm-docker-database:ci my-app:latest + - start_openai_record_replay_proxy - run: name: Run Docker container command: | @@ -1515,6 +1678,7 @@ jobs: -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + -e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ @@ -1652,6 +1816,7 @@ jobs: command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database + - start_openai_record_replay_proxy - run: name: Run Docker container # intentionally give bad redis credentials here @@ -1675,6 +1840,7 @@ jobs: -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e COHERE_API_KEY=$COHERE_API_KEY \ + -e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ --name my-app \ @@ -2240,6 +2406,7 @@ jobs: command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database + - start_openai_record_replay_proxy - run: name: Run Docker container with test config command: | @@ -2248,6 +2415,7 @@ jobs: -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e LITELLM_MASTER_KEY="sk-1234" \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e RECORDER_ANTHROPIC_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.anthropic.com \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME="us-east-1" \ @@ -2617,6 +2785,12 @@ workflows: filters: *main_branches - auth_ui_unit_tests: filters: *main_branches + - proxy_behavior_tests: + filters: *main_branches + - proxy_security_tests: + filters: *main_branches + - schema_migration_check: + filters: *main_branches - build_docker_database_image: filters: *main_branches - e2e_ui_testing: diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index f0ced6bedb8..23b520e2ad5 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -8,3 +8,6 @@ # Update pydantic code to fix warnings (GH-3600) 876840e9957bc7e9f7d6a2b58c4d7c53dad16481 + +# style(ui): run prettier --write across the dashboard (#29622) +7edf3a9cb55548b143df1692f4ed7c4681d7fcf7 diff --git a/.gitattributes b/.gitattributes index 9030923a781..5c9061f52ac 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ -*.ipynb linguist-vendored \ No newline at end of file +*.ipynb linguist-vendored +ui/litellm-dashboard/src/lib/http/schema.d.ts linguist-generated \ No newline at end of file diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 7e91341ac77..a42b2f8f9df 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -27,6 +27,11 @@ on: required: false type: number default: 10 + dist: + description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)" + required: false + type: string + default: "loadscope" artifact-name: description: "Unique name for the coverage artifact (must be unique per run)" required: true @@ -82,18 +87,31 @@ jobs: MAX_FAILURES: ${{ inputs.max-failures }} WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} + DIST: ${{ inputs.dist }} run: | - uv run --no-sync pytest ${TEST_PATH:?} \ - --tb=short -vv \ - --maxfail="${MAX_FAILURES}" \ - -n "${WORKERS}" \ - --reruns "${RERUNS}" \ - --reruns-delay 1 \ - --dist=loadscope \ - --durations=20 \ - --cov=./litellm \ - --cov-report=xml:coverage.xml \ - --cov-config=pyproject.toml + if [ "${WORKERS}" = "0" ]; then + uv run --no-sync pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --durations=20 \ + --cov=./litellm \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml + else + uv run --no-sync pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist="${DIST}" \ + --durations=20 \ + --cov=./litellm \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml + fi - name: Save coverage report if: always() diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml deleted file mode 100644 index 7f973d8cafa..00000000000 --- a/.github/workflows/_test-unit-services-base.yml +++ /dev/null @@ -1,190 +0,0 @@ -name: _Unit Test Services Base (Reusable) - -on: - workflow_call: - inputs: - test-path: - description: "Pytest path(s) to run" - required: true - type: string - workers: - description: "Number of pytest-xdist workers (0 = no parallelism)" - required: false - type: number - default: 2 - reruns: - description: "Number of reruns for flaky tests" - required: false - type: number - default: 2 - timeout-minutes: - description: "Job timeout in minutes" - required: false - type: number - default: 20 - max-failures: - description: "Stop after this many failures" - required: false - type: number - default: 10 - enable-postgres: - description: "Start a local Postgres service container and run Prisma migrations" - required: false - type: boolean - default: false - dist: - description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)" - required: false - type: string - default: "loadscope" - artifact-name: - description: "Unique name for the coverage artifact (must be unique per run)" - required: false - type: string - default: "run" - -permissions: - contents: read - -# The postgres service container below is spawned per-job on localhost and -# destroyed with the job. Nothing outside the runner can reach it. The -# user/password/database here are not secrets — they're bootstrap values -# for a throwaway container — so we hardcode them instead of attaching -# every matrix shard to a GHA environment just to read three "secrets" -# (which also produces a "temporarily deployed to …" notification on the -# PR timeline per shard per push). -jobs: - run: - name: Run tests - runs-on: ubuntu-latest - timeout-minutes: ${{ inputs.timeout-minutes }} - - services: - postgres: - image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14 - env: - POSTGRES_USER: litellm - POSTGRES_PASSWORD: litellm - POSTGRES_DB: litellm_test - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - version: "0.10.9" - - - name: Cache uv dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cache/uv - .venv - key: ${{ runner.os }}-uv-services-${{ hashFiles('uv.lock') }} - restore-keys: | - ${{ runner.os }}-uv-services- - - - name: Install dependencies - run: | - uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - - - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache - run: | - uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - - name: Run Prisma migrations - if: ${{ inputs.enable-postgres }} - env: - DATABASE_URL: "postgresql://litellm:litellm@localhost:5432/litellm_test" - run: | - uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - - name: Run tests - env: - TEST_PATH: ${{ inputs.test-path }} - MAX_FAILURES: ${{ inputs.max-failures }} - WORKERS: ${{ inputs.workers }} - RERUNS: ${{ inputs.reruns }} - DIST: ${{ inputs.dist }} - DATABASE_URL: ${{ inputs.enable-postgres && 'postgresql://litellm:litellm@localhost:5432/litellm_test' || '' }} - run: | - if [ "${WORKERS}" = "0" ]; then - uv run --no-sync pytest ${TEST_PATH:?} \ - --tb=short -vv \ - --maxfail="${MAX_FAILURES}" \ - --reruns "${RERUNS}" \ - --reruns-delay 1 \ - --durations=20 \ - --cov=./litellm \ - --cov-report=xml:coverage.xml \ - --cov-config=pyproject.toml - else - uv run --no-sync pytest ${TEST_PATH:?} \ - --tb=short -vv \ - --maxfail="${MAX_FAILURES}" \ - -n "${WORKERS}" \ - --reruns "${RERUNS}" \ - --reruns-delay 1 \ - --dist="${DIST}" \ - --durations=20 \ - --cov=./litellm \ - --cov-report=xml:coverage.xml \ - --cov-config=pyproject.toml - fi - - - name: Save coverage report - if: always() - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} - path: coverage.xml - retention-days: 1 - - upload-coverage: - name: Upload coverage to Codecov - needs: run - if: always() - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write - pull-requests: write - - steps: - - name: Checkout code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Download coverage report - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 - with: - pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} - path: coverage-reports - merge-multiple: true - - - name: Upload to Codecov - uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 - with: - use_oidc: true - directory: coverage-reports - root_dir: ${{ github.workspace }} - flags: ${{ inputs.artifact-name }} - fail_ci_if_error: false diff --git a/.github/workflows/check-ui-api-types.yml b/.github/workflows/check-ui-api-types.yml new file mode 100644 index 00000000000..eeb5545b15e --- /dev/null +++ b/.github/workflows/check-ui-api-types.yml @@ -0,0 +1,84 @@ +name: Check UI API Types Sync + +on: + pull_request: + paths: + - "litellm/proxy/**" + - "litellm/types/**" + - "ui/litellm-dashboard/src/lib/http/schema.d.ts" + - "ui/litellm-dashboard/scripts/gen-api-types.mjs" + - "ui/litellm-dashboard/package.json" + - "ui/litellm-dashboard/package-lock.json" + - ".github/workflows/check-ui-api-types.yml" + +permissions: + contents: read + +jobs: + check-sync: + name: Verify schema.d.ts matches the proxy OpenAPI spec + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Cache uv dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install backend dependencies + run: uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Set up Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: ui/litellm-dashboard/package-lock.json + + - name: Install dashboard dependencies + working-directory: ui/litellm-dashboard + run: npm ci + + - name: Regenerate types from the live spec + working-directory: ui/litellm-dashboard + env: + LITELLM_PYTHON: "uv run --no-sync python" + run: npm run gen:api + + - name: Fail if types are stale + run: | + if ! git diff --exit-code -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then + echo "::error file=ui/litellm-dashboard/src/lib/http/schema.d.ts::Generated API types are out of sync with the proxy OpenAPI spec." + echo "" + echo "A backend route or model changed without regenerating the dashboard types." + echo "To fix, run from ui/litellm-dashboard:" + echo " npm run gen:api" + echo "then commit the updated src/lib/http/schema.d.ts." + exit 1 + fi + echo "schema.d.ts is in sync with the proxy OpenAPI spec." diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index 862f98e30f1..68497b10dbb 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -36,3 +36,79 @@ jobs: - name: Build run: npm run build + + frontend-lint: + runs-on: ubuntu-latest + timeout-minutes: 8 + defaults: + run: + working-directory: ui/litellm-dashboard + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Collect changed files + id: changed + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + : > "$RUNNER_TEMP/prettier_files.txt" + : > "$RUNNER_TEMP/eslint_files.txt" + while IFS= read -r f; do + [ -f "$f" ] || continue + case "$f" in + *.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs) + printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" + printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;; + *.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html) + printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;; + esac + done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .) + if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then + echo "has_files=true" >> "$GITHUB_OUTPUT" + else + echo "has_files=false" >> "$GITHUB_OUTPUT" + echo "No lintable UI files changed in this PR; nothing to check." + fi + + - name: Setup Node.js + if: steps.changed.outputs.has_files == 'true' + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: ui/litellm-dashboard/package-lock.json + + - name: Install dependencies + if: steps.changed.outputs.has_files == 'true' + run: npm ci + + - name: Lint changed files (prettier + eslint) + if: steps.changed.outputs.has_files == 'true' + run: | + prettier_files=() + eslint_files=() + while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt" + while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt" + status=0 + if [ ${#prettier_files[@]} -gt 0 ]; then + echo "::group::Prettier (${#prettier_files[@]} files)" + npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; } + echo "::endgroup::" + fi + if [ ${#eslint_files[@]} -gt 0 ]; then + echo "::group::ESLint (${#eslint_files[@]} files)" + npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1 + echo "::endgroup::" + fi + exit $status + + - name: Check lint budgets + if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }} + run: | + npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true + node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml index 9add77ff424..a7363ac3b43 100644 --- a/.github/workflows/test-unit-misc.yml +++ b/.github/workflows/test-unit-misc.yml @@ -28,6 +28,8 @@ jobs: tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client + tests/test_litellm/models + tests/test_litellm/repositories tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index 2d4e85630dc..2ac9a3b7c1c 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -1,9 +1,10 @@ name: "Unit Tests: Proxy DB Operations" -# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. on: - push: - branches: [main, "litellm_**"] + pull_request: + branches: + - main + - litellm_internal_staging permissions: contents: read @@ -30,9 +31,6 @@ concurrency: # xdist balances its 188 parametrized cases across workers instead of # pinning the whole file to one worker (the default --dist=loadscope # behavior for single-file targets). -# * test_db_schema_migration.py is isolated because one test in it -# (test_aaaasschema_migration_check) takes ~170s — by itself it -# determines the shard's wall-clock floor. jobs: # Fast guard — fails the workflow if a test_*.py file under # tests/proxy_unit_tests/ is not referenced by any matrix entry below. @@ -166,18 +164,6 @@ jobs: dist: loadscope timeout: 15 - # ---- db-and-spend: isolate the 170s schema-migration test ---- - # test_db_schema_migration.py has exactly one test, and that test - # is mostly waiting on `prisma migrate deploy` / `prisma migrate - # diff` subprocesses (~170s). It does no CPU-bound Python work - # inside the test. Running with workers=0 (serial, no xdist) - # skips the 4-worker cold-start cost we'd otherwise pay for a - # single test, saving ~4 minutes of wall-clock. - - test-group: schema-migration - test-path: "tests/proxy_unit_tests/test_db_schema_migration.py" - workers: 0 - dist: loadscope - timeout: 15 - test-group: db-and-spend test-path: >- tests/proxy_unit_tests/test_prisma_client_backoff_retry.py @@ -232,12 +218,11 @@ jobs: workers: 4 dist: loadscope timeout: 15 - uses: ./.github/workflows/_test-unit-services-base.yml + uses: ./.github/workflows/_test-unit-base.yml with: test-path: ${{ matrix.test-path }} workers: ${{ matrix.workers }} reruns: 2 timeout-minutes: ${{ matrix.timeout }} - enable-postgres: true dist: ${{ matrix.dist }} artifact-name: proxy-db-${{ matrix.test-group }} diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml index 1ce0abfc008..0a9513ec024 100644 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -42,6 +42,7 @@ jobs: tests/test_litellm/proxy/rag_endpoints tests/test_litellm/proxy/realtime_endpoints tests/test_litellm/proxy/ui_crud_endpoints + tests/test_litellm/proxy/utils workers: 2 reruns: 2 artifact-name: proxy-endpoints diff --git a/.github/workflows/test-unit-proxy-mgmt-behavior.yml b/.github/workflows/test-unit-proxy-mgmt-behavior.yml deleted file mode 100644 index e73997323a4..00000000000 --- a/.github/workflows/test-unit-proxy-mgmt-behavior.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: "Unit Tests: Proxy Management-Endpoint Behavior Pinning" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_branch - - "litellm_**" - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - proxy-mgmt-behavior: - uses: ./.github/workflows/_test-unit-services-base.yml - with: - test-path: tests/proxy_behavior - # workers=0 (no xdist): the world seed is a single shared Postgres - # state — two xdist workers both call seed_world() and race on the - # ``behavior-pin-budget`` row, producing UniqueViolation + cascading - # missing-membership FK failures. The whole suite is ~7s sequentially, - # so the cost of disabling parallelism here is negligible. - workers: 0 - reruns: 0 - enable-postgres: true - artifact-name: proxy-mgmt-behavior - timeout-minutes: 15 diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml deleted file mode 100644 index 4ee89897024..00000000000 --- a/.github/workflows/test-unit-security.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Unit Tests: Security" - -# Kept push-only (was previously required by DATABASE_URL secret scoping; -# now the postgres credentials are ephemeral localhost values but the -# push-trigger stays to match the proxy-db workflow cadence). -on: - push: - branches: [main, "litellm_**"] - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - security: - uses: ./.github/workflows/_test-unit-services-base.yml - with: - test-path: "tests/proxy_security_tests/" - workers: 1 - reruns: 2 - timeout-minutes: 20 - enable-postgres: true - artifact-name: security diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c114a838d6d..3d2fa3e51c8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -240,6 +240,24 @@ graph LR 7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis 8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s +### Data Access Layer (Models & Repositories) + +Database entities and the operations on them live in two packages at the root of `litellm/` so both the gateway (`proxy/`) and the SDK can use them without importing proxy internals: + +- `litellm/models/` holds the canonical Pydantic definitions for every persisted entity (`LiteLLM_VerificationToken`, `LiteLLM_TeamTable`, `LiteLLM_UserTable`, etc.). `proxy/_types.py` re-exports these for backwards compatibility, so existing imports keep working. +- `litellm/repositories/` holds the data-access layer. `BaseRepository[T]` provides the generic CRUD (`find_by_id`, `find_many`, `create`, `update`, `delete`, `count`, `exists`); entity repositories such as `VerificationTokenRepository`, `TeamRepository`, and `UserRepository` add domain-specific queries and writes on top of it. + +Conventions to follow when touching this layer: + +| Concern | How it's handled | +|---------|------------------| +| JSON columns | Prisma `Json` columns are stored as JSON strings. Repositories `json.dumps()` on write and `json.loads()` on read (see `_to_model` and the `_build_*_data` helpers). | +| Archive-then-delete | `delete_team` / `delete_token` copy the row into the `LiteLLM_Deleted*` table and delete the original inside a single `prisma_client.db.tx()` transaction. Archive payloads are built explicitly so only columns that exist on the archive table are written. | +| Column vs. field names | Where a model field differs from its DB column (for example `org_id` maps to the `organization_id` column), the repository translates in both directions rather than relying on Pydantic to guess. | +| Array mutations | Adds use Prisma's atomic `push` (`add_member`, `add_admin`, `add_models`) to avoid read-modify-write races. Removals fall back to read-modify-write because Prisma has no atomic array remove. | + +To add a new entity, define the model under `litellm/models/`, re-export it from `proxy/_types.py` if existing code imports it from there, and add a repository under `litellm/repositories/` (subclass `BaseRepository` for plain CRUD, or add bespoke methods when the entity needs encryption, archiving, or atomic array updates). Mirror the tests in `tests/test_litellm/repositories/`. + --- ## 2. SDK Request Flow diff --git a/CLAUDE.md b/CLAUDE.md index d2d8601a9c9..02a9630b486 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,7 +42,7 @@ When you must use real LLM models to, for example, write e2e tests, write a QA r If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names -Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch +Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch When working on a PR, keep the PR description in sync with new commits being made diff --git a/README.md b/README.md index 9924aeb5829..d600f3952c6 100644 --- a/README.md +++ b/README.md @@ -407,7 +407,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature ### Run in Developer Mode #### Services 1. Setup .env file in root -2. Run dependant services `docker-compose up db prometheus` +2. Run dependent services `docker-compose up db prometheus` #### Backend 1. (In root) create virtual environment `python -m venv .venv` diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index 3b59c58c8bf..b355db43540 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -12,9 +12,14 @@ spec: {{- include "litellm.backend.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.backend.podAnnotations }} + {{- if or .Values.gateway.config.create .Values.backend.podAnnotations }} annotations: + {{- if .Values.gateway.config.create }} + checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.backend.podAnnotations }} {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} labels: {{- include "litellm.backend.selectorLabels" . | nindent 8 }} @@ -35,7 +40,17 @@ spec: protocol: TCP env: {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }} + {{- if .Values.gateway.config.create }} + - name: CONFIG_FILE_PATH + value: /app/config/config.yaml + {{- end }} {{- include "litellm.envFrom" .Values.backend | nindent 10 }} + {{- if .Values.gateway.config.create }} + volumeMounts: + - name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + {{- end }} {{- with .Values.backend.livenessProbe }} livenessProbe: {{- toYaml . | nindent 12 }} @@ -46,6 +61,12 @@ spec: {{- end }} resources: {{- toYaml .Values.backend.resources | nindent 12 }} + {{- if .Values.gateway.config.create }} + volumes: + - name: gateway-config + configMap: + name: {{ include "litellm.gateway.fullname" . }}-config + {{- end }} {{- with .Values.backend.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260520120000_add_mcp_env_vars/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260520120000_add_mcp_env_vars/migration.sql new file mode 100644 index 00000000000..08d35cd74a3 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260520120000_add_mcp_env_vars/migration.sql @@ -0,0 +1,23 @@ +-- AlterTable: add admin-configured env_vars to MCP server table +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "env_vars" JSONB DEFAULT '[]'; + +-- CreateTable: per-user env var values for MCP servers +CREATE TABLE IF NOT EXISTS "LiteLLM_MCPUserEnvVars" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "server_id" TEXT NOT NULL, + "values_b64" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_MCPUserEnvVars_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_MCPUserEnvVars_user_id_server_id_key" ON "LiteLLM_MCPUserEnvVars"("user_id", "server_id"); + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "LiteLLM_MCPUserEnvVars_user_id_idx" ON "LiteLLM_MCPUserEnvVars"("user_id"); + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "LiteLLM_MCPUserEnvVars_server_id_idx" ON "LiteLLM_MCPUserEnvVars"("server_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260604120000_add_oauth2_flow_to_mcp_servers/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260604120000_add_oauth2_flow_to_mcp_servers/migration.sql new file mode 100644 index 00000000000..fee6926d963 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260604120000_add_oauth2_flow_to_mcp_servers/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "oauth2_flow" TEXT; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260605182307_add_timeout_to_mcp_server_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260605182307_add_timeout_to_mcp_server_table/migration.sql new file mode 100644 index 00000000000..845ad017cbf --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260605182307_add_timeout_to_mcp_server_table/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "timeout" DOUBLE PRECISION; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index c4754ef6117..e21c0016491 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -311,6 +311,11 @@ model LiteLLM_MCPServerTable { tool_name_to_description Json? @default("{}") extra_headers String[] @default([]) static_headers Json? @default("{}") + // Admin-configured environment variables interpolated into static_headers + // via ${NAME} syntax. Stored as an array of + // {name, value, scope, description}. scope is "global" (value used as-is) + // or "user" (value supplied per-user via LiteLLM_MCPUserEnvVars). + env_vars Json? @default("[]") // Health check status status String? @default("unknown") last_health_check DateTime? @@ -322,6 +327,7 @@ model LiteLLM_MCPServerTable { authorization_url String? token_url String? registration_url String? + oauth2_flow String? allow_all_keys Boolean @default(false) available_on_public_internet Boolean @default(true) delegate_auth_to_upstream Boolean @default(false) @@ -330,6 +336,7 @@ model LiteLLM_MCPServerTable { byok_description String[] @default([]) byok_api_key_help_url String? source_url String? + timeout Float? // BYOM submission lifecycle approval_status String? @default("active") submitted_by String? @@ -364,6 +371,21 @@ model LiteLLM_MCPUserCredentials { @@unique([user_id, server_id]) } +// Per-user environment variable values for MCP servers. +// values_b64 is an encrypted JSON object: {VAR_NAME: "value", ...}. +model LiteLLM_MCPUserEnvVars { + id String @id @default(uuid()) + user_id String + server_id String + values_b64 String + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + + @@unique([user_id, server_id]) + @@index([user_id]) + @@index([server_id]) +} + // Generate Tokens for Proxy model LiteLLM_VerificationToken { token String @id diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 0654f17ec68..e2a86205fc5 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.73" +version = "0.4.74" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.73" +version = "0.4.74" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index bae15f0362c..e6c30e12286 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -16,8 +16,17 @@ import os # Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available import dotenv as _dotenv + +def _dev_env_hot_reload_enabled() -> bool: + """The proxy exports this flag when started with ``--reload``. A reloaded + worker is a fresh process that inherits the reloader's environment, so an + edited ``.env`` value stays masked by the stale inherited one unless we + let the file win; overriding makes the edit take effect on reload.""" + return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True" + + if os.getenv("LITELLM_MODE", "DEV") == "DEV": - _dotenv.load_dotenv() + _dotenv.load_dotenv(override=_dev_env_hot_reload_enabled()) from typing import ( Callable, @@ -278,6 +287,7 @@ ovhcloud_key: Optional[str] = None lemonade_key: Optional[str] = None sap_service_key: Optional[str] = None amazon_nova_api_key: Optional[str] = None +inception_key: Optional[str] = None common_cloud_provider_auth_params: dict = { "params": ["project", "region_name", "token"], "providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"], @@ -432,6 +442,13 @@ custom_prometheus_metadata_labels: List[str] = [] custom_prometheus_tags: List[str] = [] prometheus_metrics_config: Optional[List] = None prometheus_emit_stream_label: bool = False +# Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on +# `litellm_proxy_failed_requests_metric`. Off by default to preserve the +# pre-unification label set so existing dashboards / recording rules keyed on +# that metric keep matching after upgrade. Enable when downstream consumers +# are ready to split 429s by source (vendor vs. litellm) and dimension +# (RPM/TPM/concurrent/budget). +prometheus_emit_rate_limit_labels: bool = False prometheus_user_budget_label_include_email_alias: bool = False prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000 prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0 @@ -443,6 +460,7 @@ disable_copilot_system_to_assistant: bool = ( False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. ) public_mcp_servers: Optional[List[str]] = None +public_mcp_hub_strict_whitelist: bool = True public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) @@ -551,6 +569,7 @@ cohere_models: Set = set() cohere_chat_models: Set = set() mistral_chat_models: Set = set() text_completion_codestral_models: Set = set() +text_completion_inception_models: Set = set() anthropic_models: Set = set() openrouter_models: Set = set() datarobot_models: Set = set() @@ -609,6 +628,7 @@ cerebras_models: Set = set() galadriel_models: Set = set() nvidia_nim_models: Set = set() nvidia_riva_models: Set = set() +soniox_models: Set = set() sambanova_models: Set = set() sambanova_embedding_models: Set = set() novita_models: Set = set() @@ -628,6 +648,7 @@ publicai_models: Set = set() v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() +inception_models: Set = set() hyperbolic_models: Set = set() black_forest_labs_models: Set = set() recraft_models: Set = set() @@ -792,6 +813,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): fireworks_ai_embedding_models.add(key) elif value.get("litellm_provider") == "text-completion-codestral": text_completion_codestral_models.add(key) + elif value.get("litellm_provider") == "text-completion-inception": + text_completion_inception_models.add(key) elif value.get("litellm_provider") == "xai": xai_models.add(key) elif value.get("litellm_provider") == "zai": @@ -838,6 +861,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): nvidia_nim_models.add(key) elif value.get("litellm_provider") == "nvidia_riva": nvidia_riva_models.add(key) + elif value.get("litellm_provider") == "soniox": + soniox_models.add(key) elif value.get("litellm_provider") == "sambanova": sambanova_models.add(key) elif value.get("litellm_provider") == "sambanova-embedding-models": @@ -878,6 +903,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.add(key) + elif value.get("litellm_provider") == "inception": + inception_models.add(key) elif value.get("litellm_provider") == "hyperbolic": hyperbolic_models.add(key) elif value.get("litellm_provider") == "black_forest_labs": @@ -980,6 +1007,7 @@ model_list = list( | watsonx_models | gemini_models | text_completion_codestral_models + | text_completion_inception_models | xai_models | zai_models | fal_ai_models @@ -1000,6 +1028,7 @@ model_list = list( | galadriel_models | nvidia_nim_models | nvidia_riva_models + | soniox_models | sambanova_models | azure_text_models | novita_models @@ -1018,6 +1047,7 @@ model_list = list( | v0_models | morph_models | lambda_ai_models + | inception_models | black_forest_labs_models | recraft_models | cometapi_models @@ -1074,6 +1104,7 @@ models_by_provider: dict = { "fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models, "aleph_alpha": aleph_alpha_models, "text-completion-codestral": text_completion_codestral_models, + "text-completion-inception": text_completion_inception_models, "xai": xai_models, "zai": zai_models, "fal_ai": fal_ai_models, @@ -1098,6 +1129,7 @@ models_by_provider: dict = { "galadriel": galadriel_models, "nvidia_nim": nvidia_nim_models, "nvidia_riva": nvidia_riva_models, + "soniox": soniox_models, "sambanova": sambanova_models | sambanova_embedding_models, "novita": novita_models, "nebius": nebius_models | nebius_embedding_models, @@ -1118,6 +1150,7 @@ models_by_provider: dict = { "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, + "inception": inception_models, "hyperbolic": hyperbolic_models, "black_forest_labs": black_forest_labs_models, "recraft": recraft_models, @@ -1277,6 +1310,8 @@ from .exceptions import ( NotFoundError, PermissionDeniedError, RateLimitError, + RateLimitErrorCategory, + RateLimitType, ServiceUnavailableError, BadGatewayError, OpenAIError, @@ -1728,6 +1763,9 @@ if TYPE_CHECKING: from .llms.openrouter.responses.transformation import ( OpenRouterResponsesAPIConfig as OpenRouterResponsesAPIConfig, ) + from .llms.bedrock_mantle.responses.transformation import ( + BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig, + ) from .llms.gemini.interactions.transformation import ( GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, ) @@ -1869,6 +1907,9 @@ if TYPE_CHECKING: from .llms.codestral.completion.transformation import ( CodestralTextCompletionConfig as CodestralTextCompletionConfig, ) + from .llms.inception.completion.transformation import ( + InceptionTextCompletionConfig as InceptionTextCompletionConfig, + ) from .llms.azure.azure import ( AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig, ) @@ -1937,6 +1978,9 @@ if TYPE_CHECKING: from .llms.lambda_ai.chat.transformation import ( LambdaAIChatConfig as LambdaAIChatConfig, ) + from .llms.inception.chat.transformation import ( + InceptionChatConfig as InceptionChatConfig, + ) from .llms.hyperbolic.chat.transformation import ( HyperbolicChatConfig as HyperbolicChatConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 17eb6609292..bace54ffad1 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -237,6 +237,7 @@ LLM_CONFIG_NAMES = ( "PerplexityResponsesConfig", "DatabricksResponsesAPIConfig", "OpenRouterResponsesAPIConfig", + "BedrockMantleResponsesAPIConfig", "GoogleAIStudioInteractionsConfig", "OpenAIOSeriesConfig", "AnthropicSkillsConfig", @@ -267,6 +268,7 @@ LLM_CONFIG_NAMES = ( "AIMLChatConfig", "VolcEngineChatConfig", "CodestralTextCompletionConfig", + "InceptionTextCompletionConfig", "AzureOpenAIAssistantsAPIConfig", "HerokuChatConfig", "CometAPIConfig", @@ -310,6 +312,7 @@ LLM_CONFIG_NAMES = ( "MorphChatConfig", "RAGFlowConfig", "LambdaAIChatConfig", + "InceptionChatConfig", "HyperbolicChatConfig", "VercelAIGatewayConfig", "OVHCloudChatConfig", @@ -318,6 +321,7 @@ LLM_CONFIG_NAMES = ( "LemonadeChatConfig", "SnowflakeEmbeddingConfig", "AmazonNovaChatConfig", + "SonioxAudioTranscriptionConfig", ) # Types that support lazy loading via _lazy_import_types @@ -956,6 +960,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.openrouter.responses.transformation", "OpenRouterResponsesAPIConfig", ), + "BedrockMantleResponsesAPIConfig": ( + ".llms.bedrock_mantle.responses.transformation", + "BedrockMantleResponsesAPIConfig", + ), "GoogleAIStudioInteractionsConfig": ( ".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig", @@ -1040,6 +1048,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.codestral.completion.transformation", "CodestralTextCompletionConfig", ), + "InceptionTextCompletionConfig": ( + ".llms.inception.completion.transformation", + "InceptionTextCompletionConfig", + ), "AzureOpenAIAssistantsAPIConfig": ( ".llms.azure.azure", "AzureOpenAIAssistantsAPIConfig", @@ -1154,6 +1166,10 @@ _LLM_CONFIGS_IMPORT_MAP = { "MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"), "RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"), "LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"), + "InceptionChatConfig": ( + ".llms.inception.chat.transformation", + "InceptionChatConfig", + ), "HyperbolicChatConfig": ( ".llms.hyperbolic.chat.transformation", "HyperbolicChatConfig", @@ -1180,6 +1196,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.amazon_nova.chat.transformation", "AmazonNovaChatConfig", ), + "SonioxAudioTranscriptionConfig": ( + ".llms.soniox.audio_transcription.transformation", + "SonioxAudioTranscriptionConfig", + ), } # Import map for utils module lazy imports diff --git a/litellm/_service_logger.py b/litellm/_service_logger.py index 5531c418799..b290b4340e7 100644 --- a/litellm/_service_logger.py +++ b/litellm/_service_logger.py @@ -371,6 +371,8 @@ class ServiceLogging(CustomLogger): service=ServiceTypes.LITELLM, duration=_duration, call_type=kwargs.get("call_type", "unknown"), + start_time=start_time, + end_time=end_time, ) except Exception as e: raise e diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 3ad5485dea1..6979e1ac659 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -159,7 +159,9 @@ async def _send_message_via_completion_bridge( api_base=api_base, ) - return LiteLLMSendMessageResponse.from_dict(response_dict) + return LiteLLMSendMessageResponse.from_dict( + response_dict, request_id=str(request.id) + ) async def _execute_a2a_send_with_retry( @@ -317,15 +319,6 @@ async def asend_message( ) card_url = getattr(agent_card, "url", None) if agent_card else None - context_id = trace_id or str(uuid.uuid4()) - message = request.params.message - if isinstance(message, dict): - if message.get("context_id") is None: - message["context_id"] = context_id - else: - if getattr(message, "context_id", None) is None: - message.context_id = context_id - a2a_response = await _execute_a2a_send_with_retry( a2a_client=a2a_client, request=request, @@ -338,7 +331,9 @@ async def asend_message( verbose_logger.info(f"A2A send_message completed, request_id={request.id}") # Wrap in LiteLLM response type for _hidden_params support - response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response) + response = LiteLLMSendMessageResponse.from_a2a_response( + a2a_response, request_id=str(request.id) + ) # Calculate token usage from request and response response_dict = a2a_response.model_dump(mode="json", exclude_none=True) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 2de7bda6467..87c26b776e8 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -182,6 +182,14 @@ class ResponsesToCompletionBridgeHandler: client=kwargs.get("client"), ) + # Pin the resolved provider so `responses()` doesn't re-run + # `get_llm_provider()` on the model string and strip a second + # provider prefix (see GitHub issue #28505). request_data already + # carries `custom_llm_provider` via the spread of + # `sanitized_litellm_params`; overwriting it on the dict (rather + # than adding an explicit kwarg) avoids the duplicate-keyword + # TypeError that would otherwise fire on the real bridge path. + request_data["custom_llm_provider"] = custom_llm_provider result = responses( **request_data, ) @@ -268,6 +276,13 @@ class ResponsesToCompletionBridgeHandler: except Exception as e: raise e + # Pin the resolved provider so `aresponses()` doesn't re-run + # `get_llm_provider()` on the model string and strip a second + # provider prefix (see GitHub issue #28505). Set on request_data + # rather than passed as a separate kwarg to avoid the duplicate- + # keyword TypeError when `sanitized_litellm_params` already + # carries `custom_llm_provider`. + request_data["custom_llm_provider"] = custom_llm_provider result = await aresponses( **request_data, aresponses=True, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 51abbbf729b..6d8b5cf8a57 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -402,6 +402,20 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): instructions, ) = self.convert_chat_completion_messages_to_responses_api(messages) + # OpenAI's Responses API rejects an empty input. For a system-only + # request, carry the system message as a system-role input item instead + # of instructions, mirroring how non-string system content is already + # handled in convert_chat_completion_messages_to_responses_api. + if not input_items and instructions is not None: + input_items = [ + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": instructions}], + } + ] + instructions = None + optional_params = self._extract_extra_body_params(optional_params) # Build responses API request using the reverse transformation logic diff --git a/litellm/constants.py b/litellm/constants.py index df15050e652..36e578bd323 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -585,6 +585,7 @@ LITELLM_CHAT_PROVIDERS = [ "volcengine", "codestral", "text-completion-codestral", + "text-completion-inception", "deepseek", "sambanova", "maritalk", @@ -620,6 +621,7 @@ LITELLM_CHAT_PROVIDERS = [ "oci", "morph", "lambda_ai", + "inception", "vercel_ai_gateway", "wandb", "ovhcloud", @@ -676,6 +678,7 @@ OPENAI_CHAT_COMPLETION_PARAMS = [ "extra_headers", "thinking", "web_search_options", + "include_server_side_tool_invocations", "service_tier", "prompt_cache_key", "prompt_cache_retention", @@ -737,6 +740,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = { "verbosity": None, "thinking": None, "web_search_options": None, + "include_server_side_tool_invocations": None, "service_tier": None, "safety_identifier": None, "prompt_cache_key": None, @@ -779,6 +783,7 @@ openai_compatible_endpoints: List = [ "https://api.v0.dev/v1", "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", + "https://api.inceptionlabs.ai/v1", "https://api.hyperbolic.xyz/v1", "https://ai-gateway.helicone.ai/", "https://ai-gateway.vercel.sh/v1", @@ -835,6 +840,7 @@ openai_compatible_providers: List = [ "helicone", "morph", "lambda_ai", + "inception", "hyperbolic", "vercel_ai_gateway", "aiml", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9a4b158b622..88029615ba8 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2425,12 +2425,11 @@ class BaseTokenUsageProcessor: if not attr.startswith("_") and not callable( getattr(usage.completion_tokens_details, attr) ): - current_val = getattr( - combined.completion_tokens_details, attr, 0 + current_val = ( + getattr(combined.completion_tokens_details, attr, 0) or 0 ) - new_val = getattr(usage.completion_tokens_details, attr, 0) - - if new_val is not None and current_val is not None: + new_val = getattr(usage.completion_tokens_details, attr, 0) or 0 + if isinstance(new_val, (int, float)): setattr( combined.completion_tokens_details, attr, diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 17f5b43c273..1cbef6b0b49 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -9,13 +9,109 @@ ## LiteLLM versions of the OpenAI Exception Types -from typing import Any, Dict, Optional +import enum +from typing import Any, Dict, Optional, Union import httpx import openai from litellm.types.utils import LiteLLMCommonStrings + +class RateLimitErrorCategory(str, enum.Enum): + """ + Category of a rate limit error, allowing callers to distinguish where the rate + limit originated. Exposed on every :class:`RateLimitError` instance via the + ``category`` attribute. + + Use these values to switch on the rate limit source, e.g.:: + + try: + ... + except litellm.RateLimitError as e: + if e.category == RateLimitErrorCategory.LITELLM_RATE_LIMIT: + ... # litellm's own limiter (key/team/user/model RPM/TPM/budget) + elif e.category == RateLimitErrorCategory.VENDOR_RATE_LIMIT: + ... # the upstream LLM provider returned 429 + """ + + VENDOR_RATE_LIMIT = "vendor_rate_limit" + """The upstream LLM provider returned a rate-limit response (e.g. OpenAI 429).""" + + VENDOR_BATCH_RATE_LIMIT = "vendor_batch_rate_limit" + """The upstream LLM provider returned a rate-limit response on a batch endpoint.""" + + LITELLM_RATE_LIMIT = "litellm_rate_limit" + """LiteLLM's own rate limiter (key/team/user/model RPM/TPM, budget, parallel-requests, etc.) blocked the request.""" + + LITELLM_BATCH_RATE_LIMIT = "litellm_batch_rate_limit" + """LiteLLM's own batch rate limiter (token/request budget across a batch input file) blocked the request.""" + + +class RateLimitType(str, enum.Enum): + """ + The dimension that was exceeded when a rate-limit error fired. + + This is orthogonal to :class:`RateLimitErrorCategory` — *category* tells + callers **who** rate-limited the request (the upstream vendor vs. one of + litellm's own limiters), while *type* tells them **which limit dimension** + was exceeded (an RPM ceiling, a TPM ceiling, a max-parallel-requests + ceiling, a budget cap, or a max-iterations cap). + + Surfaced both on every :class:`RateLimitError` instance via the + ``rate_limit_type`` attribute and on the structured + ``StandardLoggingPayload.error_information.error_rate_limit_type`` field + so custom callbacks / metrics consumers can split rate-limit failures by + cause without parsing free-text error messages. + """ + + REQUESTS = "requests" + """Requests-per-minute (RPM) or requests-per-window ceiling exceeded.""" + + TOKENS = "tokens" + """Tokens-per-minute (TPM) or tokens-per-window ceiling exceeded.""" + + CONCURRENT_REQUESTS = "concurrent_requests" + """``max_parallel_requests`` — too many in-flight requests at once.""" + + BUDGET = "budget" + """Spend budget cap reached (key, team, user, or per-session).""" + + MAX_ITERATIONS = "max_iterations" + """Per-session max-iterations cap reached (agent-style flows).""" + + +_RATE_LIMIT_CATEGORY_VALUES = frozenset(c.value for c in RateLimitErrorCategory) +_RATE_LIMIT_TYPE_VALUES = frozenset(t.value for t in RateLimitType) + + +def validate_rate_limit_category(value: Any) -> Optional[str]: + """Return ``value`` only if it matches a known :class:`RateLimitErrorCategory`. + + Used at duck-typed read sites (StandardLoggingPayload extraction, Prometheus + labels) to reject `.category` strings set by unrelated third-party exceptions + — otherwise those would leak into custom-callback payloads and Prometheus + label cardinality. + """ + if isinstance(value, RateLimitErrorCategory): + return value.value + if isinstance(value, str) and value in _RATE_LIMIT_CATEGORY_VALUES: + return value + return None + + +def validate_rate_limit_type(value: Any) -> Optional[str]: + """Return ``value`` only if it matches a known :class:`RateLimitType`. + + See :func:`validate_rate_limit_category` for the rationale. + """ + if isinstance(value, RateLimitType): + return value.value + if isinstance(value, str) and value in _RATE_LIMIT_TYPE_VALUES: + return value + return None + + _MINIMAL_ERROR_RESPONSE: Optional[httpx.Response] = None @@ -321,6 +417,18 @@ class PermissionDeniedError(openai.PermissionDeniedError): # type: ignore class RateLimitError(openai.RateLimitError): # type: ignore + """ + Unified rate-limit error. + + Every rate-limit condition surfaced by litellm — whether it originated from + an upstream LLM provider, a vendor batch endpoint, or one of litellm's own + proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget, + max-iterations, etc.) — is raised as an instance of this class. + + The :attr:`category` attribute lets callers distinguish the source. See + :class:`RateLimitErrorCategory` for the available values. + """ + def __init__( self, message, @@ -330,6 +438,12 @@ class RateLimitError(openai.RateLimitError): # type: ignore litellm_debug_info: Optional[str] = None, max_retries: Optional[int] = None, num_retries: Optional[int] = None, + category: Union[str, RateLimitErrorCategory] = ( + RateLimitErrorCategory.VENDOR_RATE_LIMIT + ), + rate_limit_type: Optional[Union[str, RateLimitType]] = None, + headers: Optional[Dict[str, str]] = None, + detail: Any = None, ): self.status_code = 429 self.message = "litellm.RateLimitError: {}".format(message) @@ -338,9 +452,39 @@ class RateLimitError(openai.RateLimitError): # type: ignore self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries + self.category = ( + category.value if isinstance(category, RateLimitErrorCategory) else category + ) + # Which dimension was exceeded — request count, token count, parallel + # requests, budget, max iterations. None when the source didn't + # classify the failure (e.g. legacy vendor 429 with no header hints). + self.rate_limit_type: Optional[str] = ( + rate_limit_type.value + if isinstance(rate_limit_type, RateLimitType) + else rate_limit_type + ) + # Headers explicitly attached to the error (e.g. retry-after, + # rate_limit_type, reset_at). Preserved across the proxy boundary so + # clients can react appropriately. + # + # IMPORTANT: we deliberately do NOT auto-populate self.headers from + # response.headers when only `response` is provided. A vendor 429 can + # set arbitrary response headers (Set-Cookie, CORS overrides, …); if + # those leaked into e.headers and a downstream proxy serializer + # forwarded them to the client, a malicious upstream could inject + # browser-interpreted headers for the proxy origin. Vendor response + # headers stay reachable on `e.response.headers` for callers that + # explicitly want them; only the proxy-supplied `headers=` kwarg + # makes it onto `self.headers`. _response_headers = ( getattr(response, "headers", None) if response is not None else None ) + self.headers: Optional[Dict[str, str]] = ( + {k: str(v) for k, v in headers.items()} if headers else None + ) + # Mirrors FastAPI HTTPException.detail so the same instance can be + # serialized through both the ProxyException and HTTPException paths. + self.detail = detail if detail is not None else self.message self.response = httpx.Response( status_code=429, headers=_response_headers, @@ -843,11 +987,24 @@ LITELLM_EXCEPTION_TYPES = [ class BudgetExceededError(Exception): def __init__( - self, current_cost: float, max_budget: float, message: Optional[str] = None + self, + current_cost: float, + max_budget: float, + message: Optional[str] = None, + llm_provider: Optional[str] = None, ): self.current_cost = current_cost self.max_budget = max_budget self.status_code = 429 + self.llm_provider = llm_provider or "" + # Surface unified rate-limit fields without joining the RateLimitError + # hierarchy so existing `except BudgetExceededError:` handlers keep + # working; custom callbacks reading StandardLoggingPayload pick these + # up via the same `category` / `rate_limit_type` attributes the rest + # of the unified rate-limit error path uses. Stored as plain strings + # to match the normalization RateLimitError.__init__ performs. + self.category: str = RateLimitErrorCategory.LITELLM_RATE_LIMIT.value + self.rate_limit_type: str = RateLimitType.BUDGET.value message = ( message or f"Budget has been exceeded! Current cost: {current_cost}, Max budget: {max_budget}" @@ -1062,3 +1219,37 @@ class GuardrailInterventionNormalStringError( def __repr__(self): return self.__str__() + + +class SensitiveDataRouteException(Exception): + """ + Exception raised when a guardrail detects sensitive data and wants to reroute the request. + + Instead of blocking the request, this exception signals that the request should be + routed to a different model (typically an on-premise model for data privacy). + + The proxy catches this exception and: + 1. Reroutes the current request to the specified model + 2. When sticky_session_routing is True, stores the routing decision in session + cache so all subsequent requests in the same session are routed to the same model + """ + + def __init__( + self, + route_to_model: str, + session_id: str, + guardrail_name: Optional[str] = None, + detection_info: Optional[Dict[str, Any]] = None, + message: Optional[str] = None, + sticky_session_routing: bool = True, + ): + self.route_to_model = route_to_model + self.session_id = session_id + self.guardrail_name = guardrail_name + self.detection_info = detection_info or {} + self.sticky_session_routing = sticky_session_routing + self.message = ( + message + or f"Sensitive data detected by {guardrail_name}. Routing to model: {route_to_model}" + ) + super().__init__(self.message) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 7559fe142c4..c6d427e7f09 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -4,6 +4,7 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers. import asyncio import base64 +import os from typing import ( Any, Awaitable, @@ -16,7 +17,6 @@ from typing import ( TypeVar, Union, ) - import httpx from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters from mcp.client.sse import sse_client @@ -42,9 +42,8 @@ from mcp.types import ( ) from mcp.types import Tool as MCPTool from pydantic import AnyUrl - from litellm._logging import verbose_logger -from litellm.constants import MCP_CLIENT_TIMEOUT +from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_NPM_CACHE_DIR from litellm.llms.custom_httpx.http_handler import get_ssl_configuration from litellm.types.llms.custom_http import VerifyTypes from litellm.types.mcp import ( @@ -61,13 +60,33 @@ def to_basic_auth(auth_value: str) -> str: return base64.b64encode(auth_value.encode("utf-8")).decode() +def _strip_header_whitespace(headers: Dict[str, str]) -> Dict[str, str]: + return { + (key.strip() if isinstance(key, str) else key): ( + value.strip() if isinstance(value, str) else value + ) + for key, value in headers.items() + } + + +def _first_non_cancelled_cause(exc: BaseException) -> Optional[BaseException]: + queue: List[BaseException] = [exc] + while queue: + current = queue.pop(0) + nested = getattr(current, "exceptions", None) + if nested: + queue.extend(nested) + elif not isinstance(current, asyncio.CancelledError): + return current + return None + + TSessionResult = TypeVar("TSessionResult") class MCPSigV4Auth(httpx.Auth): """ httpx Auth class that signs each request with AWS SigV4. - This is used for MCP servers that require AWS SigV4 authentication, such as AWS Bedrock AgentCore MCP servers. httpx calls auth_flow() for every outgoing request, enabling per-request signature computation. @@ -92,10 +111,8 @@ class MCPSigV4Auth(httpx.Auth): "Missing botocore to use AWS SigV4 authentication. " "Run 'pip install boto3'." ) - self.service_name = aws_service_name or "bedrock-agentcore" self.region_name = aws_region_name or "us-east-1" - # Note: os.environ/ prefixed values are already resolved by # ProxyConfig._check_for_os_environ_vars() at config load time. # Values arrive here as plain strings. @@ -143,20 +160,17 @@ class MCPSigV4Auth(httpx.Auth): session_name = ( aws_session_name or f"litellm-mcp-{int(__import__('time').time())}" ) - sts_kwargs: dict = {"region_name": aws_region_name} if aws_access_key_id and aws_secret_access_key: sts_kwargs["aws_access_key_id"] = aws_access_key_id sts_kwargs["aws_secret_access_key"] = aws_secret_access_key if aws_session_token: sts_kwargs["aws_session_token"] = aws_session_token - sts_client = boto3.client("sts", **sts_kwargs) sts_response = sts_client.assume_role( RoleArn=aws_role_name, RoleSessionName=session_name, ) - sts_creds = sts_response["Credentials"] return Credentials( access_key=sts_creds["AccessKeyId"], @@ -178,17 +192,14 @@ class MCPSigV4Auth(httpx.Auth): data=request.content, headers=dict(request.headers), ) - # Sign the request — SigV4Auth.add_auth() adds Authorization, # X-Amz-Date, and X-Amz-Security-Token (if session token present). # Host header is derived automatically from the URL. sigv4 = SigV4Auth(self.credentials, self.service_name, self.region_name) sigv4.add_auth(aws_request) - # Copy SigV4 headers back to the httpx request for header_name, header_value in aws_request.headers.items(): request.headers[header_name] = header_value - yield request @@ -198,6 +209,8 @@ class MCPClient: SSE and HTTP transports Authentication via Bearer token, Basic Auth, or API Key Tool calling with error handling and result parsing + Sampling callbacks for upstream server LLM requests + Elicitation callbacks for upstream server user-input requests """ def __init__( @@ -211,6 +224,9 @@ class MCPClient: extra_headers: Optional[Dict[str, str]] = None, ssl_verify: Optional[VerifyTypes] = None, aws_auth: Optional[httpx.Auth] = None, + sampling_callback: Optional[Callable] = None, + elicitation_callback: Optional[Callable] = None, + logging_callback: Optional[Callable] = None, ): self.server_url: str = server_url self.transport_type: MCPTransport = transport_type @@ -222,6 +238,9 @@ class MCPClient: self.ssl_verify: Optional[VerifyTypes] = ssl_verify self._aws_auth: Optional[httpx.Auth] = aws_auth self._last_initialize_instructions: Optional[str] = None + self._sampling_callback: Optional[Callable] = sampling_callback + self._elicitation_callback: Optional[Callable] = elicitation_callback + self._logging_callback: Optional[Callable] = logging_callback # handle the basic auth value if provided if auth_value: self.update_auth_value(auth_value) @@ -231,23 +250,20 @@ class MCPClient: ) -> Tuple[Any, Optional[httpx.AsyncClient]]: """ Create the appropriate transport context based on transport type. - Returns: Tuple of (transport_context, http_client). http_client is only set for HTTP transport and needs cleanup. """ http_client: Optional[httpx.AsyncClient] = None - if self.transport_type == MCPTransport.stdio: if not self.stdio_config: raise ValueError("stdio_config is required for stdio transport") server_params = StdioServerParameters( command=self.stdio_config.get("command", ""), args=self.stdio_config.get("args", []), - env=self.stdio_config.get("env", {}), + env=self._get_safe_stdio_env(self.stdio_config.get("env")), ) return stdio_client(server_params), None - if self.transport_type == MCPTransport.sse: headers = self._get_auth_headers() httpx_client_factory = self._create_httpx_client_factory() @@ -260,14 +276,12 @@ class MCPClient: ), None, ) - # HTTP transport (default) if streamable_http_client is None: raise ImportError( "streamable_http_client is not available. " "Please install mcp with HTTP support." ) - headers = self._get_auth_headers() httpx_client_factory = self._create_httpx_client_factory() verbose_logger.debug("litellm headers for streamable_http_client: %s", headers) @@ -281,6 +295,54 @@ class MCPClient: ) return transport_ctx, http_client + def _get_safe_stdio_env( + self, provided_env: Optional[Dict[str, str]] + ) -> Optional[Dict[str, str]]: + """ + Return a safe environment for the stdio subprocess. + + If provided_env is set, we use it as-is. + If provided_env is None, we return a minimal allowlist from the parent environment + to avoid leaking sensitive LiteLLM keys (OPENAI_API_KEY, etc.) to sub-processes. + """ + if provided_env is not None: + return provided_env + + # Minimal allowlist of safe/standard environment variables + safe_keys = { + "PATH", + "HOME", + "USER", + "LOGNAME", + "TMPDIR", + "TMP", + "TEMP", + "SHELL", + "LANG", + "LC_ALL", + # Node/Package manager caches + "NPM_CONFIG_CACHE", + "PNPM_HOME", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + # System info + "SYSTEMROOT", + "COMSPEC", + "PATHEXT", + "WINDIR", + } + + safe_env = {} + for key in safe_keys: + if key in os.environ: + safe_env[key] = os.environ[key] + + if "NPM_CONFIG_CACHE" not in safe_env: + safe_env["NPM_CONFIG_CACHE"] = MCP_NPM_CACHE_DIR + + return safe_env + async def _execute_session_operation( self, transport_ctx: Any, @@ -288,13 +350,24 @@ class MCPClient: ) -> TSessionResult: """ Execute an operation within a transport and session context. - Handles entering/exiting contexts and running the operation. + Passes sampling/elicitation/logging callbacks to the ClientSession + so that upstream MCP servers can request LLM inference (sampling), + user input (elicitation), or send log messages. """ transport = await transport_ctx.__aenter__() + in_flight_error: Optional[BaseException] = None try: read_stream, write_stream = transport[0], transport[1] - session_ctx = ClientSession(read_stream, write_stream) + # Build session kwargs with optional callbacks + session_kwargs: Dict[str, Any] = {} + if self._sampling_callback is not None: + session_kwargs["sampling_callback"] = self._sampling_callback + if self._elicitation_callback is not None: + session_kwargs["elicitation_callback"] = self._elicitation_callback + if self._logging_callback is not None: + session_kwargs["logging_callback"] = self._logging_callback + session_ctx = ClientSession(read_stream, write_stream, **session_kwargs) session = await session_ctx.__aenter__() try: init_result = await session.initialize() @@ -309,11 +382,21 @@ class MCPClient: await session_ctx.__aexit__(None, None, None) except BaseException as e: verbose_logger.debug(f"Error during session context exit: {e}") + except BaseException as e: + in_flight_error = e + raise finally: try: await transport_ctx.__aexit__(None, None, None) - except BaseException as e: - verbose_logger.debug(f"Error during transport context exit: {e}") + except BaseException as exit_error: + verbose_logger.debug( + f"Error during transport context exit: {exit_error}" + ) + root_cause = _first_non_cancelled_cause(exit_error) + if root_cause is not None and isinstance( + in_flight_error, asyncio.CancelledError + ): + raise root_cause from in_flight_error async def run_with_session( self, operation: Callable[[ClientSession], Awaitable[TSessionResult]] @@ -351,7 +434,6 @@ class MCPClient: def _get_auth_headers(self) -> dict: """Generate authentication headers based on auth type.""" headers = {} - if self._mcp_auth_value: if isinstance(self._mcp_auth_value, str): if self.auth_type == MCPAuth.bearer_token: @@ -373,17 +455,14 @@ class MCPClient: # Note: aws_sigv4 auth is not handled here — SigV4 requires per-request # signing (including the body hash), so it uses httpx.Auth flow instead # of static headers. See MCPSigV4Auth and _create_httpx_client_factory(). - # update the headers with the extra headers if self.extra_headers: headers.update(self.extra_headers) - - return headers + return _strip_header_whitespace(headers) def _create_httpx_client_factory(self) -> Callable[..., httpx.AsyncClient]: """ Create a custom httpx client factory that uses LiteLLM's SSL configuration. - This factory follows the same CA bundle path logic as http_handler.py: 1. Check ssl_verify parameter (can be SSLContext, bool, or path to CA bundle) 2. Check SSL_VERIFY environment variable @@ -400,17 +479,14 @@ class MCPClient: """Create an httpx.AsyncClient with LiteLLM's SSL configuration.""" # Get unified SSL configuration using the same logic as http_handler.py ssl_config = get_ssl_configuration(self.ssl_verify) - verbose_logger.debug( f"MCP client using SSL configuration: {type(ssl_config).__name__}" ) - # Use SigV4 auth if configured and no explicit auth provided. # The MCP SDK's sse_client and streamable_http_client call this # factory without passing auth=, so self._aws_auth is used. # For non-SigV4 clients, self._aws_auth is None — no behavior change. effective_auth = auth if auth is not None else self._aws_auth - return httpx.AsyncClient( headers=headers, timeout=timeout, @@ -458,7 +534,6 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( @@ -491,7 +566,6 @@ class MCPClient: f"MCP Tool '{call_tool_request_params.name}' progress: " f"{progress}/{total} ({percentage:.0f}%) - {message or ''}" ) - # Forward to Host if callback provided if host_progress_callback: try: @@ -514,14 +588,15 @@ class MCPClient: ) return tool_result except asyncio.CancelledError: - verbose_logger.warning("MCP client tool call was cancelled") + verbose_logger.warning( + f"MCP client tool call timed out after {self.timeout}s for {self.server_url}" + ) raise except Exception as e: import traceback error_trace = traceback.format_exc() verbose_logger.debug(f"MCP client tool call traceback:\n{error_trace}") - # Log detailed error information error_type = type(e).__name__ verbose_logger.error( @@ -532,14 +607,12 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream - " "the MCP server may have crashed, disconnected, or timed out." ) - # Return a default error result instead of raising return MCPCallToolResult( content=[ @@ -577,14 +650,12 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream during list_tools - " "the MCP server may have crashed, disconnected, or timed out" ) - # Return empty list instead of raising to allow graceful degradation return [] @@ -617,7 +688,6 @@ class MCPClient: error_trace = traceback.format_exc() verbose_logger.debug(f"MCP client get_prompt traceback:\n{error_trace}") - # Log detailed error information error_type = type(e).__name__ verbose_logger.error( @@ -628,14 +698,12 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream during get_prompt - " "the MCP server may have crashed, disconnected, or timed out." ) - raise async def list_resources(self) -> list[Resource]: @@ -667,14 +735,12 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream during list_resources - " "the MCP server may have crashed, disconnected, or timed out" ) - # Return empty list instead of raising to allow graceful degradation return [] @@ -709,14 +775,12 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream during list_resource_templates - " "the MCP server may have crashed, disconnected, or timed out" ) - # Return empty list instead of raising to allow graceful degradation return [] @@ -742,7 +806,6 @@ class MCPClient: error_trace = traceback.format_exc() verbose_logger.debug(f"MCP client read_resource traceback:\n{error_trace}") - # Log detailed error information error_type = type(e).__name__ verbose_logger.error( @@ -753,12 +816,10 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream during read_resource - " "the MCP server may have crashed, disconnected, or timed out." ) - raise diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 0ec17bbea5d..390af2cb6e6 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -37,6 +37,8 @@ from litellm.proxy._types import ( VirtualKeyEvent, WebhookEvent, ) +from litellm.repositories.team_repository import TeamRepository +from litellm.repositories.user_repository import UserRepository from litellm.types.integrations.slack_alerting import * from ..email_templates.templates import * @@ -1231,7 +1233,7 @@ Model Info: and recipient_user_id is not None and prisma_client is not None ): - user_row = await prisma_client.db.litellm_usertable.find_unique( + user_row = await UserRepository(prisma_client).table.find_unique( where={"user_id": recipient_user_id} ) @@ -1263,7 +1265,7 @@ Model Info: team_id = webhook_event.team_id team_name = "Default Team" if team_id is not None and prisma_client is not None: - team_row = await prisma_client.db.litellm_teamtable.find_unique( + team_row = await TeamRepository(prisma_client).table.find_unique( where={"team_id": team_id} ) if team_row is not None: diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index a1bf65141c9..75710e10498 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -8,18 +8,23 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor BaseLLMObsOTELAttributes, safe_set_attribute, ) +from litellm.litellm_core_utils.redact_messages import ( + should_redact_message_logging, +) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.types.utils import StandardLoggingPayload if TYPE_CHECKING: from opentelemetry.trace import Span from litellm.integrations._types.open_inference import ( - MessageAttributes, - ImageAttributes, - SpanAttributes, AudioAttributes, EmbeddingAttributes, + ImageAttributes, + MessageAttributes, + MessageContentAttributes, OpenInferenceSpanKindValues, + SpanAttributes, + ToolCallAttributes, ) @@ -53,40 +58,24 @@ class ArizeOTELAttributes(BaseLLMObsOTELAttributes): msg.get("content", ""), ) - @staticmethod - @override - def set_response_output_messages(span: "Span", response_obj): - """ - Sets output message attributes on the span from the LLM response. - Args: - span: The OpenTelemetry span to set attributes on - response_obj: The response object containing choices with messages - """ - from litellm.integrations._types.open_inference import ( - MessageAttributes, - SpanAttributes, - ) + # Additive: emit structured tool_calls / multimodal content + # so Arize/Phoenix can render tool-using and image-bearing + # turns. These set NEW attribute keys (MESSAGE_TOOL_CALLS / + # MESSAGE_NAME / MESSAGE_TOOL_CALL_ID / MESSAGE_CONTENTS.*) — + # never replace the MESSAGE_CONTENT write above. + _safe_emit( + f"input message extras (idx={idx})", + _emit_input_message_extras, + span, + prefix, + msg, + ) - for idx, choice in enumerate(response_obj.get("choices", [])): - response_message = choice.get("message", {}) - safe_set_attribute( - span, - SpanAttributes.OUTPUT_VALUE, - response_message.get("content", ""), - ) - - # This shows up under `output_messages` tab on the span page. - prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.{idx}" - safe_set_attribute( - span, - f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", - response_message.get("role"), - ) - safe_set_attribute( - span, - f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", - response_message.get("content", ""), - ) + # Note: `BaseLLMObsOTELAttributes.set_response_output_messages` is not + # overridden here. The live code path uses `_set_choice_outputs` (called + # via `_set_response_attributes` from `set_attributes`) which handles + # tool_calls, multimodal output, embeddings, audio, images, and structured + # outputs in a single place. def _set_response_attributes(span: "Span", response_obj): @@ -106,11 +95,17 @@ def _set_response_attributes(span: "Span", response_obj): def _set_choice_outputs(span: "Span", response_obj, msg_attrs, span_attrs): for idx, choice in enumerate(response_obj.get("choices", [])): response_message = choice.get("message", {}) - safe_set_attribute( - span, - span_attrs.OUTPUT_VALUE, - response_message.get("content", ""), - ) + content = response_message.get("content", "") + + # Tool-only assistant responses have empty content; serialize the + # tool_calls into OUTPUT_VALUE so Arize's "Output" pane isn't blank. + output_value = content + if not output_value: + tool_calls = _get_tool_calls(response_message) + if tool_calls: + output_value = _summarize_tool_calls_for_output(tool_calls) + + safe_set_attribute(span, span_attrs.OUTPUT_VALUE, output_value) prefix = f"{span_attrs.LLM_OUTPUT_MESSAGES}.{idx}" safe_set_attribute( span, @@ -120,7 +115,18 @@ def _set_choice_outputs(span: "Span", response_obj, msg_attrs, span_attrs): safe_set_attribute( span, f"{prefix}.{msg_attrs.MESSAGE_CONTENT}", - response_message.get("content", ""), + content, + ) + + # Additive: emit assistant tool_calls so tool-using turns render in + # Arize/Phoenix. Sets new MESSAGE_TOOL_CALLS keys only — does not + # change MESSAGE_CONTENT/MESSAGE_ROLE writes above. + _safe_emit( + f"output tool_calls (idx={idx})", + _emit_message_tool_calls, + span, + prefix, + response_message, ) @@ -278,6 +284,43 @@ def _set_usage_outputs(span: "Span", response_obj, span_attrs): reasoning_tokens, ) + # Additive: cache token breakdown so prompt-caching savings render in + # Arize. Sources covered: + # - OpenAI Chat Completions: `prompt_tokens_details.cached_tokens` + # - Anthropic / Bedrock-Anthropic: `cache_read_input_tokens`, + # `cache_creation_input_tokens` + # All emits are conditional, so when none of these fields exist (the + # situation in the existing test fixtures) no extra attributes are set. + prompt_token_details = _safe_get(usage, "prompt_tokens_details") or _safe_get( + usage, "input_tokens_details" + ) + cache_read = _safe_get(prompt_token_details, "cached_tokens") or _safe_get( + usage, "cache_read_input_tokens" + ) + if cache_read: + safe_set_attribute( + span, + span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_READ, + cache_read, + ) + # Anthropic / Bedrock-Anthropic only — OpenAI's `prompt_tokens_details` + # does not expose a cache-write count, so we read straight off `usage`. + cache_write = _safe_get(usage, "cache_creation_input_tokens") + if cache_write: + safe_set_attribute( + span, + span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_WRITE, + cache_write, + ) + + audio_prompt_tokens = _safe_get(prompt_token_details, "audio_tokens") + if audio_prompt_tokens: + safe_set_attribute( + span, + span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_AUDIO, + audio_prompt_tokens, + ) + def _infer_open_inference_span_kind(call_type: Optional[str]) -> str: """ @@ -321,6 +364,10 @@ def _infer_open_inference_span_kind(call_type: Optional[str]) -> str: "videos", "realtime", "pass_through", + # `passthrough` (no underscore) is what real call_types use: + # `allm_passthrough_route`, `llm_passthrough_route`. Without + # this they fell through to UNKNOWN, blanking span.kind. + "passthrough", "anthropic_messages", "ocr", ) @@ -396,6 +443,18 @@ def set_attributes( """ Populates span with OpenInference-compliant LLM attributes for Arize and Phoenix tracing. """ + # Coerce non-dict response objects (e.g. httpx.Response from passthrough + # routes) into a dict so downstream `.get()` calls don't crash. Existing + # dict / `.get()`-bearing objects (incl. Pydantic OpenAI Responses API + # models) are returned unchanged, preserving the existing test behavior. + response_obj_for_attrs = _coerce_response_obj_for_attrs(response_obj) + + # Set span.kind defensively before anything else. If a downstream step + # throws, the span still has a kind so Arize can render it correctly + # (an LLM call instead of UNKNOWN). This is the single source of truth + # for span.kind — no late re-write happens below. + _safe_emit("early span kind", _set_early_span_kind, span, kwargs) + try: optional_params = _sanitize_optional_params(kwargs.get("optional_params")) litellm_params = kwargs.get("litellm_params", {}) or {} @@ -415,25 +474,22 @@ def set_attributes( metadata_tools = _extract_metadata_tools(metadata) optional_tools = _extract_optional_tools(optional_params) - call_type = standard_logging_payload.get("call_type") _set_request_attributes( span=span, kwargs=kwargs, standard_logging_payload=standard_logging_payload, optional_params=optional_params, litellm_params=litellm_params, - response_obj=response_obj, + response_obj=response_obj_for_attrs, span_attrs=SpanAttributes, ) - span_kind = _infer_open_inference_span_kind(call_type=call_type) + # span.kind was already set above by `_set_early_span_kind`. We do + # NOT re-write it here based on tool presence: a chat completion + # that passes `tools=[...]` (or returns `tool_calls`) is still an + # LLM call per the OpenInference spec — TOOL is reserved for actual + # tool execution spans, not LLM calls that request tools. _set_tool_attributes(span, optional_tools, metadata_tools) - if ( - optional_tools or metadata_tools - ) and span_kind != OpenInferenceSpanKindValues.TOOL.value: - span_kind = OpenInferenceSpanKindValues.TOOL.value - - safe_set_attribute(span, SpanAttributes.OPENINFERENCE_SPAN_KIND, span_kind) attributes.set_messages(span, kwargs) model_params = ( @@ -443,7 +499,7 @@ def set_attributes( ) _set_model_params(span, model_params, SpanAttributes) - _set_response_attributes(span=span, response_obj=response_obj) + _set_response_attributes(span=span, response_obj=response_obj_for_attrs) except Exception as e: verbose_logger.error( @@ -452,6 +508,22 @@ def set_attributes( if hasattr(span, "record_exception"): span.record_exception(e) + # Additive emitters. Each is independently guarded so a failure can never + # blank the attributes set by the main try-block above. New attributes are + # written under new keys; existing attributes are not overwritten. + slp = kwargs.get("standard_logging_object") + _safe_emit("session/user attrs", _set_session_and_user_attrs, span, kwargs, slp) + _safe_emit("response cost", _set_response_cost_attr, span, slp) + _safe_emit( + "passthrough normalization", + _maybe_normalize_passthrough, + span, + kwargs, + response_obj, + response_obj_for_attrs, + slp, + ) + def _sanitize_optional_params(optional_params: Optional[dict]) -> dict: if not isinstance(optional_params, dict): @@ -534,3 +606,529 @@ def _set_model_params(span: "Span", model_params: Optional[dict], span_attrs) -> user_id = model_params.get("user") if user_id is not None: safe_set_attribute(span, span_attrs.USER_ID, user_id) + + +# --------------------------------------------------------------------------- +# Additive rendering helpers (introduced to enhance Arize/Phoenix rendering +# without changing any previously-emitted attribute keys or values). +# --------------------------------------------------------------------------- + + +def _safe_emit(label: str, fn, *args, **kwargs) -> None: + """Run an additive attribute emitter, swallowing any error so it cannot + blank attributes set elsewhere on the span. Failures are logged at debug. + """ + try: + fn(*args, **kwargs) + except Exception as e: + verbose_logger.debug("[Arize] %s skipped: %s", label, e) + + +def _set_early_span_kind(span: "Span", kwargs: dict) -> None: + """Defensively set OPENINFERENCE_SPAN_KIND before any other logic runs.""" + slp = kwargs.get("standard_logging_object") + call_type = slp.get("call_type") if isinstance(slp, dict) else None + safe_set_attribute( + span, + SpanAttributes.OPENINFERENCE_SPAN_KIND, + _infer_open_inference_span_kind(call_type=call_type), + ) + + +def _coerce_response_obj_for_attrs(response_obj): + """Return a `.get`-compatible view of `response_obj` when possible. + + - dicts and Pydantic models that already expose `.get` are returned + unchanged (preserves all current behavior, including the Responses API + flow which relies on Pydantic attribute access). + - `httpx.Response` and other text-only responses (passthrough routes) + are JSON-decoded so the standard extraction paths can read fields like + `id`, `model`, and `usage`. On failure the original object is returned + so behavior is no worse than today. + """ + if response_obj is None or hasattr(response_obj, "get"): + return response_obj + text = getattr(response_obj, "text", None) + if isinstance(text, str) and text: + try: + parsed = json.loads(text) + if isinstance(parsed, dict): + return parsed + except Exception: + pass + return response_obj + + +def _coerce_text(value) -> Optional[str]: + """Best-effort text extraction from a message-content value. + + Returns None when no textual portion can be derived. Handles: + - plain strings + - lists of OpenAI-style content parts (`{"type": "text", "text": ...}`) + - lists of Anthropic-style content parts (`{"type": "text", "text": ...}` + or `{"type": "input_text", "text": ...}`) + """ + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, list): + parts = [] + for part in value: + if isinstance(part, str): + parts.append(part) + elif isinstance(part, dict): + text = part.get("text") or part.get("input_text") + if isinstance(text, str): + parts.append(text) + if parts: + return "\n".join(parts) + return None + + +def _to_plain_dict(value): + """Best-effort: coerce a value (Pydantic model / dict / None) to a dict. + + Returns the original value when no safe conversion exists. Used to bridge + OpenAI Pydantic message/tool_call objects into the dict-based helpers. + """ + if value is None or isinstance(value, dict): + return value + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + try: + return model_dump() + except Exception: + pass + return value + + +def _get_tool_calls(message) -> Optional[list]: + """Return ``message.tool_calls`` only when it's a non-empty list. + + Works for dicts and Pydantic message objects via ``_safe_get``. + """ + tool_calls = _safe_get(message, "tool_calls") + return tool_calls if isinstance(tool_calls, list) and tool_calls else None + + +def _normalize_tool_call(raw_tc) -> Optional[Dict[str, Any]]: + """Normalize a single tool_call (dict or Pydantic) into a stable shape: + + {"id": str|None, "type": str, "function": {"name": str|None, "arguments": str|None}} + + Arguments are coerced to a JSON string per OpenInference convention. + Returns ``None`` when ``raw_tc`` cannot be coerced to a dict. + """ + tc = _to_plain_dict(raw_tc) + if not isinstance(tc, dict): + return None + function = _to_plain_dict(tc.get("function")) + name = function.get("name") if isinstance(function, dict) else None + args = function.get("arguments") if isinstance(function, dict) else None + if args is not None and not isinstance(args, str): + try: + args = json.dumps(args) + except Exception: + args = str(args) + return { + "id": tc.get("id"), + "type": tc.get("type", "function"), + "function": {"name": name, "arguments": args}, + } + + +def _summarize_tool_calls_for_output(tool_calls) -> str: + """Render a tool_calls list as a compact JSON string for OUTPUT_VALUE. + + Best-effort: returns ``str(tool_calls)`` if anything unexpected happens + so OUTPUT_VALUE is never blanked on a malformed payload. + """ + try: + normalized = [n for n in (_normalize_tool_call(tc) for tc in tool_calls) if n] + return json.dumps({"tool_calls": normalized}) + except Exception: + return str(tool_calls) + + +def _emit_message_tool_calls(span: "Span", prefix: str, message) -> None: + """Emit ``MESSAGE_TOOL_CALLS.*`` for an assistant message that requested + tool calls. Pure addition: only writes when ``tool_calls`` is non-empty. + + Accepts dicts or Pydantic message objects (e.g. ``litellm.Message``); the + same applies to each tool_call entry. + """ + tool_calls = _get_tool_calls(message) + if not tool_calls: + return + for tc_idx, raw_tc in enumerate(tool_calls): + tc = _normalize_tool_call(raw_tc) + if tc is None: + continue + tc_prefix = f"{prefix}.{MessageAttributes.MESSAGE_TOOL_CALLS}.{tc_idx}" + if tc["id"]: + safe_set_attribute( + span, f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_ID}", tc["id"] + ) + fn = tc["function"] + if fn["name"]: + safe_set_attribute( + span, + f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}", + fn["name"], + ) + if fn["arguments"] is not None: + safe_set_attribute( + span, + f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}", + fn["arguments"], + ) + + +def _emit_input_message_extras(span: "Span", prefix: str, message: dict) -> None: + """Emit additive attributes for an input message: + + - `MESSAGE_NAME` and `MESSAGE_TOOL_CALL_ID` (commonly set on tool-result + messages so traces show which tool produced which result). + - `MESSAGE_TOOL_CALLS.*` when an assistant message requested tools. + - `MESSAGE_CONTENTS.*` structured content for list-shaped content + (multimodal text + image parts). The plain `MESSAGE_CONTENT` write is + still performed by the caller, so renderers that only read the legacy + key continue to work. + """ + if not isinstance(message, dict): + return + + name = message.get("name") + if name: + safe_set_attribute(span, f"{prefix}.{MessageAttributes.MESSAGE_NAME}", name) + + tool_call_id = message.get("tool_call_id") + if tool_call_id: + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_TOOL_CALL_ID}", + tool_call_id, + ) + + _emit_message_tool_calls(span, prefix, message) + + content = message.get("content") + if isinstance(content, list): + contents_prefix = f"{prefix}.{MessageAttributes.MESSAGE_CONTENTS}" + for part_idx, part in enumerate(content): + if not isinstance(part, dict): + continue + part_prefix = f"{contents_prefix}.{part_idx}" + part_type = part.get("type") + if part_type in ("text", "input_text"): + text = part.get("text") + if isinstance(text, str): + safe_set_attribute( + span, + f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", + "text", + ) + safe_set_attribute( + span, + f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}", + text, + ) + elif part_type in ("image_url", "image", "input_image"): + url = None + image = part.get("image_url") + if isinstance(image, dict): + url = image.get("url") + elif isinstance(image, str): + url = image + if not url: + # Anthropic-style source.{type=base64,media_type,data} + source = part.get("source") + if isinstance(source, dict) and source.get("data"): + media_type = source.get("media_type", "image/jpeg") + url = f"data:{media_type};base64,{source['data']}" + elif isinstance(part.get("url"), str): + url = part["url"] + if url: + safe_set_attribute( + span, + f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", + "image", + ) + safe_set_attribute( + span, + f"{part_prefix}.message_content.image.image.url", + url, + ) + + +def _set_session_and_user_attrs( + span: "Span", kwargs: dict, standard_logging_payload +) -> None: + """Emit `SESSION_ID` / `USER_ID` / team metadata when source data exists. + + `SESSION_ID` is emitted only when an explicit end-user identifier exists + (`metadata.user_api_key_end_user_id`). We deliberately do NOT fall back + to `trace_id`, because that would create a distinct "session" for every + single request and distort Arize's Session-grouping analytics. The + `trace_id` is still emitted under its own `litellm.trace_id` key so + spans remain filterable by trace. + + USER_ID is *only* emitted when no upstream path (model_params.user or + optional_params.user) has already set it, to avoid overwriting an + existing value with a possibly-different one from API-key metadata. + """ + if not isinstance(standard_logging_payload, dict): + return + metadata = standard_logging_payload.get("metadata") or {} + if not isinstance(metadata, dict): + return + + session_id = metadata.get("user_api_key_end_user_id") + if session_id: + safe_set_attribute(span, SpanAttributes.SESSION_ID, str(session_id)) + + trace_id = standard_logging_payload.get("trace_id") + if trace_id: + safe_set_attribute(span, "litellm.trace_id", str(trace_id)) + + optional_params = kwargs.get("optional_params") or {} + model_params = standard_logging_payload.get("model_parameters") or {} + has_user_already = bool( + (isinstance(optional_params, dict) and optional_params.get("user")) + or (isinstance(model_params, dict) and model_params.get("user")) + ) + if not has_user_already: + user_id = metadata.get("user_api_key_user_id") + if user_id: + safe_set_attribute(span, SpanAttributes.USER_ID, str(user_id)) + + team_id = metadata.get("user_api_key_team_id") + if team_id: + safe_set_attribute(span, "litellm.team_id", str(team_id)) + team_alias = metadata.get("user_api_key_team_alias") + if team_alias: + safe_set_attribute(span, "litellm.team_alias", str(team_alias)) + key_alias = metadata.get("user_api_key_alias") + if key_alias: + safe_set_attribute(span, "litellm.key_alias", str(key_alias)) + + +def _set_response_cost_attr(span: "Span", standard_logging_payload) -> None: + """Emit cost attributes from the StandardLoggingPayload when present. + + Uses the OpenInference `llm.cost.total` key so Arize / Phoenix can + surface the cost in their "Total Cost" column. LiteLLM only tracks a + single total in `StandardLoggingPayload.response_cost`, so we cannot + split it into prompt/completion. We also keep the legacy + `llm.response.cost` key for back-compat with any consumer querying it. + """ + if not isinstance(standard_logging_payload, dict): + return + cost = standard_logging_payload.get("response_cost") + if cost is None: + return + try: + cost_value = float(cost) + except (TypeError, ValueError): + return + safe_set_attribute(span, "llm.cost.total", cost_value) + safe_set_attribute(span, "llm.response.cost", cost_value) + + +def _is_passthrough_call_type(call_type: Optional[str]) -> bool: + if not call_type: + return False + lowered = str(call_type).lower() + return "passthrough" in lowered or "pass_through" in lowered + + +def _maybe_normalize_passthrough( + span: "Span", + kwargs: dict, + raw_response_obj, + coerced_response_obj, + standard_logging_payload, +) -> None: + """Surface input/output text for passthrough routes (e.g. Bedrock + InvokeModel) so the parent span renders as more than `usage` numbers. + + Only runs when `call_type` is a passthrough variant. Reads from: + - `kwargs["additional_args"]["complete_input_dict"]` for input + - the coerced response (or `kwargs["original_response"]`) for output + + All emits are best-effort: if the provider shape isn't recognized the + helper exits silently. Existing chat/completion paths never enter this + helper because their call_type doesn't contain "passthrough". + + TEMPORARY BRIDGE: passthrough handlers don't populate the + StandardLoggingPayload `messages` field today (they call + `transform_response(messages=[])`), so the input is only available via + `additional_args.complete_input_dict`. The proper fix is upstream in + `base_passthrough_logging_handler._create_response_logging_payload()`: + once that populates SLP `messages`/`response`, every callback gets + passthrough I/O (with central redaction) for free and this helper's + `complete_input_dict` fallback can be deleted. See follow-up issue. + """ + call_type = ( + standard_logging_payload.get("call_type") + if isinstance(standard_logging_payload, dict) + else None + ) + if not _is_passthrough_call_type(call_type): + return + + # Respect LiteLLM's central message-redaction contract. The normal + # chat/completion path is redacted by `perform_redaction` before + # callbacks run, but `complete_input_dict` (read below) is NOT covered by + # that layer — so without this gate, an operator who enabled redaction + # would still see raw passthrough prompts in Arize. Skip entirely when + # redaction is on so neither input nor output leaks through this bridge. + if should_redact_message_logging(kwargs): + return + + # --- INPUT -------------------------------------------------------------- + additional_args = kwargs.get("additional_args") or {} + complete_input_dict = ( + additional_args.get("complete_input_dict") + if isinstance(additional_args, dict) + else None + ) + if isinstance(complete_input_dict, dict): + _set_passthrough_input_attributes(span, complete_input_dict.get("messages")) + + # --- OUTPUT ------------------------------------------------------------- + parsed_response = _parse_passthrough_response( + raw_response_obj, coerced_response_obj, kwargs + ) + if not isinstance(parsed_response, dict): + return + + _set_passthrough_output_attributes(span, parsed_response) + + +def _set_passthrough_input_attributes(span: "Span", messages) -> None: + """Render passthrough request messages into INPUT_VALUE + LLM_INPUT_MESSAGES.""" + if not (isinstance(messages, list) and messages): + return + # Set INPUT_VALUE from the last user message text if discoverable. + last_text = None + for msg in reversed(messages): + if isinstance(msg, dict): + last_text = _coerce_text(msg.get("content")) + if last_text: + break + if last_text: + safe_set_attribute(span, SpanAttributes.INPUT_VALUE, last_text) + # Mirror messages into LLM_INPUT_MESSAGES so the input pane renders. + for idx, msg in enumerate(messages): + if not isinstance(msg, dict): + continue + prefix = f"{SpanAttributes.LLM_INPUT_MESSAGES}.{idx}" + role = msg.get("role") + if role: + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", + role, + ) + text = _coerce_text(msg.get("content")) + if text is not None: + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", + text, + ) + + +def _set_passthrough_output_attributes(span: "Span", parsed_response: dict) -> None: + """Render passthrough response into OUTPUT_VALUE + LLM_OUTPUT_MESSAGES.""" + # Anthropic / Bedrock-Anthropic: `content` is a list of typed parts. + content_list = parsed_response.get("content") + if isinstance(content_list, list) and content_list: + texts = [] + for part in content_list: + if isinstance(part, dict) and isinstance(part.get("text"), str): + texts.append(part["text"]) + joined = "\n\n".join(t for t in texts if t) + if joined: + safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, joined) + prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0" + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", + parsed_response.get("role", "assistant"), + ) + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", + joined, + ) + + # OpenAI-style passthrough: `choices[0].message.content` + choices = parsed_response.get("choices") + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + msg = first.get("message") + if isinstance(msg, dict): + text = _coerce_text(msg.get("content")) + if text: + safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, text) + prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0" + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", + msg.get("role", "assistant"), + ) + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", + text, + ) + + +def _parse_passthrough_response(raw_response_obj, coerced_response_obj, kwargs): + """Return a dict view of the provider response for passthrough routes.""" + # Prefer the coerced view (already JSON-parsed for httpx.Response). + candidates = [] + if isinstance(coerced_response_obj, dict): + candidates.append(coerced_response_obj) + if ( + isinstance(raw_response_obj, dict) + and raw_response_obj is not coerced_response_obj + ): + candidates.append(raw_response_obj) + + for candidate in candidates: + # StandardPassThroughResponseObject wrapper: {"response": "..."}. + if ( + "response" in candidate + and "content" not in candidate + and "choices" not in candidate + ): + inner = candidate.get("response") + if isinstance(inner, str): + try: + parsed = json.loads(inner) + if isinstance(parsed, dict): + return parsed + except Exception: + continue + if isinstance(inner, dict): + return inner + else: + return candidate + + # Fallback: kwargs["original_response"] from the OTel base path. + original = kwargs.get("original_response") if isinstance(kwargs, dict) else None + if isinstance(original, dict): + return original + if isinstance(original, str): + try: + parsed = json.loads(original) + if isinstance(parsed, dict): + return parsed + except Exception: + return None + return None diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index c2b0c4ddce9..3a69c9a7936 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -104,6 +104,51 @@ }, "description": "Datadog Custom Metrics Integration" }, + { + "id": "galileo", + "displayName": "Galileo", + "logo": "galileo.ico", + "supports_key_team_logging": false, + "dynamic_params": { + "GALILEO_API_KEY": { + "type": "password", + "ui_name": "API Key", + "description": "Galileo Cloud API key (app.galileo.ai). Omit for enterprise username/password auth.", + "required": false + }, + "GALILEO_PROJECT_ID": { + "type": "text", + "ui_name": "Project ID", + "description": "Galileo project ID to log traces to", + "required": true + }, + "GALILEO_LOG_STREAM_ID": { + "type": "text", + "ui_name": "Log Stream ID", + "description": "Galileo log stream ID for v2 spans logging (optional)", + "required": false + }, + "GALILEO_BASE_URL": { + "type": "text", + "ui_name": "Base URL", + "description": "Galileo API base URL (e.g. https://api.galileo.ai for Cloud, or your enterprise API URL)", + "required": false + }, + "GALILEO_USERNAME": { + "type": "text", + "ui_name": "Username", + "description": "Galileo enterprise username (legacy Observe auth; use instead of API key)", + "required": false + }, + "GALILEO_PASSWORD": { + "type": "password", + "ui_name": "Password", + "description": "Galileo enterprise password (legacy Observe auth)", + "required": false + } + }, + "description": "Galileo AI Observability Integration" + }, { "id": "datadog_cost_management", "displayName": "Datadog Cost Management", diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 82a35f2eedd..fc5f0429b63 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -47,9 +47,29 @@ from litellm.exceptions import ( BlockedPiiEntityError, GuardrailRaisedException, ModifyResponseException, + SensitiveDataRouteException, ) +def get_session_id_from_request_data(request_data: Dict[str, Any]) -> Optional[str]: + """Extract session_id from request data (litellm_session_id or metadata).""" + session_id = request_data.get("litellm_session_id") + if session_id: + return str(session_id) + + metadata = request_data.get("metadata") or {} + session_id = metadata.get("session_id") + if session_id: + return str(session_id) + + litellm_metadata = request_data.get("litellm_metadata") or {} + session_id = litellm_metadata.get("session_id") + if session_id: + return str(session_id) + + return None + + class CustomGuardrail(CustomLogger): # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path. use_native_during_call_hook: ClassVar[bool] = False @@ -68,6 +88,9 @@ class CustomGuardrail(CustomLogger): end_session_after_n_fails: Optional[int] = None, on_violation: Optional[str] = None, realtime_violation_message: Optional[str] = None, + on_sensitive_data: Optional[str] = None, + sensitive_data_route_to_model: Optional[str] = None, + sticky_session_routing: bool = True, **kwargs, ): """ @@ -83,6 +106,9 @@ class CustomGuardrail(CustomLogger): end_session_after_n_fails: For /v1/realtime sessions, end the session after this many violations on_violation: For /v1/realtime sessions, 'warn' or 'end_session' realtime_violation_message: Message the bot speaks aloud when a /v1/realtime guardrail fires + on_sensitive_data: Action when sensitive data is detected. 'block' (default) or 'route' + sensitive_data_route_to_model: Model to route to when on_sensitive_data='route' + sticky_session_routing: When True, all subsequent requests in the session use the same model """ self.guardrail_name = guardrail_name self.supported_event_hooks = supported_event_hooks @@ -96,6 +122,11 @@ class CustomGuardrail(CustomLogger): self.end_session_after_n_fails: Optional[int] = end_session_after_n_fails self.on_violation: Optional[str] = on_violation self.realtime_violation_message: Optional[str] = realtime_violation_message + self.on_sensitive_data: Optional[str] = on_sensitive_data + self.sensitive_data_route_to_model: Optional[str] = ( + sensitive_data_route_to_model + ) + self.sticky_session_routing: bool = sticky_session_routing if supported_event_hooks: ## validate event_hook is in supported_event_hooks @@ -167,6 +198,108 @@ class CustomGuardrail(CustomLogger): detection_info=detection_info, ) + def raise_sensitive_data_route_exception( + self, + route_to_model: str, + request_data: Dict[str, Any], + detection_info: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Raise an exception to reroute the request to a different model. + + Use this when sensitive data is detected and the guardrail is configured + to route to an on-premise model instead of blocking. + + The exception will reroute this request to the specified model. When + sticky_session_routing is enabled (the default), it also stores the + routing decision so subsequent requests in this session reuse the model. + + Args: + route_to_model: The model to route this request (and session) to + request_data: The original request data dictionary + detection_info: Optional non-sensitive detection metadata (e.g. matched + entity types, rule ids, scores). This is surfaced in request metadata + and logs, so it must not contain the raw detected sensitive values. + + Raises: + SensitiveDataRouteException: Always raises to trigger rerouting + """ + session_id = self._get_session_id_from_request_data(request_data) + if not session_id: + raise ValueError( + "Cannot route sensitive data without a session_id. " + "Ensure the request includes a session_id in metadata or headers." + ) + + raise SensitiveDataRouteException( + route_to_model=route_to_model, + session_id=session_id, + guardrail_name=self.guardrail_name, + detection_info=detection_info, + sticky_session_routing=self.sticky_session_routing, + ) + + def _get_session_id_from_request_data( + self, request_data: Dict[str, Any] + ) -> Optional[str]: + """Extract session_id from request data.""" + return get_session_id_from_request_data(request_data) + + def should_route_on_sensitive_data(self) -> bool: + """ + Returns True if this guardrail is configured to route requests + to a different model when sensitive data is detected. + """ + return ( + self.on_sensitive_data == "route" + and self.sensitive_data_route_to_model is not None + ) + + def handle_sensitive_data_detection( + self, + request_data: Dict[str, Any], + detection_info: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Handle sensitive data detection based on guardrail configuration. + + If on_sensitive_data='route', raises SensitiveDataRouteException to reroute. + Otherwise, raises GuardrailRaisedException to block. When routing is + configured but the request carries no session_id, routing is not possible + so the request falls back to a graceful block. + + Args: + request_data: The request data dictionary + detection_info: Optional non-sensitive detection metadata. When routing, + this is surfaced in request metadata and logs, so it must not contain + the raw detected sensitive values. + + Raises: + SensitiveDataRouteException: When configured to route and a session_id is present + GuardrailRaisedException: When configured to block, or when routing is + configured but no session_id is available + """ + if self.should_route_on_sensitive_data(): + try: + self.raise_sensitive_data_route_exception( + route_to_model=self.sensitive_data_route_to_model, # type: ignore + request_data=request_data, + detection_info=detection_info, + ) + except ValueError: + raise GuardrailRaisedException( + message=( + f"Sensitive data detected by {self.guardrail_name} " + "(routing skipped: request has no session_id)" + ), + guardrail_name=self.guardrail_name, + ) + else: + raise GuardrailRaisedException( + message=f"Sensitive data detected by {self.guardrail_name}", + guardrail_name=self.guardrail_name, + ) + @staticmethod def get_config_model() -> Optional[Type["GuardrailConfigModel"]]: """ @@ -662,6 +795,16 @@ class CustomGuardrail(CustomLogger): request_data["metadata"] = {} _append_guardrail_info(request_data["metadata"]) + # Emit the otel guardrail span here, where every guardrail execution lands, + # rather than relying on a post-call hook that does not fire on every path + # (e.g. a pass-through request that passes its guardrails). + try: + from litellm.integrations.otel.logger import emit_guardrail_span + + emit_guardrail_span(slg) + except Exception: + pass + async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, @@ -743,12 +886,20 @@ class CustomGuardrail(CustomLogger): Guardrails signal intentional blocks by raising: - GuardrailRaisedException (generic guardrail API, tool permission) - BlockedPiiEntityError (Presidio PII detection) + - SensitiveDataRouteException (sensitive-data reroute to on-premise model) - HTTPException with status 400 (content policy violation) - ModifyResponseException (passthrough mode violation) """ if isinstance(e, ModifyResponseException): return True - if isinstance(e, (GuardrailRaisedException, BlockedPiiEntityError)): + if isinstance( + e, + ( + GuardrailRaisedException, + BlockedPiiEntityError, + SensitiveDataRouteException, + ), + ): return True if ( HTTPException is not None diff --git a/litellm/integrations/email_alerting.py b/litellm/integrations/email_alerting.py index b45b9aa7f5c..b721dc50464 100644 --- a/litellm/integrations/email_alerting.py +++ b/litellm/integrations/email_alerting.py @@ -7,6 +7,7 @@ from typing import List, Optional from litellm._logging import verbose_logger, verbose_proxy_logger from litellm.proxy._types import WebhookEvent +from litellm.repositories.team_repository import TeamRepository # we use this for the email header, please send a test email if you change this. verify it looks good on email LITELLM_LOGO_URL = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" @@ -24,7 +25,7 @@ async def get_all_team_member_emails(team_id: Optional[str] = None) -> list: if prisma_client is None: raise Exception("Not connected to DB!") - team_row = await prisma_client.db.litellm_teamtable.find_unique( + team_row = await TeamRepository(prisma_client).table.find_unique( where={ "team_id": team_id, } diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index a598124f612..8fef90c24e0 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -1,8 +1,13 @@ +from __future__ import annotations + import json import os import re -from typing import Any, Dict, List, Optional, Tuple, cast +import uuid +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Tuple, Union, cast +import httpx from pydantic import BaseModel, Field import litellm @@ -12,11 +17,15 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, get_content_from_model_response, ) +from litellm.types.llms.openai import ( + AllMessageValues, + HttpxBinaryResponseContent, + ResponsesAPIResponse, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.types.llms.openai import AllMessageValues GALILEO_CLOUD_API_BASE_URL = "https://api.galileo.ai" # Cap the in-memory buffer so persistent flush failures (e.g. Galileo @@ -33,6 +42,11 @@ class LLMResponse(BaseModel): model: str num_input_tokens: int num_output_tokens: int + num_total_tokens: int + cost: Optional[float] = Field( + default=None, + description="Total cost of the LLM call in USD as computed by LiteLLM.", + ) output_logprobs: Optional[Dict[str, Any]] = Field( default=None, description="Optional. When available, logprobs are used to compute Uncertainty.", @@ -121,10 +135,14 @@ class GalileoObserve(CustomLogger): @staticmethod def _galileo_input_messages( - messages: Optional[List[Any]], input_text: str + messages: Optional[Any], input_text: str ) -> List[Dict[str, str]]: + if isinstance(messages, dict): + messages = messages.get("messages") if not messages: return [{"role": "user", "content": input_text}] + if not isinstance(messages, list): + return [{"role": "user", "content": input_text}] galileo_messages: List[Dict[str, str]] = [] for message in messages: @@ -147,13 +165,59 @@ class GalileoObserve(CustomLogger): return [{"role": "user", "content": input_text}] @staticmethod - def _record_to_v2_span(record: Dict[str, Any]) -> Dict[str, Any]: - created_at = record.get("created_at", "") + def _local_timezone(): + return datetime.now().astimezone().tzinfo or timezone.utc + + @staticmethod + def _format_created_at(dt: Union[datetime, Any]) -> str: + """Serialize timestamps as UTC ISO-8601 for Galileo.""" + if not isinstance(dt, datetime): + return str(dt) + + if dt.tzinfo is None: + # LiteLLM often passes naive datetimes in local time; convert to UTC + # instead of appending Z to local time (which shifts Traces tab sorting). + dt = dt.replace(tzinfo=GalileoObserve._local_timezone()) + + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + @staticmethod + def _normalize_created_at(created_at: str) -> str: if created_at and not re.search(r"(Z|[+-]\d{2}:?\d{2})$", created_at): - created_at = f"{created_at}Z" + return f"{created_at}Z" + return created_at + + @staticmethod + def _token_metrics_from_record(record: Dict[str, Any]) -> Dict[str, Any]: + num_input_tokens = int(record.get("num_input_tokens") or 0) + num_output_tokens = int(record.get("num_output_tokens") or 0) + num_total_tokens = int(record.get("num_total_tokens") or 0) + if num_total_tokens == 0 and (num_input_tokens or num_output_tokens): + num_total_tokens = num_input_tokens + num_output_tokens + metrics: Dict[str, Any] = { + "num_input_tokens": num_input_tokens, + "num_output_tokens": num_output_tokens, + "num_total_tokens": num_total_tokens, + } + cost = record.get("cost") + if cost is not None: + metrics["cost"] = float(cost) + return metrics + + @staticmethod + def _record_to_v2_span( + record: Dict[str, Any], + *, + trace_id: str, + span_id: str, + ) -> Dict[str, Any]: + created_at = GalileoObserve._normalize_created_at(record.get("created_at", "")) span: Dict[str, Any] = { "type": "llm", + "id": span_id, + "trace_id": trace_id, + "parent_id": trace_id, "name": record.get("node_type", "litellm"), "created_at": created_at, "input": GalileoObserve._galileo_input_messages( @@ -167,14 +231,49 @@ class GalileoObserve(CustomLogger): "model": record.get("model"), "metrics": { "duration_ns": int(record.get("latency_ms", 0)) * 1_000_000, - "num_input_tokens": record.get("num_input_tokens"), - "num_output_tokens": record.get("num_output_tokens"), + **GalileoObserve._token_metrics_from_record(record), }, } if record.get("tags"): span["tags"] = record["tags"] return span + @staticmethod + def _record_to_v2_trace(record: Dict[str, Any]) -> Dict[str, Any]: + trace_id = str(uuid.uuid4()) + span_id = str(uuid.uuid4()) + created_at = GalileoObserve._normalize_created_at(record.get("created_at", "")) + + return { + "type": "trace", + "id": trace_id, + "name": record.get("node_type", "litellm"), + "created_at": created_at, + "input": record.get("input_text", ""), + "output": record.get("output_text", ""), + "status_code": record.get("status_code", 200), + "metrics": { + "duration_ns": int(record.get("latency_ms", 0)) * 1_000_000, + **GalileoObserve._token_metrics_from_record(record), + }, + "spans": [ + GalileoObserve._record_to_v2_span( + record, trace_id=trace_id, span_id=span_id + ) + ], + } + + def _build_traces_payload(self, records: List[dict]) -> Dict[str, Any]: + payload: Dict[str, Any] = { + "traces": [self._record_to_v2_trace(record) for record in records], + "logging_method": "api_direct", + "reliable": False, + "is_complete": True, + } + if self.log_stream_id: + payload["log_stream_id"] = self.log_stream_id + return payload + def _get_ingest_request(self) -> Optional[Tuple[str, Dict[str, Any]]]: if not self.base_url or not self.project_id: return None @@ -184,105 +283,457 @@ class GalileoObserve(CustomLogger): # flush_in_memory_records) aren't silently dropped when we later clear # the in-memory buffer. records = list(self.in_memory_records) + payload = self._build_traces_payload(records) if self.use_v2_api: - payload: Dict[str, Any] = { - "spans": [self._record_to_v2_span(record) for record in records], - "reliable": False, - } - if self.log_stream_id: - payload["log_stream_id"] = self.log_stream_id return ( - f"{self.base_url}/v2/projects/{self.project_id}/spans", + f"{self.base_url}/ingest/traces/{self.project_id}", payload, ) + # Username/password auth logs in for a JWT and uses the standard v2 traces API. return ( - f"{self.base_url}/projects/{self.project_id}/observe/ingest", - {"records": records}, + f"{self.base_url}/v2/projects/{self.project_id}/traces", + payload, ) + @staticmethod + def _redact_headers(headers: Optional[Dict[str, str]]) -> Dict[str, str]: + if not headers: + return {} + redacted: Dict[str, str] = {} + for key, value in headers.items(): + if key.lower() in {"authorization", "galileo-api-key"} and value: + redacted[key] = ( + f"{value[:8]}...{value[-4:]}" if len(value) > 12 else "***" + ) + else: + redacted[key] = value + return redacted + + def _log_flush_config(self) -> None: + verbose_logger.debug( + "Galileo Logger flush config: use_v2_api=%s base_url=%s project_id=%s " + "log_stream_id=%s api_key_set=%s username_set=%s record_count=%s", + self.use_v2_api, + self.base_url, + self.project_id, + self.log_stream_id, + bool(self.api_key), + bool(self.username), + len(self.in_memory_records), + ) + + @staticmethod + def _log_v2_payload_validation(payload: Dict[str, Any]) -> None: + missing_fields: List[str] = [] + traces = payload.get("traces", []) + if not traces: + missing_fields.append("traces") + + for trace_index, trace in enumerate(traces): + if not isinstance(trace, dict): + continue + for field in ("id", "type", "spans"): + if field not in trace: + missing_fields.append(f"traces[{trace_index}].{field}") + + trace_id = trace.get("id") + for span_index, span in enumerate(trace.get("spans", [])): + if not isinstance(span, dict): + continue + for field in ("id", "trace_id", "parent_id"): + if field not in span: + missing_fields.append( + f"traces[{trace_index}].spans[{span_index}].{field}" + ) + if trace_id and span.get("trace_id") != trace_id: + missing_fields.append( + f"traces[{trace_index}].spans[{span_index}].trace_id mismatch" + ) + + if missing_fields: + verbose_logger.debug( + "Galileo Logger: ingest /traces payload validation issues: %s", + missing_fields, + ) + + def _log_flush_payload(self, url: str, payload: Dict[str, Any]) -> None: + traces = payload.get("traces", []) + verbose_logger.debug( + "Galileo Logger flush URL: %s trace_count=%s", + url, + len(traces) if isinstance(traces, list) else 0, + ) + if self.use_v2_api and "/ingest/traces/" in url: + self._log_v2_payload_validation(payload) + + @staticmethod + def _log_http_status_error(error: httpx.HTTPStatusError, url: str) -> None: + response = error.response + verbose_logger.debug( + "Galileo Logger HTTP error: status=%s url=%s", + response.status_code, + url, + ) + verbose_logger.debug( + "Galileo Logger HTTP error response body: %s", + response.text, + ) + try: + verbose_logger.debug( + "Galileo Logger HTTP error response json: %s", + response.json(), + ) + except Exception: + pass + + @staticmethod + def _build_prompt(kwargs: Dict[str, Any]) -> Dict[str, Any]: + optional_params = kwargs.get("optional_params", {}) or {} + prompt: Dict[str, Any] = {"messages": kwargs.get("messages")} + if optional_params.get("functions") is not None: + prompt["functions"] = optional_params["functions"] + if optional_params.get("tools") is not None: + prompt["tools"] = optional_params["tools"] + return prompt + + @staticmethod + def _serialize_galileo_output(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, str): + return value + + def _json_default(obj: Any) -> Any: + if hasattr(obj, "model_dump"): + return obj.model_dump() + return str(obj) + + return json.dumps(value, default=_json_default) + + @staticmethod + def _prompt_to_input_text(prompt: Dict[str, Any]) -> str: + messages = prompt.get("messages") + if messages is not None: + text = GalileoObserve._input_text_from_messages(messages) + if text: + return text + return json.dumps(prompt, default=str) + + @staticmethod + def _get_chat_content_for_galileo(response_obj: litellm.ModelResponse) -> Any: + if response_obj.choices and len(response_obj.choices) > 0: + message = response_obj["choices"][0]["message"] + if hasattr(message, "json"): + message_json = message.json() + if isinstance(message_json, str): + return json.loads(message_json) + return message_json + return message + return None + + @staticmethod + def _get_text_completion_content_for_galileo( + response_obj: litellm.TextCompletionResponse, + ) -> Optional[str]: + if response_obj.choices and len(response_obj.choices) > 0: + return response_obj.choices[0].text + return None + + @staticmethod + def _get_responses_api_content_for_galileo( + response_obj: ResponsesAPIResponse, + ) -> Any: + if hasattr(response_obj, "output") and response_obj.output: + return response_obj.output + return None + + @staticmethod + def _langfuse_style_rerank_prompt(kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Match Langfuse rerank input: prompt = {"messages": kwargs.get("messages")}.""" + return {"messages": kwargs.get("messages")} + + def _get_galileo_input_output_content( + self, + kwargs: Dict[str, Any], + response_obj: Any, + level: str = "DEFAULT", + status_message: Optional[str] = None, + ) -> Tuple[str, Optional[str], Any]: + """ + Mirror Langfuse _get_langfuse_input_output_content for Galileo ingest. + + Returns (input_text, output_text, messages_for_span). output_text None skips ingest. + """ + call_type = kwargs.get("call_type") + prompt = self._build_prompt(kwargs) + + if ( + level == "ERROR" + and status_message is not None + and isinstance(status_message, str) + ): + return self._prompt_to_input_text(prompt), status_message, prompt + + if response_obj is not None and ( + call_type == "embedding" + or isinstance(response_obj, litellm.EmbeddingResponse) + ): + return self._prompt_to_input_text(prompt), None, prompt + + if response_obj is not None and isinstance(response_obj, litellm.ModelResponse): + output = self._get_chat_content_for_galileo(response_obj) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + kwargs.get("messages") or [], + ) + + if response_obj is not None and isinstance( + response_obj, HttpxBinaryResponseContent + ): + return self._prompt_to_input_text(prompt), "speech-output", prompt + + if response_obj is not None and isinstance( + response_obj, litellm.TextCompletionResponse + ): + output = self._get_text_completion_content_for_galileo(response_obj) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + kwargs.get("messages") or [], + ) + + if response_obj is not None and isinstance(response_obj, litellm.ImageResponse): + output = response_obj.get("data", None) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + prompt, + ) + + if response_obj is not None and isinstance( + response_obj, litellm.TranscriptionResponse + ): + output = response_obj.get("text", None) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + prompt, + ) + + if response_obj is not None and isinstance( + response_obj, litellm.RerankResponse + ): + output = response_obj.results + rerank_prompt = self._langfuse_style_rerank_prompt(kwargs) + return ( + json.dumps(rerank_prompt, default=str), + self._serialize_galileo_output(output), + rerank_prompt, + ) + + if response_obj is not None and isinstance(response_obj, ResponsesAPIResponse): + output = self._get_responses_api_content_for_galileo(response_obj) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + kwargs.get("messages") or [], + ) + + if ( + call_type == "_arealtime" + and response_obj is not None + and isinstance(response_obj, list) + ): + input_val = kwargs.get("input") + return ( + self._serialize_galileo_output(input_val) or "", + self._serialize_galileo_output(response_obj), + input_val, + ) + + if ( + call_type == "pass_through_endpoint" + and response_obj is not None + and isinstance(response_obj, dict) + ): + output = response_obj.get("response", "") + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + prompt, + ) + + if response_obj is not None and isinstance(response_obj, dict): + output = get_content_from_model_response(response_obj) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + kwargs.get("messages") or [], + ) + + return self._prompt_to_input_text(prompt), None, kwargs.get("messages") or [] + def get_output_str_from_response( self, response_obj: Any, kwargs: Dict[str, Any] ) -> Optional[str]: - if response_obj is None: - return None - if kwargs.get("call_type", None) == "embedding" or isinstance( - response_obj, litellm.EmbeddingResponse - ): - return None - if isinstance(response_obj, litellm.TextCompletionResponse): - return response_obj.choices[0].text - if isinstance(response_obj, litellm.ImageResponse): - return json.dumps(response_obj["data"], default=str) - if isinstance(response_obj, (litellm.ModelResponse, dict)): - return get_content_from_model_response(response_obj) - return None + _, output_text, _ = self._get_galileo_input_output_content( + kwargs=kwargs, response_obj=response_obj + ) + return output_text + + @staticmethod + def _input_text_from_messages(messages: Any) -> str: + """Return a plain-string summary of the input suitable for the trace-level input field.""" + if isinstance(messages, str): + return messages + if not isinstance(messages, list): + return "" + # Use the last user/human message so the trace table shows the actual prompt + for msg in reversed(messages): + if not isinstance(msg, dict): + continue + if str(msg.get("role", "")).lower() in ("user", "human"): + content = msg.get("content") or "" + if isinstance(content, list): + content = " ".join( + b.get("text", "") if isinstance(b, dict) else str(b) + for b in content + ) + if content: + return str(content) + # Fallback: first non-empty content of any role + for msg in messages: + if isinstance(msg, dict): + content = msg.get("content") or "" + if isinstance(content, list): + content = " ".join( + b.get("text", "") if isinstance(b, dict) else str(b) + for b in content + ) + if content: + return str(content) + return "" async def async_log_success_event( self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any ): verbose_logger.debug("On Async Success") + try: + await self._async_log_success_event_impl( + kwargs=kwargs, + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + ) + except Exception: + verbose_logger.exception( + "Galileo Logger: unexpected error in async_log_success_event" + ) + async def _async_log_success_event_impl( + self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any + ): if not self._is_configured(): verbose_logger.debug( - "Galileo Logger: skipping flush — set GALILEO_PROJECT_ID and " - "either GALILEO_API_KEY (hosted) or GALILEO_USERNAME/GALILEO_PASSWORD " - "(enterprise Observe)." + "Galileo Logger: skipping — GALILEO_PROJECT_ID=%s GALILEO_API_KEY=%s GALILEO_BASE_URL=%s", + bool(self.project_id), + bool(self.api_key), + bool(self.base_url), ) return - _latency_ms = int((end_time - start_time).total_seconds() * 1000) - _call_type = kwargs.get("call_type", "litellm") - input_text = litellm.utils.get_formatted_prompt( - data=kwargs, call_type=_call_type + slo: Optional[Dict[str, Any]] = kwargs.get("standard_logging_object") + if slo is None: + verbose_logger.debug( + "Galileo Logger: no standard_logging_object in kwargs, skipping" + ) + return + + _call_type: str = str( + slo.get("call_type") or kwargs.get("call_type") or "litellm" ) - _usage = response_obj.get("usage", {}) or {} - num_input_tokens = _usage.get("prompt_tokens", 0) - num_output_tokens = _usage.get("completion_tokens", 0) + input_text, output_text, messages = self._get_galileo_input_output_content( + kwargs=kwargs, response_obj=response_obj + ) + if output_text is None: + verbose_logger.debug( + "Galileo Logger: skipping %s — no text output to log", _call_type + ) + return - output_text = self.get_output_str_from_response( - response_obj=response_obj, kwargs=kwargs + raw_start = slo.get("startTime") + raw_end = slo.get("endTime") + if raw_start is None or raw_end is None: + verbose_logger.debug( + "Galileo Logger: standard_logging_object missing startTime/endTime, " + "falling back to start_time/end_time params" + ) + if not isinstance(start_time, datetime) or not isinstance( + end_time, datetime + ): + return + start_ts = start_time + end_ts = end_time + if start_ts.tzinfo is None: + start_ts = start_ts.replace(tzinfo=GalileoObserve._local_timezone()) + if end_ts.tzinfo is None: + end_ts = end_ts.replace(tzinfo=GalileoObserve._local_timezone()) + start_ts = start_ts.astimezone(timezone.utc) + end_ts = end_ts.astimezone(timezone.utc) + else: + start_ts = datetime.fromtimestamp(float(raw_start), tz=timezone.utc) + end_ts = datetime.fromtimestamp(float(raw_end), tz=timezone.utc) + _latency_ms = max(0, int((end_ts - start_ts).total_seconds() * 1000)) + num_input_tokens = int(slo.get("prompt_tokens") or 0) + num_output_tokens = int(slo.get("completion_tokens") or 0) + num_total_tokens = int(slo.get("total_tokens") or 0) + if num_total_tokens == 0 and (num_input_tokens or num_output_tokens): + num_total_tokens = num_input_tokens + num_output_tokens + + request_record = LLMResponse( + latency_ms=_latency_ms, + status_code=200, + input_text=input_text, + output_text=output_text, + node_type=_call_type, + model=str(slo.get("model") or kwargs.get("model") or "-"), + num_input_tokens=num_input_tokens, + num_output_tokens=num_output_tokens, + num_total_tokens=num_total_tokens, + cost=slo.get("response_cost"), + created_at=GalileoObserve._format_created_at(start_ts), ) - if output_text is not None: - request_record = LLMResponse( - latency_ms=_latency_ms, - status_code=200, - input_text=input_text, - output_text=output_text, - node_type=_call_type, - model=kwargs.get("model", "-"), - num_input_tokens=num_input_tokens, - num_output_tokens=num_output_tokens, - created_at=start_time.strftime( - "%Y-%m-%dT%H:%M:%S" - ), # timestamp str constructed in "%Y-%m-%dT%H:%M:%S" format + request_dict = request_record.model_dump() + if isinstance(messages, dict): + messages = messages.get("messages") + if isinstance(messages, list) and messages: + request_dict["messages"] = messages + self.in_memory_records.append(request_dict) + verbose_logger.debug( + "Galileo Logger: queued record, in_memory=%d", len(self.in_memory_records) + ) + + # Bound the buffer so persistent flush failures cannot grow it + # without limit. Drop the oldest records once we exceed the cap. + if len(self.in_memory_records) > GALILEO_MAX_IN_MEMORY_RECORDS: + dropped = len(self.in_memory_records) - GALILEO_MAX_IN_MEMORY_RECORDS + self.in_memory_records = self.in_memory_records[ + -GALILEO_MAX_IN_MEMORY_RECORDS: + ] + verbose_logger.warning( + "Galileo Logger: in-memory buffer exceeded %s records; " + "dropped %s oldest record(s). Check Galileo connectivity/credentials.", + GALILEO_MAX_IN_MEMORY_RECORDS, + dropped, ) - request_dict = request_record.model_dump() - messages = kwargs.get("messages") - if messages: - request_dict["messages"] = messages - self.in_memory_records.append(request_dict) - - # Bound the buffer so persistent flush failures cannot grow it - # without limit. Drop the oldest records once we exceed the cap. - if len(self.in_memory_records) > GALILEO_MAX_IN_MEMORY_RECORDS: - dropped = len(self.in_memory_records) - GALILEO_MAX_IN_MEMORY_RECORDS - self.in_memory_records = self.in_memory_records[ - -GALILEO_MAX_IN_MEMORY_RECORDS: - ] - verbose_logger.warning( - "Galileo Logger: in-memory buffer exceeded %s records; " - "dropped %s oldest record(s). Check Galileo connectivity/credentials.", - GALILEO_MAX_IN_MEMORY_RECORDS, - dropped, - ) - - if len(self.in_memory_records) >= self.batch_size: - await self.flush_in_memory_records() + if len(self.in_memory_records) >= self.batch_size: + await self.flush_in_memory_records() async def flush_in_memory_records(self): if not self.in_memory_records: @@ -296,15 +747,23 @@ class GalileoObserve(CustomLogger): ingest_request = self._get_ingest_request() if ingest_request is None: verbose_logger.debug( - "Galileo Logger: missing GALILEO_BASE_URL or GALILEO_PROJECT_ID" + "Galileo Logger: missing GALILEO_BASE_URL or GALILEO_PROJECT_ID — skipping flush" ) return if not await self._ensure_headers(): - verbose_logger.debug("Galileo Logger: could not set request headers") + verbose_logger.debug( + "Galileo Logger: could not set request headers — skipping flush" + ) return url, payload = ingest_request + self._log_flush_config() + self._log_flush_payload(url=url, payload=payload) + verbose_logger.debug( + "Galileo Logger flush headers: %s", + self._redact_headers(self.headers), + ) verbose_logger.debug("flushing in memory records to %s", url) try: @@ -313,6 +772,12 @@ class GalileoObserve(CustomLogger): headers=self.headers, json=payload, ) + except httpx.HTTPStatusError as e: + self._log_http_status_error(error=e, url=url) + verbose_logger.debug( + "Galileo Logger: failed to flush in memory records: %s", e + ) + return except Exception as e: verbose_logger.debug( "Galileo Logger: failed to flush in memory records: %s", e @@ -323,6 +788,11 @@ class GalileoObserve(CustomLogger): verbose_logger.debug( "Galileo Logger: successfully flushed in memory records" ) + verbose_logger.debug( + "Galileo Logger flush response: status=%s body=%s", + response.status_code, + response.text, + ) del self.in_memory_records[:records_in_payload] else: verbose_logger.debug("Galileo Logger: failed to flush in memory records") diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index b7a565512c6..cae59295634 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -102,6 +102,18 @@ def langfuse_client_init( if Version(langfuse.version.__version__) >= Version("2.6.0"): parameters["sdk_integration"] = "litellm" + if Version(langfuse.version.__version__) >= Version("2.7.3"): + import httpx + + import litellm + + from ...llms.custom_httpx.http_handler import get_ssl_configuration + + parameters["httpx_client"] = httpx.Client( + verify=get_ssl_configuration(), + cert=os.getenv("SSL_CERTIFICATE", litellm.ssl_certificate), + ) + client = Langfuse(**parameters) return client diff --git a/litellm/integrations/openmeter.py b/litellm/integrations/openmeter.py index 5a8ab4bcc9f..b234ab11ddb 100644 --- a/litellm/integrations/openmeter.py +++ b/litellm/integrations/openmeter.py @@ -65,7 +65,15 @@ class OpenMeterLogger(CustomLogger): "total_tokens": response_obj["usage"].get("total_tokens"), } - user_param = kwargs.get("user", None) # end-user passed in via 'user' param + # OPENMETER_TRUST_REQUEST_USER (default "true"): when set to "false", + # the request-supplied `user` field is ignored and the subject is + # resolved solely from the key-bound user_api_key_user_id. Proxies + # serving multi-tenant traffic enable this to prevent clients from + # forging attribution by setting `user` in the request body. + trust_request_user = ( + os.getenv("OPENMETER_TRUST_REQUEST_USER", "true").lower() != "false" + ) + user_param = kwargs.get("user", None) if trust_request_user else None # If no user provided directly, try to get it from token user_id if user_param is None: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 8bc61d960e7..24780eb4bfc 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1012,6 +1012,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = litellm_params.get("metadata", {}) or {} proxy_span = _metadata.get("litellm_parent_otel_span", None) + + # Fallback: check litellm_metadata (used by /v1/messages and other + # LITELLM_METADATA_ROUTES). + if proxy_span is None: + _litellm_metadata = litellm_params.get("litellm_metadata", {}) or {} + proxy_span = _litellm_metadata.get("litellm_parent_otel_span", None) + if ( proxy_span is not None and getattr(proxy_span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME @@ -2668,6 +2675,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) def _to_ns(self, dt): + if dt is None: + return int(datetime.now().timestamp() * 1e9) + if isinstance(dt, (int, float)): + return int(dt * 1e9) return int(dt.timestamp() * 1e9) def _get_span_name(self, kwargs): @@ -2714,6 +2725,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): _metadata = litellm_params.get("metadata", {}) or {} parent_otel_span = _metadata.get("litellm_parent_otel_span", None) + # Fallback: check litellm_metadata (used by /v1/messages and other + # LITELLM_METADATA_ROUTES that store proxy-internal metadata + # separately from the provider's native "metadata" field). + if parent_otel_span is None: + _litellm_metadata = litellm_params.get("litellm_metadata", {}) or {} + parent_otel_span = _litellm_metadata.get("litellm_parent_otel_span", None) + # Priority 1: Explicit parent span from metadata if parent_otel_span is not None: verbose_logger.debug( diff --git a/litellm/integrations/opik/opik_payload_builder/extractors.py b/litellm/integrations/opik/opik_payload_builder/extractors.py index 9779ccddacf..1e3a664acc1 100644 --- a/litellm/integrations/opik/opik_payload_builder/extractors.py +++ b/litellm/integrations/opik/opik_payload_builder/extractors.py @@ -39,20 +39,32 @@ def extract_opik_metadata( standard_logging_metadata: Dict[str, Any], ) -> Dict[str, Any]: """ - Extract and merge Opik metadata from request and requester. + Merge Opik metadata from three sources in increasing priority order: + + 1. user_api_key_auth_metadata– lowest priority (operator-level defaults) + 2. litellm_metadata (request)– overrides auth-key defaults + 3. requester_metadata – highest priority (e.g. proxy header overrides) Args: - litellm_metadata: Metadata from litellm_params - standard_logging_metadata: Metadata from standard_logging_object + litellm_metadata: Metadata from litellm_params.mak + standard_logging_metadata: Metadata from standard_logging_object. Returns: - Merged Opik metadata dictionary + Merged Opik metadata dictionary. """ - opik_meta = litellm_metadata.get("opik", {}).copy() + # Start with auth-key defaults (lowest priority). + auth_meta = standard_logging_metadata.get("user_api_key_auth_metadata") or {} + opik_meta = (auth_meta.get("opik") or {}).copy() + # Request-level values override auth-key defaults. + request_opik = litellm_metadata.get("opik") or {} + opik_meta.update(request_opik) + + # Requester-level values win over everything else. requester_metadata = standard_logging_metadata.get("requester_metadata", {}) or {} requester_opik = requester_metadata.get("opik", {}) or {} - opik_meta.update(requester_opik) + if requester_opik: + opik_meta.update(requester_opik) _logging.verbose_logger.debug( f"litellm_opik_metadata - {json.dumps(opik_meta, default=str)}" diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index de5e7a7b8ab..5e683ce7b99 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -15,6 +15,7 @@ from litellm.integrations.otel.model.baggage import promoted_baggage from litellm.integrations.otel.model.config import OpenTelemetryV2Config from litellm.integrations.otel.plumbing.context import ( is_recordable_span, + request_root_span, resolve_parent_context, resolve_request_span_context, set_request_baggage, @@ -25,7 +26,6 @@ from litellm.integrations.otel.mappers import resolve_mappers from litellm.integrations.otel.model.metadata import ( LLMCallEvent, RequestIdentity, - guardrail_entries_from_request_data, model_from_request_data, ) from litellm.integrations.otel.model.payloads import ( @@ -436,8 +436,12 @@ class OpenTelemetryV2(CustomLogger): attach(set_request_baggage(bag, context=get_current())) # The server span was started by the instrumentor before this ran, # so the Baggage processor (which only fires at span start) won't - # backfill it — stamp identity on it directly. - server_span = get_current_span() + # backfill it — stamp identity on it directly. Prefer the anchored + # root span over the ambient one so identity still lands on the + # server span when seeding from inside the live ``auth`` phase span + # (the auth-failure path), where ``get_current_span`` is the phase + # span, not the request's root. + server_span = request_root_span() or get_current_span() if is_recordable_span(server_span): # Re-capture the anchor here too: this runs post-auth with the # server span active and covers entrypoints that bypass @@ -468,46 +472,27 @@ class OpenTelemetryV2(CustomLogger): ) return data - async def async_post_call_success_hook( - self, - data: Mapping[str, Any], - user_api_key_dict: Any, - response: Any, - ) -> Any: - self._emit_guardrail_spans(data) - return response - - async def async_post_call_failure_hook( - self, - request_data: Mapping[str, Any], - original_exception: BaseException | None, - user_api_key_dict: Any, - traceback_str: str | None = None, - ) -> None: - self._emit_guardrail_spans(request_data) - - def _emit_guardrail_spans(self, request_data: Mapping[str, Any]) -> None: + def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None: + # Emitted by the guardrail-recording code the moment a guardrail finishes, + # not from a post-call hook — that hook does not fire on every path (a + # pass-through request that passes its guardrails never reaches it), which + # left passing guardrails without a span. + # # A guardrail is a sibling of the LLM call under the request's root span, - # so parent it to the explicit anchor — not the active span, which on the - # failure path can be the live ``auth`` phase span (post-call failure hooks - # run from inside it on an auth rejection). Emit with the guardrail's actual - # execution window so a pre_call guardrail is placed before the LLM call - # rather than at post-call emission time. - guardrails = guardrail_entries_from_request_data(request_data) - if not guardrails: - return - parent_ctx = resolve_request_span_context() - for entry in guardrails: - data = GuardrailSpanData.from_logging_entry( - cast("StandardLoggingGuardrailInformation", entry) - ) - self._emitter.emit( - SpanRole.GUARDRAIL, - data, - parent_context=parent_ctx, - start_time_ns=to_ns(data.start_time), - end_time_ns=to_ns(data.end_time), - ) + # so parent it to the explicit anchor — never the active span, which during + # a pre_call guardrail can be the live ``auth`` phase span. Emit with the + # guardrail's actual execution window so a pre_call guardrail is placed + # before the LLM call rather than at emission time. One entry in, one span + # out — the module-level entry point routes each entry to this single + # registered logger so a guardrail is never emitted more than once. + data = GuardrailSpanData.from_logging_entry(entry) + self._emitter.emit( + SpanRole.GUARDRAIL, + data, + parent_context=resolve_request_span_context(), + start_time_ns=to_ns(data.start_time), + end_time_ns=to_ns(data.end_time), + ) def create_litellm_proxy_request_started_span( self, start_time: datetime, headers: Mapping[str, str] | None @@ -528,6 +513,26 @@ def _registered_v2_logger() -> "OpenTelemetryV2 | None": return logger if isinstance(logger, OpenTelemetryV2) else None +def emit_guardrail_span(entry: "StandardLoggingGuardrailInformation") -> None: + """Emit a guardrail span on the registered v2 OTel logger. + + Called by the guardrail-recording code the moment a guardrail finishes, so a + span is produced regardless of whether a post-call hook later runs (it does + not on the pass-through allow path). Routes through the single canonical + logger — the same one every other v2 entry point uses — so a guardrail + recorded once yields exactly one span; fanning out across every reachable + ``OpenTelemetryV2`` instance double-emits the same entry. Best-effort: span + emission must never break guardrail evaluation. + """ + logger = _registered_v2_logger() + if logger is None: + return + try: + logger.emit_guardrail_span(entry) + except Exception: + pass + + def seed_request_identity(user_api_key_dict: Any, model: Any = None) -> None: logger = _registered_v2_logger() if logger is not None: diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 4663ed59761..4c9cecfef57 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -255,26 +255,6 @@ def model_from_request_data(data: object) -> str | None: return None -def guardrail_entries_from_request_data( - request_data: Mapping[str, Any], -) -> list[dict]: - """The guardrail-information dicts buried in ``metadata`` of a post-call dict. - - ``standard_logging_guardrail_information`` is stored as either a single dict - or a list of them; normalize to a list of dicts (dropping non-dict noise) so - the caller just iterates. Empty list when none are present. - """ - metadata = request_data.get("metadata") - if not isinstance(metadata, Mapping): - return [] - info = metadata.get("standard_logging_guardrail_information") - if isinstance(info, Mapping): - return [cast(dict, info)] - if isinstance(info, list): - return [entry for entry in info if isinstance(entry, dict)] - return [] - - def resolve_provider_model(payload: "StandardLoggingPayload") -> str | None: """The model litellm dispatched to the provider, from the payload. diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 08fce09868b..bbef40ba374 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -340,7 +340,7 @@ class MCPToolCallSpanData: def from_standard_logging_payload( cls, payload: "StandardLoggingPayload", capture_content: bool = False ) -> "MCPToolCallSpanData": - meta = cast(Mapping[str, object], payload.get("mcp_tool_call_metadata") or {}) + meta = _mcp_tool_call_metadata(cast(Mapping[str, object], payload)) return cls( operation=resolve_operation(as_str(payload.get("call_type"))), method=MCPMethod.TOOLS_CALL.value, @@ -363,11 +363,22 @@ class MCPToolCallSpanData: ) +def _mcp_tool_call_metadata(payload: Mapping[str, object]) -> Mapping[str, object]: + """The MCP gateway's tool-call metadata, which lives under + ``StandardLoggingPayload.metadata`` (a ``StandardLoggingMetadata`` key), not + at the payload's top level.""" + metadata = payload.get("metadata") + if not isinstance(metadata, Mapping): + return {} + meta = metadata.get("mcp_tool_call_metadata") + return meta if isinstance(meta, Mapping) else {} + + def is_mcp_tool_call(payload: Mapping[str, object]) -> bool: """Whether a closed request's payload is an MCP tool call rather than an LLM call — true when the MCP gateway stamped its tool-call metadata, or the call type says so on a path that hasn't populated the metadata yet.""" - return bool(payload.get("mcp_tool_call_metadata")) or ( + return bool(_mcp_tool_call_metadata(payload)) or ( payload.get("call_type") == "call_mcp_tool" ) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 9fc09807369..2119527a8e5 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -24,14 +24,18 @@ from typing import ( import litellm from litellm._logging import print_verbose, verbose_logger -from litellm.integrations.custom_logger import CustomLogger -from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import ( - BoundedPrometheusSeriesTracker, +from litellm.exceptions import ( + validate_rate_limit_category, + validate_rate_limit_type, ) +from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus_helpers import ( PrometheusLabelFactoryContext, _get_cached_end_user_id_for_cost_tracking, ) +from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import ( + BoundedPrometheusSeriesTracker, +) from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -42,6 +46,9 @@ from litellm.proxy._types import ( LiteLLM_UserTable, UserAPIKeyAuth, ) +from litellm.repositories.organization_repository import OrganizationRepository +from litellm.repositories.team_repository import TeamRepository +from litellm.repositories.user_repository import UserRepository from litellm.types.integrations.prometheus import * from litellm.types.integrations.prometheus import ( _sanitize_prometheus_label_name, @@ -78,6 +85,20 @@ class PrometheusLogger(CustomLogger): # Always initialize label_filters, even for non-premium users self.label_filters = self._parse_prometheus_config() + # Cache resolved label sets per metric. Several entries in + # ``PrometheusMetricLabels.get_labels`` read module-level toggles + # (e.g. ``litellm.prometheus_emit_stream_label``, + # ``litellm.prometheus_emit_rate_limit_labels``) that can be + # changed at runtime. Prometheus counters/gauges/histograms are + # created with a *fixed* ``labelnames`` set; if a runtime call + # to ``get_labels_for_metric`` returned a different set, the + # subsequent ``counter.labels(**_labels)`` would raise a + # ``ValueError`` from the prometheus client. Snapshotting at + # logger init time pins the label set for the lifetime of the + # logger so toggling these flags only takes effect after a + # restart, keeping init-time and runtime label sets in sync. + self._cached_metric_labels: Dict[str, List[str]] = {} + _custom_buckets = litellm.prometheus_latency_buckets self.latency_buckets = ( tuple(_custom_buckets) @@ -1033,13 +1054,27 @@ class PrometheusLogger(CustomLogger): self, metric_name: DEFINED_PROMETHEUS_METRICS ) -> List[str]: """ - Get the labels for a metric, filtered if configured + Get the labels for a metric, filtered if configured. + + The result is cached on the instance so the label set used to + construct each Prometheus metric at ``__init__`` time stays in lock + step with the label set passed to ``counter.labels(...)`` at + runtime, even if the underlying module-level toggles consulted by + :meth:`PrometheusMetricLabels.get_labels` (e.g. + ``litellm.prometheus_emit_rate_limit_labels``, + ``litellm.prometheus_emit_stream_label``) are flipped after the + logger has been created. """ + cached = self._cached_metric_labels.get(metric_name) + if cached is not None: + return cached + # Get default labels for this metric from PrometheusMetricLabels default_labels = PrometheusMetricLabels.get_labels(metric_name) # If no label filtering is configured for this metric, use default labels if metric_name not in self.label_filters: + self._cached_metric_labels[metric_name] = default_labels return default_labels # Get configured labels for this metric @@ -1050,6 +1085,7 @@ class PrometheusLogger(CustomLogger): label for label in default_labels if label in configured_labels ] + self._cached_metric_labels[metric_name] = filtered_labels return filtered_labels def _track_end_user_metric_series( @@ -2029,14 +2065,8 @@ class PrometheusLogger(CustomLogger): Proxy level tracking - failed client side requests - labelnames=[ - "end_user", - "hashed_api_key", - "api_key_alias", - REQUESTED_MODEL, - "team", - "team_alias", - ] + EXCEPTION_LABELS, + See :attr:`PrometheusMetricLabels.litellm_proxy_failed_requests_metric` + for the authoritative list of labels emitted on this metric. """ from litellm.litellm_core_utils.litellm_logging import ( StandardLoggingPayloadSetup, @@ -2059,6 +2089,9 @@ class PrometheusLogger(CustomLogger): model_id = _metadata.get("model_info", {}).get("id") or request_data.get( "model_info", {} ).get("id") + rate_limit_category, rate_limit_type = self._extract_rate_limit_labels( + original_exception + ) enum_values = UserAPIKeyLabelValues( end_user=user_api_key_dict.end_user_id, user=user_api_key_dict.user_id, @@ -2073,6 +2106,8 @@ class PrometheusLogger(CustomLogger): status_code=str(status_code), exception_status=str(status_code), exception_class=self._get_exception_class_name(original_exception), + rate_limit_category=rate_limit_category, + rate_limit_type=rate_limit_type, tags=_tags, route=user_api_key_dict.request_route, client_ip=_metadata.get("requester_ip_address"), @@ -2690,7 +2725,7 @@ class PrometheusLogger(CustomLogger): Args: guardrail_name: Name of the guardrail latency_seconds: Execution latency in seconds - status: "success" or "error" + status: "success", "error", or "intervened" error_type: Type of error if any, None otherwise hook_type: "pre_call", "during_call", or "post_call" """ @@ -2843,6 +2878,33 @@ class PrometheusLogger(CustomLogger): @staticmethod def _get_exception_class_name(exception: Exception) -> str: + # Some exception types pin the ``exception_class`` label to a legacy + # value for back-compat with existing dashboards (e.g. proxy-side 429s + # keep reporting as "HTTPException"). Honor that opt-in marker before + # deriving the label from the runtime class name. Reading it via + # ``getattr`` keeps this core integrations module free of a transitive + # ``fastapi`` dependency. + legacy_class_name = getattr(exception, "prometheus_exception_class_name", None) + if isinstance(legacy_class_name, str) and legacy_class_name: + return legacy_class_name + + # Same back-compat reasoning for ``BudgetExceededError``: the unified + # rate-limit error work attached ``.llm_provider`` to budget errors + # too (so callbacks reading ``StandardLoggingPayload`` get provider + # attribution). Without this short-circuit, the provider prefix below + # would silently flip the label from "BudgetExceededError" to e.g. + # "Openai.BudgetExceededError" and break dashboards keyed on the + # original value. + try: + from litellm.exceptions import BudgetExceededError + except ImportError: + BudgetExceededError = None # type: ignore[assignment,misc] + + if BudgetExceededError is not None and isinstance( + exception, BudgetExceededError + ): + return "BudgetExceededError" + exception_class_name = "" if hasattr(exception, "llm_provider"): exception_class_name = getattr(exception, "llm_provider") or "" @@ -2857,6 +2919,27 @@ class PrometheusLogger(CustomLogger): exception_class_name += exception.__class__.__name__ return exception_class_name + @staticmethod + def _extract_rate_limit_labels( + exception: Optional[Exception], + ) -> Tuple[Optional[str], Optional[str]]: + """ + Pull the unified ``category`` / ``rate_limit_type`` fields off any + exception that declares them (``litellm.RateLimitError`` and bare- + Exception subclasses like ``BudgetExceededError``). + + Values are validated against the :class:`RateLimitErrorCategory` / + :class:`RateLimitType` enums so unrelated third-party exceptions that + happen to declare ``.category`` / ``.rate_limit_type`` string attributes + can't leak garbage into Prometheus label cardinality. + """ + if exception is None: + return None, None + return ( + validate_rate_limit_category(getattr(exception, "category", None)), + validate_rate_limit_type(getattr(exception, "rate_limit_type", None)), + ) + async def log_success_fallback_event( self, original_model_group: str, kwargs: dict, original_exception: Exception ): @@ -3198,12 +3281,12 @@ class PrometheusLogger(CustomLogger): page_size: int, page: int ) -> Tuple[List[LiteLLM_UserTable], Optional[int]]: skip = (page - 1) * page_size - users = await prisma_client.db.litellm_usertable.find_many( + users = await UserRepository(prisma_client).table.find_many( skip=skip, take=page_size, order={"created_at": "desc"}, ) - total_count = await prisma_client.db.litellm_usertable.count() + total_count = await UserRepository(prisma_client).table.count() return users, total_count await self._initialize_budget_metrics( @@ -3226,13 +3309,13 @@ class PrometheusLogger(CustomLogger): async def fetch_orgs(page_size: int, page: int) -> Tuple[list, Optional[int]]: skip = (page - 1) * page_size - orgs = await prisma_client.db.litellm_organizationtable.find_many( + orgs = await OrganizationRepository(prisma_client).table.find_many( skip=skip, take=page_size, order={"created_at": "desc"}, include={"litellm_budget_table": True}, ) - total_count = await prisma_client.db.litellm_organizationtable.count() + total_count = await OrganizationRepository(prisma_client).table.count() return orgs, total_count await self._initialize_budget_metrics( @@ -3300,14 +3383,14 @@ class PrometheusLogger(CustomLogger): try: # Get total user count - total_users = await prisma_client.db.litellm_usertable.count() + total_users = await UserRepository(prisma_client).table.count() self.litellm_total_users_metric.set(total_users) verbose_logger.debug( f"Prometheus: set litellm_total_users to {total_users}" ) # Get total team count - total_teams = await prisma_client.db.litellm_teamtable.count() + total_teams = await TeamRepository(prisma_client).table.count() self.litellm_teams_count_metric.set(total_teams) verbose_logger.debug( f"Prometheus: set litellm_teams_count to {total_teams}" diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index 3aa0a1558d7..ce7f01c5a2a 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -244,6 +244,9 @@ search_tools: - search_tool_name: "my-tavily-tool" litellm_params: search_provider: "tavily" + - search_tool_name: "my-you-com-tool" + litellm_params: + search_provider: "you_com" ``` --- diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 95658d08767..ffaa5140916 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -655,7 +655,11 @@ def exception_type( # type: ignore # noqa: PLR0915 custom_llm_provider == "anthropic" or custom_llm_provider == "anthropic_text" ): # one of the anthropics - if "prompt is too long" in error_str or "prompt: length" in error_str: + if ( + "prompt is too long" in error_str + or "prompt: length" in error_str + or ExceptionCheckers.is_error_str_context_window_exceeded(error_str) + ): exception_mapping_worked = True raise ContextWindowExceededError( message="AnthropicError - {}".format(error_str), diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index bb3f3fae9f0..de65ed93312 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -373,6 +373,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.lambda.ai/v1": custom_llm_provider = "lambda_ai" dynamic_api_key = get_secret_str("LAMBDA_API_KEY") + elif endpoint == "https://api.inceptionlabs.ai/v1": + custom_llm_provider = "inception" + dynamic_api_key = get_secret_str("INCEPTION_API_KEY") elif endpoint == "https://api.hyperbolic.xyz/v1": custom_llm_provider = "hyperbolic" dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY") @@ -656,6 +659,11 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or get_secret_str("NVIDIA_RIVA_API_KEY") or get_secret_str("NVIDIA_NIM_API_KEY") ) + elif custom_llm_provider == "soniox": + api_base = ( + api_base or get_secret_str("SONIOX_API_BASE") or "https://api.soniox.com" + ) + dynamic_api_key = api_key or get_secret_str("SONIOX_API_KEY") elif custom_llm_provider == "cerebras": api_base = ( api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1" @@ -954,6 +962,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "inception": + ( + api_base, + dynamic_api_key, + ) = litellm.InceptionChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "hyperbolic": ( api_base, diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index b8cdc8210fc..23b51faafc7 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -22,9 +22,11 @@ def get_supported_openai_params( # noqa: PLR0915 ``` Args: - base_model: For Azure, the true underlying model (e.g. ``"azure/gpt-5.2"``) - when the deployment name differs. Used for model-type detection so that - non-standard deployment names route to the correct config. + base_model: An optional capability hint for deployments whose ``model`` + label isn't recognized on its own (e.g. an Azure deployment name, or a + friendly Bedrock alias). It is additive: the result is the union of the + params supported by ``model`` and by ``base_model``, so a hint can only + add capabilities, never strip ones the real model already supports. Returns: - List if custom_llm_provider is mapped @@ -52,7 +54,15 @@ def get_supported_openai_params( # noqa: PLR0915 provider_config = None if provider_config and request_type == "chat_completion": - return provider_config.get_supported_openai_params(model=base_model or model) + supported_params = provider_config.get_supported_openai_params(model=model) + if base_model and base_model != model: + base_model_params = provider_config.get_supported_openai_params( + model=base_model + ) + supported_params = list( + dict.fromkeys([*supported_params, *base_model_params]) + ) + return supported_params if custom_llm_provider == "bedrock": return litellm.AmazonConverseConfig().get_supported_openai_params(model=model) @@ -331,6 +341,11 @@ def get_supported_openai_params( # noqa: PLR0915 return ElevenLabsAudioTranscriptionConfig().get_supported_openai_params( model=model ) + elif custom_llm_provider == "soniox": + if request_type == "transcription": + return litellm.SonioxAudioTranscriptionConfig().get_supported_openai_params( + model=model + ) elif custom_llm_provider in litellm._custom_providers: if request_type == "chat_completion": provider_config = litellm.ProviderConfigManager.get_provider_chat_config( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index abc22713be4..dbfcf55d75d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -37,6 +37,10 @@ from litellm import ( turn_off_message_logging, ) from litellm._logging import _is_debugging_on, _redact_string, verbose_logger +from litellm.exceptions import ( + validate_rate_limit_category, + validate_rate_limit_type, +) from litellm._uuid import uuid from litellm.batches.batch_utils import _handle_completed_batch from litellm.caching.caching import DualCache, InMemoryCache @@ -3503,7 +3507,9 @@ class Logging(LiteLLMLoggingBaseClass): else: return None - def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse: + def _handle_anthropic_messages_response_logging( + self, result: Any + ) -> Union[ModelResponse, ResponsesAPIResponse]: """ Handles logging for Anthropic messages responses. @@ -3522,6 +3528,15 @@ class Logging(LiteLLMLoggingBaseClass): return result elif isinstance(result, ModelResponse): return result + elif isinstance( + result, + (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent), + ): + # anthropic_messages() can route to OpenAI Responses API; in that path + # the assembled streaming result is one of these terminal events rather than + # a ModelResponse. Return the inner response so downstream handlers + # (_transform_usage_objects, normalize_logging_result) can process it. + return result.response httpx_response = self.model_call_details.get("httpx_response", None) if httpx_response and isinstance(httpx_response, httpx.Response): @@ -5307,12 +5322,27 @@ class StandardLoggingPayloadSetup: else str(original_exception) ) + # Duck-typed read so bare-Exception subclasses like + # `litellm.BudgetExceededError` can participate without joining the + # RateLimitError hierarchy (which would break `except BudgetExceededError`). + # Validated against the enum value sets so a third-party exception that + # happens to declare a `.category` or `.rate_limit_type` string attribute + # can't leak garbage into the payload or Prometheus label cardinality. + rate_limit_category = validate_rate_limit_category( + getattr(original_exception, "category", None) + ) + rate_limit_type = validate_rate_limit_type( + getattr(original_exception, "rate_limit_type", None) + ) + return StandardLoggingPayloadErrorInformation( error_code=error_status, error_class=error_class, llm_provider=_llm_provider_in_exception, traceback=traceback_info, error_message=error_message if original_exception else "", + error_rate_limit_category=rate_limit_category, + error_rate_limit_type=rate_limit_type, ) @staticmethod diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 882561ed2e8..f39c942f90f 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -34,6 +34,14 @@ _IMAGE_RESPONSE_CALL_TYPES = frozenset( _VALID_DATA_RESIDENCIES = frozenset(r.value for r in DataResidency) +def _get_token_detail_value(details: object, key: str) -> Optional[int]: + if isinstance(details, dict): + value = details.get(key) + else: + value = getattr(details, key, None) + return value if isinstance(value, int) else None + + def _is_above_128k(tokens: float) -> bool: if tokens > 128000: return True @@ -870,17 +878,47 @@ def calculate_image_response_cost_from_usage( cached_tokens=0, ) + output_tokens_details = getattr(usage, "completion_tokens_details", None) + if output_tokens_details is None: + output_tokens_details = getattr(usage, "output_tokens_details", None) + + if output_tokens_details is None: + completion_tokens_details = CompletionTokensDetailsWrapper( + text_tokens=0, + image_tokens=completion_tokens, + reasoning_tokens=0, + audio_tokens=0, + ) + else: + text_tokens = _get_token_detail_value(output_tokens_details, "text_tokens") or 0 + image_tokens = ( + _get_token_detail_value(output_tokens_details, "image_tokens") or 0 + ) + audio_tokens = ( + _get_token_detail_value(output_tokens_details, "audio_tokens") or 0 + ) + reasoning_tokens = ( + _get_token_detail_value(output_tokens_details, "reasoning_tokens") or 0 + ) + known_output_tokens = ( + text_tokens + image_tokens + audio_tokens + reasoning_tokens + ) + if completion_tokens > known_output_tokens: + text_tokens += completion_tokens - known_output_tokens + + completion_tokens_details = CompletionTokensDetailsWrapper( + text_tokens=text_tokens, + image_tokens=image_tokens, + reasoning_tokens=reasoning_tokens, + audio_tokens=audio_tokens, + ) + normalized_usage = Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens, prompt_tokens_details=prompt_tokens_details, - completion_tokens_details=CompletionTokensDetailsWrapper( - text_tokens=0, - image_tokens=completion_tokens, - reasoning_tokens=0, - audio_tokens=0, - ), + completion_tokens_details=completion_tokens_details, ) prompt_cost, completion_cost = generic_cost_per_token( diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 46e9b43a429..81a4c8b14b6 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1670,15 +1670,15 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915 if gemini_call_id: _function_response["id"] = gemini_call_id - # Create part with function_response, and optionally inline_data for images (Computer Use) _part: VertexPartType = {"function_response": _function_response} - # For Computer Use, if we have images/files, we need separate parts: - # - One part with function_response - # - One part per inline_data item - # Gemini's PartType is a oneof, so we can't have both in the same part + # For multimodal function responses, Gemini expects media parts nested + # inside functionResponse.parts instead of sibling content parts. if inline_data_list: - return [_part] + [{"inline_data": d} for d in inline_data_list] + _function_response["parts"] = [ + {"inline_data": inline_data} for inline_data in inline_data_list + ] + return [_part] return _part @@ -3653,17 +3653,13 @@ from litellm.types.llms.bedrock import ContentBlock as BedrockContentBlock from litellm.types.llms.bedrock import DocumentBlock as BedrockDocumentBlock from litellm.types.llms.bedrock import ImageBlock as BedrockImageBlock from litellm.types.llms.bedrock import SourceBlock as BedrockSourceBlock +from litellm.types.llms.bedrock import BedrockToolSpec from litellm.types.llms.bedrock import ToolBlock as BedrockToolBlock -from litellm.types.llms.bedrock import ( - ToolInputSchemaBlock as BedrockToolInputSchemaBlock, -) -from litellm.types.llms.bedrock import ToolJsonSchemaBlock as BedrockToolJsonSchemaBlock from litellm.types.llms.bedrock import SearchResultBlock from litellm.types.llms.bedrock import ToolResultBlock as BedrockToolResultBlock from litellm.types.llms.bedrock import ( ToolResultContentBlock as BedrockToolResultContentBlock, ) -from litellm.types.llms.bedrock import ToolSpecBlock as BedrockToolSpecBlock from litellm.types.llms.bedrock import ToolUseBlock as BedrockToolUseBlock from litellm.types.llms.bedrock import VideoBlock as BedrockVideoBlock @@ -5496,6 +5492,7 @@ def _bedrock_tools_pt( ] """ from litellm.llms.bedrock.common_utils import ( + get_bedrock_base_model, normalize_json_schema_custom_types_to_object, ) from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs @@ -5503,6 +5500,11 @@ def _bedrock_tools_pt( _valid_json_schema_root_types = frozenset( ("array", "boolean", "integer", "null", "number", "object", "string") ) + # Only Claude on Bedrock honours strict tool schemas; other families + # (Nova, Llama, GPT-OSS) reject the strict field outright. + supports_strict_tools = bool( + model and get_bedrock_base_model(model).startswith("anthropic") + ) tool_block_list: List[BedrockToolBlock] = [] for tool_idx, tool in enumerate(tools): # Check if tool is already a BedrockToolBlock (e.g., systemTool for Nova grounding) @@ -5548,17 +5550,16 @@ def _bedrock_tools_pt( normalize_json_schema_custom_types_to_object(parameters) if parameters.get("type") not in _valid_json_schema_root_types: parameters["type"] = "object" - tool_input_schema = BedrockToolInputSchemaBlock( - json=BedrockToolJsonSchemaBlock( - type=parameters["type"], - properties=parameters.get("properties", {}), - required=parameters.get("required", []), - ) + tool_block = cast( + BedrockToolBlock, + BedrockToolSpec( + name=name, + description=description, + parameters=parameters, + strict=tool.get("function", {}).get("strict", None), + supports_strict_tools=supports_strict_tools, + ), ) - tool_spec = BedrockToolSpecBlock( - inputSchema=tool_input_schema, name=name, description=description - ) - tool_block = BedrockToolBlock(toolSpec=tool_spec) tool_block_list.append(tool_block) ## ADD CACHE POINT TOOL BLOCK ## diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 33bb6d7d2ea..772f058d9bb 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -92,8 +92,27 @@ class RealTimeStreaming: # Track whether we have already sent the guardrail turn-detection update # that disables provider auto-response for transcription guardrails. self._guardrail_turn_detection_update_sent: bool = False + # Deferred Gemini Live setup: Pipecat may stream audio before session.update. + # Buffer client audio until the backend acknowledges setup (setupComplete). + self._backend_setup_complete: bool = ( + provider_config is None or provider_config.requires_session_configuration() + ) + self._flushing_pending_messages_until_setup: bool = False + self._pending_messages_until_setup: List[str] = [] + self._pending_messages_byte_total: int = 0 + + # Per-connection caps for pre-setup audio frames (message count + total bytes). + _MAX_BUFFERED_MESSAGES: int = 200 + _MAX_BUFFERED_BYTES: int = 10 * 1024 * 1024 # 10 MB _SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"]) + _CLIENT_AUDIO_BUFFER_TYPES = frozenset( + [ + "input_audio_buffer.append", + "input_audio_buffer.commit", + "input_audio_buffer.clear", + ] + ) _AUDIO_FORMAT_MAP: Dict[str, Dict[str, Any]] = { "pcm16": {"type": "audio/pcm", "rate": 24000}, "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000}, @@ -285,6 +304,86 @@ class RealTimeStreaming: await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined] return True + def _uses_deferred_backend_setup(self) -> bool: + """True when setup is deferred until the client's first session.update.""" + if self.provider_config is None: + return False + return not self.provider_config.requires_session_configuration() + + def _should_buffer_client_message_until_setup(self, message: str) -> bool: + if not self._uses_deferred_backend_setup(): + return False + if ( + self._backend_setup_complete + and not self._flushing_pending_messages_until_setup + ): + return False + try: + msg_obj = json.loads(message) + except (json.JSONDecodeError, TypeError): + return False + return msg_obj.get("type") in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES + + def _buffer_pending_message_until_setup(self, message: str) -> None: + msg_bytes = len(message.encode("utf-8")) + if ( + len(self._pending_messages_until_setup) + < RealTimeStreaming._MAX_BUFFERED_MESSAGES + and self._pending_messages_byte_total + msg_bytes + <= RealTimeStreaming._MAX_BUFFERED_BYTES + ): + self._pending_messages_until_setup.append(message) + self._pending_messages_byte_total += msg_bytes + else: + verbose_logger.warning( + "Pre-setup buffer full (%d messages / %d bytes); dropping frame", + len(self._pending_messages_until_setup), + self._pending_messages_byte_total, + ) + + async def _flush_pending_messages_until_setup(self) -> bool: + pending = self._pending_messages_until_setup + self._pending_messages_until_setup = [] + self._pending_messages_byte_total = 0 + for idx, message in enumerate(pending): + try: + await self._send_to_backend(message) + except Exception as e: + unsent = pending[idx:] + self._pending_messages_until_setup = ( + unsent + self._pending_messages_until_setup + ) + self._pending_messages_byte_total = sum( + len(msg.encode("utf-8")) + for msg in self._pending_messages_until_setup + ) + verbose_logger.debug( + "Failed to flush buffered client message after setup: %s " + "(%d buffered message(s) retained)", + e, + len(unsent), + ) + return False + return True + + async def _send_event_to_client(self, event: Any, event_str: str) -> bool: + if self._client_wants_beta and isinstance(event, dict): + try: + translated = self._translate_event_to_beta(event) + if translated is None: + return False + await self.websocket.send_text(json.dumps(translated)) + return True + except Exception as e: + verbose_logger.warning( + "Failed to translate %s to beta protocol, forwarding " + "untranslated event to client: %s", + event.get("type"), + e, + ) + await self.websocket.send_text(event_str) + return True + def _cache_session_configuration_request(self, transformed_message: str) -> None: """Store setup payload once sent to backend. @@ -547,6 +646,19 @@ class RealTimeStreaming: isinstance(event, dict) and event.get("type") == "session.created" ) if is_session_created_event: + if ( + self._uses_deferred_backend_setup() + and not self._backend_setup_complete + ): + self._backend_setup_complete = True + self._flushing_pending_messages_until_setup = True + try: + while self._pending_messages_until_setup: + flushed = await self._flush_pending_messages_until_setup() + if not flushed: + break + finally: + self._flushing_pending_messages_until_setup = False if self._session_created_sent_to_client: # A synthetic session.created (with placeholder defaults) was # already forwarded to the client when we connected. The @@ -569,7 +681,7 @@ class RealTimeStreaming: ## update if a prior attempt was dropped by the provider transform. if is_session_created_event and self._has_audio_transcription_guardrails(): self.store_message(event_str) - await self.websocket.send_text(event_str) + await self._send_event_to_client(event, event_str) await self._maybe_send_guardrail_turn_detection_update() continue ## GUARDRAIL: run on transcription events in provider_config path too @@ -581,7 +693,7 @@ class RealTimeStreaming: transcript = event.get("transcript", "") self._collect_user_input_from_backend_event(cast(dict, event)) self.store_message(event_str) - await self.websocket.send_text(event_str) + await self._send_event_to_client(event, event_str) blocked = await self.run_realtime_guardrails( cast(str, transcript), item_id=cast(Optional[str], event.get("item_id")), @@ -591,7 +703,7 @@ class RealTimeStreaming: continue ## LOGGING self.store_message(event_str) - await self.websocket.send_text(event_str) + await self._send_event_to_client(event, event_str) async def _handle_raw_backend_message(self, raw_response) -> bool: """Process a backend message without provider_config (raw path). @@ -880,6 +992,7 @@ class RealTimeStreaming: ## GUARDRAIL: intercept conversation.item.create for text-based injection. guardrail_turn_detection_injected = False + msg_type: Optional[str] = None try: msg_obj = json.loads(message) msg_type = msg_obj.get("type") @@ -1081,6 +1194,29 @@ class RealTimeStreaming: # actually forward to the backend. self.store_input(message=message) + if self._should_buffer_client_message_until_setup(message): + self._buffer_pending_message_until_setup(message) + continue + + if self._pending_messages_until_setup: + should_send_setup_before_buffered_messages = ( + not self._backend_setup_complete + and not self._flushing_pending_messages_until_setup + and msg_type == "session.update" + ) + if not should_send_setup_before_buffered_messages: + self._buffer_pending_message_until_setup(message) + if ( + self._backend_setup_complete + and not self._flushing_pending_messages_until_setup + ): + await self._flush_pending_messages_until_setup() + continue + + if self._flushing_pending_messages_until_setup: + self._buffer_pending_message_until_setup(message) + continue + ## FORWARD TO BACKEND # Only mark the guardrail turn_detection update as sent after the # backend actually accepted the message. Setting the flag earlier diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 55042a733ed..f3274151e5a 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1149,6 +1149,32 @@ class CustomStreamWrapper: completion_obj: Dict[str, Any] = {"content": ""} from litellm.types.utils import GenericStreamingChunk as GChunk + if ( + isinstance(chunk, ModelResponseStream) + and self.custom_llm_provider is not None + and self.custom_llm_provider in litellm._custom_providers + ): + _has_content = bool( + chunk.choices + and chunk.choices[0].delta is not None + and ( + chunk.choices[0].delta.content + or chunk.choices[0].delta.tool_calls + ) + ) + if self.received_finish_reason is not None: + if not _has_content: + raise StopIteration + if chunk.choices and chunk.choices[0].finish_reason: + self.received_finish_reason = chunk.choices[0].finish_reason + if not _has_content: + return None + # Strip finish_reason from the content chunk so it appears + # only on the trailing empty-delta chunk (OpenAI spec). + # finish_reason_handler() will emit the proper terminal chunk. + chunk.choices[0].finish_reason = None # type: ignore[assignment] + return chunk + if ( isinstance(chunk, dict) and generic_chunk_has_all_required_fields( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 4f15d1b3cef..7949e150c23 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -81,7 +81,6 @@ from litellm.types.utils import ( from litellm.utils import ( ModelResponse, Usage, - _supports_factory, add_dummy_tool, any_assistant_message_has_thinking_blocks, get_max_tokens, @@ -337,50 +336,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): v in model_lower for v in ("opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7") ) - @staticmethod - def _supports_model_capability(model: str, key: str) -> bool: - """Check a boolean capability ``key`` in the model map. - - Strips bedrock/vertex prefixes so a provider-routed Claude still - resolves to the Anthropic model-map entry. - """ - try: - if _supports_factory( - model=model, - custom_llm_provider="anthropic", - key=key, - ): - return True - except Exception: - pass - candidates = [model] - for prefix in ( - "bedrock/converse/", - "bedrock/invoke/", - "bedrock/", - "vertex_ai/", - ): - if model.startswith(prefix): - candidates.append(model[len(prefix) :]) - try: - from litellm.llms.bedrock.common_utils import BedrockModelInfo - - base = BedrockModelInfo.get_base_model(model) - if base: - candidates.append(base) - candidates.append(f"bedrock/{base}") - except Exception: - pass - try: - for cand in candidates: - if cand in litellm.model_cost and ( - litellm.model_cost[cand].get(key) is True - ): - return True - except Exception: - pass - return False - @staticmethod def _supports_effort_level(model: str, level: str) -> bool: """Check ``supports_{level}_reasoning_effort`` in the model map.""" @@ -918,7 +873,39 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_tools = [] mcp_servers = [] for tool in tools: - if "input_schema" in tool: # assume in anthropic format + if tool.get("type") == "namespace": + # Namespace is a grouping container (e.g. codex's multi_agent_v1). + # Extract its nested tools and map them individually. + for nested in tool.get("tools") or []: + if "input_schema" in nested: + # Already in Anthropic format. + anthropic_tools.append(nested) + elif "function" not in nested and "name" in nested: + # Flat format: {type, name, description, parameters, ...}. + # Normalize to OpenAI-wrapped format before mapping. + wrapped = cast( + ChatCompletionToolParam, + { + "type": nested.get("type", "function"), + "function": { + k: v for k, v in nested.items() if k != "type" + }, + }, + ) + nested_tool, nested_mcp = self._map_tool_helper(wrapped) + if nested_tool is not None: + anthropic_tools.append(nested_tool) + if nested_mcp is not None: + mcp_servers.append(nested_mcp) + elif "function" in nested: + nested_tool, nested_mcp = self._map_tool_helper( + cast(ChatCompletionToolParam, nested) + ) + if nested_tool is not None: + anthropic_tools.append(nested_tool) + if nested_mcp is not None: + mcp_servers.append(nested_mcp) + elif "input_schema" in tool: # assume in anthropic format anthropic_tools.append(tool) else: # assume openai tool call new_tool, mcp_server_tool = self._map_tool_helper(tool) @@ -1978,6 +1965,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Remove internal LiteLLM parameters that should not be sent to Anthropic API optional_params.pop("is_vertex_request", None) + optional_params.pop("client_metadata", None) data = { "model": model, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 31131d722ab..3f002d73cbc 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -272,19 +272,63 @@ class AnthropicModelInfo(BaseLLMModelInfo): ) @staticmethod - def _is_adaptive_thinking_model(model: str) -> bool: - """Claude 4.6+ models use adaptive thinking with ``output_config.effort``.""" + def _supports_model_capability(model: str, key: str) -> bool: + """Check a boolean capability ``key`` in the model map. + + Strips bedrock/vertex prefixes so a provider-routed Claude still + resolves to the Anthropic model-map entry. + """ from litellm.utils import _supports_factory try: if _supports_factory( model=model, - custom_llm_provider=None, - key="supports_adaptive_thinking", + custom_llm_provider="anthropic", + key=key, ): return True except Exception: pass + candidates = [model] + for prefix in ( + "bedrock/converse/", + "bedrock/invoke/", + "bedrock/", + "vertex_ai/", + ): + if model.startswith(prefix): + candidates.append(model[len(prefix) :]) + try: + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + base = BedrockModelInfo.get_base_model(model) + if base: + candidates.append(base) + candidates.append(f"bedrock/{base}") + except Exception: + pass + try: + for cand in candidates: + if cand in litellm.model_cost and ( + litellm.model_cost[cand].get(key) is True + ): + return True + except Exception: + pass + return False + + @staticmethod + def _is_adaptive_thinking_model(model: str) -> bool: + """Claude 4.6+ models use adaptive thinking with ``output_config.effort``. + + Driven by the ``supports_adaptive_thinking`` flag in the model map; the + 4.6/4.7 name checks remain only as a fallback for provider-routed ids + whose map entries predate the flag. + """ + if AnthropicModelInfo._supports_model_capability( + model, "supports_adaptive_thinking" + ): + return True return AnthropicModelInfo._is_claude_4_6_model( model ) or AnthropicModelInfo._is_claude_4_7_model(model) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 02e0c562654..150f056dc81 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1510,6 +1510,17 @@ class LiteLLMAnthropicMessagesAdapter: return "thinking", ChatCompletionThinkingBlock( type="thinking", thinking=thinking, signature=signature ) + # OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning + # parsers) populate ``reasoning_content`` without ``thinking_blocks``. + # ``Delta`` deletes the ``thinking_blocks`` attribute when unset, so the + # branch above is skipped entirely; open a ``thinking`` block here so the + # matching ``thinking_delta`` stream is not emitted into a text block. + elif isinstance(choice, StreamingChoices) and getattr( + choice.delta, "reasoning_content", None + ): + return "thinking", ChatCompletionThinkingBlock( + type="thinking", thinking="", signature="" + ) return "text", TextBlock(type="text", text="") diff --git a/litellm/llms/azure/image_edit/transformation.py b/litellm/llms/azure/image_edit/transformation.py index a450ee0b217..72f1eef36c0 100644 --- a/litellm/llms/azure/image_edit/transformation.py +++ b/litellm/llms/azure/image_edit/transformation.py @@ -97,8 +97,15 @@ class AzureImageEditConfig(OpenAIImageEditConfig): ) original_url = httpx.URL(api_base) - # Extract api_version or use default - api_version = cast(Optional[str], litellm_params.get("api_version")) + # Resolve api_version: litellm_params > litellm.api_version > AZURE_API_VERSION env > default. + # Mirrors the fallback chain used by the Azure chat path in common_utils.py, + # so callers that set a global / env api_version don't get an unversioned URL. + api_version = ( + cast(Optional[str], litellm_params.get("api_version")) + or litellm.api_version + or get_secret_str("AZURE_API_VERSION") + or litellm.AZURE_DEFAULT_API_VERSION + ) # Create a new dictionary with existing params query_params = dict(original_url.params) diff --git a/litellm/llms/bedrock_mantle/responses/__init__.py b/litellm/llms/bedrock_mantle/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py new file mode 100644 index 00000000000..b63fd0ecdb1 --- /dev/null +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -0,0 +1,81 @@ +""" +Amazon Bedrock Mantle - Responses API backend. + +gpt-5.5 / gpt-5.4 on Mantle are exposed ONLY on the `/openai/v1/responses` +path (not the standard `/v1/responses`). Payloads and SSE follow the OpenAI +Responses spec, so this config inherits OpenAIResponsesAPIConfig and overrides +only the endpoint URL and Bearer authentication. + +Auth: AWS Bedrock API key as Bearer token (BEDROCK_MANTLE_API_KEY or the +standard AWS_BEARER_TOKEN_BEDROCK), NOT SigV4. +""" + +from typing import Optional + +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +BEDROCK_MANTLE_DEFAULT_REGION = "us-east-1" + +# Checked longest/most-specific first so a full endpoint URL collapses to host +# in one pass and the appended path never doubles. +_BASE_SUFFIXES_TO_STRIP = ( + "/openai/v1/responses", + "/v1/responses", + "/responses", + "/openai/v1", + "/v1", +) + + +class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BEDROCK_MANTLE + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + region = ( + get_secret_str("BEDROCK_MANTLE_REGION") + or get_secret_str("AWS_REGION") + or BEDROCK_MANTLE_DEFAULT_REGION + ) + base = ( + api_base + or get_secret_str("BEDROCK_MANTLE_API_BASE") + or f"https://bedrock-mantle.{region}.api.aws" + ) + base = base.rstrip("/") + for suffix in _BASE_SUFFIXES_TO_STRIP: + if base.endswith(suffix): + base = base[: -len(suffix)] + break + return f"{base}/openai/v1/responses" + + def validate_environment( + self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + litellm_params = litellm_params or GenericLiteLLMParams() + api_key = ( + litellm_params.api_key + or get_secret_str("BEDROCK_MANTLE_API_KEY") + or get_secret_str("AWS_BEARER_TOKEN_BEDROCK") + ) + if not api_key: + raise ValueError( + "Bedrock Mantle API key is required. Set BEDROCK_MANTLE_API_KEY " + "(or AWS_BEARER_TOKEN_BEDROCK) or pass api_key." + ) + headers["Authorization"] = f"Bearer {api_key}" + return headers + + def supports_native_file_search(self) -> bool: + return False + + def supports_native_websocket(self) -> bool: + return False diff --git a/litellm/llms/cohere/chat/v2_transformation.py b/litellm/llms/cohere/chat/v2_transformation.py index 190491adfc7..9aa8c114907 100644 --- a/litellm/llms/cohere/chat/v2_transformation.py +++ b/litellm/llms/cohere/chat/v2_transformation.py @@ -120,6 +120,7 @@ class CohereV2ChatConfig(OpenAIGPTConfig): "stream", "temperature", "max_tokens", + "max_completion_tokens", "top_p", "frequency_penalty", "presence_penalty", @@ -143,7 +144,12 @@ class CohereV2ChatConfig(OpenAIGPTConfig): optional_params["stream"] = value if param == "temperature": optional_params["temperature"] = value - if param == "max_tokens": + if ( + param == "max_tokens" + and "max_completion_tokens" not in non_default_params + ): + optional_params["max_tokens"] = value + if param == "max_completion_tokens": optional_params["max_tokens"] = value if param == "n": optional_params["num_generations"] = value diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index e11d8532dbf..01c94476431 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -589,6 +589,7 @@ class AsyncHTTPHandler: params: Optional[dict] = None, headers: Optional[dict] = None, follow_redirects: Optional[bool] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, ): # Set follow_redirects to UseClientDefault if None _follow_redirects = ( @@ -599,7 +600,11 @@ class AsyncHTTPHandler: params.update(HTTPHandler.extract_query_params(url)) response = await self.client.get( - url, params=params, headers=headers, follow_redirects=_follow_redirects # type: ignore + url, + params=params, + headers=headers, # type: ignore + follow_redirects=_follow_redirects, # type: ignore + timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT, ) return response @@ -1115,6 +1120,7 @@ class HTTPHandler: params: Optional[dict] = None, headers: Optional[dict] = None, follow_redirects: Optional[bool] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, ): # Set follow_redirects to UseClientDefault if None _follow_redirects = ( @@ -1128,6 +1134,7 @@ class HTTPHandler: params=params, headers=headers, follow_redirects=_follow_redirects, + timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT, ) return response diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 8b3add5398b..31c772510ba 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1751,6 +1751,7 @@ class BaseLLMHTTPHandler: api_base=api_base, optional_params=optional_params, data=data, + api_key=api_key, ) ## LOGGING @@ -1833,6 +1834,7 @@ class BaseLLMHTTPHandler: api_base=api_base, optional_params=optional_params, data=data, + api_key=api_key, ) ## LOGGING @@ -2586,6 +2588,8 @@ class BaseLLMHTTPHandler: headers=headers, ) + headers.setdefault("Content-Type", "application/json") + ## LOGGING logging_obj.pre_call( input=input, @@ -2676,6 +2680,8 @@ class BaseLLMHTTPHandler: headers=headers, ) + headers.setdefault("Content-Type", "application/json") + ## LOGGING logging_obj.pre_call( input=input, @@ -5528,6 +5534,7 @@ class BaseLLMHTTPHandler: user_api_key_dict: Optional[Any] = None, litellm_metadata: Optional[Dict[str, Any]] = None, custom_llm_provider: Optional[str] = None, + first_message: Optional[str] = None, **kwargs: Any, ): """ @@ -5559,6 +5566,7 @@ class BaseLLMHTTPHandler: api_base=api_base, timeout=timeout, custom_llm_provider=custom_llm_provider, + first_message=first_message, **kwargs, ) await handler.run() @@ -5624,6 +5632,7 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, user_api_key_dict=user_api_key_dict, request_data=_request_data, + first_message=first_message, ) await streaming.bidirectional_forward() diff --git a/litellm/llms/fal_ai/image_generation/__init__.py b/litellm/llms/fal_ai/image_generation/__init__.py index 9deeb403c46..7f3358934a7 100644 --- a/litellm/llms/fal_ai/image_generation/__init__.py +++ b/litellm/llms/fal_ai/image_generation/__init__.py @@ -7,6 +7,7 @@ from .flux_pro_v11_transformation import FalAIFluxProV11Config from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig from .flux_schnell_transformation import FalAIFluxSchnellConfig from .imagen4_transformation import FalAIImagen4Config +from .nano_banana_transformation import FalAINanoBananaConfig from .recraft_v3_transformation import FalAIRecraftV3Config from .ideogram_v3_transformation import FalAIIdeogramV3Config from .stable_diffusion_transformation import FalAIStableDiffusionConfig @@ -20,6 +21,7 @@ __all__ = [ "FalAIBaseConfig", "FalAIImageGenerationConfig", "FalAIImagen4Config", + "FalAINanoBananaConfig", "FalAIRecraftV3Config", "FalAIBriaConfig", "FalAIFluxProV11Config", @@ -45,7 +47,9 @@ def get_fal_ai_image_generation_config(model: str) -> BaseImageGenerationConfig: model_lower = model.lower() # Map model names to their corresponding configuration classes - if "imagen4" in model_lower or "imagen-4" in model_lower: + if "nano-banana" in model_lower or "gemini-25-flash-image" in model_lower: + return FalAINanoBananaConfig() + elif "imagen4" in model_lower or "imagen-4" in model_lower: return FalAIImagen4Config() elif "recraft" in model_lower: return FalAIRecraftV3Config() diff --git a/litellm/llms/fal_ai/image_generation/nano_banana_transformation.py b/litellm/llms/fal_ai/image_generation/nano_banana_transformation.py new file mode 100644 index 00000000000..dd4758055ac --- /dev/null +++ b/litellm/llms/fal_ai/image_generation/nano_banana_transformation.py @@ -0,0 +1,105 @@ +from typing import List, Optional + +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams + +from .transformation import FalAIBaseConfig + + +class FalAINanoBananaConfig(FalAIBaseConfig): + """ + Configuration for Fal AI's Nano Banana / Gemini 2.5 Flash Image models. + + Serves the imagen4 deprecation migration path. The same underlying model is + exposed under two endpoints that share an identical schema: + - fal-ai/nano-banana + - fal-ai/gemini-25-flash-image + + Documentation: https://fal.ai/models/fal-ai/nano-banana + """ + + SUPPORTED_ASPECT_RATIOS: List[str] = [ + "21:9", + "16:9", + "3:2", + "4:3", + "5:4", + "1:1", + "4:5", + "3:4", + "2:3", + "9:16", + ] + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + base_url: str = ( + api_base or get_secret_str("FAL_AI_API_BASE") or self.DEFAULT_BASE_URL + ).rstrip("/") + endpoint = model if model.startswith("fal-ai/") else f"fal-ai/{model}" + return f"{base_url}/{endpoint}" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + return ["n", "response_format", "size"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + for key, value in non_default_params.items(): + if key == "response_format": + continue + elif key == "n": + if "num_images" not in optional_params: + optional_params["num_images"] = value + elif key == "size": + if "aspect_ratio" not in optional_params: + optional_params["aspect_ratio"] = self._map_aspect_ratio(value) + elif key not in optional_params and not drop_params: + raise ValueError( + f"Parameter {key} is not supported for model {model}. " + f"Supported parameters are {supported_params}. " + "Set drop_params=True to drop unsupported parameters." + ) + return optional_params + + def _map_aspect_ratio(self, size: str) -> str: + if not isinstance(size, str) or "x" not in size: + return "1:1" + try: + width, height = (int(part) for part in size.split("x")) + target = width / height + except (ValueError, ZeroDivisionError): + return "1:1" + + def ratio_of(aspect_ratio: str) -> float: + w, h = (int(part) for part in aspect_ratio.split(":")) + return w / h + + return min( + self.SUPPORTED_ASPECT_RATIOS, + key=lambda aspect_ratio: abs(ratio_of(aspect_ratio) - target), + ) + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + return {"prompt": prompt, **optional_params} diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 9e9d300b585..cca3b3da37a 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -170,11 +170,6 @@ class FireworksAIConfig(OpenAIGPTConfig): is_response_format_supported=False, enforce_tool_choice=False, # tools and response_format are both set, don't enforce tool_choice ) - elif "json_schema" in value: - optional_params["response_format"] = { - "type": "json_object", - "schema": value["json_schema"]["schema"], - } else: optional_params["response_format"] = value elif param == "max_completion_tokens": diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index b69b7e1913e..4e9764446c9 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -93,6 +93,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): "modalities", "parallel_tool_calls", "web_search_options", + "include_server_side_tool_invocations", "service_tier", ] if supports_reasoning(model, custom_llm_provider="gemini"): diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index bc963d62b5f..42a807983b9 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -1,6 +1,8 @@ import base64 import datetime -from typing import Any, Dict, List, Optional, Union +import json +import math +from typing import Any, Dict, List, Optional, Sequence, Union import httpx @@ -12,6 +14,245 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import TokenCountResponse +GEMINI_IMAGE_ASPECT_RATIOS: Dict[str, float] = { + "1:1": 1 / 1, + "1:4": 1 / 4, + "1:8": 1 / 8, + "2:3": 2 / 3, + "3:2": 3 / 2, + "3:4": 3 / 4, + "4:1": 4 / 1, + "4:3": 4 / 3, + "4:5": 4 / 5, + "5:4": 5 / 4, + "8:1": 8 / 1, + "9:16": 9 / 16, + "16:9": 16 / 9, + "21:9": 21 / 9, +} + +# Supported aspect ratio dimensions from Google Gemini image generation docs: +# https://ai.google.dev/gemini-api/docs/image-generation#aspect_ratios_and_image_size +GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO: Dict[tuple[int, int], str] = { + (512, 512): "1:1", + (1024, 1024): "1:1", + (2048, 2048): "1:1", + (4096, 4096): "1:1", + (256, 1024): "1:4", + (512, 2048): "1:4", + (1024, 4096): "1:4", + (2048, 8192): "1:4", + (192, 1536): "1:8", + (384, 3072): "1:8", + (768, 6144): "1:8", + (1536, 12288): "1:8", + (424, 632): "2:3", + (848, 1264): "2:3", + (1696, 2528): "2:3", + (3392, 5056): "2:3", + (632, 424): "3:2", + (1264, 848): "3:2", + (2528, 1696): "3:2", + (5056, 3392): "3:2", + (448, 600): "3:4", + (896, 1200): "3:4", + (1792, 2400): "3:4", + (3584, 4800): "3:4", + (1024, 256): "4:1", + (2048, 512): "4:1", + (4096, 1024): "4:1", + (8192, 2048): "4:1", + (600, 448): "4:3", + (1200, 896): "4:3", + (2400, 1792): "4:3", + (4800, 3584): "4:3", + (464, 576): "4:5", + (928, 1152): "4:5", + (1856, 2304): "4:5", + (3712, 4608): "4:5", + (576, 464): "5:4", + (1152, 928): "5:4", + (2304, 1856): "5:4", + (4608, 3712): "5:4", + (1536, 192): "8:1", + (3072, 384): "8:1", + (6144, 768): "8:1", + (12288, 1536): "8:1", + (384, 688): "9:16", + (768, 1376): "9:16", + (1536, 2752): "9:16", + (3072, 5504): "9:16", + (688, 384): "16:9", + (1376, 768): "16:9", + (2752, 1536): "16:9", + (5504, 3072): "16:9", + (792, 336): "21:9", + (1584, 672): "21:9", + (3168, 1344): "21:9", + (6336, 2688): "21:9", + (1280, 896): "4:3", + (896, 1280): "3:4", +} + + +def map_openai_size_to_gemini_image_config( + size: str, model: str +) -> Optional[Dict[str, str]]: + dimensions = _parse_openai_image_size(size) + if dimensions is None: + return None + + width, height = dimensions + image_config = { + "aspectRatio": _map_dimensions_to_gemini_aspect_ratio(width, height) + } + image_size = _map_dimensions_to_gemini_image_size(width, height) + if is_gemini_image_model(model): + if supports_gemini_image_size(model): + image_config["imageSize"] = image_size + else: + image_config["imageSize"] = image_size + return image_config + + +def supports_gemini_image_size(model: str) -> bool: + try: + model_info = litellm.get_model_info(model=model) + value = model_info.get("supports_image_size") + if value is not None: + return bool(value) + except Exception: + pass + return "2.5-flash" not in model + + +def is_gemini_image_model(model: str) -> bool: + base_model = model.split("/", 1)[-1] + return "gemini" in base_model + + +def map_openai_image_params_to_gemini( + params: Dict[str, Any], + model: str, + supported_params: Sequence[str], + optional_params: Optional[Dict[str, Any]] = None, + parse_image_config_string: bool = False, +) -> Dict[str, Any]: + optional_params = optional_params or {} + filtered_params = { + key: value for key, value in params.items() if key in supported_params + } + + mapped_params: Dict[str, Any] = {} + + if "n" in filtered_params and "n" not in optional_params: + mapped_params["sampleCount"] = filtered_params["n"] + + if "size" in filtered_params and "size" not in optional_params: + image_config = map_openai_size_to_gemini_image_config( + filtered_params["size"], + model, + ) + if image_config is not None: + if is_gemini_image_model(model): + mapped_params["imageConfig"] = image_config + else: + mapped_params["aspectRatio"] = image_config["aspectRatio"] + if "imageSize" in image_config: + mapped_params["imageSize"] = image_config["imageSize"] + + image_config_param = filtered_params.get("imageConfig") + if isinstance(image_config_param, str) and parse_image_config_string: + try: + image_config_param = json.loads(image_config_param) + except json.JSONDecodeError as exc: + raise litellm.UnsupportedParamsError( + model=model, + message="`imageConfig` must be valid JSON when provided as a string.", + ) from exc + if isinstance(image_config_param, dict): + mapped_params["imageConfig"] = image_config_param + + for key, value in filtered_params.items(): + if key not in ("n", "size", "imageConfig") and key not in optional_params: + mapped_params[key] = value + + return mapped_params + + +def get_gemini_image_generation_config( + model: str, + optional_params: Dict[str, Any], +) -> Dict[str, Any]: + generation_config: Dict[str, Any] = {"response_modalities": ["IMAGE", "TEXT"]} + + image_config: Dict[str, Any] = {} + if isinstance(optional_params.get("imageConfig"), dict): + image_config.update(optional_params["imageConfig"]) + + if not supports_gemini_image_size(model): + image_config.pop("imageSize", None) + + if image_config: + generation_config["imageConfig"] = image_config + + candidate_count = next( + ( + optional_params[key] + for key in ("candidateCount", "candidate_count", "sampleCount", "n") + if optional_params.get(key) is not None + ), + None, + ) + if candidate_count is not None: + generation_config["candidateCount"] = candidate_count + + return generation_config + + +def _parse_openai_image_size(size: str) -> Optional[tuple[int, int]]: + if size == "auto": + return None + + width_str, separator, height_str = size.lower().partition("x") + if not separator: + return None + + try: + width = int(width_str) + height = int(height_str) + except ValueError: + return None + + if width <= 0 or height <= 0: + return None + + return width, height + + +def _map_dimensions_to_gemini_aspect_ratio(width: int, height: int) -> str: + if (width, height) in GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO: + return GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO[(width, height)] + + requested_ratio = width / height + return min( + GEMINI_IMAGE_ASPECT_RATIOS, + key=lambda aspect_ratio: abs( + math.log(GEMINI_IMAGE_ASPECT_RATIOS[aspect_ratio] / requested_ratio) + ), + ) + + +def _map_dimensions_to_gemini_image_size(width: int, height: int) -> str: + effective_square_side = math.sqrt(width * height) + if effective_square_side < 768: + return "512" + if effective_square_side < 1536: + return "1K" + if effective_square_side < 3072: + return "2K" + return "4K" + class GeminiError(BaseLLMException): pass diff --git a/litellm/llms/gemini/image_edit/cost_calculator.py b/litellm/llms/gemini/image_edit/cost_calculator.py index 2e332a7fc00..956edb849a0 100644 --- a/litellm/llms/gemini/image_edit/cost_calculator.py +++ b/litellm/llms/gemini/image_edit/cost_calculator.py @@ -4,8 +4,9 @@ Gemini Image Edit Cost Calculator from typing import Any -import litellm -from litellm.types.utils import ImageResponse +from litellm.llms.gemini.image_generation.cost_calculator import ( + cost_calculator as image_generation_cost_calculator, +) def cost_calculator( @@ -15,20 +16,10 @@ def cost_calculator( """ Gemini image edit cost calculator. - Mirrors image generation pricing: charge per returned image based on - model metadata (`output_cost_per_image`). + Gemini image edits and generations share image response billing behavior: + use provider token usage when present, otherwise fall back to per-image pricing. """ - model_info = litellm.get_model_info( + return image_generation_cost_calculator( model=model, - custom_llm_provider="gemini", + image_response=image_response, ) - - output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 - - if not isinstance(image_response, ImageResponse): - raise ValueError( - f"image_response must be of type ImageResponse got type={type(image_response)}" - ) - - num_images = len(image_response.data or []) - return output_cost_per_image * num_images diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py index c8aaab0e14e..2316361d6e7 100644 --- a/litellm/llms/gemini/image_edit/transformation.py +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -7,10 +7,22 @@ from httpx._types import RequestFiles from litellm.images.utils import ImageEditRequestUtils from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.gemini.common_utils import ( + get_gemini_image_generation_config, + map_openai_image_params_to_gemini, +) +from litellm.llms.gemini.image_usage_transformation import ( + transform_gemini_image_usage, +) from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage +from litellm.types.utils import ( + FileTypes, + ImageObject, + ImageResponse, + OpenAIImage, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -22,7 +34,7 @@ else: class GeminiImageEditConfig(BaseImageEditConfig): DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta" - SUPPORTED_PARAMS: List[str] = ["size"] + SUPPORTED_PARAMS: List[str] = ["n", "size", "imageConfig"] def get_supported_openai_params(self, model: str) -> List[str]: return list(self.SUPPORTED_PARAMS) @@ -33,21 +45,12 @@ class GeminiImageEditConfig(BaseImageEditConfig): model: str, drop_params: bool, ) -> Dict[str, Any]: - supported_params = self.get_supported_openai_params(model) - filtered_params = { - key: value - for key, value in image_edit_optional_params.items() - if key in supported_params - } - - mapped_params: Dict[str, Any] = {} - - if "size" in filtered_params: - mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio( - filtered_params["size"] # type: ignore[arg-type] - ) - - return mapped_params + return map_openai_image_params_to_gemini( + params=image_edit_optional_params, # type: ignore[arg-type] + model=model, + supported_params=self.get_supported_openai_params(model), + parse_image_config_string=True, + ) def validate_environment( self, @@ -107,18 +110,10 @@ class GeminiImageEditConfig(BaseImageEditConfig): request_body: Dict[str, Any] = {"contents": contents} - generation_config: Dict[str, Any] = {} - - if "aspectRatio" in image_edit_optional_request_params: - # Move aspectRatio into imageConfig inside generationConfig - if "imageConfig" not in generation_config: - generation_config["imageConfig"] = {} - generation_config["imageConfig"]["aspectRatio"] = ( - image_edit_optional_request_params["aspectRatio"] - ) - - if generation_config: - request_body["generationConfig"] = generation_config + request_body["generationConfig"] = get_gemini_image_generation_config( + model=model, + optional_params=image_edit_optional_request_params, + ) empty_files = cast(RequestFiles, []) return request_body, empty_files @@ -156,18 +151,12 @@ class GeminiImageEditConfig(BaseImageEditConfig): ) model_response.data = cast(List[OpenAIImage], data_list) + if "usageMetadata" in response_json: + model_response.usage = transform_gemini_image_usage( + response_json["usageMetadata"] + ) return model_response - def _map_size_to_aspect_ratio(self, size: str) -> str: - aspect_ratio_map = { - "1024x1024": "1:1", - "1792x1024": "16:9", - "1024x1792": "9:16", - "1280x896": "4:3", - "896x1280": "3:4", - } - return aspect_ratio_map.get(size, "1:1") - def _prepare_inline_image_parts( self, image: Union[FileTypes, List[FileTypes]] ) -> List[Dict[str, Any]]: diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index 9c4cd008b8c..e6770a76bcb 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -5,18 +5,21 @@ import httpx from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) +from litellm.llms.gemini.common_utils import ( + get_gemini_image_generation_config, + is_gemini_image_model, + map_openai_image_params_to_gemini, +) +from litellm.llms.gemini.image_usage_transformation import ( + transform_gemini_image_usage, +) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.gemini import GeminiImageGenerationRequest from litellm.types.llms.openai import ( AllMessageValues, OpenAIImageGenerationOptionalParams, ) -from litellm.types.utils import ( - ImageObject, - ImageResponse, - ImageUsage, - ImageUsageInputTokensDetails, -) +from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -36,7 +39,10 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): Google AI Imagen API supported parameters https://ai.google.dev/gemini-api/docs/imagen """ - return ["n", "size"] + supported_params = ["n", "size"] + if is_gemini_image_model(model): + supported_params.append("imageConfig") + return supported_params # type: ignore[return-value] def map_openai_params( self, @@ -45,64 +51,11 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): model: str, drop_params: bool, ) -> dict: - supported_params = self.get_supported_openai_params(model) - mapped_params = {} - - for k, v in non_default_params.items(): - if k not in optional_params.keys(): - if k in supported_params: - # Map OpenAI parameters to Google format - if k == "n": - mapped_params["sampleCount"] = v - elif k == "size": - # Map OpenAI size format to Google aspectRatio - mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v) - else: - mapped_params[k] = v - return mapped_params - - def _map_size_to_aspect_ratio(self, size: str) -> str: - """ - https://ai.google.dev/gemini-api/docs/image-generation - - """ - aspect_ratio_map = { - "1024x1024": "1:1", - "1792x1024": "16:9", - "1024x1792": "9:16", - "1280x896": "4:3", - "896x1280": "3:4", - } - return aspect_ratio_map.get(size, "1:1") - - def _transform_image_usage(self, usage_metadata: dict) -> ImageUsage: - """ - Transform Gemini usageMetadata to ImageUsage format - """ - input_tokens_details = ImageUsageInputTokensDetails( - image_tokens=0, - text_tokens=0, - ) - - # Extract detailed token counts from promptTokensDetails - tokens_details = usage_metadata.get("promptTokensDetails", []) - for details in tokens_details: - if isinstance(details, dict): - modality = str(details.get("modality", "")).upper() - raw_token_count = details.get( - "tokenCount", details.get("token_count", 0) - ) - token_count = raw_token_count if isinstance(raw_token_count, int) else 0 - if modality == "TEXT": - input_tokens_details.text_tokens += token_count - elif modality == "IMAGE": - input_tokens_details.image_tokens += token_count - - return ImageUsage( - input_tokens=usage_metadata.get("promptTokenCount", 0), - input_tokens_details=input_tokens_details, - output_tokens=usage_metadata.get("candidatesTokenCount", 0), - total_tokens=usage_metadata.get("totalTokenCount", 0), + return map_openai_image_params_to_gemini( + params=non_default_params, + model=model, + supported_params=self.get_supported_openai_params(model), + optional_params=optional_params, ) def get_complete_url( @@ -127,7 +80,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): complete_url = complete_url.rstrip("/") # Gemini Flash Image Preview models use generateContent endpoint - if "gemini" in model: + if is_gemini_image_model(model): complete_url = f"{complete_url}/models/{model}:generateContent" else: # All other Imagen models use predict endpoint @@ -179,10 +132,13 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): } """ # For Gemini Flash Image Preview models, use standard Gemini format - if "gemini" in model: + if is_gemini_image_model(model): request_body: dict = { "contents": [{"parts": [{"text": prompt}]}], - "generationConfig": {"response_modalities": ["IMAGE", "TEXT"]}, + "generationConfig": get_gemini_image_generation_config( + model=model, + optional_params=optional_params, + ), } return request_body else: @@ -200,6 +156,9 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): ) return request_body_obj.model_dump(exclude_none=True) + def _transform_image_usage(self, usage_metadata: dict): + return transform_gemini_image_usage(usage_metadata) + def transform_image_generation_response( self, model: str, @@ -229,7 +188,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): model_response.data = [] # Handle different response formats based on model - if "gemini" in model: + if is_gemini_image_model(model): # Gemini Flash Image Preview models return in candidates format candidates = response_data.get("candidates", []) for candidate in candidates: @@ -255,7 +214,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): # Extract usage metadata for Gemini models if "usageMetadata" in response_data: - model_response.usage = self._transform_image_usage( + model_response.usage = transform_gemini_image_usage( response_data["usageMetadata"] ) else: diff --git a/litellm/llms/gemini/image_usage_transformation.py b/litellm/llms/gemini/image_usage_transformation.py new file mode 100644 index 00000000000..5a55bdeffb1 --- /dev/null +++ b/litellm/llms/gemini/image_usage_transformation.py @@ -0,0 +1,73 @@ +from typing import Any + +from litellm.types.utils import ImageUsage, ImageUsageInputTokensDetails + + +def _get_token_count(details: dict) -> int: + raw_token_count = details.get("tokenCount", details.get("token_count", 0)) + return raw_token_count if isinstance(raw_token_count, int) else 0 + + +def _get_modality_token_details(usage_metadata: dict, *details_keys: str) -> list: + for details_key in details_keys: + details = usage_metadata.get(details_key) + if isinstance(details, list): + return details + return [] + + +def _sum_modality_token_details( + usage_metadata: dict, *details_keys: str +) -> ImageUsageInputTokensDetails: + tokens_details = ImageUsageInputTokensDetails( + image_tokens=0, + text_tokens=0, + ) + + for details in _get_modality_token_details(usage_metadata, *details_keys): + if isinstance(details, dict): + modality = str(details.get("modality", "")).upper() + token_count = _get_token_count(details) + if modality == "TEXT": + tokens_details.text_tokens += token_count + elif modality == "IMAGE": + tokens_details.image_tokens += token_count + + return tokens_details + + +def transform_gemini_image_usage(usage_metadata: dict) -> ImageUsage: + """ + Transform Gemini usageMetadata to ImageUsage format. + """ + input_tokens_details = _sum_modality_token_details( + usage_metadata, "promptTokensDetails", "prompt_tokens_details" + ) + output_tokens = usage_metadata.get("candidatesTokenCount", 0) + output_tokens_details = _sum_modality_token_details( + usage_metadata, "candidatesTokensDetails", "candidates_tokens_details" + ) + + if not _get_modality_token_details( + usage_metadata, "candidatesTokensDetails", "candidates_tokens_details" + ): + output_tokens_details.image_tokens = output_tokens + else: + known_output_tokens = ( + output_tokens_details.text_tokens + output_tokens_details.image_tokens + ) + if output_tokens > known_output_tokens: + output_tokens_details.text_tokens += output_tokens - known_output_tokens + + usage_payload: dict[str, Any] = { + "input_tokens": usage_metadata.get("promptTokenCount", 0), + "input_tokens_details": input_tokens_details, + "output_tokens": output_tokens, + "total_tokens": usage_metadata.get("totalTokenCount", 0), + "prompt_tokens": usage_metadata.get("promptTokenCount", 0), + "prompt_tokens_details": input_tokens_details.model_dump(), + "completion_tokens": output_tokens, + "completion_tokens_details": output_tokens_details.model_dump(), + "output_tokens_details": output_tokens_details.model_dump(), + } + return ImageUsage(**usage_payload) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index cf1fc75ef10..212287fb7f8 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -27,7 +27,6 @@ from litellm.types.llms.gemini import ( ) from litellm.types.llms.openai import ( OpenAIRealtimeContentPartDone, - OpenAIRealtimeConversationItemCreated, OpenAIRealtimeDoneEvent, OpenAIRealtimeEvents, OpenAIRealtimeEventTypes, @@ -79,6 +78,12 @@ _KNOWN_GEMINI_TOP_LEVEL_KEYS: set = { map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT } +# Gemini Live native-audio model ids carry this marker (e.g. +# ``gemini-2.5-flash-native-audio-preview-09-2025``). These models reject a +# ``speechConfig`` on ``setup`` with a 1007 invalid-argument error, so it is +# stripped in ``_finalize_gemini_live_setup``. +_GEMINI_NATIVE_AUDIO_MODEL_MARKER = "native-audio" + class GeminiRealtimeConfig(BaseRealtimeConfig): # Cap the LRU of in-flight tool calls so long sessions with many tool @@ -98,6 +103,33 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # bypassing spend and budget accounting. self._pending_usage_metadata: Optional[dict] = None + @staticmethod + def _usage_detail_alias(details: Any, defaults: Dict[str, int]) -> Dict[str, Any]: + if not isinstance(details, dict): + return dict(defaults) + return { + **defaults, + **{key: value for key, value in details.items() if value is not None}, + } + + @staticmethod + def _add_pipecat_usage_detail_aliases(usage_dict: Dict[str, Any]) -> Dict[str, Any]: + usage_dict.setdefault( + "input_token_details", + GeminiRealtimeConfig._usage_detail_alias( + usage_dict.get("input_tokens_details"), + {"cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0}, + ), + ) + usage_dict.setdefault( + "output_token_details", + GeminiRealtimeConfig._usage_detail_alias( + usage_dict.get("output_tokens_details"), + {"text_tokens": 0, "audio_tokens": 0}, + ), + ) + return usage_dict + def validate_environment( self, headers: dict, model: str, api_key: Optional[str] = None ) -> dict: @@ -173,9 +205,25 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def map_automatic_turn_detection( self, value: OpenAIRealtimeTurnDetection ) -> AutomaticActivityDetection: + """Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``. + + OpenAI ``semantic_vad`` has no Gemini Live equivalent — return an empty + dict so callers omit ``realtimeInputConfig`` (mapping it with + ``disabled: true`` breaks native-audio sessions). + """ + if ( + isinstance(value, dict) + and value.get("type") == "semantic_vad" + and "create_response" not in value + ): + return AutomaticActivityDetection() + automatic_activity_dection = AutomaticActivityDetection() if "create_response" in value and isinstance(value["create_response"], bool): automatic_activity_dection["disabled"] = not value["create_response"] + elif isinstance(value, dict) and value.get("type") == "server_vad": + # OpenAI server VAD enables activity detection by default. + automatic_activity_dection["disabled"] = False else: automatic_activity_dection["disabled"] = True if "prefix_padding_ms" in value and isinstance(value["prefix_padding_ms"], int): @@ -197,6 +245,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): "tools", "input_audio_transcription", "turn_detection", + "voice", ] def map_openai_params( @@ -231,17 +280,33 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): optional_params["inputAudioTranscription"] = {} elif key == "turn_detection": value_typed = cast(OpenAIRealtimeTurnDetection, value) + if ( + isinstance(value_typed, dict) + and value_typed.get("type") == "semantic_vad" + and "create_response" not in value_typed + ): + # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD. + # Only skip when there is no create_response override so that + # a guardrail-injected create_response:false is not dropped. + continue transformed_audio_activity_config = self.map_automatic_turn_detection( value_typed ) - if ( - len(transformed_audio_activity_config) > 0 - ): # if the config is not empty, add it to the optional params + if transformed_audio_activity_config: optional_params["realtimeInputConfig"] = ( BidiGenerateContentRealtimeInputConfig( automaticActivityDetection=transformed_audio_activity_config ) ) + elif key == "voice": + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + vertex_gemini_config = VertexGeminiConfig() + speech_config = vertex_gemini_config._map_audio_params({"voice": value}) + if speech_config: + optional_params["generationConfig"]["speechConfig"] = speech_config if len(optional_params["generationConfig"]) == 0: optional_params.pop("generationConfig") return optional_params @@ -297,6 +362,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): and "transcription" in input_cfg ): normalized["input_audio_transcription"] = input_cfg["transcription"] + output_cfg = audio.get("output") + if isinstance(output_cfg, dict) and output_cfg.get("voice"): + normalized["voice"] = output_cfg["voice"] extracted_turn_detection = GeminiRealtimeConfig._extract_turn_detection( normalized @@ -308,6 +376,18 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return normalized + @staticmethod + def _finalize_gemini_live_setup( + model: str, setup: Dict[str, Any] + ) -> Dict[str, Any]: + """Drop fields Gemini Live native-audio rejects on ``setup``.""" + if _GEMINI_NATIVE_AUDIO_MODEL_MARKER not in model.lower(): + return setup + generation_config = setup.get("generationConfig") + if isinstance(generation_config, dict): + generation_config.pop("speechConfig", None) + return setup + def _handle_session_update( self, json_message: dict, @@ -351,7 +431,11 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): verbose_logger.debug( "Gemini Realtime: Sending initial setup with tools to backend" ) - return [json.dumps({"setup": new_overrides})] + return [ + json.dumps( + {"setup": self._finalize_gemini_live_setup(model, new_overrides)} + ) + ] if not new_overrides: verbose_logger.debug( @@ -420,7 +504,15 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): verbose_logger.debug( "Gemini Realtime: Forwarding session.update as follow-up setup" ) - return [json.dumps({"setup": follow_up_setup})] + return [ + json.dumps( + { + "setup": self._finalize_gemini_live_setup( + model, cast(Dict[str, Any], follow_up_setup) + ) + } + ) + ] def _handle_conversation_item(self, json_message: dict) -> List[str]: """ @@ -666,6 +758,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): "object": "realtime.response", "id": response_id, "status": "in_progress", + "status_details": None, "output": [], "conversation_id": conversation_id, "modalities": _modalities, @@ -675,9 +768,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) response_items.append(response_created) - ## - return response.output_item.added ← adds ‘item_id’ same for all subsequent events + ## - return response.output_item.added response_output_item_added = OpenAIRealtimeStreamResponseOutputItemAdded( type="response.output_item.added", + event_id="event_{}".format(uuid.uuid4()), response_id=response_id, output_index=0, item={ @@ -690,20 +784,28 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): }, ) response_items.append(response_output_item_added) - ## - return conversation.item.created - conversation_item_created = OpenAIRealtimeConversationItemCreated( - type="conversation.item.created", - event_id="event_{}".format(uuid.uuid4()), - item={ - "id": output_item_id, - "object": "realtime.item", - "type": "message", - "status": "in_progress", - "role": "assistant", - "content": [], - }, + ## - return conversation.item.added + # Pipecat 1.3.x handles "conversation.item.added" (not ".created"). + # Sending ".created" raises "Unimplemented server event type" which + # kills the receive task handler. + response_items.append( + cast( + OpenAIRealtimeEvents, + { + "type": "conversation.item.added", + "event_id": "event_{}".format(uuid.uuid4()), + "previous_item_id": None, + "item": { + "id": output_item_id, + "object": "realtime.item", + "type": "message", + "status": "in_progress", + "role": "assistant", + "content": [], + }, + }, + ) ) - response_items.append(conversation_item_created) ## - return response.content_part.added response_content_part_added = OpenAIRealtimeResponseContentPartAdded( type="response.content_part.added", @@ -749,9 +851,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return OpenAIRealtimeResponseDelta( type=( - "response.text.delta" + "response.output_text.delta" if delta_type == "text" - else "response.audio.delta" + else "response.output_audio.delta" ), content_index=0, event_id="event_{}".format(uuid.uuid4()), @@ -778,7 +880,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): current_response_id = "resp_{}".format(uuid.uuid4()) if delta_type == "text": return OpenAIRealtimeResponseTextDone( - type="response.text.done", + type="response.output_text.done", content_index=0, event_id="event_{}".format(uuid.uuid4()), item_id=current_output_item_id, @@ -788,7 +890,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) elif delta_type == "audio": return OpenAIRealtimeResponseAudioDone( - type="response.audio.done", + type="response.output_audio.done", content_index=0, event_id="event_{}".format(uuid.uuid4()), item_id=current_output_item_id, @@ -914,7 +1016,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): events: List[OpenAIRealtimeFunctionCallArgumentsDone] = [] for idx, fc in enumerate(function_calls): - call_id = fc.get("id", "") + call_id = fc.get("id", "") or f"call_{uuid.uuid4().hex[:16]}" name = fc.get("name", "") # Store call_id → name mapping for round-trip. Use an LRU so @@ -962,7 +1064,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): current_delta_chunks = [] any_delta_chunk = False for event in transformed_message: - if event["type"] == "response.text.delta": + if event["type"] == "response.output_text.delta": current_delta_chunks.append( cast(OpenAIRealtimeResponseDelta, event) ) @@ -973,7 +1075,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) else: if ( - transformed_message["type"] == "response.text.delta" + transformed_message["type"] == "response.output_text.delta" ): # ONLY ACCUMULATE TEXT DELTA CHUNKS - AUDIO WILL CAUSE SERVER MEMORY ISSUES if current_delta_chunks is None: current_delta_chunks = [] @@ -1067,6 +1169,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): responses_api_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( _chat_completion_usage, ) + _usage_dict = responses_api_usage.model_dump() + self._add_pipecat_usage_detail_aliases(_usage_dict) response_done_event = OpenAIRealtimeDoneEvent( type="response.done", event_id="event_{}".format(uuid.uuid4()), @@ -1074,6 +1178,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): object="realtime.response", id=current_response_id, status="completed", + status_details=None, # type: ignore[typeddict-item] output=( [output_item["item"] for output_item in output_items] if output_items @@ -1081,7 +1186,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ), conversation_id=current_conversation_id, modalities=_modalities, - usage=responses_api_usage.model_dump(), + usage=_usage_dict, ), ) if temperature is not None: @@ -1294,19 +1399,36 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): output_tx = server_content.get("outputTranscription") if isinstance(output_tx, dict) and output_tx.get("text"): + if current_response_id is None: + current_response_id = "resp_{}".format(uuid.uuid4()) + if current_output_item_id is None: + current_output_item_id = "item_{}".format(uuid.uuid4()) + current_conversation_id = ( + current_conversation_id or "conv_{}".format(uuid.uuid4()) + ) + returned_message.extend( + self.return_new_content_delta_events( + session_configuration_request=session_configuration_request, + response_id=current_response_id, + output_item_id=current_output_item_id, + conversation_id=current_conversation_id, + delta_type="audio", + ) + ) + # Emit as the GA event name; _GA_TO_BETA_EVENT_TYPES translates + # this back to response.audio_transcript.delta for beta clients. returned_message.append( cast( OpenAIRealtimeEvents, { - "type": "response.audio_transcript.delta", + "type": "response.output_audio_transcript.delta", "event_id": "event_{}".format(uuid.uuid4()), - "delta": output_tx["text"], - "item_id": current_output_item_id - or "item_{}".format(uuid.uuid4()), - "response_id": current_response_id - or "resp_{}".format(uuid.uuid4()), - "output_index": 0, + "transcript": output_tx["text"], + "item_id": current_output_item_id, "content_index": 0, + "output_index": 0, + "response_id": current_response_id, + "delta": output_tx["text"], }, ) ) @@ -1416,6 +1538,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): "object": "realtime.response", "id": current_response_id, "status": "in_progress", + "status_details": None, "output": [], "conversation_id": current_conversation_id, "modalities": tool_call_modalities, @@ -1460,6 +1583,29 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): }, ) ) + # conversation.item.added — Pipecat 1.3.x registers the + # call_id into _pending_function_calls inside + # _handle_evt_conversation_item_added, which is triggered + # by this event (NOT by response.output_item.added and NOT + # by the old conversation.item.created which Pipecat 1.3.x + # does not handle). Without this event the subsequent + # response.function_call_arguments.done finds an empty + # pending-calls dict and drops the tool invocation silently. + returned_message.append( + cast( + OpenAIRealtimeEvents, + { + "type": "conversation.item.added", + "event_id": f"event_{uuid.uuid4()}", + "previous_item_id": None, + "item": { + **function_call_item, + "status": "in_progress", + "arguments": "", + }, + }, + ) + ) # response.function_call_arguments.delta — Gemini delivers # the full arguments string in a single toolCall frame # rather than streaming partial chunks, so emit one delta @@ -1496,14 +1642,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): item={**function_call_item}, ) ) - # conversation.item.created - returned_message.append( - OpenAIRealtimeConversationItemCreated( - type="conversation.item.created", - event_id=f"event_{uuid.uuid4()}", - item={**function_call_item}, - ) - ) # response.done - close the response so clients can submit tool # results. Mirror the non-tool-call RESPONSE_DONE path: if Gemini @@ -1537,6 +1675,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): tool_call_responses_api_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( _tool_call_chat_completion_usage, ) + _tool_usage_dict = tool_call_responses_api_usage.model_dump() + self._add_pipecat_usage_detail_aliases(_tool_usage_dict) tool_call_done_event = OpenAIRealtimeDoneEvent( type="response.done", event_id=f"event_{uuid.uuid4()}", @@ -1544,6 +1684,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): id=current_response_id, object="realtime.response", status="completed", + status_details=None, # type: ignore[typeddict-item] output=[ { "id": te["item_id"], @@ -1558,7 +1699,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ], conversation_id=current_conversation_id, modalities=tool_call_modalities, - usage=tool_call_responses_api_usage.model_dump(), + usage=_tool_usage_dict, ), ) tool_call_temperature = tool_call_generation_config.get("temperature") diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index 77a95bfa5ab..644e96a7dd1 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -265,7 +265,11 @@ class GeminiVideoConfig(BaseVideoConfig): { "instances": [ { - "prompt": "A cat playing with a ball of yarn" + "prompt": "A cat playing with a ball of yarn", + "image": { + "bytesBase64Encoded": "...", + "mimeType": "image/jpeg" + } } ], "parameters": { @@ -275,13 +279,18 @@ class GeminiVideoConfig(BaseVideoConfig): } } """ - instance = GeminiVideoGenerationInstance(prompt=prompt) + instance: GeminiVideoGenerationInstance = {"prompt": prompt} params_copy = video_create_optional_request_params.copy() - if "image" in params_copy and params_copy["image"] is not None: - image_data = _convert_image_to_gemini_format(params_copy["image"]) - params_copy["image"] = image_data + if "image" in params_copy: + image = params_copy.pop("image") + if image is not None: + if isinstance(image, dict): + image_data = image + else: + image_data = _convert_image_to_gemini_format(image) + instance["image"] = image_data parameters = GeminiVideoGenerationParameters(**params_copy) diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index 226f6b2ebad..6be885b1f91 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -239,7 +239,7 @@ class HuggingFaceEmbedding(BaseLLM): model_response.model = model input_tokens = 0 for text in input: - input_tokens += len(encoding.encode(text)) + input_tokens += len(encoding.encode(text, disallowed_special=())) setattr( model_response, diff --git a/litellm/llms/inception/__init__.py b/litellm/llms/inception/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/inception/chat/__init__.py b/litellm/llms/inception/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/inception/chat/transformation.py b/litellm/llms/inception/chat/transformation.py new file mode 100644 index 00000000000..d591f783a99 --- /dev/null +++ b/litellm/llms/inception/chat/transformation.py @@ -0,0 +1,54 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Inception's `/v1/chat/completions` + +Inception Labs (https://www.inceptionlabs.ai) serves the Mercury family of +diffusion LLMs through an OpenAI-compatible API, so we only need to point the +OpenAI-like handler at the Inception API base and pick up the Inception API key. +""" + +from typing import List, Optional, Tuple + +import litellm +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class InceptionChatConfig(OpenAILikeChatConfig): + """ + Inception is OpenAI-compatible with standard endpoints + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "inception" + + def get_supported_openai_params(self, model: str) -> List: + return [ + "max_tokens", + "max_completion_tokens", + "temperature", + "stop", + "tools", + "tool_choice", + "stream", + "stream_options", + "response_format", + "reasoning_effort", + "reasoning_summary", + "reasoning_summary_wait", + "diffusing", + "realtime", + ] + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + passed_api_base = api_base + api_base = api_base or get_secret_str("INCEPTION_API_BASE") or "https://api.inceptionlabs.ai/v1" # type: ignore + dynamic_api_key = api_key + if passed_api_base is None or api_key: + dynamic_api_key = ( + api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY") + ) + return api_base, dynamic_api_key diff --git a/litellm/llms/inception/completion/__init__.py b/litellm/llms/inception/completion/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/inception/completion/transformation.py b/litellm/llms/inception/completion/transformation.py new file mode 100644 index 00000000000..1035042f6bf --- /dev/null +++ b/litellm/llms/inception/completion/transformation.py @@ -0,0 +1,43 @@ +""" +Inception fill-in-the-middle (FIM) completions. + +Inception's FIM endpoint is OpenAI text-completion compatible: it takes a +`prompt` (prefix) plus an optional `suffix` and returns standard +`choices[].text`. It is served at `/v1/fim/completions` rather than +`/v1/completions`, so routing points the OpenAI client at the `/v1/fim` base +(see the `text-completion-inception` branch in `main.py`). +""" + +from typing import List + +from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig + + +class InceptionTextCompletionConfig(OpenAITextCompletionConfig): + def get_supported_openai_params(self, model: str) -> List: + return [ + "suffix", + "max_tokens", + "max_completion_tokens", + "top_p", + "frequency_penalty", + "presence_penalty", + "stop", + "stream", + "stream_options", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + for param, value in non_default_params.items(): + if param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_params: + optional_params[param] = value + return optional_params diff --git a/litellm/llms/litellm_proxy/skills/handler.py b/litellm/llms/litellm_proxy/skills/handler.py index 37aabd8b477..7b259c1ed66 100644 --- a/litellm/llms/litellm_proxy/skills/handler.py +++ b/litellm/llms/litellm_proxy/skills/handler.py @@ -17,6 +17,7 @@ from litellm.proxy.common_utils.resource_ownership import ( is_proxy_admin, user_can_access_resource_owner, ) +from litellm.repositories.table_repositories import SkillsRepository # Skills are looked up on every chat completion that has skills enabled # (`SkillsInjectionHook` calls ``fetch_skill_from_db``). 60s LRU/TTL cache @@ -107,7 +108,7 @@ class LiteLLMSkillsHandler: f"LiteLLMSkillsHandler: Creating skill {skill_id} with title={data.display_title}" ) - new_skill = await prisma_client.db.litellm_skillstable.create(data=skill_data) + new_skill = await SkillsRepository(prisma_client).table.create(data=skill_data) return _prisma_skill_to_litellm(new_skill) @staticmethod @@ -133,7 +134,7 @@ class LiteLLMSkillsHandler: return [] find_many_kwargs["where"] = {"created_by": {"in": owner_scopes}} - skills = await prisma_client.db.litellm_skillstable.find_many( + skills = await SkillsRepository(prisma_client).table.find_many( **find_many_kwargs ) return [_prisma_skill_to_litellm(s) for s in skills] @@ -150,7 +151,7 @@ class LiteLLMSkillsHandler: return cached prisma_client = await LiteLLMSkillsHandler._get_prisma_client() - skill = await prisma_client.db.litellm_skillstable.find_unique( + skill = await SkillsRepository(prisma_client).table.find_unique( where={"skill_id": skill_id} ) _SKILL_CACHE.set_cache( @@ -189,7 +190,7 @@ class LiteLLMSkillsHandler: ): raise ValueError(f"Skill not found: {skill_id}") - await prisma_client.db.litellm_skillstable.delete(where={"skill_id": skill_id}) + await SkillsRepository(prisma_client).table.delete(where={"skill_id": skill_id}) _SKILL_CACHE.set_cache(skill_id, _NEGATIVE_SKILL_SENTINEL) return {"id": skill_id, "type": "skill_deleted"} diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py index 4eb00fd81d6..da8687bce72 100644 --- a/litellm/llms/moonshot/chat/transformation.py +++ b/litellm/llms/moonshot/chat/transformation.py @@ -134,11 +134,15 @@ class MoonshotChatConfig(OpenAIGPTConfig): ########################################## # temperature limitations - # 1. `temperature` on KIMI API is [0, 1] but OpenAI is [0, 2] - # 2. If temperature < 0.3 and n > 1, KIMI will raise an exception. + # 1. reasoning models (kimi-k2.5, kimi-k2.6, ...) reject every temperature + # except 1, so the param is dropped and the model's default is used + # 2. `temperature` on KIMI API is [0, 1] but OpenAI is [0, 2] + # 3. If temperature < 0.3 and n > 1, KIMI will raise an exception. # If we enter this condition, we set the temperature to 0.3 as suggested by Moonshot AI ########################################## - if "temperature" in optional_params: + if supports_reasoning(model=model, custom_llm_provider="moonshot"): + optional_params.pop("temperature", None) + elif "temperature" in optional_params: if optional_params["temperature"] > 1: optional_params["temperature"] = 1 if optional_params["temperature"] < 0.3 and optional_params.get("n", 1) > 1: diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index c9257677fd1..49b3801c82f 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -115,6 +115,15 @@ "max_completion_tokens": "max_tokens" } }, + "neosantara": { + "base_url": "https://api.neosantara.xyz/v1", + "api_key_env": "NEOSANTARA_API_KEY", + "api_base_env": "NEOSANTARA_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + }, + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] + }, "tensormesh": { "base_url": "https://serverless.tensormesh.ai/v1", "api_key_env": "TENSORMESH_INFERENCE_API_KEY", diff --git a/litellm/llms/snowflake/utils.py b/litellm/llms/snowflake/utils.py index d84efdd9fcd..4f79006f6f8 100644 --- a/litellm/llms/snowflake/utils.py +++ b/litellm/llms/snowflake/utils.py @@ -25,6 +25,7 @@ class SnowflakeBaseConfig: "temperature", "max_tokens", "top_p", + "stream", "response_format", "tools", "tool_choice", diff --git a/litellm/llms/soniox/__init__.py b/litellm/llms/soniox/__init__.py new file mode 100644 index 00000000000..778211a2a53 --- /dev/null +++ b/litellm/llms/soniox/__init__.py @@ -0,0 +1 @@ +"""Soniox LLM provider implementation.""" diff --git a/litellm/llms/soniox/audio_transcription/__init__.py b/litellm/llms/soniox/audio_transcription/__init__.py new file mode 100644 index 00000000000..3da6032ce65 --- /dev/null +++ b/litellm/llms/soniox/audio_transcription/__init__.py @@ -0,0 +1 @@ +"""Soniox audio transcription implementation.""" diff --git a/litellm/llms/soniox/audio_transcription/handler.py b/litellm/llms/soniox/audio_transcription/handler.py new file mode 100644 index 00000000000..d4774fea460 --- /dev/null +++ b/litellm/llms/soniox/audio_transcription/handler.py @@ -0,0 +1,802 @@ +""" +Handler for Soniox async speech-to-text transcription. + +Soniox's async transcription API requires multiple HTTP calls: + 1. (optional) POST /v1/files — upload a local audio file + 2. POST /v1/transcriptions — create a transcription job + 3. GET /v1/transcriptions/{id} — poll until status == "completed" + 4. GET /v1/transcriptions/{id}/transcript — fetch the transcript + 5. (optional) DELETE /v1/transcriptions/{id} — cleanup + 6. (optional) DELETE /v1/files/{id} — cleanup + +Because this does not fit the single-request shape of +`base_llm_http_handler.audio_transcriptions`, the dispatch in +`litellm.main.transcription()` routes Soniox requests directly to this +handler (analogous to the OpenAI / Azure transcription handlers). +""" + +import asyncio +import math +import time +from typing import ( + TYPE_CHECKING, + Any, + Coroutine, + Dict, + List, + Optional, + Tuple, + Union, +) + +import httpx + +from litellm.litellm_core_utils.audio_utils.utils import ( + get_audio_file_name, + process_audio_file, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, +) +from litellm.llms.soniox.audio_transcription.transformation import ( + SonioxAudioTranscriptionConfig, +) +from litellm.llms.soniox.common_utils import ( + SONIOX_DEFAULT_CLEANUP, + SONIOX_DEFAULT_MAX_POLL_ATTEMPTS, + SONIOX_DEFAULT_POLL_INTERVAL, + SONIOX_MAX_POLL_ATTEMPTS, + SONIOX_MAX_POLL_INTERVAL, + SONIOX_MIN_POLL_INTERVAL, + SONIOX_SECRET_FIELDS, + SonioxException, + get_soniox_api_base, +) +from litellm.types.utils import FileTypes, TranscriptionResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObj, + ) +else: + LiteLLMLoggingObj = Any + + +class SonioxAudioTranscriptionHandler: + """Orchestrates the Soniox async transcription flow.""" + + # ------------------------------------------------------------------ + # Public entry points + # ------------------------------------------------------------------ + + def audio_transcriptions( + self, + model: str, + audio_file: Optional[FileTypes], + optional_params: dict, + litellm_params: dict, + model_response: TranscriptionResponse, + timeout: float, + max_retries: int, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + atranscription: bool = False, + headers: Optional[Dict[str, Any]] = None, + provider_config: Optional[SonioxAudioTranscriptionConfig] = None, + ) -> Union[TranscriptionResponse, Coroutine[Any, Any, TranscriptionResponse]]: + """Sync/async dispatch for Soniox transcription requests. + + Note: ``max_retries`` is accepted for signature compatibility with + ``litellm.transcription`` but is **not yet implemented** for the Soniox + async pipeline. Transient HTTP failures during upload, create, poll, + or fetch will surface immediately. Wrap calls with the standard + ``litellm.Router`` / ``num_retries`` mechanism for retry behaviour. + """ + config = provider_config or SonioxAudioTranscriptionConfig() + + if atranscription is True: + return self._async_audio_transcriptions( + model=model, + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + model_response=model_response, + timeout=timeout, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + client=client if isinstance(client, AsyncHTTPHandler) else None, + headers=headers or {}, + provider_config=config, + ) + + return self._sync_audio_transcriptions( + model=model, + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + model_response=model_response, + timeout=timeout, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + client=client if isinstance(client, HTTPHandler) else None, + headers=headers or {}, + provider_config=config, + ) + + # ------------------------------------------------------------------ + # Helpers shared between sync and async paths + # ------------------------------------------------------------------ + + def _prepare( + self, + audio_file: Optional[FileTypes], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str], + api_base: Optional[str], + provider_config: SonioxAudioTranscriptionConfig, + headers: Dict[str, Any], + ) -> Tuple[ + Dict[str, str], # auth headers + str, # api_base (no trailing slash) + Dict[str, Any], # body for POST /v1/transcriptions (without file_id/audio_url) + Dict[str, Any], # handler-only options (poll interval, cleanup, ...) + ]: + # Validate env -> auth headers. + auth_headers = provider_config.validate_environment( + headers=headers, + model="", # unused + messages=[], + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + api_base=api_base, + ) + + base_url = get_soniox_api_base(api_base) + + # Operate on a local copy so we don't mutate the caller's dict + # (the caller may reuse `optional_params` for retries or logging). + params = dict(optional_params) + + # Pull handler-only kwargs out of params so they aren't sent + # to Soniox. + poll_interval = float( + params.pop("soniox_polling_interval", SONIOX_DEFAULT_POLL_INTERVAL) + ) + try: + max_attempts = int( + params.pop( + "soniox_max_polling_attempts", SONIOX_DEFAULT_MAX_POLL_ATTEMPTS + ) + ) + except (ValueError, OverflowError): + max_attempts = SONIOX_DEFAULT_MAX_POLL_ATTEMPTS + cleanup_raw = params.pop("soniox_cleanup", SONIOX_DEFAULT_CLEANUP) + if cleanup_raw is None: + cleanup: List[str] = [] + elif isinstance(cleanup_raw, str): + cleanup = [cleanup_raw] + else: + cleanup = list(cleanup_raw) + filename_override = params.pop("filename", None) + + # Server-side clamps. Caller-supplied poll settings (from request kwargs) + # are bounded so an authenticated caller cannot force a worker into a + # tight poll loop (zero interval) or pin it indefinitely (huge attempt + # count). Total polling time is bounded by + # SONIOX_MAX_POLL_ATTEMPTS * SONIOX_MAX_POLL_INTERVAL. + if not math.isfinite(poll_interval): + poll_interval = SONIOX_DEFAULT_POLL_INTERVAL + clamped_poll_interval = max( + SONIOX_MIN_POLL_INTERVAL, min(poll_interval, SONIOX_MAX_POLL_INTERVAL) + ) + clamped_max_attempts = max(1, min(max_attempts, SONIOX_MAX_POLL_ATTEMPTS)) + + handler_opts: Dict[str, Any] = { + "poll_interval": clamped_poll_interval, + "max_attempts": clamped_max_attempts, + "cleanup": cleanup, + "filename_override": filename_override, + "audio_url": params.pop("audio_url", None), + "file_id": params.pop("file_id", None), + } + + # Soniox does not accept `language` directly; map_openai_params should + # already have translated it, but drop any leftover to be safe. + params.pop("language", None) + + # response_format is handled by LiteLLM post-processing, not Soniox. + handler_opts["response_format"] = params.pop("response_format", None) + + return auth_headers, base_url, params, handler_opts + + def _build_create_body( + self, + model: str, + optional_params: dict, + handler_opts: Dict[str, Any], + file_id: Optional[str], + ) -> Dict[str, Any]: + body: Dict[str, Any] = {"model": model} + # Soniox-native passthrough fields + for key, value in optional_params.items(): + if value is None: + continue + body[key] = value + + if handler_opts.get("audio_url"): + body["audio_url"] = handler_opts["audio_url"] + if file_id: + body["file_id"] = file_id + + return body + + @staticmethod + def _redact_body_for_logging(body: Dict[str, Any]) -> Dict[str, Any]: + """Return a shallow copy of ``body`` with secret fields redacted. + + Soniox's create-transcription body can include + ``webhook_auth_header_value`` (a shared secret used to authenticate + webhook callbacks). Forwarding that value to logging callbacks would + let anyone with read access to those sinks forge webhook requests, so + we replace any value of a known secret-bearing field with the literal + ``"[REDACTED]"`` before logging. Non-secret fields are passed through + unchanged. + """ + if not body: + return body + redacted = dict(body) + for field in SONIOX_SECRET_FIELDS: + if field in redacted and redacted[field] is not None: + redacted[field] = "[REDACTED]" + return redacted + + @staticmethod + def _safe_log_pre_call( + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: str, + body: Dict[str, Any], + ) -> None: + try: + logging_obj.pre_call( + input=None, + api_key=api_key, + additional_args={ + "api_base": f"{api_base}/v1/transcriptions", + "atranscription": True, + "complete_input_dict": SonioxAudioTranscriptionHandler._redact_body_for_logging( + body + ), + }, + ) + except Exception: + # Logging hooks are best-effort: a misbehaving callback or third-party + # observability integration must never break a real Soniox call. + pass + + @staticmethod + def _safe_log_post_call( + logging_obj: LiteLLMLoggingObj, + audio_file: Optional[FileTypes], + api_key: Optional[str], + body: Dict[str, Any], + original_response: Any, + ) -> None: + try: + logging_obj.post_call( + input=get_audio_file_name(audio_file) if audio_file else None, + api_key=api_key, + additional_args={ + "complete_input_dict": SonioxAudioTranscriptionHandler._redact_body_for_logging( + body + ) + }, + original_response=original_response, + ) + except Exception: + # Logging hooks are best-effort: a misbehaving callback or third-party + # observability integration must never break a real Soniox call. + pass + + @staticmethod + def _raise_for_response( + response: httpx.Response, + provider_config: SonioxAudioTranscriptionConfig, + action: str, + ) -> None: + if response.status_code >= 400: + try: + payload = response.json() + message = ( + payload.get("error_message") + or payload.get("error") + or response.text + ) + except Exception: + message = response.text + raise provider_config.get_error_class( + error_message=f"Soniox {action} failed (HTTP {response.status_code}): {message}", + status_code=response.status_code, + headers=response.headers, + ) + + # ------------------------------------------------------------------ + # Sync flow + # ------------------------------------------------------------------ + + def _sync_audio_transcriptions( + self, + model: str, + audio_file: Optional[FileTypes], + optional_params: dict, + litellm_params: dict, + model_response: TranscriptionResponse, + timeout: float, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + client: Optional[HTTPHandler], + headers: Dict[str, Any], + provider_config: SonioxAudioTranscriptionConfig, + ) -> TranscriptionResponse: + auth_headers, base_url, opt_params, handler_opts = self._prepare( + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + api_base=api_base, + provider_config=provider_config, + headers=headers, + ) + + http_client = ( + client + if isinstance(client, HTTPHandler) + else ( + _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + ) + ) + + file_id = handler_opts.get("file_id") + uploaded_file_id: Optional[str] = None + transcription_id: Optional[str] = None + + try: + if not file_id and not handler_opts.get("audio_url"): + if audio_file is None: + raise SonioxException( + message=( + "Soniox transcription requires one of: a file argument, " + "an `audio_url` kwarg, or a `file_id` kwarg." + ), + status_code=400, + headers=None, + ) + uploaded_file_id = self._sync_upload_file( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + audio_file=audio_file, + filename_override=handler_opts.get("filename_override"), + timeout=timeout, + provider_config=provider_config, + ) + file_id = uploaded_file_id + + body = self._build_create_body(model, opt_params, handler_opts, file_id) + self._safe_log_pre_call(logging_obj, api_key, base_url, body) + + create_resp = http_client.post( + url=f"{base_url}/v1/transcriptions", + headers=auth_headers, + json=body, + timeout=timeout, + ) + self._raise_for_response( + create_resp, provider_config, "create transcription" + ) + transcription_id = create_resp.json()["id"] + + transcription_meta = self._sync_poll_until_completed( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + transcription_id=transcription_id, + poll_interval=handler_opts["poll_interval"], + max_attempts=handler_opts["max_attempts"], + timeout=timeout, + provider_config=provider_config, + ) + + transcript_resp = http_client.get( + url=f"{base_url}/v1/transcriptions/{transcription_id}/transcript", + headers=auth_headers, + timeout=timeout, + ) + self._raise_for_response( + transcript_resp, provider_config, "fetch transcript" + ) + transcript = transcript_resp.json() + + payload = {"transcription": transcription_meta, "transcript": transcript} + response = provider_config._build_response_from_payload( + payload, + model_response=model_response, + response_format=handler_opts.get("response_format"), + ) + + self._safe_log_post_call(logging_obj, audio_file, api_key, body, payload) + + audio_duration_ms = transcription_meta.get("audio_duration_ms") + response._hidden_params.update( + { + "model": model, + "custom_llm_provider": "soniox", + "audio_transcription_duration": ( + float(audio_duration_ms) / 1000.0 + if audio_duration_ms is not None + else None + ), + } + ) + return response + finally: + self._sync_cleanup( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + cleanup=handler_opts["cleanup"], + file_id_to_cleanup=uploaded_file_id, + transcription_id=transcription_id, + timeout=timeout, + ) + + def _sync_upload_file( + self, + http_client: HTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + audio_file: FileTypes, + filename_override: Optional[str], + timeout: float, + provider_config: SonioxAudioTranscriptionConfig, + ) -> str: + processed = process_audio_file(audio_file) + filename = filename_override or processed.filename + files = { + "file": (filename, processed.file_content, processed.content_type), + } + # `Authorization` header is fine; httpx sets multipart Content-Type. + upload_headers = {"Authorization": auth_headers["Authorization"]} + resp = http_client.post( + url=f"{base_url}/v1/files", + headers=upload_headers, + files=files, + timeout=timeout, + ) + self._raise_for_response(resp, provider_config, "upload file") + return resp.json()["id"] + + def _sync_poll_until_completed( + self, + http_client: HTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + transcription_id: str, + poll_interval: float, + max_attempts: int, + timeout: float, + provider_config: SonioxAudioTranscriptionConfig, + ) -> Dict[str, Any]: + for _ in range(max_attempts): + resp = http_client.get( + url=f"{base_url}/v1/transcriptions/{transcription_id}", + headers=auth_headers, + timeout=timeout, + ) + self._raise_for_response(resp, provider_config, "poll transcription") + data = resp.json() + status = data.get("status") + if status == "completed": + return data + if status == "error": + raise provider_config.get_error_class( + error_message=( + f"Soniox transcription {transcription_id} failed: " + f"{data.get('error_message') or data.get('error_type') or 'unknown error'}" + ), + status_code=500, + headers=resp.headers, + ) + time.sleep(poll_interval) + raise provider_config.get_error_class( + error_message=( + f"Soniox transcription {transcription_id} did not complete after " + f"{max_attempts} polling attempts (interval={poll_interval}s)." + ), + status_code=504, + headers={}, + ) + + def _sync_cleanup( + self, + http_client: HTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + cleanup: List[str], + file_id_to_cleanup: Optional[str], + transcription_id: Optional[str], + timeout: float, + ) -> None: + if not cleanup: + return + if "transcription" in cleanup and transcription_id: + try: + http_client.delete( + url=f"{base_url}/v1/transcriptions/{transcription_id}", + headers=auth_headers, + timeout=timeout, + ) + except Exception: + # Cleanup is best-effort: a failed delete leaves stale data on + # Soniox but must not mask the original transcription result + # (or, on the error path, the original error). + pass + if "file" in cleanup and file_id_to_cleanup: + try: + http_client.delete( + url=f"{base_url}/v1/files/{file_id_to_cleanup}", + headers=auth_headers, + timeout=timeout, + ) + except Exception: + # Cleanup is best-effort; see comment above. + pass + + # ------------------------------------------------------------------ + # Async flow + # ------------------------------------------------------------------ + + async def _async_audio_transcriptions( + self, + model: str, + audio_file: Optional[FileTypes], + optional_params: dict, + litellm_params: dict, + model_response: TranscriptionResponse, + timeout: float, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + client: Optional[AsyncHTTPHandler], + headers: Dict[str, Any], + provider_config: SonioxAudioTranscriptionConfig, + ) -> TranscriptionResponse: + import litellm + + auth_headers, base_url, opt_params, handler_opts = self._prepare( + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + api_base=api_base, + provider_config=provider_config, + headers=headers, + ) + + http_client = ( + client + if isinstance(client, AsyncHTTPHandler) + else ( + get_async_httpx_client( + llm_provider=litellm.LlmProviders.SONIOX, + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + ) + ) + + file_id = handler_opts.get("file_id") + uploaded_file_id: Optional[str] = None + transcription_id: Optional[str] = None + + try: + if not file_id and not handler_opts.get("audio_url"): + if audio_file is None: + raise SonioxException( + message=( + "Soniox transcription requires one of: a file argument, " + "an `audio_url` kwarg, or a `file_id` kwarg." + ), + status_code=400, + headers=None, + ) + uploaded_file_id = await self._async_upload_file( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + audio_file=audio_file, + filename_override=handler_opts.get("filename_override"), + timeout=timeout, + provider_config=provider_config, + ) + file_id = uploaded_file_id + + body = self._build_create_body(model, opt_params, handler_opts, file_id) + self._safe_log_pre_call(logging_obj, api_key, base_url, body) + + create_resp = await http_client.post( + url=f"{base_url}/v1/transcriptions", + headers=auth_headers, + json=body, + timeout=timeout, + ) + self._raise_for_response( + create_resp, provider_config, "create transcription" + ) + transcription_id = create_resp.json()["id"] + + transcription_meta = await self._async_poll_until_completed( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + transcription_id=transcription_id, + poll_interval=handler_opts["poll_interval"], + max_attempts=handler_opts["max_attempts"], + timeout=timeout, + provider_config=provider_config, + ) + + transcript_resp = await http_client.get( + url=f"{base_url}/v1/transcriptions/{transcription_id}/transcript", + headers=auth_headers, + timeout=timeout, + ) + self._raise_for_response( + transcript_resp, provider_config, "fetch transcript" + ) + transcript = transcript_resp.json() + + payload = {"transcription": transcription_meta, "transcript": transcript} + response = provider_config._build_response_from_payload( + payload, + model_response=model_response, + response_format=handler_opts.get("response_format"), + ) + + self._safe_log_post_call(logging_obj, audio_file, api_key, body, payload) + + audio_duration_ms = transcription_meta.get("audio_duration_ms") + response._hidden_params.update( + { + "model": model, + "custom_llm_provider": "soniox", + "audio_transcription_duration": ( + float(audio_duration_ms) / 1000.0 + if audio_duration_ms is not None + else None + ), + } + ) + return response + finally: + await self._async_cleanup( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + cleanup=handler_opts["cleanup"], + file_id_to_cleanup=uploaded_file_id, + transcription_id=transcription_id, + timeout=timeout, + ) + + async def _async_upload_file( + self, + http_client: AsyncHTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + audio_file: FileTypes, + filename_override: Optional[str], + timeout: float, + provider_config: SonioxAudioTranscriptionConfig, + ) -> str: + processed = process_audio_file(audio_file) + filename = filename_override or processed.filename + files = { + "file": (filename, processed.file_content, processed.content_type), + } + upload_headers = {"Authorization": auth_headers["Authorization"]} + resp = await http_client.post( + url=f"{base_url}/v1/files", + headers=upload_headers, + files=files, + timeout=timeout, + ) + self._raise_for_response(resp, provider_config, "upload file") + return resp.json()["id"] + + async def _async_poll_until_completed( + self, + http_client: AsyncHTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + transcription_id: str, + poll_interval: float, + max_attempts: int, + timeout: float, + provider_config: SonioxAudioTranscriptionConfig, + ) -> Dict[str, Any]: + for _ in range(max_attempts): + resp = await http_client.get( + url=f"{base_url}/v1/transcriptions/{transcription_id}", + headers=auth_headers, + timeout=timeout, + ) + self._raise_for_response(resp, provider_config, "poll transcription") + data = resp.json() + status = data.get("status") + if status == "completed": + return data + if status == "error": + raise provider_config.get_error_class( + error_message=( + f"Soniox transcription {transcription_id} failed: " + f"{data.get('error_message') or data.get('error_type') or 'unknown error'}" + ), + status_code=500, + headers=resp.headers, + ) + await asyncio.sleep(poll_interval) + raise provider_config.get_error_class( + error_message=( + f"Soniox transcription {transcription_id} did not complete after " + f"{max_attempts} polling attempts (interval={poll_interval}s)." + ), + status_code=504, + headers={}, + ) + + async def _async_cleanup( + self, + http_client: AsyncHTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + cleanup: List[str], + file_id_to_cleanup: Optional[str], + transcription_id: Optional[str], + timeout: float, + ) -> None: + if not cleanup: + return + if "transcription" in cleanup and transcription_id: + try: + await http_client.delete( + url=f"{base_url}/v1/transcriptions/{transcription_id}", + headers=auth_headers, + timeout=timeout, + ) + except Exception: + # Cleanup is best-effort: a failed delete leaves stale data on + # Soniox but must not mask the original transcription result + # (or, on the error path, the original error). + pass + if "file" in cleanup and file_id_to_cleanup: + try: + await http_client.delete( + url=f"{base_url}/v1/files/{file_id_to_cleanup}", + headers=auth_headers, + timeout=timeout, + ) + except Exception: + # Cleanup is best-effort; see comment above. + pass diff --git a/litellm/llms/soniox/audio_transcription/transformation.py b/litellm/llms/soniox/audio_transcription/transformation.py new file mode 100644 index 00000000000..681d4352dfe --- /dev/null +++ b/litellm/llms/soniox/audio_transcription/transformation.py @@ -0,0 +1,281 @@ +""" +Translates between OpenAI's `/v1/audio/transcriptions` shape and Soniox's +async transcription API (https://soniox.com/docs/stt/async/async-transcription). + +This config covers parameter mapping, env validation and response shaping. +The actual orchestration (file upload -> create -> poll -> fetch -> cleanup) +lives in `litellm.llms.soniox.audio_transcription.handler`, because Soniox's +async API requires multiple HTTP calls and does not fit the single-request +contract of `base_llm_http_handler.audio_transcriptions`. +""" + +from typing import Any, Dict, List, Optional, Union + +from httpx import Headers, Response + +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, + BaseAudioTranscriptionConfig, +) +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.soniox.common_utils import ( + SonioxException, + get_soniox_api_base, + get_soniox_api_key, + render_soniox_tokens, + render_soniox_tokens_as_srt, + render_soniox_tokens_as_vtt, +) +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIAudioTranscriptionOptionalParams, +) +from litellm.types.utils import FileTypes, TranscriptionResponse + +# Soniox-native kwargs the user can pass through `litellm.transcription(..., **kwargs)` +# in addition to the standard OpenAI params. +SONIOX_PASSTHROUGH_PARAMS: List[str] = [ + "language_hints", + "language_hints_strict", + "enable_language_identification", + "enable_speaker_diarization", + "context", + "translation", + "client_reference_id", + "webhook_url", + "webhook_auth_header_name", + "webhook_auth_header_value", + "audio_url", + "file_id", +] + +# Handler-only kwargs (consumed by the handler, not sent to Soniox). +SONIOX_HANDLER_ONLY_PARAMS: List[str] = [ + "soniox_polling_interval", + "soniox_max_polling_attempts", + "soniox_cleanup", + "filename", +] + + +class SonioxAudioTranscriptionConfig(BaseAudioTranscriptionConfig): + """Configuration for Soniox async speech-to-text transcription.""" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIAudioTranscriptionOptionalParams]: + # `language` is mapped onto Soniox's `language_hints`. + # `response_format` is handled by LiteLLM (Soniox doesn't support + # SRT/VTT natively but we synthesize them from token timestamps). + return ["language", "response_format"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + # Translate the OpenAI `language` param into Soniox `language_hints`. + if "language" in non_default_params and non_default_params["language"]: + language = non_default_params["language"] + existing_hints = optional_params.get("language_hints") + if not existing_hints: + optional_params["language_hints"] = [language] + elif language not in existing_hints: + optional_params["language_hints"] = [language] + list(existing_hints) + + # Capture response_format for post-processing (not sent to Soniox API). + if "response_format" in non_default_params: + optional_params["response_format"] = non_default_params["response_format"] + + # Pass through Soniox-native kwargs unchanged. + for key in SONIOX_PASSTHROUGH_PARAMS + SONIOX_HANDLER_ONLY_PARAMS: + if key in non_default_params and non_default_params[key] is not None: + optional_params[key] = non_default_params[key] + + return optional_params + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, Headers] + ) -> BaseLLMException: + return SonioxException( + message=error_message, status_code=status_code, headers=headers + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + resolved_key = get_soniox_api_key(api_key) + if not resolved_key: + raise SonioxException( + message=( + "Missing Soniox API key. Set the SONIOX_API_KEY environment " + "variable or pass api_key=... to litellm.transcription()." + ), + status_code=401, + headers=None, + ) + + merged_headers: Dict[str, str] = { + "Authorization": f"Bearer {resolved_key}", + } + if headers: + merged_headers.update(headers) + return merged_headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + # The handler builds per-call URLs (uploads, create, poll, fetch, delete); + # we just return the resolved base. + return get_soniox_api_base(api_base) + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + ) -> AudioTranscriptionRequestData: + """ + Build the JSON body for `POST /v1/transcriptions`. + + The handler is responsible for the file upload (if `audio_file` is bytes) + and for filling in `file_id`/`audio_url`. This method exists so the + config can be exercised in isolation by unit tests. + """ + body: Dict[str, Any] = {"model": model} + + for key in SONIOX_PASSTHROUGH_PARAMS: + value = optional_params.get(key) + if value is not None: + body[key] = value + + return AudioTranscriptionRequestData( + data=body, files=None, content_type="application/json" + ) + + def transform_audio_transcription_response( + self, + raw_response: Response, + model_response: Optional[TranscriptionResponse] = None, + ) -> TranscriptionResponse: + """ + Build a TranscriptionResponse from a Soniox transcript payload. + + `raw_response.json()` may be either: + - a Soniox transcript object: `{"id": "...", "text": "...", "tokens": [...]}` + - or a merged envelope: `{"transcription": {...}, "transcript": {...}}` + produced by the handler so transcription metadata is also available. + """ + try: + payload = raw_response.json() + except Exception as exc: + raise SonioxException( + message=f"Failed to parse Soniox response: {exc}", + status_code=getattr(raw_response, "status_code", 500), + headers=getattr(raw_response, "headers", None), + ) + + return self._build_response_from_payload(payload, model_response=model_response) + + def _build_response_from_payload( + self, + payload: Dict[str, Any], + model_response: Optional[TranscriptionResponse] = None, + response_format: Optional[str] = None, + ) -> TranscriptionResponse: + """Shared response-building logic (also used by the handler).""" + transcription_meta: Dict[str, Any] = {} + transcript: Dict[str, Any] + + if isinstance(payload, dict) and "transcript" in payload: + transcription_meta = payload.get("transcription") or {} + transcript = payload.get("transcript") or {} + else: + transcript = payload if isinstance(payload, dict) else {} + + tokens: List[Dict[str, Any]] = transcript.get("tokens") or [] + + # Decide what to put in `text` based on response_format: + # - "srt": render tokens as SRT subtitles (synthesized from timestamps) + # - "vtt": render tokens as WebVTT subtitles (synthesized from timestamps) + # - "verbose_json": return JSON with word-level timing (handled below) + # - "text" / "json" / None: default plain text rendering + if response_format == "srt" and tokens: + text = render_soniox_tokens_as_srt(tokens) + elif response_format == "vtt" and tokens: + text = render_soniox_tokens_as_vtt(tokens) + else: + # Default text rendering (also used for "json", "text", + # "verbose_json") + has_speaker = any(t.get("speaker") is not None for t in tokens) + has_language = any(t.get("language") is not None for t in tokens) + + if (has_speaker or has_language) and tokens: + text = render_soniox_tokens(tokens) + elif transcript.get("text"): + text = transcript["text"] + elif tokens: + text = "".join(t.get("text", "") for t in tokens) + else: + text = "" + + response = model_response or TranscriptionResponse(text=text) + response.text = text + response["task"] = "transcribe" + + # Best-effort metadata fields matching OpenAI's verbose_json shape. + if transcription_meta.get("audio_duration_ms") is not None: + try: + response["duration"] = ( + float(transcription_meta["audio_duration_ms"]) / 1000.0 + ) + except (TypeError, ValueError): + pass + + # Surface a representative language if all tokens agree. + has_language = any(t.get("language") is not None for t in tokens) + if has_language: + languages = {t.get("language") for t in tokens if t.get("language")} + if len(languages) == 1: + response["language"] = next(iter(languages)) + + # For verbose_json, include word-level timing from tokens. + if response_format == "verbose_json" and tokens: + words: List[Dict[str, Any]] = [] + for token in tokens: + word_entry: Dict[str, Any] = {"word": token.get("text", "")} + if token.get("start_ms") is not None: + word_entry["start"] = float(token["start_ms"]) / 1000.0 + if token.get("end_ms") is not None: + word_entry["end"] = float(token["end_ms"]) / 1000.0 + words.append(word_entry) + if words: + response["words"] = words + + # Stash the raw Soniox payload so power-users can read tokens, segments, + # speaker/language data, etc. + response._hidden_params.update( + { + "soniox_raw": { + "transcription": transcription_meta, + "transcript": transcript, + } + } + ) + return response diff --git a/litellm/llms/soniox/common_utils.py b/litellm/llms/soniox/common_utils.py new file mode 100644 index 00000000000..01f8062fc96 --- /dev/null +++ b/litellm/llms/soniox/common_utils.py @@ -0,0 +1,274 @@ +""" +Shared utilities for the Soniox provider (https://soniox.com). +""" + +from typing import Any, Dict, List, Optional + +from litellm.llms.base_llm.chat.transformation import BaseLLMException + +# Soniox API base URL. +SONIOX_API_BASE: str = "https://api.soniox.com" + +# Default polling interval in seconds when waiting for an async transcription +# to finish. Mirrors the Soniox SDK default. +SONIOX_DEFAULT_POLL_INTERVAL: float = 1.0 + +# Minimum polling interval (in seconds) the server will accept from caller- +# supplied `soniox_polling_interval` kwargs. Prevents an authenticated caller +# from forcing a worker into a tight poll loop with a zero/near-zero interval. +SONIOX_MIN_POLL_INTERVAL: float = 0.5 + +# Maximum polling interval (in seconds). Prevents a caller from setting an +# excessively large or non-finite interval that would keep a worker sleeping +# far longer than necessary between status checks. +SONIOX_MAX_POLL_INTERVAL: float = 60.0 + +# Default maximum number of polling attempts (1800 attempts * 1s ~= 30 minutes). +SONIOX_DEFAULT_MAX_POLL_ATTEMPTS: int = 1800 + +# Hard upper bound on polling attempts. Combined with `SONIOX_MIN_POLL_INTERVAL` +# this caps total polling time per request at ~3000s (50 minutes), preventing a +# caller from pinning a worker indefinitely via a huge attempt count. +SONIOX_MAX_POLL_ATTEMPTS: int = 6000 + +# Default cleanup behaviour: delete both the uploaded file (if any) and the +# transcription record after the transcript has been fetched. +SONIOX_DEFAULT_CLEANUP: List[str] = ["file", "transcription"] + +# Body fields that may carry secrets and must be redacted before being +# forwarded to logging callbacks. Soniox accepts a webhook auth header value +# alongside the create-transcription request; that value lets the recipient +# authenticate webhook callbacks and must not leak into observability sinks. +SONIOX_SECRET_FIELDS: List[str] = ["webhook_auth_header_value"] + + +class SonioxException(BaseLLMException): + """Provider-specific exception class for Soniox.""" + + pass + + +def get_soniox_api_key(api_key: Optional[str] = None) -> Optional[str]: + """Resolve the Soniox API key from arg or env var.""" + # Local import to avoid a circular import: litellm.secret_managers.main + # imports from litellm at top-level. + from litellm.secret_managers.main import get_secret_str + + return api_key or get_secret_str("SONIOX_API_KEY") + + +def get_soniox_api_base(api_base: Optional[str] = None) -> str: + """Resolve the Soniox API base URL from arg or env var (defaults to public API).""" + from litellm.secret_managers.main import get_secret_str + + base = api_base or get_secret_str("SONIOX_API_BASE") or SONIOX_API_BASE + return base.rstrip("/") + + +def render_soniox_tokens(tokens: List[Dict[str, Any]]) -> str: + """ + Render a list of Soniox tokens to a readable transcript string. + + Mirrors the behaviour of the official Soniox SDK's `renderTokens` helper: + - When the speaker changes, a `Speaker N:` tag is inserted. + - When the language changes, a `[lang]` (or `[Translation][lang]`) tag is + inserted. + + If neither speaker nor language information is present on any token (i.e. + diarization and language identification are disabled), the function simply + concatenates the token texts. + """ + if not tokens: + return "" + + text_parts: List[str] = [] + current_speaker: Optional[Any] = None + current_language: Optional[Any] = None + + for token in tokens: + text = token.get("text", "") + speaker = token.get("speaker") + language = token.get("language") + is_translation = token.get("translation_status") == "translation" + + # Speaker changed -> emit a speaker tag. + if speaker is not None and speaker != current_speaker: + if current_speaker is not None: + text_parts.append("\n\n") + current_speaker = speaker + current_language = None # reset language whenever speaker changes + text_parts.append(f"Speaker {current_speaker}:") + + # Language changed -> emit a language (or translation) tag. + if language is not None and language != current_language: + current_language = language + prefix = "[Translation] " if is_translation else "" + text_parts.append(f"\n{prefix}[{current_language}] ") + text = text.lstrip() if isinstance(text, str) else text + + text_parts.append(text) + + return "".join(text_parts) + + +# --------------------------------------------------------------------------- +# SRT / VTT subtitle rendering +# --------------------------------------------------------------------------- + +# Maximum number of tokens to group into a single subtitle cue. +_CUE_MAX_TOKENS: int = 15 + +# Maximum duration (in ms) for a single cue before forcing a break. +_CUE_MAX_DURATION_MS: int = 5000 + + +def _format_timestamp_srt(ms: int) -> str: + """Format milliseconds as SRT timestamp: HH:MM:SS,mmm""" + if ms < 0: + ms = 0 + hours = ms // 3_600_000 + ms %= 3_600_000 + minutes = ms // 60_000 + ms %= 60_000 + seconds = ms // 1_000 + millis = ms % 1_000 + return f"{hours:02d}:{minutes:02d}:{seconds:02d},{millis:03d}" + + +def _format_timestamp_vtt(ms: int) -> str: + """Format milliseconds as VTT timestamp: HH:MM:SS.mmm""" + if ms < 0: + ms = 0 + hours = ms // 3_600_000 + ms %= 3_600_000 + minutes = ms // 60_000 + ms %= 60_000 + seconds = ms // 1_000 + millis = ms % 1_000 + return f"{hours:02d}:{minutes:02d}:{seconds:02d}.{millis:03d}" + + +def _group_tokens_into_cues( + tokens: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """ + Group Soniox tokens into subtitle cues. + + Each cue has: + - start_ms: int + - end_ms: int + - text: str + + Grouping heuristics: + - A new cue starts when token count exceeds _CUE_MAX_TOKENS. + - A new cue starts when duration exceeds _CUE_MAX_DURATION_MS. + - A new cue starts when the speaker changes (if diarization is on). + - Tokens without timestamps are appended to the current cue. + """ + cues: List[Dict[str, Any]] = [] + current_tokens: List[str] = [] + current_start: Optional[int] = None + current_end: Optional[int] = None + current_speaker: Optional[Any] = None + + def _flush() -> None: + if current_tokens and current_start is not None: + text = "".join(current_tokens).strip() + if text: + cues.append( + { + "start_ms": current_start, + "end_ms": ( + current_end if current_end is not None else current_start + ), + "text": text, + } + ) + + for token in tokens: + start_ms = token.get("start_ms") + end_ms = token.get("end_ms") + text = token.get("text", "") + speaker = token.get("speaker") + + # Skip tokens with no timestamp data entirely if we have no cue started + if start_ms is None and current_start is None: + continue + + # Speaker change forces a new cue + if speaker is not None and speaker != current_speaker: + _flush() + current_tokens = [] + current_start = start_ms + current_end = end_ms + current_speaker = speaker + current_tokens.append(text) + continue + + # Duration or token count exceeded -> flush + should_break = False + if len(current_tokens) >= _CUE_MAX_TOKENS: + should_break = True + elif ( + current_start is not None + and start_ms is not None + and (start_ms - current_start) >= _CUE_MAX_DURATION_MS + ): + should_break = True + + if should_break: + _flush() + current_tokens = [] + current_start = start_ms + current_end = end_ms + current_tokens.append(text) + else: + if current_start is None: + current_start = start_ms + if end_ms is not None: + current_end = end_ms + current_tokens.append(text) + + _flush() + return cues + + +def render_soniox_tokens_as_srt(tokens: List[Dict[str, Any]]) -> str: + """ + Render Soniox tokens as SRT (SubRip) subtitle format. + + Returns an empty string if no tokens have timestamp data. + """ + cues = _group_tokens_into_cues(tokens) + if not cues: + return "" + + lines: List[str] = [] + for idx, cue in enumerate(cues, start=1): + start = _format_timestamp_srt(cue["start_ms"]) + end = _format_timestamp_srt(cue["end_ms"]) + lines.append(str(idx)) + lines.append(f"{start} --> {end}") + lines.append(cue["text"]) + lines.append("") # blank line between cues + + return "\n".join(lines) + + +def render_soniox_tokens_as_vtt(tokens: List[Dict[str, Any]]) -> str: + """ + Render Soniox tokens as WebVTT subtitle format. + + Returns the VTT header even if no cues are present. + """ + cues = _group_tokens_into_cues(tokens) + + lines: List[str] = ["WEBVTT", ""] + for cue in cues: + start = _format_timestamp_vtt(cue["start_ms"]) + end = _format_timestamp_vtt(cue["end_ms"]) + lines.append(f"{start} --> {end}") + lines.append(cue["text"]) + lines.append("") # blank line between cues + + return "\n".join(lines) diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 3f945adca0d..e9f08f403f9 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -337,6 +337,7 @@ class ContextCachingEndpoints(VertexBase): return messages, optional_params, None tools = optional_params.pop("tools", None) + tool_choice = optional_params.pop("tool_choice", None) ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( @@ -371,7 +372,7 @@ class ContextCachingEndpoints(VertexBase): ## CHECK IF CACHED ALREADY generated_cache_key = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, model=model + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model ) google_cache_name = self.check_cache( cache_key=generated_cache_key, @@ -402,6 +403,8 @@ class ContextCachingEndpoints(VertexBase): ) cached_content_request_body["tools"] = tools + if tool_choice is not None: + cached_content_request_body["toolConfig"] = tool_choice ## LOGGING logging_obj.pre_call( @@ -487,6 +490,7 @@ class ContextCachingEndpoints(VertexBase): return messages, optional_params, None tools = optional_params.pop("tools", None) + tool_choice = optional_params.pop("tool_choice", None) ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( @@ -518,7 +522,7 @@ class ContextCachingEndpoints(VertexBase): ## CHECK IF CACHED ALREADY generated_cache_key = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, model=model + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model ) google_cache_name = await self.async_check_cache( cache_key=generated_cache_key, @@ -550,6 +554,8 @@ class ContextCachingEndpoints(VertexBase): ) cached_content_request_body["tools"] = tools + if tool_choice is not None: + cached_content_request_body["toolConfig"] = tool_choice ## LOGGING logging_obj.pre_call( diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 4f5846cc5b6..c578d6cd28b 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -996,7 +996,19 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 excluded_keys=["thoughtSignature"], ): assistant_content.append(gemini_tool_call_part) - last_message_with_tool_calls = assistant_msg + # Only record this as the active tool-call message when it actually + # carries tool calls. The `if` guard above is also entered for a + # text-only assistant message (`assistant_msg.get("tool_calls", []) + # is not None` is True for an empty list), so without this check a + # later assistant message with no tool calls would clobber the + # reference. The following tool result would then be matched against + # an assistant message that has no tool_calls, raising "Missing + # corresponding tool call for tool response message". + if ( + assistant_msg.get("tool_calls") + or assistant_msg.get("function_call") is not None + ): + last_message_with_tool_calls = assistant_msg ## HANDLE SERVER-SIDE TOOL INVOCATIONS (context circulation) _psf = assistant_msg.get("provider_specific_fields") @@ -1109,6 +1121,61 @@ def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None: data_dict[k] = v +def _has_google_maps_tool(tools: Optional[Any]) -> bool: + """Return True if any tool object in the list has a 'googleMaps' key.""" + if not isinstance(tools, list): + return False + return any( + isinstance(t, dict) and VertexToolName.GOOGLE_MAPS.value in t for t in tools + ) + + +def _rewrite_mime_type_to_response_format(generation_config: GenerationConfig) -> None: + """ + Convert response_mime_type + response_json_schema/response_schema to the newer + responseFormat structure when googleMaps is present in tools. + + The Gemini API rejects the combination of googleMaps + response_mime_type: + 'application/json' with the error: + "Google Maps tool with a response mime type: 'application/json' is unsupported" + + The newer responseFormat field supports this combination on both the Gemini API + (generativelanguage.googleapis.com) and Vertex AI endpoints. + + Before: + generationConfig: { + response_mime_type: "application/json", + response_json_schema: {...} + } + + After: + generationConfig: { + responseFormat: { + "text": {"mimeType": "APPLICATION_JSON", "schema": {...}} + } + } + """ + schema = generation_config.pop("response_json_schema", None) # type: ignore[misc] + if schema is None: + schema = generation_config.pop("response_schema", None) # type: ignore[misc] + generation_config.pop("response_mime_type", None) # type: ignore[misc] + + response_format: Dict[str, Any] = {"text": {"mimeType": "APPLICATION_JSON"}} + if schema is not None: + response_format["text"]["schema"] = schema + generation_config["responseFormat"] = response_format # type: ignore[typeddict-unknown-key] + + +def _rewrite_google_maps_response_format(data: RequestBody) -> None: + generation_config = cast(Optional[GenerationConfig], data.get("generationConfig")) + if ( + isinstance(generation_config, dict) + and _has_google_maps_tool(data.get("tools")) + and generation_config.get("response_mime_type") == "application/json" + ): + _rewrite_mime_type_to_response_format(generation_config) + + def _transform_request_body( # noqa: PLR0915 messages: List[AllMessageValues], model: str, @@ -1234,6 +1301,7 @@ def _transform_request_body( # noqa: PLR0915 if labels and custom_llm_provider != LlmProviders.GEMINI: data["labels"] = labels _pop_and_merge_extra_body(data, optional_params) + _rewrite_google_maps_response_format(data) except Exception as e: raise e diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 189ac7a7f6a..5cd02293f14 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1147,6 +1147,26 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return cast(dict, speech_config) + @staticmethod + def _apply_include_server_side_tool_invocations( + non_default_params: Dict, + optional_params: Dict, + ) -> None: + """ + Set include_server_side_tool_invocations before tools are mapped. + + map_openai_params iterates non_default_params in request order; if tools + appear before this flag, _resolve_search_tool_conflict would drop search + tools before the flag is applied. + """ + for key in ( + "include_server_side_tool_invocations", + "includeServerSideToolInvocations", + ): + if non_default_params.get(key) is True or optional_params.get(key) is True: + optional_params["include_server_side_tool_invocations"] = True + return + def map_openai_params( # noqa: PLR0915 self, non_default_params: Dict, @@ -1154,6 +1174,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): model: str, drop_params: bool, ) -> Dict: + self._apply_include_server_side_tool_invocations( + non_default_params, optional_params + ) gemini_sampling_params_warned: bool = False for param, value in non_default_params.items(): if param == "temperature": diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 4be4c2d5e78..1e92754857b 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -159,6 +159,6 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert "model", None ) # do not pass model in request body to vertex ai - sanitize_vertex_anthropic_output_params(anthropic_messages_request) + sanitize_vertex_anthropic_output_params(anthropic_messages_request, model) return anthropic_messages_request diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py index a33ad677789..280cc1c888a 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py @@ -10,23 +10,38 @@ import; extracting the helper into a leaf module resolves the warning and keeps the parent module's import surface narrow. """ -# Keys inside ``output_config`` that Vertex AI Claude does not accept. -# Add an entry only when a 400 "Extra inputs are not permitted" is -# reproducible against the live Vertex endpoint. +# Keys inside ``output_config`` that Vertex AI Claude rejects regardless of +# the target model. Add an entry only when a 400 "Extra inputs are not +# permitted" is reproducible against the live Vertex endpoint for every model. VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS: frozenset = frozenset() -def sanitize_vertex_anthropic_output_params(data: dict) -> None: +def _model_accepts_output_config_effort(model: str) -> bool: + """Whether ``model`` accepts ``output_config.effort`` on Vertex. + + Opus/Sonnet 4.6+ advertise ``supports_output_config`` (or a reasoning + effort level) and accept it; Haiku 4.5 advertises neither and 400s on + ``output_config.effort: Extra inputs are not permitted``. Imported lazily + so this stays a leaf module (see module docstring). + """ + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + return AnthropicConfig._model_supports_effort_param(model) + + +def sanitize_vertex_anthropic_output_params(data: dict, model: str) -> None: """ Strip Vertex-unsupported keys from ``output_config`` / ``output_format`` in-place; forward whatever remains. Behavior: - * ``output_config`` containing only unsupported keys (e.g. ``effort`` - alone) is removed entirely so the request body has no empty dict. - * ``output_config`` containing a mix of supported + unsupported keys - has the unsupported subset filtered out and the rest forwarded. - * ``output_config`` that is supported in full passes through unchanged. + * ``output_config.effort`` is dropped for models that don't accept it + (e.g. Haiku 4.5) and forwarded for those that do (Opus/Sonnet 4.6+). + Clients like Claude Code inject it into every Messages payload, so the + gate has to live here rather than rely on the caller. + * Keys in ``VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS`` are always filtered. + * ``output_config`` left empty after filtering is removed so the request + body has no empty dict. * ``output_format`` is forwarded as-is (Vertex AI Claude accepts it). * Non-dict values for ``output_config`` are dropped to avoid sending malformed payloads downstream. @@ -37,11 +52,19 @@ def sanitize_vertex_anthropic_output_params(data: dict) -> None: if not isinstance(output_config, dict): data.pop("output_config", None) return - sanitized = { - k: v - for k, v in output_config.items() - if k not in VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS - } + + drop_keys = set(VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS) + if "effort" in output_config and not _model_accepts_output_config_effort(model): + from litellm._logging import verbose_logger + + verbose_logger.debug( + "Dropping unsupported output_config.effort for vertex_ai model=%s " + "(no supports_output_config in the model map)", + model, + ) + drop_keys.add("effort") + + sanitized = {k: v for k, v in output_config.items() if k not in drop_keys} if sanitized: data["output_config"] = sanitized else: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 4627d9f6df3..c852909d475 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -106,7 +106,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): data.pop("model", None) # vertex anthropic doesn't accept 'model' parameter - sanitize_vertex_anthropic_output_params(data) + sanitize_vertex_anthropic_output_params(data, model) tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) diff --git a/litellm/llms/you_com/__init__.py b/litellm/llms/you_com/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/you_com/search/__init__.py b/litellm/llms/you_com/search/__init__.py new file mode 100644 index 00000000000..41bd9ce6b1a --- /dev/null +++ b/litellm/llms/you_com/search/__init__.py @@ -0,0 +1,7 @@ +""" +You.com Search API module. +""" + +from litellm.llms.you_com.search.transformation import YouComSearchConfig + +__all__ = ["YouComSearchConfig"] diff --git a/litellm/llms/you_com/search/transformation.py b/litellm/llms/you_com/search/transformation.py new file mode 100644 index 00000000000..3c94b991735 --- /dev/null +++ b/litellm/llms/you_com/search/transformation.py @@ -0,0 +1,193 @@ +""" +Calls You.com's /v1/search endpoint to search the web. + +You.com API Reference: https://you.com/docs/api-reference/search/v1-search +OpenAPI spec: https://you.com/specs/openapi_search_v1.yaml +""" + +from typing import Dict, List, Optional, TypedDict, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + + +class _YouComSearchRequestRequired(TypedDict): + """Required fields for You.com Search API request.""" + + query: str + + +class YouComSearchRequest(_YouComSearchRequestRequired, total=False): + """ + You.com Search API request format. + Based on: https://you.com/specs/openapi_search_v1.yaml + """ + + count: int + country: str + language: str + freshness: str + include_domains: List[str] + exclude_domains: List[str] + safesearch: str + + +class YouComSearchConfig(BaseSearchConfig): + # Keyed tier (higher rate limits): authenticate with X-API-Key. + YOU_COM_API_BASE = "https://ydc-index.io" + # Keyless free tier: IP-throttled (100 queries/day) and requires no auth. + # Used automatically when YOUCOM_API_KEY is not set. + YOU_COM_FREE_API_BASE = "https://api.you.com/v1/agents/search" + + @staticmethod + def ui_friendly_name() -> str: + return "You.com" + + def validate_environment( + self, + headers: Dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, + ) -> Dict: + """ + Set headers for the You.com Search API. + + If YOUCOM_API_KEY (or an explicit api_key) is present, use the keyed + endpoint with the `X-API-Key` header. Otherwise fall through to the + keyless free tier; no auth header is required. + """ + api_key = api_key or get_secret_str("YOUCOM_API_KEY") + headers["Content-Type"] = "application/json" + # Pin Accept-Encoding to identity: the keyless `api.you.com/v1/agents/search` + # endpoint advertises gzip content-encoding but returns body bytes the + # decoder rejects, which surfaces as httpx.DecodingError through litellm's + # http handler. Identity is harmless on the keyed endpoint. + headers.setdefault("Accept-Encoding", "identity") + if api_key: + headers["X-API-Key"] = api_key + return headers + + def get_complete_url( + self, + api_base: Optional[str], + optional_params: dict, + data: Optional[Union[Dict, List[Dict]]] = None, + **kwargs, + ) -> str: + """ + Pick the endpoint based on whether an API key is configured. + + - api_base explicit override -> use it as-is (normalized) + - YOUCOM_API_KEY set -> keyed endpoint (ydc-index.io/v1/search) + - no key -> keyless free tier (api.you.com/v1/agents/search) + """ + if api_base is None: + api_base = get_secret_str("YOUCOM_API_BASE") + + if api_base is None: + api_key = kwargs.get("api_key") or get_secret_str("YOUCOM_API_KEY") + if api_key: + api_base = self.YOU_COM_API_BASE + else: + # Keyless free tier already includes the full path. + return self.YOU_COM_FREE_API_BASE + + api_base = api_base.rstrip("/") + + if not api_base.endswith("/v1/search") and not api_base.endswith( + "/v1/agents/search" + ): + api_base = f"{api_base}/v1/search" + + return api_base + + def transform_search_request( + self, + query: Union[str, List[str]], + optional_params: dict, + **kwargs, + ) -> Dict: + """ + Transform Search request to You.com API format. + + Perplexity unified spec → You.com mappings: + - query → query + - max_results → count + - search_domain_filter → include_domains + - country → country + - max_tokens_per_page → (not applicable, ignored) + """ + if isinstance(query, list): + query = " ".join(query) + + request_data: YouComSearchRequest = { + "query": query, + } + + if "max_results" in optional_params: + request_data["count"] = optional_params["max_results"] + + if "search_domain_filter" in optional_params: + request_data["include_domains"] = optional_params["search_domain_filter"] + + if "country" in optional_params: + request_data["country"] = optional_params["country"].lower() + + result_data = dict(request_data) + + for param, value in optional_params.items(): + if ( + param not in self.get_supported_perplexity_optional_params() + and param not in result_data + ): + result_data[param] = value + + return result_data + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> SearchResponse: + """ + Transform You.com API response to LiteLLM unified SearchResponse format. + + You.com → LiteLLM mappings (for both `results.web[]` and `results.news[]`): + - title → SearchResult.title + - url → SearchResult.url + - snippets[0] → SearchResult.snippet (falls back to `description`) + - page_age → SearchResult.date + """ + response_json = raw_response.json() + raw_results = response_json.get("results") or {} + + web_results = raw_results.get("web") or [] + news_results = raw_results.get("news") or [] + + results: List[SearchResult] = [] + for item in list(web_results) + list(news_results): + snippets = item.get("snippets") or [] + snippet = snippets[0] if snippets else item.get("description", "") + results.append( + SearchResult( + title=item.get("title", ""), + url=item.get("url", ""), + snippet=snippet, + date=item.get("page_age"), + last_updated=None, + ) + ) + + return SearchResponse( + results=results, + object="search", + ) diff --git a/litellm/main.py b/litellm/main.py index 3ef094042e8..64891e2def9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -437,6 +437,7 @@ async def acompletion( # noqa: PLR0915 # Optional liteLLM function params thinking: Optional[AnthropicThinkingParam] = None, web_search_options: Optional[OpenAIWebSearchOptions] = None, + include_server_side_tool_invocations: Optional[bool] = None, # Session management shared_session: Optional["ClientSession"] = None, # Per-request JSON schema validation (overrides litellm.enable_json_schema_validation) @@ -584,6 +585,7 @@ async def acompletion( # noqa: PLR0915 "acompletion": True, # assuming this is a required parameter "thinking": thinking, "web_search_options": web_search_options, + "include_server_side_tool_invocations": include_server_side_tool_invocations, "shared_session": shared_session, "enable_json_schema_validation": enable_json_schema_validation, } @@ -641,6 +643,7 @@ async def acompletion( # noqa: PLR0915 if ( custom_llm_provider == "text-completion-openai" or custom_llm_provider == "text-completion-codestral" + or custom_llm_provider == "text-completion-inception" ) and isinstance(response, TextCompletionResponse): response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( response_object=response, @@ -1115,6 +1118,7 @@ def completion( # type: ignore # noqa: PLR0915 top_logprobs: Optional[int] = None, parallel_tool_calls: Optional[bool] = None, web_search_options: Optional[OpenAIWebSearchOptions] = None, + include_server_side_tool_invocations: Optional[bool] = None, deployment_id=None, extra_headers: Optional[dict] = None, safety_identifier: Optional[str] = None, @@ -1318,7 +1322,9 @@ def completion( # type: ignore # noqa: PLR0915 preset_cache_key = kwargs.get("preset_cache_key", None) hf_model_name = kwargs.get("hf_model_name", None) supports_system_message = kwargs.get("supports_system_message", None) - base_model = kwargs.get("base_model", None) + base_model = kwargs.get("base_model", None) or ( + model_info.get("base_model") if isinstance(model_info, dict) else None + ) ### DISABLE FLAGS ### disable_add_transform_inline_image_block = kwargs.get( "disable_add_transform_inline_image_block", None @@ -1530,11 +1536,7 @@ def completion( # type: ignore # noqa: PLR0915 "logit_bias": logit_bias, "user": user, # params to identify the model - "model": ( - model_info.get("base_model") - if isinstance(model_info, dict) and model_info.get("base_model") - else model - ), + "model": model, "custom_llm_provider": custom_llm_provider, "response_format": response_format, "seed": seed, @@ -1549,6 +1551,11 @@ def completion( # type: ignore # noqa: PLR0915 "reasoning_effort": reasoning_effort, "thinking": thinking, "web_search_options": web_search_options, + "include_server_side_tool_invocations": ( + include_server_side_tool_invocations + if include_server_side_tool_invocations is not None + else kwargs.get("include_server_side_tool_invocations") + ), "safety_identifier": safety_identifier, "service_tier": service_tier, "allowed_openai_params": kwargs.get("allowed_openai_params"), @@ -3803,6 +3810,67 @@ def completion( # type: ignore # noqa: PLR0915 ): return _model_response response = _model_response + elif custom_llm_provider == "text-completion-inception": + passed_api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + ) + api_base = ( + passed_api_base + or get_secret_str("INCEPTION_API_BASE") + or "https://api.inceptionlabs.ai/v1" + ) + # FIM is served at `/v1/fim/completions`; the OpenAI client appends + # `/completions`, so point it at the `/v1/fim` base. + api_base = api_base.rstrip("/") + if not api_base.endswith("/fim"): + api_base += "/fim" + + # Don't forward the server-managed Inception key to a caller-supplied + # api_base; only resolve it for the default/server base, or when the + # caller passes their own key. + if passed_api_base is None or api_key: + api_key = ( + api_key + or litellm.inception_key + or get_secret_str("INCEPTION_API_KEY") + ) + + _response = openai_text_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, # type: ignore[arg-type] + custom_llm_provider="text-completion-inception", + api_base=api_base, + acompletion=acompletion, + client=client, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + ) + + if ( + optional_params.get("stream", False) is False + and acompletion is False + and text_completion is False + ): + _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + + if optional_params.get("stream", False) or acompletion is True: + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + response = _response elif custom_llm_provider in ("sagemaker_chat", "sagemaker_nova"): # boto3 reads keys from .env # sagemaker_chat: HF Messages API endpoints @@ -6587,7 +6655,7 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: @client -def transcription( +def transcription( # noqa: PLR0915 model: str, file: FileTypes, ## OPTIONAL OPENAI PARAMS ## @@ -6779,6 +6847,35 @@ def transcription( else None ), ) + elif custom_llm_provider == "soniox": + from litellm.llms.soniox.audio_transcription.handler import ( + SonioxAudioTranscriptionHandler, + ) + + response = SonioxAudioTranscriptionHandler().audio_transcriptions( + model=model, + audio_file=file, + optional_params=optional_params, + litellm_params=litellm_params_dict, + model_response=model_response, + atranscription=atranscription, + client=( + client + if client is not None + and ( + isinstance(client, HTTPHandler) + or isinstance(client, AsyncHTTPHandler) + ) + else None + ), + timeout=timeout, + max_retries=max_retries, + logging_obj=litellm_logging_obj, + api_base=api_base, + api_key=api_key, + headers=extra_headers, + provider_config=provider_config, # type: ignore[arg-type] + ) elif provider_config is not None: response = base_llm_http_handler.audio_transcriptions( model=model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a604aafa540..397f96fdb1e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1075,6 +1075,7 @@ }, "eu.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1104,6 +1105,7 @@ }, "au.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1241,6 +1243,7 @@ }, "eu.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1271,6 +1274,7 @@ }, "au.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1315,6 +1319,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1346,6 +1351,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1377,6 +1383,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1408,6 +1415,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1439,6 +1447,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1454,6 +1463,36 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh" }, + "jp.anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": true + }, "anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -1543,6 +1582,7 @@ }, "eu.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1571,6 +1611,7 @@ }, "au.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1599,6 +1640,7 @@ }, "jp.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1995,11 +2037,13 @@ }, "au.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -2185,6 +2229,7 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -7494,6 +7539,27 @@ "supports_video_input": true, "supports_vision": true }, + "azure_ai/kimi-k2.6": { + "input_cost_per_token": 9.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k2-6-in-microsoft-foundry/4513125", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, "litellm_provider": "azure_ai", @@ -12682,7 +12748,8 @@ "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "supports_image_size": false }, "deepinfra/google/gemini-2.5-pro": { "max_tokens": 1000000, @@ -13583,6 +13650,7 @@ }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "deprecation_date": "2026-10-15", @@ -13787,11 +13855,13 @@ }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -13944,6 +14014,22 @@ "/v1/images/generations" ] }, + "fal_ai/fal-ai/nano-banana": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.039, + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/gemini-25-flash-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.039, + "supported_endpoints": [ + "/v1/images/generations" + ] + }, "featherless_ai/featherless-ai/Qwerky-72B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, @@ -15006,7 +15092,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, @@ -15056,7 +15143,8 @@ "supports_vision": true, "supports_web_search": false, "tpm": 8000000, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -15196,10 +15284,16 @@ "supports_service_tier": true }, "gemini-3.1-flash-lite": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_per_audio_token": 9e-08, - "input_cost_per_audio_token": 9e-07, - "input_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_per_audio_token": 5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_audio_token": 5e-07, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "vertex_ai-language-models", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -15211,9 +15305,12 @@ "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", - "output_cost_per_reasoning_token": 2.7e-06, - "output_cost_per_token": 2.7e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "output_cost_per_reasoning_token": 1.5e-06, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token_flex": 7.5e-07, + "output_cost_per_token_priority": 2.7e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -15336,7 +15433,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, @@ -15386,7 +15484,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -15436,7 +15535,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -15587,7 +15687,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -16597,7 +16698,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, @@ -16653,7 +16755,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -16832,7 +16935,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, @@ -16884,7 +16988,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -16936,7 +17041,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, @@ -17093,7 +17199,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 3e-07, @@ -17318,10 +17425,16 @@ "supports_service_tier": true }, "gemini/gemini-3.1-flash-lite": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_per_audio_token": 9e-08, - "input_cost_per_audio_token": 9e-07, - "input_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_per_audio_token": 5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_audio_token": 5e-07, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "gemini", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -17333,10 +17446,13 @@ "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", - "output_cost_per_reasoning_token": 2.7e-06, - "output_cost_per_token": 2.7e-06, + "output_cost_per_reasoning_token": 1.5e-06, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token_flex": 7.5e-07, + "output_cost_per_token_priority": 2.7e-06, "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -18098,23 +18214,22 @@ }, "github_copilot/claude-haiku-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" ], "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/claude-opus-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -18122,7 +18237,6 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, - "supports_reasoning": true, "supports_output_config": true }, "github_copilot/claude-opus-4.6-fast": { @@ -18138,22 +18252,6 @@ "supports_parallel_function_calling": true, "supports_vision": true }, - "github_copilot/claude-opus-4.7": { - "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/messages" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, "github_copilot/claude-opus-41": { "litellm_provider": "github_copilot", "max_input_tokens": 80000, @@ -18180,33 +18278,16 @@ }, "github_copilot/claude-sonnet-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" ], "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_reasoning": true - }, - "github_copilot/claude-sonnet-4.6": { - "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/messages" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gemini-2.5-pro": { "litellm_provider": "github_copilot", @@ -18216,25 +18297,7 @@ "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_reasoning": true - }, - "github_copilot/gemini-3-flash-preview": { - "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gemini-3-pro-preview": { "litellm_provider": "github_copilot", @@ -18246,30 +18309,13 @@ "supports_parallel_function_calling": true, "supports_vision": true }, - "github_copilot/gemini-3.1-pro-preview": { - "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true - }, "github_copilot/gpt-3.5-turbo": { "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_function_calling": true }, "github_copilot/gpt-3.5-turbo-0613": { "litellm_provider": "github_copilot", @@ -18277,10 +18323,7 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_function_calling": true }, "github_copilot/gpt-4": { "litellm_provider": "github_copilot", @@ -18288,22 +18331,7 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] - }, - "github_copilot/gpt-4-0125-preview": { - "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_function_calling": true, - "supports_parallel_function_calling": true + "supports_function_calling": true }, "github_copilot/gpt-4-0613": { "litellm_provider": "github_copilot", @@ -18311,22 +18339,16 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_function_calling": true }, "github_copilot/gpt-4-o-preview": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-4.1": { "litellm_provider": "github_copilot", @@ -18337,10 +18359,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4.1-2025-04-14": { "litellm_provider": "github_copilot", @@ -18351,89 +18370,68 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-41-copilot": { "litellm_provider": "github_copilot", - "mode": "chat" + "mode": "completion" }, "github_copilot/gpt-4o": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4o-2024-05-13": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4o-2024-08-06": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-4o-2024-11-20": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4o-mini": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-4o-mini-2024-07-18": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-5": { "litellm_provider": "github_copilot", @@ -18452,19 +18450,14 @@ }, "github_copilot/gpt-5-mini": { "litellm_provider": "github_copilot", - "max_input_tokens": 264000, + "max_input_tokens": 128000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gpt-5.1": { "litellm_provider": "github_copilot", @@ -18497,7 +18490,7 @@ }, "github_copilot/gpt-5.2": { "litellm_provider": "github_copilot", - "max_input_tokens": 264000, + "max_input_tokens": 128000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", @@ -18508,27 +18501,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.2-codex": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "supported_endpoints": [ - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gpt-5.3-codex": { "litellm_provider": "github_copilot", - "max_input_tokens": 400000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -18538,96 +18515,25 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.4": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.4-mini": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "supported_endpoints": [ - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.5": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "supported_endpoints": [ - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "github_copilot/oswe-vscode-prime": { - "litellm_provider": "github_copilot", - "max_input_tokens": 264000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true + "supports_vision": true }, "github_copilot/text-embedding-3-small": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding", - "supported_endpoints": [ - "/v1/embeddings" - ] + "mode": "embedding" }, "github_copilot/text-embedding-3-small-inference": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding", - "supported_endpoints": [ - "/v1/embeddings" - ] + "mode": "embedding" }, "github_copilot/text-embedding-ada-002": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding", - "supported_endpoints": [ - "/v1/embeddings" - ] + "mode": "embedding" }, "chatgpt/gpt-5.4": { "litellm_provider": "chatgpt", @@ -23278,11 +23184,13 @@ }, "jp.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -23308,6 +23216,7 @@ }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", @@ -23420,6 +23329,31 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "inception/mercury-2": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "inception", + "max_input_tokens": 128000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "text-completion-inception/mercury-edit-2": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "text-completion-inception", + "max_input_tokens": 32000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "completion", + "output_cost_per_token": 7.5e-07 + }, "lambda_ai/deepseek-llama3.3-70b": { "input_cost_per_token": 2e-07, "litellm_provider": "lambda_ai", @@ -24208,6 +24142,21 @@ "max_input_tokens": 200000, "max_output_tokens": 8192 }, + "minimax/MiniMax-M3": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 1.2e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true, + "max_input_tokens": 512000, + "max_output_tokens": 128000 + }, "mistral.devstral-2-123b": { "input_cost_per_token": 4e-07, "litellm_provider": "bedrock_converse", @@ -25090,6 +25039,7 @@ }, "moonshot/kimi-k2-0711-preview": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-05-25", "input_cost_per_token": 6e-07, "litellm_provider": "moonshot", "max_input_tokens": 131072, @@ -25104,6 +25054,7 @@ }, "moonshot/kimi-k2-0905-preview": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-05-25", "input_cost_per_token": 6e-07, "litellm_provider": "moonshot", "max_input_tokens": 262144, @@ -25118,6 +25069,7 @@ }, "moonshot/kimi-k2-turbo-preview": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-05-25", "input_cost_per_token": 1.15e-06, "litellm_provider": "moonshot", "max_input_tokens": 262144, @@ -25142,6 +25094,7 @@ "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true @@ -25158,12 +25111,14 @@ "source": "https://platform.kimi.ai/docs/pricing/chat-k26", "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, "moonshot/kimi-latest": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-01-28", "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", "max_input_tokens": 131072, @@ -25178,6 +25133,7 @@ }, "moonshot/kimi-latest-128k": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-01-28", "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", "max_input_tokens": 131072, @@ -25192,6 +25148,7 @@ }, "moonshot/kimi-latest-32k": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-01-28", "input_cost_per_token": 1e-06, "litellm_provider": "moonshot", "max_input_tokens": 32768, @@ -25206,6 +25163,7 @@ }, "moonshot/kimi-latest-8k": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-01-28", "input_cost_per_token": 2e-07, "litellm_provider": "moonshot", "max_input_tokens": 8192, @@ -25220,6 +25178,7 @@ }, "moonshot/kimi-thinking-preview": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2025-11-11", "input_cost_per_token": 6e-07, "litellm_provider": "moonshot", "max_input_tokens": 131072, @@ -25232,6 +25191,7 @@ }, "moonshot/kimi-k2-thinking": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-05-25", "input_cost_per_token": 6e-07, "litellm_provider": "moonshot", "max_input_tokens": 262144, @@ -25247,6 +25207,7 @@ }, "moonshot/kimi-k2-thinking-turbo": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-05-25", "input_cost_per_token": 1.15e-06, "litellm_provider": "moonshot", "max_input_tokens": 262144, @@ -25270,9 +25231,11 @@ "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "moonshot/moonshot-v1-128k-0430": { + "deprecation_date": "2024-04-30", "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", "max_input_tokens": 131072, @@ -25294,6 +25257,7 @@ "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true }, @@ -25307,9 +25271,11 @@ "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "moonshot/moonshot-v1-32k-0430": { + "deprecation_date": "2024-04-30", "input_cost_per_token": 1e-06, "litellm_provider": "moonshot", "max_input_tokens": 32768, @@ -25331,6 +25297,7 @@ "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true }, @@ -25344,9 +25311,11 @@ "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "moonshot/moonshot-v1-8k-0430": { + "deprecation_date": "2024-04-30", "input_cost_per_token": 2e-07, "litellm_provider": "moonshot", "max_input_tokens": 8192, @@ -25368,6 +25337,7 @@ "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true }, @@ -25381,6 +25351,7 @@ "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "morph/morph-v3-fast": { @@ -26432,6 +26403,32 @@ "supports_vision": true, "supports_web_search": true }, + "oci/meta.llama-3.1-8b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/meta.llama-3.1-70b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true + }, "oci/meta.llama-3.1-405b-instruct": { "input_cost_per_token": 1.068e-05, "litellm_provider": "oci", @@ -26442,7 +26439,8 @@ "output_cost_per_token": 1.068e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/meta.llama-3.2-90b-vision-instruct": { "input_cost_per_token": 2e-06, @@ -26455,6 +26453,7 @@ "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, "supports_response_schema": false, + "supports_native_streaming": true, "supports_vision": true }, "oci/meta.llama-3.3-70b-instruct": { @@ -26467,31 +26466,35 @@ "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/meta.llama-4-maverick-17b-128e-instruct-fp8": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", - "max_input_tokens": 512000, - "max_output_tokens": 4000, - "max_tokens": 4000, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true, + "supports_vision": true }, "oci/meta.llama-4-scout-17b-16e-instruct": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", - "max_input_tokens": 192000, - "max_output_tokens": 4000, - "max_tokens": 4000, + "max_input_tokens": 10485760, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3": { "input_cost_per_token": 3e-06, @@ -26503,7 +26506,8 @@ "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-fast": { "input_cost_per_token": 5e-06, @@ -26515,7 +26519,8 @@ "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-mini": { "input_cost_per_token": 3e-07, @@ -26527,7 +26532,8 @@ "output_cost_per_token": 5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-mini-fast": { "input_cost_per_token": 6e-07, @@ -26539,7 +26545,8 @@ "output_cost_per_token": 4e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-4": { "input_cost_per_token": 3e-06, @@ -26551,7 +26558,8 @@ "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-latest": { "input_cost_per_token": 1.56e-06, @@ -26563,7 +26571,8 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-a-03-2025": { "input_cost_per_token": 1.56e-06, @@ -26575,7 +26584,8 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-plus-latest": { "input_cost_per_token": 1.56e-06, @@ -26587,7 +26597,88 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true, + "supports_image_size": false + }, + "oci/google.gemini-2.5-pro": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-flash-lite": { + "input_cost_per_token": 7.5e-08, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true, + "supports_image_size": false + }, + "oci/cohere.command-a-vision": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true, + "supports_vision": true + }, + "oci/cohere.command-a-reasoning": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_function_calling": false, + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/cohere.embed-multilingual-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "mode": "embedding", + "output_vector_size": 1024, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { "input_cost_per_token": 1.56e-06, @@ -26663,18 +26754,6 @@ "supports_response_schema": false, "supports_vision": true }, - "oci/meta.llama-3.1-70b-instruct": { - "input_cost_per_token": 7.2e-07, - "litellm_provider": "oci", - "max_input_tokens": 128000, - "max_output_tokens": 4000, - "max_tokens": 4000, - "mode": "chat", - "output_cost_per_token": 7.2e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": false - }, "oci/meta.llama-3.3-70b-instruct-fp8-dynamic": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", @@ -26792,45 +26871,6 @@ "supports_response_schema": true, "supports_vision": true }, - "oci/google.gemini-2.5-pro": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true - }, - "oci/google.gemini-2.5-flash": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true - }, - "oci/google.gemini-2.5-flash-lite": { - "input_cost_per_token": 7.5e-08, - "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true - }, "oci/cohere.embed-english-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "oci", @@ -27638,7 +27678,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_image_size": false }, "openrouter/google/gemini-2.5-pro": { "input_cost_per_audio_token": 7e-07, @@ -29508,7 +29549,8 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "supports_image_size": false }, "perplexity/xai/grok-4-1-fast-non-reasoning": { "litellm_provider": "perplexity", @@ -30090,7 +30132,8 @@ "supports_vision": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_image_size": false }, "replicate/openai/gpt-oss-120b": { "input_cost_per_token": 1.8e-07, @@ -31909,6 +31952,7 @@ }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", @@ -32788,7 +32832,8 @@ "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_image_size": false }, "vercel_ai_gateway/google/gemini-2.5-pro": { "input_cost_per_token": 2.5e-06, @@ -33555,6 +33600,7 @@ }, "vertex_ai/claude-haiku-4-5": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33576,6 +33622,7 @@ }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33626,6 +33673,7 @@ }, "vertex_ai/claude-3-7-sonnet@20250219": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-05-11", "input_cost_per_token": 3e-06, @@ -33725,6 +33773,7 @@ }, "vertex_ai/claude-opus-4": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "vertex_ai-anthropic_models", @@ -33750,6 +33799,7 @@ }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, @@ -33767,6 +33817,7 @@ }, "vertex_ai/claude-opus-4-1@20250805": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, @@ -33784,6 +33835,7 @@ }, "vertex_ai/claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33810,6 +33862,7 @@ }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33837,6 +33890,7 @@ }, "vertex_ai/claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33864,6 +33918,7 @@ }, "vertex_ai/claude-opus-4-6@default": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33891,6 +33946,7 @@ }, "vertex_ai/claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33918,6 +33974,7 @@ }, "vertex_ai/claude-opus-4-7@default": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33959,6 +34016,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -33987,6 +34045,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34001,6 +34060,7 @@ }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34027,6 +34087,7 @@ }, "vertex_ai/claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -34054,6 +34115,7 @@ }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34081,6 +34143,7 @@ }, "vertex_ai/claude-opus-4@20250514": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "vertex_ai-anthropic_models", @@ -34106,6 +34169,7 @@ }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34135,6 +34199,7 @@ }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34342,7 +34407,8 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": false, - "tpm": 8000000 + "tpm": 8000000, + "supports_image_size": false }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -34430,10 +34496,16 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-flash-lite": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_per_audio_token": 9e-08, - "input_cost_per_audio_token": 9e-07, - "input_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_per_audio_token": 5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_audio_token": 5e-07, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "vertex_ai-language-models", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -34445,8 +34517,11 @@ "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", - "output_cost_per_reasoning_token": 2.7e-06, - "output_cost_per_token": 2.7e-06, + "output_cost_per_reasoning_token": 1.5e-06, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token_flex": 7.5e-07, + "output_cost_per_token_priority": 2.7e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -36020,7 +36095,8 @@ "supports_prompt_caching": true, "supports_response_schema": false, "supports_tool_choice": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-3-beta": { "cache_read_input_token_cost": 7.5e-07, @@ -36219,7 +36295,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-fast-non-reasoning": { "cache_read_input_token_cost": 5e-08, @@ -36236,7 +36313,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-0709": { "input_cost_per_token": 3e-06, @@ -36252,7 +36330,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-latest": { "input_cost_per_token": 3e-06, @@ -36310,7 +36389,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-1-fast-reasoning-latest": { "cache_read_input_token_cost": 5e-08, @@ -36331,7 +36411,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-1-fast-non-reasoning": { "cache_read_input_token_cost": 5e-08, @@ -36351,7 +36432,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-1-fast-non-reasoning-latest": { "cache_read_input_token_cost": 5e-08, @@ -36371,7 +36453,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4.20-multi-agent-beta-0309": { "cache_read_input_token_cost": 2e-07, @@ -36522,7 +36605,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "deprecation_date": "2026-05-15" }, "xai/grok-code-fast-1-0825": { "cache_read_input_token_cost": 2e-08, @@ -36537,7 +36621,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "deprecation_date": "2026-05-15" }, "xai/grok-vision-beta": { "input_cost_per_image": 5e-06, @@ -41151,6 +41236,7 @@ }, "vertex_ai/claude-sonnet-4-6@default": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -41238,6 +41324,44 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "bedrock_mantle/openai.gpt-5.5": { + "input_cost_per_token": 5.5e-06, + "cache_read_input_token_cost": 5.5e-07, + "output_cost_per_token": 3.3e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/openai.gpt-5.4": { + "input_cost_per_token": 2.75e-06, + "cache_read_input_token_cost": 2.75e-07, + "output_cost_per_token": 1.65e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "volcengine/doubao-seed-2-0-pro-260215": { "litellm_provider": "volcengine", "max_input_tokens": 256000, @@ -41516,5 +41640,18 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_pdf_input": true + }, + "soniox/stt-async-v4": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 0.0000277778, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true } -} +} \ No newline at end of file diff --git a/litellm/models/__init__.py b/litellm/models/__init__.py new file mode 100644 index 00000000000..7e2d2c0ed9d --- /dev/null +++ b/litellm/models/__init__.py @@ -0,0 +1,66 @@ +""" +Domain models for LiteLLM backend. +""" + +from litellm.models.access_group import LiteLLM_AccessGroupTable +from litellm.models.budget import ( + LiteLLM_BudgetTable, + LiteLLM_BudgetTableFull, + LiteLLM_TeamMemberTable, +) +from litellm.models.config import LiteLLM_Config +from litellm.models.credentials import ( + CreateCredentialItem, + CredentialBase, + CredentialItem, +) +from litellm.models.end_user import LiteLLM_EndUserTable +from litellm.models.managed_files import ( + LiteLLM_ManagedFileTable, + LiteLLM_ManagedObjectTable, + LiteLLM_ManagedVectorStoresTable, + LiteLLM_ManagedVectorStoreTable, +) +from litellm.models.mcp_server import LiteLLM_MCPServerTable +from litellm.models.model import LiteLLM_ProxyModelTable +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.models.organization import LiteLLM_OrganizationTable +from litellm.models.organization_membership import LiteLLM_OrganizationMembershipTable +from litellm.models.project import LiteLLM_ProjectTable +from litellm.models.skills import LiteLLM_SkillsTable +from litellm.models.spend_logs import LiteLLM_ErrorLogs, LiteLLM_SpendLogs +from litellm.models.tag import LiteLLM_TagTable +from litellm.models.team import LiteLLM_TeamTable +from litellm.models.team_membership import LiteLLM_TeamMembership +from litellm.models.user import LiteLLM_UserTable +from litellm.models.verification_token import LiteLLM_VerificationToken + +__all__ = [ + "LiteLLM_AccessGroupTable", + "LiteLLM_BudgetTable", + "LiteLLM_BudgetTableFull", + "LiteLLM_TeamMemberTable", + "LiteLLM_Config", + "CredentialBase", + "CredentialItem", + "CreateCredentialItem", + "LiteLLM_EndUserTable", + "LiteLLM_ManagedFileTable", + "LiteLLM_ManagedObjectTable", + "LiteLLM_ManagedVectorStoreTable", + "LiteLLM_ManagedVectorStoresTable", + "LiteLLM_MCPServerTable", + "LiteLLM_ProxyModelTable", + "LiteLLM_ObjectPermissionTable", + "LiteLLM_OrganizationTable", + "LiteLLM_OrganizationMembershipTable", + "LiteLLM_ProjectTable", + "LiteLLM_SkillsTable", + "LiteLLM_ErrorLogs", + "LiteLLM_SpendLogs", + "LiteLLM_TagTable", + "LiteLLM_TeamTable", + "LiteLLM_TeamMembership", + "LiteLLM_UserTable", + "LiteLLM_VerificationToken", +] diff --git a/litellm/models/access_group.py b/litellm/models/access_group.py new file mode 100644 index 00000000000..682e779e531 --- /dev/null +++ b/litellm/models/access_group.py @@ -0,0 +1,26 @@ +""" +Access group table model. + +Canonical definition for ``litellm_accessgrouptable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from datetime import datetime +from typing import List, Optional + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_AccessGroupTable(LiteLLMPydanticObjectBase): + access_group_id: str + access_group_name: str + description: Optional[str] = None + access_model_names: List[str] = [] + access_mcp_server_ids: List[str] = [] + access_agent_ids: List[str] = [] + assigned_team_ids: List[str] = [] + assigned_key_ids: List[str] = [] + created_at: Optional[datetime] = None + created_by: Optional[str] = None + updated_at: Optional[datetime] = None + updated_by: Optional[str] = None diff --git a/litellm/models/base.py b/litellm/models/base.py new file mode 100644 index 00000000000..01981297bd5 --- /dev/null +++ b/litellm/models/base.py @@ -0,0 +1,38 @@ +""" +Base model class for domain models. +""" + +from datetime import datetime +from typing import Any, Dict, Optional + +from pydantic import BaseModel, ConfigDict + + +class DomainModel(BaseModel): + """Base class for all domain models.""" + + model_config = ConfigDict( + from_attributes=True, + protected_namespaces=(), + extra="ignore", + ) + + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + + @classmethod + def from_db_record(cls, record: Any) -> "DomainModel": + """Create a domain model from a database record.""" + if record is None: + raise ValueError("Cannot create domain model from None record") + if isinstance(record, dict): + return cls(**record) + if hasattr(record, "model_dump") and callable(record.model_dump): + return cls(**record.model_dump()) + if hasattr(record, "dict") and callable(record.dict): + return cls(**record.dict()) + return cls(**dict(record)) + + def to_db_dict(self, exclude_unset: bool = False) -> Dict[str, Any]: + """Convert domain model to a dictionary for database operations.""" + return self.model_dump(exclude_none=True, exclude_unset=exclude_unset) diff --git a/litellm/models/budget.py b/litellm/models/budget.py new file mode 100644 index 00000000000..e7dfe2f8fbc --- /dev/null +++ b/litellm/models/budget.py @@ -0,0 +1,56 @@ +""" +Budget table model. + +Canonical definition for ``litellm_budgettable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from datetime import datetime +from typing import List, Optional + +from pydantic import ConfigDict + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase): + """Represents user-controllable params for a LiteLLM_BudgetTable record. + + Budget-write paths use `model_fields.keys()` on this class as an allowlist + for user input. Keep server-managed fields (e.g. `budget_reset_at`) on + `LiteLLM_BudgetTableFull` so they aren't user-settable. + """ + + budget_id: Optional[str] = None + soft_budget: Optional[float] = None + max_budget: Optional[float] = None + max_parallel_requests: Optional[int] = None + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + model_max_budget: Optional[dict] = None + budget_duration: Optional[str] = None + allowed_models: Optional[List[str]] = ( + None # per-member model scope; empty = inherit team models + ) + + model_config = ConfigDict(protected_namespaces=()) + + +class LiteLLM_BudgetTableFull(LiteLLM_BudgetTable): + """LiteLLM_BudgetTable + server-managed fields returned on API responses.""" + + budget_reset_at: Optional[datetime] = None + created_at: datetime + + +class LiteLLM_TeamMemberTable(LiteLLM_BudgetTable): + """ + Used to track spend of a user_id within a team_id + """ + + spend: Optional[float] = None + user_id: Optional[str] = None + team_id: Optional[str] = None + budget_id: Optional[str] = None + + model_config = ConfigDict(protected_namespaces=()) diff --git a/litellm/models/config.py b/litellm/models/config.py new file mode 100644 index 00000000000..99b5c5692fd --- /dev/null +++ b/litellm/models/config.py @@ -0,0 +1,15 @@ +""" +Config table model. + +Canonical definition for ``litellm_config``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from typing import Dict + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_Config(LiteLLMPydanticObjectBase): + param_name: str + param_value: Dict diff --git a/litellm/models/credentials.py b/litellm/models/credentials.py new file mode 100644 index 00000000000..b74ea055d21 --- /dev/null +++ b/litellm/models/credentials.py @@ -0,0 +1,31 @@ +""" +Credential table models. + +These are the canonical credential types for the proxy. They live in the model +layer; ``litellm.types.utils`` re-exports them for backwards compatibility. +""" + +from typing import Optional + +from pydantic import BaseModel, model_validator + + +class CredentialBase(BaseModel): + credential_name: str + credential_info: dict + + +class CredentialItem(CredentialBase): + credential_values: dict + + +class CreateCredentialItem(CredentialBase): + credential_values: Optional[dict] = None + model_id: Optional[str] = None + + @model_validator(mode="before") + @classmethod + def check_credential_params(cls, values): + if not values.get("credential_values") and not values.get("model_id"): + raise ValueError("Either credential_values or model_id must be set") + return values diff --git a/litellm/models/end_user.py b/litellm/models/end_user.py new file mode 100644 index 00000000000..15fd03ec2ca --- /dev/null +++ b/litellm/models/end_user.py @@ -0,0 +1,35 @@ +""" +End-user table model. + +Canonical definition for ``litellm_endusertable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from typing import Literal, Optional + +from pydantic import ConfigDict, model_validator + +from litellm.models.budget import LiteLLM_BudgetTable +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_EndUserTable(LiteLLMPydanticObjectBase): + user_id: str + blocked: bool + alias: Optional[str] = None + spend: float = 0.0 + allowed_model_region: Optional[Literal["eu", "us"]] = None + default_model: Optional[str] = None + litellm_budget_table: Optional[LiteLLM_BudgetTable] = None + object_permission_id: Optional[str] = None + object_permission: Optional[LiteLLM_ObjectPermissionTable] = None + + @model_validator(mode="before") + @classmethod + def set_model_info(cls, values): + if values.get("spend") is None: + values.update({"spend": 0.0}) + return values + + model_config = ConfigDict(protected_namespaces=()) diff --git a/litellm/models/managed_files.py b/litellm/models/managed_files.py new file mode 100644 index 00000000000..24154768860 --- /dev/null +++ b/litellm/models/managed_files.py @@ -0,0 +1,62 @@ +""" +Managed file, object, and vector store table models. + +Canonical definitions for the ``litellm_managed*`` tables. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from datetime import datetime +from typing import Any, Dict, List, Literal, Optional, Union + +from litellm.types.llms.base import LiteLLMPydanticObjectBase +from litellm.types.llms.openai import OpenAIFileObject, ResponsesAPIResponse +from litellm.types.utils import LiteLLMBatch, LiteLLMFineTuningJob + + +class LiteLLM_ManagedFileTable(LiteLLMPydanticObjectBase): + unified_file_id: str + file_object: Optional[OpenAIFileObject] = None + model_mappings: Dict[str, str] + flat_model_file_ids: List[str] + created_by: Optional[str] = None + team_id: Optional[str] = None + updated_by: Optional[str] = None + storage_backend: Optional[str] = None + storage_url: Optional[str] = None + + +class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase): + unified_object_id: str + model_object_id: str + file_purpose: Literal["batch", "fine-tune", "response", "container"] + file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob, ResponsesAPIResponse] + created_by: Optional[str] = None + team_id: Optional[str] = None + + +class LiteLLM_ManagedVectorStoreTable(LiteLLMPydanticObjectBase): + """Table for managing vector stores with target_model_names support.""" + + unified_resource_id: str + resource_object: Optional[Any] = None + model_mappings: Dict[str, str] + flat_model_resource_ids: List[str] + created_by: Optional[str] = None + team_id: Optional[str] = None + updated_by: Optional[str] = None + storage_backend: Optional[str] = None + storage_url: Optional[str] = None + + +class LiteLLM_ManagedVectorStoresTable(LiteLLMPydanticObjectBase): + vector_store_id: str + custom_llm_provider: str + vector_store_name: Optional[str] + vector_store_description: Optional[str] + vector_store_metadata: Optional[Dict[str, Any]] + created_at: Optional[datetime] + updated_at: Optional[datetime] + litellm_credential_name: Optional[str] + litellm_params: Optional[Dict[str, Any]] + team_id: Optional[str] + user_id: Optional[str] diff --git a/litellm/models/mcp_server.py b/litellm/models/mcp_server.py new file mode 100644 index 00000000000..3d03eff6df8 --- /dev/null +++ b/litellm/models/mcp_server.py @@ -0,0 +1,103 @@ +""" +MCP server table model. + +Canonical definition for ``litellm_mcpservertable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +import enum +from datetime import datetime +from typing import Dict, List, Literal, Optional + +from pydantic import Field + +from litellm.types.llms.base import LiteLLMPydanticObjectBase +from litellm.types.mcp import MCPAuthType, MCPCredentials, MCPTransportType +from litellm.types.mcp_server.mcp_server_manager import MCPInfo + + +class MCPEnvVarScope(str, enum.Enum): + """Scope for an MCP server environment variable. + + - ``global``: value is provided by the admin and used for all users. + - ``user``: each user must provide their own value via the per-user + env-var endpoint. The admin-supplied ``value`` is treated as a + placeholder/hint and is not used at request time. + """ + + global_ = "global" + user = "user" + + +class MCPEnvVar(LiteLLMPydanticObjectBase): + """One environment variable for an MCP server. + + Variables can be interpolated into ``static_headers`` using ``${NAME}`` + syntax. ``scope=global`` values are stored on the server. ``scope=user`` + values are stored per-user in ``LiteLLM_MCPUserEnvVars`` and supplied by + each user. + """ + + name: str + value: str = "" + scope: MCPEnvVarScope = MCPEnvVarScope.global_ + description: Optional[str] = None + + +class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase): + """Represents a LiteLLM_MCPServerTable record""" + + server_id: str + server_name: Optional[str] = None + alias: Optional[str] = None + description: Optional[str] = None + url: Optional[str] = None + spec_path: Optional[str] = None + transport: MCPTransportType + auth_type: Optional[MCPAuthType] = None + credentials: Optional[MCPCredentials] = None + instructions: Optional[str] = None + created_at: Optional[datetime] = None + created_by: Optional[str] = None + updated_at: Optional[datetime] = None + updated_by: Optional[str] = None + teams: List[Dict[str, Optional[str]]] = Field(default_factory=list) + mcp_access_groups: List[str] = Field(default_factory=list) + allowed_tools: List[str] = Field(default_factory=list) + tool_name_to_display_name: Optional[Dict[str, str]] = None + tool_name_to_description: Optional[Dict[str, str]] = None + extra_headers: List[str] = Field(default_factory=list) + mcp_info: Optional[MCPInfo] = None + static_headers: Optional[Dict[str, str]] = None + env_vars: Optional[List[MCPEnvVar]] = None + status: Optional[Literal["healthy", "unhealthy", "unknown"]] = Field( + default="unknown", + description="Health status: 'healthy', 'unhealthy', 'unknown'", + ) + last_health_check: Optional[datetime] = None + health_check_error: Optional[str] = None + command: Optional[str] = None + args: List[str] = Field(default_factory=list) + env: Dict[str, str] = Field(default_factory=dict) + authorization_url: Optional[str] = None + token_url: Optional[str] = None + registration_url: Optional[str] = None + oauth2_flow: Optional[Literal["client_credentials", "authorization_code"]] = None + allow_all_keys: bool = False + available_on_public_internet: bool = True + delegate_auth_to_upstream: bool = False + oauth_passthrough: bool = False + is_byok: bool = False + byok_description: List[str] = Field(default_factory=list) + byok_api_key_help_url: Optional[str] = None + has_user_credential: Optional[bool] = None + source_url: Optional[str] = None + timeout: Optional[float] = None + approval_status: Optional[str] = Field( + default="active", + description="Approval status: 'pending_review', 'active', 'rejected'", + ) + submitted_by: Optional[str] = None + submitted_at: Optional[datetime] = None + reviewed_at: Optional[datetime] = None + review_notes: Optional[str] = None diff --git a/litellm/models/model.py b/litellm/models/model.py new file mode 100644 index 00000000000..7657e4d30f8 --- /dev/null +++ b/litellm/models/model.py @@ -0,0 +1,59 @@ +""" +Proxy model table model. + +Canonical definition for ``litellm_proxymodeltable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +import json +from datetime import datetime +from typing import Optional + +from pydantic import ConfigDict, model_validator + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_ProxyModelTable(LiteLLMPydanticObjectBase): + model_id: str + model_name: str + litellm_params: dict + model_info: Optional[dict] = None + blocked: bool = False + created_at: Optional[datetime] = None + created_by: Optional[str] = None + updated_at: Optional[datetime] = None + updated_by: Optional[str] = None + + model_config = ConfigDict(protected_namespaces=()) + + @model_validator(mode="before") + @classmethod + def check_potential_json_str(cls, values): + if isinstance(values.get("litellm_params"), str): + try: + values["litellm_params"] = json.loads(values["litellm_params"]) + except json.JSONDecodeError: + pass + if isinstance(values.get("model_info"), str): + try: + values["model_info"] = json.loads(values["model_info"]) + except json.JSONDecodeError: + pass + return values + + @property + def is_blocked(self) -> bool: + return self.blocked + + @property + def team_id(self) -> Optional[str]: + if self.model_info: + return self.model_info.get("team_id") + return None + + @property + def team_public_model_name(self) -> Optional[str]: + if self.model_info: + return self.model_info.get("team_public_model_name") + return None diff --git a/litellm/models/object_permission.py b/litellm/models/object_permission.py new file mode 100644 index 00000000000..6c0d100046c --- /dev/null +++ b/litellm/models/object_permission.py @@ -0,0 +1,26 @@ +""" +Object permission table model. + +Canonical definition for ``litellm_objectpermissiontable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from typing import Dict, List, Optional + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_ObjectPermissionTable(LiteLLMPydanticObjectBase): + """Represents a LiteLLM_ObjectPermissionTable record""" + + object_permission_id: str + mcp_servers: Optional[List[str]] = [] + mcp_access_groups: Optional[List[str]] = [] + mcp_tool_permissions: Optional[Dict[str, List[str]]] = None + vector_stores: Optional[List[str]] = [] + agents: Optional[List[str]] = [] + agent_access_groups: Optional[List[str]] = [] + models: Optional[List[str]] = [] + mcp_toolsets: Optional[List[str]] = None + blocked_tools: Optional[List[str]] = [] + search_tools: Optional[List[str]] = [] diff --git a/litellm/models/organization.py b/litellm/models/organization.py new file mode 100644 index 00000000000..8b2d95c3e09 --- /dev/null +++ b/litellm/models/organization.py @@ -0,0 +1,31 @@ +""" +Organization table model. + +Canonical definition for ``litellm_organizationtable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from typing import List, Optional + +from litellm.models.budget import LiteLLM_BudgetTable +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.models.user import LiteLLM_UserTable +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_OrganizationTable(LiteLLMPydanticObjectBase): + """Represents user-controllable params for a LiteLLM_OrganizationTable record""" + + organization_id: Optional[str] = None + organization_alias: Optional[str] = None + budget_id: str + spend: float = 0.0 + metadata: Optional[dict] = None + models: List[str] = [] + model_spend: Optional[dict] = {} + created_by: str + updated_by: str + users: Optional[List[LiteLLM_UserTable]] = None + litellm_budget_table: Optional[LiteLLM_BudgetTable] = None + object_permission: Optional[LiteLLM_ObjectPermissionTable] = None + object_permission_id: Optional[str] = None diff --git a/litellm/models/organization_membership.py b/litellm/models/organization_membership.py new file mode 100644 index 00000000000..9957c0c21af --- /dev/null +++ b/litellm/models/organization_membership.py @@ -0,0 +1,40 @@ +""" +Organization membership table model. + +Canonical definition for ``litellm_organizationmembership``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from datetime import datetime +from typing import Any, Optional + +from pydantic import ConfigDict, model_validator + +from litellm.models.budget import LiteLLM_BudgetTable +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase): + """Tracks which organizations a user belongs to and their spend within it.""" + + user_id: str + organization_id: str + user_role: Optional[str] = None + spend: float = 0.0 + budget_id: Optional[str] = None + created_at: datetime + updated_at: datetime + user: Optional[Any] = None + litellm_budget_table: Optional[LiteLLM_BudgetTable] = None + user_email: Optional[str] = None + + model_config = ConfigDict(protected_namespaces=()) + + @model_validator(mode="after") + def populate_user_email(self) -> "LiteLLM_OrganizationMembershipTable": + if self.user_email is None and self.user is not None: + if isinstance(self.user, dict): + self.user_email = self.user.get("user_email") + else: + self.user_email = getattr(self.user, "user_email", None) + return self diff --git a/litellm/models/project.py b/litellm/models/project.py new file mode 100644 index 00000000000..083c7ee3cc5 --- /dev/null +++ b/litellm/models/project.py @@ -0,0 +1,41 @@ +""" +Project table model. + +Canonical definition for ``litellm_projecttable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from datetime import datetime +from typing import List, Optional + +from litellm.models.budget import LiteLLM_BudgetTable +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_ProjectTable(LiteLLMPydanticObjectBase): + """Database model representation for project""" + + project_id: str + project_alias: Optional[str] = None + description: Optional[str] = None + team_id: Optional[str] = None + budget_id: Optional[str] = None + metadata: Optional[dict] = None + models: List[str] = [] + spend: float = 0.0 + model_spend: Optional[dict] = None + model_rpm_limit: Optional[dict] = None + model_tpm_limit: Optional[dict] = None + blocked: bool = False + object_permission_id: Optional[str] = None + created_by: Optional[str] = None + updated_by: Optional[str] = None + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + litellm_budget_table: Optional[LiteLLM_BudgetTable] = None + object_permission: Optional[LiteLLM_ObjectPermissionTable] = None + + @property + def is_blocked(self) -> bool: + return self.blocked diff --git a/litellm/models/skills.py b/litellm/models/skills.py new file mode 100644 index 00000000000..62091c0ca01 --- /dev/null +++ b/litellm/models/skills.py @@ -0,0 +1,30 @@ +""" +Skills table model. + +Canonical definition for ``litellm_skillstable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from datetime import datetime +from typing import Any, Dict, Optional + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_SkillsTable(LiteLLMPydanticObjectBase): + """Represents a LiteLLM_SkillsTable record""" + + skill_id: str + display_title: Optional[str] = None + description: Optional[str] = None + instructions: Optional[str] = None + source: str = "custom" + latest_version: Optional[str] = None + file_content: Optional[bytes] = None + file_name: Optional[str] = None + file_type: Optional[str] = None + metadata: Optional[Dict[str, Any]] = None + created_at: Optional[datetime] = None + created_by: Optional[str] = None + updated_at: Optional[datetime] = None + updated_by: Optional[str] = None diff --git a/litellm/models/spend_logs.py b/litellm/models/spend_logs.py new file mode 100644 index 00000000000..96bd328c3ca --- /dev/null +++ b/litellm/models/spend_logs.py @@ -0,0 +1,50 @@ +""" +Spend and error log table models. + +Canonical definitions for ``litellm_spendlogs`` and ``litellm_errorlogs``. +Re-exported from ``litellm.proxy._types`` for backwards compatibility. +""" + +from datetime import datetime +from typing import Optional, Union + +from pydantic import Json + +from litellm._uuid import uuid +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_SpendLogs(LiteLLMPydanticObjectBase): + request_id: str + api_key: str + model: Optional[str] = "" + api_base: Optional[str] = "" + call_type: str + spend: Optional[float] = 0.0 + total_tokens: Optional[int] = 0 + prompt_tokens: Optional[int] = 0 + completion_tokens: Optional[int] = 0 + startTime: Union[str, datetime, None] + endTime: Union[str, datetime, None] + user: Optional[str] = "" + metadata: Optional[Json] = {} + cache_hit: Optional[str] = "False" + cache_key: Optional[str] = None + request_tags: Optional[Json] = None + requester_ip_address: Optional[str] = None + messages: Optional[Union[str, list, dict]] + response: Optional[Union[str, list, dict]] + + +class LiteLLM_ErrorLogs(LiteLLMPydanticObjectBase): + request_id: Optional[str] = str(uuid.uuid4()) + api_base: Optional[str] = "" + model_group: Optional[str] = "" + litellm_model_name: Optional[str] = "" + model_id: Optional[str] = "" + request_kwargs: Optional[dict] = {} + exception_type: Optional[str] = "" + status_code: Optional[str] = "" + exception_string: Optional[str] = "" + startTime: Union[str, datetime, None] + endTime: Union[str, datetime, None] diff --git a/litellm/models/tag.py b/litellm/models/tag.py new file mode 100644 index 00000000000..02d8f58916d --- /dev/null +++ b/litellm/models/tag.py @@ -0,0 +1,36 @@ +""" +Tag table model. + +Canonical definition for ``litellm_tagtable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from datetime import datetime +from typing import List, Optional + +from pydantic import model_validator + +from litellm.models.budget import LiteLLM_BudgetTable +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_TagTable(LiteLLMPydanticObjectBase): + tag_name: str + description: Optional[str] = None + models: List[str] = [] + model_info: Optional[dict] = None + spend: float = 0.0 + budget_id: Optional[str] = None + litellm_budget_table: Optional[LiteLLM_BudgetTable] = None + created_at: Optional[datetime] = None + created_by: Optional[str] = None + updated_at: Optional[datetime] = None + + @model_validator(mode="before") + @classmethod + def set_model_info(cls, values): + if values.get("spend") is None: + values.update({"spend": 0.0}) + if values.get("models") is None: + values.update({"models": []}) + return values diff --git a/litellm/models/team.py b/litellm/models/team.py new file mode 100644 index 00000000000..aa0798955f2 --- /dev/null +++ b/litellm/models/team.py @@ -0,0 +1,154 @@ +""" +Team table models. + +Canonical definitions for ``litellm_teamtable`` (plus the shared Member and +budget-window value types and the team-model alias table). Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +import json +from datetime import datetime +from typing import List, Literal, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class MemberBase(LiteLLMPydanticObjectBase): + user_id: Optional[str] = Field( + default=None, + description="The unique ID of the user to add. Either user_id or user_email must be provided", + ) + user_email: Optional[str] = Field( + default=None, + description="The email address of the user to add. Either user_id or user_email must be provided", + ) + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values): + if not isinstance(values, dict): + raise ValueError("input needs to be a dictionary") + if values.get("user_id") is None and values.get("user_email") is None: + raise ValueError("Either user id or user email must be provided") + return values + + +class Member(MemberBase): + role: Literal["admin", "user"] = Field( + description="The role of the user within the team. 'admin' users can manage team settings and members, 'user' is a regular team member" + ) + + +class BudgetLimitEntry(LiteLLMPydanticObjectBase): + """A single budget window with its own limit and independent reset schedule.""" + + budget_duration: str + max_budget: float + reset_at: Optional[datetime] = None + + +class LiteLLM_ModelTable(LiteLLMPydanticObjectBase): + id: Optional[int] = None + model_aliases: Optional[Union[str, dict]] = None + created_by: str + updated_by: str + team: Optional["LiteLLM_TeamTable"] = None + + model_config = ConfigDict(protected_namespaces=()) + + +class TeamBase(LiteLLMPydanticObjectBase): + team_alias: Optional[str] = None + team_id: Optional[str] = None + organization_id: Optional[str] = None + admins: list = [] + members: list = [] + members_with_roles: List[Member] = [] + team_member_permissions: Optional[List[str]] = None + metadata: Optional[dict] = None + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + max_budget: Optional[float] = None + soft_budget: Optional[float] = None + budget_duration: Optional[str] = None + budget_limits: Optional[List[BudgetLimitEntry]] = None + models: list = [] + blocked: bool = False + router_settings: Optional[dict] = None + access_group_ids: Optional[List[str]] = None + default_team_member_models: Optional[List[str]] = None + + +class LiteLLM_TeamTable(TeamBase): + team_id: str # type: ignore + spend: Optional[float] = None + max_parallel_requests: Optional[int] = None + budget_duration: Optional[str] = None + budget_reset_at: Optional[datetime] = None + model_id: Optional[int] = None + model_spend: Optional[dict] = {} + model_max_budget: Optional[dict] = {} + policies: Optional[List[str]] = None + allow_team_guardrail_config: Optional[bool] = False + litellm_model_table: Optional[LiteLLM_ModelTable] = None + object_permission: Optional[LiteLLM_ObjectPermissionTable] = None + object_permission_id: Optional[str] = None + updated_at: Optional[datetime] = None + created_at: Optional[datetime] = None + + model_config = ConfigDict(protected_namespaces=()) + + @model_validator(mode="before") + @classmethod + def set_model_info(cls, values): + dict_fields = [ + "metadata", + "aliases", + "config", + "permissions", + "model_max_budget", + "model_aliases", + "router_settings", + "budget_limits", + ] + + if isinstance(values, BaseModel): + values = values.model_dump() + + if ( + isinstance(values.get("members_with_roles"), dict) + and not values["members_with_roles"] + ): + values["members_with_roles"] = [] + + for field in dict_fields: + value = values.get(field) + if value is not None and isinstance(value, str): + try: + values[field] = json.loads(value) + except json.JSONDecodeError: + raise ValueError(f"Field {field} should be a valid dictionary") + + return values + + +class LiteLLM_TeamTableCachedObj(LiteLLM_TeamTable): + last_refreshed_at: Optional[float] = None + + +class LiteLLM_DeletedTeamTable(LiteLLM_TeamTable): + """Audit record for deleted teams; mirrors the team plus deletion metadata.""" + + id: Optional[str] = None + deleted_at: Optional[datetime] = None + deleted_by: Optional[str] = None + deleted_by_api_key: Optional[str] = None + litellm_changed_by: Optional[str] = None + + model_config = ConfigDict(protected_namespaces=()) + + +LiteLLM_ModelTable.model_rebuild() diff --git a/litellm/models/team_membership.py b/litellm/models/team_membership.py new file mode 100644 index 00000000000..d0a1308ce7c --- /dev/null +++ b/litellm/models/team_membership.py @@ -0,0 +1,32 @@ +""" +Team membership table model. + +Canonical definition for ``litellm_teammembership``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from typing import Optional, Union + +from litellm.models.budget import LiteLLM_BudgetTable, LiteLLM_BudgetTableFull +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_TeamMembership(LiteLLMPydanticObjectBase): + user_id: str + team_id: str + budget_id: Optional[str] = None + spend: Optional[float] = 0.0 + total_spend: Optional[float] = 0.0 + litellm_budget_table: Optional[ + Union[LiteLLM_BudgetTableFull, LiteLLM_BudgetTable] + ] = None + + def safe_get_team_member_rpm_limit(self) -> Optional[int]: + if self.litellm_budget_table is not None: + return self.litellm_budget_table.rpm_limit + return None + + def safe_get_team_member_tpm_limit(self) -> Optional[int]: + if self.litellm_budget_table is not None: + return self.litellm_budget_table.tpm_limit + return None diff --git a/litellm/models/user.py b/litellm/models/user.py new file mode 100644 index 00000000000..cd7e9db4aec --- /dev/null +++ b/litellm/models/user.py @@ -0,0 +1,70 @@ +""" +User table model. + +Canonical definition for ``litellm_usertable``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from datetime import datetime +from typing import Dict, List, Optional + +from pydantic import ConfigDict, Field, model_validator + +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.models.organization_membership import ( + LiteLLM_OrganizationMembershipTable, +) +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_UserTable(LiteLLMPydanticObjectBase): + user_id: str + user_alias: Optional[str] = None + team_id: Optional[str] = None + sso_user_id: Optional[str] = None + organization_id: Optional[str] = None + object_permission_id: Optional[str] = None + password: Optional[str] = Field(default=None, exclude=True) + teams: List[str] = [] + user_role: Optional[str] = None + max_budget: Optional[float] = None + spend: float = 0.0 + user_email: Optional[str] = None + models: list = [] + metadata: Optional[dict] = None + max_parallel_requests: Optional[int] = None + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + budget_duration: Optional[str] = None + budget_reset_at: Optional[datetime] = None + allowed_cache_controls: List[str] = [] + policies: List[str] = [] + model_spend: Optional[Dict] = {} + model_max_budget: Optional[Dict] = {} + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + organization_memberships: Optional[List[LiteLLM_OrganizationMembershipTable]] = None + object_permission: Optional[LiteLLM_ObjectPermissionTable] = None + + model_config = ConfigDict(protected_namespaces=()) + + @model_validator(mode="before") + @classmethod + def set_model_info(cls, values): + if values.get("spend") is None: + values.update({"spend": 0.0}) + if values.get("models") is None: + values.update({"models": []}) + if values.get("teams") is None: + values.update({"teams": []}) + return values + + def is_over_budget(self) -> bool: + if self.max_budget is None: + return False + return self.spend >= self.max_budget + + def has_model_access(self, model_name: str) -> bool: + if not self.models: + return True + return model_name in self.models diff --git a/litellm/models/verification_token.py b/litellm/models/verification_token.py new file mode 100644 index 00000000000..8bddd1c1619 --- /dev/null +++ b/litellm/models/verification_token.py @@ -0,0 +1,74 @@ +""" +Verification token table model. + +Canonical definition for ``litellm_verificationtoken``. Re-exported from +``litellm.proxy._types`` for backwards compatibility. +""" + +from datetime import datetime +from typing import Dict, List, Optional, Union + +from pydantic import ConfigDict + +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase): + token: Optional[str] = None + key_name: Optional[str] = None + key_alias: Optional[str] = None + spend: float = 0.0 + max_budget: Optional[float] = None + expires: Optional[Union[str, datetime]] = None + models: List = [] + aliases: Dict = {} + config: Dict = {} + user_id: Optional[str] = None + team_id: Optional[str] = None + agent_id: Optional[str] = None + project_id: Optional[str] = None + max_parallel_requests: Optional[int] = None + metadata: Dict = {} + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + budget_duration: Optional[str] = None + budget_reset_at: Optional[datetime] = None + allowed_cache_controls: Optional[list] = [] + allowed_routes: Optional[list] = [] + permissions: Dict = {} + model_spend: Dict = {} + model_max_budget: Dict = {} + soft_budget_cooldown: bool = False + blocked: Optional[bool] = None + litellm_budget_table: Optional[dict] = None + budget_id: Optional[str] = None + org_id: Optional[str] = None # org id for a given key + created_at: Optional[datetime] = None + created_by: Optional[str] = None + updated_at: Optional[datetime] = None + updated_by: Optional[str] = None + last_active: Optional[datetime] = None + object_permission_id: Optional[str] = None + object_permission: Optional[LiteLLM_ObjectPermissionTable] = None + access_group_ids: Optional[List[str]] = None + rotation_count: Optional[int] = 0 + auto_rotate: Optional[bool] = False + rotation_interval: Optional[str] = None + last_rotation_at: Optional[datetime] = None + key_rotation_at: Optional[datetime] = None + router_settings: Optional[dict] = None + budget_limits: Optional[List[dict]] = None + model_config = ConfigDict(protected_namespaces=()) + + +class LiteLLM_DeletedVerificationToken(LiteLLM_VerificationToken): + """Audit record for deleted keys; mirrors the token plus deletion metadata.""" + + id: Optional[str] = None + deleted_at: Optional[datetime] = None + deleted_by: Optional[str] = None + deleted_by_api_key: Optional[str] = None + litellm_changed_by: Optional[str] = None + + model_config = ConfigDict(protected_namespaces=()) diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 0562b41d2cd..e0eeb014c51 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1539,6 +1539,23 @@ "interactions": true } }, + "neosantara": { + "display_name": "Neosantara (`neosantara`)", + "url": "https://docs.litellm.ai/docs/providers/neosantara", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "nvidia_nim": { "display_name": "Nvidia NIM (`nvidia_nim`)", "url": "https://docs.litellm.ai/docs/providers/nvidia_nim", diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 863e6acd41e..dcf7660d002 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -14,8 +14,12 @@ from litellm.proxy._types import ( SpecialHeaders, UserAPIKeyAuth, ) -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.repositories.table_repositories import ( + AgentsRepository, + MCPServerRepository, +) def _parse_mcp_server_names_from_path( @@ -1445,7 +1449,7 @@ class MCPRequestHandler: return None if object_permission_id is None: - agent_row = await prisma_client.db.litellm_agentstable.find_unique( + agent_row = await AgentsRepository(prisma_client).table.find_unique( where={"agent_id": agent_id}, ) object_permission_id = ( @@ -1600,7 +1604,7 @@ class MCPRequestHandler: server_ids: Set[str] = set() if access_groups and prisma_client is not None: try: - mcp_servers = await prisma_client.db.litellm_mcpservertable.find_many( + mcp_servers = await MCPServerRepository(prisma_client).table.find_many( where={"mcp_access_groups": {"hasSome": access_groups}} ) for server in mcp_servers: diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index d7b2224eb64..c52752940c3 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -1,16 +1,20 @@ import base64 import binascii +import hashlib import json from datetime import datetime, timedelta, timezone from typing import Any, Dict, Iterable, List, Optional, Set, Union, cast from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid +from litellm.constants import MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._types import ( LiteLLM_MCPServerTable, LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, MCPApprovalStatus, + MCPEnvVarScope, MCPSubmissionsSummary, NewMCPServerRequest, SpecialMCPServerName, @@ -22,12 +26,158 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy.utils import PrismaClient +from litellm.repositories.object_permission_repository import ObjectPermissionRepository +from litellm.repositories.table_repositories import ( + MCPServerRepository, + MCPUserCredentialsRepository, +) +from litellm.repositories.team_repository import TeamRepository +from litellm.repositories.verification_token_repository import ( + VerificationTokenRepository, +) from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.mcp import MCPCredentials +def _is_global_env_var_scope(scope: Any) -> bool: + """``scope="user"`` entries are placeholders the user fills in; everything + else (including a missing scope) is an admin-supplied global value.""" + return scope != MCPEnvVarScope.user and scope != "user" + + +def _encrypt_global_env_var_values(env_vars: Iterable[Dict[str, Any]]) -> None: + """Encrypt ``scope="global"`` env var values in place before persisting. + + Global values hold admin-supplied secrets (API keys, passwords) that get + interpolated into headers, so they are encrypted at rest like credentials + and the per-user ``values_b64`` column. Per-user placeholders are not + secrets and are stored verbatim. + """ + for entry in env_vars: + if not _is_global_env_var_scope(entry.get("scope")): + continue + value = entry.get("value") + if value: + entry["value"] = encrypt_value_helper(value) + + +def decrypt_global_env_var_values(env_vars: Optional[Iterable[Any]]) -> None: + """Decrypt ``scope="global"`` env var values in place after reading the DB. + + Accepts ``MCPEnvVar`` models (``LiteLLM_MCPServerTable``) or plain dicts + (raw rows / deserialized JSON). Global values are always stored encrypted, + so a value that no longer decrypts (e.g. after a salt-key change) is dropped + and a warning is logged rather than forwarding the ciphertext into upstream + ``${NAME}`` headers, where it would silently fail. + """ + if not env_vars: + return + for entry in env_vars: + is_dict = isinstance(entry, dict) + scope = entry.get("scope") if is_dict else getattr(entry, "scope", None) + if not _is_global_env_var_scope(scope): + continue + value = entry.get("value") if is_dict else getattr(entry, "value", None) + if not value: + continue + decrypted = decrypt_value_helper( + value=value, + key="mcp_global_env_var", + exception_type="debug", + return_original_value=False, + ) + if decrypted is None: + name = entry.get("name") if is_dict else getattr(entry, "name", None) + verbose_proxy_logger.warning( + "MCP global env var %s failed to decrypt (LITELLM_SALT_KEY " + "changed?); dropping it so ciphertext is not sent upstream", + name, + ) + decrypted = "" + if is_dict: + entry["value"] = decrypted + else: + entry.value = decrypted + + +def _decrypt_env_vars_on_returned_row(row: Any) -> None: + """Decrypt ``scope="global"`` env var values on a row returned by Prisma create/update. + + Prisma may hand back ``env_vars`` either as a parsed list (the common case for + JSONB columns) or as a raw JSON string (observed for some write paths). The + in-place decrypt helper only mutates iterables of dicts/models, so a string + payload would silently skip decryption and ciphertext would leak into the + registry via ``add_server``/``update_server`` (which trust the caller). + Parse the string back to a list so the in-place decrypt actually runs, and + write the decrypted list back onto the row so downstream consumers see plain + values. + """ + env_vars = getattr(row, "env_vars", None) + if env_vars is None: + return + if isinstance(env_vars, str): + try: + env_vars = json.loads(env_vars) + except (json.JSONDecodeError, TypeError): + return + if not isinstance(env_vars, list): + return + try: + setattr(row, "env_vars", env_vars) + except (AttributeError, TypeError): + pass + decrypt_global_env_var_values(env_vars) + + +def _reencrypt_global_env_var_values( + env_vars: Optional[Iterable[Any]], new_encryption_key: str +) -> Optional[List[Dict[str, Any]]]: + """Re-encrypt ``scope="global"`` env var values for master-key rotation. + + Each global value is decrypted with the current salt key and re-encrypted + under ``new_encryption_key``. Returns the rebuilt list when at least one + value was rotated, else ``None`` so the caller can skip the DB write. A + value that fails to decrypt is left untouched (and logged) so a corrupt + entry is preserved for recovery rather than overwritten. + """ + if not env_vars: + return None + if isinstance(env_vars, str): + try: + env_vars = json.loads(env_vars) + except (json.JSONDecodeError, TypeError): + return None + if not env_vars: + return None + rebuilt = [dict(v) for v in env_vars] + rotated = False + for entry in rebuilt: + if not _is_global_env_var_scope(entry.get("scope")): + continue + value = entry.get("value") + if not value: + continue + decrypted = decrypt_value_helper( + value=value, + key="mcp_global_env_var", + exception_type="debug", + return_original_value=False, + ) + if decrypted is None: + verbose_proxy_logger.warning( + "rotate_mcp_server_credentials_master_key: could not decrypt " + "global env var %s, skipping", + entry.get("name"), + ) + continue + entry["value"] = encrypt_value_helper( + decrypted, new_encryption_key=new_encryption_key + ) + rotated = True + return rebuilt if rotated else None + + def _prepare_mcp_server_data( data: Union[NewMCPServerRequest, UpdateMCPServerRequest], exclude_unset: bool = False, @@ -98,6 +248,16 @@ def _prepare_mcp_server_data( if data_dict.get("static_headers") is not None: data_dict["static_headers"] = safe_dumps(data_dict["static_headers"]) + # env_vars is read from ``data_dict`` (not ``data``) like every other JSON + # column so the exclude_unset filter is respected: a partial update that + # omits env_vars never overwrites the stored value. Global values are + # encrypted at rest before serialization. + env_vars = data_dict.get("env_vars") + if env_vars is not None: + serialized_env_vars = [dict(v) for v in env_vars] + _encrypt_global_env_var_values(serialized_env_vars) + data_dict["env_vars"] = safe_dumps(serialized_env_vars) + if data_dict.get("mcp_info") is not None: data_dict["mcp_info"] = safe_dumps(data_dict["mcp_info"]) @@ -203,14 +363,17 @@ async def get_all_mcp_servers( where: Dict[str, Any] = {} if approval_status is not None: where["approval_status"] = approval_status - mcp_servers = await prisma_client.db.litellm_mcpservertable.find_many( + mcp_servers = await MCPServerRepository(prisma_client).table.find_many( where=where if where else {} ) - return [ + tables = [ LiteLLM_MCPServerTable(**mcp_server.model_dump()) for mcp_server in mcp_servers ] + for table in tables: + decrypt_global_env_var_values(table.env_vars) + return tables except Exception as e: verbose_proxy_logger.debug( "litellm.proxy._experimental.mcp_server.db.py::get_all_mcp_servers - {}".format( @@ -226,14 +389,18 @@ async def get_mcp_server( """ Returns the matching mcp server from the db iff exists """ - mcp_server: Optional[LiteLLM_MCPServerTable] = ( - await prisma_client.db.litellm_mcpservertable.find_unique( - where={ - "server_id": server_id, - } - ) + mcp_server: Optional[LiteLLM_MCPServerTable] = await MCPServerRepository( + prisma_client + ).table.find_unique( + where={ + "server_id": server_id, + } ) - return mcp_server + if mcp_server is None: + return None + table = LiteLLM_MCPServerTable(**mcp_server.model_dump()) + decrypt_global_env_var_values(table.env_vars) + return table async def get_mcp_servers( @@ -242,16 +409,18 @@ async def get_mcp_servers( """ Returns the matching mcp servers from the db with the server_ids """ - _mcp_servers: List[LiteLLM_MCPServerTable] = ( - await prisma_client.db.litellm_mcpservertable.find_many( - where={ - "server_id": {"in": server_ids}, - } - ) + _mcp_servers: List[LiteLLM_MCPServerTable] = await MCPServerRepository( + prisma_client + ).table.find_many( + where={ + "server_id": {"in": server_ids}, + } ) final_mcp_servers: List[LiteLLM_MCPServerTable] = [] for _mcp_server in _mcp_servers: - final_mcp_servers.append(LiteLLM_MCPServerTable(**_mcp_server.model_dump())) + table = LiteLLM_MCPServerTable(**_mcp_server.model_dump()) + decrypt_global_env_var_values(table.env_vars) + final_mcp_servers.append(table) return final_mcp_servers @@ -262,15 +431,15 @@ async def get_mcp_servers_by_verificationtoken( """ Returns the mcp servers from the db for the verification token """ - verification_token_record: LiteLLM_TeamTable = ( - await prisma_client.db.litellm_verificationtoken.find_unique( - where={ - "token": token, - }, - include={ - "object_permission": True, - }, - ) + verification_token_record: LiteLLM_TeamTable = await VerificationTokenRepository( + prisma_client + ).table.find_unique( + where={ + "token": token, + }, + include={ + "object_permission": True, + }, ) mcp_servers: Optional[List[str]] = [] @@ -288,15 +457,15 @@ async def get_mcp_servers_by_team( """ Returns the mcp servers from the db for the team id """ - team_record: LiteLLM_TeamTable = ( - await prisma_client.db.litellm_teamtable.find_unique( - where={ - "team_id": team_id, - }, - include={ - "object_permission": True, - }, - ) + team_record: LiteLLM_TeamTable = await TeamRepository( + prisma_client + ).table.find_unique( + where={ + "team_id": team_id, + }, + include={ + "object_permission": True, + }, ) mcp_servers: Optional[List[str]] = [] @@ -347,16 +516,16 @@ async def get_objectpermissions_for_mcp_server( """ Get all the object permissions records and the associated team and verficiationtoken records that have access to the mcp server """ - object_permission_records = ( - await prisma_client.db.litellm_objectpermissiontable.find_many( - where={ - "mcp_servers": {"has": mcp_server_id}, - }, - include={ - "teams": True, - "verification_tokens": True, - }, - ) + object_permission_records = await ObjectPermissionRepository( + prisma_client + ).table.find_many( + where={ + "mcp_servers": {"has": mcp_server_id}, + }, + include={ + "teams": True, + "verification_tokens": True, + }, ) return object_permission_records @@ -368,7 +537,7 @@ async def get_virtualkeys_for_mcp_server( """ Get all the virtual keys that have access to the mcp server """ - virtual_keys = await prisma_client.db.litellm_verificationtoken.find_many( + virtual_keys = await VerificationTokenRepository(prisma_client).table.find_many( where={ "mcp_servers": {"has": server_id}, }, @@ -399,13 +568,30 @@ async def delete_mcp_server( """ Delete the mcp server from the db by server_id + The server-row delete is the commit point. Per-user env var rows have no FK + cascade, so they are cleaned up afterwards on a best-effort basis: a transient + failure there leaves only orphaned rows pointing at a now-missing server and + must not turn a successful delete into a caller-visible error. + Returns the deleted mcp server record if it exists, otherwise None """ - deleted_server = await prisma_client.db.litellm_mcpservertable.delete( + deleted_server = await MCPServerRepository(prisma_client).table.delete( where={ "server_id": server_id, }, ) + if deleted_server is not None: + try: + await prisma_client.db.litellm_mcpuserenvvars.delete_many( + where={"server_id": server_id} + ) + except Exception as e: + verbose_proxy_logger.warning( + "MCP server %s deleted but per-user env var cleanup failed; " + "orphaned rows can be removed on a later delete: %s", + server_id, + e, + ) return deleted_server @@ -425,10 +611,11 @@ async def create_mcp_server( data_dict["created_by"] = touched_by data_dict["updated_by"] = touched_by - new_mcp_server = await prisma_client.db.litellm_mcpservertable.create( + new_mcp_server = await MCPServerRepository(prisma_client).table.create( data=data_dict # type: ignore ) + _decrypt_env_vars_on_returned_row(new_mcp_server) return new_mcp_server @@ -459,7 +646,7 @@ async def update_mcp_server( "credentials" in data_dict and data_dict["credentials"] is not None ) if data.auth_type or has_credentials: - existing = await prisma_client.db.litellm_mcpservertable.find_unique( + existing = await MCPServerRepository(prisma_client).table.find_unique( where={"server_id": data.server_id} ) @@ -502,44 +689,56 @@ async def update_mcp_server( # Add audit fields data_dict["updated_by"] = touched_by - updated_mcp_server = await prisma_client.db.litellm_mcpservertable.update( + updated_mcp_server = await MCPServerRepository(prisma_client).table.update( where={"server_id": data.server_id}, data=data_dict # type: ignore ) + _decrypt_env_vars_on_returned_row(updated_mcp_server) return updated_mcp_server async def rotate_mcp_server_credentials_master_key( prisma_client: PrismaClient, touched_by: str, new_master_key: str ): - mcp_servers = await prisma_client.db.litellm_mcpservertable.find_many() + from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + mcp_servers = await MCPServerRepository(prisma_client).table.find_many() + + updated = 0 for mcp_server in mcp_servers: + update_data: Dict[str, Any] = {} + credentials = mcp_server.credentials - if not credentials: + if credentials: + # Decrypt with current key first, then re-encrypt with new key + decrypted_credentials = decrypt_credentials( + credentials=cast(MCPCredentials, dict(credentials)), + ) + encrypted_credentials = encrypt_credentials( + credentials=decrypted_credentials, + encryption_key=new_master_key, + ) + update_data["credentials"] = safe_dumps(encrypted_credentials) + + rotated_env_vars = _reencrypt_global_env_var_values( + mcp_server.env_vars, new_master_key + ) + if rotated_env_vars is not None: + update_data["env_vars"] = safe_dumps(rotated_env_vars) + + if not update_data: continue - credentials_copy = dict(credentials) - # Decrypt with current key first, then re-encrypt with new key - decrypted_credentials = decrypt_credentials( - credentials=cast(MCPCredentials, credentials_copy), - ) - encrypted_credentials = encrypt_credentials( - credentials=decrypted_credentials, - encryption_key=new_master_key, - ) - - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - - serialized_credentials = safe_dumps(encrypted_credentials) - - await prisma_client.db.litellm_mcpservertable.update( + update_data["updated_by"] = touched_by + await MCPServerRepository(prisma_client).table.update( where={"server_id": mcp_server.server_id}, - data={ - "credentials": serialized_credentials, - "updated_by": touched_by, - }, + data=update_data, ) + updated += 1 + verbose_proxy_logger.info( + "rotate_mcp_server_credentials_master_key: rotated %d MCP server row(s)", + updated, + ) def _decode_user_credential(stored: str) -> Optional[str]: @@ -593,7 +792,9 @@ async def rotate_mcp_user_credentials_master_key( under the new master key. Rows that are unreadable under both paths are logged and skipped so one corrupt row does not abort the rotation. """ - rows = await prisma_client.db.litellm_mcpusercredentials.find_many() + rows = await MCPUserCredentialsRepository(prisma_client).table.find_many() + rotated = 0 + skipped = 0 for row in rows: plaintext = _decode_user_credential(row.credential_b64) if plaintext is None: @@ -603,11 +804,12 @@ async def rotate_mcp_user_credentials_master_key( row.user_id, row.server_id, ) + skipped += 1 continue re_encrypted = encrypt_value_helper( plaintext, new_encryption_key=new_master_key ) - await prisma_client.db.litellm_mcpusercredentials.update( + await MCPUserCredentialsRepository(prisma_client).table.update( where={ "user_id_server_id": { "user_id": row.user_id, @@ -616,6 +818,61 @@ async def rotate_mcp_user_credentials_master_key( }, data={"credential_b64": re_encrypted}, ) + rotated += 1 + verbose_proxy_logger.info( + "rotate_mcp_user_credentials_master_key: rotated %d row(s), skipped %d", + rotated, + skipped, + ) + + +async def rotate_mcp_user_env_vars_master_key( + prisma_client: PrismaClient, new_master_key: str +): + """Re-encrypt every ``LiteLLM_MCPUserEnvVars`` row with ``new_master_key``. + + Reads each ``values_b64`` blob with the current salt key and writes it back + encrypted under the new master key. Rows that fail to decrypt are logged and + skipped so one corrupt row does not abort the rotation nor overwrite values + that may still be recoverable. + """ + rows = await prisma_client.db.litellm_mcpuserenvvars.find_many() + rotated = 0 + skipped = 0 + for row in rows: + plaintext = decrypt_value_helper( + value=row.values_b64, + key="mcp_user_env_vars", + exception_type="debug", + return_original_value=False, + ) + if plaintext is None: + verbose_proxy_logger.warning( + "rotate_mcp_user_env_vars_master_key: could not decrypt env vars " + "for user_id=%s server_id=%s, skipping", + row.user_id, + row.server_id, + ) + skipped += 1 + continue + re_encrypted = encrypt_value_helper( + plaintext, new_encryption_key=new_master_key + ) + await prisma_client.db.litellm_mcpuserenvvars.update( + where={ + "user_id_server_id": { + "user_id": row.user_id, + "server_id": row.server_id, + } + }, + data={"values_b64": re_encrypted}, + ) + rotated += 1 + verbose_proxy_logger.info( + "rotate_mcp_user_env_vars_master_key: rotated %d row(s), skipped %d", + rotated, + skipped, + ) async def store_user_credential( @@ -627,7 +884,7 @@ async def store_user_credential( """Store a user credential for a BYOK MCP server.""" encoded = encrypt_value_helper(credential) - await prisma_client.db.litellm_mcpusercredentials.upsert( + await MCPUserCredentialsRepository(prisma_client).table.upsert( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}, data={ "create": { @@ -647,7 +904,7 @@ async def get_user_credential( ) -> Optional[str]: """Return credential for a user+server pair, or None.""" - row = await prisma_client.db.litellm_mcpusercredentials.find_unique( + row = await MCPUserCredentialsRepository(prisma_client).table.find_unique( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} ) if row is None: @@ -661,7 +918,7 @@ async def has_user_credential( server_id: str, ) -> bool: """Return True if the user has a stored credential for this server.""" - row = await prisma_client.db.litellm_mcpusercredentials.find_unique( + row = await MCPUserCredentialsRepository(prisma_client).table.find_unique( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} ) return row is not None @@ -673,7 +930,7 @@ async def delete_user_credential( server_id: str, ) -> None: """Delete the user's stored credential for a BYOK MCP server.""" - await prisma_client.db.litellm_mcpusercredentials.delete( + await MCPUserCredentialsRepository(prisma_client).table.delete( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} ) @@ -720,7 +977,7 @@ async def store_user_oauth_credential( # Skip the guard when the caller knows the row is already an OAuth2 credential # (e.g. during token refresh), saving an extra DB round-trip. if not skip_byok_guard: - existing = await prisma_client.db.litellm_mcpusercredentials.find_unique( + existing = await MCPUserCredentialsRepository(prisma_client).table.find_unique( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} ) if ( @@ -738,7 +995,7 @@ async def store_user_oauth_credential( ) encoded = encrypt_value_helper(json.dumps(payload)) - await prisma_client.db.litellm_mcpusercredentials.upsert( + await MCPUserCredentialsRepository(prisma_client).table.upsert( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}, data={ "create": { @@ -751,11 +1008,14 @@ async def store_user_oauth_credential( ) -def is_oauth_credential_expired(cred: Dict[str, Any]) -> bool: +def is_oauth_credential_expired(cred: Dict[str, Any], buffer_seconds: int = 0) -> bool: """Return True if the OAuth2 credential's access_token has expired. Checks the ``expires_at`` ISO-format string stored in the credential payload. Returns False when ``expires_at`` is absent or unparseable (treat as non-expired). + With ``buffer_seconds`` > 0, a token that is still valid but expires within the + buffer is also treated as expired, so callers can refresh proactively instead of + handing back a token that may lapse mid-request. """ expires_at = cred.get("expires_at") if not expires_at: @@ -764,7 +1024,7 @@ def is_oauth_credential_expired(cred: Dict[str, Any]) -> bool: exp_dt = datetime.fromisoformat(expires_at) if exp_dt.tzinfo is None: exp_dt = exp_dt.replace(tzinfo=timezone.utc) - return datetime.now(timezone.utc) > exp_dt + return datetime.now(timezone.utc) + timedelta(seconds=buffer_seconds) > exp_dt except (ValueError, TypeError): return False @@ -776,7 +1036,7 @@ async def get_user_oauth_credential( ) -> Optional[Dict[str, Any]]: """Return the decoded OAuth2 payload dict for a user+server pair, or None.""" - row = await prisma_client.db.litellm_mcpusercredentials.find_unique( + row = await MCPUserCredentialsRepository(prisma_client).table.find_unique( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} ) if row is None: @@ -790,7 +1050,7 @@ async def list_user_oauth_credentials( ) -> List[Dict[str, Any]]: """Return all OAuth2 credential payloads for a user, tagged with server_id.""" - rows = await prisma_client.db.litellm_mcpusercredentials.find_many( + rows = await MCPUserCredentialsRepository(prisma_client).table.find_many( where={"user_id": user_id} ) results: List[Dict[str, Any]] = [] @@ -912,6 +1172,50 @@ async def refresh_user_oauth_token( return await get_user_oauth_credential(prisma_client, user_id, server_id) +async def resolve_valid_user_oauth_token( + user_id: str, + server: Any, + cred: Optional[Dict[str, Any]], + prisma_client: Optional[PrismaClient] = None, +) -> Optional[Dict[str, Any]]: + """Return an OAuth2 credential whose access_token is good for the next request. + + Returns the credential unchanged while its token is valid for at least + ``MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS``. Only when the token is expired (or + expiring within that buffer) and a refresh_token is stored does it mint a new one + via ``refresh_user_oauth_token``. Returns None when there is no usable token + (missing token, expired with no refresh_token, or a failed refresh). + + The refresh_token is only ever sent to the server's token_url inside + ``refresh_user_oauth_token``; it is never exposed to the caller beyond the cred + dict it already holds. ``prisma_client`` is fetched lazily and only when a refresh + actually happens, so the valid-token path never requires a DB handle. + """ + if not cred or not cred.get("access_token"): + return None + if not is_oauth_credential_expired( + cred, buffer_seconds=MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS + ): + return cred + if not cred.get("refresh_token"): + return None + if prisma_client is None: + from litellm.proxy.utils import get_prisma_client_or_throw + + prisma_client = get_prisma_client_or_throw( + "Database not connected. Cannot refresh OAuth token." + ) + refreshed = await refresh_user_oauth_token( + prisma_client=prisma_client, + user_id=user_id, + server=server, + cred=cred, + ) + if not refreshed or not refreshed.get("access_token"): + return None + return refreshed + + async def approve_mcp_server( prisma_client: PrismaClient, server_id: str, @@ -919,7 +1223,7 @@ async def approve_mcp_server( ) -> LiteLLM_MCPServerTable: """Set approval_status=active and record reviewed_at.""" now = datetime.now(timezone.utc) - updated = await prisma_client.db.litellm_mcpservertable.update( + updated = await MCPServerRepository(prisma_client).table.update( where={"server_id": server_id}, data={ "approval_status": MCPApprovalStatus.active, @@ -927,7 +1231,9 @@ async def approve_mcp_server( "updated_by": touched_by, }, ) - return LiteLLM_MCPServerTable(**updated.model_dump()) + table = LiteLLM_MCPServerTable(**updated.model_dump()) + decrypt_global_env_var_values(table.env_vars) + return table async def reject_mcp_server( @@ -945,11 +1251,13 @@ async def reject_mcp_server( } if review_notes is not None: data["review_notes"] = review_notes - updated = await prisma_client.db.litellm_mcpservertable.update( + updated = await MCPServerRepository(prisma_client).table.update( where={"server_id": server_id}, data=data, ) - return LiteLLM_MCPServerTable(**updated.model_dump()) + table = LiteLLM_MCPServerTable(**updated.model_dump()) + decrypt_global_env_var_values(table.env_vars) + return table async def get_mcp_submissions( @@ -960,12 +1268,14 @@ async def get_mcp_submissions( along with a summary count breakdown by approval_status. Mirrors get_guardrail_submissions() from guardrail_endpoints.py. """ - rows = await prisma_client.db.litellm_mcpservertable.find_many( + rows = await MCPServerRepository(prisma_client).table.find_many( where={"submitted_at": {"not": None}}, order={"submitted_at": "desc"}, take=500, # safety cap; paginate if needed in a future iteration ) items = [LiteLLM_MCPServerTable(**r.model_dump()) for r in rows] + for item in items: + decrypt_global_env_var_values(item.env_vars) pending = sum( 1 for i in items if i.approval_status == MCPApprovalStatus.pending_review @@ -980,3 +1290,121 @@ async def get_mcp_submissions( rejected=rejected, items=items, ) + + +# ── Per-user MCP environment variables ──────────────────────────────────── + + +def _decode_user_env_vars(stored: str) -> Dict[str, str]: + """Decrypt a ``values_b64`` blob and parse it as a flat ``{name: value}`` dict.""" + decrypted = decrypt_value_helper( + value=stored, + key="mcp_user_env_vars", + exception_type="debug", + return_original_value=False, + ) + if decrypted is None: + if stored: + verbose_proxy_logger.warning( + "MCP per-user env vars failed to decrypt (LITELLM_SALT_KEY " + "changed?); treating as unset so the user is prompted to " + "re-enter them rather than silently forwarding ciphertext" + ) + return {} + try: + parsed = json.loads(decrypted) + except (ValueError, TypeError): + return {} + if not isinstance(parsed, dict): + return {} + return {str(k): str(v) for k, v in parsed.items()} + + +async def get_user_env_vars( + prisma_client: PrismaClient, + user_id: str, + server_id: str, +) -> Dict[str, str]: + """Return the calling user's env var dict for ``server_id`` (empty if none).""" + row = await prisma_client.db.litellm_mcpuserenvvars.find_unique( + where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} + ) + if row is None: + return {} + return _decode_user_env_vars(row.values_b64) + + +async def get_user_env_vars_bulk( + prisma_client: PrismaClient, + user_id: str, + server_ids: Iterable[str], +) -> Dict[str, Dict[str, str]]: + """Return ``{server_id: {var_name: value}}`` for one user across many servers. + + Servers with no stored row are simply absent from the result. + """ + ids = list(server_ids) + if not ids: + return {} + rows = await prisma_client.db.litellm_mcpuserenvvars.find_many( + where={"user_id": user_id, "server_id": {"in": ids}} + ) + return {row.server_id: _decode_user_env_vars(row.values_b64) for row in rows} + + +async def merge_user_env_vars( + prisma_client: PrismaClient, + user_id: str, + server_id: str, + updates: Dict[str, str], + allowed_names: Iterable[str], +) -> Dict[str, str]: + """Merge ``updates`` into the user's stored env vars for ``server_id`` and + return the resulting set. + + The read-modify-write runs inside a transaction guarded by a + ``(user_id, server_id)`` advisory lock so two concurrent writes from the + same user can't drop one update. Names outside ``allowed_names`` are pruned, + so an admin retiring a user-scoped variable also clears its stored value. + """ + allowed = set(allowed_names) + lock_key = int.from_bytes( + hashlib.blake2b(f"{user_id}:{server_id}".encode(), digest_size=8).digest(), + "big", + signed=True, + ) + async with prisma_client.db.tx() as tx: + await tx.execute_raw("SELECT pg_advisory_xact_lock($1::bigint)", lock_key) + row = await tx.litellm_mcpuserenvvars.find_unique( + where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} + ) + existing = _decode_user_env_vars(row.values_b64) if row is not None else {} + merged = {k: v for k, v in {**existing, **updates}.items() if k in allowed} + encoded = encrypt_value_helper(json.dumps(merged)) + await tx.litellm_mcpuserenvvars.upsert( + where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}, + data={ + "create": { + "user_id": user_id, + "server_id": server_id, + "values_b64": encoded, + }, + "update": {"values_b64": encoded}, + }, + ) + return merged + + +async def delete_user_env_vars( + prisma_client: PrismaClient, + user_id: str, + server_id: str, +) -> None: + """Remove the calling user's env var values for ``server_id``. + + Uses ``delete_many`` so a missing row is a no-op; real DB errors still + propagate to the caller instead of being silently swallowed. + """ + await prisma_client.db.litellm_mcpuserenvvars.delete_many( + where={"user_id": user_id, "server_id": server_id} + ) diff --git a/litellm/proxy/_experimental/mcp_server/elicitation_handler.py b/litellm/proxy/_experimental/mcp_server/elicitation_handler.py new file mode 100644 index 00000000000..e42270bf10b --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/elicitation_handler.py @@ -0,0 +1,163 @@ +""" +MCP Elicitation Handler +Handles `elicitation/create` requests from upstream MCP servers by either: +1. Relaying them to the connected downstream MCP client (if it supports elicitation) +2. Returning a decline/error response (if no downstream client or unsupported) +Supports both Form mode (structured data collection) and URL mode (external URL +navigation for sensitive interactions like OAuth). +MCP Spec Reference: + https://modelcontextprotocol.io/specification/2025-11-25/client/elicitation +""" + +from typing import Any, Optional, Union +from litellm._logging import verbose_logger + +# Guard imports that require the mcp package +try: + from mcp.types import ( + ElicitRequestFormParams, + ElicitRequestParams, + ElicitRequestURLParams, + ElicitResult, + ErrorData, + ) + + MCP_ELICITATION_AVAILABLE = True +except ImportError: + MCP_ELICITATION_AVAILABLE = False + + +async def handle_elicitation_request( + context: Any, + params: "ElicitRequestParams", + downstream_session: Optional[Any] = None, + downstream_capabilities: Optional[Any] = None, +) -> Union["ElicitResult", "ErrorData"]: + """ + Handle an MCP elicitation/create request from an upstream MCP server. + In Gateway mode (Mode A), we relay the elicitation request to the + connected downstream client if they declared elicitation capabilities. + In Tool Bridge mode (Mode B), there's no persistent downstream MCP + client, so we return a decline response. + Args: + context: MCP RequestContext from the upstream server connection. + params: The ElicitRequestParams (either form or URL mode). + downstream_session: The ServerSession to the downstream client, + if available (for relaying). + downstream_capabilities: The downstream client's declared + capabilities, used to check elicitation support. + Returns: + ElicitResult with the user's response, or ErrorData on failure. + """ + if not MCP_ELICITATION_AVAILABLE: + return ErrorData( + code=-1, + message="MCP elicitation is not available (mcp package not installed)", + ) + try: + mode = getattr(params, "mode", "form") + verbose_logger.info( + "MCP elicitation: received request mode=%s, message=%s", + mode, + getattr(params, "message", ""), + ) + # Check if we have a downstream session to relay to + if downstream_session is not None: + return await _relay_elicitation_to_downstream( + params=params, + downstream_session=downstream_session, + downstream_capabilities=downstream_capabilities, + ) + # No downstream session — we're in Tool Bridge mode + # or the client doesn't support elicitation + verbose_logger.info( + "MCP elicitation: no downstream session available, declining" + ) + return ElicitResult( + action="decline", + ) + except Exception as e: + verbose_logger.exception("MCP elicitation handler failed: %s", e) + return ErrorData( + code=-1, + message=f"Elicitation failed: {str(e)}", + ) + + +async def _relay_elicitation_to_downstream( + params: "ElicitRequestParams", + downstream_session: Any, + downstream_capabilities: Optional[Any] = None, +) -> Union["ElicitResult", "ErrorData"]: + """ + Relay an elicitation request to the downstream MCP client. + Uses the ServerSession's elicit_form() or elicit_url() methods to + send the elicitation request back to the connected client. + Args: + params: The elicitation request parameters. + downstream_session: The ServerSession connected to the downstream client. + downstream_capabilities: Client capabilities to check support. + Returns: + ElicitResult from the downstream client. + """ + mode = getattr(params, "mode", "form") + # Check if the downstream client supports the requested mode + if downstream_capabilities is not None: + elicit_caps = getattr(downstream_capabilities, "elicitation", None) + if elicit_caps is None: + verbose_logger.info( + "MCP elicitation: downstream client does not support elicitation" + ) + return ElicitResult(action="decline") + if mode == "url": + url_cap = getattr(elicit_caps, "url", None) + if url_cap is None: + verbose_logger.info( + "MCP elicitation: downstream client does not support URL mode" + ) + return ElicitResult(action="decline") + if mode == "form": + form_cap = getattr(elicit_caps, "form", None) + if form_cap is None: + verbose_logger.info( + "MCP elicitation: downstream client does not support form mode" + ) + return ElicitResult(action="decline") + try: + if mode == "url" and isinstance(params, ElicitRequestURLParams): + # URL mode: relay URL to client for external navigation + verbose_logger.info( + "MCP elicitation: relaying URL mode to downstream, url=%s", + getattr(params, "url", ""), + ) + result = await downstream_session.elicit_url( + message=params.message, + url=params.url, + elicitation_id=getattr(params, "elicitationId", None), + ) + elif isinstance(params, ElicitRequestFormParams): + # Form mode: relay structured form to client + verbose_logger.info("MCP elicitation: relaying form mode to downstream") + result = await downstream_session.elicit_form( + message=params.message, + requestedSchema=getattr(params, "requestedSchema", None), + ) + else: + # Fallback for generic ElicitRequestParams — pass an empty schema + # since elicit() requires requestedSchema as a positional arg. + verbose_logger.info( + "MCP elicitation: relaying generic elicitation to downstream" + ) + result = await downstream_session.elicit( + message=getattr(params, "message", ""), + requestedSchema=getattr(params, "requestedSchema", {}), + ) + verbose_logger.info( + "MCP elicitation: downstream responded with action=%s", + getattr(result, "action", "unknown"), + ) + return result + except Exception as e: + verbose_logger.warning("MCP elicitation: failed to relay to downstream: %s", e) + # If relay fails, decline gracefully + return ElicitResult(action="decline") diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 739dc4a2f88..85ac6b399f4 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -42,30 +42,42 @@ from litellm.constants import ( MCP_TOOL_LISTING_TIMEOUT, ) from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException -from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get from litellm.experimental_mcp_client.client import MCPClient, MCPSigV4Auth +from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError +from litellm.proxy._experimental.mcp_server.elicitation_handler import ( + MCP_ELICITATION_AVAILABLE, +) +from litellm.proxy._experimental.mcp_server.sampling_handler import ( + MCP_SAMPLING_AVAILABLE, +) from litellm.proxy._experimental.mcp_server.oauth2_token_cache import resolve_mcp_auth from litellm.proxy._experimental.mcp_server.utils import ( MCP_TOOL_PREFIX_SEPARATOR, + MCPMissingUserEnvVarsError, add_server_prefix_to_name, + build_env_var_setup_url, + collect_env_var_references, compute_short_server_prefix, get_server_prefix, + interpolate_headers, is_short_mcp_tool_prefix_enabled, is_tool_name_prefixed, iter_known_server_prefixes, merge_mcp_headers, normalize_server_name, + parse_admin_env_vars, split_server_prefix_from_name, validate_mcp_server_name, ) from litellm.proxy._types import ( LiteLLM_MCPServerTable, MCPAuthType, + MCPEnvVar, MCPTransport, MCPTransportType, UserAPIKeyAuth, @@ -73,6 +85,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper from litellm.proxy.utils import ProxyLogging +from litellm.repositories.table_repositories import MCPServerRepository from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.mcp import MCPAuth, MCPStdioConfig from litellm.types.mcp_server.mcp_server_manager import ( @@ -118,6 +131,33 @@ _AZURE_ENTRA_HOSTS = { "login.chinacloudapi.cn", # China } +# Short-lived in-memory cache for per-user MCP env var values, mirroring the +# BYOK credential cache. Keyed by (user_id, server_id); value is +# (values_dict, monotonic_timestamp). Keeps the tool-call and tool-listing +# paths off the DB on every request within the TTL window. +_user_env_vars_cache: Dict[Tuple[str, str], Tuple[Dict[str, str], float]] = {} +_USER_ENV_VARS_CACHE_TTL = 60 # seconds +_USER_ENV_VARS_CACHE_MAX_SIZE = 4096 # cap to prevent unbounded growth + + +def invalidate_user_env_vars_cache(user_id: str, server_id: str) -> None: + """Drop a cached entry after the user stores or clears their env var values + so the next request reads the fresh value instead of a stale one.""" + _user_env_vars_cache.pop((user_id, server_id), None) + + +def _write_user_env_vars_cache( + user_id: str, server_id: str, values: Dict[str, str] +) -> None: + cache_key = (user_id, server_id) + # Re-insert at the tail so eviction drops the oldest-written entry, not a + # freshly refreshed one, and only sheds a single entry instead of wiping the + # whole cache (which would stampede the DB). + _user_env_vars_cache.pop(cache_key, None) + if len(_user_env_vars_cache) >= _USER_ENV_VARS_CACHE_MAX_SIZE: + _user_env_vars_cache.pop(next(iter(_user_env_vars_cache)), None) + _user_env_vars_cache[cache_key] = (values, time.monotonic()) + def _should_strip_caller_authorization( mcp_server: MCPServer, @@ -289,6 +329,107 @@ def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]: return data +def _deserialize_json_list(data: Any) -> Optional[List[Dict[str, Any]]]: + """Deserialize a JSON array stored in the DB (``env_vars`` and friends). + + Returns ``None`` for empty / null / unparseable input. Accepts strings + (raw JSON), already-materialized lists of dicts, and lists of Pydantic + models (Prisma may hydrate a JSON column such as ``env_vars`` into + ``MCPEnvVar`` objects); model entries are normalized to plain dicts so + downstream consumers expecting ``List[Dict[str, Any]]`` validate. + """ + if data is None or data == "" or data == []: + return None + if isinstance(data, str): + try: + parsed = json.loads(data) + except (json.JSONDecodeError, TypeError): + return None + data = parsed + if not isinstance(data, list): + return None + return [ + item.model_dump(mode="json") if hasattr(item, "model_dump") else item + for item in data + ] + + +def _create_sampling_callback(user_api_key_auth: Optional[Any] = None): + """ + Create a sampling callback for MCP ClientSession. + Returns a callable that handles sampling/createMessage requests from + upstream MCP servers by routing them through litellm.acompletion(). + """ + if not MCP_SAMPLING_AVAILABLE: + return None + + async def _sampling_callback(context, params): + from litellm.proxy._experimental.mcp_server.sampling_handler import ( + handle_sampling_create_message, + ) + import litellm + from litellm.proxy._experimental.mcp_server.server import ( + get_active_auth_context, + ) + + auth_context = get_active_auth_context() + resolved_auth = user_api_key_auth or ( + auth_context.user_api_key_auth if auth_context else None + ) + # Forward original HTTP headers and client IP so that + # header-dependent guardrails, tag-based routing, trace + # correlation, and forward_llm_provider_auth_headers work + # correctly for sampling sub-calls. + _raw_headers = getattr(auth_context, "raw_headers", None) + _client_ip = getattr(auth_context, "client_ip", None) + + return await handle_sampling_create_message( + context=context, + params=params, + default_model=getattr(litellm, "default_mcp_sampling_model", None), + user_api_key_auth=resolved_auth, + raw_headers=_raw_headers, + client_ip=_client_ip, + ) + + return _sampling_callback + + +def _create_elicitation_callback(): + """ + Create an elicitation callback for MCP ClientSession. + Returns a callable that handles elicitation/create requests from + upstream MCP servers. In gateway mode, this relays to the downstream + client; in tool bridge mode, it returns a decline response. + """ + if not MCP_ELICITATION_AVAILABLE: + return None + + async def _elicitation_callback(context, params): + from litellm.proxy._experimental.mcp_server.elicitation_handler import ( + handle_elicitation_request, + ) + from litellm.proxy._experimental.mcp_server.server import get_active_mcp_session + + # In Gateway mode, we relay the elicitation request to the downstream client + # that triggered the current operation. + downstream_session = get_active_mcp_session() + downstream_capabilities = ( + getattr(downstream_session, "capabilities", None) + if downstream_session + else None + ) + + return await handle_elicitation_request( + context=context, + params=params, + downstream_session=downstream_session, + downstream_capabilities=downstream_capabilities, + ) + + return _elicitation_callback + + class MCPServerManager: _STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$") @@ -374,7 +515,8 @@ class MCPServerManager: - server is OpenAPI (spec_path), - non-empty upstream instructions are already cached, - auth preconditions match health_check_server's skip rules - (per-user auth / missing static auth token), + (per-user auth / missing static auth token / static headers that + reference a per-user env var), - a prior probe attempt for this server is within MCP_HEALTH_CHECK_TIMEOUT seconds (the probe is a health-check-shaped op and already uses this knob for its inner call timeout; reusing it @@ -389,6 +531,8 @@ class MCPServerManager: return if server.requires_per_user_auth: return + if self._references_per_user_env_var(server): + return if ( server.auth_type and server.auth_type != MCPAuth.none @@ -413,8 +557,13 @@ class MCPServerManager: ) try: + resolved_static_headers = await self._resolve_static_headers_with_env_vars( + server=server, + user_api_key_auth=None, + raise_on_missing=False, + ) extra_headers: Optional[Dict[str, str]] = ( - dict(server.static_headers) if server.static_headers else None + dict(resolved_static_headers) if resolved_static_headers else None ) client = await self._create_mcp_client( server=server, @@ -574,6 +723,7 @@ class MCPServerManager: allowed_params=server_config.get("allowed_params", None), access_groups=server_config.get("access_groups", None), static_headers=server_config.get("static_headers", None), + env_vars=server_config.get("env_vars", None), allow_all_keys=bool(server_config.get("allow_all_keys", False)), available_on_public_internet=bool( server_config.get("available_on_public_internet", True) @@ -600,6 +750,9 @@ class MCPServerManager: "subject_token_type", "urn:ietf:params:oauth:token-type:access_token", ), + allow_sampling=bool(server_config.get("allow_sampling", False)), + allow_elicitation=bool(server_config.get("allow_elicitation", False)), + timeout=server_config.get("timeout", None), ) self._assign_unique_short_prefix(new_server) _warn_internal_delegate_pkce_if_applicable(new_server, source="config") @@ -699,8 +852,7 @@ class MCPServerManager: ) verbose_logger.debug( - f"Using headers for OpenAPI tools (excluding sensitive values): " - f"{list(headers.keys())}" + f"Using headers for OpenAPI tools (excluding sensitive values): {list(headers.keys())}" ) # Extract and register tools from OpenAPI paths @@ -836,17 +988,41 @@ class MCPServerManager: f"Server ID {mcp_server.server_id} not found in registry" ) + def _resolve_env_vars_list( + self, + mcp_server: LiteLLM_MCPServerTable, + *, + env_vars_are_encrypted: bool, + ) -> Optional[List[Dict[str, Any]]]: + env_vars_list = _deserialize_json_list(getattr(mcp_server, "env_vars", None)) + if env_vars_are_encrypted: + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + decrypt_global_env_var_values, + ) + + decrypt_global_env_var_values(env_vars_list) + return env_vars_list + async def build_mcp_server_from_table( self, mcp_server: LiteLLM_MCPServerTable, *, credentials_are_encrypted: bool = True, + env_vars_are_encrypted: Optional[bool] = None, ) -> MCPServer: _mcp_info: MCPInfo = mcp_server.mcp_info or {} env_dict = _deserialize_json_dict(getattr(mcp_server, "env", None)) static_headers_dict = _deserialize_json_dict( getattr(mcp_server, "static_headers", None) ) + env_vars_list = self._resolve_env_vars_list( + mcp_server, + env_vars_are_encrypted=( + credentials_are_encrypted + if env_vars_are_encrypted is None + else env_vars_are_encrypted + ), + ) credentials_dict = _deserialize_json_dict( getattr(mcp_server, "credentials", None) ) @@ -946,6 +1122,7 @@ class MCPServerManager: mcp_info=mcp_info, extra_headers=getattr(mcp_server, "extra_headers", None), static_headers=static_headers_dict, + env_vars=env_vars_list, client_id=client_id_value or getattr(mcp_server, "client_id", None), client_secret=client_secret_value or getattr(mcp_server, "client_secret", None), @@ -1013,6 +1190,7 @@ class MCPServerManager: credentials_dict.get("subject_token_type") if credentials_dict else None ) or "urn:ietf:params:oauth:token-type:access_token", + timeout=getattr(mcp_server, "timeout", None), ) _warn_internal_delegate_pkce_if_applicable(new_server, source="database") return new_server @@ -1043,7 +1221,14 @@ class MCPServerManager: return try: if mcp_server.server_id not in self.registry: - new_server = await self.build_mcp_server_from_table(mcp_server) + # Callers hand us a record returned by the db.py read/write + # helpers, which already decrypt global env var values (the + # `credentials` field is the only one still encrypted here). + # Re-decrypting plaintext would zero the values, so build with + # env_vars_are_encrypted=False. + new_server = await self.build_mcp_server_from_table( + mcp_server, env_vars_are_encrypted=False + ) self._assign_unique_short_prefix(new_server) self.registry[mcp_server.server_id] = new_server await self._maybe_register_openapi_tools(new_server) @@ -1066,7 +1251,11 @@ class MCPServerManager: return try: if mcp_server.server_id in self.registry: - new_server = await self.build_mcp_server_from_table(mcp_server) + # See add_server: db.py helpers already decrypted env var + # values, so don't decrypt them a second time here. + new_server = await self.build_mcp_server_from_table( + mcp_server, env_vars_are_encrypted=False + ) # Carry the previously-resolved short prefix across so the # tool names stay stable for clients holding cached lists. existing_prefix = self.registry[mcp_server.server_id].short_prefix @@ -1487,6 +1676,180 @@ class MCPServerManager: return resolved_env + def _references_per_user_env_var(self, server: MCPServer) -> bool: + """True when ``server.static_headers`` reference a per-user ``${NAME}`` env var. + + Such placeholders can only be filled from a calling user's stored values, + so a userless probe (health check / instructions prefetch) would forward + the literal ``${NAME}`` upstream and get rejected. Callers skip the probe + and report ``unknown`` instead of a misleading ``unhealthy``. + """ + static_headers = server.static_headers + env_vars = getattr(server, "env_vars", None) + if not static_headers or not env_vars: + return False + _global_values, user_specs = parse_admin_env_vars(env_vars) + user_var_names = {spec["name"] for spec in user_specs} + if not user_var_names: + return False + referenced = collect_env_var_references(strings=static_headers.values()) + return bool(referenced & user_var_names) + + async def _resolve_static_headers_with_env_vars( + self, + server: MCPServer, + user_api_key_auth: Optional[UserAPIKeyAuth], + *, + raise_on_missing: bool = True, + ) -> Optional[Dict[str, str]]: + """Return server.static_headers with ``${NAME}`` interpolated. + + Globals come from ``server.env_vars`` entries with ``scope=="global"``. + Per-user values come from the ``LiteLLM_MCPUserEnvVars`` row for the + calling user. + + When ``raise_on_missing`` is ``True`` (the tool-*call* path), raises + ``MCPMissingUserEnvVarsError`` if ``static_headers`` reference a per-user + variable the calling user has not yet supplied — converted into a + user-facing 412 by the REST layer. + + When ``raise_on_missing`` is ``False`` (the tool-*list* path), missing + per-user vars are non-blocking: we interpolate whatever is available and + leave unfilled ``${NAME}`` references untouched, so the server's tools + still appear in the listing. The user only hits the friendly error when + they actually invoke a tool that needs the missing value. + """ + static_headers = server.static_headers + env_vars = getattr(server, "env_vars", None) + if not static_headers and not env_vars: + return static_headers + + global_values, user_specs = parse_admin_env_vars(env_vars) + # An empty-valued global is treated as unset: it must not mask a per-user + # var the user still has to supply, nor override a value the user did + # supply. The unresolved ${NAME} is then left untouched, like any other + # undefined reference. + global_values = {name: value for name, value in global_values.items() if value} + user_var_names = {spec["name"] for spec in user_specs} + + # If no env vars are configured, return static_headers as-is. + if not global_values and not user_specs: + return static_headers + + # Figure out which user-scoped vars are actually referenced. A var that + # also carries a global value is always covered by that global (globals + # win in the merge below), so it can never be genuinely "missing" even if + # the user hasn't filled it in -- only vars without a global fallback do. + referenced = collect_env_var_references(strings=(static_headers or {}).values()) + referenced_user_vars = referenced & user_var_names + required_user_vars = { + name for name in referenced_user_vars if name not in global_values + } + + user_values: Dict[str, str] = {} + if required_user_vars: + try: + user_values = await self._load_user_env_vars(server, user_api_key_auth) + except Exception as exc: + # On the tool-call path a DB failure must surface as a real + # server error, not a misleading "set up your credentials" 412. + # On the listing path we stay best-effort and leave the + # unfilled ${NAME} references untouched so tools still appear. + if raise_on_missing: + raise + verbose_logger.warning( + "MCPServerManager: best-effort user env var load failed for " + "server=%s: %s", + server.server_id, + exc, + ) + + if raise_on_missing: + missing = sorted( + name for name in required_user_vars if not user_values.get(name) + ) + if missing: + # A cached negative must never produce a 412: cache + # invalidation is process-local, so a user who just stored + # values on another worker would otherwise be told their + # credentials are missing until the entry expires. Confirm + # against the DB before raising. + user_values = await self._load_user_env_vars( + server, user_api_key_auth, force_refresh=True + ) + missing = sorted( + name for name in required_user_vars if not user_values.get(name) + ) + if missing: + raise MCPMissingUserEnvVarsError( + server_id=server.server_id, + server_name=server.server_name or server.name, + missing=missing, + setup_url=build_env_var_setup_url(server.server_id), + ) + + # Only honor stored user values for currently user-scoped vars, and let + # admin globals win, so a stale row from when a var was user-scoped can + # never override the global value the admin set after switching it. + scoped_user_values = { + name: value for name, value in user_values.items() if name in user_var_names + } + merged_vars: Dict[str, str] = {**scoped_user_values, **global_values} + if not static_headers: + return static_headers + return interpolate_headers(static_headers, merged_vars) + + async def _load_user_env_vars( + self, + server: MCPServer, + user_api_key_auth: Optional[UserAPIKeyAuth], + *, + force_refresh: bool = False, + ) -> Dict[str, str]: + """Look up the calling user's env var values for ``server``. + + Returns an empty dict when no user is available. Results are cached in a + short-lived in-memory map keyed by (user_id, server_id) so the tool-call + and tool-listing paths avoid a DB round-trip per request within the TTL + window; the cache is invalidated when the user stores or clears values. + Pass ``force_refresh`` to bypass the cache read and re-fetch from the DB + (used before raising a "missing credentials" error so a process-local + stale entry cannot mask values stored on another worker). A missing DB + connection and any other DB error propagate so the caller can decide + between failing the request (tool-call path) and staying best-effort + (listing path); they must never be mistaken for "user has no values", + which would send the user a misleading "set up your credentials" 412. + """ + if user_api_key_auth is None: + return {} + user_id = getattr(user_api_key_auth, "user_id", None) + if not user_id: + return {} + + cache_key = (user_id, server.server_id) + if not force_refresh: + cached = _user_env_vars_cache.get(cache_key) + if cached is not None: + values, ts = cached + if time.monotonic() - ts < _USER_ENV_VARS_CACHE_TTL: + return values + + from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 + + if prisma_client is None: + raise RuntimeError( + "MCP per-user env vars require a database connection, but none " + "is configured. Connect a database to your proxy to use per-user " + "MCP env vars." + ) + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + get_user_env_vars, + ) + + values = await get_user_env_vars(prisma_client, user_id, server.server_id) + _write_user_env_vars_cache(user_id, server.server_id, values) + return values + async def _create_mcp_client( self, server: MCPServer, @@ -1494,6 +1857,7 @@ class MCPServerManager: extra_headers: Optional[Dict[str, str]] = None, stdio_env: Optional[Dict[str, str]] = None, subject_token: Optional[str] = None, + user_api_key_auth: Optional[UserAPIKeyAuth] = None, ) -> MCPClient: """ Create an MCPClient instance for the given server. @@ -1510,6 +1874,7 @@ class MCPServerManager: extra_headers: Additional headers to forward. stdio_env: Environment variables for stdio transport. subject_token: Optional user JWT for token exchange (OBO) flow. + user_api_key_auth: Optional auth context for sampling callbacks. Returns: Configured MCP client instance. @@ -1520,23 +1885,44 @@ class MCPServerManager: transport = server.transport or MCPTransport.sse + # Create sampling and elicitation callbacks for this client + sampling_cb = ( + _create_sampling_callback(user_api_key_auth=user_api_key_auth) + if server.allow_sampling + else None + ) + elicitation_cb = ( + _create_elicitation_callback() if server.allow_elicitation else None + ) + # Handle stdio transport if transport == MCPTransport.stdio: resolved_env = ( - stdio_env if stdio_env is not None else dict(server.env or {}) + stdio_env + if stdio_env is not None + else (dict(server.env) if server.env is not None else None) ) # Ensure npm-based STDIO MCP servers have a writable cache dir. # In containers the default (~/.npm or /app/.npm) may not exist # or be read-only, causing npx to fail with ENOENT. - if "NPM_CONFIG_CACHE" not in resolved_env: + if resolved_env is not None and "NPM_CONFIG_CACHE" not in resolved_env: resolved_env["NPM_CONFIG_CACHE"] = MCP_NPM_CACHE_DIR # Defense-in-depth: block commands not in the allowlist. # The Pydantic validator blocks new servers; this catches legacy # config/DB records predating the allowlist. if server.command: base_command = os.path.basename(server.command) - if base_command not in MCP_STDIO_ALLOWED_COMMANDS: + # Strip .exe/.cmd/.bat/.com suffix for Windows compatibility + base_command_no_ext = base_command.lower() + for ext in [".exe", ".cmd", ".bat", ".com"]: + if base_command.lower().endswith(ext): + base_command_no_ext = base_command[: -len(ext)].lower() + break + if ( + base_command.lower() not in MCP_STDIO_ALLOWED_COMMANDS + and base_command_no_ext not in MCP_STDIO_ALLOWED_COMMANDS + ): raise HTTPException( status_code=403, detail=f"MCP stdio command '{server.command}' is not in the allowlist ({sorted(MCP_STDIO_ALLOWED_COMMANDS)}). " @@ -1556,9 +1942,13 @@ class MCPServerManager: transport_type=transport, auth_type=server.auth_type, auth_value=auth_value, - timeout=MCP_CLIENT_TIMEOUT, + timeout=( + server.timeout if server.timeout is not None else MCP_CLIENT_TIMEOUT + ), stdio_config=stdio_config, extra_headers=extra_headers, + sampling_callback=sampling_cb, + elicitation_callback=elicitation_cb, ) else: # For HTTP/SSE transports @@ -1582,9 +1972,13 @@ class MCPServerManager: transport_type=transport, auth_type=server.auth_type, auth_value=auth_value, - timeout=MCP_CLIENT_TIMEOUT, + timeout=( + server.timeout if server.timeout is not None else MCP_CLIENT_TIMEOUT + ), extra_headers=extra_headers, aws_auth=aws_auth, + sampling_callback=sampling_cb, + elicitation_callback=elicitation_cb, ) async def _get_tools_from_server( @@ -1616,10 +2010,17 @@ class MCPServerManager: client = None try: - if server.static_headers: + # Tool *listing* must not be blocked by missing per-user env vars — + # the server's tools should still appear so the client connects. The + # friendly "missing vars" error is raised only on the tool-*call* + # path (see _call_regular_mcp_tool). + resolved_static_headers = await self._resolve_static_headers_with_env_vars( + server, user_api_key_auth, raise_on_missing=False + ) + if resolved_static_headers: if extra_headers is None: extra_headers = {} - extra_headers.update(server.static_headers) + extra_headers.update(resolved_static_headers) # MCPJWTSigner: inject signed JWT for tools/list (list path skips pre_call_hook). # Skip entirely when the signer is not configured (avoid an unnecessary @@ -1668,6 +2069,7 @@ class MCPServerManager: mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, stdio_env=stdio_env, + user_api_key_auth=user_api_key_auth, ) ## HANDLE OPENAPI TOOLS @@ -2988,10 +3390,17 @@ class MCPServerManager: continue extra_headers[header] = header_value - if mcp_server.static_headers: + # Interpolate env vars into static_headers. Raises + # MCPMissingUserEnvVarsError when the calling user has not filled in + # a required per-user variable — the REST layer converts that into + # a friendly 412 with a setup URL. + resolved_static_headers = await self._resolve_static_headers_with_env_vars( + mcp_server, user_api_key_auth + ) + if resolved_static_headers: if extra_headers is None: extra_headers = {} - extra_headers.update(mcp_server.static_headers) + extra_headers.update(resolved_static_headers) if hook_extra_headers: if extra_headers is None: @@ -3030,6 +3439,7 @@ class MCPServerManager: extra_headers=extra_headers, stdio_env=stdio_env, subject_token=subject_token, + user_api_key_auth=user_api_key_auth, ) call_tool_params = MCPCallToolRequestParams( @@ -3046,14 +3456,26 @@ class MCPServerManager: asyncio.create_task(_call_tool_via_client(client, call_tool_params)) ) + _timeout = ( + mcp_server.timeout if mcp_server.timeout is not None else MCP_CLIENT_TIMEOUT + ) try: - mcp_responses = await asyncio.gather(*tasks) + mcp_responses = await asyncio.wait_for( + asyncio.gather(*tasks), timeout=_timeout + ) + except asyncio.TimeoutError: + raise HTTPException( + status_code=504, + detail={ + "error": "timeout", + "message": f"MCP tool call timed out after {_timeout}s", + }, + ) except ( BlockedPiiEntityError, GuardrailRaisedException, HTTPException, ) as e: - # Re-raise guardrail exceptions to properly fail the MCP call verbose_logger.error( f"Guardrail blocked MCP tool call during result check: {str(e)}" ) @@ -3260,7 +3682,6 @@ class MCPServerManager: ) ) else: - # For regular MCP servers, use the MCP client return await self._call_regular_mcp_tool( mcp_server=mcp_server, original_tool_name=name, @@ -3397,7 +3818,7 @@ class MCPServerManager: # Pending/rejected servers are excluded at the DB level so we never load them. from litellm.proxy._experimental.mcp_server.db import LiteLLM_MCPServerTable - raw_rows = await prisma_client.db.litellm_mcpservertable.find_many( + raw_rows = await MCPServerRepository(prisma_client).table.find_many( where={ "OR": [ {"approval_status": None}, @@ -3437,7 +3858,13 @@ class MCPServerManager: verbose_logger.debug( f"Building server from DB: {server.server_id} ({server.server_name})" ) - new_server = await self.build_mcp_server_from_table(server) + # raw_rows come straight from the DB, so their global env var + # values (like credentials) are still encrypted here, unlike the + # already-decrypted records add_server/update_server are handed. + # Decrypt them while building the registry entry. + new_server = await self.build_mcp_server_from_table( + server, env_vars_are_encrypted=True + ) # Carry the cached short_prefix from the previous registry entry # (if any) so the prefix is stable across reloads. if existing_server is not None and existing_server.short_prefix: @@ -3540,15 +3967,37 @@ class MCPServerManager: def get_public_mcp_servers(self) -> List[MCPServer]: """ - Get the public MCP servers (available_on_public_internet=True flag on server). - Also includes servers from litellm.public_mcp_servers for backwards compat. + Return the MCP servers published to the AI Hub via /v1/mcp/make_public. + + Default (litellm.public_mcp_hub_strict_whitelist=True): mirrors + /public/model_hub and /public/agent_hub — gates strictly on the + litellm.public_mcp_servers whitelist. Returns an empty list when no + servers have been published. The per-server available_on_public_internet + flag is unrelated — it governs IP-based access in + _is_server_accessible_from_ip, not hub visibility. + + Legacy (litellm.public_mcp_hub_strict_whitelist=False): preserves the + pre-fix behavior where any server with available_on_public_internet=True + is also included. Intended as a one-release migration window for + deployments that relied on the OR-with-default semantics; will be + removed in a future release. """ - servers: List[MCPServer] = [] + if litellm.public_mcp_hub_strict_whitelist: + if litellm.public_mcp_servers is None: + return [] + public_ids = set(litellm.public_mcp_servers) + return [ + server + for server in self.get_registry().values() + if server.server_id in public_ids + ] + public_ids = set(litellm.public_mcp_servers or []) - for server in self.get_registry().values(): - if server.available_on_public_internet or server.server_id in public_ids: - servers.append(server) - return servers + return [ + server + for server in self.get_registry().values() + if server.available_on_public_internet or server.server_id in public_ids + ] def expand_permission_list(self, identifiers: List[str]) -> List[str]: """ @@ -3755,11 +4204,21 @@ class MCPServerManager: and not server.authentication_token ): should_skip_health_check = True + # Skip if static_headers reference a per-user env var: a userless probe + # can't fill ${NAME} and would forward the literal placeholder upstream, + # flipping the server to unhealthy even though real user calls succeed. + elif self._references_per_user_env_var(server): + should_skip_health_check = True if not should_skip_health_check: - extra_headers = {} - if server.static_headers: - extra_headers.update(server.static_headers) + resolved_static_headers = await self._resolve_static_headers_with_env_vars( + server=server, + user_api_key_auth=None, + raise_on_missing=False, + ) + extra_headers = ( + dict(resolved_static_headers) if resolved_static_headers else {} + ) client = await self._create_mcp_client( server=server, @@ -3809,6 +4268,7 @@ class MCPServerManager: extra_headers=server.extra_headers or [], mcp_info=server.mcp_info, static_headers=server.static_headers, + env_vars=self._env_vars_to_models(server.env_vars), status=status, last_health_check=datetime.now(), health_check_error=health_check_error, @@ -3820,6 +4280,7 @@ class MCPServerManager: registration_url=server.registration_url, allow_all_keys=server.allow_all_keys, instructions=server.instructions, + timeout=server.timeout, ) async def get_all_mcp_servers_with_health_and_teams( @@ -3881,6 +4342,14 @@ class MCPServerManager: return list_mcp_servers + @staticmethod + def _env_vars_to_models( + env_vars: Optional[List[Dict[str, Any]]], + ) -> Optional[List[MCPEnvVar]]: + if env_vars is None: + return None + return [MCPEnvVar.model_validate(env_var) for env_var in env_vars] + def _build_mcp_server_table(self, server: MCPServer) -> LiteLLM_MCPServerTable: return LiteLLM_MCPServerTable( server_id=server.server_id, @@ -3901,6 +4370,7 @@ class MCPServerManager: extra_headers=server.extra_headers or [], mcp_info=server.mcp_info, static_headers=server.static_headers, + env_vars=self._env_vars_to_models(server.env_vars), status=None, # No health check performed last_health_check=None, # No health check performed health_check_error=None, @@ -3919,6 +4389,7 @@ class MCPServerManager: byok_api_key_help_url=server.byok_api_key_help_url, source_url=server.source_url, instructions=server.instructions, + timeout=server.timeout, ) async def get_all_mcp_servers_unfiltered(self) -> List[LiteLLM_MCPServerTable]: diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index e20c9f3a082..725f7a335bc 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1,3 +1,4 @@ +import asyncio import importlib from datetime import datetime from typing import ( @@ -13,6 +14,7 @@ from typing import ( Union, ) +import httpx from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from litellm._logging import verbose_logger @@ -20,7 +22,10 @@ from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthErr from litellm.proxy._experimental.mcp_server.ui_session_utils import ( build_effective_auth_contexts, ) -from litellm.proxy._experimental.mcp_server.utils import merge_mcp_headers +from litellm.proxy._experimental.mcp_server.utils import ( + MCPMissingUserEnvVarsError, + merge_mcp_headers, +) from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -41,6 +46,28 @@ router = APIRouter( tags=["mcp"], ) + +def _connection_error_message(exc: BaseException) -> str: + if isinstance(exc, httpx.LocalProtocolError): + return ( + "Failed to connect to MCP server: a request header is malformed. " + "Check static headers for leading/trailing spaces or illegal characters." + ) + if isinstance(exc, (httpx.ConnectError, httpx.ConnectTimeout)): + return ( + "Failed to connect to MCP server: the server is unreachable. " + "Check the URL and that the server is running." + ) + if isinstance(exc, httpx.TimeoutException): + return "Failed to connect to MCP server: the connection timed out." + if isinstance(exc, httpx.HTTPStatusError): + return ( + f"Failed to connect to MCP server: it returned HTTP " + f"{exc.response.status_code}." + ) + return "Failed to connect to MCP server. Check proxy logs for details." + + if MCP_AVAILABLE: from mcp.types import Tool as MCPTool @@ -119,9 +146,10 @@ if MCP_AVAILABLE: try: from litellm.proxy._experimental.mcp_server.db import ( get_user_oauth_credential, - is_oauth_credential_expired, + resolve_valid_user_oauth_token, ) + prisma_client = None if prefetched_creds is not None: cred = prefetched_creds.get(server_id) else: @@ -133,13 +161,13 @@ if MCP_AVAILABLE: cred = await get_user_oauth_credential( prisma_client, user_id, server_id ) + cred = await resolve_valid_user_oauth_token( + user_id=user_id, + server=server, + cred=cred, + prisma_client=prisma_client, + ) if cred and cred.get("access_token"): - if is_oauth_credential_expired(cred): - verbose_logger.debug( - f"_get_user_oauth_extra_headers: token expired for " - f"user={user_id} server={server_id}" - ) - return None return {"Authorization": f"Bearer {cred['access_token']}"} except Exception as e: verbose_logger.warning( @@ -812,6 +840,23 @@ if MCP_AVAILABLE: requested_server_id=canonical_server_id, ) return result + except MCPMissingUserEnvVarsError as e: + verbose_logger.info( + "MCP tool call missing per-user env vars: server_id=%s missing=%s", + e.server_id, + e.missing, + ) + raise HTTPException( + status_code=412, + detail={ + "error": "missing_user_env_vars", + "message": str(e), + "server_id": e.server_id, + "server_name": e.server_name, + "missing": e.missing, + "setup_url": e.setup_url, + }, + ) except BlockedPiiEntityError as e: verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {str(e)}") raise HTTPException( @@ -961,14 +1006,14 @@ if MCP_AVAILABLE: return await operation(client) - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit, asyncio.CancelledError): raise except BaseException as e: verbose_logger.error("Error in MCP operation: %s", e, exc_info=True) return { "status": "error", "error": True, - "message": "Failed to connect to MCP server. Check proxy logs for details.", + "message": _connection_error_message(e), } async def _preview_openapi_tools(spec_path: str) -> dict: diff --git a/litellm/proxy/_experimental/mcp_server/sampling_handler.py b/litellm/proxy/_experimental/mcp_server/sampling_handler.py new file mode 100644 index 00000000000..1637c9eb0b9 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/sampling_handler.py @@ -0,0 +1,1279 @@ +""" +MCP Sampling Handler +Handles `sampling/createMessage` requests from upstream MCP servers by +routing them through LiteLLM's internal completion infrastructure. +This allows MCP servers to perform agentic reasoning (e.g., multi-step +tool calling, chain-of-thought) without needing their own LLM API keys — +LiteLLM acts as the LLM provider using its existing 100+ provider support, +cost tracking, rate limiting, and model routing. +MCP Spec Reference: + https://modelcontextprotocol.io/specification/2025-11-25/client/sampling +""" + +from typing import Any, Dict, List, Optional, Union +import typing + +if typing.TYPE_CHECKING: + from litellm.proxy.utils import ProxyLogging + +from litellm._logging import verbose_logger + +from fastapi import HTTPException + +# Guard imports that require the mcp package +try: + from mcp.types import ( + CreateMessageRequestParams, + CreateMessageResult, + CreateMessageResultWithTools, + ErrorData, + ModelPreferences, + SamplingMessage, + TextContent, + Tool, + ToolChoice, + ToolUseContent, + ) + + MCP_SAMPLING_AVAILABLE = True +except ImportError as _sampling_import_err: + MCP_SAMPLING_AVAILABLE = False + verbose_logger.warning( + "MCP sampling disabled: failed to import required types from mcp.types — %s. " + "This usually means the 'mcp' package is not installed or is an older version " + "that does not support sampling. Install/upgrade with: pip install 'mcp>=1.1'", + _sampling_import_err, + ) + + +def _resolve_model_from_preferences( + model_preferences: Optional["ModelPreferences"], + default_model: Optional[str] = None, +) -> str: + """ + Resolve an LLM model name from MCP ModelPreferences. + Strategy: + 1. Check hints for substring matches against known model names. + 2. Fall back to priority-based selection (cost/speed/intelligence). + 3. Fall back to the configured default model. + Args: + model_preferences: MCP ModelPreferences with hints and priorities. + default_model: Fallback model if no hint matches. + Returns: + A model string suitable for litellm.acompletion(). + """ + import litellm + + # Build list of available model names from proxy Router or litellm.model_list + available_model_names: list = [] + try: + from litellm.proxy.proxy_server import llm_router + + if llm_router is not None: + available_model_names = llm_router.get_model_names() + except Exception: + pass + if not available_model_names and litellm.model_list: + for entry in litellm.model_list: + if isinstance(entry, dict): + name = entry.get("model_name") + if name: + available_model_names.append(name) + elif isinstance(entry, str): + available_model_names.append(entry) + if model_preferences and model_preferences.hints: + for hint in model_preferences.hints: + hint_name = getattr(hint, "name", None) + if not hint_name: + continue + # Try direct match first + if hint_name in available_model_names: + verbose_logger.debug( + "MCP sampling model resolution: direct hint match '%s'", + hint_name, + ) + return hint_name + # Try substring match against known models + for model_name in available_model_names: + if hint_name.lower() in model_name.lower(): + verbose_logger.debug( + "MCP sampling model resolution: substring hint match " + "'%s' -> '%s'", + hint_name, + model_name, + ) + return model_name + verbose_logger.debug( + "MCP sampling model resolution: no hint matched from %s " + "against %d available models", + [getattr(h, "name", None) for h in model_preferences.hints], + len(available_model_names), + ) + + # 2. Priority-based selection (cost/speed/intelligence) + if ( + model_preferences + and available_model_names + and _has_priorities(model_preferences) + ): + best = _select_model_by_priority(available_model_names, model_preferences) + if best is not None: + verbose_logger.debug( + "MCP sampling model resolution: priority-based selection chose '%s'", + best, + ) + return best + + # 3. Use default model from caller + if default_model: + verbose_logger.debug( + "MCP sampling model resolution: using caller-provided default '%s'", + default_model, + ) + return default_model + # Fall back to first available model + if available_model_names: + verbose_logger.debug( + "MCP sampling model resolution: no default configured, " + "falling back to first available model '%s'", + available_model_names[0], + ) + return available_model_names[0] + # Last resort - use LiteLLM default or raise error + default_sampling_model = getattr(litellm, "default_mcp_sampling_model", None) + if default_sampling_model: + verbose_logger.debug( + "MCP sampling model resolution: using litellm.default_mcp_sampling_model='%s'", + default_sampling_model, + ) + return default_sampling_model + raise ValueError( + "No model could be resolved for MCP sampling. Please configure 'default_mcp_sampling_model' in your LiteLLM configuration." + ) + + +def _has_priorities(model_preferences: "ModelPreferences") -> bool: + """Return True if any priority weight is set (non-None and > 0).""" + return any( + (getattr(model_preferences, attr, None) or 0) > 0 + for attr in ("costPriority", "speedPriority", "intelligencePriority") + ) + + +def _select_model_by_priority( + model_names: List[str], + model_preferences: "ModelPreferences", +) -> Optional[str]: + """Score available models by MCP priority weights and return the best. + + Scoring strategy (per the MCP spec, priorities are 0-1 floats): + + * **costPriority** — higher means "prefer cheaper models". + Metric: combined (input + output) cost per token from + ``model_prices_and_context_window.json``. Lower cost → higher score. + + * **speedPriority** — higher means "prefer faster models". + Metric: ``output_tokens_per_second`` from model info when available; + otherwise a neutral score for every candidate, since no reliable + latency proxy exists (context-window size does not track speed). + + * **intelligencePriority** — higher means "prefer smarter models". + Metric: ``max_output_tokens`` is used as a rough capability proxy + (frontier models expose larger context windows). + + Each metric is min-max normalised across the candidate set so that + every model gets a 0-1 score per dimension. The final score is the + weighted sum of the three normalised dimensions. + + Returns the highest-scoring model name, or None if scoring fails for + all candidates (e.g. no model_info available). + """ + import litellm as _litellm + + cost_weight = getattr(model_preferences, "costPriority", None) or 0.0 + speed_weight = getattr(model_preferences, "speedPriority", None) or 0.0 + intel_weight = getattr(model_preferences, "intelligencePriority", None) or 0.0 + + # Gather raw metrics for each model + scored: List[Dict[str, Any]] = [] + for name in model_names: + try: + info = _litellm.get_model_info(name) + except Exception: + continue + input_cost = info.get("input_cost_per_token") or 0.0 + output_cost = info.get("output_cost_per_token") or 0.0 + total_cost = input_cost + output_cost + max_output = info.get("max_output_tokens") or info.get("max_tokens") or 0 + output_tps = info.get("output_tokens_per_second") or 0.0 + scored.append( + { + "name": name, + "cost": total_cost, + "max_output": max_output, + "output_tps": output_tps, + } + ) + + if not scored: + return None + + # Min-max normalisation helpers + def _normalise(values: List[float], invert: bool = False) -> List[float]: + """Normalise to [0, 1]. If *invert*, lower raw → higher score.""" + lo, hi = min(values), max(values) + if hi == lo: + return [0.5] * len(values) # all equal → neutral score + normed = [(v - lo) / (hi - lo) for v in values] + if invert: + normed = [1.0 - n for n in normed] + return normed + + costs = [s["cost"] for s in scored] + max_outputs = [float(s["max_output"]) for s in scored] + output_tps_values = [s["output_tps"] for s in scored] + + # costPriority: lower cost → higher score (invert) + cost_scores = _normalise(costs, invert=True) + # speedPriority: use output_tokens_per_second if any model has it, + # otherwise a neutral score (no reliable latency proxy is available). + if any(v > 0 for v in output_tps_values): + speed_scores = _normalise(output_tps_values, invert=False) + else: + speed_scores = [0.5] * len(scored) + # intelligencePriority: higher max_output → smarter + intel_scores = _normalise(max_outputs, invert=False) + + best_name = None + best_score = -1.0 + for i, entry in enumerate(scored): + score = ( + cost_weight * cost_scores[i] + + speed_weight * speed_scores[i] + + intel_weight * intel_scores[i] + ) + verbose_logger.debug( + "MCP priority scoring: model=%s cost_score=%.3f speed_score=%.3f " + "intel_score=%.3f → weighted=%.3f", + entry["name"], + cost_scores[i], + speed_scores[i], + intel_scores[i], + score, + ) + if score > best_score: + best_score = score + best_name = entry["name"] + + return best_name + + +def _convert_mcp_content_to_openai( + content: Any, +) -> Union[str, Dict[str, Any], List[Dict[str, Any]]]: + """ + Convert MCP SamplingMessage content to OpenAI message content format. + Handles: + - TextContent → string or {"type": "text", "text": ...} + - ImageContent → {"type": "image_url", "image_url": {"url": "data:..."}} + - AudioContent → {"type": "input_audio", "input_audio": {...}} + - ToolUseContent → function call representation + - ToolResultContent → tool result representation + - List of mixed content → list of content parts + """ + if isinstance(content, list): + parts = [] + for item in content: + converted = _convert_single_content(item) + if isinstance(converted, list): + parts.extend(converted) + else: + parts.append(converted) + return parts + return _convert_single_content(content) + + +def _convert_single_content( + content: Any, +) -> Union[Dict[str, Any], List[Dict[str, Any]]]: + """Convert a single MCP content item to OpenAI format. + + For text/image/audio content, returns a single content-part dict. + For tool_use/tool_result, returns a dict with a ``_marker_type`` key + so the caller (``_convert_mcp_messages_to_openai``) can hoist it to + the correct message-level position (``tool_calls`` array or a + separate ``role: "tool"`` message). + """ + import json + + content_type = getattr(content, "type", None) + if content_type == "text": + return {"type": "text", "text": content.text} + elif content_type == "image": + data = getattr(content, "data", "") + mime_type = getattr(content, "mimeType", "image/png") + return { + "type": "image_url", + "image_url": {"url": f"data:{mime_type};base64,{data}"}, + } + elif content_type == "audio": + data = getattr(content, "data", "") + mime_type = getattr(content, "mimeType", "audio/wav") + # Map MIME type to OpenAI audio format + format_map = { + "audio/wav": "wav", + "audio/mp3": "mp3", + "audio/mpeg": "mp3", + "audio/flac": "flac", + "audio/ogg": "ogg", + } + audio_format = format_map.get(mime_type, "wav") + return { + "type": "input_audio", + "input_audio": {"data": data, "format": audio_format}, + } + elif content_type == "tool_use": + # ToolUseContent → proper OpenAI function-call representation. + # The ``_marker_type`` key lets the message-level converter + # hoist this into the ``tool_calls`` array on the assistant + # message instead of embedding it inline as a content part. + return { + "_marker_type": "tool_use", + "id": getattr(content, "id", f"call_{id(content)}"), + "type": "function", + "function": { + "name": getattr(content, "name", ""), + "arguments": json.dumps(getattr(content, "input", {}), default=str), + }, + } + elif content_type == "tool_result": + # ToolResultContent → proper OpenAI tool-role message. + # Marked so the message-level converter can emit it as a + # separate ``{"role": "tool", ...}`` message. + tool_use_id = getattr(content, "toolUseId", "") + nested_content = getattr(content, "content", []) + if isinstance(nested_content, list): + text_parts = [ + getattr(c, "text", str(c)) + for c in nested_content + if getattr(c, "type", None) == "text" + ] + result_text = "\n".join(text_parts) if text_parts else "" + else: + result_text = str(nested_content) + return { + "_marker_type": "tool_result", + "role": "tool", + "tool_call_id": tool_use_id, + "content": result_text, + } + # Fallback: treat as text + return {"type": "text", "text": str(content)} + + +def _convert_mcp_messages_to_openai( + messages: List["SamplingMessage"], + system_prompt: Optional[str] = None, +) -> List[Dict[str, Any]]: + """ + Convert MCP SamplingMessage list to OpenAI messages format. + MCP messages use: + - role: "user" | "assistant" + - content: TextContent | ImageContent | AudioContent | ToolUseContent + | ToolResultContent | list[...] + OpenAI messages use: + - role: "system" | "user" | "assistant" | "tool" + - content: str | list[content_part] + """ + openai_messages: List[Dict[str, Any]] = [] + # Add system prompt if provided + if system_prompt: + openai_messages.append({"role": "system", "content": system_prompt}) + for msg in messages: + role = msg.role + content = msg.content + # Handle tool use content from assistant + if role == "assistant" and _has_tool_use(content): + tool_calls = _extract_tool_calls(content) + if tool_calls: + openai_msg: Dict[str, Any] = { + "role": "assistant", + "tool_calls": tool_calls, + } + # Also include any text content alongside tool calls + text_parts = _extract_text_parts(content) + if text_parts: + openai_msg["content"] = text_parts + openai_messages.append(openai_msg) + continue + # Handle tool result content from user + if role == "user" and _has_tool_result(content): + tool_results = _extract_tool_results(content) + for tool_result in tool_results: + openai_messages.append(tool_result) + continue + # Standard text/image/audio message — also handles any stray + # tool_use / tool_result that slipped past the fast-path checks + # above (e.g. unexpected role, single non-list content). + converted = _convert_mcp_content_to_openai(content) + converted_parts = ( + converted + if isinstance(converted, list) + else ([converted] if isinstance(converted, dict) else []) + ) + + # Separate marker items from regular content parts + tool_call_markers = [] + tool_result_markers = [] + regular_parts = [] + for part in converted_parts: + marker = part.get("_marker_type") if isinstance(part, dict) else None + if marker == "tool_use": + # Strip the internal marker before emitting + tc = {k: v for k, v in part.items() if k != "_marker_type"} + tool_call_markers.append(tc) + elif marker == "tool_result": + tr = {k: v for k, v in part.items() if k != "_marker_type"} + tool_result_markers.append(tr) + else: + regular_parts.append(part) + + # Emit assistant message with tool_calls if any were found + if tool_call_markers: + openai_msg_tc: Dict[str, Any] = { + "role": "assistant", + "tool_calls": tool_call_markers, + } + if regular_parts: + openai_msg_tc["content"] = regular_parts + openai_messages.append(openai_msg_tc) + elif regular_parts: + if isinstance(converted, str): + openai_messages.append({"role": role, "content": converted}) + else: + openai_messages.append({"role": role, "content": regular_parts}) + + # Emit separate tool-result messages + for tr in tool_result_markers: + openai_messages.append(tr) + + return openai_messages + + +def _has_tool_use(content: Any) -> bool: + """Check if content contains ToolUseContent.""" + if isinstance(content, list): + return any(getattr(c, "type", None) == "tool_use" for c in content) + return getattr(content, "type", None) == "tool_use" + + +def _has_tool_result(content: Any) -> bool: + """Check if content contains ToolResultContent.""" + if isinstance(content, list): + return any(getattr(c, "type", None) == "tool_result" for c in content) + return getattr(content, "type", None) == "tool_result" + + +def _extract_tool_calls(content: Any) -> List[Dict[str, Any]]: + """Extract OpenAI-format tool_calls from MCP ToolUseContent.""" + import json + + items = content if isinstance(content, list) else [content] + tool_calls = [] + for item in items: + if getattr(item, "type", None) == "tool_use": + tool_calls.append( + { + "id": getattr(item, "id", f"call_{id(item)}"), + "type": "function", + "function": { + "name": getattr(item, "name", ""), + "arguments": json.dumps( + getattr(item, "input", {}), default=str + ), + }, + } + ) + return tool_calls + + +def _extract_text_parts(content: Any) -> Optional[str]: + """Extract text parts from mixed content.""" + items = content if isinstance(content, list) else [content] + texts = [] + for item in items: + if getattr(item, "type", None) == "text": + texts.append(getattr(item, "text", "")) + return "\n".join(texts) if texts else None + + +def _extract_tool_results(content: Any) -> List[Dict[str, Any]]: + """Extract OpenAI-format tool messages from MCP ToolResultContent.""" + items = content if isinstance(content, list) else [content] + results = [] + for item in items: + if getattr(item, "type", None) == "tool_result": + tool_use_id = getattr(item, "toolUseId", "") + # Extract text from nested content + nested_content = getattr(item, "content", []) + if isinstance(nested_content, list): + text_parts = [ + getattr(c, "text", str(c)) + for c in nested_content + if getattr(c, "type", None) == "text" + ] + result_text = "\n".join(text_parts) if text_parts else "" + else: + result_text = str(nested_content) + results.append( + { + "role": "tool", + "tool_call_id": tool_use_id, + "content": result_text, + } + ) + return results + + +def _convert_mcp_tools_to_openai( + tools: Optional[List["Tool"]], +) -> Optional[List[Dict[str, Any]]]: + """ + Convert MCP Tool definitions to OpenAI function calling format. + MCP Tool: {name, description, inputSchema} + OpenAI Tool: {type: "function", function: {name, description, parameters}} + """ + if not tools: + return None + openai_tools = [] + for tool in tools: + openai_tool = { + "type": "function", + "function": { + "name": tool.name, + "description": tool.description or "", + "parameters": tool.inputSchema + or { + "type": "object", + "properties": {}, + }, + }, + } + openai_tools.append(openai_tool) + return openai_tools + + +def _convert_mcp_tool_choice_to_openai( + tool_choice: Optional["ToolChoice"], +) -> Optional[Union[str, Dict[str, Any]]]: + """ + Convert MCP ToolChoice to OpenAI tool_choice format. + MCP: {mode: "auto"} | {mode: "required"} | {mode: "none"} + OpenAI: "auto" | "required" | "none" + """ + if not tool_choice: + return None + mode = getattr(tool_choice, "mode", "auto") + if mode == "auto": + return "auto" + elif mode == "required": + return "required" + elif mode == "none": + return "none" + return "auto" + + +def _convert_openai_response_to_mcp_result( + response: Any, + model_name: str, +) -> Union["CreateMessageResult", "CreateMessageResultWithTools", "ErrorData"]: + """ + Convert a litellm completion response to MCP CreateMessageResult. + Args: + response: The litellm ModelResponse. + model_name: The model that was used. + Returns: + MCP CreateMessageResult or CreateMessageResultWithTools. + """ + if not response.choices: + verbose_logger.warning( + "MCP sampling: LLM returned empty choices list for model=%s " + "(possible content filter or provider error)", + model_name, + ) + return ErrorData( + code=-1, + message=( + f"LLM returned no choices for model '{model_name}'. " + "This may indicate content filtering or a provider-side error." + ), + ) + choice = response.choices[0] + message = choice.message + # Determine stop reason + finish_reason = getattr(choice, "finish_reason", "stop") + if finish_reason == "tool_calls": + stop_reason = "toolUse" + elif finish_reason == "length": + stop_reason = "maxTokens" + else: + stop_reason = "endTurn" + actual_model = getattr(response, "model", model_name) or model_name + # Check if response has tool calls + tool_calls = getattr(message, "tool_calls", None) + if tool_calls: + # Build ToolUseContent items + content_parts: "List[Any]" = [] + # Include text content if present + if message.content: + content_parts.append(TextContent(type="text", text=message.content)) + # Convert tool calls to MCP ToolUseContent + for tc in tool_calls: + import json + + tool_input = tc.function.arguments + if isinstance(tool_input, str): + try: + tool_input = json.loads(tool_input) + except (json.JSONDecodeError, TypeError): + tool_input = {"raw": tool_input} + content_parts.append( + ToolUseContent( + type="tool_use", + id=tc.id, + name=tc.function.name, + input=tool_input, + ) + ) + return CreateMessageResultWithTools( + role="assistant", + content=content_parts, + model=actual_model, + stopReason=stop_reason, + ) + # Simple text response + text = message.content or "" + return CreateMessageResult( + role="assistant", + content=TextContent(type="text", text=text), + model=actual_model, + stopReason=stop_reason, + ) + + +async def _check_model_access( # noqa: PLR0915 + model: str, user_api_key_auth: Any +) -> Optional["ErrorData"]: + """Enforce model-permission checks for MCP sampling requests. + + Runs the same authorization checks as ``/chat/completions``: + key-level, team-level, per-member, user-level, and project-level + model restrictions. The model name comes from the upstream MCP + server (untrusted input). + + Returns None if authorized, or an ErrorData describing the denial. + """ + if user_api_key_auth is None: + return None + + _api_key = getattr(user_api_key_auth, "api_key", None) + _token = getattr(user_api_key_auth, "token", None) + _user_role = getattr(user_api_key_auth, "user_role", None) + + _has_real_credential = bool(_api_key) or bool(_token) + _is_admin = ( + _user_role in ("proxy_admin", "proxy_admin_viewer") if _user_role else False + ) + + if not _has_real_credential and not _is_admin: + verbose_logger.warning( + "MCP sampling: denying model access for model=%s — " + "auth context has no real LiteLLM credential (possible " + "OAuth passthrough placeholder). api_key=%s, token=%s, role=%s", + model, + bool(_api_key), + bool(_token), + _user_role, + ) + return ErrorData( + code=-1, + message=( + "Model access denied: sampling requires a valid LiteLLM " + "API key or admin credential. OAuth-only sessions cannot " + "trigger proxy model calls without explicit authorization." + ), + ) + + try: + import litellm + from litellm.proxy.auth.auth_checks import ( + can_key_call_model, + can_team_access_model, + can_user_call_model, + can_project_access_model, + _check_team_member_model_access, + get_team_object, + get_user_object, + get_project_object, + ) + + try: + from litellm.proxy.proxy_server import llm_router as _llm_router + except ImportError: + _llm_router = None + + await can_key_call_model( + model=model, + llm_model_list=getattr(litellm, "model_list", None), + valid_token=user_api_key_auth, + llm_router=_llm_router, + ) + + _team_id = getattr(user_api_key_auth, "team_id", None) + _user_id = getattr(user_api_key_auth, "user_id", None) + _project_id = getattr(user_api_key_auth, "project_id", None) + + try: + from litellm.proxy.proxy_server import ( + prisma_client as _prisma_client, + user_api_key_cache as _user_api_key_cache, + proxy_logging_obj as _proxy_logging_obj, + ) + except ImportError: + _prisma_client = None + _user_api_key_cache = None # type: ignore[assignment] + _proxy_logging_obj = None # type: ignore[assignment] + + if _team_id and _prisma_client and _user_api_key_cache: + try: + team_obj = await get_team_object( + team_id=_team_id, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + proxy_logging_obj=_proxy_logging_obj, + ) + except Exception: + team_obj = None + + if team_obj: + await can_team_access_model( + model=model, + team_object=team_obj, + llm_router=_llm_router, + team_model_aliases=getattr( + user_api_key_auth, "team_model_aliases", None + ), + ) + if _user_id and _proxy_logging_obj: + await _check_team_member_model_access( + model=model, + team_object=team_obj, + valid_token=user_api_key_auth, + llm_router=_llm_router, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + proxy_logging_obj=_proxy_logging_obj, + ) + elif not _team_id and _user_id and _prisma_client and _user_api_key_cache: + try: + user_obj = await get_user_object( + user_id=_user_id, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + user_id_upsert=False, + proxy_logging_obj=_proxy_logging_obj, + ) + except Exception: + user_obj = None + + if user_obj: + await can_user_call_model( + model=model, + llm_router=_llm_router, + user_object=user_obj, + ) + + if _project_id and _prisma_client and _user_api_key_cache: + try: + project_obj = await get_project_object( + project_id=_project_id, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + proxy_logging_obj=_proxy_logging_obj, + ) + except Exception: + project_obj = None + + if project_obj: + can_project_access_model( + model=model, + project_object=project_obj, + llm_router=_llm_router, + ) + + verbose_logger.debug( + "MCP sampling: model access check passed for model=%s", + model, + ) + return None + except Exception as access_err: + verbose_logger.warning( + "MCP sampling: model access denied for model=%s: %s", + model, + access_err, + ) + return ErrorData( + code=-1, + message=( + f"Model access denied: the API key is not authorized " + f"to use model '{model}'. {access_err}" + ), + ) + + +async def _run_budget_checks( + model: str, + user_api_key_auth: Any, + raw_headers: Optional[Dict[str, str]] = None, + client_ip: Optional[str] = None, +) -> Optional["ErrorData"]: + """Enforce key/team/user/org/global budget checks for sampling requests. + + Runs the same ``common_checks`` path that ``/chat/completions`` uses, + so sampling cannot bypass budget limits. + + Returns None if all checks pass, or an ErrorData describing the denial. + """ + try: + from litellm.proxy.auth.auth_checks import common_checks + from litellm.proxy.proxy_server import ( + general_settings, + llm_router as _llm_router, + prisma_client as _prisma_client, + proxy_logging_obj as _proxy_logging_obj, + user_api_key_cache as _user_api_key_cache, + ) + from litellm.proxy.auth.auth_checks import ( + get_team_object, + get_user_object, + ) + import litellm + except ImportError as import_err: + verbose_logger.warning( + "MCP sampling: budget check imports unavailable: %s", import_err + ) + return None # Can't enforce budgets without the modules + + _team_id = getattr(user_api_key_auth, "team_id", None) + _user_id = getattr(user_api_key_auth, "user_id", None) + + team_obj = None + if _team_id and _prisma_client and _user_api_key_cache: + try: + team_obj = await get_team_object( + team_id=_team_id, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + proxy_logging_obj=_proxy_logging_obj, + ) + except Exception: + pass + + user_obj = None + if _user_id and _prisma_client and _user_api_key_cache: + try: + user_obj = await get_user_object( + user_id=_user_id, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + user_id_upsert=False, + proxy_logging_obj=_proxy_logging_obj, + ) + except Exception: + pass + + dummy_request = _build_sampling_request( + raw_headers=raw_headers, + client_ip=client_ip, + ) + + # Enforce virtual-key route restrictions: a key limited to MCP routes + # must not be able to trigger a /chat/completions call via sampling. + # This mirrors the RouteChecks.should_call_route gate that runs in + # user_api_key_auth before common_checks for regular requests. + try: + from litellm.proxy.auth.route_checks import RouteChecks + + RouteChecks.should_call_route( + route="/chat/completions", + valid_token=user_api_key_auth, + request=dummy_request, + ) + except HTTPException as route_err: + verbose_logger.warning( + "MCP sampling: route check denied /chat/completions for key: %s", + route_err.detail, + ) + return ErrorData( + code=-1, + message=f"Sampling denied: virtual key is not allowed to call /chat/completions. {route_err.detail}", + ) + + global_proxy_spend = getattr(litellm, "_global_proxy_spend", None) + + # Build request body and merge x-litellm-tags from MCP headers BEFORE + # common_checks runs. _tag_max_budget_check inside common_checks only + # inspects request_body; without this pre-merge, header-supplied tags + # bypass per-tag budget enforcement (mirroring the regular auth path). + request_body: Dict[str, Any] = {"model": model} + try: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + LiteLLMProxyRequestSetup.apply_client_tag_policy_pre_auth( + request=dummy_request, + request_data=request_body, + user_api_key_dict=user_api_key_auth, + ) + except Exception: + # Non-fatal: tag merge is defense-in-depth; don't block sampling + # if the merge utility is unavailable or fails. + pass + + try: + await common_checks( + request_body=request_body, + team_object=team_obj, + user_object=user_obj, + end_user_object=None, + global_proxy_spend=global_proxy_spend, + general_settings=general_settings or {}, + route="/chat/completions", + llm_router=_llm_router, + proxy_logging_obj=typing.cast("ProxyLogging", _proxy_logging_obj), + valid_token=user_api_key_auth, + request=dummy_request, + ) + except Exception as budget_err: + verbose_logger.warning( + "MCP sampling: budget check failed for model=%s: %s", + model, + budget_err, + ) + return ErrorData( + code=-1, + message=f"Sampling denied: {budget_err}", + ) + + verbose_logger.debug("MCP sampling: budget checks passed for model=%s", model) + return None + + +def _build_sampling_request( + raw_headers: Optional[Dict[str, str]] = None, + client_ip: Optional[str] = None, +) -> Any: + """Build a synthetic FastAPI Request for sampling sub-calls. + + Converts the original MCP connection's HTTP headers into ASGI + scope format so that ``add_litellm_data_to_request`` can apply + header-dependent guardrails, tag-based routing, trace correlation, + and ``forward_llm_provider_auth_headers``. + + Key fields populated: + - **headers**: All original HTTP headers are forwarded (except + hop-by-hop: content-length, transfer-encoding). This ensures + ``traceparent``, ``authorization``, ``user-agent``, and + ``x-litellm-api-key`` are visible to pre-call utils. + - **client**: The ASGI ``(host, port)`` tuple so that + ``request.client.host`` returns the real client IP for + IP-based routing and guardrails. + - **server**: Derived from the running proxy's ``server_host`` + / ``server_port`` when available, avoiding the misleading + ``127.0.0.1:0`` placeholder. + - **x-forwarded-for**: Injected from ``client_ip`` if the + original headers don't already carry it, as a fallback for + IP attribution. + """ + from fastapi import Request + + # --- Build ASGI headers --- + _scope_headers: list = [(b"content-type", b"application/json")] + # Hop-by-hop headers that must NOT be forwarded into the + # synthetic request (they describe the original HTTP framing, + # not the logical request). + _HOP_BY_HOP = frozenset( + { + "content-length", + "transfer-encoding", + "connection", + "keep-alive", + "upgrade", + "te", + "trailer", + } + ) + if raw_headers: + for hdr_name, hdr_value in raw_headers.items(): + _key = hdr_name.lower() + # Skip content-type (already set), x-forwarded-for (use resolved + # client_ip instead to prevent spoofing), and hop-by-hop headers + if _key in {"content-type", "x-forwarded-for"} or _key in _HOP_BY_HOP: + continue + _scope_headers.append( + ( + _key.encode("latin-1", errors="replace"), + hdr_value.encode("utf-8"), + ) + ) + + # Inject x-forwarded-for from captured client_ip if the + # original headers don't already carry it + if client_ip and not any(h[0] == b"x-forwarded-for" for h in _scope_headers): + _scope_headers.append((b"x-forwarded-for", client_ip.encode("utf-8"))) + + # --- Derive server (host, port) from the running proxy --- + _server_host = "127.0.0.1" + _server_port = 4000 # LiteLLM default + try: + import litellm.proxy.proxy_server as proxy_server + + _proxy_host = getattr(proxy_server, "server_host", None) + _proxy_port = getattr(proxy_server, "server_port", None) + + if _proxy_host: + _server_host = str(_proxy_host) + if _proxy_port: + _server_port = int(_proxy_port) + except (ImportError, AttributeError, TypeError, ValueError): + pass + + # --- Build ASGI client tuple for request.client.host --- + _client_tuple = None + if client_ip: + _client_tuple = (client_ip, 0) + + scope: Dict[str, Any] = { + "type": "http", + "method": "POST", + "path": "/mcp/sampling/createMessage", + "scheme": "http", + "server": (_server_host, _server_port), + "query_string": b"", + "root_path": "", + "headers": _scope_headers, + } + if _client_tuple is not None: + scope["client"] = _client_tuple + + return Request(scope=scope) + + +async def _build_completion_kwargs( + params: "CreateMessageRequestParams", + model: str, + user_api_key_auth: Any, + raw_headers: Optional[Dict[str, str]], + client_ip: Optional[str], +) -> Dict[str, Any]: + openai_messages = _convert_mcp_messages_to_openai( + messages=params.messages, + system_prompt=params.systemPrompt, + ) + completion_kwargs: Dict[str, Any] = { + "model": model, + "messages": openai_messages, + "max_tokens": params.maxTokens, + } + if params.temperature is not None: + completion_kwargs["temperature"] = params.temperature + if params.stopSequences: + completion_kwargs["stop"] = params.stopSequences + openai_tools = _convert_mcp_tools_to_openai(params.tools) + if openai_tools: + completion_kwargs["tools"] = openai_tools + openai_tool_choice = _convert_mcp_tool_choice_to_openai(params.toolChoice) + if openai_tool_choice is not None: + completion_kwargs["tool_choice"] = openai_tool_choice + completion_kwargs["metadata"] = {} + if params.metadata: + completion_kwargs["metadata"]["mcp_metadata"] = params.metadata + + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.proxy_server import proxy_config + + completion_kwargs["user"] = getattr(user_api_key_auth, "user_id", None) + _dummy_request = _build_sampling_request( + raw_headers=raw_headers, client_ip=client_ip + ) + completion_kwargs = await add_litellm_data_to_request( + data=completion_kwargs, + request=_dummy_request, + user_api_key_dict=user_api_key_auth, + proxy_config=proxy_config, + ) + return completion_kwargs + + +async def _run_guardrails_and_call_llm( + completion_kwargs: Dict[str, Any], + user_api_key_auth: Any, +) -> Any: + try: + from litellm.proxy.proxy_server import proxy_logging_obj as _plo + + if _plo is not None: + completion_kwargs = await typing.cast("ProxyLogging", _plo).pre_call_hook( + user_api_key_dict=user_api_key_auth, + data=completion_kwargs, + call_type="acompletion", + ) + except ImportError: + pass + except Exception as guardrail_err: + verbose_logger.warning( + "MCP sampling: pre-call guardrail rejected request: %s", + guardrail_err, + ) + raise + + import litellm + + try: + from litellm.proxy.proxy_server import llm_router + + if llm_router is not None: + return await llm_router.acompletion(**completion_kwargs) + return await litellm.acompletion(**completion_kwargs) + except ImportError: + return await litellm.acompletion(**completion_kwargs) + + +async def handle_sampling_create_message( + context: Any, + params: "CreateMessageRequestParams", + default_model: Optional[str] = None, + user_api_key_auth: Optional[Any] = None, + raw_headers: Optional[Dict[str, str]] = None, + client_ip: Optional[str] = None, +) -> Union["CreateMessageResult", "CreateMessageResultWithTools", "ErrorData"]: + """ + Handle an MCP sampling/createMessage request by routing through LiteLLM. + This is the main entry point called by the MCP client session when an + upstream MCP server requests LLM inference. + Args: + context: MCP RequestContext (contains session info). + params: The CreateMessageRequestParams from the MCP server. + default_model: Default model to use if no preferences match. + user_api_key_auth: Auth context for the requesting user. + raw_headers: Original HTTP headers from the MCP connection. + Forwarded into the internal acompletion call so that + header-dependent guardrails, IP-routing, trace-id + correlation, and forward_llm_provider_auth_headers + work correctly for sampling sub-calls. + client_ip: Original client IP address for IP-based guardrails. + Returns: + CreateMessageResult with the LLM's response, or ErrorData on failure. + """ + if not MCP_SAMPLING_AVAILABLE: + return ErrorData( + code=-1, + message="MCP sampling is not available (mcp package not installed)", + ) + + if user_api_key_auth is None: + return ErrorData( + code=-1, + message=( + "Sampling requires an authenticated user context. " + "Internal or unauthenticated sessions cannot trigger " + "upstream-initiated model calls." + ), + ) + + try: + model = _resolve_model_from_preferences( + model_preferences=params.modelPreferences, + default_model=default_model, + ) + verbose_logger.info( + "MCP sampling: resolved model=%s from preferences=%s", + model, + params.modelPreferences, + ) + + access_denial = await _check_model_access(model, user_api_key_auth) + if access_denial is not None: + return access_denial + + budget_denial = await _run_budget_checks( + model=model, + user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, + ) + if budget_denial is not None: + return budget_denial + + completion_kwargs = await _build_completion_kwargs( + params=params, + model=model, + user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, + ) + + openai_messages = completion_kwargs["messages"] + openai_tools = completion_kwargs.get("tools") + verbose_logger.debug( + "MCP sampling: calling litellm.acompletion with model=%s, num_messages=%d, has_tools=%s", + model, + len(openai_messages), + bool(openai_tools), + ) + + response = await _run_guardrails_and_call_llm( + completion_kwargs=completion_kwargs, + user_api_key_auth=user_api_key_auth, + ) + + result = _convert_openai_response_to_mcp_result( + response=response, model_name=model + ) + verbose_logger.info( + "MCP sampling: completed successfully, model=%s, stopReason=%s", + getattr(result, "model", "unknown"), + getattr(result, "stopReason", "unknown"), + ) + return result + except Exception as e: + from litellm.exceptions import ( + AuthenticationError, + BudgetExceededError, + ContextWindowExceededError, + PermissionDeniedError, + RateLimitError, + ServiceUnavailableError, + ) + + from litellm.proxy._types import ProxyException + + if isinstance( + e, + ( + HTTPException, + BudgetExceededError, + RateLimitError, + AuthenticationError, + PermissionDeniedError, + ContextWindowExceededError, + ServiceUnavailableError, + ProxyException, + ), + ): + raise + + verbose_logger.exception("MCP sampling handler failed: %s", e) + return ErrorData( + code=-1, + message=f"Sampling failed: {str(e)}", + ) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 6e33a105ec8..0477a5d3244 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -6,6 +6,7 @@ LiteLLM MCP Server Routes import asyncio import contextlib +import contextvars import hashlib import json import time @@ -52,6 +53,7 @@ from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, LITELLM_MCP_SERVER_NAME, LITELLM_MCP_SERVER_VERSION, + MCPMissingUserEnvVarsError, add_server_prefix_to_name, get_server_prefix, iter_known_server_prefixes, @@ -125,6 +127,18 @@ try: GetPromptResult, ResourceTemplate, TextResourceContents, + Tool, + ) + from mcp.server.session import ServerSession as _McpServerSession + import weakref + + # Robust auth lookup keyed by session_object. + _session_obj_auth_storage: ( + "weakref.WeakKeyDictionary[Any, MCPAuthenticatedUser]" + ) = weakref.WeakKeyDictionary() + + active_mcp_session_var: contextvars.ContextVar[Optional[_McpServerSession]] = ( + contextvars.ContextVar("active_mcp_session", default=None) ) except ImportError as e: verbose_logger.debug(f"MCP module not found: {e}") @@ -160,6 +174,60 @@ def _mcp_session_id_from_headers( return None +def _jsonrpc_text_has_top_level_method(text: str) -> bool: + """Whether a (possibly truncated) JSON-RPC envelope has a ``method`` key at + the root object's top level. + + Used to tell a request/notification (carries ``method``) apart from a + response (carries ``result``/``error`` and no top-level ``method``). A + response payload can itself nest a ``method`` field, so only keys at the + root object's depth are inspected rather than searching the whole string. + Returns ``True`` only when a top-level ``method`` key is positively found; + truncation that hides it yields ``False``. + """ + depth = 0 + in_string = False + escaped = False + in_object: List[bool] = [] + reading_key = False + expect_key = False + key_chars: List[str] = [] + for ch in text: + if in_string: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_string = False + if reading_key and depth == 1 and "".join(key_chars) == "method": + return True + elif reading_key: + key_chars.append(ch) + continue + if ch == '"': + in_string = True + reading_key = expect_key and depth >= 1 and in_object[-1] + key_chars = [] + expect_key = False + elif ch == "{" or ch == "[": + depth += 1 + in_object.append(ch == "{") + expect_key = ch == "{" + elif ch == "}" or ch == "]": + if in_object: + in_object.pop() + depth -= 1 + if depth <= 0: + break + expect_key = False + elif ch == ",": + expect_key = bool(in_object) and in_object[-1] + elif ch == ":": + expect_key = False + return False + + if MCP_AVAILABLE: from mcp.server import Server from mcp.server.lowlevel.server import NotificationOptions @@ -483,10 +551,18 @@ if MCP_AVAILABLE: ######################################################## @server.list_tools() - async def list_tools() -> List[MCPTool]: + async def handle_list_tools() -> List[Tool]: """ - List all available tools + List all available tools. + Also captures the active session for propagation to callbacks. """ + from mcp.server.lowlevel.server import request_ctx + + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) + try: # Get user authentication from context variable ( @@ -497,7 +573,7 @@ if MCP_AVAILABLE: oauth2_headers, raw_headers, _client_ip, - ) = get_auth_context() + ) = await get_or_extract_auth_context() verbose_logger.debug( f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}" ) @@ -528,152 +604,188 @@ if MCP_AVAILABLE: # Return empty list instead of failing completely # This prevents the HTTP stream from failing and allows the client to get a response return [] + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) @server.call_tool() - async def mcp_server_tool_call( - name: str, arguments: Optional[Dict[str, Any]] + async def mcp_server_tool_call( # noqa: PLR0915 + name: str, arguments: Dict[str, Any] | None ) -> CallToolResult: """ Call a specific tool with the provided arguments - Args: name (str): Name of the tool to call arguments (Dict[str, Any] | None): Arguments to pass to the tool - Returns: List[Union[MCPTextContent, MCPImageContent, MCPEmbeddedResource]]: Tool execution results - Raises: HTTPException: If tool not found or arguments missing """ from fastapi import Request - from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.proxy_server import proxy_config + from mcp.types import CallToolResult + from mcp.server.lowlevel.server import request_ctx - # Validate arguments - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = get_auth_context() + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) - verbose_logger.debug( - f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" - ) - host_progress_callback = None try: - host_ctx = server.request_context - if host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta: - host_token = getattr(host_ctx.meta, "progressToken", None) - if host_token and hasattr(host_ctx, "session") and host_ctx.session: - host_session = host_ctx.session - - async def forward_progress(progress: float, total: Optional[float]): - """Forward progress notifications from external MCP to Host""" - try: - await host_session.send_progress_notification( - progress_token=host_token, - progress=progress, - total=total, - ) - verbose_logger.debug( - f"Forwarded progress {progress}/{total} to Host" - ) - except Exception as e: - verbose_logger.error( - f"Failed to forward progress to Host: {e}" - ) - - host_progress_callback = forward_progress - verbose_logger.debug( - f"Host progressToken captured: {host_token[:8]}..." - ) - except Exception as e: - verbose_logger.warning(f"Could not capture host progress context: {e}") - try: - # Create a body date for logging - body_data = {"name": name, "arguments": arguments} - # Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A) - chain_id = get_chain_id_from_headers(raw_headers) - if chain_id: - body_data["litellm_trace_id"] = chain_id - body_data["litellm_session_id"] = chain_id - - request = Request( - scope={ - "type": "http", - "method": "POST", - "path": "/mcp/tools/call", - "headers": [(b"content-type", b"application/json")], - } + # Validate arguments + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = await get_or_extract_auth_context() + verbose_logger.debug( + f"MCP mcp_server_tool_call - user_api_key_auth={user_api_key_auth}, user_role={getattr(user_api_key_auth, 'user_role', 'N/A')}" ) - if user_api_key_auth is not None: - data = await add_litellm_data_to_request( - data=body_data, - request=request, - user_api_key_dict=user_api_key_auth, - proxy_config=proxy_config, + + verbose_logger.debug( + f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" + ) + host_progress_callback = None + try: + host_ctx = server.request_context + if host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta: + host_token = getattr(host_ctx.meta, "progressToken", None) + if host_token and hasattr(host_ctx, "session") and host_ctx.session: + host_session = host_ctx.session + + async def forward_progress( + progress: float, total: Optional[float] + ): + """Forward progress notifications from external MCP to Host""" + try: + await host_session.send_progress_notification( + progress_token=host_token, + progress=progress, + total=total, + ) + verbose_logger.debug( + f"Forwarded progress {progress}/{total} to Host" + ) + except Exception as e: + verbose_logger.error( + f"Failed to forward progress to Host: {e}" + ) + + host_progress_callback = forward_progress + verbose_logger.debug( + f"Host progressToken captured: {host_token[:8]}..." + ) + except Exception as e: + verbose_logger.warning(f"Could not capture host progress context: {e}") + try: + # Create a body date for logging + body_data = {"name": name, "arguments": arguments} + # Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A) + chain_id = get_chain_id_from_headers(raw_headers) + if chain_id: + body_data["litellm_trace_id"] = chain_id + body_data["litellm_session_id"] = chain_id + + request = Request( + scope={ + "type": "http", + "method": "POST", + "path": "/mcp/tools/call", + "headers": [(b"content-type", b"application/json")], + } ) - else: - data = body_data - - response = await call_mcp_tool( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - host_progress_callback=host_progress_callback, - **data, # for logging - ) - except BlockedPiiEntityError as e: - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {str(e)}") - return CallToolResult( - content=[ - TextContent( - text=f"Error: Blocked PII entity detected - {str(e)}", - type="text", + if user_api_key_auth is not None: + data = await add_litellm_data_to_request( + data=body_data, + request=request, + user_api_key_dict=user_api_key_auth, + proxy_config=proxy_config, ) - ], - isError=True, - ) - except GuardrailRaisedException as e: - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {str(e)}") - return CallToolResult( - content=[ - TextContent( - text=f"Error: Guardrail violation - {str(e)}", type="text" - ) - ], - isError=True, - ) - except HTTPException as e: - verbose_logger.error(f"HTTPException in MCP tool call: {str(e)}") - return CallToolResult( - content=[TextContent(text=f"Error: {str(e.detail)}", type="text")], - isError=True, - ) - except Exception as e: - verbose_logger.exception(f"MCP mcp_server_tool_call - error: {e}") - return CallToolResult( - content=[TextContent(text=f"Error: {str(e)}", type="text")], - isError=True, - ) + else: + data = body_data - return response + response = await call_mcp_tool( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + host_progress_callback=host_progress_callback, + **data, # for logging + ) + except MCPMissingUserEnvVarsError as e: + verbose_logger.info( + "MCP mcp_server_tool_call missing per-user env vars: server_id=%s missing=%s", + e.server_id, + e.missing, + ) + return CallToolResult( + content=[TextContent(text=str(e), type="text")], + isError=True, + ) + except BlockedPiiEntityError as e: + verbose_logger.error( + f"BlockedPiiEntityError in MCP tool call: {str(e)}" + ) + return CallToolResult( + content=[ + TextContent( + text=f"Error: Blocked PII entity detected - {str(e)}", + type="text", + ) + ], + isError=True, + ) + except GuardrailRaisedException as e: + verbose_logger.error( + f"GuardrailRaisedException in MCP tool call: {str(e)}" + ) + return CallToolResult( + content=[ + TextContent( + text=f"Error: Guardrail violation - {str(e)}", type="text" + ) + ], + isError=True, + ) + except HTTPException as e: + verbose_logger.error(f"HTTPException in MCP tool call: {str(e)}") + return CallToolResult( + content=[TextContent(text=f"Error: {str(e.detail)}", type="text")], + isError=True, + ) + except Exception as e: + verbose_logger.exception(f"MCP mcp_server_tool_call - error: {e}") + return CallToolResult( + content=[TextContent(text=f"Error: {str(e)}", type="text")], + isError=True, + ) + + return response + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) @server.list_prompts() async def list_prompts() -> List[Prompt]: """ List all available prompts """ + from mcp.server.lowlevel.server import request_ctx + + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) + try: # Get user authentication from context variable ( @@ -684,7 +796,7 @@ if MCP_AVAILABLE: oauth2_headers, raw_headers, _client_ip, - ) = get_auth_context() + ) = await get_or_extract_auth_context() verbose_logger.debug( f"MCP list_prompts - User API Key Auth from context: {user_api_key_auth}" ) @@ -713,6 +825,9 @@ if MCP_AVAILABLE: # Return empty list instead of failing completely # This prevents the HTTP stream from failing and allows the client to get a response return [] + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) @server.get_prompt() async def get_prompt( @@ -730,33 +845,13 @@ if MCP_AVAILABLE: """ # Validate arguments - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = get_auth_context() + from mcp.server.lowlevel.server import request_ctx - verbose_logger.debug( - f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" - ) - return await mcp_get_prompt( - name=name, - arguments=arguments, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) - @server.list_resources() - async def list_resources() -> List[Resource]: - """List all available resources.""" try: ( user_api_key_auth, @@ -766,7 +861,45 @@ if MCP_AVAILABLE: oauth2_headers, raw_headers, _client_ip, - ) = get_auth_context() + ) = await get_or_extract_auth_context() + + verbose_logger.debug( + f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" + ) + return await mcp_get_prompt( + name=name, + arguments=arguments, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) + + @server.list_resources() + async def list_resources() -> List[Resource]: + """List all available resources.""" + from mcp.server.lowlevel.server import request_ctx + + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) + + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = await get_or_extract_auth_context() verbose_logger.debug( f"MCP list_resources - User API Key Auth from context: {user_api_key_auth}" ) @@ -792,10 +925,20 @@ if MCP_AVAILABLE: except Exception as e: verbose_logger.exception(f"Error in list_resources endpoint: {str(e)}") return [] + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) @server.list_resource_templates() async def list_resource_templates() -> List[ResourceTemplate]: """List all available resource templates.""" + from mcp.server.lowlevel.server import request_ctx + + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) + try: ( user_api_key_auth, @@ -805,7 +948,7 @@ if MCP_AVAILABLE: oauth2_headers, raw_headers, _client_ip, - ) = get_auth_context() + ) = await get_or_extract_auth_context() verbose_logger.debug( f"MCP list_resource_templates - User API Key Auth from context: {user_api_key_auth}" ) @@ -825,8 +968,7 @@ if MCP_AVAILABLE: raw_headers=raw_headers, ) verbose_logger.info( - "MCP list_resource_templates - Successfully returned " - f"{len(resource_templates)} resource templates" + f"MCP list_resource_templates - Successfully returned {len(resource_templates)} resource templates" ) return resource_templates except Exception as e: @@ -834,30 +976,44 @@ if MCP_AVAILABLE: f"Error in list_resource_templates endpoint: {str(e)}" ) return [] + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) @server.read_resource() async def read_resource(url: AnyUrl) -> list[ReadResourceContents]: - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = get_auth_context() + from mcp.server.lowlevel.server import request_ctx - read_resource_result = await mcp_read_resource( - url=url, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) - return _normalize_resource_contents(read_resource_result.contents) + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = await get_or_extract_auth_context() + + read_resource_result = await mcp_read_resource( + url=url, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + return _normalize_resource_contents(read_resource_result.contents) + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) ######################################################## ############ End of MCP Server Routes ################## @@ -1166,8 +1322,7 @@ if MCP_AVAILABLE: try: from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 get_user_oauth_credential, - is_oauth_credential_expired, - refresh_user_oauth_token, + resolve_valid_user_oauth_token, ) from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( # noqa: PLC0415 _compute_per_user_token_ttl, @@ -1180,14 +1335,14 @@ if MCP_AVAILABLE: cached_token = await mcp_per_user_token_cache.get(user_id, server_id) if cached_token is not None: verbose_logger.debug( - "_get_user_oauth_extra_headers_from_db: Redis hit for " - "user=%s server=%s", + "_get_user_oauth_extra_headers_from_db: Redis hit for user=%s server=%s", user_id, server_id, ) return {"Authorization": f"Bearer {cached_token}"} # ── Slow path: DB lookup ────────────────────────────────────────── + prisma_client = None if prefetched_creds is not None: cred = prefetched_creds.get(server_id) else: @@ -1205,45 +1360,17 @@ if MCP_AVAILABLE: if not cred or not cred.get("access_token"): return None - if is_oauth_credential_expired(cred): - verbose_logger.debug( - "_get_user_oauth_extra_headers_from_db: token expired for " - "user=%s server=%s — attempting refresh", - user_id, - server_id, - ) - # Attempt token refresh; requires a DB client (not available from prefetch) - if cred.get("refresh_token"): - try: - from litellm.proxy.utils import ( # noqa: PLC0415 - get_prisma_client_or_throw, - ) - - prisma_client = get_prisma_client_or_throw( - "Database not connected. Cannot refresh OAuth token." - ) - cred = await refresh_user_oauth_token( - prisma_client=prisma_client, - user_id=user_id, - server=server, - cred=cred, - ) - except Exception as refresh_exc: - verbose_logger.warning( - "_get_user_oauth_extra_headers_from_db: refresh failed " - "for user=%s server=%s: %s", - user_id, - server_id, - refresh_exc, - ) - cred = None - - if not cred or not cred.get("access_token"): - # Clear stale Redis/cache entry so we don't serve it again. - # Do this for both the individual and prefetch paths so the - # next request doesn't get a stale cache hit. - await mcp_per_user_token_cache.delete(user_id, server_id) - return None + cred = await resolve_valid_user_oauth_token( + user_id=user_id, + server=server, + cred=cred, + prisma_client=prisma_client, + ) + if cred is None: + # Refresh failed or token expired with no usable refresh_token — + # clear the stale Redis entry so the next request doesn't reuse it. + await mcp_per_user_token_cache.delete(user_id, server_id) + return None access_token: str = cred["access_token"] @@ -1275,8 +1402,7 @@ if MCP_AVAILABLE: return {"Authorization": f"Bearer {access_token}"} except Exception as e: verbose_logger.warning( - "_get_user_oauth_extra_headers_from_db: failed to retrieve credential for " - "user=%s server=%s: %s", + "_get_user_oauth_extra_headers_from_db: failed to retrieve credential for user=%s server=%s: %s", user_id, server_id, e, @@ -2485,7 +2611,7 @@ if MCP_AVAILABLE: arguments=arguments or {}, server_name=server_name or mcp_server.name, user_api_key_auth=user_api_key_auth, - proxy_logging_obj=proxy_logging_obj, + proxy_logging_obj=proxy_logging_obj, # type: ignore[arg-type] server=mcp_server, raw_headers=raw_headers, ) @@ -2744,8 +2870,7 @@ if MCP_AVAILABLE: raise HTTPException( status_code=400, detail=( - "Multiple MCP servers configured; read_resource currently " - "supports exactly one allowed server." + "Multiple MCP servers configured; read_resource currently supports exactly one allowed server." ), ) @@ -3124,8 +3249,7 @@ if MCP_AVAILABLE: return False except Exception: verbose_logger.debug( - "Unable to inspect active MCP sessions for '%s'. " - "Deferring to session manager.", + "Unable to inspect active MCP sessions for '%s'. Deferring to session manager.", _session_id, ) return False @@ -3136,8 +3260,7 @@ if MCP_AVAILABLE: if method == "DELETE": _remove_stateful_session_tracking(_session_id) verbose_logger.info( - "DELETE request for non-existent MCP session '%s'. " - "Returning success (idempotent DELETE).", + "DELETE request for non-existent MCP session '%s'. Returning success (idempotent DELETE).", _session_id, ) success_response = JSONResponse( @@ -3615,6 +3738,7 @@ if MCP_AVAILABLE: return session_id = _get_session_id_from_scope(scope) + body = b"" if scope.get("method") == "POST": consumed_messages, body = await _read_request_body_for_routing(receive) is_initialize = _is_initialize_request(body) @@ -3639,8 +3763,7 @@ if MCP_AVAILABLE: ) if not await _enforce_stateful_session_cap_for_owner(request_owner): verbose_logger.warning( - "Rejecting MCP initialize: caller already holds the maximum " - "number of active stateful sessions." + "Rejecting MCP initialize: caller already holds the maximum number of active stateful sessions." ) too_many_response = JSONResponse( status_code=429, @@ -3672,9 +3795,56 @@ if MCP_AVAILABLE: # POST/DELETE are the methods that actually mutate the shared # auth context, so serializing those is sufficient for the # clobbering race between concurrent JSON-RPC calls. - session_lock: Optional[asyncio.Lock] = None + # + # Also skip the lock for JSON-RPC *responses* (POSTs that carry + # a ``result`` or ``error`` but no ``method``). These are replies + # to server-initiated requests such as ``elicitation/create`` or + # ``sampling/createMessage``. The in-flight tool-call POST that + # triggered the server request already holds the session lock, so + # trying to acquire it again for the response POST would deadlock. + is_jsonrpc_response = False request_method = (scope.get("method") or "").upper() - if use_stateful and session_id and request_method in ("POST", "DELETE"): + if body and request_method == "POST": + try: + _peeked = json.loads(body) + if ( + isinstance(_peeked, dict) + and _peeked.get("jsonrpc") == "2.0" + and "id" in _peeked + and "method" not in _peeked + and ("result" in _peeked or "error" in _peeked) + ): + is_jsonrpc_response = True + verbose_logger.debug( + "MCP: detected JSON-RPC response POST (id=%s), skipping session lock to avoid deadlock", + _peeked.get("id"), + ) + except (json.JSONDecodeError, TypeError): + # Peek cap truncated the body, so it can't be fully parsed. + # Scan the top-level keys (depth-aware) instead of a flat + # substring search: a response's result payload may nest a + # "method" field, and misreading that would acquire the lock + # and deadlock the in-flight tool call awaiting this + # response. A false skip is harmless; a false acquire is not. + _body_str = body.decode("utf-8", errors="replace") + if ( + '"jsonrpc"' in _body_str + and ('"result"' in _body_str or '"error"' in _body_str) + and not _jsonrpc_text_has_top_level_method(_body_str) + ): + is_jsonrpc_response = True + verbose_logger.debug( + "MCP: detected truncated JSON-RPC response POST via " + "top-level key scan, skipping session lock to avoid deadlock" + ) + + session_lock: Optional[asyncio.Lock] = None + if ( + use_stateful + and session_id + and request_method in ("POST", "DELETE") + and not is_jsonrpc_response + ): session_lock = _stateful_session_locks.setdefault( session_id, asyncio.Lock() ) @@ -4099,6 +4269,119 @@ if MCP_AVAILABLE: ) return None, None, None, None, None, None, None + def _get_current_session(): + try: + from mcp.server.lowlevel.server import request_ctx + + return request_ctx.get().session + except (LookupError, ImportError): + return None + + def _cache_auth_context_lazily(): + session = _get_current_session() + if session is None: + return + try: + if session in _session_obj_auth_storage: + return + except TypeError: + verbose_logger.debug( + "_cache_auth_context_lazily: session object is unhashable (type=%s), cannot cache auth context", + type(session).__name__, + ) + return + + auth = auth_context_var.get() + if auth and isinstance(auth, MCPAuthenticatedUser): + try: + _session_obj_auth_storage[session] = auth + except TypeError: + verbose_logger.debug( + "_cache_auth_context_lazily: could not store auth via " + "session identity — session object is unhashable" + ) + + def _recover_auth_from_session() -> Optional[MCPAuthenticatedUser]: + session = _get_current_session() + if session is None: + return None + + stored: Optional[MCPAuthenticatedUser] = None + try: + stored = _session_obj_auth_storage.get(session) + except TypeError: + verbose_logger.debug( + "_recover_auth_from_session: session object is unhashable " + "(type=%s), skipping _session_obj_auth_storage lookup", + type(session).__name__, + ) + + return stored + + async def get_or_extract_auth_context() -> Tuple[ + Optional[UserAPIKeyAuth], + Optional[str], + Optional[List[str]], + Optional[Dict[str, Dict[str, str]]], + Optional[Dict[str, str]], + Optional[Dict[str, str]], + Optional[str], + ]: + """ + Get auth context from ContextVar first, then fall back to session + storage (which survives cross-task boundaries in the MCP SDK). + """ + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = get_auth_context() + + if user_api_key_auth is not None: + _cache_auth_context_lazily() + else: + stored = _recover_auth_from_session() + + if stored: + user_api_key_auth = stored.user_api_key_auth + mcp_auth_header = stored.mcp_auth_header + mcp_servers = stored.mcp_servers + mcp_server_auth_headers = stored.mcp_server_auth_headers + oauth2_headers = stored.oauth2_headers + raw_headers = stored.raw_headers + _client_ip = stored.client_ip + return ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) + + def get_active_mcp_session() -> Optional[_McpServerSession]: + """Return the active MCP session captured during handler execution.""" + session = active_mcp_session_var.get() + if session is not None: + return session + return _get_current_session() + + def get_active_auth_context() -> Optional[MCPAuthenticatedUser]: + """Return auth context from ContextVar or session storage.""" + auth = auth_context_var.get() + if auth and isinstance(auth, MCPAuthenticatedUser): + return auth + + stored = _recover_auth_from_session() + if stored is not None: + return stored + return None + ######################################################## ############ End of Auth Context Functions ############# ######################################################## diff --git a/litellm/proxy/_experimental/mcp_server/toolset_db.py b/litellm/proxy/_experimental/mcp_server/toolset_db.py index 08ac7dbd33b..a996131653f 100644 --- a/litellm/proxy/_experimental/mcp_server/toolset_db.py +++ b/litellm/proxy/_experimental/mcp_server/toolset_db.py @@ -4,6 +4,7 @@ from typing import List, Optional from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.proxy.utils import PrismaClient +from litellm.repositories.table_repositories import MCPToolsetRepository from litellm.types.mcp_server.mcp_toolset import ( MCPToolset, NewMCPToolsetRequest, @@ -30,7 +31,7 @@ async def create_mcp_toolset( data_dict["tools"] = json.dumps(data_dict.get("tools", [])) data_dict["created_by"] = touched_by data_dict["updated_by"] = touched_by - row = await prisma_client.db.litellm_mcptoolsettable.create(data=data_dict) + row = await MCPToolsetRepository(prisma_client).table.create(data=data_dict) return _toolset_from_row(row) @@ -38,7 +39,7 @@ async def get_mcp_toolset( prisma_client: PrismaClient, toolset_id: str, ) -> Optional[MCPToolset]: - row = await prisma_client.db.litellm_mcptoolsettable.find_unique( + row = await MCPToolsetRepository(prisma_client).table.find_unique( where={"toolset_id": toolset_id} ) if row is None: @@ -54,7 +55,7 @@ async def list_mcp_toolsets( where = {} if toolset_ids is not None: where = {"toolset_id": {"in": toolset_ids}} - rows = await prisma_client.db.litellm_mcptoolsettable.find_many(where=where) + rows = await MCPToolsetRepository(prisma_client).table.find_many(where=where) return [_toolset_from_row(r) for r in rows] except Exception as e: verbose_proxy_logger.warning( @@ -69,7 +70,7 @@ async def get_mcp_toolset_by_name( prisma_client: PrismaClient, toolset_name: str, ) -> Optional[MCPToolset]: - row = await prisma_client.db.litellm_mcptoolsettable.find_first( + row = await MCPToolsetRepository(prisma_client).table.find_first( where={"toolset_name": toolset_name} ) if row is None: @@ -87,7 +88,7 @@ async def update_mcp_toolset( data_dict["tools"] = json.dumps(data_dict["tools"]) data_dict["updated_by"] = touched_by try: - row = await prisma_client.db.litellm_mcptoolsettable.update( + row = await MCPToolsetRepository(prisma_client).table.update( where={"toolset_id": data.toolset_id}, data=data_dict, ) @@ -105,7 +106,7 @@ async def delete_mcp_toolset( toolset_id: str, ) -> Optional[MCPToolset]: try: - row = await prisma_client.db.litellm_mcptoolsettable.delete( + row = await MCPToolsetRepository(prisma_client).table.delete( where={"toolset_id": toolset_id} ) except Exception as e: diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index b66dfa85b9c..97cfa74ea45 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -4,11 +4,23 @@ MCP Server Utilities import json import re -from typing import Any, Dict, Iterator, Mapping, Optional, Tuple, Union +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Set, + Tuple, + Union, +) import hashlib import importlib import os +from urllib.parse import quote # Constants LITELLM_MCP_SERVER_NAME = "litellm-mcp-server" @@ -370,6 +382,130 @@ def validate_mcp_server_name( raise Exception(error_message) +class MCPMissingUserEnvVarsError(Exception): + """Raised when an MCP request can't be built because the calling user has + not supplied one or more required per-user environment variables. + + The error message is user-facing and includes a URL the user can visit + to fill them in. + """ + + def __init__( + self, + *, + server_id: str, + server_name: Optional[str], + missing: List[str], + setup_url: str, + ) -> None: + self.server_id = server_id + self.server_name = server_name + self.missing = missing + self.setup_url = setup_url + label = server_name or server_id + bullet_list = "\n".join(f"- {name}" for name in missing) + message = ( + f'Cannot connect to MCP server "{label}".\n\n' + f"Your administrator configured this server to require per-user " + f"variables, but you haven't set the following yet:\n" + f"{bullet_list}\n\n" + f"Set your credentials here:\n" + f"{setup_url}" + ) + super().__init__(message) + + +# Pattern for ``${NAME}`` substitution. Matches the standard env-var +# identifier rules — letters, digits, underscores, can't start with a digit. +_ENV_VAR_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") + + +def parse_admin_env_vars( + env_vars: Optional[Iterable[Any]], +) -> Tuple[Dict[str, str], List[Dict[str, Any]]]: + """Split admin-configured env var entries into globals and per-user specs. + + Accepts the raw value of ``MCPServer.env_vars`` (list of dicts or Pydantic + models). Returns: + + - ``global_values``: ``{name: value}`` for entries with ``scope=="global"``. + - ``user_specs``: list of ``{name, description}`` for entries with + ``scope=="user"`` — these are the names the user must fill in. + + Unknown / malformed entries are skipped silently. + """ + global_values: Dict[str, str] = {} + user_specs: List[Dict[str, Any]] = [] + if not env_vars: + return global_values, user_specs + for raw in env_vars: + if raw is None: + continue + if hasattr(raw, "model_dump"): + entry = raw.model_dump() + elif isinstance(raw, dict): + entry = raw + else: + continue + name = entry.get("name") + if not isinstance(name, str) or not name: + continue + scope = entry.get("scope") or "global" + if scope == "user": + user_specs.append({"name": name, "description": entry.get("description")}) + else: + value = entry.get("value") + global_values[name] = "" if value is None else str(value) + return global_values, user_specs + + +def find_env_var_references(value: str) -> Set[str]: + """Return the set of ``${NAME}`` identifiers referenced inside ``value``.""" + if not value: + return set() + return set(_ENV_VAR_PATTERN.findall(value)) + + +def collect_env_var_references(*, strings: Iterable[str]) -> Set[str]: + """Union of every ``${NAME}`` reference across a collection of strings.""" + refs: Set[str] = set() + for s in strings: + if isinstance(s, str): + refs |= find_env_var_references(s) + return refs + + +def interpolate_env_vars(value: str, variables: Mapping[str, str]) -> str: + """Replace ``${NAME}`` references in ``value`` with the matching mapping + entry. Unknown names are left untouched so callers can detect them via + ``find_env_var_references`` on the result if needed. + """ + if not value: + return value + + def _sub(match: "re.Match[str]") -> str: + name = match.group(1) + if name in variables: + return variables[name] + return match.group(0) + + return _ENV_VAR_PATTERN.sub(_sub, value) + + +def interpolate_headers( + headers: Mapping[str, str], variables: Mapping[str, str] +) -> Dict[str, str]: + """Return a copy of ``headers`` with every value passed through ``interpolate_env_vars``.""" + return {k: interpolate_env_vars(v, variables) for k, v in headers.items()} + + +def build_env_var_setup_url(server_id: str) -> str: + """The frontend URL where a user can fill in their per-user env vars.""" + base = os.environ.get("PROXY_BASE_URL", "").rstrip("/") + path = f"/ui/?page=mcp-servers&fill_env_vars={quote(server_id, safe='')}" + return f"{base}{path}" if base else path + + def merge_mcp_headers( *, extra_headers: Optional[Mapping[str, str]] = None, diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html index f27612ff54e..45de348c4d5 100644 --- a/litellm/proxy/_experimental/out/404.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/404/index.html b/litellm/proxy/_experimental/out/404/index.html index f27612ff54e..45de348c4d5 100644 --- a/litellm/proxy/_experimental/out/404/index.html +++ b/litellm/proxy/_experimental/out/404/index.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/__next.__PAGE__.txt b/litellm/proxy/_experimental/out/__next.__PAGE__.txt index c024136e8dc..095c8f4339f 100644 --- a/litellm/proxy/_experimental/out/__next.__PAGE__.txt +++ b/litellm/proxy/_experimental/out/__next.__PAGE__.txt @@ -1,10 +1,10 @@ 1:"$Sreact.fragment" 2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -3:I[952683,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js"],"default"] +3:I[952683,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js","/litellm-asset-prefix/_next/static/chunks/bee4095c26818f05.js","/litellm-asset-prefix/_next/static/chunks/81937424fe90f746.js","/litellm-asset-prefix/_next/static/chunks/e2257d8308d35cf4.js","/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","/litellm-asset-prefix/_next/static/chunks/eb1ba04e211a533f.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/4cb93eefa53f21a3.js","/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","/litellm-asset-prefix/_next/static/chunks/40a2744137b1aec2.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/84a27349dda457cd.js","/litellm-asset-prefix/_next/static/chunks/8ddf82e7e0b331fc.js","/litellm-asset-prefix/_next/static/chunks/1d7b3500478e93ae.js","/litellm-asset-prefix/_next/static/chunks/f0e079183e7bb90c.js","/litellm-asset-prefix/_next/static/chunks/10757c2146f43db4.js","/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","/litellm-asset-prefix/_next/static/chunks/ffa46de7b8384155.js","/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","/litellm-asset-prefix/_next/static/chunks/80f4410629229bf9.js","/litellm-asset-prefix/_next/static/chunks/75ee9aba04c74e23.js","/litellm-asset-prefix/_next/static/chunks/193886179a5779b5.js","/litellm-asset-prefix/_next/static/chunks/2063ca6435a47940.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","/litellm-asset-prefix/_next/static/chunks/d7c18aec4a87a237.js","/litellm-asset-prefix/_next/static/chunks/dac86522fa98e760.js"],"default"] 6:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] 7:"$Sreact.suspense" :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false} +0:{"buildId":"LpqGBJeKQM0vUG-9uVaiY","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/bee4095c26818f05.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/81937424fe90f746.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/e2257d8308d35cf4.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/eb1ba04e211a533f.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/4cb93eefa53f21a3.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/40a2744137b1aec2.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/84a27349dda457cd.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/8ddf82e7e0b331fc.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/1d7b3500478e93ae.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/f0e079183e7bb90c.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/10757c2146f43db4.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ffa46de7b8384155.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/80f4410629229bf9.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/75ee9aba04c74e23.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/193886179a5779b5.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/2063ca6435a47940.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/d7c18aec4a87a237.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/dac86522fa98e760.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false} 4:{} 5:"$0:rsc:props:children:0:props:serverProvidedParams:params" 8:null diff --git a/litellm/proxy/_experimental/out/__next._full.txt b/litellm/proxy/_experimental/out/__next._full.txt index 0c119086b9e..2b2b3850207 100644 --- a/litellm/proxy/_experimental/out/__next._full.txt +++ b/litellm/proxy/_experimental/out/__next._full.txt @@ -1,34 +1,34 @@ 1:"$Sreact.fragment" -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] -4:I[557951,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"AuthProvider"] +2:I[867271,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"default"] +3:I[71195,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"default"] +4:I[557951,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"AuthProvider"] 5:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] 6:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] 7:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -8:I[952683,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js"],"default"] +8:I[952683,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js","/litellm-asset-prefix/_next/static/chunks/bee4095c26818f05.js","/litellm-asset-prefix/_next/static/chunks/81937424fe90f746.js","/litellm-asset-prefix/_next/static/chunks/e2257d8308d35cf4.js","/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","/litellm-asset-prefix/_next/static/chunks/eb1ba04e211a533f.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/4cb93eefa53f21a3.js","/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","/litellm-asset-prefix/_next/static/chunks/40a2744137b1aec2.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/84a27349dda457cd.js","/litellm-asset-prefix/_next/static/chunks/8ddf82e7e0b331fc.js","/litellm-asset-prefix/_next/static/chunks/1d7b3500478e93ae.js","/litellm-asset-prefix/_next/static/chunks/f0e079183e7bb90c.js","/litellm-asset-prefix/_next/static/chunks/10757c2146f43db4.js","/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","/litellm-asset-prefix/_next/static/chunks/ffa46de7b8384155.js","/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","/litellm-asset-prefix/_next/static/chunks/80f4410629229bf9.js","/litellm-asset-prefix/_next/static/chunks/75ee9aba04c74e23.js","/litellm-asset-prefix/_next/static/chunks/193886179a5779b5.js","/litellm-asset-prefix/_next/static/chunks/2063ca6435a47940.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","/litellm-asset-prefix/_next/static/chunks/d7c18aec4a87a237.js","/litellm-asset-prefix/_next/static/chunks/dac86522fa98e760.js"],"default"] 1a:I[168027,[],"default"] :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/47150bfa067220d3.css","style"] :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"P":null,"b":"TrcGiQpTupSbDYFFfkFHY","c":["",""],"q":"","i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[["$","$L7",null,{"Component":"$8","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@9","$@a"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","async":true,"nonce":"$undefined"}],"$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17"],"$L18"]}],{},null,false,false]},null,false,false],"$L19",false]],"m":"$undefined","G":["$1a",[]],"S":true} +0:{"P":null,"b":"LpqGBJeKQM0vUG-9uVaiY","c":["",""],"q":"","i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/47150bfa067220d3.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[["$","$L7",null,{"Component":"$8","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@9","$@a"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/bee4095c26818f05.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/81937424fe90f746.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/e2257d8308d35cf4.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/eb1ba04e211a533f.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/4cb93eefa53f21a3.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/40a2744137b1aec2.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/84a27349dda457cd.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/8ddf82e7e0b331fc.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/1d7b3500478e93ae.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/f0e079183e7bb90c.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/10757c2146f43db4.js","async":true,"nonce":"$undefined"}],"$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17"],"$L18"]}],{},null,false,false]},null,false,false],"$L19",false]],"m":"$undefined","G":["$1a",[]],"S":true} 1b:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] 1c:"$Sreact.suspense" 1e:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"] 20:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] b:["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","async":true,"nonce":"$undefined"}] -c:["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","async":true,"nonce":"$undefined"}] +c:["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ffa46de7b8384155.js","async":true,"nonce":"$undefined"}] d:["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","async":true,"nonce":"$undefined"}] -e:["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","async":true,"nonce":"$undefined"}] -f:["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","async":true,"nonce":"$undefined"}] -10:["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","async":true,"nonce":"$undefined"}] -11:["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","async":true,"nonce":"$undefined"}] +e:["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/80f4410629229bf9.js","async":true,"nonce":"$undefined"}] +f:["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/75ee9aba04c74e23.js","async":true,"nonce":"$undefined"}] +10:["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/193886179a5779b5.js","async":true,"nonce":"$undefined"}] +11:["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/2063ca6435a47940.js","async":true,"nonce":"$undefined"}] 12:["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true,"nonce":"$undefined"}] 13:["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","async":true,"nonce":"$undefined"}] 14:["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","async":true,"nonce":"$undefined"}] 15:["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","async":true,"nonce":"$undefined"}] -16:["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true,"nonce":"$undefined"}] -17:["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js","async":true,"nonce":"$undefined"}] +16:["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/d7c18aec4a87a237.js","async":true,"nonce":"$undefined"}] +17:["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/dac86522fa98e760.js","async":true,"nonce":"$undefined"}] 18:["$","$L1b",null,{"children":["$","$1c",null,{"name":"Next.MetadataOutlet","children":"$@1d"}]}] 19:["$","$1","h",{"children":[null,["$","$L1e",null,{"children":"$L1f"}],["$","div",null,{"hidden":true,"children":["$","$L20",null,{"children":["$","$1c",null,{"name":"Next.Metadata","children":"$L21"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}] 9:{} @@ -36,4 +36,4 @@ a:"$0:f:0:1:1:children:0:props:children:0:props:serverProvidedParams:params" 1f:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]] 22:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"] 1d:null -21:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L22","4",{}]] +21:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L22","4",{}]] diff --git a/litellm/proxy/_experimental/out/__next._head.txt b/litellm/proxy/_experimental/out/__next._head.txt index ea6e5095458..870c89c7e11 100644 --- a/litellm/proxy/_experimental/out/__next._head.txt +++ b/litellm/proxy/_experimental/out/__next._head.txt @@ -3,4 +3,4 @@ 3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] 4:"$Sreact.suspense" 5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"] -0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false} +0:{"buildId":"LpqGBJeKQM0vUG-9uVaiY","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_experimental/out/__next._index.txt b/litellm/proxy/_experimental/out/__next._index.txt index ebf6d8fec08..67c452e8c21 100644 --- a/litellm/proxy/_experimental/out/__next._index.txt +++ b/litellm/proxy/_experimental/out/__next._index.txt @@ -1,9 +1,9 @@ 1:"$Sreact.fragment" -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] -4:I[557951,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"AuthProvider"] +2:I[867271,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"default"] +3:I[71195,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"default"] +4:I[557951,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"AuthProvider"] 5:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] 6:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","style"] -0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"loading":null,"isPartial":false} +:HL["/litellm-asset-prefix/_next/static/chunks/47150bfa067220d3.css","style"] +0:{"buildId":"LpqGBJeKQM0vUG-9uVaiY","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/47150bfa067220d3.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_experimental/out/__next._tree.txt b/litellm/proxy/_experimental/out/__next._tree.txt index 4a08f4f9e11..86dc121c5f9 100644 --- a/litellm/proxy/_experimental/out/__next._tree.txt +++ b/litellm/proxy/_experimental/out/__next._tree.txt @@ -1,5 +1,5 @@ :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/47150bfa067220d3.css","style"] :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300} +0:{"buildId":"LpqGBJeKQM0vUG-9uVaiY","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300} diff --git a/litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_clientMiddlewareManifest.json b/litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_clientMiddlewareManifest.json similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_clientMiddlewareManifest.json rename to litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_clientMiddlewareManifest.json diff --git a/litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/adfb3758f3e2c464.js b/litellm/proxy/_experimental/out/_next/static/chunks/10376d0955336027.js similarity index 98% rename from litellm/proxy/_experimental/out/_next/static/chunks/adfb3758f3e2c464.js rename to litellm/proxy/_experimental/out/_next/static/chunks/10376d0955336027.js index 825fe327e97..55ce00c27b0 100644 --- a/litellm/proxy/_experimental/out/_next/static/chunks/adfb3758f3e2c464.js +++ b/litellm/proxy/_experimental/out/_next/static/chunks/10376d0955336027.js @@ -1,4 +1,4 @@ -(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,295320,e=>{"use strict";e.i(247167);var t=e.i(931067),i=e.i(271645);let r={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M704 446H320c-4.4 0-8 3.6-8 8v402c0 4.4 3.6 8 8 8h384c4.4 0 8-3.6 8-8V454c0-4.4-3.6-8-8-8zm-328 64h272v117H376V510zm272 290H376V683h272v117z"}},{tag:"path",attrs:{d:"M424 748a32 32 0 1064 0 32 32 0 10-64 0zm0-178a32 32 0 1064 0 32 32 0 10-64 0z"}},{tag:"path",attrs:{d:"M811.4 368.9C765.6 248 648.9 162 512.2 162S258.8 247.9 213 368.8C126.9 391.5 63.5 470.2 64 563.6 64.6 668 145.6 752.9 247.6 762c4.7.4 8.7-3.3 8.7-8v-60.4c0-4-3-7.4-7-7.9-27-3.4-52.5-15.2-72.1-34.5-24-23.5-37.2-55.1-37.2-88.6 0-28 9.1-54.4 26.2-76.4 16.7-21.4 40.2-36.9 66.1-43.7l37.9-10 13.9-36.7c8.6-22.8 20.6-44.2 35.7-63.5 14.9-19.2 32.6-36 52.4-50 41.1-28.9 89.5-44.2 140-44.2s98.9 15.3 140 44.3c19.9 14 37.5 30.8 52.4 50 15.1 19.3 27.1 40.7 35.7 63.5l13.8 36.6 37.8 10c54.2 14.4 92.1 63.7 92.1 120 0 33.6-13.2 65.1-37.2 88.6-19.5 19.2-44.9 31.1-71.9 34.5-4 .5-6.9 3.9-6.9 7.9V754c0 4.7 4.1 8.4 8.8 8 101.7-9.2 182.5-94 183.2-198.2.6-93.4-62.7-172.1-148.6-194.9z"}}]},name:"cloud-server",theme:"outlined"};var n=e.i(9583),a=i.forwardRef(function(e,a){return i.createElement(n.default,(0,t.default)({},e,{ref:a,icon:r}))});e.s(["CloudServerOutlined",0,a],295320)},283713,e=>{"use strict";var t=e.i(271645),i=e.i(764205),r=e.i(612256);let n="litellm_selected_worker_id";e.s(["useWorker",0,()=>{let{data:e}=(0,r.useUIConfig)(),a=e?.is_control_plane??!1,o=e?.workers??[],[l,s]=(0,t.useState)(()=>localStorage.getItem(n));(0,t.useEffect)(()=>{if(!l||0===o.length)return;let e=o.find(e=>e.worker_id===l);e&&(0,i.switchToWorkerUrl)(e.url)},[l,o]);let c=o.find(e=>e.worker_id===l)??null,d=(0,t.useCallback)(e=>{let t=o.find(t=>t.worker_id===e);t&&(s(e),localStorage.setItem(n,e),(0,i.switchToWorkerUrl)(t.url))},[o]);return{isControlPlane:a,workers:o,selectedWorkerId:l,selectedWorker:c,selectWorker:d,disconnectFromWorker:(0,t.useCallback)(()=>{s(null),localStorage.removeItem(n),(0,i.switchToWorkerUrl)(null)},[])}}])},954616,e=>{"use strict";var t=e.i(271645),i=e.i(114272),r=e.i(540143),n=e.i(915823),a=e.i(619273),o=class extends n.Subscribable{#e;#t=void 0;#i;#r;constructor(e,t){super(),this.#e=e,this.setOptions(t),this.bindMethods(),this.#n()}bindMethods(){this.mutate=this.mutate.bind(this),this.reset=this.reset.bind(this)}setOptions(e){let t=this.options;this.options=this.#e.defaultMutationOptions(e),(0,a.shallowEqualObjects)(this.options,t)||this.#e.getMutationCache().notify({type:"observerOptionsUpdated",mutation:this.#i,observer:this}),t?.mutationKey&&this.options.mutationKey&&(0,a.hashKey)(t.mutationKey)!==(0,a.hashKey)(this.options.mutationKey)?this.reset():this.#i?.state.status==="pending"&&this.#i.setOptions(this.options)}onUnsubscribe(){this.hasListeners()||this.#i?.removeObserver(this)}onMutationUpdate(e){this.#n(),this.#a(e)}getCurrentResult(){return this.#t}reset(){this.#i?.removeObserver(this),this.#i=void 0,this.#n(),this.#a()}mutate(e,t){return this.#r=t,this.#i?.removeObserver(this),this.#i=this.#e.getMutationCache().build(this.#e,this.options),this.#i.addObserver(this),this.#i.execute(e)}#n(){let e=this.#i?.state??(0,i.getDefaultState)();this.#t={...e,isPending:"pending"===e.status,isSuccess:"success"===e.status,isError:"error"===e.status,isIdle:"idle"===e.status,mutate:this.mutate,reset:this.reset}}#a(e){r.notifyManager.batch(()=>{if(this.#r&&this.hasListeners()){let t=this.#t.variables,i=this.#t.context,r={client:this.#e,meta:this.options.meta,mutationKey:this.options.mutationKey};if(e?.type==="success"){try{this.#r.onSuccess?.(e.data,t,i,r)}catch(e){Promise.reject(e)}try{this.#r.onSettled?.(e.data,null,t,i,r)}catch(e){Promise.reject(e)}}else if(e?.type==="error"){try{this.#r.onError?.(e.error,t,i,r)}catch(e){Promise.reject(e)}try{this.#r.onSettled?.(void 0,e.error,t,i,r)}catch(e){Promise.reject(e)}}}this.listeners.forEach(e=>{e(this.#t)})})}},l=e.i(912598);function s(e,i){let n=(0,l.useQueryClient)(i),[s]=t.useState(()=>new o(n,e));t.useEffect(()=>{s.setOptions(e)},[s,e]);let c=t.useSyncExternalStore(t.useCallback(e=>s.subscribe(r.notifyManager.batchCalls(e)),[s]),()=>s.getCurrentResult(),()=>s.getCurrentResult()),d=t.useCallback((e,t)=>{s.mutate(e,t).catch(a.noop)},[s]);if(c.error&&(0,a.shouldThrowError)(s.options.throwOnError,[c.error]))throw c.error;return{...c,mutate:d,mutateAsync:c.mutate}}e.s(["useMutation",()=>s],954616)},175712,e=>{"use strict";e.i(247167);var t=e.i(271645),i=e.i(343794),r=e.i(529681),n=e.i(242064),a=e.i(517455),o=e.i(185793),l=e.i(721369),s=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let c=e=>{var{prefixCls:r,className:a,hoverable:o=!0}=e,l=s(e,["prefixCls","className","hoverable"]);let{getPrefixCls:c}=t.useContext(n.ConfigContext),d=c("card",r),u=(0,i.default)(`${d}-grid`,a,{[`${d}-grid-hoverable`]:o});return t.createElement("div",Object.assign({},l,{className:u}))};e.i(296059);var d=e.i(915654),u=e.i(183293),m=e.i(246422),p=e.i(838378);let g=(0,m.genStyleHooks)("Card",e=>{let t=(0,p.mergeToken)(e,{cardShadow:e.boxShadowCard,cardHeadPadding:e.padding,cardPaddingBase:e.paddingLG,cardActionsIconSize:e.fontSize});return[(e=>{let{componentCls:t,cardShadow:i,cardHeadPadding:r,colorBorderSecondary:n,boxShadowTertiary:a,bodyPadding:o,extraColor:l}=e;return{[t]:Object.assign(Object.assign({},(0,u.resetComponent)(e)),{position:"relative",background:e.colorBgContainer,borderRadius:e.borderRadiusLG,[`&:not(${t}-bordered)`]:{boxShadow:a},[`${t}-head`]:(e=>{let{antCls:t,componentCls:i,headerHeight:r,headerPadding:n,tabsMarginBottom:a}=e;return Object.assign(Object.assign({display:"flex",justifyContent:"center",flexDirection:"column",minHeight:r,marginBottom:-1,padding:`0 ${(0,d.unit)(n)}`,color:e.colorTextHeading,fontWeight:e.fontWeightStrong,fontSize:e.headerFontSize,background:e.headerBg,borderBottom:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${e.colorBorderSecondary}`,borderRadius:`${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)} 0 0`},(0,u.clearFix)()),{"&-wrapper":{width:"100%",display:"flex",alignItems:"center"},"&-title":Object.assign(Object.assign({display:"inline-block",flex:1},u.textEllipsis),{[` +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,295320,e=>{"use strict";e.i(247167);var t=e.i(931067),i=e.i(271645);let r={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M704 446H320c-4.4 0-8 3.6-8 8v402c0 4.4 3.6 8 8 8h384c4.4 0 8-3.6 8-8V454c0-4.4-3.6-8-8-8zm-328 64h272v117H376V510zm272 290H376V683h272v117z"}},{tag:"path",attrs:{d:"M424 748a32 32 0 1064 0 32 32 0 10-64 0zm0-178a32 32 0 1064 0 32 32 0 10-64 0z"}},{tag:"path",attrs:{d:"M811.4 368.9C765.6 248 648.9 162 512.2 162S258.8 247.9 213 368.8C126.9 391.5 63.5 470.2 64 563.6 64.6 668 145.6 752.9 247.6 762c4.7.4 8.7-3.3 8.7-8v-60.4c0-4-3-7.4-7-7.9-27-3.4-52.5-15.2-72.1-34.5-24-23.5-37.2-55.1-37.2-88.6 0-28 9.1-54.4 26.2-76.4 16.7-21.4 40.2-36.9 66.1-43.7l37.9-10 13.9-36.7c8.6-22.8 20.6-44.2 35.7-63.5 14.9-19.2 32.6-36 52.4-50 41.1-28.9 89.5-44.2 140-44.2s98.9 15.3 140 44.3c19.9 14 37.5 30.8 52.4 50 15.1 19.3 27.1 40.7 35.7 63.5l13.8 36.6 37.8 10c54.2 14.4 92.1 63.7 92.1 120 0 33.6-13.2 65.1-37.2 88.6-19.5 19.2-44.9 31.1-71.9 34.5-4 .5-6.9 3.9-6.9 7.9V754c0 4.7 4.1 8.4 8.8 8 101.7-9.2 182.5-94 183.2-198.2.6-93.4-62.7-172.1-148.6-194.9z"}}]},name:"cloud-server",theme:"outlined"};var n=e.i(9583),a=i.forwardRef(function(e,a){return i.createElement(n.default,(0,t.default)({},e,{ref:a,icon:r}))});e.s(["CloudServerOutlined",0,a],295320)},283713,e=>{"use strict";var t=e.i(271645),i=e.i(602869),r=e.i(612256);let n="litellm_selected_worker_id";e.s(["useWorker",0,()=>{let{data:e}=(0,r.useUIConfig)(),a=e?.is_control_plane??!1,o=e?.workers??[],[l,s]=(0,t.useState)(()=>localStorage.getItem(n));(0,t.useEffect)(()=>{if(!l||0===o.length)return;let e=o.find(e=>e.worker_id===l);e&&(0,i.switchToWorkerUrl)(e.url)},[l,o]);let c=o.find(e=>e.worker_id===l)??null,d=(0,t.useCallback)(e=>{let t=o.find(t=>t.worker_id===e);t&&(s(e),localStorage.setItem(n,e),(0,i.switchToWorkerUrl)(t.url))},[o]);return{isControlPlane:a,workers:o,selectedWorkerId:l,selectedWorker:c,selectWorker:d,disconnectFromWorker:(0,t.useCallback)(()=>{s(null),localStorage.removeItem(n),(0,i.switchToWorkerUrl)(null)},[])}}])},954616,e=>{"use strict";var t=e.i(271645),i=e.i(114272),r=e.i(540143),n=e.i(915823),a=e.i(619273),o=class extends n.Subscribable{#e;#t=void 0;#i;#r;constructor(e,t){super(),this.#e=e,this.setOptions(t),this.bindMethods(),this.#n()}bindMethods(){this.mutate=this.mutate.bind(this),this.reset=this.reset.bind(this)}setOptions(e){let t=this.options;this.options=this.#e.defaultMutationOptions(e),(0,a.shallowEqualObjects)(this.options,t)||this.#e.getMutationCache().notify({type:"observerOptionsUpdated",mutation:this.#i,observer:this}),t?.mutationKey&&this.options.mutationKey&&(0,a.hashKey)(t.mutationKey)!==(0,a.hashKey)(this.options.mutationKey)?this.reset():this.#i?.state.status==="pending"&&this.#i.setOptions(this.options)}onUnsubscribe(){this.hasListeners()||this.#i?.removeObserver(this)}onMutationUpdate(e){this.#n(),this.#a(e)}getCurrentResult(){return this.#t}reset(){this.#i?.removeObserver(this),this.#i=void 0,this.#n(),this.#a()}mutate(e,t){return this.#r=t,this.#i?.removeObserver(this),this.#i=this.#e.getMutationCache().build(this.#e,this.options),this.#i.addObserver(this),this.#i.execute(e)}#n(){let e=this.#i?.state??(0,i.getDefaultState)();this.#t={...e,isPending:"pending"===e.status,isSuccess:"success"===e.status,isError:"error"===e.status,isIdle:"idle"===e.status,mutate:this.mutate,reset:this.reset}}#a(e){r.notifyManager.batch(()=>{if(this.#r&&this.hasListeners()){let t=this.#t.variables,i=this.#t.context,r={client:this.#e,meta:this.options.meta,mutationKey:this.options.mutationKey};if(e?.type==="success"){try{this.#r.onSuccess?.(e.data,t,i,r)}catch(e){Promise.reject(e)}try{this.#r.onSettled?.(e.data,null,t,i,r)}catch(e){Promise.reject(e)}}else if(e?.type==="error"){try{this.#r.onError?.(e.error,t,i,r)}catch(e){Promise.reject(e)}try{this.#r.onSettled?.(void 0,e.error,t,i,r)}catch(e){Promise.reject(e)}}}this.listeners.forEach(e=>{e(this.#t)})})}},l=e.i(912598);function s(e,i){let n=(0,l.useQueryClient)(i),[s]=t.useState(()=>new o(n,e));t.useEffect(()=>{s.setOptions(e)},[s,e]);let c=t.useSyncExternalStore(t.useCallback(e=>s.subscribe(r.notifyManager.batchCalls(e)),[s]),()=>s.getCurrentResult(),()=>s.getCurrentResult()),d=t.useCallback((e,t)=>{s.mutate(e,t).catch(a.noop)},[s]);if(c.error&&(0,a.shouldThrowError)(s.options.throwOnError,[c.error]))throw c.error;return{...c,mutate:d,mutateAsync:c.mutate}}e.s(["useMutation",()=>s],954616)},175712,e=>{"use strict";e.i(247167);var t=e.i(271645),i=e.i(343794),r=e.i(529681),n=e.i(242064),a=e.i(517455),o=e.i(185793),l=e.i(721369),s=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let c=e=>{var{prefixCls:r,className:a,hoverable:o=!0}=e,l=s(e,["prefixCls","className","hoverable"]);let{getPrefixCls:c}=t.useContext(n.ConfigContext),d=c("card",r),u=(0,i.default)(`${d}-grid`,a,{[`${d}-grid-hoverable`]:o});return t.createElement("div",Object.assign({},l,{className:u}))};e.i(296059);var d=e.i(915654),u=e.i(183293),m=e.i(246422),p=e.i(838378);let g=(0,m.genStyleHooks)("Card",e=>{let t=(0,p.mergeToken)(e,{cardShadow:e.boxShadowCard,cardHeadPadding:e.padding,cardPaddingBase:e.paddingLG,cardActionsIconSize:e.fontSize});return[(e=>{let{componentCls:t,cardShadow:i,cardHeadPadding:r,colorBorderSecondary:n,boxShadowTertiary:a,bodyPadding:o,extraColor:l}=e;return{[t]:Object.assign(Object.assign({},(0,u.resetComponent)(e)),{position:"relative",background:e.colorBgContainer,borderRadius:e.borderRadiusLG,[`&:not(${t}-bordered)`]:{boxShadow:a},[`${t}-head`]:(e=>{let{antCls:t,componentCls:i,headerHeight:r,headerPadding:n,tabsMarginBottom:a}=e;return Object.assign(Object.assign({display:"flex",justifyContent:"center",flexDirection:"column",minHeight:r,marginBottom:-1,padding:`0 ${(0,d.unit)(n)}`,color:e.colorTextHeading,fontWeight:e.fontWeightStrong,fontSize:e.headerFontSize,background:e.headerBg,borderBottom:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${e.colorBorderSecondary}`,borderRadius:`${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)} 0 0`},(0,u.clearFix)()),{"&-wrapper":{width:"100%",display:"flex",alignItems:"center"},"&-title":Object.assign(Object.assign({display:"inline-block",flex:1},u.textEllipsis),{[` > ${i}-typography, > ${i}-typography-edit-content `]:{insetInlineStart:0,marginTop:0,marginBottom:0}}),[`${t}-tabs-top`]:{clear:"both",marginBottom:a,color:e.colorText,fontWeight:"normal",fontSize:e.fontSize,"&-bar":{borderBottom:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${e.colorBorderSecondary}`}}})})(e),[`${t}-extra`]:{marginInlineStart:"auto",color:l,fontWeight:"normal",fontSize:e.fontSize},[`${t}-body`]:{padding:o,borderRadius:`0 0 ${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)}`},[`${t}-grid`]:(e=>{let{cardPaddingBase:t,colorBorderSecondary:i,cardShadow:r,lineWidth:n}=e;return{width:"33.33%",padding:t,border:0,borderRadius:0,boxShadow:` @@ -9,4 +9,4 @@ 0 ${(0,d.unit)(n)} 0 0 ${i} inset; `,transition:`all ${e.motionDurationMid}`,"&-hoverable:hover":{position:"relative",zIndex:1,boxShadow:r}}})(e),[`${t}-cover`]:{"> *":{display:"block",width:"100%",borderRadius:`${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)} 0 0`}},[`${t}-actions`]:(e=>{let{componentCls:t,iconCls:i,actionsLiMargin:r,cardActionsIconSize:n,colorBorderSecondary:a,actionsBg:o}=e;return Object.assign(Object.assign({margin:0,padding:0,listStyle:"none",background:o,borderTop:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${a}`,display:"flex",borderRadius:`0 0 ${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)}`},(0,u.clearFix)()),{"& > li":{margin:r,color:e.colorTextDescription,textAlign:"center","> span":{position:"relative",display:"block",minWidth:e.calc(e.cardActionsIconSize).mul(2).equal(),fontSize:e.fontSize,lineHeight:e.lineHeight,cursor:"pointer","&:hover":{color:e.colorPrimary,transition:`color ${e.motionDurationMid}`},[`a:not(${t}-btn), > ${i}`]:{display:"inline-block",width:"100%",color:e.colorIcon,lineHeight:(0,d.unit)(e.fontHeight),transition:`color ${e.motionDurationMid}`,"&:hover":{color:e.colorPrimary}},[`> ${i}`]:{fontSize:n,lineHeight:(0,d.unit)(e.calc(n).mul(e.lineHeight).equal())}},"&:not(:last-child)":{borderInlineEnd:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${a}`}}})})(e),[`${t}-meta`]:Object.assign(Object.assign({margin:`${(0,d.unit)(e.calc(e.marginXXS).mul(-1).equal())} 0`,display:"flex"},(0,u.clearFix)()),{"&-avatar":{paddingInlineEnd:e.padding},"&-detail":{overflow:"hidden",flex:1,"> div:not(:last-child)":{marginBottom:e.marginXS}},"&-title":Object.assign({color:e.colorTextHeading,fontWeight:e.fontWeightStrong,fontSize:e.fontSizeLG},u.textEllipsis),"&-description":{color:e.colorTextDescription}})}),[`${t}-bordered`]:{border:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${n}`,[`${t}-cover`]:{marginTop:-1,marginInlineStart:-1,marginInlineEnd:-1}},[`${t}-hoverable`]:{cursor:"pointer",transition:`box-shadow ${e.motionDurationMid}, border-color ${e.motionDurationMid}`,"&:hover":{borderColor:"transparent",boxShadow:i}},[`${t}-contain-grid`]:{borderRadius:`${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)} 0 0 `,[`${t}-body`]:{display:"flex",flexWrap:"wrap"},[`&:not(${t}-loading) ${t}-body`]:{marginBlockStart:e.calc(e.lineWidth).mul(-1).equal(),marginInlineStart:e.calc(e.lineWidth).mul(-1).equal(),padding:0}},[`${t}-contain-tabs`]:{[`> div${t}-head`]:{minHeight:0,[`${t}-head-title, ${t}-extra`]:{paddingTop:r}}},[`${t}-type-inner`]:(e=>{let{componentCls:t,colorFillAlter:i,headerPadding:r,bodyPadding:n}=e;return{[`${t}-head`]:{padding:`0 ${(0,d.unit)(r)}`,background:i,"&-title":{fontSize:e.fontSize}},[`${t}-body`]:{padding:`${(0,d.unit)(e.padding)} ${(0,d.unit)(n)}`}}})(e),[`${t}-loading`]:(e=>{let{componentCls:t}=e;return{overflow:"hidden",[`${t}-body`]:{userSelect:"none"}}})(e),[`${t}-rtl`]:{direction:"rtl"}}})(t),(e=>{let{componentCls:t,bodyPaddingSM:i,headerPaddingSM:r,headerHeightSM:n,headerFontSizeSM:a}=e;return{[`${t}-small`]:{[`> ${t}-head`]:{minHeight:n,padding:`0 ${(0,d.unit)(r)}`,fontSize:a,[`> ${t}-head-wrapper`]:{[`> ${t}-extra`]:{fontSize:e.fontSize}}},[`> ${t}-body`]:{padding:i}},[`${t}-small${t}-contain-tabs`]:{[`> ${t}-head`]:{[`${t}-head-title, ${t}-extra`]:{paddingTop:0,display:"flex",alignItems:"center"}}}}})(t)]},e=>{var t,i;return{headerBg:"transparent",headerFontSize:e.fontSizeLG,headerFontSizeSM:e.fontSize,headerHeight:e.fontSizeLG*e.lineHeightLG+2*e.padding,headerHeightSM:e.fontSize*e.lineHeight+2*e.paddingXS,actionsBg:e.colorBgContainer,actionsLiMargin:`${e.paddingSM}px 0`,tabsMarginBottom:-e.padding-e.lineWidth,extraColor:e.colorText,bodyPaddingSM:12,headerPaddingSM:12,bodyPadding:null!=(t=e.bodyPadding)?t:e.paddingLG,headerPadding:null!=(i=e.headerPadding)?i:e.paddingLG}});var h=e.i(792812),f=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let b=e=>{let{actionClasses:i,actions:r=[],actionStyle:n}=e;return t.createElement("ul",{className:i,style:n},r.map((e,i)=>{let n=`action-${i}`;return t.createElement("li",{style:{width:`${100/r.length}%`},key:n},t.createElement("span",null,e))}))},y=t.forwardRef((e,s)=>{let d,{prefixCls:u,className:m,rootClassName:p,style:y,extra:v,headStyle:x={},bodyStyle:$={},title:S,loading:j,bordered:w,variant:O,size:C,type:E,cover:I,actions:N,tabList:k,children:z,activeTabKey:L,defaultActiveTabKey:M,tabBarExtraContent:R,hoverable:P,tabProps:T={},classNames:_,styles:G}=e,B=f(e,["prefixCls","className","rootClassName","style","extra","headStyle","bodyStyle","title","loading","bordered","variant","size","type","cover","actions","tabList","children","activeTabKey","defaultActiveTabKey","tabBarExtraContent","hoverable","tabProps","classNames","styles"]),{getPrefixCls:A,direction:H,card:U}=t.useContext(n.ConfigContext),[W]=(0,h.default)("card",O,w),D=e=>{var t;return(0,i.default)(null==(t=null==U?void 0:U.classNames)?void 0:t[e],null==_?void 0:_[e])},F=e=>{var t;return Object.assign(Object.assign({},null==(t=null==U?void 0:U.styles)?void 0:t[e]),null==G?void 0:G[e])},K=t.useMemo(()=>{let e=!1;return t.Children.forEach(z,t=>{(null==t?void 0:t.type)===c&&(e=!0)}),e},[z]),q=A("card",u),[V,X,J]=g(q),Q=t.createElement(o.default,{loading:!0,active:!0,paragraph:{rows:4},title:!1},z),Y=void 0!==L,Z=Object.assign(Object.assign({},T),{[Y?"activeKey":"defaultActiveKey"]:Y?L:M,tabBarExtraContent:R}),ee=(0,a.default)(C),et=ee&&"default"!==ee?ee:"large",ei=k?t.createElement(l.default,Object.assign({size:et},Z,{className:`${q}-head-tabs`,onChange:t=>{var i;null==(i=e.onTabChange)||i.call(e,t)},items:k.map(e=>{var{tab:t}=e;return Object.assign({label:t},f(e,["tab"]))})})):null;if(S||v||ei){let e=(0,i.default)(`${q}-head`,D("header")),r=(0,i.default)(`${q}-head-title`,D("title")),n=(0,i.default)(`${q}-extra`,D("extra")),a=Object.assign(Object.assign({},x),F("header"));d=t.createElement("div",{className:e,style:a},t.createElement("div",{className:`${q}-head-wrapper`},S&&t.createElement("div",{className:r,style:F("title")},S),v&&t.createElement("div",{className:n,style:F("extra")},v)),ei)}let er=(0,i.default)(`${q}-cover`,D("cover")),en=I?t.createElement("div",{className:er,style:F("cover")},I):null,ea=(0,i.default)(`${q}-body`,D("body")),eo=Object.assign(Object.assign({},$),F("body")),el=t.createElement("div",{className:ea,style:eo},j?Q:z),es=(0,i.default)(`${q}-actions`,D("actions")),ec=(null==N?void 0:N.length)?t.createElement(b,{actionClasses:es,actionStyle:F("actions"),actions:N}):null,ed=(0,r.default)(B,["onTabChange"]),eu=(0,i.default)(q,null==U?void 0:U.className,{[`${q}-loading`]:j,[`${q}-bordered`]:"borderless"!==W,[`${q}-hoverable`]:P,[`${q}-contain-grid`]:K,[`${q}-contain-tabs`]:null==k?void 0:k.length,[`${q}-${ee}`]:ee,[`${q}-type-${E}`]:!!E,[`${q}-rtl`]:"rtl"===H},m,p,X,J),em=Object.assign(Object.assign({},null==U?void 0:U.style),y);return V(t.createElement("div",Object.assign({ref:s},ed,{className:eu,style:em}),d,en,el,ec))});var v=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};y.Grid=c,y.Meta=e=>{let{prefixCls:r,className:a,avatar:o,title:l,description:s}=e,c=v(e,["prefixCls","className","avatar","title","description"]),{getPrefixCls:d}=t.useContext(n.ConfigContext),u=d("card",r),m=(0,i.default)(`${u}-meta`,a),p=o?t.createElement("div",{className:`${u}-meta-avatar`},o):null,g=l?t.createElement("div",{className:`${u}-meta-title`},l):null,h=s?t.createElement("div",{className:`${u}-meta-description`},s):null,f=g||h?t.createElement("div",{className:`${u}-meta-detail`},g,h):null;return t.createElement("div",Object.assign({},c,{className:m}),p,f)},e.s(["Card",0,y],175712)},770914,908286,38243,e=>{"use strict";e.i(247167);var t=e.i(271645),i=e.i(343794),r=e.i(876556);function n(e){return["small","middle","large"].includes(e)}function a(e){return!!e&&"number"==typeof e&&!Number.isNaN(e)}e.s(["isPresetSize",()=>n,"isValidGapNumber",()=>a],908286);var o=e.i(242064),l=e.i(249616),s=e.i(372409),c=e.i(246422);let d=(0,c.genStyleHooks)(["Space","Addon"],e=>[(e=>{let{componentCls:t,borderRadius:i,paddingSM:r,colorBorder:n,paddingXS:a,fontSizeLG:o,fontSizeSM:l,borderRadiusLG:c,borderRadiusSM:d,colorBgContainerDisabled:u,lineWidth:m}=e;return{[t]:[{display:"inline-flex",alignItems:"center",gap:0,paddingInline:r,margin:0,background:u,borderWidth:m,borderStyle:"solid",borderColor:n,borderRadius:i,"&-large":{fontSize:o,borderRadius:c},"&-small":{paddingInline:a,borderRadius:d,fontSize:l},"&-compact-last-item":{borderEndStartRadius:0,borderStartStartRadius:0},"&-compact-first-item":{borderEndEndRadius:0,borderStartEndRadius:0},"&-compact-item:not(:first-child):not(:last-child)":{borderRadius:0},"&-compact-item:not(:last-child)":{borderInlineEndWidth:0}},(0,s.genCompactItemStyle)(e,{focus:!1})]}})(e)]);var u=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let m=t.default.forwardRef((e,r)=>{let{className:n,children:a,style:s,prefixCls:c}=e,m=u(e,["className","children","style","prefixCls"]),{getPrefixCls:p,direction:g}=t.default.useContext(o.ConfigContext),h=p("space-addon",c),[f,b,y]=d(h),{compactItemClassnames:v,compactSize:x}=(0,l.useCompactItemContext)(h,g),$=(0,i.default)(h,b,v,y,{[`${h}-${x}`]:x},n);return f(t.default.createElement("div",Object.assign({ref:r,className:$,style:s},m),a))}),p=t.default.createContext({latestIndex:0}),g=p.Provider,h=({className:e,index:i,children:r,split:n,style:a})=>{let{latestIndex:o}=t.useContext(p);return null==r?null:t.createElement(t.Fragment,null,t.createElement("div",{className:e,style:a},r),i{let t=(0,f.mergeToken)(e,{spaceGapSmallSize:e.paddingXS,spaceGapMiddleSize:e.padding,spaceGapLargeSize:e.paddingLG});return[(e=>{let{componentCls:t,antCls:i}=e;return{[t]:{display:"inline-flex","&-rtl":{direction:"rtl"},"&-vertical":{flexDirection:"column"},"&-align":{flexDirection:"column","&-center":{alignItems:"center"},"&-start":{alignItems:"flex-start"},"&-end":{alignItems:"flex-end"},"&-baseline":{alignItems:"baseline"}},[`${t}-item:empty`]:{display:"none"},[`${t}-item > ${i}-badge-not-a-wrapper:only-child`]:{display:"block"}}}})(t),(e=>{let{componentCls:t}=e;return{[t]:{"&-gap-row-small":{rowGap:e.spaceGapSmallSize},"&-gap-row-middle":{rowGap:e.spaceGapMiddleSize},"&-gap-row-large":{rowGap:e.spaceGapLargeSize},"&-gap-col-small":{columnGap:e.spaceGapSmallSize},"&-gap-col-middle":{columnGap:e.spaceGapMiddleSize},"&-gap-col-large":{columnGap:e.spaceGapLargeSize}}}})(t)]},()=>({}),{resetStyle:!1});var y=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let v=t.forwardRef((e,l)=>{var s;let{getPrefixCls:c,direction:d,size:u,className:m,style:p,classNames:f,styles:v}=(0,o.useComponentConfig)("space"),{size:x=null!=u?u:"small",align:$,className:S,rootClassName:j,children:w,direction:O="horizontal",prefixCls:C,split:E,style:I,wrap:N=!1,classNames:k,styles:z}=e,L=y(e,["size","align","className","rootClassName","children","direction","prefixCls","split","style","wrap","classNames","styles"]),[M,R]=Array.isArray(x)?x:[x,x],P=n(R),T=n(M),_=a(R),G=a(M),B=(0,r.default)(w,{keepEmpty:!0}),A=void 0===$&&"horizontal"===O?"center":$,H=c("space",C),[U,W,D]=b(H),F=(0,i.default)(H,m,W,`${H}-${O}`,{[`${H}-rtl`]:"rtl"===d,[`${H}-align-${A}`]:A,[`${H}-gap-row-${R}`]:P,[`${H}-gap-col-${M}`]:T},S,j,D),K=(0,i.default)(`${H}-item`,null!=(s=null==k?void 0:k.item)?s:f.item),q=Object.assign(Object.assign({},v.item),null==z?void 0:z.item),V=B.map((e,i)=>{let r=(null==e?void 0:e.key)||`${K}-${i}`;return t.createElement(h,{className:K,key:r,index:i,split:E,style:q},e)}),X=t.useMemo(()=>({latestIndex:B.reduce((e,t,i)=>null!=t?i:e,0)}),[B]);if(0===B.length)return null;let J={};return N&&(J.flexWrap="wrap"),!T&&G&&(J.columnGap=M),!P&&_&&(J.rowGap=R),U(t.createElement("div",Object.assign({ref:l,className:F,style:Object.assign(Object.assign(Object.assign({},J),p),I)},L),t.createElement(g,{value:X},V)))});v.Compact=l.default,v.Addon=m,e.s(["default",0,v],38243),e.s(["Space",0,v],770914)},560445,e=>{"use strict";e.i(247167);var t=e.i(271645),i=e.i(201072),r=e.i(726289),n=e.i(864517),a=e.i(562901),o=e.i(779573),l=e.i(343794),s=e.i(361275),c=e.i(244009),d=e.i(611935),u=e.i(763731),m=e.i(242064);e.i(296059);var p=e.i(915654),g=e.i(183293),h=e.i(246422);let f=(e,t,i,r,n)=>({background:e,border:`${(0,p.unit)(r.lineWidth)} ${r.lineType} ${t}`,[`${n}-icon`]:{color:i}}),b=(0,h.genStyleHooks)("Alert",e=>[(e=>{let{componentCls:t,motionDurationSlow:i,marginXS:r,marginSM:n,fontSize:a,fontSizeLG:o,lineHeight:l,borderRadiusLG:s,motionEaseInOutCirc:c,withDescriptionIconSize:d,colorText:u,colorTextHeading:m,withDescriptionPadding:p,defaultPadding:h}=e;return{[t]:Object.assign(Object.assign({},(0,g.resetComponent)(e)),{position:"relative",display:"flex",alignItems:"center",padding:h,wordWrap:"break-word",borderRadius:s,[`&${t}-rtl`]:{direction:"rtl"},[`${t}-content`]:{flex:1,minWidth:0},[`${t}-icon`]:{marginInlineEnd:r,lineHeight:0},"&-description":{display:"none",fontSize:a,lineHeight:l},"&-message":{color:m},[`&${t}-motion-leave`]:{overflow:"hidden",opacity:1,transition:`max-height ${i} ${c}, opacity ${i} ${c}, padding-top ${i} ${c}, padding-bottom ${i} ${c}, - margin-bottom ${i} ${c}`},[`&${t}-motion-leave-active`]:{maxHeight:0,marginBottom:"0 !important",paddingTop:0,paddingBottom:0,opacity:0}}),[`${t}-with-description`]:{alignItems:"flex-start",padding:p,[`${t}-icon`]:{marginInlineEnd:n,fontSize:d,lineHeight:0},[`${t}-message`]:{display:"block",marginBottom:r,color:m,fontSize:o},[`${t}-description`]:{display:"block",color:u}},[`${t}-banner`]:{marginBottom:0,border:"0 !important",borderRadius:0}}})(e),(e=>{let{componentCls:t,colorSuccess:i,colorSuccessBorder:r,colorSuccessBg:n,colorWarning:a,colorWarningBorder:o,colorWarningBg:l,colorError:s,colorErrorBorder:c,colorErrorBg:d,colorInfo:u,colorInfoBorder:m,colorInfoBg:p}=e;return{[t]:{"&-success":f(n,r,i,e,t),"&-info":f(p,m,u,e,t),"&-warning":f(l,o,a,e,t),"&-error":Object.assign(Object.assign({},f(d,c,s,e,t)),{[`${t}-description > pre`]:{margin:0,padding:0}})}}})(e),(e=>{let{componentCls:t,iconCls:i,motionDurationMid:r,marginXS:n,fontSizeIcon:a,colorIcon:o,colorIconHover:l}=e;return{[t]:{"&-action":{marginInlineStart:n},[`${t}-close-icon`]:{marginInlineStart:n,padding:0,overflow:"hidden",fontSize:a,lineHeight:(0,p.unit)(a),backgroundColor:"transparent",border:"none",outline:"none",cursor:"pointer",[`${i}-close`]:{color:o,transition:`color ${r}`,"&:hover":{color:l}}},"&-close-text":{color:o,transition:`color ${r}`,"&:hover":{color:l}}}}})(e)],e=>({withDescriptionIconSize:e.fontSizeHeading3,defaultPadding:`${e.paddingContentVerticalSM}px 12px`,withDescriptionPadding:`${e.paddingMD}px ${e.paddingContentHorizontalLG}px`}));var y=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let v={success:i.default,info:o.default,error:r.default,warning:a.default},x=e=>{let{icon:i,prefixCls:r,type:n}=e,a=v[n]||null;return i?(0,u.replaceElement)(i,t.createElement("span",{className:`${r}-icon`},i),()=>({className:(0,l.default)(`${r}-icon`,i.props.className)})):t.createElement(a,{className:`${r}-icon`})},$=e=>{let{isClosable:i,prefixCls:r,closeIcon:a,handleClose:o,ariaProps:l}=e,s=!0===a||void 0===a?t.createElement(n.default,null):a;return i?t.createElement("button",Object.assign({type:"button",onClick:o,className:`${r}-close-icon`,tabIndex:0},l),s):null},S=t.forwardRef((e,i)=>{let{description:r,prefixCls:n,message:a,banner:o,className:u,rootClassName:p,style:g,onMouseEnter:h,onMouseLeave:f,onClick:v,afterClose:S,showIcon:j,closable:w,closeText:O,closeIcon:C,action:E,id:I}=e,N=y(e,["description","prefixCls","message","banner","className","rootClassName","style","onMouseEnter","onMouseLeave","onClick","afterClose","showIcon","closable","closeText","closeIcon","action","id"]),[k,z]=t.useState(!1),L=t.useRef(null);t.useImperativeHandle(i,()=>({nativeElement:L.current}));let{getPrefixCls:M,direction:R,closable:P,closeIcon:T,className:_,style:G}=(0,m.useComponentConfig)("alert"),B=M("alert",n),[A,H,U]=b(B),W=t=>{var i;z(!0),null==(i=e.onClose)||i.call(e,t)},D=t.useMemo(()=>void 0!==e.type?e.type:o?"warning":"info",[e.type,o]),F=t.useMemo(()=>"object"==typeof w&&!!w.closeIcon||!!O||("boolean"==typeof w?w:!1!==C&&null!=C||!!P),[O,C,w,P]),K=!!o&&void 0===j||j,q=(0,l.default)(B,`${B}-${D}`,{[`${B}-with-description`]:!!r,[`${B}-no-icon`]:!K,[`${B}-banner`]:!!o,[`${B}-rtl`]:"rtl"===R},_,u,p,U,H),V=(0,c.default)(N,{aria:!0,data:!0}),X=t.useMemo(()=>"object"==typeof w&&w.closeIcon?w.closeIcon:O||(void 0!==C?C:"object"==typeof P&&P.closeIcon?P.closeIcon:T),[C,w,P,O,T]),J=t.useMemo(()=>{let e=null!=w?w:P;if("object"==typeof e){let{closeIcon:t}=e;return y(e,["closeIcon"])}return{}},[w,P]);return A(t.createElement(s.default,{visible:!k,motionName:`${B}-motion`,motionAppear:!1,motionEnter:!1,onLeaveStart:e=>({maxHeight:e.offsetHeight}),onLeaveEnd:S},({className:i,style:n},o)=>t.createElement("div",Object.assign({id:I,ref:(0,d.composeRef)(L,o),"data-show":!k,className:(0,l.default)(q,i),style:Object.assign(Object.assign(Object.assign({},G),g),n),onMouseEnter:h,onMouseLeave:f,onClick:v,role:"alert"},V),K?t.createElement(x,{description:r,icon:e.icon,prefixCls:B,type:D}):null,t.createElement("div",{className:`${B}-content`},a?t.createElement("div",{className:`${B}-message`},a):null,r?t.createElement("div",{className:`${B}-description`},r):null),E?t.createElement("div",{className:`${B}-action`},E):null,t.createElement($,{isClosable:F,prefixCls:B,closeIcon:X,handleClose:W,ariaProps:J}))))});var j=e.i(278409),w=e.i(233848),O=e.i(487806),C=e.i(479671),E=e.i(480002),I=e.i(868917);let N=function(e){function i(){var e,t,r;return(0,j.default)(this,i),t=i,r=arguments,t=(0,O.default)(t),(e=(0,E.default)(this,(0,C.default)()?Reflect.construct(t,r||[],(0,O.default)(this).constructor):t.apply(this,r))).state={error:void 0,info:{componentStack:""}},e}return(0,I.default)(i,e),(0,w.default)(i,[{key:"componentDidCatch",value:function(e,t){this.setState({error:e,info:t})}},{key:"render",value:function(){let{message:e,description:i,id:r,children:n}=this.props,{error:a,info:o}=this.state,l=(null==o?void 0:o.componentStack)||null,s=void 0===e?(a||"").toString():e;return a?t.createElement(S,{id:r,type:"error",message:s,description:t.createElement("pre",{style:{fontSize:"0.9em",overflowX:"auto"}},void 0===i?l:i)}):n}}])}(t.Component);S.ErrorBoundary=N,e.s(["Alert",0,S],560445)},936578,571303,e=>{"use strict";var t=e.i(843476),i=e.i(115504),r=e.i(271645);function n({className:e="",...n}){var a,o;let l=(0,r.useId)();return a=()=>{let e=document.getAnimations().filter(e=>e instanceof CSSAnimation&&"spin"===e.animationName),t=e.find(e=>e.effect.target?.getAttribute("data-spinner-id")===l),i=e.find(e=>e.effect instanceof KeyframeEffect&&e.effect.target?.getAttribute("data-spinner-id")!==l);t&&i&&(t.currentTime=i.currentTime)},o=[l],(0,r.useLayoutEffect)(a,o),(0,t.jsxs)("svg",{"data-spinner-id":l,className:(0,i.cx)("pointer-events-none size-12 animate-spin text-current",e),fill:"none",viewBox:"0 0 24 24",...n,children:[(0,t.jsx)("circle",{className:"opacity-25",cx:"12",cy:"12",r:"10",stroke:"currentColor",strokeWidth:"4"}),(0,t.jsx)("path",{className:"opacity-75",fill:"currentColor",d:"M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"})]})}function a(){return(0,t.jsxs)("div",{className:(0,i.cx)("h-screen","flex items-center justify-center gap-4"),children:[(0,t.jsx)("div",{className:"text-lg font-medium py-2 pr-4 border-r border-r-gray-200",children:"🚅 LiteLLM"}),(0,t.jsxs)("div",{className:"flex items-center justify-center gap-2",children:[(0,t.jsx)(n,{className:"size-4"}),(0,t.jsx)("span",{className:"text-gray-600 text-sm",children:"Loading..."})]})]})}e.s(["UiLoadingSpinner",()=>n],571303),e.s(["default",()=>a],936578)},594542,e=>{"use strict";var t=e.i(843476),i=e.i(954616),r=e.i(764205),n=e.i(612256),a=e.i(936578),o=e.i(268004),l=e.i(161281),s=e.i(321836),c=e.i(827252),d=e.i(295320),u=e.i(560445),m=e.i(464571),p=e.i(175712),g=e.i(808613),h=e.i(311451),f=e.i(282786),b=e.i(199133),y=e.i(770914),v=e.i(898586),x=e.i(618566),$=e.i(271645),S=e.i(283713);function j(){let[e,j]=(0,$.useState)(""),[w,O]=(0,$.useState)(""),[C,E]=(0,$.useState)(!0),{data:I,isLoading:N}=(0,n.useUIConfig)(),k=(0,i.useMutation)({mutationFn:async({username:e,password:t,useV3:i})=>await (0,r.loginCall)(e,t,i)}),z=(0,x.useRouter)(),{workers:L,selectWorker:M}=(0,S.useWorker)(),[R,P]=(0,$.useState)(null);(0,$.useEffect)(()=>{let e=new URLSearchParams(window.location.search).get("worker");e&&P(e)},[]),(0,$.useEffect)(()=>{if(N)return;if(I&&I.admin_ui_disabled)return void E(!1);let e=new URLSearchParams(window.location.search),t=e.get("code"),i=t&&/^[a-zA-Z0-9._~+/=-]+$/.test(t)?t:null;if(i){let t=localStorage.getItem("litellm_worker_url"),n=t&&/^https?:\/\/.+/.test(t)?t:null;(0,r.exchangeLoginCode)(i,n).then(()=>{e.delete("code");let t=e.toString();window.history.replaceState(null,"",window.location.pathname+(t?`?${t}`:"")),z.replace("/ui/?login=success")});return}if(e.has("worker")&&I?.is_control_plane){(0,o.clearTokenCookies)(),E(!1);return}let n=(0,o.getCookieFromDocument)("token");if(n&&!(0,l.isJwtExpired)(n)){let e=(0,s.consumeReturnUrl)();e?z.replace(e):z.replace("/ui");return}if(I&&I.auto_redirect_to_sso){let e=(0,s.getReturnUrl)(),t=`${(0,r.getProxyBaseUrl)()}/sso/key/generate`;e&&(0,s.isValidReturnUrl)(e)&&(t+=`?redirect_to=${encodeURIComponent(e)}`),z.push(t);return}E(!1)},[N,z,I]);let T=k.error instanceof Error?k.error.message:null,_=k.isPending,{Title:G,Text:B,Paragraph:A}=v.Typography;return N||C?(0,t.jsx)(a.default,{}):I&&I.admin_ui_disabled?(0,t.jsx)("div",{className:"min-h-screen flex items-center justify-center bg-gray-50",children:(0,t.jsx)(p.Card,{className:"w-full max-w-lg shadow-md",children:(0,t.jsxs)(y.Space,{direction:"vertical",size:"middle",className:"w-full",children:[(0,t.jsx)("div",{className:"text-center",children:(0,t.jsx)(G,{level:2,children:"🚅 LiteLLM"})}),(0,t.jsx)(u.Alert,{message:"Admin UI Disabled",description:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(A,{className:"text-sm",children:"The Admin UI has been disabled by the administrator. To re-enable it, please update the following environment variable:"}),(0,t.jsx)(A,{className:"text-sm",children:(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"DISABLE_ADMIN_UI=False"})})]}),type:"warning",showIcon:!0})]})})}):(0,t.jsx)("div",{className:"min-h-screen flex items-center justify-center bg-gray-50",children:(0,t.jsxs)(p.Card,{className:"w-full max-w-lg shadow-md",children:[(0,t.jsxs)(y.Space,{direction:"vertical",size:"middle",className:"w-full",children:[(0,t.jsx)("div",{className:"text-center",children:(0,t.jsx)(G,{level:2,children:"🚅 LiteLLM"})}),(0,t.jsxs)("div",{className:"text-center",children:[(0,t.jsx)(G,{level:3,children:"Login"}),(0,t.jsx)(B,{type:"secondary",children:"Access your LiteLLM Admin UI."})]}),(0,t.jsx)(u.Alert,{message:"Default Credentials",description:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsxs)(A,{className:"text-sm",children:["By default, Username is ",(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"admin"})," and Password is your set LiteLLM Proxy",(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"MASTER_KEY"}),"."]}),(0,t.jsxs)(A,{className:"text-sm",children:["Need to set UI credentials or SSO?"," ",(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/proxy/ui",target:"_blank",rel:"noopener noreferrer",children:"Check the documentation"}),"."]})]}),type:"info",icon:(0,t.jsx)(c.InfoCircleOutlined,{}),showIcon:!0}),T&&(0,t.jsx)(u.Alert,{message:T,type:"error",showIcon:!0}),(0,t.jsxs)(g.Form,{onFinish:()=>{let t=L.find(e=>e.worker_id===R);t&&(0,r.switchToWorkerUrl)(t.url),k.mutate({username:e,password:w,useV3:!!t},{onSuccess:e=>{if(t)M(t.worker_id),z.push("/ui/?login=success");else{let t=(0,s.consumeReturnUrl)();t?z.push(t):z.push(e.redirect_url)}},onError:()=>{t&&(0,r.switchToWorkerUrl)(null)}})},layout:"vertical",requiredMark:!1,children:[I?.is_control_plane&&L.length>0&&(0,t.jsx)(g.Form.Item,{label:"Worker",style:{marginBottom:16},children:(0,t.jsx)(b.Select,{value:R||void 0,onChange:e=>P(e),placeholder:"Choose a worker to connect to",size:"large",suffixIcon:(0,t.jsx)(d.CloudServerOutlined,{}),options:L.map(e=>({label:e.name,value:e.worker_id}))})}),(0,t.jsx)(g.Form.Item,{label:"Username",name:"username",rules:[{required:!0,message:"Please enter your username"}],children:(0,t.jsx)(h.Input,{placeholder:"Enter your username",autoComplete:"username",value:e,onChange:e=>j(e.target.value),disabled:_,size:"large",className:"rounded-md border-gray-300"})}),(0,t.jsx)(g.Form.Item,{label:"Password",name:"password",rules:[{required:!0,message:"Please enter your password"}],children:(0,t.jsx)(h.Input.Password,{placeholder:"Enter your password",autoComplete:"current-password",value:w,onChange:e=>O(e.target.value),disabled:_,size:"large"})}),(0,t.jsx)(g.Form.Item,{children:(0,t.jsx)(m.Button,{type:"primary",htmlType:"submit",loading:_,disabled:_,block:!0,size:"large",children:_?"Logging in...":"Login"})}),(0,t.jsx)(g.Form.Item,{children:I?.sso_configured?(0,t.jsx)(m.Button,{disabled:_||!!R&&0===L.length,onClick:()=>{let e=L.find(e=>e.worker_id===R);e&&(localStorage.setItem("litellm_selected_worker_id",R),(0,r.switchToWorkerUrl)(e.url));let t=e?.url??(0,r.getProxyBaseUrl)(),i=encodeURIComponent(window.location.origin+"/ui/login");z.push(`${t}/sso/key/generate?return_to=${i}`)},block:!0,size:"large",children:"Login with SSO"}):(0,t.jsx)(f.Popover,{content:"Please configure SSO to log in with SSO.",trigger:"hover",children:(0,t.jsx)(m.Button,{disabled:!0,block:!0,size:"large",children:"Login with SSO"})})})]})]}),I?.sso_configured&&(0,t.jsx)(u.Alert,{type:"info",showIcon:!0,closable:!0,message:(0,t.jsxs)(B,{children:["Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow upon loading this page. To re-enable auto-redirect-to-SSO, set ",(0,t.jsx)(B,{code:!0,children:"AUTO_REDIRECT_UI_LOGIN_TO_SSO=true"})," in your environment configuration."]})})]})})}e.s(["default",0,function(){return(0,t.jsx)(j,{})}],594542)}]); \ No newline at end of file + margin-bottom ${i} ${c}`},[`&${t}-motion-leave-active`]:{maxHeight:0,marginBottom:"0 !important",paddingTop:0,paddingBottom:0,opacity:0}}),[`${t}-with-description`]:{alignItems:"flex-start",padding:p,[`${t}-icon`]:{marginInlineEnd:n,fontSize:d,lineHeight:0},[`${t}-message`]:{display:"block",marginBottom:r,color:m,fontSize:o},[`${t}-description`]:{display:"block",color:u}},[`${t}-banner`]:{marginBottom:0,border:"0 !important",borderRadius:0}}})(e),(e=>{let{componentCls:t,colorSuccess:i,colorSuccessBorder:r,colorSuccessBg:n,colorWarning:a,colorWarningBorder:o,colorWarningBg:l,colorError:s,colorErrorBorder:c,colorErrorBg:d,colorInfo:u,colorInfoBorder:m,colorInfoBg:p}=e;return{[t]:{"&-success":f(n,r,i,e,t),"&-info":f(p,m,u,e,t),"&-warning":f(l,o,a,e,t),"&-error":Object.assign(Object.assign({},f(d,c,s,e,t)),{[`${t}-description > pre`]:{margin:0,padding:0}})}}})(e),(e=>{let{componentCls:t,iconCls:i,motionDurationMid:r,marginXS:n,fontSizeIcon:a,colorIcon:o,colorIconHover:l}=e;return{[t]:{"&-action":{marginInlineStart:n},[`${t}-close-icon`]:{marginInlineStart:n,padding:0,overflow:"hidden",fontSize:a,lineHeight:(0,p.unit)(a),backgroundColor:"transparent",border:"none",outline:"none",cursor:"pointer",[`${i}-close`]:{color:o,transition:`color ${r}`,"&:hover":{color:l}}},"&-close-text":{color:o,transition:`color ${r}`,"&:hover":{color:l}}}}})(e)],e=>({withDescriptionIconSize:e.fontSizeHeading3,defaultPadding:`${e.paddingContentVerticalSM}px 12px`,withDescriptionPadding:`${e.paddingMD}px ${e.paddingContentHorizontalLG}px`}));var y=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let v={success:i.default,info:o.default,error:r.default,warning:a.default},x=e=>{let{icon:i,prefixCls:r,type:n}=e,a=v[n]||null;return i?(0,u.replaceElement)(i,t.createElement("span",{className:`${r}-icon`},i),()=>({className:(0,l.default)(`${r}-icon`,i.props.className)})):t.createElement(a,{className:`${r}-icon`})},$=e=>{let{isClosable:i,prefixCls:r,closeIcon:a,handleClose:o,ariaProps:l}=e,s=!0===a||void 0===a?t.createElement(n.default,null):a;return i?t.createElement("button",Object.assign({type:"button",onClick:o,className:`${r}-close-icon`,tabIndex:0},l),s):null},S=t.forwardRef((e,i)=>{let{description:r,prefixCls:n,message:a,banner:o,className:u,rootClassName:p,style:g,onMouseEnter:h,onMouseLeave:f,onClick:v,afterClose:S,showIcon:j,closable:w,closeText:O,closeIcon:C,action:E,id:I}=e,N=y(e,["description","prefixCls","message","banner","className","rootClassName","style","onMouseEnter","onMouseLeave","onClick","afterClose","showIcon","closable","closeText","closeIcon","action","id"]),[k,z]=t.useState(!1),L=t.useRef(null);t.useImperativeHandle(i,()=>({nativeElement:L.current}));let{getPrefixCls:M,direction:R,closable:P,closeIcon:T,className:_,style:G}=(0,m.useComponentConfig)("alert"),B=M("alert",n),[A,H,U]=b(B),W=t=>{var i;z(!0),null==(i=e.onClose)||i.call(e,t)},D=t.useMemo(()=>void 0!==e.type?e.type:o?"warning":"info",[e.type,o]),F=t.useMemo(()=>"object"==typeof w&&!!w.closeIcon||!!O||("boolean"==typeof w?w:!1!==C&&null!=C||!!P),[O,C,w,P]),K=!!o&&void 0===j||j,q=(0,l.default)(B,`${B}-${D}`,{[`${B}-with-description`]:!!r,[`${B}-no-icon`]:!K,[`${B}-banner`]:!!o,[`${B}-rtl`]:"rtl"===R},_,u,p,U,H),V=(0,c.default)(N,{aria:!0,data:!0}),X=t.useMemo(()=>"object"==typeof w&&w.closeIcon?w.closeIcon:O||(void 0!==C?C:"object"==typeof P&&P.closeIcon?P.closeIcon:T),[C,w,P,O,T]),J=t.useMemo(()=>{let e=null!=w?w:P;if("object"==typeof e){let{closeIcon:t}=e;return y(e,["closeIcon"])}return{}},[w,P]);return A(t.createElement(s.default,{visible:!k,motionName:`${B}-motion`,motionAppear:!1,motionEnter:!1,onLeaveStart:e=>({maxHeight:e.offsetHeight}),onLeaveEnd:S},({className:i,style:n},o)=>t.createElement("div",Object.assign({id:I,ref:(0,d.composeRef)(L,o),"data-show":!k,className:(0,l.default)(q,i),style:Object.assign(Object.assign(Object.assign({},G),g),n),onMouseEnter:h,onMouseLeave:f,onClick:v,role:"alert"},V),K?t.createElement(x,{description:r,icon:e.icon,prefixCls:B,type:D}):null,t.createElement("div",{className:`${B}-content`},a?t.createElement("div",{className:`${B}-message`},a):null,r?t.createElement("div",{className:`${B}-description`},r):null),E?t.createElement("div",{className:`${B}-action`},E):null,t.createElement($,{isClosable:F,prefixCls:B,closeIcon:X,handleClose:W,ariaProps:J}))))});var j=e.i(278409),w=e.i(233848),O=e.i(487806),C=e.i(479671),E=e.i(480002),I=e.i(868917);let N=function(e){function i(){var e,t,r;return(0,j.default)(this,i),t=i,r=arguments,t=(0,O.default)(t),(e=(0,E.default)(this,(0,C.default)()?Reflect.construct(t,r||[],(0,O.default)(this).constructor):t.apply(this,r))).state={error:void 0,info:{componentStack:""}},e}return(0,I.default)(i,e),(0,w.default)(i,[{key:"componentDidCatch",value:function(e,t){this.setState({error:e,info:t})}},{key:"render",value:function(){let{message:e,description:i,id:r,children:n}=this.props,{error:a,info:o}=this.state,l=(null==o?void 0:o.componentStack)||null,s=void 0===e?(a||"").toString():e;return a?t.createElement(S,{id:r,type:"error",message:s,description:t.createElement("pre",{style:{fontSize:"0.9em",overflowX:"auto"}},void 0===i?l:i)}):n}}])}(t.Component);S.ErrorBoundary=N,e.s(["Alert",0,S],560445)},936578,571303,e=>{"use strict";var t=e.i(843476),i=e.i(115504),r=e.i(271645);function n({className:e="",...n}){var a,o;let l=(0,r.useId)();return a=()=>{let e=document.getAnimations().filter(e=>e instanceof CSSAnimation&&"spin"===e.animationName),t=e.find(e=>e.effect.target?.getAttribute("data-spinner-id")===l),i=e.find(e=>e.effect instanceof KeyframeEffect&&e.effect.target?.getAttribute("data-spinner-id")!==l);t&&i&&(t.currentTime=i.currentTime)},o=[l],(0,r.useLayoutEffect)(a,o),(0,t.jsxs)("svg",{"data-spinner-id":l,className:(0,i.cx)("pointer-events-none size-12 animate-spin text-current",e),fill:"none",viewBox:"0 0 24 24",...n,children:[(0,t.jsx)("circle",{className:"opacity-25",cx:"12",cy:"12",r:"10",stroke:"currentColor",strokeWidth:"4"}),(0,t.jsx)("path",{className:"opacity-75",fill:"currentColor",d:"M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"})]})}function a(){return(0,t.jsxs)("div",{className:(0,i.cx)("h-screen","flex items-center justify-center gap-4"),children:[(0,t.jsx)("div",{className:"text-lg font-medium py-2 pr-4 border-r border-r-gray-200",children:"🚅 LiteLLM"}),(0,t.jsxs)("div",{className:"flex items-center justify-center gap-2",children:[(0,t.jsx)(n,{className:"size-4"}),(0,t.jsx)("span",{className:"text-gray-600 text-sm",children:"Loading..."})]})]})}e.s(["UiLoadingSpinner",()=>n],571303),e.s(["default",()=>a],936578)},594542,e=>{"use strict";var t=e.i(843476),i=e.i(954616),r=e.i(602869),n=e.i(612256),a=e.i(936578),o=e.i(268004),l=e.i(161281),s=e.i(321836),c=e.i(827252),d=e.i(295320),u=e.i(560445),m=e.i(464571),p=e.i(175712),g=e.i(808613),h=e.i(311451),f=e.i(282786),b=e.i(199133),y=e.i(770914),v=e.i(898586),x=e.i(618566),$=e.i(271645),S=e.i(283713);function j(){let[e,j]=(0,$.useState)(""),[w,O]=(0,$.useState)(""),[C,E]=(0,$.useState)(!0),{data:I,isLoading:N}=(0,n.useUIConfig)(),k=(0,i.useMutation)({mutationFn:async({username:e,password:t,useV3:i})=>await (0,r.loginCall)(e,t,i)}),z=(0,x.useRouter)(),{workers:L,selectWorker:M}=(0,S.useWorker)(),[R,P]=(0,$.useState)(null);(0,$.useEffect)(()=>{let e=new URLSearchParams(window.location.search).get("worker");e&&P(e)},[]),(0,$.useEffect)(()=>{if(N)return;if(I&&I.admin_ui_disabled)return void E(!1);let e=new URLSearchParams(window.location.search),t=e.get("code"),i=t&&/^[a-zA-Z0-9._~+/=-]+$/.test(t)?t:null;if(i){let t=localStorage.getItem("litellm_worker_url"),n=t&&/^https?:\/\/.+/.test(t)?t:null;(0,r.exchangeLoginCode)(i,n).then(()=>{e.delete("code");let t=e.toString();window.history.replaceState(null,"",window.location.pathname+(t?`?${t}`:"")),z.replace("/ui/?login=success")});return}if(e.has("worker")&&I?.is_control_plane){(0,o.clearTokenCookies)(),E(!1);return}let n=(0,o.getCookieFromDocument)("token");if(n&&!(0,l.isJwtExpired)(n)){let e=(0,s.consumeReturnUrl)();e?z.replace(e):z.replace("/ui");return}if(I&&I.auto_redirect_to_sso){let e=(0,s.getReturnUrl)(),t=`${(0,r.getProxyBaseUrl)()}/sso/key/generate`;e&&(0,s.isValidReturnUrl)(e)&&(t+=`?redirect_to=${encodeURIComponent(e)}`),z.push(t);return}E(!1)},[N,z,I]);let T=k.error instanceof Error?k.error.message:null,_=k.isPending,{Title:G,Text:B,Paragraph:A}=v.Typography;return N||C?(0,t.jsx)(a.default,{}):I&&I.admin_ui_disabled?(0,t.jsx)("div",{className:"min-h-screen flex items-center justify-center bg-gray-50",children:(0,t.jsx)(p.Card,{className:"w-full max-w-lg shadow-md",children:(0,t.jsxs)(y.Space,{direction:"vertical",size:"middle",className:"w-full",children:[(0,t.jsx)("div",{className:"text-center",children:(0,t.jsx)(G,{level:2,children:"🚅 LiteLLM"})}),(0,t.jsx)(u.Alert,{message:"Admin UI Disabled",description:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(A,{className:"text-sm",children:"The Admin UI has been disabled by the administrator. To re-enable it, please update the following environment variable:"}),(0,t.jsx)(A,{className:"text-sm",children:(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"DISABLE_ADMIN_UI=False"})})]}),type:"warning",showIcon:!0})]})})}):(0,t.jsx)("div",{className:"min-h-screen flex items-center justify-center bg-gray-50",children:(0,t.jsxs)(p.Card,{className:"w-full max-w-lg shadow-md",children:[(0,t.jsxs)(y.Space,{direction:"vertical",size:"middle",className:"w-full",children:[(0,t.jsx)("div",{className:"text-center",children:(0,t.jsx)(G,{level:2,children:"🚅 LiteLLM"})}),(0,t.jsxs)("div",{className:"text-center",children:[(0,t.jsx)(G,{level:3,children:"Login"}),(0,t.jsx)(B,{type:"secondary",children:"Access your LiteLLM Admin UI."})]}),(0,t.jsx)(u.Alert,{message:"Default Credentials",description:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsxs)(A,{className:"text-sm",children:["By default, Username is ",(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"admin"})," and Password is your set LiteLLM Proxy",(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"MASTER_KEY"}),"."]}),(0,t.jsxs)(A,{className:"text-sm",children:["Need to set UI credentials or SSO?"," ",(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/proxy/ui",target:"_blank",rel:"noopener noreferrer",children:"Check the documentation"}),"."]})]}),type:"info",icon:(0,t.jsx)(c.InfoCircleOutlined,{}),showIcon:!0}),T&&(0,t.jsx)(u.Alert,{message:T,type:"error",showIcon:!0}),(0,t.jsxs)(g.Form,{onFinish:()=>{let t=L.find(e=>e.worker_id===R);t&&(0,r.switchToWorkerUrl)(t.url),k.mutate({username:e,password:w,useV3:!!t},{onSuccess:e=>{if(t)M(t.worker_id),z.push("/ui/?login=success");else{let t=(0,s.consumeReturnUrl)();t?z.push(t):z.push(e.redirect_url)}},onError:()=>{t&&(0,r.switchToWorkerUrl)(null)}})},layout:"vertical",requiredMark:!1,children:[I?.is_control_plane&&L.length>0&&(0,t.jsx)(g.Form.Item,{label:"Worker",style:{marginBottom:16},children:(0,t.jsx)(b.Select,{value:R||void 0,onChange:e=>P(e),placeholder:"Choose a worker to connect to",size:"large",suffixIcon:(0,t.jsx)(d.CloudServerOutlined,{}),options:L.map(e=>({label:e.name,value:e.worker_id}))})}),(0,t.jsx)(g.Form.Item,{label:"Username",name:"username",rules:[{required:!0,message:"Please enter your username"}],children:(0,t.jsx)(h.Input,{placeholder:"Enter your username",autoComplete:"username",value:e,onChange:e=>j(e.target.value),disabled:_,size:"large",className:"rounded-md border-gray-300"})}),(0,t.jsx)(g.Form.Item,{label:"Password",name:"password",rules:[{required:!0,message:"Please enter your password"}],children:(0,t.jsx)(h.Input.Password,{placeholder:"Enter your password",autoComplete:"current-password",value:w,onChange:e=>O(e.target.value),disabled:_,size:"large"})}),(0,t.jsx)(g.Form.Item,{children:(0,t.jsx)(m.Button,{type:"primary",htmlType:"submit",loading:_,disabled:_,block:!0,size:"large",children:_?"Logging in...":"Login"})}),(0,t.jsx)(g.Form.Item,{children:I?.sso_configured?(0,t.jsx)(m.Button,{disabled:_||!!R&&0===L.length,onClick:()=>{let e=L.find(e=>e.worker_id===R);e&&(localStorage.setItem("litellm_selected_worker_id",R),(0,r.switchToWorkerUrl)(e.url));let t=e?.url??(0,r.getProxyBaseUrl)(),i=encodeURIComponent(window.location.origin+"/ui/login");z.push(`${t}/sso/key/generate?return_to=${i}`)},block:!0,size:"large",children:"Login with SSO"}):(0,t.jsx)(f.Popover,{content:"Please configure SSO to log in with SSO.",trigger:"hover",children:(0,t.jsx)(m.Button,{disabled:!0,block:!0,size:"large",children:"Login with SSO"})})})]})]}),I?.sso_configured&&(0,t.jsx)(u.Alert,{type:"info",showIcon:!0,closable:!0,message:(0,t.jsxs)(B,{children:["Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow upon loading this page. To re-enable auto-redirect-to-SSO, set"," ",(0,t.jsx)(B,{code:!0,children:"AUTO_REDIRECT_UI_LOGIN_TO_SSO=true"})," in your environment configuration."]})})]})})}e.s(["default",0,function(){return(0,t.jsx)(j,{})}],594542)}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/36d1c027ba991a4f.js b/litellm/proxy/_experimental/out/_next/static/chunks/10757c2146f43db4.js similarity index 66% rename from litellm/proxy/_experimental/out/_next/static/chunks/36d1c027ba991a4f.js rename to litellm/proxy/_experimental/out/_next/static/chunks/10757c2146f43db4.js index 091c3b70110..3b6538f90e1 100644 --- a/litellm/proxy/_experimental/out/_next/static/chunks/36d1c027ba991a4f.js +++ b/litellm/proxy/_experimental/out/_next/static/chunks/10757c2146f43db4.js @@ -1,6 +1,6 @@ (globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,127952,869216,368869,e=>{"use strict";var t=e.i(843476),n=e.i(560445),r=e.i(175712);e.i(247167);var l=e.i(271645),a=e.i(343794),o=e.i(908206),i=e.i(242064),s=e.i(517455),d=e.i(150073);let c={xxl:3,xl:3,lg:3,md:3,sm:2,xs:1},u=l.default.createContext({});var f=e.i(876556),m=function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var l=0,r=Object.getOwnPropertySymbols(e);lt.indexOf(r[l])&&Object.prototype.propertyIsEnumerable.call(e,r[l])&&(n[r[l]]=e[r[l]]);return n},p=function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var l=0,r=Object.getOwnPropertySymbols(e);lt.indexOf(r[l])&&Object.prototype.propertyIsEnumerable.call(e,r[l])&&(n[r[l]]=e[r[l]]);return n};let g=e=>{let{itemPrefixCls:t,component:n,span:r,className:o,style:i,labelStyle:s,contentStyle:d,bordered:c,label:f,content:m,colon:p,type:g,styles:h}=e,{classNames:x}=l.useContext(u),v=Object.assign(Object.assign({},s),null==h?void 0:h.label),b=Object.assign(Object.assign({},d),null==h?void 0:h.content);if(c)return l.createElement(n,{colSpan:r,style:i,className:(0,a.default)(o,{[`${t}-item-${g}`]:"label"===g||"content"===g,[null==x?void 0:x.label]:(null==x?void 0:x.label)&&"label"===g,[null==x?void 0:x.content]:(null==x?void 0:x.content)&&"content"===g})},null!=f&&l.createElement("span",{style:v},f),null!=m&&l.createElement("span",{style:b},m));return l.createElement(n,{colSpan:r,style:i,className:(0,a.default)(`${t}-item`,o)},l.createElement("div",{className:`${t}-item-container`},null!=f&&l.createElement("span",{style:v,className:(0,a.default)(`${t}-item-label`,null==x?void 0:x.label,{[`${t}-item-no-colon`]:!p})},f),null!=m&&l.createElement("span",{style:b,className:(0,a.default)(`${t}-item-content`,null==x?void 0:x.content)},m)))};function h(e,{colon:t,prefixCls:n,bordered:r},{component:a,type:o,showLabel:i,showContent:s,labelStyle:d,contentStyle:c,styles:u}){return e.map(({label:e,children:f,prefixCls:m=n,className:p,style:h,labelStyle:x,contentStyle:v,span:b=1,key:y,styles:w},j)=>"string"==typeof a?l.createElement(g,{key:`${o}-${y||j}`,className:p,style:h,styles:{label:Object.assign(Object.assign(Object.assign(Object.assign({},d),null==u?void 0:u.label),x),null==w?void 0:w.label),content:Object.assign(Object.assign(Object.assign(Object.assign({},c),null==u?void 0:u.content),v),null==w?void 0:w.content)},span:b,colon:t,component:a,itemPrefixCls:m,bordered:r,label:i?e:null,content:s?f:null,type:o}):[l.createElement(g,{key:`label-${y||j}`,className:p,style:Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({},d),null==u?void 0:u.label),h),x),null==w?void 0:w.label),span:1,colon:t,component:a[0],itemPrefixCls:m,bordered:r,label:e,type:"label"}),l.createElement(g,{key:`content-${y||j}`,className:p,style:Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({},c),null==u?void 0:u.content),h),v),null==w?void 0:w.content),span:2*b-1,component:a[1],itemPrefixCls:m,bordered:r,content:f,type:"content"})])}let x=e=>{let t=l.useContext(u),{prefixCls:n,vertical:r,row:a,index:o,bordered:i}=e;return r?l.createElement(l.Fragment,null,l.createElement("tr",{key:`label-${o}`,className:`${n}-row`},h(a,e,Object.assign({component:"th",type:"label",showLabel:!0},t))),l.createElement("tr",{key:`content-${o}`,className:`${n}-row`},h(a,e,Object.assign({component:"td",type:"content",showContent:!0},t)))):l.createElement("tr",{key:o,className:`${n}-row`},h(a,e,Object.assign({component:i?["th","td"]:"td",type:"item",showLabel:!0,showContent:!0},t)))};e.i(296059);var v=e.i(915654),b=e.i(183293),y=e.i(246422),w=e.i(838378);let j=(0,y.genStyleHooks)("Descriptions",e=>(e=>{let{componentCls:t,extraColor:n,itemPaddingBottom:r,itemPaddingEnd:l,colonMarginRight:a,colonMarginLeft:o,titleMarginBottom:i}=e;return{[t]:Object.assign(Object.assign(Object.assign({},(0,b.resetComponent)(e)),(e=>{let{componentCls:t,labelBg:n}=e;return{[`&${t}-bordered`]:{[`> ${t}-view`]:{border:`${(0,v.unit)(e.lineWidth)} ${e.lineType} ${e.colorSplit}`,"> table":{tableLayout:"auto"},[`${t}-row`]:{borderBottom:`${(0,v.unit)(e.lineWidth)} ${e.lineType} ${e.colorSplit}`,"&:first-child":{"> th:first-child, > td:first-child":{borderStartStartRadius:e.borderRadiusLG}},"&:last-child":{borderBottom:"none","> th:first-child, > td:first-child":{borderEndStartRadius:e.borderRadiusLG}},[`> ${t}-item-label, > ${t}-item-content`]:{padding:`${(0,v.unit)(e.padding)} ${(0,v.unit)(e.paddingLG)}`,borderInlineEnd:`${(0,v.unit)(e.lineWidth)} ${e.lineType} ${e.colorSplit}`,"&:last-child":{borderInlineEnd:"none"}},[`> ${t}-item-label`]:{color:e.colorTextSecondary,backgroundColor:n,"&::after":{display:"none"}}}},[`&${t}-middle`]:{[`${t}-row`]:{[`> ${t}-item-label, > ${t}-item-content`]:{padding:`${(0,v.unit)(e.paddingSM)} ${(0,v.unit)(e.paddingLG)}`}}},[`&${t}-small`]:{[`${t}-row`]:{[`> ${t}-item-label, > ${t}-item-content`]:{padding:`${(0,v.unit)(e.paddingXS)} ${(0,v.unit)(e.padding)}`}}}}}})(e)),{"&-rtl":{direction:"rtl"},[`${t}-header`]:{display:"flex",alignItems:"center",marginBottom:i},[`${t}-title`]:Object.assign(Object.assign({},b.textEllipsis),{flex:"auto",color:e.titleColor,fontWeight:e.fontWeightStrong,fontSize:e.fontSizeLG,lineHeight:e.lineHeightLG}),[`${t}-extra`]:{marginInlineStart:"auto",color:n,fontSize:e.fontSize},[`${t}-view`]:{width:"100%",borderRadius:e.borderRadiusLG,table:{width:"100%",tableLayout:"fixed",borderCollapse:"collapse"}},[`${t}-row`]:{"> th, > td":{paddingBottom:r,paddingInlineEnd:l},"> th:last-child, > td:last-child":{paddingInlineEnd:0},"&:last-child":{borderBottom:"none","> th, > td":{paddingBottom:0}}},[`${t}-item-label`]:{color:e.labelColor,fontWeight:"normal",fontSize:e.fontSize,lineHeight:e.lineHeight,textAlign:"start","&::after":{content:'":"',position:"relative",top:-.5,marginInline:`${(0,v.unit)(o)} ${(0,v.unit)(a)}`},[`&${t}-item-no-colon::after`]:{content:'""'}},[`${t}-item-no-label`]:{"&::after":{margin:0,content:'""'}},[`${t}-item-content`]:{display:"table-cell",flex:1,color:e.contentColor,fontSize:e.fontSize,lineHeight:e.lineHeight,wordBreak:"break-word",overflowWrap:"break-word"},[`${t}-item`]:{paddingBottom:0,verticalAlign:"top","&-container":{display:"flex",[`${t}-item-label`]:{display:"inline-flex",alignItems:"baseline"},[`${t}-item-content`]:{display:"inline-flex",alignItems:"baseline",minWidth:"1em"}}},"&-middle":{[`${t}-row`]:{"> th, > td":{paddingBottom:e.paddingSM}}},"&-small":{[`${t}-row`]:{"> th, > td":{paddingBottom:e.paddingXS}}}})}})((0,w.mergeToken)(e,{})),e=>({labelBg:e.colorFillAlter,labelColor:e.colorTextTertiary,titleColor:e.colorText,titleMarginBottom:e.fontSizeSM*e.lineHeightSM,itemPaddingBottom:e.padding,itemPaddingEnd:e.padding,colonMarginRight:e.marginXS,colonMarginLeft:e.marginXXS/2,contentColor:e.colorText,extraColor:e.colorText}));var k=function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var l=0,r=Object.getOwnPropertySymbols(e);lt.indexOf(r[l])&&Object.prototype.propertyIsEnumerable.call(e,r[l])&&(n[r[l]]=e[r[l]]);return n};let C=e=>{let t,{prefixCls:n,title:r,extra:g,column:h,colon:v=!0,bordered:b,layout:y,children:w,className:C,rootClassName:S,style:N,size:E,labelStyle:_,contentStyle:O,styles:$,items:T,classNames:I}=e,P=k(e,["prefixCls","title","extra","column","colon","bordered","layout","children","className","rootClassName","style","size","labelStyle","contentStyle","styles","items","classNames"]),{getPrefixCls:M,direction:R,className:L,style:D,classNames:A,styles:K}=(0,i.useComponentConfig)("descriptions"),B=M("descriptions",n),F=(0,d.default)(),z=l.useMemo(()=>{var e;return"number"==typeof h?h:null!=(e=(0,o.matchScreen)(F,Object.assign(Object.assign({},c),h)))?e:3},[F,h]),H=(t=l.useMemo(()=>T||(0,f.default)(w).map(e=>Object.assign(Object.assign({},null==e?void 0:e.props),{key:e.key})),[T,w]),l.useMemo(()=>t.map(e=>{var{span:t}=e,n=m(e,["span"]);return"filled"===t?Object.assign(Object.assign({},n),{filled:!0}):Object.assign(Object.assign({},n),{span:"number"==typeof t?t:(0,o.matchScreen)(F,t)})}),[t,F])),V=(0,s.default)(E),W=((e,t)=>{let[n,r]=(0,l.useMemo)(()=>{let n,r,l,a;return n=[],r=[],l=!1,a=0,t.filter(e=>e).forEach(t=>{let{filled:o}=t,i=p(t,["filled"]);if(o){r.push(i),n.push(r),r=[],a=0;return}let s=e-a;(a+=t.span||1)>=e?(a>e?(l=!0,r.push(Object.assign(Object.assign({},i),{span:s}))):r.push(i),n.push(r),r=[],a=0):r.push(i)}),r.length>0&&n.push(r),[n=n.map(t=>{let n=t.reduce((e,t)=>e+(t.span||1),0);if(n({labelStyle:_,contentStyle:O,styles:{content:Object.assign(Object.assign({},K.content),null==$?void 0:$.content),label:Object.assign(Object.assign({},K.label),null==$?void 0:$.label)},classNames:{label:(0,a.default)(A.label,null==I?void 0:I.label),content:(0,a.default)(A.content,null==I?void 0:I.content)}}),[_,O,$,I,A,K]);return U(l.createElement(u.Provider,{value:X},l.createElement("div",Object.assign({className:(0,a.default)(B,L,A.root,null==I?void 0:I.root,{[`${B}-${V}`]:V&&"default"!==V,[`${B}-bordered`]:!!b,[`${B}-rtl`]:"rtl"===R},C,S,q,G),style:Object.assign(Object.assign(Object.assign(Object.assign({},D),K.root),null==$?void 0:$.root),N)},P),(r||g)&&l.createElement("div",{className:(0,a.default)(`${B}-header`,A.header,null==I?void 0:I.header),style:Object.assign(Object.assign({},K.header),null==$?void 0:$.header)},r&&l.createElement("div",{className:(0,a.default)(`${B}-title`,A.title,null==I?void 0:I.title),style:Object.assign(Object.assign({},K.title),null==$?void 0:$.title)},r),g&&l.createElement("div",{className:(0,a.default)(`${B}-extra`,A.extra,null==I?void 0:I.extra),style:Object.assign(Object.assign({},K.extra),null==$?void 0:$.extra)},g)),l.createElement("div",{className:`${B}-view`},l.createElement("table",null,l.createElement("tbody",null,W.map((e,t)=>l.createElement(x,{key:t,index:t,colon:v,prefixCls:B,vertical:"vertical"===y,bordered:b,row:e}))))))))};C.Item=({children:e})=>e,e.s(["Descriptions",0,C],869216);var S=e.i(311451),N=e.i(212931),E=e.i(898586),_=e.i(868297),O=e.i(732961),$=e.i(289882),T=e.i(170517),I=e.i(628882),P=e.i(320890),M=e.i(104458),R=e.i(722319),L=e.i(8398),D=e.i(279728);e.i(765846);var A=e.i(602716),K=e.i(328052);e.i(262370);var B=e.i(135551);let F=(e,t)=>new B.FastColor(e).setA(t).toRgbString(),z=(e,t)=>new B.FastColor(e).lighten(t).toHexString(),H=e=>{let t=(0,A.generate)(e,{theme:"dark"});return{1:t[0],2:t[1],3:t[2],4:t[3],5:t[6],6:t[5],7:t[4],8:t[6],9:t[5],10:t[4]}},V=(e,t)=>{let n=e||"#000",r=t||"#fff";return{colorBgBase:n,colorTextBase:r,colorText:F(r,.85),colorTextSecondary:F(r,.65),colorTextTertiary:F(r,.45),colorTextQuaternary:F(r,.25),colorFill:F(r,.18),colorFillSecondary:F(r,.12),colorFillTertiary:F(r,.08),colorFillQuaternary:F(r,.04),colorBgSolid:F(r,.95),colorBgSolidHover:F(r,1),colorBgSolidActive:F(r,.9),colorBgElevated:z(n,12),colorBgContainer:z(n,8),colorBgLayout:z(n,0),colorBgSpotlight:z(n,26),colorBgBlur:F(r,.04),colorBorder:z(n,26),colorBorderSecondary:z(n,19)}},W={defaultSeed:P.defaultConfig.token,useToken:function(){let[e,t,n]=(0,M.useToken)();return{theme:e,token:t,hashId:n}},defaultAlgorithm:R.default,darkAlgorithm:(e,t)=>{let n=Object.keys(T.defaultPresetColors).map(t=>{let n=(0,A.generate)(e[t],{theme:"dark"});return Array.from({length:10},()=>1).reduce((e,r,l)=>(e[`${t}-${l+1}`]=n[l],e[`${t}${l+1}`]=n[l],e),{})}).reduce((e,t)=>e=Object.assign(Object.assign({},e),t),{}),r=null!=t?t:(0,R.default)(e),l=(0,K.default)(e,{generateColorPalettes:H,generateNeutralColorPalettes:V});return Object.assign(Object.assign(Object.assign(Object.assign({},r),n),l),{colorPrimaryBg:l.colorPrimaryBorder,colorPrimaryBgHover:l.colorPrimaryBorderHover})},compactAlgorithm:(e,t)=>{let n=null!=t?t:(0,R.default)(e),r=n.fontSizeSM,l=n.controlHeight-4;return Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({},n),function(e){let{sizeUnit:t,sizeStep:n}=e,r=n-2;return{sizeXXL:t*(r+10),sizeXL:t*(r+6),sizeLG:t*(r+2),sizeMD:t*(r+2),sizeMS:t*(r+1),size:t*r,sizeSM:t*r,sizeXS:t*(r-1),sizeXXS:t*(r-1)}}(null!=t?t:e)),(0,D.default)(r)),{controlHeight:l}),(0,L.default)(Object.assign(Object.assign({},n),{controlHeight:l})))},getDesignToken:e=>{let t=(null==e?void 0:e.algorithm)?(0,_.createTheme)(e.algorithm):$.default,n=Object.assign(Object.assign({},T.default),null==e?void 0:e.token);return(0,O.getComputedToken)(n,{override:null==e?void 0:e.token},t,I.default)},defaultConfig:P.defaultConfig,_internalContext:P.DesignTokenContext};e.s(["theme",0,W],368869);var U=e.i(270377);function q({isOpen:e,title:a,alertMessage:o,message:i,resourceInformationTitle:s,resourceInformation:d,onCancel:c,onOk:u,confirmLoading:f,requiredConfirmation:m}){let{Title:p,Text:g}=E.Typography,{token:h}=W.useToken(),[x,v]=(0,l.useState)("");return(0,l.useEffect)(()=>{e&&v("")},[e]),(0,t.jsx)(N.Modal,{title:a,open:e,onOk:u,onCancel:c,confirmLoading:f,okText:f?"Deleting...":"Delete",cancelText:"Cancel",okButtonProps:{danger:!0,disabled:!!m&&x!==m||f},cancelButtonProps:{disabled:f},children:(0,t.jsxs)("div",{className:"space-y-4",children:[o&&(0,t.jsx)(n.Alert,{message:o,type:"warning"}),(0,t.jsx)(r.Card,{title:s,className:"mt-4",styles:{body:{padding:"16px"},header:{backgroundColor:h.colorErrorBg,borderColor:h.colorErrorBorder}},style:{backgroundColor:h.colorErrorBg,borderColor:h.colorErrorBorder},children:(0,t.jsx)(C,{column:1,size:"small",children:d&&d.map(({label:e,value:n,...r})=>(0,t.jsx)(C.Item,{label:(0,t.jsx)("span",{className:"font-semibold",children:e}),children:(0,t.jsx)(g,{...r,children:n??"-"})},e))})}),(0,t.jsx)("div",{children:(0,t.jsx)(g,{children:i})}),m&&(0,t.jsxs)("div",{className:"mb-6 mt-4 pt-4 border-t border-gray-200 dark:border-gray-700",children:[(0,t.jsxs)(g,{className:"block text-base font-medium text-gray-700 dark:text-gray-300 mb-2",children:[(0,t.jsx)(g,{children:"Type "}),(0,t.jsx)(g,{strong:!0,type:"danger",children:m}),(0,t.jsx)(g,{children:" to confirm deletion:"})]}),(0,t.jsx)(S.Input,{value:x,onChange:e=>v(e.target.value),placeholder:m,className:"rounded-md",prefix:(0,t.jsx)(U.ExclamationCircleOutlined,{style:{color:h.colorError}}),autoFocus:!0})]})]})})}e.s(["default",()=>q],127952)},950724,(e,t,n)=>{t.exports=function(e){var t=typeof e;return null!=e&&("object"==t||"function"==t)}},100236,(e,t,n)=>{t.exports=e.g&&e.g.Object===Object&&e.g},139088,(e,t,n)=>{var r=e.r(100236),l="object"==typeof self&&self&&self.Object===Object&&self;t.exports=r||l||Function("return this")()},631926,(e,t,n)=>{var r=e.r(139088);t.exports=function(){return r.Date.now()}},748891,(e,t,n)=>{var r=/\s/;t.exports=function(e){for(var t=e.length;t--&&r.test(e.charAt(t)););return t}},830364,(e,t,n)=>{var r=e.r(748891),l=/^\s+/;t.exports=function(e){return e?e.slice(0,r(e)+1).replace(l,""):e}},630353,(e,t,n)=>{t.exports=e.r(139088).Symbol},243436,(e,t,n)=>{var r=e.r(630353),l=Object.prototype,a=l.hasOwnProperty,o=l.toString,i=r?r.toStringTag:void 0;t.exports=function(e){var t=a.call(e,i),n=e[i];try{e[i]=void 0;var r=!0}catch(e){}var l=o.call(e);return r&&(t?e[i]=n:delete e[i]),l}},223243,(e,t,n)=>{var r=Object.prototype.toString;t.exports=function(e){return r.call(e)}},377684,(e,t,n)=>{var r=e.r(630353),l=e.r(243436),a=e.r(223243),o=r?r.toStringTag:void 0;t.exports=function(e){return null==e?void 0===e?"[object Undefined]":"[object Null]":o&&o in Object(e)?l(e):a(e)}},877289,(e,t,n)=>{t.exports=function(e){return null!=e&&"object"==typeof e}},361884,(e,t,n)=>{var r=e.r(377684),l=e.r(877289);t.exports=function(e){return"symbol"==typeof e||l(e)&&"[object Symbol]"==r(e)}},773759,(e,t,n)=>{var r=e.r(830364),l=e.r(950724),a=e.r(361884),o=0/0,i=/^[-+]0x[0-9a-f]+$/i,s=/^0b[01]+$/i,d=/^0o[0-7]+$/i,c=parseInt;t.exports=function(e){if("number"==typeof e)return e;if(a(e))return o;if(l(e)){var t="function"==typeof e.valueOf?e.valueOf():e;e=l(t)?t+"":t}if("string"!=typeof e)return 0===e?e:+e;e=r(e);var n=s.test(e);return n||d.test(e)?c(e.slice(2),n?2:8):i.test(e)?o:+e}},374009,(e,t,n)=>{var r=e.r(950724),l=e.r(631926),a=e.r(773759),o=Math.max,i=Math.min;t.exports=function(e,t,n){var s,d,c,u,f,m,p=0,g=!1,h=!1,x=!0;if("function"!=typeof e)throw TypeError("Expected a function");function v(t){var n=s,r=d;return s=d=void 0,p=t,u=e.apply(r,n)}function b(e){var n=e-m,r=e-p;return void 0===m||n>=t||n<0||h&&r>=c}function y(){var e,n,r,a=l();if(b(a))return w(a);f=setTimeout(y,(e=a-m,n=a-p,r=t-e,h?i(r,c-n):r))}function w(e){return(f=void 0,x&&s)?v(e):(s=d=void 0,u)}function j(){var e,n=l(),r=b(n);if(s=arguments,d=this,m=n,r){if(void 0===f)return p=e=m,f=setTimeout(y,t),g?v(e):u;if(h)return clearTimeout(f),f=setTimeout(y,t),v(m)}return void 0===f&&(f=setTimeout(y,t)),u}return t=a(t)||0,r(n)&&(g=!!n.leading,c=(h="maxWait"in n)?o(a(n.maxWait)||0,t):c,x="trailing"in n?!!n.trailing:x),j.cancel=function(){void 0!==f&&clearTimeout(f),p=0,s=m=d=f=void 0},j.flush=function(){return void 0===f?u:w(l())},j}},436289,503269,214520,814379,992704,684653,877891,401141,952744,605083,101852,249578,571616,e=>{"use strict";var t=e.i(271645);function n(e,t){return null!==e&&null!==t&&"object"==typeof e&&"object"==typeof t&&"id"in e&&"id"in t?e.id===t.id:e===t}function r(e=n){return(0,t.useCallback)((t,n)=>"string"==typeof e?(null==t?void 0:t[e])===(null==n?void 0:n[e]):e(t,n),[e])}e.s(["useByComparator",()=>r],436289);var l=e.i(914189);function a(e,n,r){let[a,o]=(0,t.useState)(r),i=void 0!==e,s=(0,t.useRef)(i),d=(0,t.useRef)(!1),c=(0,t.useRef)(!1);return!i||s.current||d.current?i||!s.current||c.current||(c.current=!0,s.current=i,console.error("A component is changing from controlled to uncontrolled. This may be caused by the value changing from a defined value to undefined, which should not happen.")):(d.current=!0,s.current=i,console.error("A component is changing from uncontrolled to controlled. This may be caused by the value changing from undefined to a defined value, which should not happen.")),[i?e:a,(0,l.useEvent)(e=>(i||o(e),null==n?void 0:n(e)))]}function o(e){let[n]=(0,t.useState)(e);return n}e.s(["useControllable",()=>a],503269),e.s(["useDefaultValue",()=>o],214520);var i=e.i(835696);function s(e,n){let r=(0,t.useRef)({left:0,top:0});if((0,i.useIsoMorphicEffect)(()=>{if(!n)return;let e=n.getBoundingClientRect();e&&(r.current=e)},[e,n]),null==n||!e||n===document.activeElement)return!1;let l=n.getBoundingClientRect();return l.top!==r.current.top||l.left!==r.current.left}function d(e,n=!1){let[r,l]=(0,t.useReducer)(()=>({}),{}),a=(0,t.useMemo)(()=>(function(e){if(null===e)return{width:0,height:0};let{width:t,height:n}=e.getBoundingClientRect();return{width:t,height:n}})(e),[e,r]);return(0,i.useIsoMorphicEffect)(()=>{if(!e)return;let t=new ResizeObserver(l);return t.observe(e),()=>{t.disconnect()}},[e]),n?{width:`${a.width}px`,height:`${a.height}px`}:a}e.s(["useDidElementMove",()=>s],814379),e.s(["useElementSize",()=>d],992704);var c=e.i(544508),u=e.i(402155);class f extends Map{constructor(e){super(),this.factory=e}get(e){let t=super.get(e);return void 0===t&&(t=this.factory(e),this.set(e,t)),t}}function m(e,t){let n=e(),r=new Set;return{getSnapshot:()=>n,subscribe:e=>(r.add(e),()=>r.delete(e)),dispatch(e,...l){let a=t[e].call(n,...l);a&&(n=a,r.forEach(e=>e()))}}}function p(e){return(0,t.useSyncExternalStore)(e.subscribe,e.getSnapshot,e.getSnapshot)}let g=new f(()=>m(()=>[],{ADD(e){return this.includes(e)?this:[...this,e]},REMOVE(e){let t=this.indexOf(e);if(-1===t)return this;let n=this.slice();return n.splice(t,1),n}}));function h(e,n){let r=g.get(n),l=(0,t.useId)(),a=p(r);if((0,i.useIsoMorphicEffect)(()=>{if(e)return r.dispatch("ADD",l),()=>r.dispatch("REMOVE",l)},[r,e]),!e)return!1;let o=a.indexOf(l),s=a.length;return -1===o&&(o=s,s+=1),o===s-1}let x=new Map,v=new Map;function b(e){var t;let n=null!=(t=v.get(e))?t:0;return v.set(e,n+1),0!==n||(x.set(e,{"aria-hidden":e.getAttribute("aria-hidden"),inert:e.inert}),e.setAttribute("aria-hidden","true"),e.inert=!0),()=>(function(e){var t;let n=null!=(t=v.get(e))?t:1;if(1===n?v.delete(e):v.set(e,n-1),1!==n)return;let r=x.get(e);r&&(null===r["aria-hidden"]?e.removeAttribute("aria-hidden"):e.setAttribute("aria-hidden",r["aria-hidden"]),e.inert=r.inert,x.delete(e))})(e)}function y(e,{allowed:t,disallowed:n}={}){let r=h(e,"inert-others");(0,i.useIsoMorphicEffect)(()=>{var e,l;if(!r)return;let a=(0,c.disposables)();for(let t of null!=(e=null==n?void 0:n())?e:[])t&&a.add(b(t));let o=null!=(l=null==t?void 0:t())?l:[];for(let e of o){if(!e)continue;let t=(0,u.getOwnerDocument)(e);if(!t)continue;let n=e.parentElement;for(;n&&n!==t.body;){for(let e of n.children)o.some(t=>e.contains(t))||a.add(b(e));n=n.parentElement}}return a.dispose},[r,t,n])}e.s(["useInertOthers",()=>y],684653);var w=e.i(941444);function j(e,n,r){let l=(0,w.useLatestValue)(e=>{let t=e.getBoundingClientRect();0===t.x&&0===t.y&&0===t.width&&0===t.height&&r()});(0,t.useEffect)(()=>{if(!e)return;let t=null===n?null:n instanceof HTMLElement?n:n.current;if(!t)return;let r=(0,c.disposables)();if("u">typeof ResizeObserver){let e=new ResizeObserver(()=>l.current(t));e.observe(t),r.add(()=>e.disconnect())}if("u">typeof IntersectionObserver){let e=new IntersectionObserver(()=>l.current(t));e.observe(t),r.add(()=>e.disconnect())}return()=>r.dispose()},[n,l,e])}e.s(["useOnDisappear",()=>j],877891);var k=e.i(652265);function C(){return/iPhone/gi.test(window.navigator.platform)||/Mac/gi.test(window.navigator.platform)&&window.navigator.maxTouchPoints>0}function S(e,n,r,l){let a=(0,w.useLatestValue)(r);(0,t.useEffect)(()=>{if(e)return document.addEventListener(n,t,l),()=>document.removeEventListener(n,t,l);function t(e){a.current(e)}},[e,n,l])}function N(e,n,r,l){let a=(0,w.useLatestValue)(r);(0,t.useEffect)(()=>{if(e)return window.addEventListener(n,t,l),()=>window.removeEventListener(n,t,l);function t(e){a.current(e)}},[e,n,l])}function E(e,n,r){let l=h(e,"outside-click"),a=(0,w.useLatestValue)(r),o=(0,t.useCallback)(function(e,t){if(e.defaultPrevented)return;let r=t(e);if(null!==r&&r.getRootNode().contains(r)&&r.isConnected){for(let t of function e(t){return"function"==typeof t?e(t()):Array.isArray(t)||t instanceof Set?t:[t]}(n))if(null!==t&&(t.contains(r)||e.composed&&e.composedPath().includes(t)))return;return(0,k.isFocusableElement)(r,k.FocusableMode.Loose)||-1===r.tabIndex||e.preventDefault(),a.current(e,r)}},[a,n]),i=(0,t.useRef)(null);S(l,"pointerdown",e=>{var t,n;i.current=(null==(n=null==(t=e.composedPath)?void 0:t.call(e))?void 0:n[0])||e.target},!0),S(l,"mousedown",e=>{var t,n;i.current=(null==(n=null==(t=e.composedPath)?void 0:t.call(e))?void 0:n[0])||e.target},!0),S(l,"click",e=>{C()||/Android/gi.test(window.navigator.userAgent)||i.current&&(o(e,()=>i.current),i.current=null)},!0);let s=(0,t.useRef)({x:0,y:0});S(l,"touchstart",e=>{s.current.x=e.touches[0].clientX,s.current.y=e.touches[0].clientY},!0),S(l,"touchend",e=>{let t={x:e.changedTouches[0].clientX,y:e.changedTouches[0].clientY};if(!(Math.abs(t.x-s.current.x)>=30||Math.abs(t.y-s.current.y)>=30))return o(e,()=>e.target instanceof HTMLElement?e.target:null)},!0),N(l,"blur",e=>o(e,()=>window.document.activeElement instanceof HTMLIFrameElement?window.document.activeElement:null),!0)}function _(...e){return(0,t.useMemo)(()=>(0,u.getOwnerDocument)(...e),[...e])}e.s(["useWindowEvent",()=>N],401141),e.s(["useOutsideClick",()=>E],952744),e.s(["useOwnerDocument",()=>_],605083);let O=m(()=>new Map,{PUSH(e,t){var n;let r=null!=(n=this.get(e))?n:{doc:e,count:0,d:(0,c.disposables)(),meta:new Set};return r.count++,r.meta.add(t),this.set(e,r),this},POP(e,t){let n=this.get(e);return n&&(n.count--,n.meta.delete(t)),this},SCROLL_PREVENT({doc:e,d:t,meta:n}){let r,l={doc:e,d:t,meta:function(e){let t={};for(let n of e)Object.assign(t,n(t));return t}(n)},a=[C()?{before({doc:e,d:t,meta:n}){function r(e){return n.containers.flatMap(e=>e()).some(t=>t.contains(e))}t.microTask(()=>{var n;if("auto"!==window.getComputedStyle(e.documentElement).scrollBehavior){let n=(0,c.disposables)();n.style(e.documentElement,"scrollBehavior","auto"),t.add(()=>t.microTask(()=>n.dispose()))}let l=null!=(n=window.scrollY)?n:window.pageYOffset,a=null;t.addEventListener(e,"click",t=>{if(t.target instanceof HTMLElement)try{let n=t.target.closest("a");if(!n)return;let{hash:l}=new URL(n.href),o=e.querySelector(l);o&&!r(o)&&(a=o)}catch{}},!0),t.addEventListener(e,"touchstart",e=>{if(e.target instanceof HTMLElement)if(r(e.target)){let n=e.target;for(;n.parentElement&&r(n.parentElement);)n=n.parentElement;t.style(n,"overscrollBehavior","contain")}else t.style(e.target,"touchAction","none")}),t.addEventListener(e,"touchmove",e=>{if(e.target instanceof HTMLElement&&"INPUT"!==e.target.tagName)if(r(e.target)){let t=e.target;for(;t.parentElement&&""!==t.dataset.headlessuiPortal&&!(t.scrollHeight>t.clientHeight||t.scrollWidth>t.clientWidth);)t=t.parentElement;""===t.dataset.headlessuiPortal&&e.preventDefault()}else e.preventDefault()},{passive:!1}),t.add(()=>{var e;l!==(null!=(e=window.scrollY)?e:window.pageYOffset)&&window.scrollTo(0,l),a&&a.isConnected&&(a.scrollIntoView({block:"nearest"}),a=null)})})}}:{},{before({doc:e}){var t;let n=e.documentElement;r=Math.max(0,(null!=(t=e.defaultView)?t:window).innerWidth-n.clientWidth)},after({doc:e,d:t}){let n=e.documentElement,l=Math.max(0,n.clientWidth-n.offsetWidth),a=Math.max(0,r-l);t.style(n,"paddingRight",`${a}px`)}},{before({doc:e,d:t}){t.style(e.documentElement,"overflow","hidden")}}];a.forEach(({before:e})=>null==e?void 0:e(l)),a.forEach(({after:e})=>null==e?void 0:e(l))},SCROLL_ALLOW({d:e}){e.dispose()},TEARDOWN({doc:e}){this.delete(e)}});function $(e,t,n=()=>[document.body]){!function(e,t,n=()=>({containers:[]})){let r=p(O),l=t?r.get(t):void 0;l&&l.count,(0,i.useIsoMorphicEffect)(()=>{if(!(!t||!e))return O.dispatch("PUSH",t,n),()=>O.dispatch("POP",t,n)},[e,t])}(h(e,"scroll-lock"),t,e=>{var t;return{containers:[...null!=(t=e.containers)?t:[],n]}})}O.subscribe(()=>{let e=O.getSnapshot(),t=new Map;for(let[n]of e)t.set(n,n.documentElement.style.overflow);for(let n of e.values()){let e="hidden"===t.get(n.doc),r=0!==n.count;(r&&!e||!r&&e)&&O.dispatch(n.count>0?"SCROLL_PREVENT":"SCROLL_ALLOW",n),0===n.count&&O.dispatch("TEARDOWN",n)}}),e.s(["useScrollLock",()=>$],101852);let T=/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g;function I(e){var t,n;let r=null!=(t=e.innerText)?t:"",l=e.cloneNode(!0);if(!(l instanceof HTMLElement))return r;let a=!1;for(let e of l.querySelectorAll('[hidden],[aria-hidden],[role="img"]'))e.remove(),a=!0;let o=a?null!=(n=l.innerText)?n:"":r;return T.test(o)&&(o=o.replace(T,"")),o}function P(e){let n=(0,t.useRef)(""),r=(0,t.useRef)("");return(0,l.useEvent)(()=>{let t=e.current;if(!t)return"";let l=t.innerText;if(n.current===l)return r.current;let a=(function(e){let t=e.getAttribute("aria-label");if("string"==typeof t)return t.trim();let n=e.getAttribute("aria-labelledby");if(n){let e=n.split(" ").map(e=>{let t=document.getElementById(e);if(t){let e=t.getAttribute("aria-label");return"string"==typeof e?e.trim():I(t).trim()}return null}).filter(Boolean);if(e.length>0)return e.join(", ")}return I(e).trim()})(t).trim().toLowerCase();return n.current=l,r.current=a,a})}function M(e){return[e.screenX,e.screenY]}function R(){let e=(0,t.useRef)([-1,-1]);return{wasMoved(t){let n=M(t);return(e.current[0]!==n[0]||e.current[1]!==n[1])&&(e.current=n,!0)},update(t){e.current=M(t)}}}e.s(["useTextValue",()=>P],249578),e.s(["useTrackedPointer",()=>R],571616)},83733,e=>{"use strict";let t;var n,r,l=e.i(247167),a=e.i(271645),o=e.i(544508),i=e.i(746725),s=e.i(835696);void 0!==l.default&&"u">typeof globalThis&&"u">typeof Element&&(null==(n=null==l.default?void 0:l.default.env)?void 0:n.NODE_ENV)==="test"&&void 0===(null==(r=null==Element?void 0:Element.prototype)?void 0:r.getAnimations)&&(Element.prototype.getAnimations=function(){return console.warn(["Headless UI has polyfilled `Element.prototype.getAnimations` for your tests.","Please install a proper polyfill e.g. `jsdom-testing-mocks`, to silence these warnings.","","Example usage:","```js","import { mockAnimationsApi } from 'jsdom-testing-mocks'","mockAnimationsApi()","```"].join(` `)),[]});var d=((t=d||{})[t.None=0]="None",t[t.Closed=1]="Closed",t[t.Enter=2]="Enter",t[t.Leave=4]="Leave",t);function c(e){let t={};for(let n in e)!0===e[n]&&(t[`data-${n}`]="");return t}function u(e,t,n,r){let[l,d]=(0,a.useState)(n),{hasFlag:c,addFlag:u,removeFlag:f}=function(e=0){let[t,n]=(0,a.useState)(e),r=(0,a.useCallback)(e=>n(e),[t]),l=(0,a.useCallback)(e=>n(t=>t|e),[t]),o=(0,a.useCallback)(e=>(t&e)===e,[t]);return{flags:t,setFlag:r,addFlag:l,hasFlag:o,removeFlag:(0,a.useCallback)(e=>n(t=>t&~e),[n]),toggleFlag:(0,a.useCallback)(e=>n(t=>t^e),[n])}}(e&&l?3:0),m=(0,a.useRef)(!1),p=(0,a.useRef)(!1),g=(0,i.useDisposables)();return(0,s.useIsoMorphicEffect)(()=>{var l;if(e){if(n&&d(!0),!t){n&&u(3);return}return null==(l=null==r?void 0:r.start)||l.call(r,n),function(e,{prepare:t,run:n,done:r,inFlight:l}){let a=(0,o.disposables)();return function(e,{inFlight:t,prepare:n}){if(null!=t&&t.current)return n();let r=e.style.transition;e.style.transition="none",n(),e.offsetHeight,e.style.transition=r}(e,{prepare:t,inFlight:l}),a.nextFrame(()=>{n(),a.requestAnimationFrame(()=>{a.add(function(e,t){var n,r;let l=(0,o.disposables)();if(!e)return l.dispose;let a=!1;l.add(()=>{a=!0});let i=null!=(r=null==(n=e.getAnimations)?void 0:n.call(e).filter(e=>e instanceof CSSTransition))?r:[];return 0===i.length?t():Promise.allSettled(i.map(e=>e.finished)).then(()=>{a||t()}),l.dispose}(e,r))})}),a.dispose}(t,{inFlight:m,prepare(){p.current?p.current=!1:p.current=m.current,m.current=!0,p.current||(n?(u(3),f(4)):(u(4),f(2)))},run(){p.current?n?(f(3),u(4)):(f(4),u(3)):n?f(1):u(1)},done(){var e;p.current&&"function"==typeof t.getAnimations&&t.getAnimations().length>0||(m.current=!1,f(7),n||d(!1),null==(e=null==r?void 0:r.end)||e.call(r,n))}})}},[e,n,t,g]),e?[l,{closed:c(1),enter:c(2),leave:c(4),transition:c(2)||c(4)}]:[n,{closed:void 0,enter:void 0,leave:void 0,transition:void 0}]}e.s(["transitionDataAttributes",()=>c,"useTransition",()=>u],83733)},601893,919751,694421,140721,904016,942803,e=>{"use strict";var t=e.i(271645);let n=(0,t.createContext)(void 0);function r(){return(0,t.useContext)(n)}e.s(["useDisabled",()=>r],601893);var l=e.i(953760),a=e.i(174080),o="u">typeof document?t.useLayoutEffect:function(){};function i(e,t){let n,r,l;if(e===t)return!0;if(typeof e!=typeof t)return!1;if("function"==typeof e&&e.toString()===t.toString())return!0;if(e&&t&&"object"==typeof e){if(Array.isArray(e)){if((n=e.length)!==t.length)return!1;for(r=n;0!=r--;)if(!i(e[r],t[r]))return!1;return!0}if((n=(l=Object.keys(e)).length)!==Object.keys(t).length)return!1;for(r=n;0!=r--;)if(!({}).hasOwnProperty.call(t,l[r]))return!1;for(r=n;0!=r--;){let n=l[r];if(("_owner"!==n||!e.$$typeof)&&!i(e[n],t[n]))return!1}return!0}return e!=e&&t!=t}function s(e){return"u"{n.current=e}),n}let u=(e,t)=>({...(0,l.offset)(e),options:[e,t]});e.i(247167);var f=e.i(229315),m=e.i(343084);e.i(397126);let p={...t},g=p.useInsertionEffect||(e=>e());function h(e){let n=t.useRef(()=>{});return g(()=>{n.current=e}),t.useCallback(function(){for(var e=arguments.length,t=Array(e),r=0;rtypeof document?t.useLayoutEffect:t.useEffect;let v=!1,b=0,y=()=>"floating-ui-"+Math.random().toString(36).slice(2,6)+b++,w=p.useId||function(){let[e,n]=t.useState(()=>v?y():void 0);return x(()=>{null==e&&n(y())},[]),t.useEffect(()=>{v=!0},[]),e},j=t.createContext(null),k=t.createContext(null),C="active",S="selected";function N(e,t,n){let r=new Map,l="item"===n,a=e;if(l&&e){let{[C]:t,[S]:n,...r}=e;a=r}return{..."floating"===n&&{tabIndex:-1,"data-floating-ui-focusable":""},...a,...t.map(t=>{let r=t?t[n]:null;return"function"==typeof r?e?r(e):null:r}).concat(e).reduce((e,t)=>(t&&Object.entries(t).forEach(t=>{let[n,a]=t;if(!(l&&[C,S].includes(n)))if(0===n.indexOf("on")){if(r.has(n)||r.set(n,[]),"function"==typeof a){var o;null==(o=r.get(n))||o.push(a),e[n]=function(){for(var e,t=arguments.length,l=Array(t),a=0;ae(...l)).find(e=>void 0!==e)}}}else e[n]=a}),e),{})}}function E(e,t){return{...e,rects:{...e.rects,floating:{...e.rects.floating,height:t}}}}var _=e.i(746725),O=e.i(914189),$=e.i(835696);let T=(0,t.createContext)({styles:void 0,setReference:()=>{},setFloating:()=>{},getReferenceProps:()=>({}),getFloatingProps:()=>({}),slot:{}});T.displayName="FloatingContext";let I=(0,t.createContext)(null);function P(e){return(0,t.useMemo)(()=>e?"string"==typeof e?{to:e}:e:null,[e])}function M(){return(0,t.useContext)(T).setReference}function R(){return(0,t.useContext)(T).getReferenceProps}function L(){let{getFloatingProps:e,slot:n}=(0,t.useContext)(T);return(0,t.useCallback)((...t)=>Object.assign({},e(...t),{"data-anchor":n.anchor}),[e,n])}function D(e=null){!1===e&&(e=null),"string"==typeof e&&(e={to:e});let n=(0,t.useContext)(I),r=(0,t.useMemo)(()=>e,[JSON.stringify(e,(e,t)=>{var n;return null!=(n=null==t?void 0:t.outerHTML)?n:t})]);(0,$.useIsoMorphicEffect)(()=>{null==n||n(null!=r?r:null)},[n,r]);let l=(0,t.useContext)(T);return(0,t.useMemo)(()=>[l.setFloating,e?l.styles:{}],[l.setFloating,e,l.styles])}function A({children:e,enabled:n=!0}){var r,p,g,v,b,y,C;let S,_,P,M,R,L,D,A,B,F,z,H,V,W,U,q,[G,X]=(0,t.useState)(null),[Q,Y]=(0,t.useState)(0),J=(0,t.useRef)(null),[Z,ee]=(0,t.useState)(null);p=Z,(0,$.useIsoMorphicEffect)(()=>{if(!p)return;let e=new MutationObserver(()=>{let e=window.getComputedStyle(p).maxHeight,t=parseFloat(e);if(isNaN(t))return;let n=parseInt(e);isNaN(n)||t!==n&&(p.style.maxHeight=`${Math.ceil(t)}px`)});return e.observe(p,{attributes:!0,attributeFilter:["style"]}),()=>{e.disconnect()}},[p]);let et=n&&null!==G&&null!==Z,{to:en="bottom",gap:er=0,offset:el=0,padding:ea=0,inner:eo}=(g=G,v=Z,S=K(null!=(b=null==g?void 0:g.gap)?b:"var(--anchor-gap, 0)",v),_=K(null!=(y=null==g?void 0:g.offset)?y:"var(--anchor-offset, 0)",v),P=K(null!=(C=null==g?void 0:g.padding)?C:"var(--anchor-padding, 0)",v),{...g,gap:S,offset:_,padding:P}),[ei,es="center"]=en.split(" ");(0,$.useIsoMorphicEffect)(()=>{et&&Y(0)},[et]);let{refs:ed,floatingStyles:ec,context:eu}=function(e){void 0===e&&(e={});let{nodeId:n}=e,r=function(e){var n;let{open:r=!1,onOpenChange:l,elements:a}=e,o=w(),i=t.useRef({}),[s]=t.useState(()=>{let e;return e=new Map,{emit(t,n){var r;null==(r=e.get(t))||r.forEach(e=>e(n))},on(t,n){e.set(t,[...e.get(t)||[],n])},off(t,n){var r;e.set(t,(null==(r=e.get(t))?void 0:r.filter(e=>e!==n))||[])}}}),d=null!=((null==(n=t.useContext(j))?void 0:n.id)||null),[c,u]=t.useState(a.reference),f=h((e,t,n)=>{i.current.openEvent=e?t:void 0,s.emit("openchange",{open:e,event:t,reason:n,nested:d}),null==l||l(e,t,n)}),m=t.useMemo(()=>({setPositionReference:u}),[]),p=t.useMemo(()=>({reference:c||a.reference||null,floating:a.floating||null,domReference:a.reference}),[c,a.reference,a.floating]);return t.useMemo(()=>({dataRef:i,open:r,onOpenChange:f,elements:p,events:s,floatingId:o,refs:m}),[r,f,p,s,o,m])}({...e,elements:{reference:null,floating:null,...e.elements}}),u=e.rootContext||r,m=u.elements,[p,g]=t.useState(null),[v,b]=t.useState(null),y=(null==m?void 0:m.domReference)||p,C=t.useRef(null),S=t.useContext(k);x(()=>{y&&(C.current=y)},[y]);let N=function(e){void 0===e&&(e={});let{placement:n="bottom",strategy:r="absolute",middleware:u=[],platform:f,elements:{reference:m,floating:p}={},transform:g=!0,whileElementsMounted:h,open:x}=e,[v,b]=t.useState({x:0,y:0,strategy:r,placement:n,middlewareData:{},isPositioned:!1}),[y,w]=t.useState(u);i(y,u)||w(u);let[j,k]=t.useState(null),[C,S]=t.useState(null),N=t.useCallback(e=>{e!==$.current&&($.current=e,k(e))},[]),E=t.useCallback(e=>{e!==T.current&&(T.current=e,S(e))},[]),_=m||j,O=p||C,$=t.useRef(null),T=t.useRef(null),I=t.useRef(v),P=null!=h,M=c(h),R=c(f),L=c(x),D=t.useCallback(()=>{if(!$.current||!T.current)return;let e={placement:n,strategy:r,middleware:y};R.current&&(e.platform=R.current),(0,l.computePosition)($.current,T.current,e).then(e=>{let t={...e,isPositioned:!1!==L.current};A.current&&!i(I.current,t)&&(I.current=t,a.flushSync(()=>{b(t)}))})},[y,n,r,R,L]);o(()=>{!1===x&&I.current.isPositioned&&(I.current.isPositioned=!1,b(e=>({...e,isPositioned:!1})))},[x]);let A=t.useRef(!1);o(()=>(A.current=!0,()=>{A.current=!1}),[]),o(()=>{if(_&&($.current=_),O&&(T.current=O),_&&O){if(M.current)return M.current(_,O,D);D()}},[_,O,D,M,P]);let K=t.useMemo(()=>({reference:$,floating:T,setReference:N,setFloating:E}),[N,E]),B=t.useMemo(()=>({reference:_,floating:O}),[_,O]),F=t.useMemo(()=>{let e={position:r,left:0,top:0};if(!B.floating)return e;let t=d(B.floating,v.x),n=d(B.floating,v.y);return g?{...e,transform:"translate("+t+"px, "+n+"px)",...s(B.floating)>=1.5&&{willChange:"transform"}}:{position:r,left:t,top:n}},[r,g,B.floating,v.x,v.y]);return t.useMemo(()=>({...v,update:D,refs:K,elements:B,floatingStyles:F}),[v,D,K,B,F])}({...e,elements:{...m,...v&&{reference:v}}}),E=t.useCallback(e=>{let t=(0,f.isElement)(e)?{getBoundingClientRect:()=>e.getBoundingClientRect(),contextElement:e}:e;b(t),N.refs.setReference(t)},[N.refs]),_=t.useCallback(e=>{((0,f.isElement)(e)||null===e)&&(C.current=e,g(e)),((0,f.isElement)(N.refs.reference.current)||null===N.refs.reference.current||null!==e&&!(0,f.isElement)(e))&&N.refs.setReference(e)},[N.refs]),O=t.useMemo(()=>({...N.refs,setReference:_,setPositionReference:E,domReference:C}),[N.refs,_,E]),$=t.useMemo(()=>({...N.elements,domReference:y}),[N.elements,y]),T=t.useMemo(()=>({...N,...u,refs:O,elements:$,nodeId:n}),[N,O,$,n,u]);return x(()=>{u.dataRef.current.floatingContext=T;let e=null==S?void 0:S.nodesRef.current.find(e=>e.id===n);e&&(e.context=T)}),t.useMemo(()=>({...N,context:T,refs:O,elements:$}),[N,O,$,T])}({open:et,placement:"selection"===ei?"center"===es?"bottom":`bottom-${es}`:"center"===es?`${ei}`:`${ei}-${es}`,strategy:"absolute",transform:!1,middleware:[u({mainAxis:"selection"===ei?0:er,crossAxis:el}),(M={padding:ea},{...(0,l.shift)(M),options:[M,R]}),"selection"!==ei&&(L={padding:ea},{...(0,l.flip)(L),options:[L,D]}),"selection"===ei&&eo?{name:"inner",options:A={...eo,padding:ea,overflowRef:J,offset:Q,minItemsVisible:4,referenceOverflowThreshold:ea,onFallbackChange(e){var t,n;if(!e)return;let r=eu.elements.floating;if(!r)return;let l=parseFloat(getComputedStyle(r).scrollPaddingBottom)||0,a=Math.min(4,r.childElementCount),o=0,i=0;for(let e of null!=(n=null==(t=eu.elements.floating)?void 0:t.childNodes)?n:[])if(e instanceof HTMLElement){let t=e.offsetTop,n=t+e.clientHeight+l,s=r.scrollTop,d=s+r.clientHeight;if(t>=s&&n<=d)a--;else{i=Math.max(0,Math.min(n,d)-Math.max(t,s)),o=e.clientHeight;break}}a>=1&&Y(e=>{let t=o*a-i+l;return e>=t?e:t})}},async fn(e){let{listRef:t,overflowRef:n,onFallbackChange:r,offset:o=0,index:i=0,minItemsVisible:s=4,referenceOverflowThreshold:d=0,scrollRef:c,...f}=(0,m.evaluate)(A,e),{rects:p,elements:{floating:g}}=e,h=t.current[i],x=(null==c?void 0:c.current)||g,v=g.clientTop||x.clientTop,b=0!==g.clientTop,y=0!==x.clientTop,w=g===x;if(!h)return{};let j={...e,...await u(-h.offsetTop-g.clientTop-p.reference.height/2-h.offsetHeight/2-o).fn(e)},k=await (0,l.detectOverflow)(E(j,x.scrollHeight+v+g.clientTop),f),C=await (0,l.detectOverflow)(j,{...f,elementContext:"reference"}),S=(0,m.max)(0,k.top),N=j.y+S,_=(x.scrollHeight>x.clientHeight?e=>e:m.round)((0,m.max)(0,x.scrollHeight+(b&&w||y?2*v:0)-S-(0,m.max)(0,k.bottom)));if(x.style.maxHeight=_+"px",x.scrollTop=S,r){let e=x.offsetHeight=-d||C.bottom>=-d;a.flushSync(()=>r(e))}return n&&(n.current=await (0,l.detectOverflow)(E({...j,y:N},x.offsetHeight+v+g.clientTop),f)),{y:N}}}:null,(B={padding:ea,apply({availableWidth:e,availableHeight:t,elements:n}){Object.assign(n.floating.style,{overflow:"auto",maxWidth:`${e}px`,maxHeight:`min(var(--anchor-max-height, 100vh), ${t}px)`})}},{...(0,l.size)(B),options:[B,F]})].filter(Boolean),whileElementsMounted:l.autoUpdate}),[ef=ei,em=es]=eu.placement.split("-");"selection"===ei&&(ef="selection");let ep=(0,t.useMemo)(()=>({anchor:[ef,em].filter(Boolean).join(" ")}),[ef,em]),{getReferenceProps:eg,getFloatingProps:eh}=(z=(r=[function(e,n){let{open:r,elements:l}=e,{enabled:o=!0,overflowRef:i,scrollRef:s,onChange:d}=n,c=h(d),u=t.useRef(!1),f=t.useRef(null),m=t.useRef(null);t.useEffect(()=>{if(!o)return;function e(e){if(e.ctrlKey||!t||null==i.current)return;let n=e.deltaY,r=i.current.top>=-.5,l=i.current.bottom>=-.5,o=t.scrollHeight-t.clientHeight,s=n<0?-1:1,d=n<0?"max":"min";if(!(t.scrollHeight<=t.clientHeight))if(!r&&n>0||!l&&n<0)e.preventDefault(),a.flushSync(()=>{c(e=>e+Math[d](n,o*s))});else{let e;/firefox/i.test((e=navigator.userAgentData)&&Array.isArray(e.brands)?e.brands.map(e=>{let{brand:t,version:n}=e;return t+"/"+n}).join(" "):navigator.userAgent)&&(t.scrollTop+=n)}}let t=(null==s?void 0:s.current)||l.floating;if(r&&t)return t.addEventListener("wheel",e),requestAnimationFrame(()=>{f.current=t.scrollTop,null!=i.current&&(m.current={...i.current})}),()=>{f.current=null,m.current=null,t.removeEventListener("wheel",e)}},[o,r,l.floating,i,s,c]);let p=t.useMemo(()=>({onKeyDown(){u.current=!0},onWheel(){u.current=!1},onPointerMove(){u.current=!1},onScroll(){let e=(null==s?void 0:s.current)||l.floating;if(i.current&&e&&u.current){if(null!==f.current){let t=e.scrollTop-f.current;(i.current.bottom<-.5&&t<-1||i.current.top<-.5&&t>1)&&a.flushSync(()=>c(e=>e+t))}requestAnimationFrame(()=>{f.current=e.scrollTop})}}}),[l.floating,c,i,s]);return t.useMemo(()=>o?{floating:p}:{},[o,p])}(eu,{overflowRef:J,onChange:Y})]).map(e=>null==e?void 0:e.reference),H=r.map(e=>null==e?void 0:e.floating),V=r.map(e=>null==e?void 0:e.item),W=t.useCallback(e=>N(e,r,"reference"),z),U=t.useCallback(e=>N(e,r,"floating"),H),q=t.useCallback(e=>N(e,r,"item"),V),t.useMemo(()=>({getReferenceProps:W,getFloatingProps:U,getItemProps:q}),[W,U,q])),ex=(0,O.useEvent)(e=>{ee(e),ed.setFloating(e)});return t.createElement(I.Provider,{value:X},t.createElement(T.Provider,{value:{setFloating:ex,setReference:ed.setReference,styles:ec,getReferenceProps:eg,getFloatingProps:eh,slot:ep}},e))}function K(e,n,r){let l=(0,_.useDisposables)(),a=(0,O.useEvent)((e,t)=>{if(null==e)return[r,null];if("number"==typeof e)return[e,null];if("string"==typeof e){if(!t)return[r,null];let n=B(e,t);return[n,r=>{let a=function e(t){let n=/var\((.*)\)/.exec(t);if(n){let t=n[1].indexOf(",");if(-1===t)return[n[1]];let r=n[1].slice(0,t).trim(),l=n[1].slice(t+1).trim();return l?[r,...e(l)]:[r]}return[]}(e);{let o=a.map(e=>window.getComputedStyle(t).getPropertyValue(e));l.requestAnimationFrame(function i(){l.nextFrame(i);let s=!1;for(let[e,n]of a.entries()){let r=window.getComputedStyle(t).getPropertyValue(n);if(o[e]!==r){o[e]=r,s=!0;break}}if(!s)return;let d=B(e,t);n!==d&&(r(d),n=d)})}return l.dispose}]}return[r,null]}),o=(0,t.useMemo)(()=>a(e,n)[0],[e,n]),[i=o,s]=(0,t.useState)();return(0,$.useIsoMorphicEffect)(()=>{let[t,r]=a(e,n);if(s(t),r)return r(s)},[e,n]),i}function B(e,t){let n=document.createElement("div");t.appendChild(n),n.style.setProperty("margin-top","0px","important"),n.style.setProperty("margin-top",e,"important");let r=parseFloat(window.getComputedStyle(n).marginTop)||0;return t.removeChild(n),r}function F(e={},t=null,n=[]){for(let[r,l]of Object.entries(e))!function e(t,n,r){if(Array.isArray(r))for(let[l,a]of r.entries())e(t,z(n,l.toString()),a);else r instanceof Date?t.push([n,r.toISOString()]):"boolean"==typeof r?t.push([n,r?"1":"0"]):"string"==typeof r?t.push([n,r]):"number"==typeof r?t.push([n,`${r}`]):null==r?t.push([n,""]):F(r,n,t)}(n,z(t,r),l);return n}function z(e,t){return e?e+"["+t+"]":t}function H(e){var t,n;let r=null!=(t=null==e?void 0:e.form)?t:e.closest("form");if(r){for(let t of r.elements)if(t!==e&&("INPUT"===t.tagName&&"submit"===t.type||"BUTTON"===t.tagName&&"submit"===t.type||"INPUT"===t.nodeName&&"image"===t.type))return void t.click();null==(n=r.requestSubmit)||n.call(r)}}I.displayName="PlacementContext",e.s(["FloatingProvider",()=>A,"useFloatingPanel",()=>D,"useFloatingPanelProps",()=>L,"useFloatingReference",()=>M,"useFloatingReferenceProps",()=>R,"useResolvedAnchor",()=>P],919751),e.s(["attemptSubmit",()=>H,"objectToFormEntries",()=>F],694421);var V=e.i(700020),W=e.i(2788);let U=(0,t.createContext)(null);function q({children:e}){let n=(0,t.useContext)(U);if(!n)return t.default.createElement(t.default.Fragment,null,e);let{target:r}=n;return r?(0,a.createPortal)(t.default.createElement(t.default.Fragment,null,e),r):null}function G({data:e,form:n,disabled:r,onReset:l,overrides:a}){let[o,i]=(0,t.useState)(null),s=(0,_.useDisposables)();return(0,t.useEffect)(()=>{if(l&&o)return s.addEventListener(o,"reset",l)},[o,n,l]),t.default.createElement(q,null,t.default.createElement(X,{setForm:i,formId:n}),F(e).map(([e,l])=>t.default.createElement(W.Hidden,{features:W.HiddenFeatures.Hidden,...(0,V.compact)({key:e,as:"input",type:"hidden",hidden:!0,readOnly:!0,form:n,disabled:r,name:e,value:l,...a})})))}function X({setForm:e,formId:n}){return(0,t.useEffect)(()=>{if(n){let t=document.getElementById(n);t&&e(t)}},[e,n]),n?null:t.default.createElement(W.Hidden,{features:W.HiddenFeatures.Hidden,as:"input",type:"hidden",hidden:!0,readOnly:!0,ref:t=>{if(!t)return;let n=t.closest("form");n&&e(n)}})}function Q(e,n){let[r,l]=(0,t.useState)(n);return e||r===n||l(n),e?r:n}e.s(["FormFields",()=>G],140721),e.s(["useFrozenData",()=>Q],904016);let Y=(0,t.createContext)(void 0);function J(){return(0,t.useContext)(Y)}e.s(["useProvidedId",()=>J],942803)},233137,233538,e=>{"use strict";let t;var n=e.i(271645);let r=(0,n.createContext)(null);r.displayName="OpenClosedContext";var l=((t=l||{})[t.Open=1]="Open",t[t.Closed=2]="Closed",t[t.Closing=4]="Closing",t[t.Opening=8]="Opening",t);function a(){return(0,n.useContext)(r)}function o({value:e,children:t}){return n.default.createElement(r.Provider,{value:e},t)}function i({children:e}){return n.default.createElement(r.Provider,{value:null},e)}function s(e){let t=e.parentElement,n=null;for(;t&&!(t instanceof HTMLFieldSetElement);)t instanceof HTMLLegendElement&&(n=t),t=t.parentElement;let r=(null==t?void 0:t.getAttribute("disabled"))==="";return!(r&&function(e){if(!e)return!1;let t=e.previousElementSibling;for(;null!==t;){if(t instanceof HTMLLegendElement)return!1;t=t.previousElementSibling}return!0}(n))&&r}e.s(["OpenClosedProvider",()=>o,"ResetOpenClosedProvider",()=>i,"State",()=>l,"useOpenClosed",()=>a],233137),e.s(["isDisabledReactIssue7711",()=>s],233538)},35983,35889,722678,178677,635307,495470,333771,e=>{"use strict";let t,n,r,l,a;var o=e.i(290571),i=e.i(271645),s=e.i(429427),d=e.i(371330),c=e.i(174080),u=e.i(394487),f=e.i(436289),m=e.i(503269),p=e.i(214520),g=e.i(814379),h=e.i(746725),x=e.i(992704),v=e.i(914189),b=e.i(684653),y=e.i(835696),w=e.i(941444),j=e.i(877891),k=e.i(952744),C=e.i(605083),S=e.i(144279),N=e.i(101852),E=e.i(294316),_=e.i(249578),O=e.i(571616),$=e.i(83733),T=e.i(601893),I=e.i(919751),P=e.i(140721),M=e.i(904016),R=e.i(942803),L=e.i(233137),D=e.i(233538),A=((t=A||{})[t.First=0]="First",t[t.Previous=1]="Previous",t[t.Next=2]="Next",t[t.Last=3]="Last",t[t.Specific=4]="Specific",t[t.Nothing=5]="Nothing",t);function K(e,t){let n=t.resolveItems();if(n.length<=0)return null;let r=t.resolveActiveIndex(),l=null!=r?r:-1;switch(e.focus){case 0:for(let e=0;e=0;--e)if(!t.resolveDisabled(n[e],e,n))return e;return r;case 2:for(let e=l+1;e=0;--e)if(!t.resolveDisabled(n[e],e,n))return e;return r;case 4:for(let r=0;r0?e.join(" "):void 0,(0,i.useMemo)(()=>function(e){let n=(0,v.useEvent)(e=>(t(t=>[...t,e]),()=>t(t=>{let n=t.slice(),r=n.indexOf(e);return -1!==r&&n.splice(r,1),n}))),r=(0,i.useMemo)(()=>({register:n,slot:e.slot,name:e.name,props:e.props,value:e.value}),[n,e.slot,e.name,e.props,e.value]);return i.default.createElement(U.Provider,{value:r},e.children)},[t])]}U.displayName="DescriptionContext";let X=Object.assign((0,W.forwardRefWithAs)(function(e,t){let n=(0,i.useId)(),r=(0,T.useDisabled)(),{id:l=`headlessui-description-${n}`,...a}=e,o=function e(){let t=(0,i.useContext)(U);if(null===t){let t=Error("You used a component, but it is not inside a relevant parent.");throw Error.captureStackTrace&&Error.captureStackTrace(t,e),t}return t}(),s=(0,E.useSyncRefs)(t);(0,y.useIsoMorphicEffect)(()=>o.register(l),[l,o.register]);let d=r||!1,c=(0,i.useMemo)(()=>({...o.slot,disabled:d}),[o.slot,d]),u={ref:s,...o.props,id:l};return(0,W.useRender)()({ourProps:u,theirProps:a,slot:c,defaultTag:"p",name:o.name||"Description"})}),{});e.s(["Description",()=>X,"useDescribedBy",()=>q,"useDescriptions",()=>G],35889);var Q=e.i(998348);let Y=(0,i.createContext)(null);function J(e){var t,n,r;let l=null!=(n=null==(t=(0,i.useContext)(Y))?void 0:t.value)?n:void 0;return(null!=(r=null==e?void 0:e.length)?r:0)>0?[l,...e].filter(Boolean).join(" "):l}function Z({inherit:e=!1}={}){let t=J(),[n,r]=(0,i.useState)([]),l=e?[t,...n].filter(Boolean):n;return[l.length>0?l.join(" "):void 0,(0,i.useMemo)(()=>function(e){let t=(0,v.useEvent)(e=>(r(t=>[...t,e]),()=>r(t=>{let n=t.slice(),r=n.indexOf(e);return -1!==r&&n.splice(r,1),n}))),n=(0,i.useMemo)(()=>({register:t,slot:e.slot,name:e.name,props:e.props,value:e.value}),[t,e.slot,e.name,e.props,e.value]);return i.default.createElement(Y.Provider,{value:n},e.children)},[r])]}Y.displayName="LabelContext";let ee=Object.assign((0,W.forwardRefWithAs)(function(e,t){var n;let r=(0,i.useId)(),l=function e(){let t=(0,i.useContext)(Y);if(null===t){let t=Error("You used a