diff --git a/.circleci/config.yml b/.circleci/config.yml index f13e9bf66f1..ce9aaa9be8a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,16 @@ orbs: win: circleci/windows@5.0 # Add Windows orb commands: + skip_if_unrelated_changes: + parameters: + category: + type: enum + enum: ["backend", "client"] + default: "backend" + steps: + - run: + name: "Skip job when no << parameters.category >>-relevant files changed" + command: bash .circleci/scripts/path_filter.sh << parameters.category >> setup_google_dns: steps: - run: @@ -282,6 +292,7 @@ jobs: parallelism: 4 steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: @@ -354,6 +365,7 @@ jobs: parallelism: 4 steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: @@ -427,6 +439,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: @@ -480,6 +493,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -545,6 +559,7 @@ jobs: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -584,6 +599,7 @@ jobs: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -624,6 +640,7 @@ jobs: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -656,6 +673,7 @@ jobs: FAKE_OPENAI_API_BASE: http://127.0.0.1:8190 steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - restore_cache: @@ -705,6 +723,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - restore_cache: @@ -755,6 +774,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -787,6 +807,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - restore_cache: @@ -832,6 +853,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -877,6 +899,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -918,6 +941,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -963,6 +987,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1007,6 +1032,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - restore_cache: @@ -1045,6 +1071,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1089,6 +1116,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1132,6 +1160,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1163,6 +1192,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1205,6 +1235,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1248,6 +1279,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1291,6 +1323,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1321,6 +1354,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1366,6 +1400,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1407,6 +1442,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: @@ -1459,6 +1495,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1482,6 +1519,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1507,6 +1545,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1531,6 +1570,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns @@ -1570,14 +1610,14 @@ jobs: - run: name: Run helm lint command: | - helm lint ./deploy/charts/litellm-helm + helm lint ./helm/litellm-helm # Run helm tests - run: name: Run helm tests command: | IMAGE_TAG=${CIRCLE_SHA1:-ci} - helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \ + helm install litellm ./helm/litellm-helm -f ./helm/litellm-helm/ci/test-values.yaml \ --set image.repository=litellm-ci \ --set image.tag=${IMAGE_TAG} \ --set image.pullPolicy=Never @@ -1606,6 +1646,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1698,6 +1739,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns @@ -1746,13 +1788,13 @@ jobs: -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ -e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ my-app:latest \ --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ + --port 4000 - run: name: Start outputting logs command: docker logs -f my-app @@ -1787,6 +1829,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1832,13 +1875,13 @@ jobs: -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ + --port 4000 - run: name: Start outputting logs command: docker logs -f my-app @@ -1869,6 +1912,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1911,14 +1955,14 @@ jobs: -e COHERE_API_KEY=$COHERE_API_KEY \ -e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \ -e GCS_FLUSH_INTERVAL="1" \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \ litellm-docker-database:ci \ --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ + --port 4000 - run: name: Start outputting logs command: docker logs -f my-app @@ -1960,13 +2004,13 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE="bad-license" \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app-3 \ -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ - --port 4000 \ - --detailed_debug + --port 4000 - run: name: Start outputting logs for second container @@ -2000,6 +2044,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2041,13 +2086,13 @@ jobs: -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e PROXY_BATCH_WRITE_AT=2 \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ + --port 4000 - run: name: Start outputting logs command: docker logs -f my-app @@ -2085,6 +2130,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2117,13 +2163,13 @@ jobs: -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ + --port 4000 - run: name: Run Docker container 2 command: | @@ -2139,13 +2185,13 @@ jobs: -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app-2 \ -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ - --port 4001 \ - --detailed_debug + --port 4001 - run: name: Start outputting logs command: docker logs -f my-app @@ -2180,6 +2226,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2201,19 +2248,20 @@ jobs: # the OTEL test - should get this as a trace command: | docker run -d \ + --restart on-failure \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e STORE_MODEL_IN_DB="True" \ -e LITELLM_MASTER_KEY="sk-1234" \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ + --port 4000 - run: name: Start outputting logs command: docker logs -f my-app @@ -2252,6 +2300,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns # Remove Docker CLI installation since it's already available in machine executor - install_uv @@ -2289,13 +2338,13 @@ jobs: -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e GCS_FLUSH_INTERVAL="1" \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ my-app:latest \ --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ + --port 4000 - run: name: Start outputting logs command: docker logs -f my-app @@ -2333,6 +2382,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2365,14 +2415,14 @@ jobs: -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \ litellm-docker-database:ci \ --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ + --port 4000 - run: name: Start outputting logs command: docker logs -f my-app @@ -2471,6 +2521,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2499,13 +2550,13 @@ jobs: -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME="us-east-1" \ -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \ + -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ - --port 4000 \ - --detailed_debug + --port 4000 - run: name: Start outputting logs command: docker logs -f my-app @@ -2537,6 +2588,7 @@ jobs: - *python312_image steps: - checkout + - skip_if_unrelated_changes - attach_workspace: at: . # Check file locations @@ -2567,6 +2619,8 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes: + category: client - setup_google_dns - restore_cache: keys: @@ -2609,6 +2663,8 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes: + category: client - setup_google_dns - restore_cache: keys: @@ -2629,7 +2685,7 @@ jobs: cd ui/litellm-dashboard CI=true npm run test -- --run \ - --pool forks --poolOptions.forks.maxForks=8 + --pool forks --poolOptions.forks.maxForks=6 e2e_ui_testing: docker: @@ -2654,6 +2710,8 @@ jobs: PROXY_LOGOUT_URL: "https://www.example.com" steps: - checkout + - skip_if_unrelated_changes: + category: client - setup_google_dns - install_uv - restore_cache: @@ -2791,6 +2849,8 @@ jobs: SERVER_ROOT_PATH: "/litellm" steps: - checkout + - skip_if_unrelated_changes: + category: client - setup_google_dns - install_uv - restore_cache: @@ -2892,6 +2952,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - run: name: Build Docker image @@ -2917,6 +2978,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh new file mode 100755 index 00000000000..2c15428be6a --- /dev/null +++ b/.circleci/scripts/classify_changes.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -uo pipefail + +category="${1:?usage: classify_changes.sh }" + +has_client=false +has_backend=false +while IFS= read -r file || [ -n "$file" ]; do + [ -n "$file" ] || continue + case "$file" in + ui/*) has_client=true ;; + docs/* | *.md | *.mdx) : ;; + *) has_backend=true ;; + esac +done + +case "$category" in + backend) + [ "$has_backend" = true ] && echo run || echo skip + ;; + client) + { [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip + ;; + *) + echo run + ;; +esac diff --git a/.circleci/scripts/path_filter.sh b/.circleci/scripts/path_filter.sh new file mode 100755 index 00000000000..dcf64a24399 --- /dev/null +++ b/.circleci/scripts/path_filter.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -uo pipefail + +category="${1:?usage: path_filter.sh }" +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +run_full() { + echo "path-filter[$category]: running job ($1)" + exit 0 +} + +[ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request" + +candidate_bases="main litellm_internal_staging litellm_oss_staging" +merge_base="" +for base in $candidate_bases; do + git fetch --quiet origin "$base" 2>/dev/null || continue + candidate="$(git merge-base HEAD FETCH_HEAD 2>/dev/null)" || continue + [ -n "$candidate" ] || continue + if [ -z "$merge_base" ] || git merge-base --is-ancestor "$merge_base" "$candidate" 2>/dev/null; then + merge_base="$candidate" + fi +done + +[ -n "$merge_base" ] || run_full "could not resolve a merge base against $candidate_bases" + +changed="$(git diff --name-only "$merge_base" HEAD 2>/dev/null)" || run_full "git diff failed" +[ -n "$changed" ] || run_full "no files changed vs $merge_base" + +echo "path-filter[$category]: changed files vs ${merge_base}:" +printf '%s\n' "$changed" | sed 's/^/ /' || true + +decision="$(printf '%s\n' "$changed" | bash "$here/classify_changes.sh" "$category")" || run_full "classify_changes.sh failed" + +if [ "$decision" = run ]; then + run_full "$category-relevant changes detected" +fi + +echo "path-filter[$category]: only unrelated (docs/client) changes detected; halting job as successful" +circleci-agent step halt diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 12ad124fa20..bd9fc2285d1 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,7 +4,7 @@ ## Linear ticket - + ## Pre-Submission checklist @@ -13,7 +13,7 @@ - [ ] I have added meaningful tests - [ ] My PR passes all CI/CD checks (e.g., lint, format, unit tests) - [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem -- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review +- [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes) ## Delays in PR merge? @@ -24,6 +24,7 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 49f1d906069..a1772102b89 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -4,9 +4,11 @@ on: push: branches: - main + - litellm_internal_staging pull_request: branches: - main + - litellm_internal_staging # Allow CodSpeed to trigger backtest performance analysis # in order to generate initial data workflow_dispatch: @@ -22,7 +24,7 @@ concurrency: jobs: benchmarks: runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 60 steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 4834775e329..0ad84cd3ceb 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -122,10 +122,28 @@ jobs: makeLatest = (!latestVersion || isAtLeast(newVersion, latestVersion)) ? "true" : "false"; } + try { + await github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/tags/${tag}`, + sha: commitHash, + }); + } catch (error) { + if (error.status !== 422) throw error; + const existing = await github.rest.git.getRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `tags/${tag}`, + }); + if (existing.data.object.sha !== commitHash) { + throw new Error(`Tag ${tag} already exists at ${existing.data.object.sha}, expected ${commitHash}`); + } + } + const response = await github.rest.repos.createRelease({ draft: true, generate_release_notes: true, - target_commitish: commitHash, name: tag, owner: context.repo.owner, prerelease: isPrerelease, @@ -138,11 +156,21 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, release_id: response.data.id, + tag_name: tag, body: updatedBody, draft: false, - make_latest: makeLatest, }); + if (!isPrerelease) { + await github.rest.repos.updateRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: response.data.id, + tag_name: tag, + make_latest: makeLatest, + }); + } + } catch (error) { core.setFailed(error.message); } diff --git a/.github/workflows/helm_unit_test.yml b/.github/workflows/helm_unit_test.yml index 06836b1d1cd..5b9d20d97f3 100644 --- a/.github/workflows/helm_unit_test.yml +++ b/.github/workflows/helm_unit_test.yml @@ -38,4 +38,6 @@ jobs: echo "Helm unittest plugin integrity verified: $ACTUAL_SHA" - name: Run unit tests - run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm + run: | + helm unittest -f 'tests/*.yaml' helm/litellm-helm + helm unittest -f 'tests/*.yaml' helm/litellm diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 6deb28c95c7..c2fc3453261 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -63,7 +63,7 @@ jobs: env: BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | - git diff --name-only "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true + git diff --name-only --diff-filter=ACMR "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then echo "No changed litellm Python files to check with ruff format." exit 0 diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml new file mode 100644 index 00000000000..03d8ff3461c --- /dev/null +++ b/.github/workflows/test-terraform-provider.yml @@ -0,0 +1,113 @@ +name: Terraform Provider + +on: + push: + paths: + - "terraform/provider/**" + - ".github/workflows/test-terraform-provider.yml" + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_staging + - "litellm_**" + paths: + - "terraform/provider/**" + - "litellm/proxy/**" + - ".github/workflows/test-terraform-provider.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + provider-checks: + name: gofmt, vet, build, test + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: terraform/provider + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + with: + go-version-file: terraform/provider/go.mod + cache: true + cache-dependency-path: terraform/provider/go.sum + + - name: gofmt + run: | + UNFORMATTED=$(gofmt -l .) + if [ -n "${UNFORMATTED}" ]; then + echo "::error::gofmt required for: ${UNFORMATTED}" + exit 1 + fi + + - name: go vet + run: go vet ./... + + - name: Build + run: go build ./... + + - name: Test + run: go test -timeout 120s ./... + + endpoint-drift: + name: Provider endpoints vs proxy OpenAPI schema + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Cache uv dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Generate proxy OpenAPI schema + run: | + uv run --no-sync python terraform/provider/tools/dump_openapi.py "${RUNNER_TEMP}/openapi.json" + + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + with: + go-version-file: terraform/provider/go.mod + cache: true + cache-dependency-path: terraform/provider/go.sum + + - name: Audit provider endpoints against the schema + working-directory: terraform/provider + run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json" diff --git a/.gitignore b/.gitignore index 59fa5803abe..e3ccf50508f 100644 --- a/.gitignore +++ b/.gitignore @@ -52,9 +52,8 @@ ui/litellm-dashboard/node_modules ui/litellm-dashboard/next-env.d.ts ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json -deploy/charts/litellm/*.tgz -deploy/charts/litellm/charts/* -deploy/charts/*.tgz +helm/litellm-helm/*.tgz +helm/*.tgz litellm/proxy/vertex_key.json **/.vim/ **/node_modules @@ -130,3 +129,5 @@ crash.*.log # pytest coverage data .coverage + +ui/litellm-dashboard/out/ diff --git a/CLAUDE.md b/CLAUDE.md index 7d9a6367f18..5255d39b4b6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,11 +17,15 @@ Same thing for bug fixes. The tests should make it so that this specific bug can `tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_.py` if you're the first test there). One focused regression test beats many shallow ones +End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md` + When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose -Always use @.github/pull_request_template.md as a guide for your PR body +When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout -Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR +If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank + +Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y: - don't use emojis @@ -43,9 +47,11 @@ If you're trying to create a new function that relies on untyped stuff, instead If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason +Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType] # stubs lack async overload` or `# noqa: TID251 # `. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing + Commit and push your work when you're done without asking -When you must use real LLM models to, for example, write e2e tests, write a QA runbook, etc., make sure to use the latest models (doesn't have to be smartest, can also be a modern small, fast one. No strong preference for smart vs fast here, just use something modern) as of the year and month of the current date. Do a web search as necessary to figure that out +When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names diff --git a/Dockerfile b/Dockerfile index b6fef1a21fc..bc0e6a5ca6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6 diff --git a/Makefile b/Makefile index 2cc4ec3e45a..f2753b09ff5 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ .PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \ test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \ test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ - info lint lint-dev format \ + info lint lint-dev lint-checks format \ lint-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \ lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \ install-dev install-proxy-dev install-test-deps install-hooks \ @@ -53,6 +53,11 @@ help: UV := uv UV_RUN := $(UV) run --no-sync +LINT_DEP_INSTALL ?= install-dev +LINT_DEP_BASE ?= lint-fetch-base +LINT_JOBS := $(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4) +LINT_OUTPUT_SYNC := $(if $(filter output-sync,$(.FEATURES)),--output-sync=target,) + # Show info info: @echo "UV: $(UV)" @@ -107,12 +112,12 @@ lint-fetch-base: # running proxy need. lint-install: $(UV) sync --inexact --frozen --group proxy-dev - $(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma + $(UV_RUN) python scripts/prisma_generate_if_needed.py # Diff-scoped format check, identical to test-linting.yml's "Check ruff format" step: # only the litellm Python files changed vs the base are checked, so a pre-existing # format issue elsewhere doesn't block an unrelated commit. -lint-format-check-changed: install-dev lint-fetch-base +lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) @files=$$(git diff --name-only origin/litellm_internal_staging...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' || true); \ if [ -z "$$files" ]; then \ echo "No changed litellm Python files to format-check."; \ @@ -121,7 +126,7 @@ lint-format-check-changed: install-dev lint-fetch-base fi # Linting targets -lint-ruff: install-dev +lint-ruff: $(LINT_DEP_INSTALL) cd litellm && $(UV_RUN) ruff check . && cd .. # faster linter for developing ... @@ -156,12 +161,12 @@ lint-ruff-FULL-dev: install-dev if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \ else echo "No changed .py files to check."; fi -lint-basedpyright: install-dev lint-fetch-base +lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) ($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging # Type-discipline budget (mutable collections / casts / type guards / kwargs / # unexplained suppressions), the test-linting.yml step `make lint` used to omit. -lint-type-discipline: install-dev lint-fetch-base +lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) $(UV_RUN) python scripts/type_discipline_gate.py --base origin/litellm_internal_staging # --update lowers each limit by what this branch fixed since its branch point, so @@ -176,7 +181,7 @@ lint-ruff-budget: install-dev # Strict gate, invoked the same way CI does in test-linting.yml so a local pass # means the CI check will pass too. -lint-gate: install-dev lint-fetch-base +lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) $(UV_RUN) python scripts/ruff_strict_gate.py --base origin/litellm_internal_staging lint-ruff-budget-update: install-dev lint-fetch-base @@ -188,10 +193,10 @@ lint-type-discipline-budget-update: install-dev lint-fetch-base # Ratchet all budgets in one shot (ruff strict + type-discipline + basedpyright) lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-basedpyright-budget-update -check-circular-imports: install-dev +check-circular-imports: $(LINT_DEP_INSTALL) cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd .. -check-import-safety: install-dev +check-import-safety: $(LINT_DEP_INSTALL) @$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) # Combined linting, isomorphic to test-linting.yml's lint job so a local pass means a @@ -199,9 +204,13 @@ check-import-safety: install-dev # runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule / # type-discipline / basedpyright budgets as a delta vs the base, then the circular-import # and import-safety checks. Steps that compare against the base resolve it the same way CI -# does (merge-base with origin/litellm_internal_staging). lint-install is first so the -# Prisma client exists before basedpyright runs. -lint: lint-install lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety +# does (merge-base with origin/litellm_internal_staging). Setup (env sync, Prisma client, +# base fetch) runs once up front; the checks themselves are independent, so a sub-make +# fans them out with -j and the fast ones finish under basedpyright's shadow. +lint: lint-install lint-fetch-base + $(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_DEP_BASE= lint-checks + +lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety # Faster linting for local development (only checks changed code) lint-dev: lint-format-changed check-circular-imports check-import-safety @@ -256,7 +265,7 @@ test-integration: install-test-deps $(UV_RUN) pytest tests/ -k "not test_litellm" test-unit-helm: install-helm-unittest - helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm + helm unittest -f 'tests/*.yaml' helm/litellm-helm # LLM Translation testing targets test-llm-translation: install-test-deps diff --git a/backend/Dockerfile b/backend/Dockerfile index 667bdb073eb..62bd8b56483 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 79e6af05978..cb3427bed4d 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -3,16 +3,16 @@ "limit": 37484 }, "reportArgumentType": { - "limit": 2721 + "limit": 2704 }, "reportAssignmentType": { "limit": 330 }, "reportAttributeAccessIssue": { - "limit": 519 + "limit": 516 }, "reportCallIssue": { - "limit": 131 + "limit": 124 }, "reportConstantRedefinition": { "limit": 59 @@ -42,7 +42,7 @@ "limit": 18 }, "reportIndexIssue": { - "limit": 39 + "limit": 37 }, "reportInvalidTypeForm": { "limit": 35 @@ -51,7 +51,7 @@ "limit": 5 }, "reportMatchNotExhaustive": { - "limit": 2 + "limit": 0 }, "reportMissingParameterType": { "limit": 5900 @@ -63,25 +63,25 @@ "limit": 41 }, "reportOperatorIssue": { - "limit": 9 + "limit": 0 }, "reportOptionalCall": { - "limit": 7 + "limit": 0 }, "reportOptionalIterable": { - "limit": 6 + "limit": 0 }, "reportOptionalMemberAccess": { - "limit": 1086 + "limit": 1085 }, "reportOptionalOperand": { - "limit": 6 + "limit": 0 }, "reportOptionalSubscript": { - "limit": 17 + "limit": 0 }, "reportPossiblyUnboundVariable": { - "limit": 78 + "limit": 77 }, "reportPrivateUsage": { "limit": 2438 @@ -90,28 +90,28 @@ "limit": 12 }, "reportReturnType": { - "limit": 226 + "limit": 225 }, "reportTypedDictNotRequiredAccess": { - "limit": 30 + "limit": 27 }, "reportUndefinedVariable": { - "limit": 5 + "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45905 + "limit": 45894 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 40556 + "limit": 40541 }, "reportUnknownParameterType": { "limit": 20418 }, "reportUnknownVariableType": { - "limit": 32168 + "limit": 32151 }, "reportUnnecessaryCast": { "limit": 177 @@ -141,6 +141,6 @@ "limit": 1005 }, "reportUnusedVariable": { - "limit": 1298 + "limit": 1297 } } diff --git a/codecov.yaml b/codecov.yaml index f5acdd39136..bc0b3604329 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -15,6 +15,16 @@ ignore: flag_management: default_rules: carryforward: true + # Dead flags no CI job uploads anymore: their carried-forward sessions were + # measured against old revisions, and the stale line maps mark comment lines + # of since-edited files as missed, sinking patch coverage on unrelated PRs. + individual_flags: + - name: proxy-mgmt-behavior + carryforward: false + - name: security + carryforward: false + - name: proxy-db-schema-migration + carryforward: false component_management: individual_components: diff --git a/deploy/azure_resource_manager/azure_marketplace.zip b/deploy/azure_resource_manager/azure_marketplace.zip deleted file mode 100644 index 34751258637..00000000000 Binary files a/deploy/azure_resource_manager/azure_marketplace.zip and /dev/null differ diff --git a/deploy/azure_resource_manager/azure_marketplace/createUiDefinition.json b/deploy/azure_resource_manager/azure_marketplace/createUiDefinition.json deleted file mode 100644 index 4eba73bdba4..00000000000 --- a/deploy/azure_resource_manager/azure_marketplace/createUiDefinition.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/0.1.2-preview/CreateUIDefinition.MultiVm.json#", - "handler": "Microsoft.Azure.CreateUIDef", - "version": "0.1.2-preview", - "parameters": { - "config": { - "isWizard": false, - "basics": { } - }, - "basics": [ ], - "steps": [ ], - "outputs": { }, - "resourceTypes": [ ] - } -} \ No newline at end of file diff --git a/deploy/azure_resource_manager/azure_marketplace/mainTemplate.json b/deploy/azure_resource_manager/azure_marketplace/mainTemplate.json deleted file mode 100644 index 114e855bf54..00000000000 --- a/deploy/azure_resource_manager/azure_marketplace/mainTemplate.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "imageName": { - "type": "string", - "defaultValue": "ghcr.io/berriai/litellm:main-latest" - }, - "containerName": { - "type": "string", - "defaultValue": "litellm-container" - }, - "dnsLabelName": { - "type": "string", - "defaultValue": "litellm" - }, - "portNumber": { - "type": "int", - "defaultValue": 4000 - } - }, - "resources": [ - { - "type": "Microsoft.ContainerInstance/containerGroups", - "apiVersion": "2021-03-01", - "name": "[parameters('containerName')]", - "location": "[resourceGroup().location]", - "properties": { - "containers": [ - { - "name": "[parameters('containerName')]", - "properties": { - "image": "[parameters('imageName')]", - "resources": { - "requests": { - "cpu": 1, - "memoryInGB": 2 - } - }, - "ports": [ - { - "port": "[parameters('portNumber')]" - } - ] - } - } - ], - "osType": "Linux", - "restartPolicy": "Always", - "ipAddress": { - "type": "Public", - "ports": [ - { - "protocol": "tcp", - "port": "[parameters('portNumber')]" - } - ], - "dnsNameLabel": "[parameters('dnsLabelName')]" - } - } - } - ] - } \ No newline at end of file diff --git a/deploy/azure_resource_manager/main.bicep b/deploy/azure_resource_manager/main.bicep deleted file mode 100644 index b104cefe1e1..00000000000 --- a/deploy/azure_resource_manager/main.bicep +++ /dev/null @@ -1,42 +0,0 @@ -param imageName string = 'ghcr.io/berriai/litellm:main-latest' -param containerName string = 'litellm-container' -param dnsLabelName string = 'litellm' -param portNumber int = 4000 - -resource containerGroupName 'Microsoft.ContainerInstance/containerGroups@2021-03-01' = { - name: containerName - location: resourceGroup().location - properties: { - containers: [ - { - name: containerName - properties: { - image: imageName - resources: { - requests: { - cpu: 1 - memoryInGB: 2 - } - } - ports: [ - { - port: portNumber - } - ] - } - } - ] - osType: 'Linux' - restartPolicy: 'Always' - ipAddress: { - type: 'Public' - ports: [ - { - protocol: 'tcp' - port: portNumber - } - ] - dnsNameLabel: dnsLabelName - } - } -} diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index b3af953511d..4564ee403fe 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6 diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index c24cb9008f0..1883e87be60 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1.7 # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f ARG PROXY_EXTRAS_SOURCE=published ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. diff --git a/docker/build_admin_ui.sh b/docker/build_admin_ui.sh index efb2bac3535..68acdd78e3e 100755 --- a/docker/build_admin_ui.sh +++ b/docker/build_admin_ui.sh @@ -57,8 +57,6 @@ source ~/.nvm/nvm.sh nvm install v18.17.0 nvm use v18.17.0 -# copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json -cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json # cd in to /ui/litellm-dashboard cd ui/litellm-dashboard diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py index 9d15f45079f..be80a12c80a 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py @@ -477,9 +477,12 @@ class BaseEmailLogger(CustomLogger): _id = user_info.token or user_info.user_id or "default_id" _cache_key = f"email_budget_alerts:soft_budget_crossed:{_id}" - # Check if we've already sent this alert - result = await _cache.async_get_cache(key=_cache_key) - if result is None: + send_count = await _cache.async_increment_cache( + key=_cache_key, + value=1, + ttl=EMAIL_BUDGET_ALERT_TTL, + ) + if send_count is None or send_count <= 1: # Create WebhookEvent for soft budget alert event_message = f"Soft Budget Crossed - Total Soft Budget: ${user_info.soft_budget}" webhook_event = WebhookEvent( @@ -508,18 +511,12 @@ class BaseEmailLogger(CustomLogger): await self.send_team_soft_budget_alert_email(webhook_event) else: await self.send_soft_budget_alert_email(webhook_event) - - # Cache the alert to prevent duplicate sends - await _cache.async_set_cache( - key=_cache_key, - value="SENT", - ttl=EMAIL_BUDGET_ALERT_TTL, - ) except Exception as e: verbose_proxy_logger.error( f"Error sending soft budget alert email: {e}", exc_info=True, ) + await self._release_budget_alert_claim(_cache, _cache_key) return # For max_budget_alert, check if we've already sent an alert @@ -545,9 +542,12 @@ class BaseEmailLogger(CustomLogger): _id = user_info.token or user_info.user_id or "default_id" _cache_key = f"email_budget_alerts:max_budget_alert:{_id}" - # Check if we've already sent this alert - result = await _cache.async_get_cache(key=_cache_key) - if result is None: + send_count = await _cache.async_increment_cache( + key=_cache_key, + value=1, + ttl=EMAIL_BUDGET_ALERT_TTL, + ) + if send_count is None or send_count <= 1: # Calculate percentage percentage = int( EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100 @@ -576,18 +576,12 @@ class BaseEmailLogger(CustomLogger): try: await self.send_max_budget_alert_email(webhook_event) - - # Cache the alert to prevent duplicate sends - await _cache.async_set_cache( - key=_cache_key, - value="SENT", - ttl=EMAIL_BUDGET_ALERT_TTL, - ) except Exception as e: verbose_proxy_logger.error( f"Error sending max budget alert email: {e}", exc_info=True, ) + await self._release_budget_alert_claim(_cache, _cache_key) return async def _handle_multi_threshold_max_budget_alert( @@ -617,10 +611,6 @@ class BaseEmailLogger(CustomLogger): f"email_budget_alerts:max_budget_alert:{threshold_pct}:{_id}" ) - result = await _cache.async_get_cache(key=_cache_key) - if result is not None: - continue - # Parse emails + auto-include owner emails = _parse_email_list(raw_emails) if user_info.user_email: @@ -634,6 +624,14 @@ class BaseEmailLogger(CustomLogger): continue recipient_emails = list(set(emails)) + send_count = await _cache.async_increment_cache( + key=_cache_key, + value=1, + ttl=EMAIL_BUDGET_ALERT_TTL, + ) + if send_count is not None and send_count > 1: + continue + event_message = f"Max Budget Alert - {threshold_pct}% of Maximum Budget Reached" webhook_event = WebhookEvent( event="max_budget_alert", @@ -660,16 +658,21 @@ class BaseEmailLogger(CustomLogger): threshold_pct=threshold_pct, recipient_emails=recipient_emails, ) - await _cache.async_set_cache( - key=_cache_key, - value="SENT", - ttl=EMAIL_BUDGET_ALERT_TTL, - ) except Exception as e: verbose_proxy_logger.error( f"Error sending multi-threshold max budget alert email for {threshold_pct}%: {e}", exc_info=True, ) + await self._release_budget_alert_claim(_cache, _cache_key) + + async def _release_budget_alert_claim(self, cache: DualCache, cache_key: str) -> None: + try: + await cache.async_delete_cache(key=cache_key) + except Exception: + verbose_proxy_logger.debug( + "Failed to release budget alert claim for %s; it expires with the TTL", + cache_key, + ) async def _get_email_params( self, diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 831a23ff3cd..b9ac98f515c 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -17,6 +17,7 @@ if TYPE_CHECKING: from litellm.proxy._types import LiteLLM_ManagedObjectTable from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router + from litellm.types.utils import LiteLLMBatch CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost" @@ -277,13 +278,20 @@ class CheckBatchCost: except Exception: return None - async def check_batch_cost(self): + async def _track_completed_batch_cost( + self, + job: "LiteLLM_ManagedObjectTable", + response: "LiteLLMBatch", + model_id: str, + batch_id: str, + prom_logger: Optional["PrometheusLogger"], + ) -> Optional[Tuple[Optional[str], Optional[str]]]: """ - Check if the batch JOB has been tracked. - - get all status="validating" and file_purpose="batch" jobs - - check if batch is now complete - - if not, return False - - if so, return True + Fetch a completed batch's results, compute cost/usage, and emit the + aretrieve_batch spend log. Returns (model_name, llm_provider) on + success, None when the job can't be routed to a deployment. Raises on + results-fetch or cost-computation failures so the caller can leave the + job unprocessed and retry it on a later poll. """ from litellm.batches.batch_utils import ( _get_file_content_as_dictionary, @@ -296,6 +304,184 @@ class CheckBatchCost: _is_base64_encoded_unified_file_id, ) + verbose_proxy_logger.info( + f"Batch ID: {batch_id} is complete, tracking cost and usage" + ) + + # aretrieve_batch is called with the raw provider batch ID, so response.id + # is the raw provider value (e.g. "batch_20260223-0518.234"). We need the + # unified base64 ID in the S3 log so downstream consumers can correlate it + # back to the batch they submitted via the proxy. + # + # CheckBatchCost builds its own LiteLLMLogging object (logging_obj below) and + # calls async_success_handler(result=response) directly. That handler calls + # _build_standard_logging_payload(response, ...) which reads response.id at + # that point — so setting response.id here is sufficient. + # + # The HTTP endpoint does this substitution via the managed files hook + # (async_post_call_success_hook). CheckBatchCost bypasses that hook entirely, + # so we do it explicitly here. + response.id = job.unified_object_id + + # This background job runs as default_user_id, so going through the HTTP endpoint + # would trigger check_managed_file_id_access and get 403. Instead, extract the raw + # provider file ID and call afile_content directly with deployment credentials. + raw_output_file_id = response.output_file_id + decoded = _is_base64_encoded_unified_file_id(raw_output_file_id) + if decoded: + try: + raw_output_file_id = decoded.split("llm_output_file_id,")[1].split(";")[0] + except (IndexError, AttributeError): + pass + + credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {} + _file_content = await afile_content( + file_id=raw_output_file_id, + **credentials, + ) + + # Access content - handle both direct attribute and method call + if hasattr(_file_content, 'content'): + content_bytes = _file_content.content # type: ignore[union-attr] + elif hasattr(_file_content, 'read'): + content_bytes = await _file_content.read() # type: ignore[misc] + else: + content_bytes = _file_content # type: ignore[assignment] + + file_content_as_dict = _get_file_content_as_dictionary( + content_bytes # type: ignore[arg-type] + ) + + # Record output file size + if prom_logger and content_bytes: + try: + prom_logger.record_managed_file_size( + size_bytes=len(content_bytes), # type: ignore + purpose="batch", + file_type="output", + model=model_id, + ) + except Exception: + pass + + deployment_info = self.llm_router.get_deployment(model_id=model_id) + if deployment_info is None: + verbose_proxy_logger.info( + f"Skipping job {job.unified_object_id} because it is not a valid deployment info" + ) + self._record_error(prom_logger, "deployment_not_found") + return None + custom_llm_provider = deployment_info.litellm_params.custom_llm_provider + litellm_model_name = deployment_info.litellm_params.model + + model_name, llm_provider, _, _ = get_llm_provider( + model=litellm_model_name, + custom_llm_provider=custom_llm_provider, + ) + + # CheckBatchCost bypasses async_post_call_success_hook, so convert raw + # output/error file IDs to managed base64 IDs before the DB write here. + managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files") + if managed_files_hook is not None: + from litellm.proxy._types import UserAPIKeyAuth + _minimal_auth = UserAPIKeyAuth( + user_id=job.created_by or "default-user-id", + team_id=getattr(job, "team_id", None), + ) + for _file_attr in ["output_file_id", "error_file_id"]: + _raw_file_id = getattr(response, _file_attr, None) + if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id): + try: + _unified_file_id = managed_files_hook.get_unified_output_file_id( + output_file_id=_raw_file_id, + model_id=model_id, + model_name=str(model_name) if model_name else deployment_info.model_name or None, + ) + await managed_files_hook.store_unified_file_id( + file_id=_unified_file_id, + file_object=None, + litellm_parent_otel_span=None, + model_mappings={model_id: _raw_file_id}, + user_api_key_dict=_minimal_auth, + ) + setattr(response, _file_attr, _unified_file_id) + verbose_proxy_logger.info( + f"CheckBatchCost: converted {_file_attr} " + f"{_raw_file_id!r} -> managed ID for batch {batch_id}" + ) + except Exception as _e: + verbose_proxy_logger.warning( + f"CheckBatchCost: failed to create managed file ID for " + f"{_file_attr}={_raw_file_id!r}: {_e}" + ) + + # Pass deployment model_info so custom batch pricing + # (input_cost_per_token_batches etc.) is used for cost calc + deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {} + batch_cost, batch_usage, batch_models = ( + await calculate_batch_cost_and_usage( + file_content_dictionary=file_content_as_dict, + custom_llm_provider=llm_provider, # type: ignore + model_name=model_name, + model_info=deployment_model_info, # type: ignore[arg-type] + ) + ) + logging_obj = LiteLLMLogging( + model=batch_models[0], + messages=[{"role": "user", "content": ""}], + stream=False, + call_type="aretrieve_batch", + start_time=datetime.now(), + litellm_call_id=str(uuid.uuid4()), + function_id=str(uuid.uuid4()), + ) + + creator_user_id = job.created_by + user_info = await self._get_user_info(batch_id, job.created_by) + + logging_obj.update_environment_variables( + litellm_params={ + # set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks + "proxy_server_request": { + "headers": { + "user-agent": CHECK_BATCH_COST_USER_AGENT, + } + }, + "metadata": { + "user_api_key_user_id": creator_user_id, + **user_info, + }, + }, + optional_params={}, + ) + + await logging_obj.async_success_handler( + result=response, + batch_cost=batch_cost, + batch_usage=batch_usage, + batch_models=batch_models, + ) + + # Record batch duration (completed_at - created_at) + if prom_logger and response.completed_at and response.created_at: + duration_seconds = float(response.completed_at - response.created_at) + if duration_seconds >= 0: + prom_logger.record_managed_batch_duration( + duration_seconds=duration_seconds, + model=model_name, + api_provider=str(llm_provider) if llm_provider else None, + ) + + return model_name, str(llm_provider) if llm_provider else None + + async def check_batch_cost(self): + """ + Check if the batch JOB has been tracked. + - get all status="validating" and file_purpose="batch" jobs + - check if batch is now complete + - if not, return False + - if so, return True + """ try: from litellm.integrations.prometheus import PrometheusLogger prom_logger = PrometheusLogger.get_instance() @@ -381,177 +567,26 @@ class CheckBatchCost: response.status == "completed" and response.output_file_id is not None ): - verbose_proxy_logger.info( - f"Batch ID: {batch_id} is complete, tracking cost and usage" - ) - - # aretrieve_batch is called with the raw provider batch ID, so response.id - # is the raw provider value (e.g. "batch_20260223-0518.234"). We need the - # unified base64 ID in the S3 log so downstream consumers can correlate it - # back to the batch they submitted via the proxy. - # - # CheckBatchCost builds its own LiteLLMLogging object (logging_obj below) and - # calls async_success_handler(result=response) directly. That handler calls - # _build_standard_logging_payload(response, ...) which reads response.id at - # that point — so setting response.id here is sufficient. - # - # The HTTP endpoint does this substitution via the managed files hook - # (async_post_call_success_hook). CheckBatchCost bypasses that hook entirely, - # so we do it explicitly here. - response.id = job.unified_object_id - - # This background job runs as default_user_id, so going through the HTTP endpoint - # would trigger check_managed_file_id_access and get 403. Instead, extract the raw - # provider file ID and call afile_content directly with deployment credentials. - raw_output_file_id = response.output_file_id - decoded = _is_base64_encoded_unified_file_id(raw_output_file_id) - if decoded: - try: - raw_output_file_id = decoded.split("llm_output_file_id,")[1].split(";")[0] - except (IndexError, AttributeError): - pass - - credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {} - _file_content = await afile_content( - file_id=raw_output_file_id, - **credentials, - ) - - # Access content - handle both direct attribute and method call - if hasattr(_file_content, 'content'): - content_bytes = _file_content.content # type: ignore[union-attr] - elif hasattr(_file_content, 'read'): - content_bytes = await _file_content.read() # type: ignore[misc] - else: - content_bytes = _file_content # type: ignore[assignment] - - file_content_as_dict = _get_file_content_as_dictionary( - content_bytes # type: ignore[arg-type] - ) - - # Record output file size - if prom_logger and content_bytes: - try: - prom_logger.record_managed_file_size( - size_bytes=len(content_bytes), # type: ignore - purpose="batch", - file_type="output", - model=model_id, - ) - except Exception: - pass - - deployment_info = self.llm_router.get_deployment(model_id=model_id) - if deployment_info is None: - verbose_proxy_logger.info( - f"Skipping job {job.unified_object_id} because it is not a valid deployment info" + try: + tracked = await self._track_completed_batch_cost( + job=job, + response=response, + model_id=model_id, + batch_id=batch_id, + prom_logger=prom_logger, ) - if prom_logger: - prom_logger.record_check_batch_cost_error("deployment_not_found") + except Exception as tracking_err: + verbose_proxy_logger.error( + f"CheckBatchCost: failed to track cost for batch {batch_id} " + f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}" + ) + self._record_error(prom_logger, "cost_tracking_error") + continue + if tracked is None: continue - custom_llm_provider = deployment_info.litellm_params.custom_llm_provider - litellm_model_name = deployment_info.litellm_params.model - - model_name, llm_provider, _, _ = get_llm_provider( - model=litellm_model_name, - custom_llm_provider=custom_llm_provider, - ) - - # CheckBatchCost bypasses async_post_call_success_hook, so convert raw - # output/error file IDs to managed base64 IDs before the DB write here. - managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files") - if managed_files_hook is not None: - from litellm.proxy._types import UserAPIKeyAuth - _minimal_auth = UserAPIKeyAuth( - user_id=job.created_by or "default-user-id", - team_id=getattr(job, "team_id", None), - ) - for _file_attr in ["output_file_id", "error_file_id"]: - _raw_file_id = getattr(response, _file_attr, None) - if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id): - try: - _unified_file_id = managed_files_hook.get_unified_output_file_id( - output_file_id=_raw_file_id, - model_id=model_id, - model_name=str(model_name) if model_name else deployment_info.model_name or None, - ) - await managed_files_hook.store_unified_file_id( - file_id=_unified_file_id, - file_object=None, - litellm_parent_otel_span=None, - model_mappings={model_id: _raw_file_id}, - user_api_key_dict=_minimal_auth, - ) - setattr(response, _file_attr, _unified_file_id) - verbose_proxy_logger.info( - f"CheckBatchCost: converted {_file_attr} " - f"{_raw_file_id!r} -> managed ID for batch {batch_id}" - ) - except Exception as _e: - verbose_proxy_logger.warning( - f"CheckBatchCost: failed to create managed file ID for " - f"{_file_attr}={_raw_file_id!r}: {_e}" - ) - - # Pass deployment model_info so custom batch pricing - # (input_cost_per_token_batches etc.) is used for cost calc - deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {} - batch_cost, batch_usage, batch_models = ( - await calculate_batch_cost_and_usage( - file_content_dictionary=file_content_as_dict, - custom_llm_provider=llm_provider, # type: ignore - model_name=model_name, - model_info=deployment_model_info, # type: ignore[arg-type] - ) - ) - logging_obj = LiteLLMLogging( - model=batch_models[0], - messages=[{"role": "user", "content": ""}], - stream=False, - call_type="aretrieve_batch", - start_time=datetime.now(), - litellm_call_id=str(uuid.uuid4()), - function_id=str(uuid.uuid4()), - ) - - creator_user_id = job.created_by - user_info = await self._get_user_info(batch_id, job.created_by) - - logging_obj.update_environment_variables( - litellm_params={ - # set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks - "proxy_server_request": { - "headers": { - "user-agent": CHECK_BATCH_COST_USER_AGENT, - } - }, - "metadata": { - "user_api_key_user_id": creator_user_id, - **user_info, - }, - }, - optional_params={}, - ) - - await logging_obj.async_success_handler( - result=response, - batch_cost=batch_cost, - batch_usage=batch_usage, - batch_models=batch_models, - ) - - # Record batch duration (completed_at - created_at) - if prom_logger and response.completed_at and response.created_at: - duration_seconds = float(response.completed_at - response.created_at) - if duration_seconds >= 0: - prom_logger.record_managed_batch_duration( - duration_seconds=duration_seconds, - model=model_name, - api_provider=str(llm_provider) if llm_provider else None, - ) # Track this job for the final metrics summary - processed_models.append((model_name, str(llm_provider) if llm_provider else None)) + processed_models.append(tracked) # mark the job as complete try: diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index f2ad04510a8..b3864ce7878 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.45" +version = "0.1.48" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.45" +version = "0.1.48" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 716b2fa09d1..da2f2c9c1e0 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/gateway/main.py b/gateway/main.py index 09d30f5da3f..61b885b27e4 100644 --- a/gateway/main.py +++ b/gateway/main.py @@ -25,17 +25,25 @@ DatabaseURLSettings.from_env().apply_to_env() from litellm.proxy.proxy_server import app -from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES +from gateway.routes.allowlist import ( + GATEWAY_EXACT_PATHS, + GATEWAY_MOUNT_PATHS, + GATEWAY_PATH_PREFIXES, +) def _is_gateway_route(route) -> bool: - """Keep the route on the gateway if its path is in the LLM data-plane surface.""" + """Keep the route on the gateway if its path is in the LLM data-plane surface. + + Prometheus registers /metrics as a Mount (``app.mount("/metrics", make_asgi_app())``), + so Mounts are matched against GATEWAY_MOUNT_PATHS instead of being dropped with + the UI static mounts. + """ path = getattr(route, "path", None) if path is None: return False if isinstance(route, Mount): - # Gateway never serves the static UI or its asset bundles. - return False + return path in GATEWAY_MOUNT_PATHS if path in GATEWAY_EXACT_PATHS: return True return any(path.startswith(prefix) for prefix in GATEWAY_PATH_PREFIXES) diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 97dc0691a0e..6cb55327dd4 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -107,7 +107,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( # Health & ops "/health", "/metrics", - "/watsonx" + "/watsonx", ) GATEWAY_EXACT_PATHS: frozenset[str] = frozenset( @@ -121,3 +121,9 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset( "/test", } ) + +GATEWAY_MOUNT_PATHS: frozenset[str] = frozenset( + { + "/metrics", + } +) diff --git a/deploy/charts/litellm-helm/.helmignore b/helm/litellm-helm/.helmignore similarity index 100% rename from deploy/charts/litellm-helm/.helmignore rename to helm/litellm-helm/.helmignore diff --git a/deploy/charts/litellm-helm/Chart.lock b/helm/litellm-helm/Chart.lock similarity index 100% rename from deploy/charts/litellm-helm/Chart.lock rename to helm/litellm-helm/Chart.lock diff --git a/deploy/charts/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml similarity index 100% rename from deploy/charts/litellm-helm/Chart.yaml rename to helm/litellm-helm/Chart.yaml diff --git a/deploy/charts/litellm-helm/README.md b/helm/litellm-helm/README.md similarity index 100% rename from deploy/charts/litellm-helm/README.md rename to helm/litellm-helm/README.md diff --git a/deploy/charts/litellm-helm/charts/postgresql-14.3.1.tgz b/helm/litellm-helm/charts/postgresql-14.3.1.tgz similarity index 100% rename from deploy/charts/litellm-helm/charts/postgresql-14.3.1.tgz rename to helm/litellm-helm/charts/postgresql-14.3.1.tgz diff --git a/deploy/charts/litellm-helm/charts/redis-18.19.1.tgz b/helm/litellm-helm/charts/redis-18.19.1.tgz similarity index 100% rename from deploy/charts/litellm-helm/charts/redis-18.19.1.tgz rename to helm/litellm-helm/charts/redis-18.19.1.tgz diff --git a/deploy/charts/litellm-helm/ci/test-values.yaml b/helm/litellm-helm/ci/test-values.yaml similarity index 100% rename from deploy/charts/litellm-helm/ci/test-values.yaml rename to helm/litellm-helm/ci/test-values.yaml diff --git a/deploy/charts/litellm-helm/templates/NOTES.txt b/helm/litellm-helm/templates/NOTES.txt similarity index 100% rename from deploy/charts/litellm-helm/templates/NOTES.txt rename to helm/litellm-helm/templates/NOTES.txt diff --git a/deploy/charts/litellm-helm/templates/_helpers.tpl b/helm/litellm-helm/templates/_helpers.tpl similarity index 100% rename from deploy/charts/litellm-helm/templates/_helpers.tpl rename to helm/litellm-helm/templates/_helpers.tpl diff --git a/deploy/charts/litellm-helm/templates/configmap-litellm.yaml b/helm/litellm-helm/templates/configmap-litellm.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/configmap-litellm.yaml rename to helm/litellm-helm/templates/configmap-litellm.yaml diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/deployment.yaml rename to helm/litellm-helm/templates/deployment.yaml diff --git a/deploy/charts/litellm-helm/templates/extra-resources.yaml b/helm/litellm-helm/templates/extra-resources.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/extra-resources.yaml rename to helm/litellm-helm/templates/extra-resources.yaml diff --git a/deploy/charts/litellm-helm/templates/hpa.yaml b/helm/litellm-helm/templates/hpa.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/hpa.yaml rename to helm/litellm-helm/templates/hpa.yaml diff --git a/deploy/charts/litellm-helm/templates/ingress.yaml b/helm/litellm-helm/templates/ingress.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/ingress.yaml rename to helm/litellm-helm/templates/ingress.yaml diff --git a/deploy/charts/litellm-helm/templates/keda.yaml b/helm/litellm-helm/templates/keda.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/keda.yaml rename to helm/litellm-helm/templates/keda.yaml diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/helm/litellm-helm/templates/migrations-job.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/migrations-job.yaml rename to helm/litellm-helm/templates/migrations-job.yaml diff --git a/deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml b/helm/litellm-helm/templates/poddisruptionbudget.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml rename to helm/litellm-helm/templates/poddisruptionbudget.yaml diff --git a/deploy/charts/litellm-helm/templates/secret-dbcredentials.yaml b/helm/litellm-helm/templates/secret-dbcredentials.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/secret-dbcredentials.yaml rename to helm/litellm-helm/templates/secret-dbcredentials.yaml diff --git a/deploy/charts/litellm-helm/templates/secret-masterkey.yaml b/helm/litellm-helm/templates/secret-masterkey.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/secret-masterkey.yaml rename to helm/litellm-helm/templates/secret-masterkey.yaml diff --git a/deploy/charts/litellm-helm/templates/service.yaml b/helm/litellm-helm/templates/service.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/service.yaml rename to helm/litellm-helm/templates/service.yaml diff --git a/deploy/charts/litellm-helm/templates/serviceaccount.yaml b/helm/litellm-helm/templates/serviceaccount.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/serviceaccount.yaml rename to helm/litellm-helm/templates/serviceaccount.yaml diff --git a/deploy/charts/litellm-helm/templates/servicemonitor.yaml b/helm/litellm-helm/templates/servicemonitor.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/servicemonitor.yaml rename to helm/litellm-helm/templates/servicemonitor.yaml diff --git a/deploy/charts/litellm-helm/templates/tests/test-connection.yaml b/helm/litellm-helm/templates/tests/test-connection.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/tests/test-connection.yaml rename to helm/litellm-helm/templates/tests/test-connection.yaml diff --git a/deploy/charts/litellm-helm/templates/tests/test-env-vars.yaml b/helm/litellm-helm/templates/tests/test-env-vars.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/tests/test-env-vars.yaml rename to helm/litellm-helm/templates/tests/test-env-vars.yaml diff --git a/deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml b/helm/litellm-helm/templates/tests/test-servicemonitor.yaml similarity index 100% rename from deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml rename to helm/litellm-helm/templates/tests/test-servicemonitor.yaml diff --git a/deploy/charts/litellm-helm/tests/deployment_command_args_labels_tests.yaml b/helm/litellm-helm/tests/deployment_command_args_labels_tests.yaml similarity index 100% rename from deploy/charts/litellm-helm/tests/deployment_command_args_labels_tests.yaml rename to helm/litellm-helm/tests/deployment_command_args_labels_tests.yaml diff --git a/deploy/charts/litellm-helm/tests/deployment_tests.yaml b/helm/litellm-helm/tests/deployment_tests.yaml similarity index 100% rename from deploy/charts/litellm-helm/tests/deployment_tests.yaml rename to helm/litellm-helm/tests/deployment_tests.yaml diff --git a/deploy/charts/litellm-helm/tests/hpa_tests.yaml b/helm/litellm-helm/tests/hpa_tests.yaml similarity index 100% rename from deploy/charts/litellm-helm/tests/hpa_tests.yaml rename to helm/litellm-helm/tests/hpa_tests.yaml diff --git a/deploy/charts/litellm-helm/tests/ingress_tests.yaml b/helm/litellm-helm/tests/ingress_tests.yaml similarity index 100% rename from deploy/charts/litellm-helm/tests/ingress_tests.yaml rename to helm/litellm-helm/tests/ingress_tests.yaml diff --git a/deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml b/helm/litellm-helm/tests/masterkey-secret_tests.yaml similarity index 100% rename from deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml rename to helm/litellm-helm/tests/masterkey-secret_tests.yaml diff --git a/deploy/charts/litellm-helm/tests/migrations-job_tests.yaml b/helm/litellm-helm/tests/migrations-job_tests.yaml similarity index 100% rename from deploy/charts/litellm-helm/tests/migrations-job_tests.yaml rename to helm/litellm-helm/tests/migrations-job_tests.yaml diff --git a/deploy/charts/litellm-helm/tests/pdb_tests.yaml b/helm/litellm-helm/tests/pdb_tests.yaml similarity index 100% rename from deploy/charts/litellm-helm/tests/pdb_tests.yaml rename to helm/litellm-helm/tests/pdb_tests.yaml diff --git a/deploy/charts/litellm-helm/tests/service_tests.yaml b/helm/litellm-helm/tests/service_tests.yaml similarity index 100% rename from deploy/charts/litellm-helm/tests/service_tests.yaml rename to helm/litellm-helm/tests/service_tests.yaml diff --git a/deploy/charts/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml similarity index 100% rename from deploy/charts/litellm-helm/values.yaml rename to helm/litellm-helm/values.yaml diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index b355db43540..8b4552bf302 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -45,11 +45,16 @@ spec: value: /app/config/config.yaml {{- end }} {{- include "litellm.envFrom" .Values.backend | nindent 10 }} - {{- if .Values.gateway.config.create }} + {{- if or .Values.gateway.config.create .Values.backend.volumeMounts }} volumeMounts: + {{- if .Values.gateway.config.create }} - name: gateway-config mountPath: /app/config/config.yaml subPath: config.yaml + {{- end }} + {{- with .Values.backend.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} {{- end }} {{- with .Values.backend.livenessProbe }} livenessProbe: @@ -61,11 +66,16 @@ spec: {{- end }} resources: {{- toYaml .Values.backend.resources | nindent 12 }} - {{- if .Values.gateway.config.create }} + {{- if or .Values.gateway.config.create .Values.backend.volumes }} volumes: + {{- if .Values.gateway.config.create }} - name: gateway-config configMap: name: {{ include "litellm.gateway.fullname" . }}-config + {{- end }} + {{- with .Values.backend.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} {{- with .Values.backend.nodeSelector }} nodeSelector: diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 05ea4052159..bd491b69e0f 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -47,11 +47,16 @@ spec: value: {{ .Values.gateway.numWorkers | quote }} {{- end }} {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} - {{- if .Values.gateway.config.create }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts }} volumeMounts: + {{- if .Values.gateway.config.create }} - name: gateway-config mountPath: /app/config/config.yaml subPath: config.yaml + {{- end }} + {{- with .Values.gateway.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} {{- end }} {{- with .Values.gateway.livenessProbe }} livenessProbe: @@ -63,11 +68,16 @@ spec: {{- end }} resources: {{- toYaml .Values.gateway.resources | nindent 12 }} - {{- if .Values.gateway.config.create }} + {{- if or .Values.gateway.config.create .Values.gateway.volumes }} volumes: + {{- if .Values.gateway.config.create }} - name: gateway-config configMap: name: {{ include "litellm.gateway.fullname" . }}-config + {{- end }} + {{- with .Values.gateway.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} {{- with .Values.gateway.nodeSelector }} nodeSelector: diff --git a/helm/litellm/templates/ui/deployment.yaml b/helm/litellm/templates/ui/deployment.yaml index b40b44cca53..79e9a3e43bb 100644 --- a/helm/litellm/templates/ui/deployment.yaml +++ b/helm/litellm/templates/ui/deployment.yaml @@ -46,6 +46,10 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- include "litellm.envFrom" .Values.ui | nindent 10 }} + {{- with .Values.ui.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} {{- with .Values.ui.livenessProbe }} livenessProbe: {{- toYaml . | nindent 12 }} @@ -56,6 +60,10 @@ spec: {{- end }} resources: {{- toYaml .Values.ui.resources | nindent 12 }} + {{- with .Values.ui.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.ui.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/tests/deployment_volumes_tests.yaml b/helm/litellm/tests/deployment_volumes_tests.yaml new file mode 100644 index 00000000000..3a64300b86c --- /dev/null +++ b/helm/litellm/tests/deployment_volumes_tests.yaml @@ -0,0 +1,172 @@ +suite: test deployment volumes and volumeMounts +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - backend/deployment.yaml + - ui/deployment.yaml +values: + - ./values/required.yaml +tests: + - it: gateway renders only the config volume by default + template: gateway/deployment.yaml + asserts: + - equal: + path: spec.template.spec.volumes + value: + - name: gateway-config + configMap: + name: RELEASE-NAME-litellm-gateway-config + - equal: + path: spec.template.spec.containers[0].volumeMounts + value: + - name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + + - it: gateway merges user volumes and volumeMounts with the config volume + template: gateway/deployment.yaml + set: + gateway.volumes: + - name: custom-callbacks + configMap: + name: custom-callbacks + gateway.volumeMounts: + - name: custom-callbacks + mountPath: /app/custom_callbacks.py + subPath: custom_callbacks.py + asserts: + - equal: + path: spec.template.spec.volumes[0].name + value: gateway-config + - equal: + path: spec.template.spec.volumes[1] + value: + name: custom-callbacks + configMap: + name: custom-callbacks + - equal: + path: spec.template.spec.containers[0].volumeMounts[0].name + value: gateway-config + - equal: + path: spec.template.spec.containers[0].volumeMounts[1] + value: + name: custom-callbacks + mountPath: /app/custom_callbacks.py + subPath: custom_callbacks.py + + - it: gateway renders user volumes even when config creation is disabled + template: gateway/deployment.yaml + set: + gateway.config.create: false + gateway.volumes: + - name: certs + secret: + secretName: tls-certs + gateway.volumeMounts: + - name: certs + mountPath: /etc/certs + readOnly: true + asserts: + - equal: + path: spec.template.spec.volumes + value: + - name: certs + secret: + secretName: tls-certs + - equal: + path: spec.template.spec.containers[0].volumeMounts + value: + - name: certs + mountPath: /etc/certs + readOnly: true + + - it: gateway omits volumes when config creation is disabled and no user volumes are set + template: gateway/deployment.yaml + set: + gateway.config.create: false + asserts: + - isNull: + path: spec.template.spec.volumes + - isNull: + path: spec.template.spec.containers[0].volumeMounts + + - it: backend merges user volumes and volumeMounts with the shared config volume + template: backend/deployment.yaml + set: + backend.volumes: + - name: sso-handler + configMap: + name: sso-handler + backend.volumeMounts: + - name: sso-handler + mountPath: /app/custom_sso.py + subPath: custom_sso.py + asserts: + - equal: + path: spec.template.spec.volumes[0].name + value: gateway-config + - equal: + path: spec.template.spec.volumes[1] + value: + name: sso-handler + configMap: + name: sso-handler + - equal: + path: spec.template.spec.containers[0].volumeMounts[1] + value: + name: sso-handler + mountPath: /app/custom_sso.py + subPath: custom_sso.py + + - it: backend renders user volumes even when config creation is disabled + template: backend/deployment.yaml + set: + gateway.config.create: false + backend.volumes: + - name: data + emptyDir: {} + backend.volumeMounts: + - name: data + mountPath: /data + asserts: + - equal: + path: spec.template.spec.volumes + value: + - name: data + emptyDir: {} + - equal: + path: spec.template.spec.containers[0].volumeMounts + value: + - name: data + mountPath: /data + + - it: ui renders no volumes by default + template: ui/deployment.yaml + asserts: + - isNull: + path: spec.template.spec.volumes + - isNull: + path: spec.template.spec.containers[0].volumeMounts + + - it: ui renders user volumes and volumeMounts + template: ui/deployment.yaml + set: + ui.volumes: + - name: nginx-config + configMap: + name: custom-nginx + ui.volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/conf.d + asserts: + - equal: + path: spec.template.spec.volumes + value: + - name: nginx-config + configMap: + name: custom-nginx + - equal: + path: spec.template.spec.containers[0].volumeMounts + value: + - name: nginx-config + mountPath: /etc/nginx/conf.d diff --git a/helm/litellm/tests/values/required.yaml b/helm/litellm/tests/values/required.yaml new file mode 100644 index 00000000000..21d3f7a5a6a --- /dev/null +++ b/helm/litellm/tests/values/required.yaml @@ -0,0 +1,4 @@ +database: + writer: + host: postgres.example.com + dbname: litellm diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 934661643bd..6aa5dd39cd0 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -124,6 +124,11 @@ gateway: extraEnv: [] # Add extra environment variables to the gateway envConfigMaps: [] # Add extra environment variables to the gateway from config maps envSecrets: [] # Add extra environment variables to the gateway from secrets + # Additional volumes on the gateway Deployment (e.g. a ConfigMap holding + # custom callback / SSO handler code, mounted next to the proxy config). + volumes: [] + # Additional volumeMounts on the gateway container. + volumeMounts: [] config: create: true proxy_config: {} @@ -167,6 +172,10 @@ backend: extraEnv: [] envConfigMaps: [] envSecrets: [] + # Additional volumes on the backend Deployment. + volumes: [] + # Additional volumeMounts on the backend container. + volumeMounts: [] image: repository: ghcr.io/berriai/litellm-backend tag: "" @@ -206,6 +215,10 @@ ui: extraEnv: [] envConfigMaps: [] envSecrets: [] + # Additional volumes on the ui Deployment. + volumes: [] + # Additional volumeMounts on the ui container. + volumeMounts: [] image: repository: ghcr.io/berriai/litellm-ui tag: "" diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260629000000_add_max_concurrent_requests_to_mcp_server_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260629000000_add_max_concurrent_requests_to_mcp_server_table/migration.sql new file mode 100644 index 00000000000..eeeecce741d --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260629000000_add_max_concurrent_requests_to_mcp_server_table/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "max_concurrent_requests" INTEGER; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260630190000_add_budget_fallbacks_to_litellm_verification_token/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260630190000_add_budget_fallbacks_to_litellm_verification_token/migration.sql new file mode 100644 index 00000000000..1a5c16288de --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260630190000_add_budget_fallbacks_to_litellm_verification_token/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}'; + +-- AlterTable +ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}'; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 5351e0a1470..f9ab5e6aefd 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -338,6 +338,7 @@ model LiteLLM_MCPServerTable { byok_api_key_help_url String? source_url String? timeout Float? + max_concurrent_requests Int? // BYOM submission lifecycle approval_status String? @default("active") submitted_by String? @@ -418,6 +419,7 @@ model LiteLLM_VerificationToken { access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") + budget_fallbacks Json @default("{}") budget_id String? organization_id String? object_permission_id String? @@ -511,6 +513,7 @@ model LiteLLM_DeletedVerificationToken { access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") + budget_fallbacks Json @default("{}") router_settings Json? @default("{}") budget_id String? organization_id String? diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index e2a86205fc5..4d237622da2 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.74" +version = "0.4.75" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.74" +version = "0.4.75" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index 9327e121b1d..6e2a03b7c7c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -379,6 +379,7 @@ budget_duration: Optional[str] = ( None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). ) default_soft_budget: float = DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0 +budget_exceeded_throttle_percentage: Optional[float] = None forward_traceparent_to_llm_provider: bool = False @@ -588,6 +589,7 @@ gemini_models: Set = set() xai_models: Set = set() zai_models: Set = set() deepseek_models: Set = set() +tencent_models: Set = set() runwayml_models: Set = set() azure_ai_models: Set = set() jina_ai_models: Set = set() @@ -801,6 +803,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): fal_ai_models.add(key) elif value.get("litellm_provider") == "deepseek": deepseek_models.add(key) + elif value.get("litellm_provider") == "tencent": + tencent_models.add(key) elif value.get("litellm_provider") == "runwayml": runwayml_models.add(key) elif value.get("litellm_provider") == "meta_llama": @@ -1093,6 +1097,7 @@ models_by_provider: dict = { "zai": zai_models, "fal_ai": fal_ai_models, "deepseek": deepseek_models, + "tencent": tencent_models, "runwayml": runwayml_models, "mistral": mistral_chat_models, "azure_ai": azure_ai_models, @@ -1804,6 +1809,9 @@ if TYPE_CHECKING: from .llms.deepseek.chat.transformation import ( DeepSeekChatConfig as _DeepSeekChatConfig, ) + from .llms.tencent.chat.transformation import ( + TencentChatConfig as _TencentChatConfig, + ) from .llms.sap.chat.transformation import ( GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig, ) @@ -1846,6 +1854,7 @@ if TYPE_CHECKING: # Type stubs for lazy-loaded config classes (to help mypy understand types) VLLMConfig: Type[_VLLMConfig] DeepSeekChatConfig: Type[_DeepSeekChatConfig] + TencentChatConfig: Type[_TencentChatConfig] GenAIHubOrchestrationConfig: Type[_GenAIHubOrchestrationConfig] GenAIHubEmbeddingConfig: Type[_GenAIHubEmbeddingConfig] AzureOpenAIO1Config: Type[_AzureOpenAIO1Config] diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 0f9d3a560d1..488331e3895 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -284,6 +284,7 @@ LLM_CONFIG_NAMES = ( "LiteLLMProxyChatConfig", "VLLMConfig", "DeepSeekChatConfig", + "TencentChatConfig", "LMStudioChatConfig", "LmStudioEmbeddingConfig", "NscaleConfig", @@ -1096,6 +1097,7 @@ _LLM_CONFIGS_IMPORT_MAP = { ), "VLLMConfig": (".llms.vllm.completion.transformation", "VLLMConfig"), "DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"), + "TencentChatConfig": (".llms.tencent.chat.transformation", "TencentChatConfig"), "LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"), "LmStudioEmbeddingConfig": ( ".llms.lm_studio.embed.transformation", diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 37bf7c34f02..4c23ecfed54 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -129,6 +129,33 @@ def _set_agent_id_on_logging_obj( litellm_logging_obj.model_call_details["agent_id"] = agent_id +_A2A_COST_PARAM_KEYS = ("cost_per_query", "input_cost_per_token", "output_cost_per_token") + + +def _set_litellm_params_on_logging_obj( + kwargs: dict[str, Any], + litellm_params: dict[str, Any], +) -> None: + """ + Merge the agent's pricing params into model_call_details["litellm_params"] + so A2ACostCalculator can read them. + + The non-streaming path reuses the proxy-built logging object, whose + litellm_params already carries metadata / proxy_server_request / user-key + context, so merge the pricing keys in rather than replacing the dict. + """ + logging_obj = kwargs.get("litellm_logging_obj") + if logging_obj is None: + return + + cost_params = {key: litellm_params[key] for key in _A2A_COST_PARAM_KEYS if litellm_params.get(key) is not None} + if not cost_params: + return + + existing = logging_obj.model_call_details.get("litellm_params") or {} + logging_obj.model_call_details["litellm_params"] = {**existing, **cost_params} + + def _get_a2a_model_info(a2a_client: Any, kwargs: Dict[str, Any]) -> str: """ Extract agent info and set model/custom_llm_provider for cost tracking. @@ -477,6 +504,9 @@ async def asend_message( completion_tokens=completion_tokens, ) + # Merge agent pricing params into the logging obj so cost is calculated + _set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params) + # Set agent_id on logging obj for SpendLogs tracking _set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id) diff --git a/litellm/a2a_protocol/streaming_iterator.py b/litellm/a2a_protocol/streaming_iterator.py index 529154919f3..1ef174a5eee 100644 --- a/litellm/a2a_protocol/streaming_iterator.py +++ b/litellm/a2a_protocol/streaming_iterator.py @@ -11,7 +11,6 @@ from litellm._logging import verbose_logger from litellm.a2a_protocol.cost_calculator import A2ACostCalculator from litellm.a2a_protocol.utils import A2ARequestUtils from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.thread_pool_executor import executor if TYPE_CHECKING: from a2a.types import SendStreamingMessageRequest, SendStreamingMessageResponse @@ -128,22 +127,15 @@ class A2AStreamingIterator: # Call success handlers - they will build standard_logging_object asyncio.create_task( - self.logging_obj.async_success_handler( - result=result, + self.logging_obj.dispatch_success_handlers( + result, start_time=self.start_time, end_time=end_time, cache_hit=None, + prefer_async_handlers=True, ) ) - executor.submit( - self.logging_obj.success_handler, - result=result, - cache_hit=None, - start_time=self.start_time, - end_time=end_time, - ) - verbose_logger.info( f"A2A streaming completed: prompt_tokens={prompt_tokens}, " f"completion_tokens={completion_tokens}, total_tokens={total_tokens}, " diff --git a/litellm/a2a_protocol/utils.py b/litellm/a2a_protocol/utils.py index 0dbd1eefc63..ce5a168c3ac 100644 --- a/litellm/a2a_protocol/utils.py +++ b/litellm/a2a_protocol/utils.py @@ -121,8 +121,13 @@ class A2ARequestUtils: Returns: Tuple of (prompt_tokens, completion_tokens, total_tokens) """ - # Count input tokens + # Count input tokens. Dump the message to a dict first so extraction hits + # the dict branch — request-side parts are a2a-sdk Part RootModels whose + # kind/text live on part.root, which the object branch cannot read. This + # mirrors how the response side already works (it operates on model_dump). input_message = A2ARequestUtils.get_input_message_from_request(request) + if input_message is not None and hasattr(input_message, "model_dump"): + input_message = input_message.model_dump(mode="json") input_text = A2ARequestUtils.extract_text_from_message(input_message) prompt_tokens = A2ARequestUtils.count_tokens(input_text) diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 985198ce7ce..11b07d39981 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -3,6 +3,7 @@ from typing import Any, Iterator, List, Literal, Optional, Tuple import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.llm_cost_calc.utils import _parse_prompt_tokens_details from litellm.types.llms.openai import Batch from litellm.types.utils import CallTypes, ModelInfo, Usage from litellm.utils import token_counter @@ -34,7 +35,7 @@ async def calculate_batch_cost_and_usage( custom_llm_provider=custom_llm_provider, model_name=model_name, ) - batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name) + batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name, custom_llm_provider) return batch_cost, batch_usage, batch_models @@ -70,7 +71,7 @@ async def _handle_completed_batch( model_name=model_name, ) - batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name) + batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name, custom_llm_provider) return batch_cost, batch_usage, batch_models @@ -78,6 +79,7 @@ async def _handle_completed_batch( def _get_batch_models_from_file_content( file_content_dictionary: List[dict], model_name: Optional[str] = None, + custom_llm_provider: str = "openai", ) -> List[str]: """ Get the models from the file content @@ -86,8 +88,8 @@ def _get_batch_models_from_file_content( return [model_name] batch_models = [] for _item in file_content_dictionary: - if _batch_response_was_successful(_item): - _response_body = _get_response_from_batch_job_output_file(_item) + if _batch_response_was_successful(_item, custom_llm_provider): + _response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider) _model = _response_body.get("model") if _model: batch_models.append(_model) @@ -373,10 +375,10 @@ def _get_batch_job_cost_from_file_content( # parse the file content as json verbose_logger.debug("file_content_dictionary=%s", json.dumps(file_content_dictionary, indent=4)) for _item in file_content_dictionary: - if _batch_response_was_successful(_item): - _response_body = _get_response_from_batch_job_output_file(_item) - if model_info is not None: - usage = _get_batch_job_usage_from_response_body(_response_body) + if _batch_response_was_successful(_item, custom_llm_provider): + _response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider) + if model_info is not None or custom_llm_provider == "anthropic": + usage = _get_batch_job_usage_from_response_body(_response_body, custom_llm_provider) model = _response_body.get("model", "") prompt_cost, completion_cost = batch_cost_calculator( usage=usage, @@ -418,17 +420,31 @@ def _get_batch_job_total_usage_from_file_content( total_tokens: int = 0 prompt_tokens: int = 0 completion_tokens: int = 0 + cache_read_tokens: int = 0 + cache_creation_tokens: int = 0 for _item in file_content_dictionary: - if _batch_response_was_successful(_item): - _response_body = _get_response_from_batch_job_output_file(_item) - usage: Usage = _get_batch_job_usage_from_response_body(_response_body) + if _batch_response_was_successful(_item, custom_llm_provider): + _response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider) + usage: Usage = _get_batch_job_usage_from_response_body(_response_body, custom_llm_provider) total_tokens += usage.total_tokens prompt_tokens += usage.prompt_tokens completion_tokens += usage.completion_tokens + prompt_details = _parse_prompt_tokens_details(usage) + cache_read_tokens += prompt_details["cache_hit_tokens"] + cache_creation_tokens += prompt_details["cache_creation_tokens"] + cache_token_params = { + key: tokens + for key, tokens in ( + ("cache_read_input_tokens", cache_read_tokens), + ("cache_creation_input_tokens", cache_creation_tokens), + ) + if tokens > 0 + } return Usage( total_tokens=total_tokens, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, + **cache_token_params, ) @@ -465,27 +481,51 @@ def _count_prompt_or_input_tokens(model: str, value: Any) -> int: return 0 -def _get_batch_job_usage_from_response_body(response_body: dict) -> Usage: +def _get_batch_job_usage_from_response_body(response_body: dict, custom_llm_provider: str = "openai") -> Usage: """ Get the tokens of a batch job from the response body """ + if custom_llm_provider == "anthropic": + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + return AnthropicConfig().calculate_usage( + usage_object=response_body.get("usage", None) or {}, + reasoning_content=None, + ) _usage_dict = response_body.get("usage", None) or {} usage: Usage = Usage(**_usage_dict) return usage -def _get_response_from_batch_job_output_file(batch_job_output_file: dict) -> Any: +def _get_anthropic_result_from_batch_results_line(batch_results_line: dict) -> dict: + """ + Get the ``result`` object from a line of an Anthropic message batch results JSONL file. + + Anthropic batch results lines look like: + ``{"custom_id": ..., "result": {"type": "succeeded", "message": {..., "usage": {...}}}}`` + """ + return batch_results_line.get("result", None) or {} + + +def _get_response_from_batch_job_output_file(batch_job_output_file: dict, custom_llm_provider: str = "openai") -> Any: """ Get the response from the batch job output file """ + if custom_llm_provider == "anthropic": + return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("message", None) or {} _response: dict = batch_job_output_file.get("response", None) or {} _response_body = _response.get("body", None) or {} return _response_body -def _batch_response_was_successful(batch_job_output_file: dict) -> bool: +def _batch_response_was_successful(batch_job_output_file: dict, custom_llm_provider: str = "openai") -> bool: """ - Check if the batch job response status == 200 + Check if the batch job response was successful + + OpenAI-shaped output rows report ``response.status_code == 200``; Anthropic + message batch results lines report ``result.type == "succeeded"``. """ + if custom_llm_provider == "anthropic": + return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded" _response: dict = batch_job_output_file.get("response", None) or {} return _response.get("status_code", None) == 200 diff --git a/litellm/caching/disk_cache.py b/litellm/caching/disk_cache.py index b51acbe9cfd..d9f65ce949e 100644 --- a/litellm/caching/disk_cache.py +++ b/litellm/caching/disk_cache.py @@ -59,8 +59,9 @@ class DiskCache(BaseCache): def increment_cache(self, key, value: int, **kwargs) -> int: # get the value - init_value = self.get_cache(key=key) or 0 - value = init_value + value # type: ignore + cached_value = self.get_cache(key=key) + init_value = cached_value if isinstance(cached_value, int) else 0 + value = init_value + value self.set_cache(key, value, **kwargs) return value @@ -76,8 +77,9 @@ class DiskCache(BaseCache): async def async_increment(self, key, value: int, **kwargs) -> int: # get the value - init_value = await self.async_get_cache(key=key) or 0 - value = init_value + value # type: ignore + cached_value = await self.async_get_cache(key=key) + init_value = cached_value if isinstance(cached_value, int) else 0 + value = init_value + value await self.async_set_cache(key, value, **kwargs) return value diff --git a/litellm/caching/valkey_semantic_cache.py b/litellm/caching/valkey_semantic_cache.py index 746e91207d8..76b7f7d5b87 100644 --- a/litellm/caching/valkey_semantic_cache.py +++ b/litellm/caching/valkey_semantic_cache.py @@ -279,7 +279,7 @@ class ValkeySemanticCache(RedisSemanticCache): print_verbose("No prompt provided for semantic caching") return - embedding = await self._get_async_embedding(prompt, **kwargs) + embedding = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata")) await self._ensure_index_async(len(embedding)) doc_key = self._doc_key(key) @@ -298,7 +298,7 @@ class ValkeySemanticCache(RedisSemanticCache): kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None - embedding = await self._get_async_embedding(prompt, **kwargs) + embedding = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata")) await self._ensure_index_async(len(embedding)) search_result = await self.async_client.ft(self.index_name).search( diff --git a/litellm/constants.py b/litellm/constants.py index 6eb2779dcae..7423d9b2211 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -508,6 +508,7 @@ LITELLM_CHAT_PROVIDERS = [ "text-completion-codestral", "text-completion-inception", "deepseek", + "tencent", "sambanova", "maritalk", "cloudflare", @@ -729,6 +730,7 @@ openai_compatible_providers: List = [ "volcengine", "codestral", "deepseek", + "tencent", "deepinfra", "perplexity", "xinference", @@ -1502,6 +1504,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [ "public_model_groups_links", "cost_discount_config", "cost_margin_config", + "budget_exceeded_throttle_percentage", ] SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e8535a570c8..74dc0e19da3 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -52,6 +52,9 @@ from litellm.llms.databricks.cost_calculator import ( from litellm.llms.deepseek.cost_calculator import ( cost_per_token as deepseek_cost_per_token, ) +from litellm.llms.tencent.cost_calculator import ( + cost_per_token as tencent_cost_per_token, +) from litellm.llms.fireworks_ai.cost_calculator import ( cost_per_token as fireworks_ai_cost_per_token, ) @@ -219,7 +222,7 @@ def _cost_per_token_custom_pricing_helper( output_cost = completion_tokens * output_cost_per_token return input_cost, output_cost elif custom_cost_per_second is not None: - output_cost = custom_cost_per_second * response_time_ms / 1000 # type: ignore + output_cost = custom_cost_per_second * (response_time_ms or 0.0) / 1000 return 0, output_cost return None @@ -625,6 +628,8 @@ def cost_per_token( return gemini_cost_per_token(model=model, usage=usage_block, service_tier=service_tier) elif custom_llm_provider == "deepseek": return deepseek_cost_per_token(model=model, usage=usage_block) + elif custom_llm_provider == "tencent": + return tencent_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "perplexity": return perplexity_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "xai": @@ -657,29 +662,27 @@ def cost_per_token( data_residency=data_residency, ) - if model_info.get("input_cost_per_second", None) is not None and response_time_ms is not None: + input_cost_per_second = model_info.get("input_cost_per_second") + if input_cost_per_second is not None and response_time_ms is not None: verbose_logger.debug( "For model=%s - input_cost_per_second: %s; response time: %s", model, - model_info.get("input_cost_per_second", None), + input_cost_per_second, response_time_ms, ) ## COST PER SECOND ## - prompt_tokens_cost_usd_dollar = ( - model_info["input_cost_per_second"] * response_time_ms / 1000 # type: ignore - ) + prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000 - if model_info.get("output_cost_per_second", None) is not None and response_time_ms is not None: + output_cost_per_second = model_info.get("output_cost_per_second") + if output_cost_per_second is not None and response_time_ms is not None: verbose_logger.debug( "For model=%s - output_cost_per_second: %s; response time: %s", model, - model_info.get("output_cost_per_second", None), + output_cost_per_second, response_time_ms, ) ## COST PER SECOND ## - completion_tokens_cost_usd_dollar = ( - model_info["output_cost_per_second"] * response_time_ms / 1000 # type: ignore - ) + completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000 verbose_logger.debug( "Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s", @@ -1495,6 +1498,7 @@ def completion_cost( custom_llm_provider=custom_llm_provider, litellm_model_name=model, data_residency=data_residency, + litellm_logging_obj=litellm_logging_obj, ) elif call_type == _MCP_CALL_TYPE: from litellm.proxy._experimental.mcp_server.cost_calculator import ( @@ -2151,17 +2155,23 @@ def batch_cost_calculator( if input_cost_per_token_batches: total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches elif input_cost_per_token: + details = _parse_prompt_tokens_details(usage) + cache_read_tokens = details["cache_hit_tokens"] + cache_creation_tokens = details["cache_creation_tokens"] + # Subtract cached tokens from prompt_tokens before calculating cost # Fixes issue where cached tokens are being charged again + base_input_tokens = get_billable_input_tokens(usage) - cache_creation_tokens total_prompt_cost = ( - get_billable_input_tokens(usage) * (input_cost_per_token) / 2 + base_input_tokens * (input_cost_per_token) / 2 ) # batch cost is usually half of the regular token cost # Add cache read cost if applicable - details = _parse_prompt_tokens_details(usage) - cache_read_tokens = details["cache_hit_tokens"] cache_read_cost_key = _get_service_tier_cost_key("cache_read_input_token_cost", None) total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2 + + cache_creation_cost = model_info.get("cache_creation_input_token_cost") or input_cost_per_token + total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2 if output_cost_per_token_batches: total_completion_cost = usage.completion_tokens * output_cost_per_token_batches elif output_cost_per_token: @@ -2297,6 +2307,7 @@ def handle_realtime_stream_cost_calculation( custom_llm_provider: str, litellm_model_name: str, data_residency: Optional[str] = None, + litellm_logging_obj: Optional[LitellmLoggingObject] = None, ) -> float: """ Handles the cost calculation for realtime stream responses. @@ -2332,14 +2343,25 @@ def handle_realtime_stream_cost_calculation( input_cost_per_token += _input_cost_per_token output_cost_per_token += _output_cost_per_token break # exit if we find a valid model - total_cost = input_cost_per_token + output_cost_per_token - - if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results): - total_cost += handle_realtime_transcription_cost_calculation( + transcription_cost = ( + handle_realtime_transcription_cost_calculation( results=results, custom_llm_provider=custom_llm_provider, litellm_model_name=litellm_model_name, ) + if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results) + else 0.0 + ) + total_cost = input_cost_per_token + output_cost_per_token + transcription_cost + + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=input_cost_per_token, + completion_tokens_cost_usd_dollar=output_cost_per_token, + cost_for_built_in_tools_cost_usd_dollar=0.0, + total_cost_usd_dollar=total_cost, + additional_costs={"transcription_cost": transcription_cost} if transcription_cost > 0 else None, + ) return total_cost diff --git a/litellm/exceptions.py b/litellm/exceptions.py index d97ba347b07..adf7b3ef05a 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1165,12 +1165,18 @@ class ModifyResponseException(Exception): request_data: Dict[str, Any], guardrail_name: Optional[str] = None, detection_info: Optional[Dict[str, Any]] = None, + original_response: Optional[Any] = None, ): self.message = message self.model = model self.request_data = request_data self.guardrail_name = guardrail_name self.detection_info = detection_info or {} + # The LLM response that was blocked (post-call). Carries the real token + # usage the upstream call consumed, so the synthetic block response can + # report it instead of discarding it. None for pre-call blocks (the LLM + # was never invoked). + self.original_response = original_response super().__init__(message) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 831e588e5ba..c1c90233bee 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -520,13 +520,28 @@ class MCPClient: # Return empty list instead of raising to allow graceful degradation return [] + @staticmethod + def error_tool_result(exc: Exception) -> MCPCallToolResult: + """The error result ``call_tool`` returns when it swallows a failure (no re-execution).""" + return MCPCallToolResult( + content=[TextContent(type="text", text=f"{type(exc).__name__}: {str(exc)}")], + isError=True, + ) + async def call_tool( self, call_tool_request_params: MCPCallToolRequestParams, host_progress_callback: Optional[Callable] = None, + raise_on_error: bool = False, ) -> MCPCallToolResult: """ Call an MCP Tool. + + Args: + raise_on_error: When True, re-raise the underlying exception instead of returning an + ``isError=True`` result. The token-exchange (OBO) tool-call path uses this to detect + an upstream 401 so it can re-mint the exchanged token and retry once; every other + caller keeps the default and gets graceful ``isError`` degradation. """ verbose_logger.info(f"MCP client calling tool '{call_tool_request_params.name}'") @@ -579,11 +594,10 @@ class MCPClient: "MCP client detected broken connection/stream - " "the MCP server may have crashed, disconnected, or timed out." ) + if raise_on_error: + raise # Return a default error result instead of raising - return MCPCallToolResult( - content=[TextContent(type="text", text=f"{error_type}: {str(e)}")], # Empty content for error case - isError=True, - ) + return self.error_tool_result(e) async def list_prompts(self) -> List[Prompt]: """List available prompts from the server.""" diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 8a8a916fa9c..ce5074cdaf5 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -256,8 +256,6 @@ def create_fine_tuning_job( extra_body = optional_params.get("extra_body", {}) if extra_body is not None: extra_body.pop("azure_ad_token", None) - else: - get_secret_str("AZURE_AD_TOKEN") # type: ignore # Prepare Azure-specific parameters for extra_body extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams) @@ -442,7 +440,7 @@ def cancel_fine_tuning_job( ) # Azure OpenAI elif custom_llm_provider == "azure": - api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore + api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore @@ -457,8 +455,6 @@ def cancel_fine_tuning_job( extra_body = optional_params.get("extra_body", {}) if extra_body is not None: extra_body.pop("azure_ad_token", None) - else: - get_secret_str("AZURE_AD_TOKEN") # type: ignore response = azure_fine_tuning_apis_instance.cancel_fine_tuning_job( api_base=api_base, @@ -616,8 +612,6 @@ def list_fine_tuning_jobs( extra_body = optional_params.get("extra_body", {}) if extra_body is not None: extra_body.pop("azure_ad_token", None) - else: - get_secret("AZURE_AD_TOKEN") # type: ignore response = azure_fine_tuning_apis_instance.list_fine_tuning_jobs( api_base=api_base, @@ -759,8 +753,6 @@ def retrieve_fine_tuning_job( extra_body = optional_params.get("extra_body", {}) if extra_body is not None: extra_body.pop("azure_ad_token", None) - else: - get_secret_str("AZURE_AD_TOKEN") # type: ignore response = azure_fine_tuning_apis_instance.retrieve_fine_tuning_job( api_base=api_base, diff --git a/litellm/integrations/azure_sentinel/azure_sentinel.py b/litellm/integrations/azure_sentinel/azure_sentinel.py index 182a2e185ef..5f8afe58cb0 100644 --- a/litellm/integrations/azure_sentinel/azure_sentinel.py +++ b/litellm/integrations/azure_sentinel/azure_sentinel.py @@ -61,13 +61,15 @@ class AzureSentinelLogger(CustomBatchLogger): client_secret (str, optional): Azure Client Secret for OAuth2 authentication. If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var. audit_stream_name (str, optional): Stream name from DCR for audit logs. - If not provided, audit logs use the standard stream name. + If not provided, will use AZURE_SENTINEL_AUDIT_STREAM_NAME env var or the standard stream name. """ self.async_httpx_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) resolved_dcr_immutable_id = dcr_immutable_id or os.getenv("AZURE_SENTINEL_DCR_IMMUTABLE_ID") resolved_stream_name = stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM" - resolved_audit_stream_name = audit_stream_name or resolved_stream_name + resolved_audit_stream_name = ( + audit_stream_name or os.getenv("AZURE_SENTINEL_AUDIT_STREAM_NAME") or resolved_stream_name + ) resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT") resolved_tenant_id = tenant_id or os.getenv("AZURE_SENTINEL_TENANT_ID") or os.getenv("AZURE_TENANT_ID") resolved_client_id = client_id or os.getenv("AZURE_SENTINEL_CLIENT_ID") or os.getenv("AZURE_CLIENT_ID") diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 6775858c124..bd62d1e303a 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -354,14 +354,14 @@ class DataDogLogger( Raises: Raises a NON Blocking verbose_logger.exception if an error occurs """ + if not self.log_queue: + verbose_logger.exception("Datadog: log_queue does not exist") + return + + batch_to_send = self.log_queue[:] + self.log_queue = [] + try: - if not self.log_queue: - verbose_logger.exception("Datadog: log_queue does not exist") - return - - batch_to_send = self.log_queue[:] - self.log_queue = [] - verbose_logger.debug( "Datadog - about to flush %s events on %s", len(batch_to_send), diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 5e729e12be0..e258b239d93 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -368,7 +368,11 @@ class OpenTelemetryV2(CustomLogger): # it (named provisionally) so it isn't leaked as an open span. carrier.span.end(end_time=to_ns(end_time)) return None - data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=self.config.capture_span_content) + data = LLMCallSpanData.from_standard_logging_payload( + payload, + capture_content=self.config.capture_span_content, + time_to_first_chunk_seconds=call.time_to_first_chunk_seconds, + ) end_time_ns = to_ns(end_time) if carrier.span is not None: # Born at the boundary: stamp attributes from the typed payload, set diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index c5d8c35de7d..f568afa9e3e 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -55,6 +55,7 @@ class GenAIMapper: GenAI.RESPONSE_MODEL: lambda d: d.response_model, GenAI.RESPONSE_ID: lambda d: d.response_id, GenAI.RESPONSE_FINISH_REASONS: lambda d: list(d.finish_reasons) if d.finish_reasons else None, + GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds, GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens, GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens, Error.TYPE: lambda d: d.error.error_type if d.error else None, diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 37bb5464315..7ff4f540908 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -41,7 +41,7 @@ from typing import TYPE_CHECKING, Any, Mapping, cast from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL from litellm.integrations.otel.model.semconv import resolve_operation -from litellm.integrations.otel.model.utils import as_str +from litellm.integrations.otel.model.utils import as_str, to_seconds if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload @@ -201,6 +201,7 @@ class LLMCallEvent: # span is renamed from the typed payload at close (``finish_span``); this only # needs to be reasonable for a span that never gets closed (a leak). provisional_span_name: str + time_to_first_chunk_seconds: float | None @classmethod def from_dict(cls, kwargs: Mapping[str, Any]) -> "LLMCallEvent": @@ -214,9 +215,25 @@ class LLMCallEvent: dynamic_params=kwargs.get("standard_callback_dynamic_params"), is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)), provisional_span_name=f"{operation.value} {model}".strip(), + time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs), ) +def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: + """Seconds from the upstream request being issued (``api_call_start_time``) + to the first streamed chunk (``completion_start_time``); ``None`` for + non-streaming calls, where ``completion_start_time`` is backfilled with the + end time and would not measure first-chunk latency.""" + optional_params = cast(Mapping[str, Any], kwargs.get("optional_params") or {}) + if not optional_params.get("stream"): + return None + api_call_start = to_seconds(kwargs.get("api_call_start_time")) + completion_start = to_seconds(kwargs.get("completion_start_time")) + if api_call_start is None or completion_start is None: + return None + return completion_start - api_call_start + + def _call_id(payload: "StandardLoggingPayload | None", kwargs: Mapping[str, Any]) -> str | None: """The call id from the payload (when closed) or the bare kwargs (at pre_call).""" if payload is not None: diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index b0dcf97b787..fcd710492f0 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -305,10 +305,14 @@ class LLMCallSpanData: messages_in: tuple[Mapping[str, object], ...] = () choices_out: tuple[Mapping[str, object], ...] = () system_fingerprint: str | None = None + time_to_first_chunk_seconds: float | None = None @classmethod def from_standard_logging_payload( - cls, payload: "StandardLoggingPayload", capture_content: bool = False + cls, + payload: "StandardLoggingPayload", + capture_content: bool = False, + time_to_first_chunk_seconds: float | None = None, ) -> "LLMCallSpanData": params = cast(Mapping[str, object], payload.get("model_parameters") or {}) # The single parse of the request's metadata — the request-vs-provider @@ -349,6 +353,7 @@ class LLMCallSpanData: messages_in=_dicts(payload.get("messages")) if capture_content else (), choices_out=choices_out if capture_content else (), system_fingerprint=as_str(response.get("system_fingerprint")), + time_to_first_chunk_seconds=time_to_first_chunk_seconds, ) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 6315a5a4a89..4e725ae0a29 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -69,6 +69,7 @@ class GenAI: RESPONSE_ID: Final = "gen_ai.response.id" RESPONSE_MODEL: Final = "gen_ai.response.model" RESPONSE_FINISH_REASONS: Final = "gen_ai.response.finish_reasons" + RESPONSE_TIME_TO_FIRST_CHUNK: Final = "gen_ai.response.time_to_first_chunk" # usage USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens" USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens" diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py index cb1f9214876..50d0fb75962 100644 --- a/litellm/integrations/otel/plumbing/metrics.py +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -21,6 +21,7 @@ from litellm.integrations.opentelemetry import ( _build_metric_attribute_filter, _resolve_metric_attribute_filter, ) +from litellm.integrations.otel.model.metadata import time_to_first_chunk_seconds from litellm.integrations.otel.model.semconv import Metric, resolve_operation from litellm.integrations.otel.model.utils import to_seconds from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -181,13 +182,10 @@ class GenAIMetricRecorder: self._metrics.token_usage.record(usage.get("completion_tokens", 0), attributes=out_attrs) def _record_time_to_first_token(self, kwargs: Mapping[str, Any], common_attrs: dict) -> None: - if not kwargs.get("optional_params", {}).get("stream", False): + time_to_first_chunk = time_to_first_chunk_seconds(kwargs) + if time_to_first_chunk is None: return - api_call_start = to_seconds(kwargs.get("api_call_start_time")) - completion_start = to_seconds(kwargs.get("completion_start_time")) - if api_call_start is None or completion_start is None: - return - self._metrics.time_to_first_token.record(completion_start - api_call_start, attributes=common_attrs) + self._metrics.time_to_first_token.record(time_to_first_chunk, attributes=common_attrs) def _record_time_per_output_token( self, diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 4ebe312e301..e374068ca35 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import math import os import sys from datetime import datetime, timedelta @@ -65,6 +66,26 @@ if TYPE_CHECKING: else: AsyncIOScheduler = Any +_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT = 5.0 + + +def _get_budget_metrics_per_request_timeout() -> float: + raw = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT") + if raw is None: + return _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT + try: + parsed = float(raw) + except ValueError: + parsed = None + if parsed is None or not math.isfinite(parsed) or parsed <= 0: + verbose_logger.debug( + "[Non-Blocking] Prometheus: invalid PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT=%r; using default %ss.", + raw, + _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT, + ) + return _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT + return parsed + class PrometheusLogger(CustomLogger): # Class variables or attributes @@ -1607,7 +1628,15 @@ class PrometheusLogger(CustomLogger): _user_spend = _metadata.get("user_api_key_user_spend", None) _user_max_budget = _metadata.get("user_api_key_user_max_budget", None) - results = await asyncio.gather( + # Bound the per-request budget-metric emission so that slow Redis/DB + # lookups under load cannot consume the whole LoggingWorker watchdog + # (LOGGING_WORKER_MAX_TIME_PER_COROUTINE, default 20s) and get the entire + # success-logging event cancelled. Budget gauges are also refreshed by the + # periodic cron every PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES, + # so dropping one slow per-request emission only loses sub-cron real-time + # detail, not correctness. + budget_metrics_timeout = _get_budget_metrics_per_request_timeout() + gather_coro = asyncio.gather( self._set_api_key_budget_metrics_after_api_request( user_api_key=user_api_key, user_api_key_alias=user_api_key_alias, @@ -1634,6 +1663,16 @@ class PrometheusLogger(CustomLogger): ), return_exceptions=True, ) + try: + results = await asyncio.wait_for(gather_coro, timeout=budget_metrics_timeout) + except asyncio.TimeoutError: + verbose_logger.debug( + "[Non-Blocking] Prometheus: per-request budget metric emission " + "exceeded %ss under load; skipping (values are refreshed by the " + "periodic budget-metrics cron job).", + budget_metrics_timeout, + ) + return for i, r in enumerate(results): if isinstance(r, Exception): verbose_logger.debug( @@ -2004,6 +2043,43 @@ class PrometheusLogger(CustomLogger): return False + @staticmethod + def _extract_api_provider_from_request_data(request_data: dict) -> Optional[str]: + """ + Best-effort provider for the client-side failure path. + + A request can fail before a deployment is resolved, so the provider is + not always known. Prefer the resolved ``custom_llm_provider`` on + ``litellm_params``, then any provider recovered onto a partial + ``standard_logging_object`` (e.g. a stream that broke mid-flight), and + finally infer it from the requested model name (e.g. ``gpt-4o-mini`` -> + ``openai``) since the proxy's failure ``request_data`` usually carries + only the client-supplied model. Return ``None`` when it cannot be + determined so the label emits empty rather than a guess. + """ + litellm_params = request_data.get("litellm_params") or {} + provider = litellm_params.get("custom_llm_provider") + if provider: + return provider + standard_logging_object = request_data.get("standard_logging_object") or {} + provider = standard_logging_object.get("custom_llm_provider") + if provider: + return provider + model = litellm_params.get("model") or request_data.get("model") + if not model: + return None + try: + return litellm.get_llm_provider(model=model)[1] or None + except litellm.exceptions.BadRequestError: + return None + except Exception as e: # noqa: BLE001 - metrics labeling must never break request/failure handling + verbose_logger.debug( + "prometheus: unexpected error inferring api_provider from model=%s: %s", + model, + e, + ) + return None + async def async_post_call_failure_hook( self, request_data: dict, @@ -2039,6 +2115,7 @@ class PrometheusLogger(CustomLogger): _metadata = request_data.get("metadata", {}) or {} model_id = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id") rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception) + api_provider = self._extract_api_provider_from_request_data(request_data) enum_values = UserAPIKeyLabelValues( end_user=user_api_key_dict.end_user_id, user=user_api_key_dict.user_id, @@ -2060,6 +2137,7 @@ class PrometheusLogger(CustomLogger): client_ip=_metadata.get("requester_ip_address"), user_agent=_metadata.get("user_agent"), model_id=model_id, + api_provider=api_provider, stream=(str(request_data.get("stream")) if litellm.prometheus_emit_stream_label else None), ) _label_ctx = PrometheusLabelFactoryContext(enum_values) diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 939289f96ea..5b953035cfd 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -54,6 +54,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_strip_base64_files: bool = False, s3_use_key_prefix: bool = False, s3_use_virtual_hosted_style: bool = False, + s3_server_side_encryption: Optional[str] = None, s3_callback_params_override: Optional[dict] = None, **kwargs, ): @@ -92,6 +93,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_strip_base64_files=s3_strip_base64_files, s3_use_key_prefix=s3_use_key_prefix, s3_use_virtual_hosted_style=s3_use_virtual_hosted_style, + s3_server_side_encryption=s3_server_side_encryption, ) verbose_logger.debug(f"s3 logger using endpoint url {s3_endpoint_url}") @@ -145,6 +147,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_strip_base64_files: bool = False, s3_use_key_prefix: bool = False, s3_use_virtual_hosted_style: bool = False, + s3_server_side_encryption: Optional[str] = None, params_source: Optional[dict] = None, ): """ @@ -194,6 +197,8 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): bool(params.get("s3_use_virtual_hosted_style", False)) or s3_use_virtual_hosted_style ) + self.s3_server_side_encryption = params.get("s3_server_side_encryption") or s3_server_side_encryption + return async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -273,6 +278,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): async def async_upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement): try: + import base64 import hashlib import requests @@ -317,14 +323,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): # Calculate SHA256 hash of the content content_hash = hashlib.sha256(json_string.encode("utf-8")).hexdigest() + content_md5 = base64.b64encode( + hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest() + ).decode() # Prepare the request headers = { "Content-Type": "application/json", + "Content-MD5": content_md5, "x-amz-content-sha256": content_hash, "Content-Language": "en", "Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"', "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0", + **( + {"x-amz-server-side-encryption": self.s3_server_side_encryption} + if self.s3_server_side_encryption + else {} + ), } req = requests.Request("PUT", url, data=json_string, headers=headers) prepped = req.prepare() @@ -447,6 +462,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): def upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement): try: + import base64 import hashlib import requests @@ -482,14 +498,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): # Calculate SHA256 hash of the content content_hash = hashlib.sha256(json_string.encode("utf-8")).hexdigest() + content_md5 = base64.b64encode( + hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest() + ).decode() # Prepare the request headers = { "Content-Type": "application/json", + "Content-MD5": content_md5, "x-amz-content-sha256": content_hash, "Content-Language": "en", "Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"', "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0", + **( + {"x-amz-server-side-encryption": self.s3_server_side_encryption} + if self.s3_server_side_encryption + else {} + ), } req = requests.Request("PUT", url, data=json_string, headers=headers) prepped = req.prepare() diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 60100e8c2fd..00c67e9f0fb 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -91,6 +91,7 @@ class WebSearchInterceptionLogger(CustomLogger): messages: List[Dict], tools: Optional[List[Dict]], custom_llm_provider: Optional[str], + kwargs: Optional[dict[str, Any]] = None, ) -> Optional[Dict[str, Any]]: """ Short-circuit web-search-only requests by executing the search directly. @@ -176,7 +177,10 @@ class WebSearchInterceptionLogger(CustomLogger): # Execute search — keep the structured SearchResponse so the native # block can carry per-result url/title/page_age. try: - search_result_text, structured = await self._execute_search(query) + if kwargs is None: + search_result_text, structured = await self._execute_search(query) + else: + search_result_text, structured = await self._execute_search(query, kwargs=kwargs) except Exception as e: verbose_logger.error(f"WebSearchInterception: Short-circuit search failed: {e}") search_result_text, structured = f"Search failed: {e}", None @@ -936,7 +940,7 @@ class WebSearchInterceptionLogger(CustomLogger): query = tool_call["input"].get("query") if query: verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'") - search_tasks.append(self._execute_search(query)) + search_tasks.append(self._execute_search(query, kwargs=kwargs)) else: verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call['id']} has no query") # Add empty result for tools without query @@ -1009,7 +1013,9 @@ class WebSearchInterceptionLogger(CustomLogger): ) return patch, structured_results - async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchResponse]]: + async def _execute_search( + self, query: str, kwargs: Optional[dict[str, Any]] = None + ) -> Tuple[str, Optional[SearchResponse]]: """ Execute a single web search using router's search tools. @@ -1031,36 +1037,13 @@ class WebSearchInterceptionLogger(CustomLogger): ) llm_router = None - # Determine search provider from router's search_tools + search_tool = self._select_search_tool_from_router(llm_router=llm_router) search_provider: Optional[str] = None - if llm_router is not None and hasattr(llm_router, "search_tools"): - if self.search_tool_name: - # Find specific search tool by name - matching_tools = [ - tool - for tool in llm_router.search_tools - if tool.get("search_tool_name") == self.search_tool_name - ] - if matching_tools: - search_tool = matching_tools[0] - search_provider = search_tool.get("litellm_params", {}).get("search_provider") - verbose_logger.debug( - f"WebSearchInterception: Found search tool '{self.search_tool_name}' " - f"with provider '{search_provider}'" - ) - else: - verbose_logger.debug( - f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in router, " - "falling back to first available or perplexity" - ) - - # If no specific tool or not found, use first available - if not search_provider and llm_router.search_tools: - first_tool = llm_router.search_tools[0] - search_provider = first_tool.get("litellm_params", {}).get("search_provider") - verbose_logger.debug( - f"WebSearchInterception: Using first available search tool with provider '{search_provider}'" - ) + search_litellm_params: dict[str, Any] = {} + if search_tool is not None: + await self._authorize_search_tool(search_tool=search_tool, kwargs=kwargs) + search_litellm_params = dict(search_tool.get("litellm_params", {}) or {}) + search_provider = search_litellm_params.get("search_provider") # Fallback to perplexity if no router or no search tools configured if not search_provider: @@ -1073,7 +1056,12 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.debug( f"WebSearchInterception: Executing search for '{query}' using provider '{search_provider}'" ) - result = await litellm.asearch(query=query, search_provider=search_provider) + search_kwargs = { + key: value + for key, value in search_litellm_params.items() + if key != "search_provider" and value is not None + } + result = await litellm.asearch(query=query, search_provider=search_provider, **search_kwargs) # Format using transformation function search_result_text = WebSearchTransformation.format_search_response(result) @@ -1086,6 +1074,107 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.error(f"WebSearchInterception: Search failed for '{query}': {str(e)}") raise + async def _authorize_search_tool( + self, + search_tool: dict[str, Any], + kwargs: Optional[dict[str, Any]], + ) -> None: + search_tool_name = search_tool.get("search_tool_name") + if not isinstance(search_tool_name, str) or not search_tool_name: + return + + user_api_key_auth = self._get_user_api_key_auth_from_kwargs(kwargs) + if user_api_key_auth is None: + return + + from litellm.proxy.auth.auth_checks import ( + can_key_call_search_tool, + can_team_call_search_tool, + get_team_object, + ) + + await can_key_call_search_tool( + search_tool_name=search_tool_name, + valid_token=user_api_key_auth, + ) + + team_id = getattr(user_api_key_auth, "team_id", None) + if team_id: + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + team_object = await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=getattr(user_api_key_auth, "parent_otel_span", None), + proxy_logging_obj=proxy_logging_obj, + ) + await can_team_call_search_tool( + search_tool_name=search_tool_name, + team_object=team_object, + ) + + @staticmethod + def _get_user_api_key_auth_from_kwargs(kwargs: Optional[dict[str, Any]]) -> Any: + if not kwargs: + return None + + for metadata_key in ("metadata", "litellm_metadata"): + metadata = kwargs.get(metadata_key) + if isinstance(metadata, dict) and metadata.get("user_api_key_auth") is not None: + return metadata["user_api_key_auth"] + + litellm_params = kwargs.get("litellm_params") + if not isinstance(litellm_params, dict): + return None + + for metadata_key in ("metadata", "litellm_metadata"): + metadata = litellm_params.get(metadata_key) + if isinstance(metadata, dict) and metadata.get("user_api_key_auth") is not None: + return metadata["user_api_key_auth"] + + return None + + def _select_search_tool_from_router(self, llm_router: Any) -> Optional[dict[str, Any]]: + if llm_router is None or not hasattr(llm_router, "search_tools"): + return None + search_tools = list(getattr(llm_router, "search_tools") or []) + return self._select_search_tool_from_list(search_tools=search_tools, source="router") + + def _select_search_tool_from_list( + self, + search_tools: list[dict[str, Any]], + source: str, + ) -> Optional[dict[str, Any]]: + if self.search_tool_name: + matching_tools = [tool for tool in search_tools if tool.get("search_tool_name") == self.search_tool_name] + if matching_tools: + search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider") + verbose_logger.debug( + f"WebSearchInterception: Found search tool '{self.search_tool_name}' " + f"from {source} with provider '{search_provider}'" + ) + return matching_tools[0] + verbose_logger.debug( + f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in {source}, " + "falling back to first available or perplexity" + ) + + if search_tools: + first_tool = search_tools[0] + search_provider = (first_tool.get("litellm_params", {}) or {}).get("search_provider") + verbose_logger.debug( + f"WebSearchInterception: Using first available search tool from {source} " + f"with provider '{search_provider}'" + ) + return first_tool + + return None + async def _execute_chat_completion_agentic_loop( self, model: str, @@ -1145,7 +1234,7 @@ class WebSearchInterceptionLogger(CustomLogger): if query: verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'") - search_tasks.append(self._execute_search(query)) + search_tasks.append(self._execute_search(query, kwargs=kwargs)) else: verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call.get('id')} has no query") # Add empty result for tools without query diff --git a/litellm/interactions/streaming_iterator.py b/litellm/interactions/streaming_iterator.py index 45c5443cfd2..0d9d1b4579c 100644 --- a/litellm/interactions/streaming_iterator.py +++ b/litellm/interactions/streaming_iterator.py @@ -174,22 +174,15 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator): logging_response = copy.deepcopy(self.completed_response) asyncio.create_task( - self.logging_obj.async_success_handler( - result=logging_response, + self.logging_obj.dispatch_success_handlers( + logging_response, start_time=self.start_time, end_time=datetime.now(), cache_hit=None, + prefer_async_handlers=True, ) ) - executor.submit( - self.logging_obj.success_handler, - result=logging_response, - cache_hit=None, - start_time=self.start_time, - end_time=datetime.now(), - ) - class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator): """ diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index f86243c73b7..e5007ceec34 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -123,6 +123,34 @@ def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile: return ProcessedAudioFile(file_content=file_content, filename=filename, content_type=content_type) +BARE_ISO_639_1_TO_BCP47 = { + "en": "en-US", + "es": "es-ES", + "de": "de-DE", + "fr": "fr-FR", + "it": "it-IT", + "pt": "pt-BR", + "ja": "ja-JP", + "ko": "ko-KR", + "zh": "zh-CN", + "ru": "ru-RU", + "hi": "hi-IN", + "ar": "ar-SA", +} + + +def normalize_transcription_language_to_bcp47(language: str) -> str: + """ + OpenAI's transcription `language` param accepts bare ISO-639-1 codes like + ``en``; speech APIs such as Google Speech-to-Text and NVIDIA Riva require + BCP-47 like ``en-US``. Map the most common bare codes and pass through + anything already region-qualified (or unknown, for a clear provider error). + """ + if "-" in language: + return language + return BARE_ISO_639_1_TO_BCP47.get(language.lower(), language) + + def get_audio_file_name(file_obj: FileTypes) -> str: """ Safely get the name of a file-like object or return its string representation. diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 2441cbb3903..9dc202c4717 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -1944,7 +1944,7 @@ def _map_azure_exception( response=getattr(original_exception, "response", None), body=getattr(original_exception, "body", None), ) - elif "invalid_request_error" in error_str: + elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400): raise BadRequestError( message=f"AzureException BadRequestError - {message}", llm_provider="azure", @@ -1986,6 +1986,14 @@ def _map_azure_exception( litellm_debug_info=extra_information, response=getattr(original_exception, "response", None), ) + elif original_exception.status_code == 404: + raise NotFoundError( + message=f"AzureException NotFoundError - {message}", + llm_provider="azure", + model=model, + litellm_debug_info=extra_information, + response=getattr(original_exception, "response", None), + ) elif original_exception.status_code == 408: raise Timeout( message=f"AzureException Timeout - {message}", @@ -2173,7 +2181,7 @@ def exception_type( # type: ignore litellm_response_headers = _get_response_headers(original_exception=original_exception) try: error_str = redact_string(str(original_exception)) if _ENABLE_SECRET_REDACTION else str(original_exception) - if model: + if model or custom_llm_provider: if hasattr(original_exception, "message"): error_str = ( redact_string(str(original_exception.message)) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index fd82ad72d93..b70eadb4c18 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -39,6 +39,8 @@ OPTIONAL_KWARGS_KEYS = frozenset( "gigachat_access_token", "tpm", "rpm", + "itpm", + "otpm", "use_xai_oauth", } ) @@ -77,6 +79,7 @@ def get_litellm_params( proxy_server_request=None, acompletion=None, aembedding=None, + allm_passthrough_route=None, preset_cache_key=None, no_log=None, input_cost_per_second=None, @@ -119,6 +122,7 @@ def get_litellm_params( # Build base dict with explicit parameters (always included) litellm_params = { "acompletion": acompletion, + "allm_passthrough_route": allm_passthrough_route, "api_key": api_key, "force_timeout": force_timeout, "logger_fn": logger_fn, diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index e98e61f5fdd..ff095d007cc 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -657,6 +657,10 @@ def _get_openai_compatible_provider_info( api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta" # type: ignore dynamic_api_key = api_key or get_secret_str("DEEPSEEK_API_KEY") + elif custom_llm_provider == "tencent": + api_base = api_base or get_secret("TENCENT_API_BASE") or "https://tokenhub-intl.tencentcloudmaas.com/v1" + + dynamic_api_key = api_key or get_secret_str("TENCENT_API_KEY") elif custom_llm_provider == "fireworks_ai": # fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1 ( diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index c4ddb4b7ee0..19149da0316 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -106,6 +106,8 @@ def get_supported_openai_params( return litellm.VLLMConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "deepseek": return litellm.DeepSeekChatConfig().get_supported_openai_params(model=model) + elif custom_llm_provider == "tencent": + return litellm.TencentChatConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere": return litellm.CohereChatConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "maritalk": diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index d0a6ec30c9e..06a9e98c5ac 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -1,7 +1,23 @@ -from typing import Dict, Optional +from typing import Any, Dict, Iterator, Optional from litellm.types.utils import StandardCallbackDynamicParams +_CLIENT_CALLBACK_METADATA_SLOTS: tuple[str, ...] = ("litellm_metadata", "metadata") + + +def iter_client_callback_metadata_dicts( + kwargs: dict[str, Any], +) -> Iterator[tuple[str, dict[str, Any]]]: + litellm_params = kwargs.get("litellm_params") + if isinstance(litellm_params, dict): + nested = litellm_params.get("metadata") + if isinstance(nested, dict): + yield "litellm_params.metadata", nested + for key in _CLIENT_CALLBACK_METADATA_SLOTS: + candidate = kwargs.get(key) + if isinstance(candidate, dict): + yield key, candidate + def _is_env_reference(value: object) -> bool: return isinstance(value, str) and "os.environ/" in value @@ -55,6 +71,7 @@ _supported_callback_params = [ "dd_site", "dd_agent_host", "dd_agent_port", + "turn_off_message_logging", ] _request_blocked_callback_params = { @@ -87,19 +104,13 @@ def initialize_standard_callback_dynamic_params( validate_no_callback_env_reference(param, _param_value, source="request body") standard_callback_dynamic_params[param] = _param_value # type: ignore - # 2. Fallback: check "metadata" or "litellm_params" -> "metadata" - metadata = (kwargs.get("metadata") or {}).copy() - litellm_params = kwargs.get("litellm_params") or {} - if isinstance(litellm_params, dict): - metadata.update(litellm_params.get("metadata") or {}) - - if isinstance(metadata, dict): + for slot_label, metadata in iter_client_callback_metadata_dicts(kwargs): for param in _supported_callback_params: if param in _request_blocked_callback_params: continue if param not in standard_callback_dynamic_params and param in metadata: _param_value = metadata.get(param) - validate_no_callback_env_reference(param, _param_value, source="metadata") + validate_no_callback_env_reference(param, _param_value, source=slot_label) standard_callback_dynamic_params[param] = _param_value # type: ignore return standard_callback_dynamic_params diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f01d8bf59f6..5010010e14f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1528,6 +1528,7 @@ class Logging(LiteLLMLoggingBaseClass): and litellm_params.get(CallTypes.aembedding.value, False) is not True and litellm_params.get(CallTypes.aimage_generation.value, False) is not True and litellm_params.get(CallTypes.atranscription.value, False) is not True + and litellm_params.get(CallTypes.allm_passthrough_route.value, False) is not True ) def _is_assembled_stream_success(self, result=None) -> bool: diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 58107d9804b..47daf33824e 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -656,7 +656,7 @@ def convert_to_model_response_object( message: Optional[Message] = None finish_reason: Optional[str] = None - if _should_convert_tool_call_to_json_mode( + if tool_calls is not None and _should_convert_tool_call_to_json_mode( tool_calls=tool_calls, convert_tool_call_to_json_mode=convert_tool_call_to_json_mode, ): diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index a1a070eb5b7..220d1caa3d2 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,5 +1,4 @@ import asyncio -import concurrent.futures import json from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol, Union, cast @@ -25,9 +24,6 @@ if TYPE_CHECKING: else: CLIENT_CONNECTION_CLASS = Any -# Create a thread pool with a maximum of 10 threads -executor = concurrent.futures.ThreadPoolExecutor(max_workers=10) - class RealtimeEventNormalizer(Protocol): def should_drop(self, event: object) -> bool: ... @@ -315,13 +311,12 @@ class RealTimeStreaming: if self.session_tools or self.tool_calls: self.logging_obj.model_call_details["realtime_tools"] = self.session_tools self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls - ## ASYNC LOGGING # Route through the bounded logging worker (per-coroutine timeout + # concurrency cap) instead of a bare create_task, so a slow callback # can't leave suspended tasks pinning each call's response in memory. - GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(self.logging_obj.async_success_handler(self.messages)) - ## SYNC LOGGING - executor.submit(self.logging_obj.success_handler(self.messages)) + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( + self.logging_obj.dispatch_success_handlers(self.messages, prefer_async_handlers=True) + ) async def _send_to_backend(self, message: str) -> bool: """Send a message to the backend WebSocket. diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index daca48120cd..1f3a6961f39 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -131,8 +131,26 @@ class SensitiveDataMasker: return masked_data + def mask(self, data: object) -> object: + if isinstance(data, Mapping): + return self.mask_dict(dict(data)) + if isinstance(data, list): + return self._mask_sequence( + data, + 0, + DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER, + None, + False, + ) + return data + _default_masker = SensitiveDataMasker() +_error_masker = SensitiveDataMasker(visible_prefix=4, visible_suffix=0) + + +def mask_sensitive_structure(data: object) -> object: + return _error_masker.mask(data) def mask_sensitive_keys(data: Dict[str, Any], sensitive_fields: Set[str]) -> Dict[str, Any]: diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index deeee3b7daf..38bc68f2f78 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -7,6 +7,7 @@ from litellm.types.llms.openai import ( ChatCompletionAudioDelta, ) from litellm.types.utils import ( + CacheCreationTokenDetails, ChatCompletionAudioResponse, ChatCompletionMessageToolCall, Choices, @@ -541,6 +542,12 @@ class ChunkProcessor: web_search_requests: Optional[int] = None completion_tokens_details: Optional[CompletionTokensDetails] = None prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None + # Anthropic emits the cache-creation TTL breakdown (5m/1h split) only on + # the `message_start` event; the later `message_delta` carries the flat + # cache-creation count but drops the nested breakdown. prompt_tokens_details + # is last-wins, so without preserving this separately the 1h breakdown is + # lost and 1h cache writes get billed at the 5m rate. + cache_creation_token_details: Optional[CacheCreationTokenDetails] = None for chunk in chunks: usage_chunk: Optional[Usage] = None if "usage" in chunk: @@ -594,7 +601,18 @@ class ChunkProcessor: "web_search_requests", ) - prompt_tokens_details = usage_chunk_dict["prompt_tokens_details"] + prompt_tokens_details = cast( + Optional[PromptTokensDetailsWrapper], + usage_chunk_dict["prompt_tokens_details"], + ) + + cache_creation_token_details = self._capture_cache_creation_token_details( + prompt_tokens_details, cache_creation_token_details + ) + + prompt_tokens_details = self._attach_cache_creation_token_details( + prompt_tokens_details, cache_creation_token_details + ) completion_tokens = self._reset_anthropic_cursor_completion_tokens( chunks=chunks, @@ -613,6 +631,34 @@ class ChunkProcessor: prompt_tokens_details=prompt_tokens_details, ) + @staticmethod + def _capture_cache_creation_token_details( + prompt_tokens_details: Optional[PromptTokensDetailsWrapper], + current: Optional[CacheCreationTokenDetails], + ) -> Optional[CacheCreationTokenDetails]: + incoming = cast( + Optional[CacheCreationTokenDetails], + getattr(prompt_tokens_details, "cache_creation_token_details", None), + ) + if incoming is not None: + return incoming + return current + + @staticmethod + def _attach_cache_creation_token_details( + prompt_tokens_details: Optional[PromptTokensDetailsWrapper], + cache_creation_token_details: Optional[CacheCreationTokenDetails], + ) -> Optional[PromptTokensDetailsWrapper]: + if prompt_tokens_details is None or cache_creation_token_details is None: + return prompt_tokens_details + existing = cast( + Optional[CacheCreationTokenDetails], + getattr(prompt_tokens_details, "cache_creation_token_details", None), + ) + if existing is not None: + return prompt_tokens_details + return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details}) + @staticmethod def _reset_anthropic_cursor_completion_tokens( chunks: list[dict[str, Any] | ModelResponse], diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 587a3a58a94..128ba0bf3ab 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1884,7 +1884,7 @@ class CustomStreamWrapper: await self.fetch_stream() if is_async_iterable(self.completion_stream): - async for chunk in self.completion_stream: # type: ignore[union-attr] + async for chunk in self.completion_stream: # pyright: ignore[reportOptionalIterable] # is_async_iterable guard proves __aiter__ if chunk == "None" or chunk is None: continue # skip None chunks diff --git a/litellm/llms/a2a/chat/transformation.py b/litellm/llms/a2a/chat/transformation.py index c9623a817bf..113c000f352 100644 --- a/litellm/llms/a2a/chat/transformation.py +++ b/litellm/llms/a2a/chat/transformation.py @@ -10,7 +10,7 @@ import httpx from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import Choices, Message, ModelResponse +from litellm.types.utils import Choices, Message, ModelResponse, Usage from ..common_utils import ( A2AError, @@ -312,6 +312,25 @@ class A2AConfig(BaseConfig): # Set ID from response model_response.id = response_json.get("id", str(uuid.uuid4())) + # A2A agents don't return token usage; estimate it so per-token pricing + # produces real cost and callers don't receive usage of 0/0/0. + try: + from litellm.utils import token_counter + + prompt_tokens = token_counter(model="gpt-3.5-turbo", messages=messages) + completion_tokens = token_counter(model="gpt-3.5-turbo", text=text, count_response_tokens=True) + setattr( + model_response, + "usage", + Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + except Exception: # noqa: BLE001 - best-effort estimate; a tokenizer hiccup must not break the response + pass + return model_response def get_model_response_iterator( diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 4506c114208..7000c20d9c4 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -48,7 +48,10 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + ModifyResponseException, + ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -70,6 +73,170 @@ class AnthropicMessagesHandler(BaseTranslation): super().__init__() self.adapter = LiteLLMAnthropicMessagesAdapter() + @staticmethod + def _build_streaming_usage_response( + responses_so_far: list[Any], + request_data: Optional[dict], + ) -> Optional[ModelResponse]: + chunks = tuple(response for response in responses_so_far if isinstance(response, (str, bytes))) + if not chunks: + return None + try: + return AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( + all_chunks=chunks, + model=str((request_data or {}).get("model") or ""), + ) + except (AttributeError, TypeError, ValueError): + return None + + def build_block_sse_chunks( + self, + exc: "ModifyResponseException", + stream_started: bool = False, + responses_so_far: Optional[list[Any]] = None, + ) -> list[bytes]: + """ + Build an Anthropic SSE sequence delivering the guardrail block message + and terminating the stream cleanly. + + - ``stream_started`` False (buffered / pre-stream): nothing has been + sent, so emit a complete standalone message (message_start -> + content_block_* -> message_delta -> message_stop) via + FakeAnthropicMessagesStreamIterator, the same converter the + /v1/messages pre-stream block handler uses. + - ``stream_started`` True (sampling / detect-only end-of-stream): real + chunks were already sent, so *continue* the in-progress message -- + close the open content block, append the block message as a new text + block, then end the message. Emitting a second ``message_start`` here + would make Anthropic clients reject the stream. + """ + if stream_started: + return self._block_continuation_chunks(exc, responses_so_far or []) + return self._standalone_block_chunks(exc) + + def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + import uuid + + from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + from litellm.llms.base_llm.guardrail_translation.utils import ( + blocked_response_usage, + ) + from litellm.types.utils import AnthropicMessagesResponse + + block_response = AnthropicMessagesResponse( + id=f"msg_{uuid.uuid4()}", + type="message", + role="assistant", + content=[{"type": "text", "text": exc.message}], + model=exc.model, + stop_reason="end_turn", + usage=blocked_response_usage(getattr(exc, "original_response", None)), + ) + return list(FakeAnthropicMessagesStreamIterator(response=block_response)) + + def _block_continuation_chunks(self, exc: "ModifyResponseException", responses_so_far: list[Any]) -> list[bytes]: + """Continue an already-started message: close the open content block, + append the block message as a new text block, then end the message -- + without a second message_start.""" + + from litellm.llms.base_llm.guardrail_translation.utils import ( + blocked_response_usage, + ) + + def _sse(event_type: str, payload: dict) -> bytes: + return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode() + + output_tokens = blocked_response_usage(getattr(exc, "original_response", None))["output_tokens"] + open_index, max_index = self._content_block_state(responses_so_far) + new_index = (max_index + 1) if max_index is not None else 0 + chunks: list[bytes] = [] + if open_index is not None: + chunks.append(_sse("content_block_stop", {"type": "content_block_stop", "index": open_index})) + chunks += [ + _sse( + "content_block_start", + { + "type": "content_block_start", + "index": new_index, + "content_block": {"type": "text", "text": ""}, + }, + ), + _sse( + "content_block_delta", + { + "type": "content_block_delta", + "index": new_index, + "delta": {"type": "text_delta", "text": exc.message}, + }, + ), + _sse("content_block_stop", {"type": "content_block_stop", "index": new_index}), + _sse( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": output_tokens}, + }, + ), + _sse("message_stop", {"type": "message_stop"}), + ] + return chunks + + @staticmethod + def _content_block_state( + responses_so_far: list[Any], + ) -> tuple[Optional[int], Optional[int]]: + """From the SSE chunks already sent to the client, return (open + content-block index or None, highest content-block index seen or None). + + A single streamed item may bundle multiple SSE events (raw bytes) or be + an already-parsed event dict, so every event across every item is + considered -- matching how ``get_streaming_string_so_far`` reads the + same stream.""" + open_indices: set[int] = set() + max_index: Optional[int] = None + for item in responses_so_far: + for data in AnthropicMessagesHandler._iter_sse_events(item): + event_type = data.get("type") + index = data.get("index") + if not isinstance(index, int): + continue + if event_type == "content_block_start": + open_indices.add(index) + max_index = index if max_index is None else max(max_index, index) + elif event_type == "content_block_stop": + open_indices.discard(index) + open_index = max(open_indices) if open_indices else None + return open_index, max_index + + @staticmethod + def _iter_sse_events(item: Any) -> list[dict]: + """Yield the event-data dicts in one stream chunk. + + Handles both formats this stream can carry (see + ``get_streaming_string_so_far``): raw SSE ``bytes`` -- which may bundle + several events separated by a blank line -- and an already-parsed event + ``dict``.""" + if isinstance(item, dict): + return [item] + if not isinstance(item, (bytes, bytearray)): + return [] + events: list[dict] = [] + for block in item.decode("utf-8", errors="replace").split("\n\n"): + for line in block.split("\n"): + line = line.strip() + if not line.startswith("data:"): + continue + try: + parsed = json.loads(line[len("data:") :].strip()) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + events.append(parsed) + return events + def _translate_to_openai(self, data: dict) -> ChatCompletionRequest: """Translate Anthropic request to OpenAI chat completion format.""" ( @@ -406,6 +573,8 @@ class AnthropicMessagesHandler(BaseTranslation): Get the string so far, check the apply guardrail to the string so far, and return the list of responses so far. """ + from litellm.integrations.custom_guardrail import ModifyResponseException + has_ended = self._check_streaming_has_ended(responses_so_far) if has_ended: # build the model response from the responses_so_far @@ -430,25 +599,35 @@ class AnthropicMessagesHandler(BaseTranslation): if tool_calls_list: guardrail_inputs["tool_calls"] = tool_calls_list - _guardrailed_inputs = ( - await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid + try: + _guardrailed_inputs = await guardrail_to_apply.apply_guardrail( inputs=guardrail_inputs, request_data=request_data if request_data is not None else {}, input_type="response", logging_obj=litellm_logging_obj, ) - ) + except ModifyResponseException as e: + if e.original_response is None: + e.original_response = built_response or self._build_streaming_usage_response( + responses_so_far, request_data + ) + raise else: verbose_proxy_logger.debug("Skipping output guardrail - model response has no choices") return responses_so_far string_so_far = self.get_streaming_string_so_far(responses_so_far) - _guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid - inputs={"texts": [string_so_far]}, - request_data=request_data if request_data is not None else {}, - input_type="response", - logging_obj=litellm_logging_obj, - ) + try: + _guardrailed_inputs = await guardrail_to_apply.apply_guardrail( + inputs={"texts": [string_so_far]}, + request_data=request_data if request_data is not None else {}, + input_type="response", + logging_obj=litellm_logging_obj, + ) + except ModifyResponseException as e: + if e.original_response is None: + e.original_response = self._build_streaming_usage_response(responses_so_far, request_data) + raise return responses_so_far def _prepare_request_data( diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 812e0f62c96..7299fc16897 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -78,7 +78,7 @@ async def _prepare_context_managed_request( system: Optional[Any], context_management_spec: Any, litellm_metadata: Optional[Dict], - drop_params: Optional[bool], + additional_drop_params: Optional[list[str]], llm_router: Any, user_api_key_auth: Any = None, ) -> Optional[PolyfillResult]: @@ -95,7 +95,7 @@ async def _prepare_context_managed_request( # silently drop intermediate turns. polyfill_will_run = _polyfill_will_run( context_management_spec=context_management_spec, - drop_params=drop_params, + additional_drop_params=additional_drop_params, ) if polyfill_will_run: @@ -117,7 +117,7 @@ async def _prepare_context_managed_request( system=working_system, context_management_spec=context_management_spec, litellm_metadata=litellm_metadata, - drop_params=drop_params, + additional_drop_params=additional_drop_params, llm_router=llm_router, user_api_key_auth=user_api_key_auth, ) @@ -143,18 +143,19 @@ async def _prepare_context_managed_request( def _polyfill_will_run( *, context_management_spec: Any, - drop_params: Optional[bool], + additional_drop_params: Optional[list[str]], ) -> bool: """Return True when ``compact_20260112`` will run via the polyfill dispatcher. - Mirrors the gating in ``_run_polyfill_if_enabled``: an empty spec or - effective ``drop_params`` short-circuits the polyfill. The pre-processing - skip only applies when the dispatcher will actually invoke - ``apply_compact_20260112`` (which has its own compaction-block slicing). + Mirrors the gating in ``_run_polyfill_if_enabled``: an empty spec or an + explicit ``context_management`` entry in ``additional_drop_params`` + short-circuits the polyfill. The pre-processing skip only applies when the + dispatcher will actually invoke ``apply_compact_20260112`` (which has its + own compaction-block slicing). """ edits = _normalize_spec_edits( context_management_spec=context_management_spec, - drop_params=drop_params, + additional_drop_params=additional_drop_params, ) if edits is None: return False @@ -169,7 +170,7 @@ def _polyfill_will_run( def _spec_has_non_compact_edits( *, context_management_spec: Any, - drop_params: Optional[bool], + additional_drop_params: Optional[list[str]], ) -> bool: """Return True when the spec includes edits other than ``compact_20260112``. @@ -180,7 +181,7 @@ def _spec_has_non_compact_edits( """ edits = _normalize_spec_edits( context_management_spec=context_management_spec, - drop_params=drop_params, + additional_drop_params=additional_drop_params, ) if edits is None: return False @@ -195,10 +196,22 @@ def _spec_has_non_compact_edits( ) +def _context_management_explicitly_dropped(additional_drop_params: Optional[list[str]]) -> bool: + """True when the caller opted out of context_management via ``additional_drop_params``. + + ``drop_params`` deliberately does NOT gate the polyfill: ``context_management`` + is a LiteLLM-supported param (native on Anthropic, polyfilled elsewhere), and + ``drop_params`` only exists to drop genuinely unsupported params. + """ + if not isinstance(additional_drop_params, list): + return False + return "context_management" in additional_drop_params + + def _normalize_spec_edits( *, context_management_spec: Any, - drop_params: Optional[bool], + additional_drop_params: Optional[list[str]], ) -> Optional[List[Dict[str, Any]]]: """Return the normalized ``edits`` list, or ``None`` if the polyfill won't run. @@ -208,8 +221,7 @@ def _normalize_spec_edits( if not context_management_spec: return None - effective_drop_params = drop_params if drop_params is not None else litellm.drop_params - if effective_drop_params: + if _context_management_explicitly_dropped(additional_drop_params): return None from litellm.llms.anthropic.experimental_pass_through.context_management.dispatcher import ( @@ -230,22 +242,23 @@ async def _run_polyfill_if_enabled( system: Optional[Any], context_management_spec: Any, litellm_metadata: Optional[Dict], - drop_params: Optional[bool], + additional_drop_params: Optional[list[str]], llm_router: Any, user_api_key_auth: Any = None, ) -> Optional[PolyfillResult]: """Run the async context_management polyfill if a spec is present. - Returns ``None`` when the spec is empty or drop_params is on. Raises - ``AnthropicContextManagementError`` so the /v1/messages endpoint can - emit an Anthropic-format 400. All other exceptions are best-effort - swallowed (matches v0 behavior). + Returns ``None`` when the spec is empty or ``context_management`` is + listed in ``additional_drop_params`` (the explicit opt-out; ``drop_params`` + does not disable the polyfill because context_management is a supported + param). Raises ``AnthropicContextManagementError`` so the /v1/messages + endpoint can emit an Anthropic-format 400. All other exceptions are + best-effort swallowed (matches v0 behavior). """ if not context_management_spec: return None - effective_drop_params = drop_params if drop_params is not None else litellm.drop_params - if effective_drop_params: + if _context_management_explicitly_dropped(additional_drop_params): return None try: @@ -274,7 +287,7 @@ async def _run_polyfill_if_enabled( # emits an Anthropic-format error. if _spec_has_non_compact_edits( context_management_spec=context_management_spec, - drop_params=drop_params, + additional_drop_params=additional_drop_params, ): raise AnthropicContextManagementError( status_code=500, @@ -533,7 +546,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) -> Union[AnthropicMessagesResponse, AsyncIterator[Any], Iterator[bytes]]: """Handle non-Anthropic models asynchronously using the adapter""" context_management = kwargs.pop("context_management", None) - drop_params: Optional[bool] = kwargs.get("drop_params", None) + additional_drop_params: Optional[list[str]] = kwargs.get("additional_drop_params", None) litellm_router = kwargs.pop("litellm_router", None) if litellm_router is None: try: @@ -555,7 +568,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: system=system, context_management_spec=context_management, litellm_metadata=proxy_litellm_metadata, - drop_params=drop_params, + additional_drop_params=additional_drop_params, llm_router=litellm_router, user_api_key_auth=user_api_key_auth, ) @@ -661,7 +674,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: # ``compact_20260112`` editor can ``await`` the summarization model); # bridge to it via ``run_async_function``. context_management = kwargs.pop("context_management", None) - drop_params: Optional[bool] = kwargs.get("drop_params", None) + additional_drop_params: Optional[list[str]] = kwargs.get("additional_drop_params", None) # Deliberately do NOT auto-attach the proxy ``llm_router`` here: # ``run_async_function`` spawns a new event loop in a worker thread # to bridge to the async dispatcher, but the proxy router's httpx @@ -696,7 +709,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: system=system, context_management_spec=context_management, litellm_metadata=proxy_litellm_metadata, - drop_params=drop_params, + additional_drop_params=additional_drop_params, llm_router=litellm_router, user_api_key_auth=user_api_key_auth, ) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py index cb37725d79c..4bf36a0d5c6 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py @@ -193,6 +193,14 @@ class AgenticAnthropicStreamingIterator: raise StopAsyncIteration + async def aclose(self) -> None: + from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( + aclose_if_supported, + ) + + await aclose_if_supported(self._inner) + await aclose_if_supported(self._follow_up_iterator) + async def _process_agentic_hooks(self) -> None: """Rebuild the Anthropic response from collected SSE bytes and call hooks.""" if self._hook_processing_done: diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index effd7dda6a0..dd983f0c344 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -148,6 +148,7 @@ async def _try_websearch_short_circuit( tools: Optional[List[Dict]], custom_llm_provider: Optional[str], stream: Optional[bool], + kwargs: Optional[dict] = None, ) -> Optional[Union[AnthropicMessagesResponse, AsyncIterator]]: """ Attempt to short-circuit a web-search-only request. @@ -177,6 +178,7 @@ async def _try_websearch_short_circuit( messages=messages, tools=tools, custom_llm_provider=custom_llm_provider, + kwargs=kwargs, ) if response is not None: anthropic_response = cast(AnthropicMessagesResponse, response) @@ -292,6 +294,7 @@ async def anthropic_messages( tools=tools, custom_llm_provider=custom_llm_provider, stream=original_stream, + kwargs={**kwargs, "metadata": metadata}, ) if short_circuit_response is not None: return short_circuit_response diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py index 6c72b7a3e00..79faa39c7a2 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py @@ -17,7 +17,9 @@ How it works: import uuid from typing import Any, AsyncIterator, Dict, List, Optional, Union +import litellm import litellm.constants as _c +from litellm.litellm_core_utils.url_utils import validate_url from litellm.llms.anthropic.common_utils import strip_advisor_blocks_from_messages from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -76,16 +78,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): raise ValueError("advisor tool definition must include a 'model' field specifying the advisor model") _raw_max_uses = advisor_tool.get("max_uses") max_uses: int = ADVISOR_MAX_USES if _raw_max_uses is None else int(_raw_max_uses) - # Optional routing overrides for the advisor sub-call (e.g. proxy routing). - # If not set in the tool definition, litellm resolves from env vars. - # The advisor tool is caller-controlled; only honor a client-supplied - # api_base/api_key when the proxy has enabled clientside credentials, - # otherwise let litellm resolve from server config. - advisor_api_key: Optional[str] = None - advisor_api_base: Optional[str] = None - if _allow_client_side_advisor_credentials(): - advisor_api_key = advisor_tool.get("api_key") - advisor_api_base = advisor_tool.get("api_base") + advisor_api_key, advisor_api_base = _resolve_advisor_credentials(advisor_tool) # Build the synthetic tool definition the provider will receive. synthetic_advisor_tool = _make_synthetic_advisor_tool() @@ -186,6 +179,49 @@ def _allow_client_side_advisor_credentials() -> bool: return general_settings.get("allow_client_side_credentials") is True +def _resolve_advisor_credentials(advisor_tool: dict) -> tuple[Optional[str], Optional[str]]: + """Resolve the (api_key, api_base) override for the advisor sub-call. + + A caller-supplied ``api_base`` is only honored alongside a caller-supplied + ``api_key``: without one, ``AnthropicModelInfo.get_auth_header()`` falls + back to the proxy's own Anthropic credentials, which would then be sent to + the caller-chosen ``api_base``. A caller-supplied ``api_base`` is also + required to be https with TLS verification on, and SSRF-validated so it + can't target a private/internal/cloud-metadata address, mirroring + ``proxy.auth.auth_utils.check_complete_credentials``. https with TLS + verification is required because ``validate_url`` only rewrites the + connection to a DNS-pinned IP for http, or for https with + ``litellm.ssl_verify`` disabled; otherwise it returns the URL unchanged + and relies on certificate validation to block DNS rebinding, so this + closes the same gap without threading the pinned URL through the whole + ``anthropic_messages()`` call chain. + """ + if not _allow_client_side_advisor_credentials(): + return None, None + api_key: Optional[str] = advisor_tool.get("api_key") + api_base: Optional[str] = advisor_tool.get("api_base") + if api_base is None: + return api_key, None + if not api_key: + raise ValueError( + "advisor tool definition sets 'api_base' without 'api_key'. A " + "caller-supplied api_base is only honored alongside a " + "caller-supplied api_key, so the proxy's own credentials are " + "never sent to a caller-chosen destination." + ) + if not api_base.startswith("https://"): + raise ValueError(f"advisor tool definition sets 'api_base'={api_base!r}, which must use the https scheme.") + if getattr(litellm, "ssl_verify", True) is False: + raise ValueError( + "advisor tool definition sets 'api_base' but the proxy has TLS verification " + "disabled (litellm.ssl_verify=False), so a caller-supplied api_base can't be " + "safely validated against DNS rebinding." + ) + if getattr(litellm, "user_url_validation", True): + validate_url(api_base) + return api_key, api_base + + def _make_synthetic_advisor_tool() -> Dict: """Build a regular tool definition the executor provider can understand.""" return { diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py index 2357960f716..5f2b23d7eca 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py @@ -1,8 +1,13 @@ import asyncio import json from datetime import datetime -from typing import Any, AsyncIterator, List, Union +from typing import Any, AsyncIterator, List, Protocol, Union, runtime_checkable +import httpx +from pydantic import TypeAdapter +from typing_extensions import TypedDict + +from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, @@ -12,6 +17,93 @@ from litellm.types.utils import GenericStreamingChunk, ModelResponseStream GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ = PassThroughEndpointLogging() +INCOMPLETE_STREAM_ERROR_MESSAGE = ( + "Provider stream ended before emitting a message_stop event; " + "the response is incomplete and any partial content (e.g. tool_use input JSON) may be truncated." +) + + +def _is_message_stop_chunk(chunk: object) -> bool: + if isinstance(chunk, dict): + return chunk.get("type") == "message_stop" + if isinstance(chunk, (bytes, bytearray)): + return any(line == b"event: message_stop" for line in chunk.splitlines()) + return False + + +def _is_provider_error_chunk(chunk: object) -> bool: + if isinstance(chunk, dict): + return chunk.get("type") == "error" + if isinstance(chunk, (bytes, bytearray)): + return any(line == b"event: error" for line in chunk.splitlines()) + return False + + +def _is_terminal_stream_chunk(chunk: object) -> bool: + return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk) + + +def _incomplete_stream_error_sse_event() -> bytes: + payload = json.dumps( + { + "type": "error", + "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE}, + } + ) + return f"event: error\ndata: {payload}\n\n".encode() + + +class AnthropicMessagesStreamHiddenParams(TypedDict): + additional_headers: dict[str, str] + + +@runtime_checkable +class SupportsAclose(Protocol): + async def aclose(self) -> None: ... + + +async def aclose_if_supported(stream: object) -> None: + if isinstance(stream, SupportsAclose): + await stream.aclose() + + +_RESPONSE_HEADERS_ADAPTER: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str]) + + +def anthropic_messages_stream_hidden_params( + response_headers: httpx.Headers, +) -> AnthropicMessagesStreamHiddenParams: + return AnthropicMessagesStreamHiddenParams( + additional_headers=_RESPONSE_HEADERS_ADAPTER.validate_python(process_response_headers(response_headers)) + ) + + +class AnthropicMessagesStreamingResponse: + """ + Wraps the /v1/messages SSE byte stream so upstream provider response + headers (e.g. Bedrock's x-amzn-requestid / x-amzn-trace-id) survive as + ``_hidden_params["additional_headers"]``, which the proxy forwards to + clients as ``llm_provider-*`` response headers. Bare async generators + cannot carry attributes, so header context was previously dropped. + """ + + def __init__( + self, + completion_stream: AsyncIterator[bytes], + hidden_params: AnthropicMessagesStreamHiddenParams, + ) -> None: + self.completion_stream = completion_stream + self._hidden_params = hidden_params + + def __aiter__(self) -> "AnthropicMessagesStreamingResponse": + return self + + async def __anext__(self) -> bytes: + return await self.completion_stream.__anext__() + + async def aclose(self) -> None: + await aclose_if_supported(self.completion_stream) + class BaseAnthropicMessagesStreamingIterator: """ @@ -102,13 +194,18 @@ class BaseAnthropicMessagesStreamingIterator: This method provides the common logic for both Anthropic and Bedrock implementations. """ collected_chunks = [] + saw_terminal_event = False async for chunk in completion_stream: if self.completion_start_time is None: self.completion_start_time = datetime.now() + saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk) encoded_chunk = self._convert_chunk_to_sse_format(chunk) collected_chunks.append(encoded_chunk) yield encoded_chunk + if not saw_terminal_event: + yield _incomplete_stream_error_sse_event() + # Handle logging after all chunks are processed await self._handle_streaming_logging(collected_chunks) diff --git a/litellm/llms/anthropic/files/transformation.py b/litellm/llms/anthropic/files/transformation.py index cf12ad9ab32..0fa01e09492 100644 --- a/litellm/llms/anthropic/files/transformation.py +++ b/litellm/llms/anthropic/files/transformation.py @@ -39,6 +39,7 @@ from ..common_utils import AnthropicError, AnthropicModelInfo ANTHROPIC_FILES_API_BASE = "https://api.anthropic.com" ANTHROPIC_FILES_BETA_HEADER = "files-api-2025-04-14" +ANTHROPIC_MESSAGE_BATCH_ID_PREFIX = "msgbatch_" class AnthropicFilesConfig(BaseFilesConfig): @@ -258,6 +259,8 @@ class AnthropicFilesConfig(BaseFilesConfig): file_id = file_content_request.get("file_id") api_base = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + if file_id.startswith(ANTHROPIC_MESSAGE_BATCH_ID_PREFIX): + return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_file_id}/results", {} return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}/content", {} def transform_file_content_response( diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index fef9b7d0154..d0b0dbb070d 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -205,7 +205,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): ######################################################### ########## DELETE RESPONSE API TRANSFORMATION ############## ######################################################### - def _construct_url_for_response_id_in_path(self, api_base: str, response_id: str) -> str: + def _construct_url_for_response_id_in_path(self, api_base: str, response_id: str, path_suffix: str = "") -> str: """ Constructs a URL for the API request with the response_id in the path. """ @@ -218,14 +218,14 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): # Remove trailing slash if present to avoid double slashes path = parsed_url.path.rstrip("/") encoded_response_id = encode_url_path_segment(response_id, field_name="response_id") - new_path = f"{path}/{encoded_response_id}" + new_path = f"{path}/{encoded_response_id}{path_suffix}" # Reconstruct the URL with all original components but with the modified path constructed_url = urlunparse( ( parsed_url.scheme, # http, https parsed_url.netloc, # domain name, port - new_path, # path with response_id added + new_path, parsed_url.params, # parameters parsed_url.query, # query string parsed_url.fragment, # fragment @@ -288,7 +288,9 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): limit: int = 20, order: Literal["asc", "desc"] = "desc", ) -> Tuple[str, Dict]: - url = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id) + "/input_items" + url = self._construct_url_for_response_id_in_path( + api_base=api_base, response_id=response_id, path_suffix="/input_items" + ) params: Dict[str, Any] = {} if after is not None: params["after"] = after @@ -322,27 +324,8 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): This function handles URLs with query parameters by inserting the response_id at the correct location (before any query parameters). """ - from urllib.parse import urlparse, urlunparse - - # Parse the URL to separate its components - parsed_url = urlparse(api_base) - - # Insert the response_id and /cancel at the end of the path component - # Remove trailing slash if present to avoid double slashes - path = parsed_url.path.rstrip("/") - encoded_response_id = encode_url_path_segment(response_id, field_name="response_id") - new_path = f"{path}/{encoded_response_id}/cancel" - - # Reconstruct the URL with all original components but with the modified path - cancel_url = urlunparse( - ( - parsed_url.scheme, # http, https - parsed_url.netloc, # domain name, port - new_path, # path with response_id and /cancel added - parsed_url.params, # parameters - parsed_url.query, # query string - parsed_url.fragment, # fragment - ) + cancel_url = self._construct_url_for_response_id_in_path( + api_base=api_base, response_id=response_id, path_suffix="/cancel" ) data: Dict = {} diff --git a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py index c67703f64d7..7d915892a28 100644 --- a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py +++ b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py @@ -15,6 +15,7 @@ from typing import Any, Dict from urllib.parse import quote import httpx +from pydantic import BaseModel from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin @@ -38,6 +39,30 @@ from litellm.secret_managers.main import get_secret_str AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY" +class AzureDocumentIntelligenceLine(BaseModel): + content: str | None = None + + +class AzureDocumentIntelligencePage(BaseModel): + pageNumber: int | None = None + width: float | None = None + height: float | None = None + unit: str | None = None + lines: tuple[AzureDocumentIntelligenceLine, ...] = () + + +class AzureDocumentIntelligenceAnalyzeResult(BaseModel): + content: str | None = None + pages: tuple[AzureDocumentIntelligencePage, ...] = () + tables: list[dict[str, object]] | None = None + keyValuePairs: list[dict[str, object]] | None = None + + +class AzureDocumentIntelligenceOperation(BaseModel): + status: str | None = None + analyzeResult: AzureDocumentIntelligenceAnalyzeResult | None = None + + class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): """ Azure Document Intelligence OCR transformation configuration. @@ -67,11 +92,14 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): (1-based, e.g. "1-3,5,7-9"). To keep the public request shape aligned with Mistral OCR, callers pass `pages` using Mistral semantics — a list of 0-based integers — or a pre-formatted - Azure-style string. Other Mistral-specific params (e.g. + Azure-style string. Azure DI also exposes a `features` query + parameter enabling add-on capabilities (e.g. "keyValuePairs", + "languages"), passed as a list of feature names or a + comma-separated string. Other Mistral-specific params (e.g. `include_image_base64`) are not supported by Azure DI and are ignored during transformation. """ - return ["pages"] + return ["pages", "features"] def map_ocr_params( self, @@ -85,16 +113,18 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): Translates Mistral-style `pages` (list[int], 0-based) into Azure's `pages` query string (1-based, e.g. "1,2,3" or "1-3,5"). A raw string that already matches Azure's format is passed through - unchanged. + unchanged. `features` (list[str] or comma-separated string) is + normalized into Azure's comma-joined `features` query string. """ pages = non_default_params.get("pages") - if pages is None: - return optional_params - - normalized = self._normalize_pages_param(pages) - if normalized: - optional_params["pages"] = normalized - return optional_params + features = non_default_params.get("features") + normalized_pages = self._normalize_pages_param(pages) if pages is not None else "" + normalized_features = self._normalize_features_param(features) if features is not None else "" + return { + **optional_params, + **({"pages": normalized_pages} if normalized_pages else {}), + **({"features": normalized_features} if normalized_features else {}), + } @staticmethod def _normalize_pages_param(pages: Any) -> str: @@ -140,6 +170,39 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): raise ValueError("`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'.") + @staticmethod + def _normalize_features_param(features: object) -> str: + """ + Convert a caller-provided `features` value to Azure DI's query-string + form (comma-joined feature names, e.g. "keyValuePairs,languages"). + + Accepted inputs: + - list[str]: feature names like ["keyValuePairs", "languages"]. + - str: a single feature name or comma-separated names. + """ + invalid_features_error = ValueError( + f"Invalid `features` for Azure Document Intelligence: {features!r}. " + f"Expected a list of feature names or a comma-separated string like " + f"'keyValuePairs' or 'keyValuePairs,languages'." + ) + + if isinstance(features, str): + raw_tokens = features.split(",") + elif isinstance(features, list): + if len(features) == 0: + return "" + raw_tokens = [feature for feature in features if isinstance(feature, str)] + if len(raw_tokens) != len(features): + raise invalid_features_error + else: + raise invalid_features_error + + tokens = tuple(token.strip() for token in raw_tokens) + feature_pattern = re.compile(r"^[A-Za-z][A-Za-z0-9]*$") + if not all(feature_pattern.match(token) for token in tokens): + raise invalid_features_error + return ",".join(tokens) + def validate_environment( self, headers: Dict, @@ -228,13 +291,15 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): f"?api-version={AZURE_DOCUMENT_INTELLIGENCE_API_VERSION}" ) - # Azure DI accepts `pages` as a query param (1-based, e.g. "1-3,5"). + # Azure DI accepts `pages` (1-based, e.g. "1-3,5") and `features` + # (comma-joined names, e.g. "keyValuePairs") as query params. # `optional_params` has already been normalized in `map_ocr_params`. pages = optional_params.get("pages") if optional_params else None - if pages: - url += f"&pages={quote(str(pages), safe=',-')}" + features = optional_params.get("features") if optional_params else None + pages_query = f"&pages={quote(str(pages), safe=',-')}" if pages else "" + features_query = f"&features={quote(str(features), safe=',')}" if features else "" - return url + return f"{url}{pages_query}{features_query}" def _extract_base64_from_data_uri(self, data_uri: str) -> str: """ @@ -328,27 +393,15 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): return OCRRequestData(data=data, files=None) - def _extract_page_markdown(self, page_data: Dict[str, Any]) -> str: - """ - Extract text from Azure DI page and format as markdown. - - Azure DI provides text in 'lines' array. We concatenate them with newlines. - - Args: - page_data: Azure DI page object - - Returns: - Markdown-formatted text - """ - lines = page_data.get("lines", []) - if not lines: - return "" - - # Extract text content from each line - text_lines = [line.get("content", "") for line in lines] - - # Join with newlines to preserve structure - return "\n".join(text_lines) + def _transform_azure_page(self, azure_page: AzureDocumentIntelligencePage) -> OCRPage: + page_number = azure_page.pageNumber if azure_page.pageNumber is not None else 1 + markdown = "\n".join(line.content or "" for line in azure_page.lines) + dimensions = self._convert_dimensions( + width=azure_page.width if azure_page.width is not None else 8.5, + height=azure_page.height if azure_page.height is not None else 11, + unit=azure_page.unit if azure_page.unit is not None else "inch", + ) + return OCRPage(index=page_number - 1, markdown=markdown, dimensions=dimensions) def _convert_dimensions(self, width: float, height: float, unit: str) -> OCRPageDimensions: """ @@ -526,6 +579,52 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): retry_after = self._get_retry_after(response=response) await asyncio.sleep(retry_after) + def _get_polling_target(self, raw_response: httpx.Response) -> tuple[str, Dict[str, str]]: + operation_url = raw_response.headers.get("Operation-Location") + if not operation_url: + raise ValueError("Azure Document Intelligence returned 202 but no Operation-Location header found") + + # Reject cross-origin polling URLs — the auth headers + # below would otherwise leak to whatever URL the upstream + # (or an attacker-controlled upstream) returns. VERIA-51. + try: + assert_same_origin(operation_url, str(raw_response.request.url)) + except SSRFError as ssrf_err: + raise ValueError(f"Azure Document Intelligence: rejected polling URL ({ssrf_err})") + + poll_headers = {"Ocp-Apim-Subscription-Key": raw_response.request.headers.get("Ocp-Apim-Subscription-Key", "")} + return operation_url, poll_headers + + def _transform_completed_response(self, model: str, raw_response: httpx.Response) -> OCRResponse: + """ + Transform a completed Azure Document Intelligence analyze operation + into the Mistral OCR response shape, preserving Azure-native + `analyzeResult` fields (`content`, `tables`, `keyValuePairs`) as + top-level response fields. + """ + operation = AzureDocumentIntelligenceOperation.model_validate(raw_response.json()) + + verbose_logger.debug(f"Azure Document Intelligence response status: {operation.status}") + + if operation.status != "succeeded": + raise ValueError(f"Azure Document Intelligence analysis failed with status: {operation.status}") + + analyze_result = ( + operation.analyzeResult if operation.analyzeResult is not None else AzureDocumentIntelligenceAnalyzeResult() + ) + mistral_pages = [self._transform_azure_page(azure_page) for azure_page in analyze_result.pages] + usage_info = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None) + + return OCRResponse( + pages=mistral_pages, + model=model, + usage_info=usage_info, + object="ocr", + content=analyze_result.content, + tables=analyze_result.tables, + keyValuePairs=analyze_result.keyValuePairs, + ) + def transform_ocr_response( self, model: str, @@ -552,11 +651,13 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): "unit": "inch", "lines": [{"content": "text", "boundingBox": [...]}] } - ] + ], + "tables": [...], + "keyValuePairs": [...] } } - Mistral OCR format: + Mistral OCR format (with Azure-native fields preserved): { "pages": [ { @@ -567,7 +668,10 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): ], "model": "azure_ai/doc-intelligence/prebuilt-layout", "usage_info": {"pages_processed": 1}, - "object": "ocr" + "object": "ocr", + "content": "Full document text...", + "tables": [...], + "keyValuePairs": [...] } Args: @@ -578,86 +682,17 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): Returns: OCRResponse in Mistral format """ - try: - # Check if we got 202 Accepted (async operation started) - if raw_response.status_code == 202: - verbose_logger.debug("Azure DI returned 202 Accepted, polling operation...") + if raw_response.status_code != 202: + return self._transform_completed_response(model=model, raw_response=raw_response) - # Get Operation-Location header - operation_url = raw_response.headers.get("Operation-Location") - if not operation_url: - raise ValueError("Azure Document Intelligence returned 202 but no Operation-Location header found") - - # Reject cross-origin polling URLs — the auth headers - # below would otherwise leak to whatever URL the upstream - # (or an attacker-controlled upstream) returns. VERIA-51. - try: - assert_same_origin(operation_url, str(raw_response.request.url)) - except SSRFError as ssrf_err: - raise ValueError(f"Azure Document Intelligence: rejected polling URL ({ssrf_err})") - - # Get headers for polling (need auth) - poll_headers = { - "Ocp-Apim-Subscription-Key": raw_response.request.headers.get("Ocp-Apim-Subscription-Key", "") - } - - # Get timeout from kwargs or use default - timeout_secs = AZURE_OPERATION_POLLING_TIMEOUT - - # Poll until operation completes - raw_response = self._poll_operation_sync( - operation_url=operation_url, - headers=poll_headers, - timeout_secs=timeout_secs, - ) - - # Now parse the completed response - response_json = raw_response.json() - - verbose_logger.debug(f"Azure Document Intelligence response status: {response_json.get('status')}") - - # Check if request succeeded - status = response_json.get("status") - if status != "succeeded": - raise ValueError(f"Azure Document Intelligence analysis failed with status: {status}") - - # Extract analyze result - analyze_result = response_json.get("analyzeResult", {}) - azure_pages = analyze_result.get("pages", []) - - # Transform pages to Mistral format - mistral_pages = [] - for azure_page in azure_pages: - page_number = azure_page.get("pageNumber", 1) - index = page_number - 1 # Convert to 0-based index - - # Extract markdown text - markdown = self._extract_page_markdown(azure_page) - - # Convert dimensions - width = azure_page.get("width", 8.5) - height = azure_page.get("height", 11) - unit = azure_page.get("unit", "inch") - dimensions = self._convert_dimensions(width=width, height=height, unit=unit) - - # Build OCR page - ocr_page = OCRPage(index=index, markdown=markdown, dimensions=dimensions) - mistral_pages.append(ocr_page) - - # Build usage info - usage_info = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None) - - # Return Mistral OCR response - return OCRResponse( - pages=mistral_pages, - model=model, - usage_info=usage_info, - object="ocr", - ) - - except Exception as e: - verbose_logger.error(f"Error parsing Azure Document Intelligence response: {e}") - raise e + verbose_logger.debug("Azure DI returned 202 Accepted, polling operation...") + operation_url, poll_headers = self._get_polling_target(raw_response) + completed_response = self._poll_operation_sync( + operation_url=operation_url, + headers=poll_headers, + timeout_secs=AZURE_OPERATION_POLLING_TIMEOUT, + ) + return self._transform_completed_response(model=model, raw_response=completed_response) async def async_transform_ocr_response( self, @@ -680,81 +715,14 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): Returns: OCRResponse in Mistral format """ - try: - # Check if we got 202 Accepted (async operation started) - if raw_response.status_code == 202: - verbose_logger.debug("Azure DI returned 202 Accepted, polling operation (async)...") + if raw_response.status_code != 202: + return self._transform_completed_response(model=model, raw_response=raw_response) - # Get Operation-Location header - operation_url = raw_response.headers.get("Operation-Location") - if not operation_url: - raise ValueError("Azure Document Intelligence returned 202 but no Operation-Location header found") - - # Reject cross-origin polling URLs (see sync path). VERIA-51. - try: - assert_same_origin(operation_url, str(raw_response.request.url)) - except SSRFError as ssrf_err: - raise ValueError(f"Azure Document Intelligence: rejected polling URL ({ssrf_err})") - - # Get headers for polling (need auth) - poll_headers = { - "Ocp-Apim-Subscription-Key": raw_response.request.headers.get("Ocp-Apim-Subscription-Key", "") - } - - # Get timeout from kwargs or use default - timeout_secs = AZURE_OPERATION_POLLING_TIMEOUT - - # Poll until operation completes (async) - raw_response = await self._poll_operation_async( - operation_url=operation_url, - headers=poll_headers, - timeout_secs=timeout_secs, - ) - - # Now parse the completed response - response_json = raw_response.json() - - verbose_logger.debug(f"Azure Document Intelligence response status: {response_json.get('status')}") - - # Check if request succeeded - status = response_json.get("status") - if status != "succeeded": - raise ValueError(f"Azure Document Intelligence analysis failed with status: {status}") - - # Extract analyze result - analyze_result = response_json.get("analyzeResult", {}) - azure_pages = analyze_result.get("pages", []) - - # Transform pages to Mistral format - mistral_pages = [] - for azure_page in azure_pages: - page_number = azure_page.get("pageNumber", 1) - index = page_number - 1 # Convert to 0-based index - - # Extract markdown text - markdown = self._extract_page_markdown(azure_page) - - # Convert dimensions - width = azure_page.get("width", 8.5) - height = azure_page.get("height", 11) - unit = azure_page.get("unit", "inch") - dimensions = self._convert_dimensions(width=width, height=height, unit=unit) - - # Build OCR page - ocr_page = OCRPage(index=index, markdown=markdown, dimensions=dimensions) - mistral_pages.append(ocr_page) - - # Build usage info - usage_info = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None) - - # Return Mistral OCR response - return OCRResponse( - pages=mistral_pages, - model=model, - usage_info=usage_info, - object="ocr", - ) - - except Exception as e: - verbose_logger.error(f"Error parsing Azure Document Intelligence response (async): {e}") - raise e + verbose_logger.debug("Azure DI returned 202 Accepted, polling operation (async)...") + operation_url, poll_headers = self._get_polling_target(raw_response) + completed_response = await self._poll_operation_async( + operation_url=operation_url, + headers=poll_headers, + timeout_secs=AZURE_OPERATION_POLLING_TIMEOUT, + ) + return self._transform_completed_response(model=model, raw_response=completed_response) diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 68db36b529e..6c41b46cfa0 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -2,7 +2,10 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Dict, List, Optional if TYPE_CHECKING: - from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + ModifyResponseException, + ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import AllMessageValues @@ -98,6 +101,30 @@ class BaseTranslation(ABC): """ return responses_so_far + def build_block_sse_chunks( + self, + exc: "ModifyResponseException", + stream_started: bool = False, + responses_so_far: Optional[list[Any]] = None, + ) -> Optional[list[bytes]]: + """ + Build the streaming chunks that deliver a guardrail block message and + cleanly terminate the stream in this provider's wire format. + + ``stream_started`` is True when real chunks were already sent to the + client: the result must *continue* the in-progress message (e.g. close + the open content block and append the block message) rather than start + a new one, which clients reject. ``responses_so_far`` provides the prior + chunks needed to do so. When False, nothing has been sent and a + standalone block message is emitted. + + Returns None when the format has no safe terminator; the caller then + re-raises ``exc`` so the proxy can surface a clean error instead. + Override in provider subclasses that support synthesizing a block + stream. + """ + return None + def get_structured_messages(self, data: dict) -> Optional[List["AllMessageValues"]]: """ Convert request data to OpenAI-spec structured messages. diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index 97ece6b5eab..8a06dd4ea52 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -1,10 +1,100 @@ from __future__ import annotations -from typing import Any, List +import json +from typing import Any, List, Optional +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues +def _anthropic_stream_chunk_events(item: Any) -> list[dict]: + if isinstance(item, dict): + return [item] + if isinstance(item, bytes): + chunk = item.decode("utf-8", errors="replace") + elif isinstance(item, str): + chunk = item + else: + return [] + + events: list[dict] = [] + for block in chunk.split("\n\n"): + for line in block.splitlines(): + stripped = line.strip() + if not stripped.startswith("data:"): + continue + payload = stripped[len("data:") :].strip() + if not payload or payload == "[DONE]": + continue + try: + parsed = json.loads(payload) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + events.append(parsed) + return events + + +def _usage_from_anthropic_stream_chunks(original_response: list[Any]) -> Optional[AnthropicUsage]: + input_tokens = 0 + output_tokens = 0 + found_usage = False + + for item in original_response: + for event in _anthropic_stream_chunk_events(item): + event_type = event.get("type") + if event_type == "message_start": + message = event.get("message") or {} + usage_obj = message.get("usage") or {} + elif event_type == "message_delta": + usage_obj = event.get("usage") or {} + else: + usage_obj = {} + if not isinstance(usage_obj, dict): + continue + if usage_obj.get("input_tokens") is not None: + input_tokens = int(usage_obj.get("input_tokens") or 0) + found_usage = True + if usage_obj.get("output_tokens") is not None: + output_tokens = int(usage_obj.get("output_tokens") or 0) + found_usage = True + + if not found_usage: + return None + return AnthropicUsage(input_tokens=input_tokens, output_tokens=output_tokens) + + +def blocked_response_usage(original_response: Optional[Any]) -> AnthropicUsage: + """ + Token usage for a synthetic guardrail-blocked response. + + A post-call block replaces the LLM's response with the violation message, + but the upstream call already consumed tokens -- report that real usage + (carried on ``ModifyResponseException.original_response``) rather than + discarding it. Pre-call blocks never invoked the LLM (no original_response), + so usage is zero. + """ + usage_obj: Any = None + if isinstance(original_response, list): + stream_usage = _usage_from_anthropic_stream_chunks(original_response) + if stream_usage is not None: + return stream_usage + elif isinstance(original_response, dict): + usage_obj = original_response.get("usage") + elif original_response is not None: + usage_obj = getattr(original_response, "usage", None) + + def _tokens(key: str, fallback_key: str) -> int: + if isinstance(usage_obj, dict): + return int(usage_obj.get(key, usage_obj.get(fallback_key, 0)) or 0) + return int(getattr(usage_obj, key, getattr(usage_obj, fallback_key, 0)) or 0) + + return AnthropicUsage( + input_tokens=_tokens("input_tokens", "prompt_tokens"), + output_tokens=_tokens("output_tokens", "completion_tokens"), + ) + + def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool: per = getattr(guardrail_to_apply, "skip_system_message_in_guardrail", None) if per is not None: diff --git a/litellm/llms/base_llm/ocr/transformation.py b/litellm/llms/base_llm/ocr/transformation.py index a38e5bfdcd6..0d878bd308c 100644 --- a/litellm/llms/base_llm/ocr/transformation.py +++ b/litellm/llms/base_llm/ocr/transformation.py @@ -70,6 +70,9 @@ class OCRResponse(LiteLLMPydanticObjectBase): model: str document_annotation: Any | None = None usage_info: OCRUsageInfo | None = None + content: str | None = None + tables: list[dict[str, object]] | None = None + keyValuePairs: list[dict[str, object]] | None = None object: str = "ocr" model_config = {"extra": "allow"} diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index b135a116753..5a8ada45651 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -76,6 +76,7 @@ from litellm.utils import ( from ..common_utils import ( BedrockError, BedrockModelInfo, + bedrock_converse_supports_parallel_tool_use_config, get_anthropic_beta_from_headers, get_bedrock_tool_name, is_claude_4_5_on_bedrock, @@ -1106,18 +1107,28 @@ class AmazonConverseConfig(BaseConfig): if cache_control is None: return None - cache_point = CachePointBlock(type="default") - if isinstance(cache_control, dict) and "ttl" in cache_control: - ttl = cache_control["ttl"] - if ttl in ["5m", "1h"] and model is not None: - if is_claude_4_5_on_bedrock(model): - cache_point["ttl"] = ttl + cache_point = self._build_cache_point_block(cache_control, model) if block_type == "system": return SystemContentBlock(cachePoint=cache_point) else: return ContentBlock(cachePoint=cache_point) + @staticmethod + def _build_cache_point_block(control: Optional[dict], model: Optional[str] = None) -> CachePointBlock: + """Build a Bedrock ``cachePoint`` block from an OpenAI-style ``cache_control``/``control`` dict. + + ``type`` is always ``"default"`` (the only value Bedrock's Converse API + accepts). ``ttl`` is only honored for models that support extended TTL + caching (Claude 4.5 family on Bedrock). + """ + cache_point = CachePointBlock(type="default") + if isinstance(control, dict) and "ttl" in control: + ttl = control["ttl"] + if ttl in ["5m", "1h"] and model is not None and is_claude_4_5_on_bedrock(model): + cache_point["ttl"] = ttl + return cache_point + def _transform_system_message( self, messages: List[AllMessageValues], model: Optional[str] = None ) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]: @@ -1241,7 +1252,7 @@ class AmazonConverseConfig(BaseConfig): # Handle parallel_tool_calls configuration parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None) - if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model): + if parallel_tool_use_config is not None and bedrock_converse_supports_parallel_tool_use_config(model): for key, value in parallel_tool_use_config.items(): if ( key in additional_request_params @@ -1526,7 +1537,8 @@ class AmazonConverseConfig(BaseConfig): if cache_injection_points and len(bedrock_tools) > 0: for point in cache_injection_points: if point.get("location") == "tool_config": - bedrock_tools.append({"cachePoint": {"type": "default"}}) + cache_point = self._build_cache_point_block(point.get("control"), model) + bedrock_tools.append(ToolBlock(cachePoint=cache_point)) break bedrock_tool_config: Optional[ToolConfigBlock] = None diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index b381b5a85fe..4c256be1ab8 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -1558,7 +1558,7 @@ class AWSEventStreamDecoder: text = chunk_data["outputText"] # ai21 mapping elif "ai21" in self.model: # fake ai21 streaming - text = chunk_data.get("completions")[0].get("data").get("text") # type: ignore + text = chunk_data["completions"][0]["data"]["text"] is_finished = True finish_reason = "stop" ######## /bedrock/converse mappings ############### diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py index d63642c806f..a2aa98d6676 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py @@ -51,10 +51,7 @@ class AmazonQwen2Config(AmazonQwen3Config): Qwen2 uses "text" field, but we also support "generation" field for compatibility. """ try: - if hasattr(raw_response, "json"): - response_data = raw_response.json() - else: - response_data = raw_response + response_data = raw_response.json() # Extract the generated text - Qwen2 uses "text" field, but also support "generation" for compatibility generated_text = response_data.get("generation", "") or response_data.get("text", "") diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py index 762631cac5e..4f496df084e 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py @@ -175,10 +175,7 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): Transform Qwen3 Bedrock response to OpenAI format """ try: - if hasattr(raw_response, "json"): - response_data = raw_response.json() - else: - response_data = raw_response + response_data = raw_response.json() # Extract the generated text - Qwen3 uses "generation" field generated_text = response_data.get("generation", "") diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index bbe16e26713..dd7cf12604d 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -5,6 +5,7 @@ from functools import partial from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast, get_args import httpx +from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger @@ -24,6 +25,7 @@ from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, _get_httpx_client, ) +from litellm.types.llms.bedrock import GuardrailConfigBlock from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage from litellm.utils import CustomStreamWrapper @@ -37,6 +39,38 @@ else: from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +_GUARDRAIL_CONFIG_VALIDATOR: "TypeAdapter[GuardrailConfigBlock]" = TypeAdapter(GuardrailConfigBlock) + +_GUARDRAIL_CONFIG_EXPECTED_FORMAT = ( + "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}" +) + + +def _bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]": + try: + guardrail_config = _GUARDRAIL_CONFIG_VALIDATOR.validate_python(raw_guardrail_config) + except ValidationError as e: + raise BedrockError( + status_code=400, + message="Invalid guardrailConfig={}. Expected format: {}. Error: {}".format( + raw_guardrail_config, _GUARDRAIL_CONFIG_EXPECTED_FORMAT, e + ), + ) + if "guardrailIdentifier" not in guardrail_config: + raise BedrockError( + status_code=400, + message="guardrailConfig={} is missing 'guardrailIdentifier'. Expected format: {}".format( + raw_guardrail_config, _GUARDRAIL_CONFIG_EXPECTED_FORMAT + ), + ) + trace = guardrail_config.get("trace") + candidate_headers = { + "X-Amzn-Bedrock-GuardrailIdentifier": guardrail_config.get("guardrailIdentifier"), + "X-Amzn-Bedrock-GuardrailVersion": guardrail_config.get("guardrailVersion"), + "X-Amzn-Bedrock-Trace": trace.upper() if trace is not None else None, + } + return {name: value for name, value in candidate_headers.items() if value is not None} + class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): def __init__(self, **kwargs): @@ -390,7 +424,16 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> dict: - return headers + raw_guardrail_config = optional_params.pop("guardrailConfig", None) + if raw_guardrail_config is None: + return headers + existing_header_names = frozenset(name.lower() for name in headers) + guardrail_headers = { + name: value + for name, value in _bedrock_invoke_guardrail_headers(raw_guardrail_config).items() + if name.lower() not in existing_header_names + } + return {**headers, **guardrail_headers} def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index d84e077c37b..d7ffff65ff0 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -7,13 +7,14 @@ The bedrock-mantle endpoint uses the Anthropic Messages API format but is served at a different endpoint (bedrock-mantle.{region}.api.aws) with AWS SigV4 auth. """ -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, ) from litellm.llms.bedrock.common_utils import build_mantle_messages_url from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -91,10 +92,14 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): litellm_params=litellm_params, headers=headers, ) - # The parent strips "model" from the body (Invoke API puts it in URL). - # The mantle endpoint (Messages API) requires "model" in the body. - request["model"] = model_id - return request + # The parent strips "model" and "stream" from the body (Invoke API puts + # the model in the URL and streams via a dedicated endpoint). The mantle + # endpoint (Messages API) requires both in the body. + return self._restore_mantle_body_fields( + request=request, + model_id=model_id, + optional_params=optional_params, + ) async def async_transform_request( self, @@ -114,5 +119,31 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): headers=headers, ) await self._async_convert_document_url_sources_to_base64(request) - request["model"] = model_id - return request + return self._restore_mantle_body_fields( + request=request, + model_id=model_id, + optional_params=optional_params, + ) + + @staticmethod + def _restore_mantle_body_fields(request: dict, model_id: str, optional_params: dict) -> dict: + stream_fields: dict = {"stream": True} if optional_params.get("stream") is True else {} + return {**request, "model": model_id, **stream_fields} + + @property + def has_custom_stream_wrapper(self) -> bool: + return False + + def get_model_response_iterator( + self, + streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, + sync_stream: bool, + json_mode: Optional[bool] = False, + ) -> Any: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + return ModelResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index df432a4d7e3..5114677ffc0 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -685,39 +685,27 @@ def get_bedrock_base_model(model: str) -> str: return model +def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: + return any( + (litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True + for candidate in (model, get_bedrock_base_model(model)) + ) + + def is_claude_4_5_on_bedrock(model: str) -> bool: """ - Check if the model is a Claude 4.5 model on Bedrock. - Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock. + Check if the model supports Bedrock prompt caching with an extended '1h' TTL + (in addition to the default 5m TTL). + + Backed by the ``cache_creation_input_token_cost_above_1hr`` field in + ``model_prices_and_context_window.json`` instead of a hardcoded list of + model-name patterns, so newly released models pick up support as soon as + their pricing entry ships, with no code change required here. """ - model_lower = model.lower() - claude_4_5_patterns = [ - "sonnet-4.5", - "sonnet_4.5", - "sonnet-4-5", - "sonnet_4_5", - "haiku-4.5", - "haiku_4.5", - "haiku-4-5", - "haiku_4_5", - "opus-4.5", - "opus_4.5", - "opus-4-5", - "opus_4_5", - "sonnet-4.6", - "sonnet_4.6", - "sonnet-4-6", - "sonnet_4_6", - "opus-4.6", - "opus_4.6", - "opus-4-6", - "opus_4_6", - "opus-4.7", - "opus_4.7", - "opus-4-7", - "opus_4_7", - ] - return any(pattern in model_lower for pattern in claude_4_5_patterns) + return any( + (litellm.model_cost.get(candidate) or {}).get("cache_creation_input_token_cost_above_1hr") is not None + for candidate in (model, get_bedrock_base_model(model)) + ) _BEDROCK_MODEL_VERSION_SUFFIX_RE = re.compile(r"-v\d+(?::\d+)?$") diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index a8a7b7ed1d5..da7b8697a6b 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -6,8 +6,13 @@ AmazonAnthropicClaudeMessagesConfig. Overrides only the URL and model-prefix stripping that are specific to the bedrock-mantle endpoint. """ -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Tuple +import httpx + +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) from litellm.llms.bedrock.common_utils import build_mantle_messages_url from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeMessagesConfig, @@ -89,8 +94,26 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): headers=headers, ) - # Parent (AmazonAnthropicClaudeMessagesConfig) removes "model" from the - # body (Bedrock Invoke puts model in the URL). The mantle endpoint - # (Messages API) requires "model" in the request body. - request["model"] = model_id - return request + # Parent (AmazonAnthropicClaudeMessagesConfig) removes "model" and + # "stream" from the body (Bedrock Invoke puts the model in the URL and + # streams via a dedicated endpoint). The mantle endpoint (Messages API) + # requires both in the request body. + stream_fields: dict[str, bool] = ( + {"stream": True} if anthropic_messages_optional_request_params.get("stream") is True else {} + ) + return {**request, "model": model_id, **stream_fields} + + def get_async_streaming_response_iterator( + self, + model: str, + httpx_response: httpx.Response, + request_body: dict, + litellm_logging_obj: LiteLLMLoggingObj, + ) -> AsyncIterator: + return AnthropicMessagesConfig.get_async_streaming_response_iterator( + self, + model=model, + httpx_response=httpx_response, + request_body=request_body, + litellm_logging_obj=litellm_logging_obj, + ) diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 557ee3348d5..b48c37791c4 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -13,6 +13,7 @@ from litellm._logging import _redact_string, verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ..base_aws_llm import BaseAWSLLM +from ..common_utils import BedrockError from .transformation import BedrockRealtimeConfig @@ -59,9 +60,7 @@ class BedrockRealtime(BaseAWSLLM): InvokeModelWithBidirectionalStreamOperationInput, ) from aws_sdk_bedrock_runtime.config import Config - from smithy_aws_core.identity.environment import ( - EnvironmentCredentialsResolver, - ) + from smithy_aws_core.identity import StaticCredentialsResolver except ImportError: raise ImportError("Missing aws_sdk_bedrock_runtime. Install with: pip install aws-sdk-bedrock-runtime") @@ -82,11 +81,36 @@ class BedrockRealtime(BaseAWSLLM): verbose_proxy_logger.debug(f"Bedrock Realtime: Connecting to {endpoint_uri} with model {model}") + credentials = self.get_credentials( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + aws_region_name=aws_region_name, + aws_session_name=aws_session_name, + aws_profile_name=aws_profile_name, + aws_role_name=aws_role_name, + aws_web_identity_token=aws_web_identity_token, + aws_sts_endpoint=aws_sts_endpoint, + aws_external_id=aws_external_id, + ) + if credentials is None: + raise BedrockError( + status_code=401, + message=( + "No AWS credentials found for Bedrock realtime. Set aws_* params in litellm_params " + "or configure credentials in the environment" + ), + ) + frozen_credentials = credentials.get_frozen_credentials() + # Initialize Bedrock client with aws_sdk_bedrock_runtime config = Config( endpoint_uri=endpoint_uri, region=aws_region_name, - aws_credentials_identity_resolver=EnvironmentCredentialsResolver(), + aws_access_key_id=frozen_credentials.access_key, + aws_secret_access_key=frozen_credentials.secret_key, + aws_session_token=frozen_credentials.token, + aws_credentials_identity_resolver=StaticCredentialsResolver(), ) bedrock_client = BedrockRuntimeClient(config=config) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3c10239f868..c426714a1bd 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1300,16 +1300,15 @@ class BaseLLMHTTPHandler: if client is None or not isinstance(client, HTTPHandler): client = _get_httpx_client() + json_data = data if files is None and isinstance(data, dict) else None + try: - # Make the POST request - clean and simple, always use data and files response = client.post( url=complete_url, headers=headers, - data=data, + data=data if json_data is None else None, files=files, - json=( - data if files is None and isinstance(data, dict) else None - ), # Use json param only when no files and data is dict + json=json_data, timeout=timeout, ) except Exception as e: @@ -1373,16 +1372,15 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client + json_data = data if files is None and isinstance(data, dict) else None + try: - # Make the async POST request - clean and simple, always use data and files response = await async_httpx_client.post( url=complete_url, headers=headers, - data=data, + data=data if json_data is None else None, files=files, - json=( - data if files is None and isinstance(data, dict) else None - ), # Use json param only when no files and data is dict + json=json_data, timeout=timeout, ) except Exception as e: @@ -2084,12 +2082,18 @@ class BaseLLMHTTPHandler: initial_response: Union[AsyncIterator, AnthropicMessagesResponse] if stream: + from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( + AnthropicMessagesStreamingResponse, + anthropic_messages_stream_hidden_params, + ) + completion_stream = anthropic_messages_provider_config.get_async_streaming_response_iterator( model=model, httpx_response=response, request_body=request_body, litellm_logging_obj=logging_obj, ) + stream_hidden_params = anthropic_messages_stream_hidden_params(response.headers) if not self._has_agentic_completion_hook(logging_obj): # No callback overrides async_should_run_agentic_loop, so the @@ -2097,7 +2101,10 @@ class BaseLLMHTTPHandler: # and rebuilding the response from SSE at end-of-stream to call # hooks that all return (False, {}). Stream through directly and # skip that per-chunk + end-of-stream overhead. - return completion_stream + return AnthropicMessagesStreamingResponse( + completion_stream=completion_stream, + hidden_params=stream_hidden_params, + ) from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, @@ -2114,7 +2121,10 @@ class BaseLLMHTTPHandler: custom_llm_provider=custom_llm_provider, kwargs={**kwargs, "api_key": api_key} if api_key else kwargs, ) - return initial_response + return AnthropicMessagesStreamingResponse( + completion_stream=initial_response, + hidden_params=stream_hidden_params, + ) else: initial_response = anthropic_messages_provider_config.transform_anthropic_messages_response( model=model, @@ -2902,6 +2912,7 @@ class BaseLLMHTTPHandler: try: response = sync_httpx_client.get(url=url, headers=headers, params=data) + response.raise_for_status() except Exception as e: raise self._handle_error( e=e, @@ -2973,9 +2984,9 @@ class BaseLLMHTTPHandler: try: response = await async_httpx_client.get(url=url, headers=headers, params=data) - + response.raise_for_status() except Exception as e: - verbose_logger.exception(f"Error retrieving response: {e}") + verbose_logger.debug(f"Error retrieving response: {e}") raise self._handle_error( e=e, provider_config=responses_api_provider_config, @@ -3066,6 +3077,7 @@ class BaseLLMHTTPHandler: try: response = sync_httpx_client.get(url=url, headers=headers, params=params) + response.raise_for_status() except Exception as e: raise self._handle_error(e=e, provider_config=responses_api_provider_config) @@ -3139,6 +3151,7 @@ class BaseLLMHTTPHandler: try: response = await async_httpx_client.get(url=url, headers=headers, params=params) + response.raise_for_status() except Exception as e: raise self._handle_error(e=e, provider_config=responses_api_provider_config) @@ -4725,6 +4738,13 @@ class BaseLLMHTTPHandler: except Exception as e: raise self._handle_error(e=e, provider_config=provider_config) + if response.status_code >= 400: + raise provider_config.get_error_class( + error_message=response.text, + status_code=response.status_code, + headers=response.headers, + ) + return provider_config.transform_file_content_response( raw_response=response, logging_obj=logging_obj, @@ -4781,6 +4801,13 @@ class BaseLLMHTTPHandler: except Exception as e: raise self._handle_error(e=e, provider_config=provider_config) + if response.status_code >= 400: + raise provider_config.get_error_class( + error_message=response.text, + status_code=response.status_code, + headers=response.headers, + ) + return provider_config.transform_file_content_response( raw_response=response, logging_obj=logging_obj, diff --git a/litellm/llms/oci/chat/cohere.py b/litellm/llms/oci/chat/cohere.py index 3661ac908d2..ca85a6309d7 100644 --- a/litellm/llms/oci/chat/cohere.py +++ b/litellm/llms/oci/chat/cohere.py @@ -110,7 +110,7 @@ def adapt_messages_to_cohere_standard( tool_calls: Optional[List[CohereToolCall]] = None if role == "assistant" and msg.get("tool_calls"): # type: ignore[union-attr,typeddict-item] tool_calls = [] - for tc in msg["tool_calls"]: # type: ignore[union-attr,typeddict-item] + for tc in msg["tool_calls"]: # pyright: ignore[reportOptionalIterable] # truthiness check above rules out None raw_arguments: Any = tc.get("function", {}).get("arguments", {}) if isinstance(raw_arguments, str): try: diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 3152ded2367..694f8cdd6c2 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -363,7 +363,11 @@ class OllamaChatConfig(BaseConfig): response_json_message["reasoning_content"] = reasoning_content response_json_message["content"] = content - if request_data.get("format", "") == "json" and litellm_params.get("function_name") is not None: + if ( + request_data.get("format", "") == "json" + and litellm_params.get("function_name") is not None + and response_json_message is not None + ): function_call = json.loads(response_json_message["content"]) message = litellm.Message( content=None, diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 396ad5b105e..f2498c0a7e2 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -2,6 +2,7 @@ Support for gpt model family """ +import json from typing import ( TYPE_CHECKING, Any, @@ -782,8 +783,30 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator): delta["reasoning_content"] = delta.pop("reasoning") return choices + @staticmethod + def _extract_error_from_chunk(chunk: dict) -> Optional[tuple[str, int]]: + """OpenAI-compatible backends (vLLM, sglang) can return an HTTP 200 + stream whose body carries an error payload, e.g. + ``data: {"error": {"message": "...", "code": 400}}``.""" + error = chunk.get("error") + if not error: + return None + if not isinstance(error, dict): + return str(error), 500 + message = error.get("message") + code = error.get("code") + status_code = code if isinstance(code, int) and 400 <= code < 600 else 500 + return (message if isinstance(message, str) else json.dumps(error)), status_code + def chunk_parser(self, chunk: dict) -> ModelResponseStream: try: + error_details = self._extract_error_from_chunk(chunk) + if error_details is not None: + error_message, error_status_code = error_details + raise OpenAIError( + status_code=error_status_code, + message=error_message, + ) choices = chunk.get("choices", []) choices = self._map_reasoning_to_reasoning_content(choices) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index c7a49a2e47f..a6b6a6267c3 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -716,7 +716,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): elif isinstance(content, list) and content_idx_optional is not None: # Replace specific text item in list content - choice.message.content[content_idx_optional]["text"] = guardrail_response # type: ignore + content[content_idx_optional]["text"] = guardrail_response async def _apply_guardrail_responses_to_output_tool_calls( self, diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 6ac33ffa44a..093dffccac0 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -45,6 +45,7 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolParam, + ResponsesAPIStreamEvents, ) from litellm.types.responses.main import ( GenericResponseOutputItem, @@ -196,12 +197,13 @@ class OpenAIResponsesHandler(BaseTranslation): return data def extract_request_tool_names(self, data: dict) -> List[str]: - """Extract tool names from Responses API request (tools[].name for function, tools[].server_label for mcp).""" + """Extract tool names from Responses API request (tools[].name for function + and custom, tools[].server_label for mcp).""" names: List[str] = [] for tool in data.get("tools") or []: if not isinstance(tool, dict): continue - if tool.get("type") == "function" and tool.get("name"): + if tool.get("type") in ("function", "custom") and tool.get("name"): names.append(str(tool["name"])) elif tool.get("type") == "mcp" and tool.get("server_label"): names.append(str(tool["server_label"])) @@ -586,7 +588,14 @@ class OpenAIResponsesHandler(BaseTranslation): """ Check if the streaming has ended. """ - return all(response.choices[0].finish_reason is not None for response in responses_so_far) + if not responses_so_far: + return False + terminal_types = { + ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value, + ResponsesAPIStreamEvents.RESPONSE_FAILED.value, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value, + } + return responses_so_far[-1].get("type") in terminal_types def get_streaming_string_so_far(self, responses_so_far: List[Any]) -> str: """ diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index d9d0f9bc236..4bf8272a334 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -155,7 +155,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): def headers(self) -> Dict[str, str]: if self.token_creator is None: self.run_env_setup() - access_token = self.token_creator() # type: ignore + access_token = self.token_creator() # pyright: ignore[reportOptionalCall] # run_env_setup set it or raised return { "Authorization": access_token, "AI-Resource-Group": self.resource_group, diff --git a/litellm/llms/tencent/__init__.py b/litellm/llms/tencent/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/tencent/chat/__init__.py b/litellm/llms/tencent/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/tencent/chat/transformation.py b/litellm/llms/tencent/chat/transformation.py new file mode 100644 index 00000000000..4dea0c4b8c7 --- /dev/null +++ b/litellm/llms/tencent/chat/transformation.py @@ -0,0 +1,68 @@ +""" +Translates from OpenAI's `/v1/chat/completions` to Tencent TokenHub's +OpenAI-compatible endpoint. +""" + +from typing import Optional + +from litellm.secret_managers.main import get_secret_str +from litellm.utils import supports_reasoning + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + + +class TencentChatConfig(OpenAIGPTConfig): + def get_supported_openai_params(self, model: str) -> list: + params = super().get_supported_openai_params(model) + if supports_reasoning(model, custom_llm_provider="tencent"): + params.extend(["thinking", "reasoning_effort"]) + return params + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params) + + thinking_value = optional_params.pop("thinking", None) + reasoning_effort = optional_params.pop("reasoning_effort", None) + + if thinking_value is not None: + if isinstance(thinking_value, dict): + optional_params["thinking"] = thinking_value + elif reasoning_effort is not None and reasoning_effort != "none": + optional_params["thinking"] = {"type": "enabled"} + + return optional_params + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> tuple[Optional[str], Optional[str]]: + api_base = api_base or get_secret_str("TENCENT_API_BASE") or "https://tokenhub-intl.tencentcloudmaas.com/v1" + dynamic_api_key = api_key or get_secret_str("TENCENT_API_KEY") + return api_base, dynamic_api_key + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + if not api_base: + api_base = "https://tokenhub-intl.tencentcloudmaas.com/v1" + + api_base = api_base.rstrip("/") + + if api_base.endswith("/chat/completions"): + return api_base + + if not api_base.endswith("/v1"): + api_base = f"{api_base}/v1" + + return f"{api_base}/chat/completions" diff --git a/litellm/llms/tencent/cost_calculator.py b/litellm/llms/tencent/cost_calculator.py new file mode 100644 index 00000000000..d9aebdc3284 --- /dev/null +++ b/litellm/llms/tencent/cost_calculator.py @@ -0,0 +1,6 @@ +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.types.utils import Usage + + +def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: + return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent") diff --git a/litellm/llms/tencent/messages/__init__.py b/litellm/llms/tencent/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/tencent/messages/transformation.py b/litellm/llms/tencent/messages/transformation.py new file mode 100644 index 00000000000..e0f13aa9ca4 --- /dev/null +++ b/litellm/llms/tencent/messages/transformation.py @@ -0,0 +1,85 @@ +""" +Tencent Anthropic-compatible messages transformation config. + +Tencent TokenHub exposes an Anthropic-compatible Messages API endpoint +alongside its standard OpenAI-compatible chat completions endpoint. +""" + +from typing import Any, Optional + +import litellm +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) +from litellm.secret_managers.main import get_secret_str + + +class TencentAnthropicMessagesConfig(AnthropicMessagesConfig): + """ + Tencent TokenHub exposes an Anthropic-compatible Messages API. + + Unlike the chat completions endpoint (which uses /v1), the Anthropic + endpoint may use a different base URL. Configure via + TENCENT_ANTHROPIC_API_BASE or TENCENT_API_BASE. + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "tencent" + + def should_strip_billing_metadata(self) -> bool: + return True + + @staticmethod + def get_api_key(api_key: Optional[str] = None) -> Optional[str]: + return api_key or get_secret_str("TENCENT_API_KEY") or litellm.api_key + + @staticmethod + def get_api_base(api_base: Optional[str] = None) -> str: + return ( + api_base + or get_secret_str("TENCENT_ANTHROPIC_API_BASE") + or get_secret_str("TENCENT_API_BASE") + or "https://tokenhub-intl.tencentcloudmaas.com" + ) + + def validate_anthropic_messages_environment( + self, + headers: dict, + model: str, + messages: list[Any], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> tuple[dict, Optional[str]]: + return super().validate_anthropic_messages_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=self.get_api_key(api_key=api_key), + api_base=api_base, + ) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + base_url = self.get_api_base(api_base=api_base).rstrip("/") + + if base_url.endswith("/v1/messages"): + return base_url + + if base_url.endswith("/v1/chat/completions"): + base_url = base_url[: -len("/v1/chat/completions")] + elif base_url.endswith("/v1"): + base_url = base_url[: -len("/v1")] + + return f"{base_url}/v1/messages" diff --git a/litellm/llms/tinyfish/search/transformation.py b/litellm/llms/tinyfish/search/transformation.py index 4b7d38e3661..cef5f9cd02e 100644 --- a/litellm/llms/tinyfish/search/transformation.py +++ b/litellm/llms/tinyfish/search/transformation.py @@ -6,53 +6,42 @@ Docs: https://docs.tinyfish.ai/search-api from __future__ import annotations -from typing import Literal, TypedDict +import json +from typing import Literal from urllib.parse import urlencode import httpx -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import TypeAdapter, ValidationError +from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, - SearchResult, ) from litellm.secret_managers.main import get_secret_str - -class _TinyfishSearchRequestRequired(TypedDict): - query: str - - -class TinyfishSearchRequest(_TinyfishSearchRequestRequired, total=False): - location: str - language: str - page: int - include_thumbnail: bool - max_results: int - - -class _TinyfishResultItem(BaseModel, frozen=True): - title: str = "" - url: str = "" - snippet: str = "" - - -class _TinyfishApiResponse(BaseModel, frozen=True): - results: tuple[_TinyfishResultItem, ...] = () - - _UrlEncodableParams = TypeAdapter(dict[str, str | int | bool]) _StrList = TypeAdapter(list[str]) _StrFrozenSet = TypeAdapter(frozenset[str]) _TINYFISH_PARAMS_KEY = "_tinyfish_params" +_TINYFISH_DOCS_URL = "https://docs.tinyfish.ai/search-api" +_TINYFISH_RESULT_CAP = 10 # TinyFish's natural per-page SERP ceiling class TinyfishSearchConfig(BaseSearchConfig): TINYFISH_API_BASE = "https://api.search.tinyfish.ai" + def __init__(self) -> None: + super().__init__() + # Threaded from transform_search_request → transform_search_response so the + # response slice honors the caller's max_results without re-sending it on + # the wire (TinyFish doesn't honor it server-side). Safe because the + # config is instantiated per-call via ProviderConfigManager. + self._caller_max_results: int | None = None + @staticmethod def ui_friendly_name() -> str: return "TinyFish" @@ -97,36 +86,77 @@ class TinyfishSearchConfig(BaseSearchConfig): optional_params: dict[str, object], **kwargs: object, ) -> dict[str, object]: + """ + Transform a LiteLLM search request to TinyFish's querystring format. + + Maps LiteLLM's unified-spec params (see + ``BaseSearchConfig.get_supported_perplexity_optional_params``) to + TinyFish equivalents: + - ``query`` (str or list[str]) → ``query`` (list joined by spaces) + - ``country`` → ``location`` + - ``search_domain_filter`` (list[str]) → folded into the query as + ``() (site:a OR site:b ...)`` (TinyFish has no first-class + field today; see ML-2084 for the planned ``include_domains``) + - ``max_results`` → not sent on the wire; stashed on + ``self._caller_max_results`` for client-side response truncation + (TinyFish doesn't honor it server-side) + - ``max_tokens_per_page`` → silently dropped (no TinyFish equivalent) + + Any other ``optional_params`` keys are forwarded to TinyFish as-is. + dict/list values are JSON-encoded so they survive ``urlencode``. + + Returns: + ``{_TINYFISH_PARAMS_KEY: }``. + ``get_complete_url`` reads this back to build the final URL. + """ resolved_query = " ".join(query) if isinstance(query, list) else query - request_data: TinyfishSearchRequest = {"query": resolved_query} - - country = optional_params.get("country") - if isinstance(country, str): - request_data["location"] = country - - raw_max = optional_params.get("max_results") - if isinstance(raw_max, (int, float, str)): - request_data["max_results"] = max(1, min(int(raw_max), 20)) - try: domains = _StrList.validate_python(optional_params.get("search_domain_filter")) except (ValidationError, TypeError): domains = [] if domains: - request_data["query"] = _append_domain_filters(request_data["query"], domains) + resolved_query = _append_domain_filters(resolved_query, domains) - result_data: dict[str, object] = dict(request_data) + request_data: dict[str, object] = {"query": resolved_query} + + country = optional_params.get("country") + if isinstance(country, str): + request_data["location"] = country + + # max_results is enforced client-side on the response (TinyFish ignores + # the param and always returns ~10). Clamp to [1, 10] and stash on self + # so transform_search_response can slice without re-reading the URL. + raw_max = optional_params.get("max_results") + if isinstance(raw_max, (int, float, str)): + try: + self._caller_max_results = max(1, min(int(raw_max), _TINYFISH_RESULT_CAP)) + except (ValueError, TypeError, OverflowError): + # OverflowError covers int(float('inf')) and similar non-finite floats. + verbose_logger.warning( + "TinyFish Search: max_results=%r is not a valid integer; ignoring.", + raw_max, + ) raw_supported: object = ( self.get_supported_perplexity_optional_params() # any-ok: base class returns bare set ) supported_perplexity = _StrFrozenSet.validate_python(raw_supported) for param, value in optional_params.items(): - if param not in supported_perplexity and param not in result_data: - result_data[param] = value + if param not in supported_perplexity and param not in request_data: + # `fetch` expects a JSON-encoded object on the wire; accept the + # natural Python dict form and serialize here so callers don't + # have to pre-stringify. + if isinstance(value, dict): + value = json.dumps(value, separators=(",", ":")) + # `urlencode` would render Python bool as "True"/"False" + # (capitalized). ux-labs validators require lowercase + # "true"/"false" (e.g. `include_thumbnail`); normalize here. + elif isinstance(value, bool): + value = "true" if value else "false" + request_data[param] = value - return {_TINYFISH_PARAMS_KEY: result_data} + return {_TINYFISH_PARAMS_KEY: request_data} def transform_search_response( self, @@ -134,24 +164,158 @@ class TinyfishSearchConfig(BaseSearchConfig): logging_obj: LiteLLMLoggingObj | None, **kwargs: object, ) -> SearchResponse: - raw_json: object = raw_response.json() # any-ok: httpx Response.json() -> Any - parsed = _TinyfishApiResponse.model_validate(raw_json) + """ + Transform a TinyFish response to LiteLLM's unified ``SearchResponse``. - max_results_str: str = "20" - if raw_response.request: - raw_param: object = raw_response.request.url.params.get( # any-ok: httpx QueryParams.get() -> Any - "max_results", "20" + Mappings (per-result): + - ``title`` → ``SearchResult.title`` (defaults to ``""`` if missing/null) + - ``url`` → ``SearchResult.url`` (defaults to ``""``) + - ``snippet`` → ``SearchResult.snippet`` (defaults to ``""``) + - all other per-result fields (``position``, ``site_name``, + ``thumbnail_url``, ``fetch``, ``fetch_error``, ...) ride through as + extras on ``SearchResult`` via its ``extra="allow"`` config. + + Top-level ``parameter_warnings`` (see ML-2085) is read when present and + each entry is re-fired via ``verbose_logger.warning``. Absent or + malformed entries are silently skipped — never throws. + + Error paths routed through ``self._wrap_error`` for uniform + ``"TinyFish Search: . See for details."`` wrapping: + - non-2xx HTTP status (caught here because ``AsyncHTTPHandler.get`` + does not call ``raise_for_status``) + - 200 with non-JSON body + - 200 with valid JSON whose shape doesn't satisfy ``SearchResponse`` + + Returns: + ``SearchResponse`` truncated to ``self._caller_max_results`` (or + ``_TINYFISH_RESULT_CAP`` when the caller didn't set ``max_results``). + """ + # AsyncHTTPHandler.get does not call raise_for_status, so non-2xx + # responses arrive here looking successful. Dispatch through + # get_error_class so callers see a uniform attributed error. + if not (200 <= raw_response.status_code < 300): + raise self._wrap_error( + error_message=raw_response.text, + status_code=raw_response.status_code, + headers=dict(raw_response.headers), ) - max_results_str = str(raw_param) - max_results: int = min(int(max_results_str), 20) - results = [ - SearchResult(title=item.title, url=item.url, snippet=item.snippet) for item in parsed.results[:max_results] - ] + try: + raw_json: object = raw_response.json() # any-ok: httpx Response.json() -> Any + except json.JSONDecodeError: + raise self._wrap_error( + error_message=f"Expected JSON response, got: {raw_response.text[:200]}", + status_code=raw_response.status_code, + headers=dict(raw_response.headers), + ) - return SearchResponse(results=results, object="search") + _default_missing_result_fields(raw_json) + + try: + parsed = SearchResponse.model_validate(raw_json) + except ValidationError as e: + raise self._wrap_error( + error_message=(f"Response shape does not match LiteLLM's SearchResponse schema: {e}"), + status_code=raw_response.status_code, + headers=dict(raw_response.headers), + ) + + _emit_parameter_warnings(parsed) + + max_results = self._caller_max_results or _TINYFISH_RESULT_CAP + return SearchResponse(results=list(parsed.results[:max_results])) + + def _wrap_error( + self, + error_message: str, + status_code: int, + headers: dict[str, str], + ) -> Exception: + """ + Build an attributed ``BaseLLMException`` from a TinyFish error body. + + Used only at the call sites we control inside + ``transform_search_response`` (non-2xx, JSONDecodeError, ValidationError). + Not an override of ``BaseSearchConfig.get_error_class``: that path is + left to inherit from the base so it auto-picks-up any future LiteLLM + improvements. Trade-off: network failures (routed through LiteLLM + core's ``_handle_error`` → ``BaseSearchConfig.get_error_class``) won't + carry the ``TinyFish Search:`` prefix — the bare error already names + the host in the URL, so attribution is implicit there. + """ + # ux-labs frontend wraps every error body as {"error": {"code", "message", "details"?}}. + # Best-effort unwrap to surface the inner message; fall back to the raw body + # for non-ux-labs responses (CDN HTML pages, other JSON envelopes, plain text). + inner_message = error_message + try: + body: object = json.loads(error_message) # any-ok: json.loads -> Any + if isinstance(body, dict): + error_obj: object = body.get("error") # any-ok: untyped dict + if isinstance(error_obj, dict): + candidate: object = error_obj.get("message") # any-ok: untyped dict + if isinstance(candidate, str) and candidate: + inner_message = candidate + except (json.JSONDecodeError, TypeError): + pass + + return BaseLLMException( + status_code=status_code, + message=f"TinyFish Search: {inner_message}. See {_TINYFISH_DOCS_URL} for details.", + headers=headers, + ) def _append_domain_filters(query: str, domains: list[str]) -> str: domain_clauses = " OR ".join(f"site:{d}" for d in domains) return f"({query}) ({domain_clauses})" + + +def _default_missing_result_fields(raw_json: object) -> None: + """Default missing/null title/url/snippet to "" on each result item in place. + + SearchResult requires these three fields; a degraded TinyFish result flows + through with empty strings instead of failing the whole call. + """ + if not isinstance(raw_json, dict): + return + results_in = raw_json.get("results") + if not isinstance(results_in, list): + return + for item in results_in: + if not isinstance(item, dict): + continue + for field in ("title", "url", "snippet"): + if not isinstance(item.get(field), str): + item[field] = "" + + +def _emit_parameter_warnings(parsed: SearchResponse) -> None: + """Re-fire TinyFish-side ``parameter_warnings`` (see ML-2085) as warnings. + + Defensive: skip silently on any shape we don't recognize so a malformed + entry (or an early/partial rollout of the field) never throws. + Schema per entry: ``{type, parameter, message, docs_url?}``. + """ + warnings_field: object = ( + getattr(parsed, "parameter_warnings", None) # any-ok: extras=allow field + ) + if not isinstance(warnings_field, list): + return + for entry in warnings_field: + if not isinstance(entry, dict): + continue + warning_type: object = entry.get("type") # any-ok: untyped dict + parameter: object = entry.get("parameter") # any-ok: untyped dict + message: object = entry.get("message") # any-ok: untyped dict + if not isinstance(warning_type, str) or not warning_type: + continue + if not isinstance(parameter, str) or not parameter: + continue + if not isinstance(message, str) or not message: + continue + verbose_logger.warning( + "TinyFish Search parameter_warning (%s) `%s`: %s", + warning_type, + parameter, + message, + ) diff --git a/litellm/llms/vertex_ai/audio_transcription/transformation.py b/litellm/llms/vertex_ai/audio_transcription/transformation.py new file mode 100644 index 00000000000..03769bf2601 --- /dev/null +++ b/litellm/llms/vertex_ai/audio_transcription/transformation.py @@ -0,0 +1,194 @@ +import base64 + +from httpx import Headers, Response + +import litellm +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.audio_utils.utils import ( + normalize_transcription_language_to_bcp47, + process_audio_file, +) +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, + BaseAudioTranscriptionConfig, +) +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.vertex_ai.common_utils import VertexAIError, validate_vertex_location +from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIAudioTranscriptionOptionalParams, +) +from litellm.types.llms.vertex_ai_speech_to_text import ( + VertexSpeechToTextAutoDecodingConfig, + VertexSpeechToTextRecognitionConfig, + VertexSpeechToTextRecognitionFeatures, + VertexSpeechToTextRecognizeRequest, + VertexSpeechToTextRecognizeResponse, +) +from litellm.types.utils import FileTypes, TranscriptionResponse + +DEFAULT_SPEECH_TO_TEXT_LOCATION = "us" +AUTO_LANGUAGE_CODE = "auto" +SUPPORTED_RESPONSE_FORMATS = ("json", "text") +_URL_UNSAFE_PROJECT_CHARS = ("/", "?", "#", "\\", ":", " ", "\t", "\n", "\r") + + +class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase): + def __init__(self) -> None: + BaseAudioTranscriptionConfig.__init__(self) + VertexBase.__init__(self) + + def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]: + return ["language", "response_format"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + mapped = { + **optional_params, + **{k: v for k, v in non_default_params.items() if k in supported_params}, + } + response_format = mapped.get("response_format") + if response_format is None or response_format in SUPPORTED_RESPONSE_FORMATS: + return mapped + if drop_params or litellm.drop_params: + return {k: v for k, v in mapped.items() if k != "response_format"} + raise UnsupportedParamsError( + status_code=400, + message=( + f"Google Speech-to-Text does not support response_format={response_format!r}. " + f"Supported values: {', '.join(SUPPORTED_RESPONSE_FORMATS)}. " + "To drop unsupported openai params from the call, set `litellm.drop_params = True`" + ), + ) + + def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> BaseLLMException: + return VertexAIError(status_code=status_code, message=error_message, headers=headers) + + def validate_environment( + self, + headers: dict, + model: str, + messages: list[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: str | None = None, + api_base: str | None = None, + ) -> dict: + access_token, project_id = self._ensure_access_token( + credentials=self.safe_get_vertex_ai_credentials(litellm_params), + project_id=self.safe_get_vertex_ai_project(litellm_params), + custom_llm_provider="vertex_ai", + ) + return { + **headers, + "Authorization": f"Bearer {access_token}", + "x-goog-user-project": project_id, + "Content-Type": "application/json", + } + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict, + litellm_params: dict, + stream: bool | None = None, + ) -> str: + location = self._validate_location(self.safe_get_vertex_ai_location(litellm_params)) + project_id = self._validate_project_id( + self.safe_get_vertex_ai_project(litellm_params) or self._resolve_project_id_from_credentials(litellm_params) + ) + host = "speech.googleapis.com" if location == "global" else f"{location}-speech.googleapis.com" + base_url = (api_base or f"https://{host}").rstrip("/") + return f"{base_url}/v2/projects/{project_id}/locations/{location}/recognizers/_:recognize" + + @staticmethod + def _validate_location(location: str | None) -> str: + try: + return validate_vertex_location(location or DEFAULT_SPEECH_TO_TEXT_LOCATION) + except ValueError as e: + raise VertexAIError(status_code=400, message=str(e)) from e + + @staticmethod + def _validate_project_id(project_id: str) -> str: + if not project_id or ".." in project_id or any(c in project_id for c in _URL_UNSAFE_PROJECT_CHARS): + raise VertexAIError(status_code=400, message=f"Invalid vertex_project format: {project_id!r}") + return project_id + + def _resolve_project_id_from_credentials(self, litellm_params: dict) -> str: + _, project_id = self._ensure_access_token( + credentials=self.safe_get_vertex_ai_credentials(litellm_params), + project_id=None, + custom_llm_provider="vertex_ai", + ) + return project_id + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + ) -> AudioTranscriptionRequestData: + processed_audio = process_audio_file(audio_file) + language = optional_params.get("language") + language_codes = ( + [normalize_transcription_language_to_bcp47(language)] + if isinstance(language, str) and language + else [AUTO_LANGUAGE_CODE] + ) + request_body = VertexSpeechToTextRecognizeRequest( + config=VertexSpeechToTextRecognitionConfig( + model=model.removeprefix("vertex_ai/"), + languageCodes=language_codes, + features=VertexSpeechToTextRecognitionFeatures(enableAutomaticPunctuation=True), + autoDecodingConfig=VertexSpeechToTextAutoDecodingConfig(), + ), + content=base64.b64encode(processed_audio.file_content).decode("utf-8"), + ) + return AudioTranscriptionRequestData(data=dict(request_body)) + + def transform_audio_transcription_response( + self, + raw_response: Response, + ) -> TranscriptionResponse: + try: + response_json = raw_response.json() + except ValueError: + raise VertexAIError( + status_code=raw_response.status_code, + message=f"Received non-JSON response from Google Speech-to-Text: {raw_response.text}", + ) + parsed = VertexSpeechToTextRecognizeResponse.model_validate(response_json) + transcripts = tuple( + result.alternatives[0].transcript + for result in parsed.results + if result.alternatives and result.alternatives[0].transcript + ) + response = TranscriptionResponse(text=" ".join(transcripts)) + response["task"] = "transcribe" + detected_language = next((result.languageCode for result in parsed.results if result.languageCode), None) + if detected_language is not None: + response["language"] = detected_language + billed_duration = _parse_duration_seconds(parsed.metadata.totalBilledDuration if parsed.metadata else None) + if billed_duration is not None: + response["duration"] = billed_duration + response._hidden_params = response_json + return response + + +def _parse_duration_seconds(duration: str | None) -> float | None: + if duration is None or not duration.endswith("s"): + return None + try: + return float(duration[:-1]) + except ValueError: + return None diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 36522dfe396..7dcb4dcf2e8 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -311,6 +311,28 @@ def get_vertex_base_model_name(model: str) -> str: return model +def validate_vertex_location(vertex_location: Optional[str]) -> str: + """ + Validate a Vertex AI location before interpolating it into a request host or + URL path. + + ``vertex_location`` is client-controllable on the proxy (it flows in from the + request body), so it must never be trusted verbatim in a URL or an attacker + could point the host at their own server and exfiltrate the admin's Google + access token. Allow the special ``global`` control plane and otherwise require + a lowercase alphanumeric-plus-hyphen token (e.g. ``us``, ``us-central1``, + ``eu``), which rejects host injection like ``attacker.example/`` or + ``evil.com#``. + """ + if vertex_location == "global": + return vertex_location + if vertex_location is None: + raise ValueError("vertex_location is required") + if not re.match(r"^[a-z][a-z0-9-]*$", vertex_location): + raise ValueError("Invalid vertex_location format") + return vertex_location + + def get_vertex_base_url( vertex_location: Optional[str], ) -> str: @@ -321,15 +343,12 @@ def get_vertex_base_url( - Multi-region geographies (e.g. ``us``, ``eu``) use ``aiplatform.{geo}.rep.googleapis.com``. - Regional locations (e.g. ``us-central1``) use ``{region}-aiplatform.googleapis.com``. """ - if vertex_location == "global": + validated_location = validate_vertex_location(vertex_location) + if validated_location == "global": return "https://aiplatform.googleapis.com" - if vertex_location is None: - raise ValueError("vertex_location is required") - if not re.match(r"^[a-z][a-z0-9-]*$", vertex_location): - raise ValueError("Invalid vertex_location format") - if "-" not in vertex_location: - return f"https://aiplatform.{vertex_location}.rep.googleapis.com" - return f"https://{vertex_location}-aiplatform.googleapis.com" + if "-" not in validated_location: + return f"https://aiplatform.{validated_location}.rep.googleapis.com" + return f"https://{validated_location}-aiplatform.googleapis.com" def _get_embedding_url( diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 678877c0721..f9ed8cea9b5 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2315,17 +2315,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # Store thoughtSignatures in provider_specific_fields if thought_signatures is not None: - if "provider_specific_fields" not in chat_completion_message: - chat_completion_message["provider_specific_fields"] = {} - chat_completion_message["provider_specific_fields"]["thought_signatures"] = thought_signatures # type: ignore + thought_signature_fields = chat_completion_message.get("provider_specific_fields") or {} + thought_signature_fields["thought_signatures"] = thought_signatures + chat_completion_message["provider_specific_fields"] = thought_signature_fields # Store server-side tool invocations in provider_specific_fields if server_side_tool_invocations is not None: - if "provider_specific_fields" not in chat_completion_message: - chat_completion_message["provider_specific_fields"] = {} - chat_completion_message["provider_specific_fields"]["server_side_tool_invocations"] = ( - server_side_tool_invocations # type: ignore - ) + tool_invocation_fields = chat_completion_message.get("provider_specific_fields") or {} + tool_invocation_fields["server_side_tool_invocations"] = server_side_tool_invocations + chat_completion_message["provider_specific_fields"] = tool_invocation_fields if isinstance(model_response, ModelResponseStream): choice = VertexGeminiConfig._create_streaming_choice( diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index 4bca3e8f71d..572725ac789 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -1,5 +1,5 @@ import os -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Optional from litellm._logging import verbose_logger @@ -175,7 +175,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): self, headers: dict, model: str, - messages: List[AllMessageValues], + messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, api_key: Optional[str] = None, @@ -217,10 +217,10 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): contents = [{"role": "user", "parts": [{"text": prompt}]}] # Prepare generation config - generation_config: Dict[str, Any] = {"responseModalities": ["IMAGE"]} + generation_config: dict[str, Any] = {"responseModalities": ["IMAGE"]} # Seed from user-supplied imageConfig dict; flat params are overlaid for backward compat. - image_config: Dict[str, Any] = dict(optional_params.get("imageConfig") or {}) + image_config: dict[str, Any] = dict(optional_params.get("imageConfig") or {}) if "aspectRatio" in optional_params: image_config["aspectRatio"] = optional_params["aspectRatio"] @@ -241,7 +241,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): elif "n" in optional_params: generation_config["candidateCount"] = optional_params["n"] - request_body: Dict[str, Any] = { + request_body: dict[str, Any] = { "contents": contents, "generationConfig": generation_config, } diff --git a/litellm/main.py b/litellm/main.py index c07d88342b6..095b5f8d825 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -582,6 +582,7 @@ async def acompletion( "api_key": api_key, "model_list": model_list, "reasoning_effort": reasoning_effort, + "verbosity": verbosity, "safety_identifier": safety_identifier, "service_tier": service_tier, "extra_headers": extra_headers, @@ -1081,6 +1082,54 @@ def _build_custom_pricing_entry( return entry +def _get_router_deployment_id(kwargs: dict) -> Optional[str]: + for metadata_key in ("litellm_metadata", "metadata"): + metadata = kwargs.get(metadata_key) or {} + if not isinstance(metadata, dict): + continue + deployment_model_info = metadata.get("model_info") or {} + if not isinstance(deployment_model_info, dict): + continue + deployment_id = deployment_model_info.get("id") + if deployment_id is not None: + return str(deployment_id) + return None + + +def _register_custom_pricing_for_request( + model: str, + custom_llm_provider: str, + kwargs: dict, + model_info: Optional[dict], +) -> None: + """Register per-request custom pricing in litellm.model_cost. + + Router-originated requests (identified by the deployment id the router puts + in metadata) get their full pricing registered under that unique id only; + the shared ``{provider}/{model}`` key receives the entry with pricing fields + stripped, mirroring Router._create_deployment. This keeps one deployment's + pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in + pricing used by sibling deployments of the same backend model. Direct SDK + calls keep the legacy behavior of registering the shared key with pricing. + """ + entry = _build_custom_pricing_entry( + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=model_info, + ) + shared_key = f"{custom_llm_provider}/{model}" + deployment_id = _get_router_deployment_id(kwargs) + if deployment_id is None: + litellm.register_model({shared_key: entry}) + return + litellm.register_model( + { + deployment_id: entry, + shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry), + } + ) + + def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: _azure_detection_model = ctx._azure_detection_model acompletion = ctx.acompletion @@ -5107,14 +5156,11 @@ def completion( # type: ignore if ( input_cost_per_token is not None and output_cost_per_token is not None ) or input_cost_per_second is not None: - litellm.register_model( - { - f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - } + _register_custom_pricing_for_request( + model=model, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=model_info, ) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} # type: ignore @@ -5193,6 +5239,7 @@ def completion( # type: ignore "parallel_tool_calls": parallel_tool_calls, "messages": messages, "reasoning_effort": reasoning_effort, + "verbosity": verbosity, "thinking": thinking, "web_search_options": web_search_options, "include_server_side_tool_invocations": ( @@ -5960,14 +6007,11 @@ def embedding( ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### if (input_cost_per_token is not None and output_cost_per_token is not None) or input_cost_per_second is not None: - litellm.register_model( - { - f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=kwargs.get("model_info"), - ) - } + _register_custom_pricing_for_request( + model=model, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=kwargs.get("model_info"), ) litellm_params_dict = get_litellm_params(**kwargs) @@ -8307,26 +8351,6 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Optional[List] finish_reason = chunks[-1]["choices"][0]["finish_reason"] logprobs = chunks[-1]["choices"][0]["logprobs"] - response = { - "id": id, - "object": object, - "created": created, - "model": model, - "system_fingerprint": system_fingerprint, - "choices": [ - { - "text": None, - "index": 0, - "logprobs": logprobs, - "finish_reason": finish_reason, - } - ], - "usage": { - "prompt_tokens": None, - "completion_tokens": None, - "total_tokens": None, - }, - } content_list = [] for chunk in chunks: choices = chunk["choices"] @@ -8338,25 +8362,37 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Optional[List] # Combine the "content" strings into a single string || combine the 'function' strings into a single string combined_content = "".join(content_list) - # Update the "content" field within the response dictionary - response["choices"][0]["text"] = combined_content - - if len(combined_content) > 0: - pass - else: - pass - # # Update usage information if needed try: - response["usage"]["prompt_tokens"] = token_counter(model=model, messages=messages) + prompt_tokens = token_counter(model=model, messages=messages) except Exception: # don't allow this failing to block a complete streaming response from being returned print_verbose("token_counter failed, assuming prompt tokens is 0") - response["usage"]["prompt_tokens"] = 0 - response["usage"]["completion_tokens"] = token_counter( + prompt_tokens = 0 + completion_tokens = token_counter( model=model, text=combined_content, count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need to add extra tokens we do for input messages ) - response["usage"]["total_tokens"] = response["usage"]["prompt_tokens"] + response["usage"]["completion_tokens"] + + response = { + "id": id, + "object": object, + "created": created, + "model": model, + "system_fingerprint": system_fingerprint, + "choices": [ + { + "text": combined_content, + "index": 0, + "logprobs": logprobs, + "finish_reason": finish_reason, + } + ], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + }, + } return TextCompletionResponse(**response) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index bf63ef73c22..cbca0744ed9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -744,7 +744,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "anthropic.claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -768,7 +769,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_streaming": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -787,8 +789,6 @@ "output_cost_per_token_above_200k_tokens": 3e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "cache_creation_input_token_cost_above_1hr": 7.5e-06, - "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.5e-05, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07 }, @@ -813,9 +813,7 @@ "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 3e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "cache_creation_input_token_cost_above_1hr": 7.5e-06, - "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.5e-05 + "cache_read_input_token_cost_above_200k_tokens": 6e-07 }, "anthropic.claude-3-7-sonnet-20240620-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -991,7 +989,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "high" + "bedrock_output_config_effort_ceiling": "high", + "supports_parallel_tool_use_config": true }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1010,7 +1009,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1023,7 +1021,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "bedrock_output_config_effort_ceiling": "max" + "bedrock_output_config_effort_ceiling": "max", + "supports_parallel_tool_use_config": true }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1042,7 +1041,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1055,7 +1053,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "bedrock_output_config_effort_ceiling": "max" + "bedrock_output_config_effort_ceiling": "max", + "supports_parallel_tool_use_config": true }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1074,7 +1073,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1087,7 +1085,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "bedrock_output_config_effort_ceiling": "max" + "bedrock_output_config_effort_ceiling": "max", + "supports_parallel_tool_use_config": true }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1106,7 +1105,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1119,7 +1117,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "bedrock_output_config_effort_ceiling": "max" + "bedrock_output_config_effort_ceiling": "max", + "supports_parallel_tool_use_config": true }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1138,7 +1137,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1151,7 +1149,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "bedrock_output_config_effort_ceiling": "max" + "bedrock_output_config_effort_ceiling": "max", + "supports_parallel_tool_use_config": true }, "anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1171,7 +1170,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1186,7 +1184,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1221,7 +1220,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1236,7 +1234,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1256,7 +1255,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1271,7 +1269,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1291,7 +1290,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1306,7 +1304,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "au.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1326,7 +1325,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1341,7 +1339,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1374,7 +1373,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1407,7 +1407,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1440,7 +1441,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1473,7 +1475,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1493,7 +1496,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1508,7 +1510,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1528,7 +1531,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1543,7 +1545,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1563,7 +1566,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1578,7 +1580,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1598,7 +1601,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1613,7 +1615,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1633,7 +1636,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1648,7 +1650,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "jp.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1680,7 +1683,8 @@ "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "supports_parallel_tool_use_config": true }, "anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -1713,7 +1717,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "global.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -1746,7 +1751,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "us.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1779,7 +1785,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "eu.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1812,7 +1819,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "au.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1845,7 +1853,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "jp.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1878,7 +1887,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh" + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -1897,7 +1907,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1909,7 +1918,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_output_config": true + "supports_output_config": true, + "supports_parallel_tool_use_config": true }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -1928,7 +1938,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1940,7 +1949,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_output_config": true + "supports_output_config": true, + "supports_parallel_tool_use_config": true }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -1959,7 +1969,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1971,7 +1980,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_output_config": true + "supports_output_config": true, + "supports_parallel_tool_use_config": true }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -1990,7 +2000,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -2002,7 +2011,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_output_config": true + "supports_output_config": true, + "supports_parallel_tool_use_config": true }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2021,7 +2031,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -2033,7 +2042,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_output_config": true + "supports_output_config": true, + "supports_parallel_tool_use_config": true }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2052,7 +2062,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -2064,7 +2073,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_output_config": true + "supports_output_config": true, + "supports_parallel_tool_use_config": true }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2126,7 +2136,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "anthropic.claude-v1": { "input_cost_per_token": 8e-06, @@ -2376,7 +2387,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "apac.anthropic.claude-3-sonnet-20240229-v1:0": { "input_cost_per_token": 3e-06, @@ -2466,7 +2478,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "azure/ada": { "input_cost_per_token": 1e-07, @@ -2575,7 +2588,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2605,7 +2617,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2666,7 +2677,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2764,7 +2774,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -5752,6 +5761,76 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/gpt-5.4": { + "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost_priority": 5.5e-07, + "input_cost_per_token": 2.75e-06, + "input_cost_per_token_priority": 5.5e-06, + "output_cost_per_token": 1.65e-05, + "output_cost_per_token_priority": 3.3e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5.4": { + "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost_priority": 5.5e-07, + "input_cost_per_token": 2.75e-06, + "input_cost_per_token_priority": 5.5e-06, + "output_cost_per_token": 1.65e-05, + "output_cost_per_token_priority": 3.3e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, @@ -5793,6 +5872,76 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/gpt-5.4-2026-03-05": { + "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost_priority": 5.5e-07, + "input_cost_per_token": 2.75e-06, + "input_cost_per_token_priority": 5.5e-06, + "output_cost_per_token": 1.65e-05, + "output_cost_per_token_priority": 3.3e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5.4-2026-03-05": { + "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost_priority": 5.5e-07, + "input_cost_per_token": 2.75e-06, + "input_cost_per_token_priority": 5.5e-06, + "output_cost_per_token": 1.65e-05, + "output_cost_per_token_priority": 3.3e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, @@ -5908,6 +6057,90 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure/us/gpt-5.5": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_priority": 1.38e-06, + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_priority": 1.375e-05, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, + "output_cost_per_token_priority": 8.25e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false + }, + "azure/eu/gpt-5.5": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_priority": 1.38e-06, + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_priority": 1.375e-05, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, + "output_cost_per_token_priority": 8.25e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false + }, "azure/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, @@ -5950,6 +6183,84 @@ "supports_vision": true, "supports_web_search": true }, + "azure/us/gpt-5.5-2026-04-23": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_priority": 1.38e-06, + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_priority": 1.375e-05, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, + "output_cost_per_token_priority": 8.25e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "azure/eu/gpt-5.5-2026-04-23": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_priority": 1.38e-06, + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_priority": 1.375e-05, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, + "output_cost_per_token_priority": 8.25e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, @@ -9373,17 +9684,16 @@ }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "output_cost_per_token": 1.2e-06 + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -9612,7 +9922,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -9634,7 +9945,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -9787,7 +10099,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -9809,7 +10122,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -9995,17 +10309,16 @@ }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "output_cost_per_token": 1.2e-06 + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -10573,7 +10886,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_parallel_tool_use_config": true }, "claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -14645,7 +14959,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -14859,7 +15174,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "eu.meta.llama3-2-1b-instruct-v1:0": { "input_cost_per_token": 1.3e-07, @@ -19222,7 +19538,6 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_adaptive_thinking": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true @@ -20059,7 +20374,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -20112,7 +20428,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "global.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -21976,8 +22293,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22025,8 +22342,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22070,8 +22387,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22115,8 +22432,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22164,8 +22481,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22212,8 +22529,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22253,8 +22570,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22297,8 +22614,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22342,8 +22659,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22388,8 +22705,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22431,8 +22748,8 @@ "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22474,8 +22791,8 @@ "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -24181,7 +24498,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -24204,7 +24522,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "crusoe/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, @@ -24992,14 +25311,13 @@ }, "minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.2e-06, "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -28455,7 +28773,6 @@ "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_200k_tokens": 2.25e-05, "source": "https://openrouter.ai/anthropic/claude-sonnet-4.6", - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -28495,7 +28812,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -28557,7 +28873,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -30480,7 +30795,6 @@ "supports_adaptive_thinking": true, "litellm_provider": "perplexity", "mode": "responses", - "supports_adaptive_thinking": true, "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, @@ -30490,7 +30804,6 @@ "supports_adaptive_thinking": true, "litellm_provider": "perplexity", "mode": "responses", - "supports_adaptive_thinking": true, "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, @@ -31406,7 +31719,7 @@ "supports_tool_choice": true }, "sambanova/Meta-Llama-3.2-1B-Instruct": { - "deprecation_date": "2025-06-25", + "deprecation_date": "2025-06-25", "input_cost_per_token": 4e-08, "litellm_provider": "sambanova", "max_input_tokens": 16384, @@ -31537,15 +31850,15 @@ "supports_vision": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "snowflake/claude-3-5-sonnet": { + "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 200000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, "supports_computer_use": true, "supports_function_calling": true, "supports_vision": true, @@ -31553,14 +31866,14 @@ "supports_system_messages": true, "supports_response_schema": true }, - "snowflake/deepseek-r1": { + "snowflake/deepseek-r1": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000135, - "output_cost_per_token": 0.0000054, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 5.4e-06, "supports_reasoning": true, "supports_system_messages": true }, @@ -31619,8 +31932,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.0000012, - "output_cost_per_token": 0.0000012, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 1.2e-06, "supports_function_calling": true, "supports_system_messages": true }, @@ -31630,8 +31943,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true }, @@ -31641,8 +31954,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000024, - "output_cost_per_token": 0.00000024, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 2.4e-07, "supports_system_messages": true }, "snowflake/llama3.2-1b": { @@ -31659,17 +31972,17 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/llama3.3-70b": { + "snowflake/llama3.3-70b": { "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "snowflake", "mode": "chat", "supports_function_calling": true, "supports_system_messages": true - }, + }, "snowflake/mistral-7b": { "litellm_provider": "snowflake", "max_input_tokens": 32000, @@ -31684,14 +31997,14 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/mistral-large2": { + "snowflake/mistral-large2": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_response_schema": true @@ -31731,17 +32044,17 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/snowflake-llama-3.3-70b": { + "snowflake/snowflake-llama-3.3-70b": { "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "snowflake", "mode": "chat", "supports_function_calling": true, "supports_system_messages": true - }, + }, "stability/sd3": { "litellm_provider": "stability", "mode": "image_generation", @@ -32825,7 +33138,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -32984,7 +33298,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -32993,7 +33308,7 @@ "input_cost_per_token": 3.6e-06, "input_cost_per_token_above_200k_tokens": 7.2e-06, "output_cost_per_token_above_200k_tokens": 2.7e-05, - "cache_creation_input_token_cost_above_200k_tokens": 9.0e-06, + "cache_creation_input_token_cost_above_200k_tokens": 9e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, "litellm_provider": "bedrock_converse", @@ -33011,7 +33326,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -33033,7 +33349,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_parallel_tool_use_config": true }, "us.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -33087,7 +33404,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "high" + "bedrock_output_config_effort_ceiling": "high", + "supports_parallel_tool_use_config": true }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -33116,7 +33434,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "high" + "bedrock_output_config_effort_ceiling": "high", + "supports_parallel_tool_use_config": true }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -33144,7 +33463,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "high" + "bedrock_output_config_effort_ceiling": "high", + "supports_parallel_tool_use_config": true }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -33736,7 +34056,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -34638,6 +34957,19 @@ "/v1/audio/speech" ] }, + "vertex_ai/chirp_3": { + "input_cost_per_second": 0.00026667, + "litellm_provider": "vertex_ai", + "metadata": { + "calculation": "$0.016/60 seconds = $0.00026667 per second", + "original_pricing_per_minute": 0.016 + }, + "mode": "audio_transcription", + "source": "https://cloud.google.com/speech-to-text/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, "vertex_ai/claude-3-5-haiku": { "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -34971,7 +35303,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35001,7 +35332,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35031,7 +35361,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35062,7 +35391,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35153,7 +35481,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35184,7 +35511,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35267,7 +35593,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -37796,12 +38121,12 @@ }, "zai.glm-5": { "input_cost_per_token": 1e-06, + "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3.2e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -37822,20 +38147,6 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, - "zai.glm-5": { - "input_cost_per_token": 1e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, @@ -42734,7 +43045,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -42768,7 +43078,10 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -42783,7 +43096,10 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -42798,7 +43114,9 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42812,7 +43130,9 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42828,9 +43148,16 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/responses"], - "supported_modalities": ["text", "image"], - "supported_output_modalities": ["text"], + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -42848,9 +43175,16 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/responses"], - "supported_modalities": ["text", "image"], - "supported_output_modalities": ["text"], + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -42867,7 +43201,10 @@ "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42883,7 +43220,10 @@ "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42899,7 +43239,10 @@ "max_tokens": 128000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -43078,20 +43421,6 @@ } ] }, - "zai.glm-5": { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3.2e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, @@ -43120,45 +43449,6 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, - "minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "bedrock/us-east-1/minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "bedrock/us-west-2/minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, "cache_creation_input_token_cost_above_1hr": 2.4e-06, @@ -43180,7 +43470,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "supports_parallel_tool_use_config": true }, "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, @@ -43203,364 +43494,368 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "supports_parallel_tool_use_config": true + }, + "snowflake/claude-sonnet-4-5": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-sonnet-4-6": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-4-sonnet": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-4-opus": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/claude-haiku-4-5": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-3-7-sonnet": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-4.1": { + "max_tokens": 16384, + "max_input_tokens": 300000, + "max_output_tokens": 16384, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5": { + "max_tokens": 16384, + "max_input_tokens": 300000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5-mini": { + "max_tokens": 16384, + "max_input_tokens": 1000000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5-nano": { + "max_tokens": 16384, + "max_input_tokens": 5000000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/llama4-maverick": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 9.7e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true + }, + "snowflake/snowflake-arctic-embed-l-v2.0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "snowflake", + "mode": "embedding" + }, + "snowflake/snowflake-arctic-embed-m-v2.0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "snowflake", + "mode": "embedding" + }, + "soniox/stt-async-v4": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true + }, + "soniox/stt-async-v5": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true + }, + "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 6e-07, + "output_cost_per_token": 3.6e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 1.8e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/Qwen/Qwen3.6-27B-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 3.2e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 202752, + "max_output_tokens": 202752, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/deepseek-ai/DeepSeek-V4-Flash": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/moonshotai/Kimi-K2.6": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 9.6e-07, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/MiniMaxAI/MiniMax-M2.5": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/google/gemma-4-31B-it": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 5.6e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/openai/gpt-oss-120b": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/openai/gpt-oss-20b": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" }, - "snowflake/claude-sonnet-4-5": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-sonnet-4-6": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-4-sonnet": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-4-opus": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 0.0000005, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/claude-haiku-4-5": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 0.0000001, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-3-7-sonnet": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-4.1": { - "max_tokens": 16384, - "max_input_tokens": 300000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 0.0000005, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5": { - "max_tokens": 16384, - "max_input_tokens": 300000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.000000125, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5-mini": { - "max_tokens": 16384, - "max_input_tokens": 1000000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000012, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5-nano": { - "max_tokens": 16384, - "max_input_tokens": 5000000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.0000006, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/llama4-maverick": { - "max_tokens": 16384, - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000024, - "output_cost_per_token": 0.00000097, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true - }, - "snowflake/snowflake-arctic-embed-l-v2.0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.0, - "litellm_provider": "snowflake", - "mode": "embedding" - }, - "snowflake/snowflake-arctic-embed-m-v2.0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.0, - "litellm_provider": "snowflake", - "mode": "embedding" - }, - "soniox/stt-async-v4": { - "litellm_provider": "soniox", - "max_output_tokens": 8000, - "max_tokens": 8000, - "input_cost_per_second": 0.0, - "output_cost_per_second": 0.0000277778, - "mode": "audio_transcription", - "source": "https://soniox.com/pricing", - "supported_endpoints": ["/v1/audio/transcriptions"], - "supports_audio_input": true - }, - "soniox/stt-async-v5": { - "litellm_provider": "soniox", - "max_output_tokens": 8000, - "max_tokens": 8000, - "input_cost_per_second": 0.0, - "output_cost_per_second": 0.0000277778, - "mode": "audio_transcription", - "source": "https://soniox.com/pricing", - "supported_endpoints": ["/v1/audio/transcriptions"], - "supports_audio_input": true - }, - "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 6e-07, - "output_cost_per_token": 3.6e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 4.5e-07, - "output_cost_per_token": 1.8e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/Qwen/Qwen3.6-27B-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 3.2e-07, - "output_cost_per_token": 3.2e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 202752, - "max_output_tokens": 202752, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/deepseek-ai/DeepSeek-V4-Flash": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/moonshotai/Kimi-K2.6": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 9.6e-07, - "output_cost_per_token": 4e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/MiniMaxAI/MiniMax-M2.5": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/google/gemma-4-31B-it": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 5.6e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/openai/gpt-oss-120b": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 6e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/openai/gpt-oss-20b": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 7e-08, - "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - } - , "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 2.8e-09, @@ -43695,12 +43990,64 @@ "supports_tool_choice": true, "supports_vision": false }, + "tencent/deepseek-v4-pro": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 3.625e-09, + "input_cost_per_token": 4.35e-07, + "input_cost_per_token_cache_hit": 3.625e-09, + "litellm_provider": "tencent", + "max_input_tokens": 1000000, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "source": "https://www.tencentcloud.com/products/tokenhub", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "tencent/deepseek-v4-flash": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 2.8e-09, + "input_cost_per_token": 1.4e-07, + "input_cost_per_token_cache_hit": 2.8e-09, + "litellm_provider": "tencent", + "max_input_tokens": 1000000, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://www.tencentcloud.com/products/tokenhub", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, "pinstripes/ps/glm-4.5-air": { "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.00000045, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 4.5e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43712,8 +44059,8 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.00000014, - "output_cost_per_token": 0.00000045, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 4.5e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43725,8 +44072,8 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.00000009, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 2e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43738,8 +44085,8 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000006, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43751,8 +44098,8 @@ "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43764,8 +44111,8 @@ "max_tokens": 1000192, "max_input_tokens": 1000192, "max_output_tokens": 1000192, - "input_cost_per_token": 0.000000255, - "output_cost_per_token": 0.00000055, + "input_cost_per_token": 2.55e-07, + "output_cost_per_token": 5.5e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43773,39 +44120,39 @@ "supports_reasoning": false, "source": "https://pinstripes.io/pricing" }, - "fallback_generalizations": { - "rules": [ - { - "name": "anthropic-claude-adaptive-thinking", - "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", - "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", - "extends": "anthropic-claude", - "model_info": { - "supports_adaptive_thinking": true - } - }, - { - "name": "anthropic-claude", - "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", - "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", - "model_info": { - "litellm_provider": "anthropic", - "mode": "chat", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_tool_choice": true, - "supports_assistant_prefill": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_reasoning": true, - "supports_pdf_input": true, - "supports_system_messages": true - } - } - ] - } + "fallback_generalizations": { + "rules": [ + { + "name": "anthropic-claude-adaptive-thinking", + "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", + "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", + "extends": "anthropic-claude", + "model_info": { + "supports_adaptive_thinking": true + } + }, + { + "name": "anthropic-claude", + "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", + "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "model_info": { + "litellm_provider": "anthropic", + "mode": "chat", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_tool_choice": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_pdf_input": true, + "supports_system_messages": true + } + } + ] + } } diff --git a/litellm/models/managed_files.py b/litellm/models/managed_files.py index 24154768860..99ba764dd98 100644 --- a/litellm/models/managed_files.py +++ b/litellm/models/managed_files.py @@ -51,12 +51,12 @@ class LiteLLM_ManagedVectorStoreTable(LiteLLMPydanticObjectBase): class LiteLLM_ManagedVectorStoresTable(LiteLLMPydanticObjectBase): vector_store_id: str custom_llm_provider: str - vector_store_name: Optional[str] - vector_store_description: Optional[str] - vector_store_metadata: Optional[Dict[str, Any]] - created_at: Optional[datetime] - updated_at: Optional[datetime] - litellm_credential_name: Optional[str] - litellm_params: Optional[Dict[str, Any]] - team_id: Optional[str] - user_id: Optional[str] + vector_store_name: Optional[str] = None + vector_store_description: Optional[str] = None + vector_store_metadata: Optional[Dict[str, Any]] = None + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + litellm_credential_name: Optional[str] = None + litellm_params: Optional[Dict[str, Any]] = None + team_id: Optional[str] = None + user_id: Optional[str] = None diff --git a/litellm/models/mcp_server.py b/litellm/models/mcp_server.py index 3d03eff6df8..5d3bc176134 100644 --- a/litellm/models/mcp_server.py +++ b/litellm/models/mcp_server.py @@ -93,6 +93,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase): has_user_credential: Optional[bool] = None source_url: Optional[str] = None timeout: Optional[float] = None + max_concurrent_requests: Optional[int] = None approval_status: Optional[str] = Field( default="active", description="Approval status: 'pending_review', 'active', 'rejected'", diff --git a/litellm/models/verification_token.py b/litellm/models/verification_token.py index 8bddd1c1619..d67726be584 100644 --- a/litellm/models/verification_token.py +++ b/litellm/models/verification_token.py @@ -39,6 +39,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase): permissions: Dict = {} model_spend: Dict = {} model_max_budget: Dict = {} + budget_fallbacks: dict[str, list[str]] = {} soft_budget_cooldown: bool = False blocked: Optional[bool] = None litellm_budget_table: Optional[dict] = None diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index 75ed91c1d3b..829aa769491 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -355,7 +355,6 @@ def llm_passthrough_route( api_key: str | None = None, request_query_params: dict | None = None, request_headers: dict | None = None, - allm_passthrough_route: bool = False, content: Any | None = None, data: dict | None = None, files: RequestFiles | None = None, @@ -382,7 +381,7 @@ def llm_passthrough_route( from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager - _is_async = allm_passthrough_route + _is_async = bool(kwargs.get("allm_passthrough_route", False)) litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj")) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 2520c7e82a1..d2986a3cd82 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -8,9 +8,11 @@ from starlette.types import Scope from litellm._logging import verbose_logger from litellm.proxy._types import ( + UI_TEAM_ID, LiteLLM_TeamTable, ProxyException, SpecialHeaders, + SpecialMCPServerName, SpecialMCPServerNames, UserAPIKeyAuth, ) @@ -356,6 +358,7 @@ class MCPRequestHandler: # Inline imports avoid a circular dependency: mcp_server_manager imports # from this module. from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, global_mcp_server_manager, ) from litellm.types.mcp import MCPAuth @@ -381,7 +384,18 @@ class MCPRequestHandler: # fetches the upstream token automatically using stored credentials, # so allowing anonymous bypass would let any external caller invoke # tools authenticated as LiteLLM's service account. - if server.has_client_credentials: + # + # Resolve the flow rather than reading has_client_credentials directly: + # this is a security gate, and a legacy row whose oauth2_flow was never + # stamped still carries the M2M credential shape (client_id/secret + + # token_url, no authorization_url). Treating an unstamped-but-M2M-shaped + # row as non-M2M here would reopen the anonymous bypass the explicit + # column no longer closes on its own. Shares the one resolution helper + # with the egress backstop and the anonymous-delegate allowlist; all fail + # closed on the ambiguous shape and are removed together once no null rows + # remain. A pure-PKCE delegate server (no stored credentials) resolves to a + # non-M2M flow and keeps its bypass. + if MCPServerManager.effective_oauth2_flow(server) == "client_credentials": return False return True @@ -725,6 +739,9 @@ class MCPRequestHandler: if not user_api_key_auth or not user_api_key_auth.team_id or not prisma_client: return None + if user_api_key_auth.team_id == UI_TEAM_ID: + return None + # Get the team object (which has object_permission already loaded) team_obj: Optional[LiteLLM_TeamTable] = await get_team_object( team_id=user_api_key_auth.team_id, @@ -1020,6 +1037,9 @@ class MCPRequestHandler: if user_api_key_auth is None or not user_api_key_auth.team_id or prisma_client is None: return [] + if user_api_key_auth.team_id == UI_TEAM_ID: + return [] + team_obj: Optional[LiteLLM_TeamTable] = await get_team_object( team_id=user_api_key_auth.team_id, prisma_client=prisma_client, @@ -1041,6 +1061,9 @@ class MCPRequestHandler: if object_permissions is None: return list(set(team_access_group_servers)) + if SpecialMCPServerName.all_proxy_servers.value in (object_permissions.mcp_servers or []): + return list(global_mcp_server_manager.get_registry().keys()) + direct_mcp_servers = global_mcp_server_manager.expand_permission_list(object_permissions.mcp_servers or []) legacy_access_group_servers = await MCPRequestHandler._get_mcp_servers_from_access_groups( @@ -1499,6 +1522,9 @@ class MCPRequestHandler: verbose_logger.debug("prisma_client is None") return [] + if user_api_key_auth.team_id == UI_TEAM_ID: + return [] + try: team_obj: Optional[LiteLLM_TeamTable] = await get_team_object( team_id=user_api_key_auth.team_id, diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 6933aa06b2d..89d645b6f8a 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -1,14 +1,16 @@ import asyncio import html as _html import json +import secrets import time from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse import httpx from fastapi import APIRouter, Form, HTTPException, Request -from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response +from pydantic import BaseModel, ValidationError from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import ( @@ -31,11 +33,11 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.utils import get_server_root_path -from litellm.types.mcp import MCPAuth +from litellm.types.mcp import MCPAuth, MCPCredentials from litellm.types.mcp_server.mcp_server_manager import MCPServer if TYPE_CHECKING: - from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy._types import LiteLLM_MCPServerTable, UserAPIKeyAuth # TTL cache for upstream OAuth metadata fetched from pass-through MCP servers. # Keeps us from hammering the upstream IdP on each discovery request. @@ -136,6 +138,72 @@ def decode_state_hash(encrypted_state: str) -> dict: return state_data +# LIT-4197: some upstream authorization servers reject an over-long ``state`` +# (the encrypted OAuth session blob routinely exceeds their limit). The upstream +# only needs an opaque value it echoes back on ``/callback``, so we forward a +# short random handle and keep the encrypted session in a per-flow HttpOnly +# cookie bound to that handle. The browser carries the cookie across the +# upstream round trip, so the flow stays correct with no server-side session +# store (works across proxy replicas, unlike an in-process map). +_OAUTH_STATE_COOKIE_PREFIX = "mcp_oauth_state_" +_OAUTH_STATE_COOKIE_TTL_SECONDS = 600 +_OAUTH_STATE_HANDLE_BYTES = 32 + + +def _oauth_state_cookie_name(relay_state: str) -> str: + return f"{_OAUTH_STATE_COOKIE_PREFIX}{relay_state}" + + +def _oauth_state_cookie_path_and_secure(request: Request) -> tuple[str, bool]: + parsed = urlparse(get_request_base_url(request)) + return parsed.path or "/", parsed.scheme == "https" + + +def _set_oauth_state_cookie( + response: Response, + request: Request, + relay_state: str, + encoded_state: str, +) -> None: + path, secure = _oauth_state_cookie_path_and_secure(request) + response.set_cookie( + key=_oauth_state_cookie_name(relay_state), + value=encoded_state, + max_age=_OAUTH_STATE_COOKIE_TTL_SECONDS, + path=path, + secure=secure, + httponly=True, + samesite="lax", + ) + + +def _resolve_encoded_oauth_state(request: Request, state: str) -> str: + """Return the encrypted OAuth session for a ``/callback`` request. + + New flows carry it in a per-flow cookie keyed by the short handle we + forwarded upstream (the IdP echoes that handle back as ``state``). Flows + started before this change - or in flight across a deploy - carry the + encrypted blob directly in ``state``, so fall back to it when the cookie + is absent. + """ + cookie_value = request.cookies.get(_oauth_state_cookie_name(state)) + return cookie_value if cookie_value else state + + +def _clear_oauth_state_cookie(response: Response, request: Request, state: str) -> None: + cookie_name = _oauth_state_cookie_name(state) + if cookie_name not in request.cookies: + return + path, secure = _oauth_state_cookie_path_and_secure(request) + response.delete_cookie( + key=cookie_name, + path=path, + secure=secure, + httponly=True, + samesite="lax", + ) + + def _get_validated_client_redirect_uri(request: Request, state_data: Dict[str, Any]) -> str: """Return a trusted (same-origin, loopback, or ops-allowlisted) client redirect URI from OAuth state. @@ -390,6 +458,46 @@ async def _store_per_user_token_server_side( ) +def _raise_if_not_oauth2(mcp_server: MCPServer) -> None: + """Reject a non-oauth2 server from the gateway's OAuth authorize/token/register flow.""" + if mcp_server.auth_type == MCPAuth.oauth2: + return + raise HTTPException( + status_code=400, + detail={ + "error": "server_not_oauth2", + "message": ( + f"MCP server '{mcp_server.server_name or mcp_server.name}' does not use OAuth " + f"(auth_type={mcp_server.auth_type}). This server does not support the authorization-code " + "flow; it has no client_id, authorize, token, or registration endpoint. " + "Access is controlled by the server's configured auth_type and access groups" + ), + }, + ) + + +def _raise_unless_oauth2_discovery_server( + mcp_server: Optional[MCPServer], + mcp_server_name: Optional[str], + description: str, +) -> None: + """404 a NAMED discovery request unless it resolves to an oauth2 server. + + A named server that is unknown (or hidden from the caller) and one that exists + but is non-oauth2 both return the same 404, so the well-known discovery paths + cannot be used to enumerate non-OAuth server names. Root discovery (no name) is + unaffected, and pass-through servers are resolved by the caller before this runs. + """ + if mcp_server_name is None: + return + if mcp_server is not None and mcp_server.auth_type == MCPAuth.oauth2: + return + raise HTTPException( + status_code=404, + detail=f"MCP server '{mcp_server_name}' is {description}", + ) + + async def authorize_with_server( request: Request, mcp_server: MCPServer, @@ -421,11 +529,12 @@ async def authorize_with_server( code_challenge_method=code_challenge_method, client_redirect_uri=redirect_uri, ) + relay_state = secrets.token_urlsafe(_OAUTH_STATE_HANDLE_BYTES) params = { "client_id": mcp_server.client_id if mcp_server.client_id else client_id, "redirect_uri": f"{request_base_url}/callback", - "state": encoded_state, + "state": relay_state, "response_type": response_type or "code", } if scope: @@ -442,7 +551,9 @@ async def authorize_with_server( existing_params = dict(parse_qsl(parsed_auth_url.query)) existing_params.update(params) final_url = urlunparse(parsed_auth_url._replace(query=urlencode(existing_params))) - return RedirectResponse(final_url) + response = RedirectResponse(final_url) + _set_oauth_state_cookie(response, request, relay_state, encoded_state) + return response async def exchange_token_with_server( @@ -457,6 +568,7 @@ async def exchange_token_with_server( refresh_token: Optional[str] = None, scope: Optional[str] = None, ): + _raise_if_not_oauth2(mcp_server) if grant_type not in ("authorization_code", "refresh_token"): raise HTTPException(status_code=400, detail="Unsupported grant_type") @@ -573,6 +685,179 @@ async def exchange_token_with_server( return JSONResponse(result, headers=TOKEN_NO_CACHE_HEADERS) +class _DcrClientRegistration(BaseModel): + """RFC 7591 dynamic client registration response, narrowed to the fields the gateway + must persist to authenticate later token-endpoint calls. Extra members are ignored.""" + + client_id: str + client_secret: Optional[str] = None + token_endpoint_auth_method: Optional[str] = None + + +class _PersistedDcrCredentials(BaseModel): + client_id: Optional[str] = None + client_secret: Optional[str] = None + token_endpoint_auth_method: Optional[str] = None + + +def _get_persisted_dcr_credentials(credentials: object) -> Optional[_PersistedDcrCredentials]: + if not credentials: + return None + try: + return ( + _PersistedDcrCredentials.model_validate_json(credentials) + if isinstance(credentials, str) + else _PersistedDcrCredentials.model_validate(credentials) + ) + except ValidationError: + return None + + +def _decrypt_persisted_dcr_credential(value: Optional[str], key: str) -> Optional[str]: + if value is None: + return None + return decrypt_value_helper( + value=value, + key=key, + exception_type="debug", + return_original_value=True, + ) + + +def _apply_persisted_dcr_credentials(mcp_server: MCPServer, credentials: _PersistedDcrCredentials) -> bool: + client_id = _decrypt_persisted_dcr_credential(credentials.client_id, "client_id") + if not client_id: + return False + mcp_server.client_id = client_id + mcp_server.client_secret = _decrypt_persisted_dcr_credential(credentials.client_secret, "client_secret") + mcp_server.token_endpoint_auth_method = credentials.token_endpoint_auth_method + return True + + +async def _get_persisted_mcp_server_with_dcr_client_id( + mcp_server: MCPServer, +) -> Optional[tuple["LiteLLM_MCPServerTable", _PersistedDcrCredentials]]: + from litellm.proxy._experimental.mcp_server.db import get_mcp_server # noqa: PLC0415 + from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 + + try: + prisma_client = get_prisma_client_or_throw("Database not connected. Cannot read MCP OAuth client registration.") + persisted_mcp_server = await get_mcp_server( + prisma_client=prisma_client, + server_id=mcp_server.server_id, + ) + except Exception as exc: # noqa: BLE001 + verbose_logger.debug( + "register_client_with_server: failed to read persisted DCR client registration for server_id=%s: %s", + mcp_server.server_id, + exc, + ) + return None + + if persisted_mcp_server is None: + return None + + credentials = _get_persisted_dcr_credentials(persisted_mcp_server.credentials) + if credentials is None or not credentials.client_id: + return None + + return persisted_mcp_server, credentials + + +async def _reuse_persisted_dcr_client_if_available(mcp_server: MCPServer) -> bool: + persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server) + if persisted is None: + return False + persisted_mcp_server, credentials = persisted + if not _apply_persisted_dcr_credentials(mcp_server, credentials): + return False + + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + global_mcp_server_manager, + ) + + try: + await global_mcp_server_manager.update_server(persisted_mcp_server) + except Exception as exc: # noqa: BLE001 + verbose_logger.warning( + "register_client_with_server: failed to refresh persisted DCR client registration for server_id=%s: %s", + mcp_server.server_id, + exc, + ) + return bool(mcp_server.client_id) + + +DcrRegistrationPersistenceResult = Literal["persisted", "reused", "failed"] + + +async def _persist_dcr_client_registration( + mcp_server: MCPServer, registration_response: object +) -> DcrRegistrationPersistenceResult: + """Persist the dynamically registered OAuth client (RFC 7591) onto the MCP server row. + + The interactive authorization_code flow mints a ``client_id`` via Dynamic Client + Registration that discovery cannot re-derive; without persisting it the autonomous + ``refresh_token`` grant has no client identity, so an expired access token forces a + full re-authorization instead of a silent refresh. Mirrors the ``encrypt_credentials`` + write that ``client_credentials`` and token exchange already use. Failures are logged, + never raised: registration still returns to the caller even when persistence fails. + """ + try: + registration = _DcrClientRegistration.model_validate(registration_response) + except ValidationError as exc: + verbose_logger.warning( + "register_client_with_server: DCR response has no usable client_id for server_id=%s; " + "client registration not persisted (%s)", + mcp_server.server_id, + exc, + ) + return "failed" + + if await _reuse_persisted_dcr_client_if_available(mcp_server): + return "reused" + + credentials: MCPCredentials = { + "client_id": registration.client_id, + **({"client_secret": registration.client_secret} if registration.client_secret is not None else {}), + **( + {"token_endpoint_auth_method": "client_secret_basic"} + if registration.token_endpoint_auth_method == "client_secret_basic" + else {} + ), + } + + from litellm.proxy._experimental.mcp_server.db import update_mcp_server # noqa: PLC0415 + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + global_mcp_server_manager, + ) + from litellm.proxy._types import UpdateMCPServerRequest # noqa: PLC0415 + from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 + + try: + prisma_client = get_prisma_client_or_throw( + "Database not connected. Cannot persist MCP OAuth client registration." + ) + updated_row = await update_mcp_server( + prisma_client=prisma_client, + data=UpdateMCPServerRequest( + server_id=mcp_server.server_id, + credentials=credentials, + oauth2_flow="authorization_code", + **({"token_url": mcp_server.token_url} if mcp_server.token_url else {}), + ), + touched_by="mcp_oauth_dcr", + ) + await global_mcp_server_manager.update_server(updated_row) + return "persisted" + except Exception as exc: # noqa: BLE001 + verbose_logger.warning( + "register_client_with_server: failed to persist DCR client registration for server_id=%s: %s", + mcp_server.server_id, + exc, + ) + return "failed" + + async def register_client_with_server( request: Request, mcp_server: MCPServer, @@ -581,7 +866,9 @@ async def register_client_with_server( response_types: Optional[list], token_endpoint_auth_method: Optional[str], fallback_client_id: Optional[str] = None, + persist_credentials: bool = False, ): + _raise_if_not_oauth2(mcp_server) request_base_url = get_request_base_url(request) dummy_return = { "client_id": fallback_client_id or mcp_server.server_name, @@ -589,7 +876,10 @@ async def register_client_with_server( "redirect_uris": [f"{request_base_url}/callback"], } - if mcp_server.client_id and mcp_server.client_secret: + if mcp_server.client_id: + return dummy_return + + if await _reuse_persisted_dcr_client_if_available(mcp_server): return dummy_return if mcp_server.authorization_url is None: @@ -625,6 +915,11 @@ async def register_client_with_server( token_response = response.json() + if persist_credentials: + persistence_result = await _persist_dcr_client_registration(mcp_server, token_response) + if persistence_result == "reused": + return dummy_return + return JSONResponse(token_response) @@ -655,6 +950,7 @@ async def authorize( mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if mcp_server is None: raise HTTPException(status_code=404, detail="MCP server not found") + _raise_if_not_oauth2(mcp_server) # Use server's stored client_id when caller doesn't supply one. # Raise a clear error instead of passing an empty string — an empty # client_id would silently produce a broken authorization URL. @@ -791,17 +1087,19 @@ async def callback( error_description, ) if state: + encoded_state = _resolve_encoded_oauth_state(request, state) try: - state_data = decode_state_hash(state) + state_data = decode_state_hash(encoded_state) original_state = state_data.get("original_state") redirect_uri = _get_validated_client_redirect_uri(request, state_data) - except HTTPException: - # Untrusted/invalid client redirect_uri — surface inline rather - # than blindly forwarding the error to an attacker-controlled URL. - return _render_oauth_error_html(error, error_description) except Exception: - # State could not be decrypted (expired key, tampered, etc.). - return _render_oauth_error_html(error, error_description) + # Untrusted/invalid client redirect_uri (HTTPException), or an + # undecryptable state (expired key, tampered): surface the IdP + # error inline rather than forwarding it to an attacker-controlled + # URL, and drop the one-time cookie we can no longer consume. + response = _render_oauth_error_html(error, error_description) + _clear_oauth_state_cookie(response, request, state) + return response params: Dict[str, str] = {"error": error} if error_description: @@ -811,7 +1109,9 @@ async def callback( if original_state is not None: params["state"] = original_state complete_returned_url = _append_query_params(redirect_uri, params) - return RedirectResponse(url=complete_returned_url, status_code=302) + response = RedirectResponse(url=complete_returned_url, status_code=302) + _clear_oauth_state_cookie(response, request, state) + return response # No state — nothing to round-trip to. Show the user the error. return _render_oauth_error_html(error, error_description) @@ -827,7 +1127,8 @@ async def callback( # 3. Successful authorization response. try: - state_data = decode_state_hash(state) + encoded_state = _resolve_encoded_oauth_state(request, state) + state_data = decode_state_hash(encoded_state) original_state = state_data["original_state"] # Re-validate the client redirect URI at the sink. /authorize @@ -840,14 +1141,18 @@ async def callback( params = {"code": code, "state": original_state} complete_returned_url = _append_query_params(redirect_uri, params) - return RedirectResponse(url=complete_returned_url, status_code=302) + response = RedirectResponse(url=complete_returned_url, status_code=302) + _clear_oauth_state_cookie(response, request, state) + return response except HTTPException: # Re-raise so a non-loopback base_url surfaces as 400 instead of # a generic "authentication incomplete" redirect. raise except Exception: - return HTMLResponse("Authentication incomplete. You can close this window.") + response = HTMLResponse("Authentication incomplete. You can close this window.") + _clear_oauth_state_cookie(response, request, state) + return response # ------------------------------ @@ -1063,6 +1368,15 @@ async def _build_oauth_protected_resource_response( detail=(f"Upstream oauth-protected-resource metadata unavailable for MCP server {mcp_server.name!r}"), ) + obo_response = _obo_protected_resource_response(mcp_server, resource_url) + if obo_response is not None: + return obo_response + + # An OBO server with no configured issuer falls through to the gateway default so discovery still + # returns metadata; every other non-oauth2 named server 404s to avoid enumeration. + if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange: + _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth-protected resource") + return { "authorization_servers": [ (f"{request_base_url}/{mcp_server_name}" if mcp_server_name else f"{request_base_url}") @@ -1072,6 +1386,51 @@ async def _build_oauth_protected_resource_response( } +def _obo_protected_resource_response(mcp_server: Optional[MCPServer], resource_url: str) -> Optional[dict]: + """The OBO (token_exchange) PRM, or None when this server is not OBO / no issuer is configured. + + The client SSOs with the IdP to obtain a subject token, which LiteLLM then exchanges, so discovery + points at the JWT-auth issuer(s) LiteLLM trusts (the same IdP that issues and validates the + subject), not the gateway. None falls the caller back to the gateway default so discovery still + returns metadata; it just can't name the IdP. + """ + if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange: + return None + issuers = _jwt_auth_issuers() + if not issuers: + return None + return { + "authorization_servers": issuers, + "resource": resource_url, + "scopes_supported": (mcp_server.scopes if mcp_server.scopes else []), + } + + +def _jwt_auth_issuers() -> list: + """The OAuth issuer identifier(s) LiteLLM's JWT auth trusts, for the OBO PRM authorization_servers. + + In token_exchange the IdP that issues the subject JWT is the same one LiteLLM validates it + against, so OBO discovery points clients at the JWT-auth issuer to obtain a subject token. + Sourced from ``JWT_ISSUER`` and any configured ``litellm_jwtauth.issuers``. + """ + import os # noqa: PLC0415 + + from litellm.proxy.proxy_server import general_settings # noqa: PLC0415 + + issuers: list = [] + env_issuer = os.getenv("JWT_ISSUER") + if env_issuer: + issuers.append(env_issuer) + + jwtauth = general_settings.get("litellm_jwtauth") if isinstance(general_settings, dict) else None + raw_issuers = jwtauth.get("issuers") if isinstance(jwtauth, dict) else getattr(jwtauth, "issuers", None) + for cfg in raw_issuers or []: + issuer = cfg.get("issuer") if isinstance(cfg, dict) else getattr(cfg, "issuer", None) + if issuer and issuer not in issuers: + issuers.append(issuer) + return issuers + + # Standard MCP pattern: /.well-known/oauth-protected-resource/mcp/{server_name} # This is the pattern expected by standard MCP clients (mcp-inspector, VSCode Copilot) @router.get( @@ -1149,6 +1508,8 @@ def _build_oauth_authorization_server_response( if mcp_server_name: mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip) + _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth authorization server") + return { "issuer": request_base_url, # point to your proxy "authorization_endpoint": authorization_endpoint, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 95d00554034..d347c694366 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -13,10 +13,12 @@ import json import os import re import time -from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union, cast +from contextlib import asynccontextmanager +from typing import Any, AsyncIterator, Callable, Literal, Optional, Union, cast from urllib.parse import urlparse import anyio +import httpx from fastapi import HTTPException from httpx import HTTPStatusError from mcp import ReadResourceResult, Resource @@ -63,6 +65,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials import ( ) from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( raise_public, + raise_token_exchange_challenge, raise_user_oauth_challenge, to_server_spec, to_subject, @@ -70,8 +73,13 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import ( LazyPerUserOAuthTokenStore, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchange_provider import ( + build_token_exchanger, +) from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( AuthorizationCodeConfig, + ServerSpec, + TokenExchangeConfig, ) from litellm.proxy._experimental.mcp_server.utils import ( MCP_TOOL_PREFIX_SEPARATOR, @@ -104,7 +112,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl -from litellm.proxy.utils import ProxyLogging +from litellm.proxy.utils import ProxyLogging, get_server_root_path from litellm.repositories.table_repositories import MCPServerRepository from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.mcp import MCPAuth, MCPStdioConfig @@ -155,7 +163,7 @@ _AZURE_ENTRA_HOSTS = { # BYOK credential cache. Keyed by (user_id, server_id); value is # (values_dict, monotonic_timestamp). Keeps the tool-call and tool-listing # paths off the DB on every request within the TTL window. -_user_env_vars_cache: Dict[Tuple[str, str], Tuple[Dict[str, str], float]] = {} +_user_env_vars_cache: dict[tuple[str, str], tuple[dict[str, str], float]] = {} _USER_ENV_VARS_CACHE_TTL = 60 # seconds _USER_ENV_VARS_CACHE_MAX_SIZE = 4096 # cap to prevent unbounded growth @@ -166,7 +174,7 @@ def invalidate_user_env_vars_cache(user_id: str, server_id: str) -> None: _user_env_vars_cache.pop((user_id, server_id), None) -def _write_user_env_vars_cache(user_id: str, server_id: str, values: Dict[str, str]) -> None: +def _write_user_env_vars_cache(user_id: str, server_id: str, values: dict[str, str]) -> None: cache_key = (user_id, server_id) # Re-insert at the tail so eviction drops the oldest-written entry, not a # freshly refreshed one, and only sheds a single entry instead of wiping the @@ -179,7 +187,7 @@ def _write_user_env_vars_cache(user_id: str, server_id: str, values: Dict[str, s def _should_strip_caller_authorization( mcp_server: MCPServer, - raw_headers: Optional[Dict[str, str]], + raw_headers: Optional[dict[str, str]], user_api_key_auth: Optional[UserAPIKeyAuth], ) -> bool: """Decide whether the caller's ``Authorization`` header must NOT be @@ -206,6 +214,10 @@ def _should_strip_caller_authorization( ``Authorization`` is the upstream OAuth token and must be forwarded, so we keep it. """ + if mcp_server.auth_type == MCPAuth.oauth2_token_exchange: + # OBO: the inbound Authorization is the subject token. It is exchanged at the IdP and only the + # exchanged token is sent upstream, so the raw caller bearer must never be forwarded. + return True if mcp_server.has_client_credentials: return True if mcp_server.auth_type == MCPAuth.oauth2 and to_server_spec(mcp_server) is not None: @@ -241,9 +253,79 @@ def _without_authorization( return filtered or None +def _format_byok_openapi_auth_header(mcp_server: MCPServer, mcp_auth_header: str) -> str: + """Format a raw BYOK credential for OpenAPI tool ``Authorization`` injection.""" + if mcp_server.auth_type == MCPAuth.api_key: + return f"ApiKey {mcp_auth_header}" + if mcp_server.auth_type == MCPAuth.basic: + return f"Basic {mcp_auth_header}" + return f"Bearer {mcp_auth_header}" + + +def _openapi_forwarded_extra_headers( + mcp_server: MCPServer, + raw_headers: Optional[dict[str, str]], + user_api_key_auth: Optional[UserAPIKeyAuth], +) -> Optional[dict[str, str]]: + if not mcp_server.extra_headers or not raw_headers: + return None + normalized_raw = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)} + skip_caller_authorization = _should_strip_caller_authorization( + mcp_server=mcp_server, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) + forwarded: dict[str, str] = {} + for header_name in mcp_server.extra_headers: + if not isinstance(header_name, str): + continue + if skip_caller_authorization and header_name.lower() == "authorization": + continue + value = normalized_raw.get(header_name.lower()) + if value is not None: + forwarded[header_name] = value + return forwarded or None + + +async def _resolve_byok_mcp_auth_header( + mcp_server: MCPServer, + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_auth_header: Optional[str], +) -> Optional[str]: + """Resolve BYOK credential for tool calls that bypass ``execute_mcp_tool``.""" + if not mcp_server.is_byok: + return mcp_auth_header + + from litellm.proxy._experimental.mcp_server.server import ( + _check_byok_credential, + _get_byok_credential, + ) + + if not mcp_auth_header: + byok_cred = await _get_byok_credential(mcp_server, user_api_key_auth) + if byok_cred is None: + raise HTTPException( + status_code=401, + detail={ + "error": "byok_auth_required", + "server_id": mcp_server.server_id, + "server_name": mcp_server.server_name or mcp_server.name, + "message": ( + "No stored credential found for this BYOK server. " + "Complete the OAuth authorization flow to provide your API key." + ), + }, + headers={"WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"'}, + ) + return byok_cred + + await _check_byok_credential(mcp_server, user_api_key_auth) + return mcp_auth_header + + def _extract_upstream_auth_failure( exc: BaseException, -) -> Optional[Tuple[int, Optional[str]]]: +) -> Optional[tuple[int, Optional[str]]]: """Walk the exception tree looking for an HTTP 401/403 response from the upstream MCP server. @@ -255,8 +337,8 @@ def _extract_upstream_auth_failure( Returns ``(status_code, www_authenticate)`` on match, else ``None``. """ - seen: Set[int] = set() - stack: List[BaseException] = [exc] + seen: set[int] = set() + stack: list[BaseException] = [exc] while stack: current = stack.pop() if id(current) in seen: @@ -337,7 +419,7 @@ def _warn_internal_delegate_pkce_if_applicable(server: MCPServer, *, source: str ) -def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]: +def _deserialize_json_dict(data: Any) -> Optional[dict[str, str]]: """ Deserialize optional JSON mappings stored in the database. @@ -358,7 +440,7 @@ def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]: return data -def _deserialize_json_list(data: Any) -> Optional[List[Dict[str, Any]]]: +def _deserialize_json_list(data: Any) -> Optional[list[dict[str, Any]]]: """Deserialize a JSON array stored in the DB (``env_vars`` and friends). Returns ``None`` for empty / null / unparseable input. Accepts strings @@ -499,6 +581,21 @@ def _create_elicitation_callback(): class MCPServerManager: _STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$") + @staticmethod + def _explicit_oauth2_flow( + oauth2_flow: Optional[str], + ) -> Optional[Literal["client_credentials", "authorization_code"]]: + """DB rows persist their flow (write-time stamps plus the startup backfill) and + config servers must declare it (validated at load), so both builds read the + value verbatim: unknown or null resolves to None, which + ``needs_user_oauth_token`` already treats as interactive. Field-shape inference + survives only in the request-time security helpers (``effective_oauth2_flow`` / + ``resolve_oauth2_flow_for_request``). + """ + if oauth2_flow in ("client_credentials", "authorization_code"): + return cast(Literal["client_credentials", "authorization_code"], oauth2_flow) + return None + @staticmethod def _resolve_oauth2_flow( *, @@ -509,11 +606,18 @@ class MCPServerManager: client_id: Optional[str], client_secret: Optional[str], ) -> Optional[Literal["client_credentials", "authorization_code"]]: - """Infer oauth2_flow for legacy records that omit the field. + """Infer oauth2_flow from field shape when the value is omitted. - DB rows created before oauth2_flow support may have OAuth2 client - credentials + token_url but a null oauth2_flow. Treat these as M2M, - unless authorization_url is present (interactive OAuth). + SECURITY-SENSITIVE: this is the shape-inference engine both request-time security + helpers delegate to, so it is what decides M2M-vs-interactive for an unstamped row. + Always access it through ``effective_oauth2_flow`` (boolean/enum decisions) or + ``resolve_oauth2_flow_for_request`` (the egress object backstop), which are the single + choke points for request-time resolution; do not call it directly from security sites + and do not weaken its M2M-shape branch without accounting for those callers. DB rows + are stamped at write time and by the startup backfill, config servers must declare + oauth2_flow (validated at load), and both builds read the value verbatim via + ``_explicit_oauth2_flow``. Delete this whole request-time layer only once the backstop + warning stays silent in production. """ if oauth2_flow in ("client_credentials", "authorization_code"): return cast(Literal["client_credentials", "authorization_code"], oauth2_flow) @@ -528,12 +632,70 @@ class MCPServerManager: return "client_credentials" return None + @staticmethod + def effective_oauth2_flow(server: "MCPServer") -> Optional[Literal["client_credentials", "authorization_code"]]: + """The oauth2_flow a security decision must use for ``server`` this request. + + Column-first, shape-fallback: a stamped row returns its explicit value; an + unstamped (null) row whose fields carry the M2M shape resolves to + ``client_credentials`` so it is treated as M2M and fails closed. Every + security-sensitive reader (anonymous-delegate allowlist and gate, egress flow + resolution) goes through this one helper rather than reading the bare + ``has_client_credentials`` column, which is unreliable for null rows. + """ + return MCPServerManager._resolve_oauth2_flow( + auth_type=server.auth_type, + oauth2_flow=server.oauth2_flow, + token_url=server.token_url, + authorization_url=server.authorization_url, + client_id=server.client_id, + client_secret=server.client_secret, + ) + + @staticmethod + def resolve_oauth2_flow_for_request(server: "MCPServer") -> "MCPServer": + """Return ``server`` with its effective oauth2_flow applied, for egress paths. + + A stamped row is returned unchanged (its effective flow equals the stored value). + An unstamped M2M-shape row is returned as a per-request copy carrying + ``oauth2_flow=client_credentials`` so downstream ``has_client_credentials`` / + ``needs_user_oauth_token`` compute correctly and the stored client credentials are + used instead of forwarding the caller's Authorization. Use this at every point that + resolves an allowed server id into an ``MCPServer`` for a tool call or listing. + """ + effective = MCPServerManager.effective_oauth2_flow(server) + if effective is None or effective == server.oauth2_flow: + return server + verbose_logger.warning( + "MCP server %s has no persisted oauth2_flow but matches the %s shape; using the " + "inferred flow for this request. The startup backfill leaves this ambiguous M2M " + "shape unstamped on purpose, so it will NOT self-heal: set oauth2_flow explicitly " + "in the dashboard or via PUT /v1/mcp/server (client_credentials for M2M, or " + "authorization_code after an interactive sign-in).", + server.server_id, + effective, + ) + return server.model_copy(update={"oauth2_flow": effective}) + + @staticmethod + def _obo_needs_endpoint_discovery( + auth_type: Optional[MCPAuthType], + token_exchange_endpoint: Optional[str], + token_url: Optional[str], + ) -> bool: + """An ``oauth2_token_exchange`` server with no configured token endpoint can have it + discovered (RFC 9728 -> RFC 8414) the same way the ``oauth2`` flow already does; an explicitly + configured ``token_exchange_endpoint``/``token_url`` wins and skips the discovery round-trip. + """ + return auth_type == MCPAuth.oauth2_token_exchange and not (token_exchange_endpoint or token_url) + def __init__(self, cred_provider: Optional[UpstreamCredentialProvider] = None): self._cred_provider = cred_provider or UpstreamCredentialProvider( - oauth_token_store=LazyPerUserOAuthTokenStore(self.get_mcp_server_by_id) + oauth_token_store=LazyPerUserOAuthTokenStore(self.get_mcp_server_by_id), + token_exchanger=build_token_exchanger(), ) - self.registry: Dict[str, MCPServer] = {} - self.config_mcp_servers: Dict[str, MCPServer] = {} + self.registry: dict[str, MCPServer] = {} + self.config_mcp_servers: dict[str, MCPServer] = {} """ eg. [ @@ -550,17 +712,22 @@ class MCPServerManager: ] """ - self.tool_name_to_mcp_server_name_mapping: Dict[str, str] = {} + # Per-server outbound tool-call concurrency limiters, lazily created from + # each server's max_concurrent_requests. Keyed by server_id so the cap + # survives the registry atomic-swap on config reload; a missing key means + # the server has no configured limit. + self._server_call_semaphores: dict[str, asyncio.Semaphore] = {} + self.tool_name_to_mcp_server_name_mapping: dict[str, str] = {} """ { "gmail_send_email": "zapier_mcp_server", } """ - self._upstream_initialize_instructions_by_server_id: Dict[str, str] = {} + self._upstream_initialize_instructions_by_server_id: dict[str, str] = {} # Per-server monotonic timestamp of last upstream prefetch attempt (success, # empty result, or failure). Used to throttle re-probes for servers that do # not return instructions, and to apply a short cooldown after failures. - self._upstream_initialize_instructions_probed_at: Dict[str, float] = {} + self._upstream_initialize_instructions_probed_at: dict[str, float] = {} def _remember_upstream_initialize_instructions(self, server: MCPServer, client: MCPClient) -> None: raw = getattr(client, "_last_initialize_instructions", None) @@ -616,7 +783,7 @@ class MCPServerManager: user_api_key_auth=None, raise_on_missing=False, ) - extra_headers: Optional[Dict[str, str]] = dict(resolved_static_headers) if resolved_static_headers else None + extra_headers: Optional[dict[str, str]] = dict(resolved_static_headers) if resolved_static_headers else None client = await self._create_mcp_client( server=server, mcp_auth_header=None, @@ -636,7 +803,7 @@ class MCPServerManager: e, ) - def get_registry(self) -> Dict[str, MCPServer]: + def get_registry(self) -> dict[str, MCPServer]: """ Get the registered MCP Servers from the registry and union with the config MCP Servers """ @@ -644,8 +811,8 @@ class MCPServerManager: async def load_servers_from_config( self, - mcp_servers_config: Dict[str, Any], - mcp_aliases: Optional[Dict[str, str]] = None, + mcp_servers_config: dict[str, Any], + mcp_aliases: Optional[dict[str, str]] = None, ): """ Load the MCP Servers from the config @@ -663,7 +830,7 @@ class MCPServerManager: for server_name, server_config in mcp_servers_config.items(): validate_mcp_server_name(server_name) - _mcp_info: Dict[str, Any] = server_config.get("mcp_info", None) or {} + _mcp_info: dict[str, Any] = server_config.get("mcp_info", None) or {} # Preserve all custom fields from config while setting defaults for core fields mcp_info: MCPInfo = _mcp_info.copy() # Set default values for core fields if not present @@ -711,14 +878,27 @@ class MCPServerManager: ) auth_type = server_config.get("auth_type", None) - if server_url and auth_type is not None and auth_type == MCPAuth.oauth2: + if server_url and ( + auth_type == MCPAuth.oauth2 + or self._obo_needs_endpoint_discovery( + auth_type, + server_config.get("token_exchange_endpoint"), + server_config.get("token_url"), + ) + ): mcp_oauth_metadata = await self._descovery_metadata( server_url=server_url, + allow_origin_fallback=auth_type == MCPAuth.oauth2, ) else: mcp_oauth_metadata = None - resolved_scopes = server_config.get("scopes") or (mcp_oauth_metadata.scopes if mcp_oauth_metadata else None) + # Filter blank scopes (e.g. YAML ``scopes: [""]``) the same way the DB-build path does, so + # an all-blank list normalizes to None rather than a ``("",)`` tuple that skips the + # entra_obo fail-closed scope precondition and POSTs an empty scope to the IdP. + resolved_scopes = self._extract_scopes(server_config.get("scopes")) or ( + mcp_oauth_metadata.scopes if mcp_oauth_metadata else None + ) resolved_authorization_url = server_config.get("authorization_url") or ( mcp_oauth_metadata.authorization_url if mcp_oauth_metadata else None ) @@ -729,6 +909,20 @@ class MCPServerManager: mcp_oauth_metadata.registration_url if mcp_oauth_metadata else None ) + config_oauth2_flow = server_config.get("oauth2_flow", None) + if auth_type == MCPAuth.oauth2 and config_oauth2_flow not in ( + "client_credentials", + "authorization_code", + ): + raise ValueError( + f"Invalid config for MCP server '{server_name or server_id}': auth_type oauth2 " + f"requires an explicit oauth2_flow (got {config_oauth2_flow!r}). Set " + "oauth2_flow: client_credentials for machine-to-machine servers (the proxy mints " + "a shared token at token_url using client_id/client_secret, no user interaction) " + "or oauth2_flow: authorization_code for interactive servers (per-user tokens via " + "browser sign-in, including delegate_auth_to_upstream)." + ) + new_server = MCPServer( server_id=server_id, name=name_for_prefix, @@ -742,14 +936,7 @@ class MCPServerManager: # oauth specific fields client_id=server_config.get("client_id", None), client_secret=server_config.get("client_secret", None), - oauth2_flow=self._resolve_oauth2_flow( - auth_type=auth_type, - oauth2_flow=server_config.get("oauth2_flow", None), - token_url=resolved_token_url, - authorization_url=resolved_authorization_url, - client_id=server_config.get("client_id", None), - client_secret=server_config.get("client_secret", None), - ), + oauth2_flow=self._explicit_oauth2_flow(config_oauth2_flow), scopes=resolved_scopes, authorization_url=resolved_authorization_url, token_url=resolved_token_url, @@ -787,9 +974,11 @@ class MCPServerManager: "subject_token_type", "urn:ietf:params:oauth:token-type:access_token", ), + token_exchange_profile=server_config.get("token_exchange_profile", "rfc8693"), allow_sampling=bool(server_config.get("allow_sampling", False)), allow_elicitation=bool(server_config.get("allow_elicitation", False)), timeout=server_config.get("timeout", None), + max_concurrent_requests=server_config.get("max_concurrent_requests", None), ) self._assign_unique_short_prefix(new_server) _warn_internal_delegate_pkce_if_applicable(new_server, source="config") @@ -851,7 +1040,7 @@ class MCPServerManager: server_prefix = get_server_prefix(server) # Build headers from server configuration - headers: Dict[str, str] = {} + headers: dict[str, str] = {} # Add authentication headers if configured if server.authentication_token: @@ -961,7 +1150,7 @@ class MCPServerManager: openapi_key_prefix = prefix_root + MCP_TOOL_PREFIX_SEPARATOR global_mcp_tool_registry.unregister_tools_with_prefix(openapi_key_prefix) - owned_raw: Set[str] = set() + owned_raw: set[str] = set() for p in iter_known_server_prefixes(server): if p: owned_raw.add(p) @@ -970,7 +1159,7 @@ class MCPServerManager: owned_normalized = {normalize_server_name(x) for x in owned_raw} - stale_mapping_keys: List[str] = [] + stale_mapping_keys: list[str] = [] for tool_name, mapped_server in list(self.tool_name_to_mcp_server_name_mapping.items()): if mapped_server in owned_raw: stale_mapping_keys.append(tool_name) @@ -998,7 +1187,7 @@ class MCPServerManager: mcp_server: LiteLLM_MCPServerTable, *, env_vars_are_encrypted: bool, - ) -> Optional[List[Dict[str, Any]]]: + ) -> Optional[list[dict[str, Any]]]: env_vars_list = _deserialize_json_list(getattr(mcp_server, "env_vars", None)) if env_vars_are_encrypted: from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 @@ -1073,7 +1262,7 @@ class MCPServerManager: # AWS SigV4 credential fields aws_creds = self._extract_aws_credentials(credentials_dict, credentials_are_encrypted) - scopes: Optional[List[str]] = None + scopes: Optional[list[str]] = None if credentials_dict: scopes_value = credentials_dict.get("scopes") if scopes_value is not None: @@ -1090,9 +1279,19 @@ class MCPServerManager: auth_type = cast(MCPAuthType, mcp_server.auth_type) server_url = mcp_server.url - needs_discovery = bool(server_url) and auth_type == MCPAuth.oauth2 and not mcp_server.authorization_url + needs_discovery = bool(server_url) and ( + (auth_type == MCPAuth.oauth2 and not mcp_server.authorization_url) + or self._obo_needs_endpoint_discovery( + auth_type, + credentials_dict.get("token_exchange_endpoint") if credentials_dict else None, + mcp_server.token_url, + ) + ) mcp_oauth_metadata = ( - await self._descovery_metadata(server_url=server_url) # type: ignore[arg-type] + await self._descovery_metadata( + server_url=server_url, # type: ignore[arg-type] + allow_origin_fallback=auth_type == MCPAuth.oauth2, + ) if needs_discovery else None ) @@ -1115,15 +1314,7 @@ class MCPServerManager: env_vars=env_vars_list, client_id=client_id_value or getattr(mcp_server, "client_id", None), client_secret=client_secret_value or getattr(mcp_server, "client_secret", None), - oauth2_flow=self._resolve_oauth2_flow( - auth_type=auth_type, - oauth2_flow=getattr(mcp_server, "oauth2_flow", None), - token_url=mcp_server.token_url or getattr(mcp_oauth_metadata, "token_url", None), - authorization_url=mcp_server.authorization_url - or getattr(mcp_oauth_metadata, "authorization_url", None), - client_id=client_id_value or getattr(mcp_server, "client_id", None), - client_secret=client_secret_value or getattr(mcp_server, "client_secret", None), - ), + oauth2_flow=self._explicit_oauth2_flow(getattr(mcp_server, "oauth2_flow", None)), scopes=resolved_scopes, authorization_url=mcp_server.authorization_url or getattr(mcp_oauth_metadata, "authorization_url", None), token_url=mcp_server.token_url or getattr(mcp_oauth_metadata, "token_url", None), @@ -1163,11 +1354,55 @@ class MCPServerManager: audience=(credentials_dict.get("audience") if credentials_dict else None), subject_token_type=(credentials_dict.get("subject_token_type") if credentials_dict else None) or "urn:ietf:params:oauth:token-type:access_token", + token_exchange_profile=(credentials_dict.get("token_exchange_profile") if credentials_dict else None) + or "rfc8693", timeout=getattr(mcp_server, "timeout", None), + max_concurrent_requests=getattr(mcp_server, "max_concurrent_requests", None), ) _warn_internal_delegate_pkce_if_applicable(new_server, source="database") + await self._persist_discovered_obo_token_url( + server_id=mcp_server.server_id, + auth_type=auth_type, + existing_token_url=mcp_server.token_url, + discovered_token_url=new_server.token_url, + ) return new_server + async def _persist_discovered_obo_token_url( + self, + *, + server_id: str, + auth_type: Optional[MCPAuthType], + existing_token_url: Optional[str], + discovered_token_url: Optional[str], + ) -> None: + """Write a freshly discovered OBO token endpoint back onto the DB row. + + ``build_mcp_server_from_table`` resolves ``token_url`` via RFC 9728 -> RFC 8414 for an + ``oauth2_token_exchange`` server that has none configured, but that resolved value otherwise + lives only on the returned in-memory object; the row keeps ``token_url=None`` so every rebuild + re-runs discovery, and a transient upstream outage during a rebuild leaves the server with no + endpoint until discovery next succeeds. Persisting it makes ``_obo_needs_endpoint_discovery`` + return False on the next build. Fires at most once per server (skipped once the row has a + value), and is best-effort: a write failure just means discovery runs again next time. + """ + if auth_type != MCPAuth.oauth2_token_exchange: + return + if existing_token_url or not discovered_token_url: + return + from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 + + if prisma_client is None: + return + try: + await MCPServerRepository(prisma_client).table.update( + where={"server_id": server_id}, + data={"token_url": discovered_token_url}, + ) + verbose_logger.debug("Persisted discovered OBO token_url for MCP server %s", server_id) + except Exception as exc: # noqa: BLE001 - best-effort; a failed write re-discovers next build + verbose_logger.warning("Failed to persist discovered OBO token_url for MCP server %s: %s", server_id, exc) + async def _maybe_register_openapi_tools(self, server: MCPServer, *, initialize_mapping: bool = True): """Register OpenAPI tools if the server has a spec_path configured.""" if server.spec_path: @@ -1235,14 +1470,14 @@ class MCPServerManager: verbose_logger.debug(f"Failed to udpate MCP server: {str(e)}") raise e - def get_all_mcp_server_ids(self) -> Set[str]: + def get_all_mcp_server_ids(self) -> set[str]: """ Get all MCP server IDs """ all_servers = list(self.get_registry().values()) return {server.server_id for server in all_servers} - def get_allow_all_keys_server_ids(self) -> List[str]: + def get_allow_all_keys_server_ids(self) -> list[str]: """Return server IDs that bypass per-key restrictions.""" return [server.server_id for server in self.get_registry().values() if server.allow_all_keys is True] @@ -1307,7 +1542,7 @@ class MCPServerManager: return [server_id for server_id in submitted_server_ids if self.get_mcp_server_by_id(server_id) is not None] - async def get_allowed_mcp_servers(self, user_api_key_auth: Optional[UserAPIKeyAuth] = None) -> List[str]: + async def get_allowed_mcp_servers(self, user_api_key_auth: Optional[UserAPIKeyAuth] = None) -> list[str]: """ Get the allowed MCP Servers for the user. @@ -1387,8 +1622,11 @@ class MCPServerManager: and getattr(server, "delegate_auth_to_upstream", False) is True # M2M servers must not be exposed anonymously: an # unauthenticated caller would get LiteLLM to proxy tool - # calls using its stored client_credentials. - and not server.has_client_credentials + # calls using its stored client_credentials. Resolve the flow + # rather than reading has_client_credentials so an unstamped + # M2M-shape row (null column, verbatim-read as non-M2M) still + # fails closed here, matching the anonymous-delegate auth gate. + and MCPServerManager.effective_oauth2_flow(server) != "client_credentials" ] combined_servers.update(delegate_server_ids) @@ -1404,8 +1642,8 @@ class MCPServerManager: async def resolve_toolset_tool_permissions( self, - toolset_ids: List[str], - ) -> Dict[str, List[str]]: + toolset_ids: list[str], + ) -> dict[str, list[str]]: """ Resolve a list of toolset IDs into a mcp_tool_permissions dict. @@ -1427,7 +1665,7 @@ class MCPServerManager: try: toolsets = await list_mcp_toolsets(prisma_client, toolset_ids=toolset_ids) - tool_permissions: Dict[str, List[str]] = {} + tool_permissions: dict[str, list[str]] = {} for toolset in toolsets: for tool in toolset.tools: raw_name = tool["tool_name"] @@ -1520,7 +1758,7 @@ class MCPServerManager: ) return toolset - def filter_server_ids_by_ip(self, server_ids: List[str], client_ip: Optional[str]) -> List[str]: + def filter_server_ids_by_ip(self, server_ids: list[str], client_ip: Optional[str]) -> list[str]: """ Filter server IDs by client IP — external callers only see public servers. @@ -1530,8 +1768,8 @@ class MCPServerManager: return filtered def filter_server_ids_by_ip_with_info( - self, server_ids: List[str], client_ip: Optional[str] - ) -> Tuple[List[str], int]: + self, server_ids: list[str], client_ip: Optional[str] + ) -> tuple[list[str], int]: """ Filter server IDs by client IP — external callers only see public servers. @@ -1551,7 +1789,7 @@ class MCPServerManager: blocked += 1 return allowed, blocked - async def get_tools_for_server(self, server_id: str) -> List[MCPTool]: + async def get_tools_for_server(self, server_id: str) -> list[MCPTool]: """ Get the tools for a given server """ @@ -1569,8 +1807,8 @@ class MCPServerManager: self, user_api_key_auth: Optional[UserAPIKeyAuth] = None, mcp_auth_header: Optional[str] = None, - mcp_server_auth_headers: Optional[Dict[str, Union[str, Dict[str, str]]]] = None, - ) -> List[MCPTool]: + mcp_server_auth_headers: Optional[dict[str, Union[str, dict[str, str]]]] = None, + ) -> list[MCPTool]: """ List all tools available across all MCP Servers. @@ -1587,7 +1825,7 @@ class MCPServerManager: verbose_logger.debug("SERVER MANAGER LISTING TOOLS") - async def _fetch_server_tools(server_id: str) -> List[MCPTool]: + async def _fetch_server_tools(server_id: str) -> list[MCPTool]: """Fetch tools from a single server with error handling.""" server = self.get_mcp_server_by_id(server_id) if server is None: @@ -1595,7 +1833,7 @@ class MCPServerManager: return [] # Get server-specific auth header if available - server_auth_header: Optional[Union[str, Dict[str, str]]] = None + server_auth_header: Optional[Union[str, dict[str, str]]] = None if mcp_server_auth_headers: from litellm.proxy._experimental.mcp_server.utils import ( lookup_mcp_server_auth_in_headers, @@ -1629,7 +1867,7 @@ class MCPServerManager: results = await asyncio.gather(*tasks) # Flatten results into single list - list_tools_result: List[MCPTool] = [tool for tools in results for tool in tools] + list_tools_result: list[MCPTool] = [tool for tools in results for tool in tools] verbose_logger.info(f"Successfully fetched {len(list_tools_result)} tools total from all servers") return list_tools_result @@ -1639,8 +1877,8 @@ class MCPServerManager: ######################################################### @staticmethod def _extract_bearer_token( - oauth2_headers: Optional[Dict[str, str]], - raw_headers: Optional[Dict[str, str]], + oauth2_headers: Optional[dict[str, str]], + raw_headers: Optional[dict[str, str]], ) -> Optional[str]: """Extract the bare Bearer token from oauth2_headers or raw_headers. @@ -1660,17 +1898,32 @@ class MCPServerManager: return auth_value return None + def _obo_subject_token( + self, + server: MCPServer, + raw_headers: Optional[dict[str, str]], + ) -> Optional[str]: + """The caller's bearer as the token_exchange (OBO) subject token, for that mode only. + + Prompts/resources discovery and reads on a token_exchange server must exchange the caller's + token like the tools paths do, not connect with no credential. Other modes never read the + inbound bearer, so return None to avoid forwarding it. + """ + if server.auth_type != MCPAuth.oauth2_token_exchange: + return None + return self._extract_bearer_token(None, raw_headers) + def _build_stdio_env( self, server: MCPServer, - raw_headers: Optional[Dict[str, str]] = None, - ) -> Optional[Dict[str, str]]: + raw_headers: Optional[dict[str, str]] = None, + ) -> Optional[dict[str, str]]: """Resolve stdio env values, supporting header-driven placeholders.""" if server.transport != MCPTransport.stdio or not server.env: return None - resolved_env: Dict[str, str] = {} + resolved_env: dict[str, str] = {} normalized_headers = {k.lower(): v for k, v in (raw_headers or {}).items()} for env_key, env_value in server.env.items(): @@ -1712,7 +1965,7 @@ class MCPServerManager: user_api_key_auth: Optional[UserAPIKeyAuth], *, raise_on_missing: bool = True, - ) -> Optional[Dict[str, str]]: + ) -> Optional[dict[str, str]]: """Return server.static_headers with ``${NAME}`` interpolated. Globals come from ``server.env_vars`` entries with ``scope=="global"``. @@ -1755,7 +2008,7 @@ class MCPServerManager: referenced_user_vars = referenced & user_var_names required_user_vars = {name for name in referenced_user_vars if name not in global_values} - user_values: Dict[str, str] = {} + user_values: dict[str, str] = {} if required_user_vars: try: user_values = await self._load_user_env_vars(server, user_api_key_auth) @@ -1794,7 +2047,7 @@ class MCPServerManager: # admin globals win, so a stale row from when a var was user-scoped can # never override the global value the admin set after switching it. scoped_user_values = {name: value for name, value in user_values.items() if name in user_var_names} - merged_vars: Dict[str, str] = {**scoped_user_values, **global_values} + merged_vars: dict[str, str] = {**scoped_user_values, **global_values} if not static_headers: return static_headers return interpolate_headers(static_headers, merged_vars) @@ -1805,7 +2058,7 @@ class MCPServerManager: user_api_key_auth: Optional[UserAPIKeyAuth], *, force_refresh: bool = False, - ) -> Dict[str, str]: + ) -> dict[str, str]: """Look up the calling user's env var values for ``server``. Returns an empty dict when no user is available. Results are cached in a @@ -1850,12 +2103,99 @@ class MCPServerManager: _write_user_env_vars_cache(user_id, server.server_id, values) return values + async def _resolve_v2_auth( + self, + *, + server: MCPServer, + spec: ServerSpec, + provider: UpstreamCredentialProvider, + subject_token: Optional[str], + user_api_key_auth: Optional[UserAPIKeyAuth], + extra_headers: Optional[dict[str, str]], + ) -> tuple[Optional[httpx.Auth], Optional[dict[str, str]]]: + """Resolve a v2-owned server's upstream credential into ``(resolved_auth, extra_headers)``. + + On a missing/rejected per-user credential this raises the mode's discovery challenge + (authorization_code's browser-OAuth 401, token_exchange's RFC 9728 challenge) or maps any + other ``CredError`` onto its public HTTP status; it never returns an error as a value. + """ + match await provider.resolve_credentials(to_subject(user_api_key_auth, subject_token), spec): + case Ok(auth): + # NoOpAuth has no header_name and so never conflicts. + header_name = getattr(auth, "header_name", None) + conflicts = bool( + header_name and extra_headers and any(key.lower() == header_name.lower() for key in extra_headers) + ) + if not conflicts: + return auth, extra_headers + if isinstance(spec.config, (TokenExchangeConfig, AuthorizationCodeConfig)): + # The resolver owns the per-user credential here (token_exchange's exchanged + # token, authorization_code's stored token). It is authoritative: a guardrail such + # as MCPJWTSigner, static_headers, or any other injected Authorization must NOT + # shadow it (otherwise the upstream gets e.g. the signer's JWT instead of the + # exchanged token and rejects it). Drop the conflicting header so the resolved + # token reaches upstream. + return auth, _without_authorization(extra_headers) + # Other modes: an Authorization already supplied via extra_headers (a forwarded caller + # header or static_headers) is intentional and wins; v1 applies those last. + return None, extra_headers + case Error(err): + if err.tag == "unauthorized" and isinstance(spec.config, AuthorizationCodeConfig): + # authorization_code's missing per-user token -> the per-server browser-OAuth + # challenge, built here where the full MCPServer is in hand. + raise_user_oauth_challenge(server, root_path=get_server_root_path()) + if err.tag == "unauthorized" and isinstance(spec.config, TokenExchangeConfig): + # token_exchange (OBO): a missing/rejected subject token -> the RFC 9728 challenge + # pointing at the IdP the client must SSO with to obtain one, rather than an opaque + # 401. No gateway-side browser flow. An IdP step-up rejection (Entra Conditional + # Access) threads its claims blob into the challenge for the client to satisfy. + raise_token_exchange_challenge( + server, + root_path=get_server_root_path(), + claims=err.unauthorized.claims, + ) + raise_public(err) + + async def preflight_token_exchange( + self, + server: MCPServer, + oauth2_headers: Optional[dict[str, str]], + user_api_key_auth: Optional[UserAPIKeyAuth], + ) -> None: + """Run the OBO exchange for a caller-supplied subject at the transport edge. + + Single-server routes call this before the MCP session opens, where an HTTP status and + ``WWW-Authenticate`` still reach the client. A rejected subject raises the RFC 9728 + challenge and any other ``CredError`` maps onto its public HTTP status, so an exchange + failure surfaces as a failure instead of the session continuing into an empty tool list. + A successful exchange is cached by the exchanger, so the session's list/call reuses it. + """ + if server.auth_type != MCPAuth.oauth2_token_exchange: + return + subject_token = self._extract_bearer_token(oauth2_headers, None) + if not subject_token: + return + spec = to_server_spec(server) + if spec is None or not isinstance(spec.config, TokenExchangeConfig): + return + match await self._cred_provider.resolve_credentials(to_subject(user_api_key_auth, subject_token), spec): + case Ok(_): + return + case Error(err): + if err.tag == "unauthorized": + raise_token_exchange_challenge( + server, + root_path=get_server_root_path(), + claims=err.unauthorized.claims, + ) + raise_public(err) + async def _create_mcp_client( self, server: MCPServer, - mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, - extra_headers: Optional[Dict[str, str]] = None, - stdio_env: Optional[Dict[str, str]] = None, + mcp_auth_header: Optional[Union[str, dict[str, str]]] = None, + extra_headers: Optional[dict[str, str]] = None, + stdio_env: Optional[dict[str, str]] = None, subject_token: Optional[str] = None, user_api_key_auth: Optional[UserAPIKeyAuth] = None, cred_provider: Optional[UpstreamCredentialProvider] = None, @@ -1884,11 +2224,17 @@ class MCPServerManager: spec = None if transport == MCPTransport.stdio else to_server_spec(server) provider = cred_provider or self._cred_provider # A caller-supplied per-request override (mcp_auth_header / x-mcp-*) defers to the v1 path - # so it wins - except for authorization_code, whose per-user token the v2 resolver owns. A - # caller must not be able to substitute another user's stored credential, so we keep the v2 - # spec and ignore the override there; the REST tools preview supplies its not-yet-persisted - # token through the resolver (cred_provider), never this path. - if spec is not None and mcp_auth_header and not isinstance(spec.config, AuthorizationCodeConfig): + # so it wins - except for the per-user modes the v2 resolver owns (authorization_code's + # stored token and token_exchange's RFC 8693 minted token). A caller must not be able to + # substitute another user's stored credential, nor silently disable the OBO exchange and + # forward an arbitrary bearer upstream, so we keep the v2 spec and ignore the override for + # both; the REST tools preview supplies its not-yet-persisted token through the resolver + # (cred_provider), never this path. + if ( + spec is not None + and mcp_auth_header + and not isinstance(spec.config, (AuthorizationCodeConfig, TokenExchangeConfig)) + ): spec = None auth_value = ( await resolve_mcp_auth(server, mcp_auth_header, subject_token=subject_token) if spec is None else None @@ -1954,26 +2300,14 @@ class MCPServerManager: server_url = server.url or "" if spec is not None: - match await provider.resolve_credentials(to_subject(user_api_key_auth, subject_token), spec): - case Ok(auth): - resolved_auth = auth - # Do not override an Authorization already supplied via extra_headers - # (a guardrail hook such as the JWT signer, static_headers, or a - # forwarded caller header): v1 applies those last, so they win. NoOpAuth - # has no header_name and so never skips. - header_name = getattr(resolved_auth, "header_name", None) - if ( - header_name - and extra_headers - and any(key.lower() == header_name.lower() for key in extra_headers) - ): - resolved_auth = None - case Error(err): - if err.tag == "unauthorized": - # The arm signals a missing per-user token semantically; raise the - # per-server OAuth challenge here, where the full MCPServer is in hand. - raise_user_oauth_challenge(server) - raise_public(err) + resolved_auth, extra_headers = await self._resolve_v2_auth( + server=server, + spec=spec, + provider=provider, + subject_token=subject_token, + user_api_key_auth=user_api_key_auth, + extra_headers=extra_headers, + ) return MCPClient( server_url=server_url, transport_type=transport, @@ -2013,12 +2347,13 @@ class MCPServerManager: async def _get_tools_from_server( self, server: MCPServer, - mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, - extra_headers: Optional[Dict[str, str]] = None, + mcp_auth_header: Optional[Union[str, dict[str, str]]] = None, + extra_headers: Optional[dict[str, str]] = None, add_prefix: bool = True, - raw_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[dict[str, str]] = None, user_api_key_auth: Optional[UserAPIKeyAuth] = None, - ) -> List[MCPTool]: + oauth2_headers: Optional[dict[str, str]] = None, + ) -> list[MCPTool]: """ Helper method to get tools from a single MCP server with prefixed names. @@ -2091,11 +2426,21 @@ class MCPServerManager: stdio_env = self._build_stdio_env(server, raw_headers) + # token_exchange (OBO) discovery needs the caller's token too: list it with the user's own + # token (mirrors the call path), not v1's deleted client_credentials fallback. Other modes + # never read the inbound bearer, so leave subject_token None to avoid forwarding it. + subject_token = ( + self._extract_bearer_token(oauth2_headers, raw_headers) + if server.auth_type == MCPAuth.oauth2_token_exchange + else None + ) + client = await self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, stdio_env=stdio_env, + subject_token=subject_token, user_api_key_auth=user_api_key_auth, ) @@ -2122,7 +2467,7 @@ class MCPServerManager: ] return tools else: - tools = await self._fetch_tools_with_timeout(client, server.name, server=server) + tools = await self._fetch_tools_with_timeout(client, server.name) self._remember_upstream_initialize_instructions(server, client) prefixed_or_original_tools = self._create_prefixed_tools(tools, server, add_prefix=add_prefix) @@ -2134,6 +2479,21 @@ class MCPServerManager: # client triggers the upstream OAuth flow. The multi-server # aggregator catches this explicitly to keep absorbing. raise + except HTTPException as e: + # A v2 resolver auth challenge (token_exchange's RFC 9728 401, authorization_code's + # browser-OAuth 401, or a 403) is raised at client-build time, inside this try. Route it + # through the same MCPUpstreamAuthError channel as pass-through so single-server routes + # surface the challenge (the client re-authenticates) while the aggregator keeps absorbing. + # Non-auth HTTP errors stay absorbed so one misconfigured server can't blank the listing. + if e.status_code in (401, 403): + headers = e.headers or {} + raise MCPUpstreamAuthError( + status_code=e.status_code, + www_authenticate=headers.get("WWW-Authenticate") or headers.get("www-authenticate"), + server_name=server.name, + ) from e + verbose_logger.warning(f"Failed to get tools from server {server.name}: {str(e)}") + return [] except Exception as e: verbose_logger.warning(f"Failed to get tools from server {server.name}: {str(e)}") return [] @@ -2141,11 +2501,11 @@ class MCPServerManager: async def get_prompts_from_server( self, server: MCPServer, - mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, - extra_headers: Optional[Dict[str, str]] = None, + mcp_auth_header: Optional[Union[str, dict[str, str]]] = None, + extra_headers: Optional[dict[str, str]] = None, add_prefix: bool = True, - raw_headers: Optional[Dict[str, str]] = None, - ) -> List[Prompt]: + raw_headers: Optional[dict[str, str]] = None, + ) -> list[Prompt]: """ Helper method to get prompts from a single MCP server with prefixed names. @@ -2169,12 +2529,14 @@ class MCPServerManager: extra_headers.update(server.static_headers) stdio_env = self._build_stdio_env(server, raw_headers) + subject_token = self._obo_subject_token(server, raw_headers) client = await self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, stdio_env=stdio_env, + subject_token=subject_token, ) prompts = await client.list_prompts() @@ -2190,11 +2552,11 @@ class MCPServerManager: async def get_resources_from_server( self, server: MCPServer, - mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, - extra_headers: Optional[Dict[str, str]] = None, + mcp_auth_header: Optional[Union[str, dict[str, str]]] = None, + extra_headers: Optional[dict[str, str]] = None, add_prefix: bool = True, - raw_headers: Optional[Dict[str, str]] = None, - ) -> List[Resource]: + raw_headers: Optional[dict[str, str]] = None, + ) -> list[Resource]: """Fetch available resources from a single MCP server.""" verbose_logger.debug(f"Connecting to url: {server.url}") @@ -2209,12 +2571,14 @@ class MCPServerManager: extra_headers.update(server.static_headers) stdio_env = self._build_stdio_env(server, raw_headers) + subject_token = self._obo_subject_token(server, raw_headers) client = await self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, stdio_env=stdio_env, + subject_token=subject_token, ) resources = await client.list_resources() @@ -2230,11 +2594,11 @@ class MCPServerManager: async def get_resource_templates_from_server( self, server: MCPServer, - mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, - extra_headers: Optional[Dict[str, str]] = None, + mcp_auth_header: Optional[Union[str, dict[str, str]]] = None, + extra_headers: Optional[dict[str, str]] = None, add_prefix: bool = True, - raw_headers: Optional[Dict[str, str]] = None, - ) -> List[ResourceTemplate]: + raw_headers: Optional[dict[str, str]] = None, + ) -> list[ResourceTemplate]: """Fetch available resource templates from a single MCP server.""" verbose_logger.debug(f"Connecting to url: {server.url}") @@ -2249,12 +2613,14 @@ class MCPServerManager: extra_headers.update(server.static_headers) stdio_env = self._build_stdio_env(server, raw_headers) + subject_token = self._obo_subject_token(server, raw_headers) client = await self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, stdio_env=stdio_env, + subject_token=subject_token, ) resource_templates = await client.list_resource_templates() @@ -2273,9 +2639,9 @@ class MCPServerManager: self, server: MCPServer, url: AnyUrl, - mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, - extra_headers: Optional[Dict[str, str]] = None, - raw_headers: Optional[Dict[str, str]] = None, + mcp_auth_header: Optional[Union[str, dict[str, str]]] = None, + extra_headers: Optional[dict[str, str]] = None, + raw_headers: Optional[dict[str, str]] = None, ) -> ReadResourceResult: """Read resource contents from a specific MCP server.""" @@ -2288,12 +2654,14 @@ class MCPServerManager: extra_headers.update(server.static_headers) stdio_env = self._build_stdio_env(server, raw_headers) + subject_token = self._obo_subject_token(server, raw_headers) client = await self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, stdio_env=stdio_env, + subject_token=subject_token, ) return await client.read_resource(url) @@ -2302,10 +2670,10 @@ class MCPServerManager: self, server: MCPServer, prompt_name: str, - arguments: Optional[Dict[str, Any]] = None, - mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, - extra_headers: Optional[Dict[str, str]] = None, - raw_headers: Optional[Dict[str, str]] = None, + arguments: Optional[dict[str, Any]] = None, + mcp_auth_header: Optional[Union[str, dict[str, str]]] = None, + extra_headers: Optional[dict[str, str]] = None, + raw_headers: Optional[dict[str, str]] = None, ) -> GetPromptResult: """Fetch a specific prompt definition from a single MCP server.""" @@ -2318,12 +2686,14 @@ class MCPServerManager: extra_headers.update(server.static_headers) stdio_env = self._build_stdio_env(server, raw_headers) + subject_token = self._obo_subject_token(server, raw_headers) client = await self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, stdio_env=stdio_env, + subject_token=subject_token, ) get_prompt_request_params = GetPromptRequestParams( @@ -2376,8 +2746,17 @@ class MCPServerManager: async def _descovery_metadata( self, server_url: str, + *, + allow_origin_fallback: bool = True, ) -> Optional[MCPOAuthMetadata]: - """Discover OAuth metadata by following RFC 9728 (protected resource metadata discovery).""" + """Discover OAuth metadata by following RFC 9728 (protected resource metadata discovery). + + ``allow_origin_fallback`` controls the last-resort guess that treats the resource server's own + origin as its authorization server when nothing is advertised. The browser ``oauth2`` flow keeps + it (a human sees the redirect), but token_exchange (OBO) sets it False so the gateway never + exchanges a subject token against an endpoint it inferred rather than one explicitly configured + or authoritatively advertised via RFC 9728 / RFC 8414. + """ try: client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) @@ -2427,7 +2806,7 @@ class MCPServerManager: ) = await self._attempt_well_known_discovery(server_url) metadata = None - if not authorization_servers: + if allow_origin_fallback and not authorization_servers: try: parsed_url = urlparse(server_url) if parsed_url.scheme and parsed_url.netloc: @@ -2449,7 +2828,7 @@ class MCPServerManager: verbose_logger.debug("MCP OAuth discovery failed for %s: %s", server_url, exc) return None - def _parse_www_authenticate_header(self, header_value: Optional[str]) -> Tuple[Optional[str], Optional[List[str]]]: + def _parse_www_authenticate_header(self, header_value: Optional[str]) -> tuple[Optional[str], Optional[list[str]]]: if not header_value: return None, None @@ -2457,7 +2836,7 @@ class MCPServerManager: params_section = params_section or header_value param_pattern = re.compile(r"([a-zA-Z0-9_]+)\s*=\s*\"?([^\",]+)\"?") - params: Dict[str, str] = { + params: dict[str, str] = { match.group(1).lower(): match.group(2).strip() for match in param_pattern.finditer(params_section) } @@ -2471,7 +2850,7 @@ class MCPServerManager: async def _fetch_oauth_metadata_from_resource( self, resource_metadata_url: str, server_url: str - ) -> Tuple[List[str], Optional[List[str]]]: + ) -> tuple[list[str], Optional[list[str]]]: if not resource_metadata_url: return [], None @@ -2506,7 +2885,7 @@ class MCPServerManager: return authorization_servers, scopes - async def _attempt_well_known_discovery(self, server_url: str) -> Tuple[List[str], Optional[List[str]]]: + async def _attempt_well_known_discovery(self, server_url: str) -> tuple[list[str], Optional[list[str]]]: try: parsed = urlparse(server_url) except Exception: @@ -2519,7 +2898,7 @@ class MCPServerManager: path = parsed.path or "" path = path.strip("/") - candidate_urls: List[str] = [] + candidate_urls: list[str] = [] if path: candidate_urls.append(f"{base}/.well-known/oauth-protected-resource/{path}") candidate_urls.append(f"{base}/.well-known/oauth-protected-resource") @@ -2535,7 +2914,7 @@ class MCPServerManager: return [], None async def _fetch_authorization_server_metadata( - self, authorization_servers: List[str], server_url: str + self, authorization_servers: list[str], server_url: str ) -> Optional[MCPOAuthMetadata]: for issuer in authorization_servers: metadata = await self._fetch_single_authorization_server_metadata(issuer, server_url) @@ -2557,7 +2936,7 @@ class MCPServerManager: base = f"{parsed.scheme}://{parsed.netloc}" path = (parsed.path or "").strip("/") - candidate_urls: List[str] = [] + candidate_urls: list[str] = [] if path: candidate_urls.append(f"{base}/.well-known/oauth-authorization-server/{path}") candidate_urls.append(f"{base}/.well-known/openid-configuration/{path}") @@ -2589,6 +2968,14 @@ class MCPServerManager: continue scopes = self._extract_scopes(data.get("scopes_supported")) + verbose_logger.debug( + "Authorization server metadata from %s: issuer=%s grant_types_supported=%s " + "token_endpoint_auth_methods_supported=%s", + url, + data.get("issuer"), + data.get("grant_types_supported"), + data.get("token_endpoint_auth_methods_supported"), + ) metadata = MCPOAuthMetadata( scopes=scopes, authorization_url=data.get("authorization_endpoint"), @@ -2643,9 +3030,9 @@ class MCPServerManager: def _extract_aws_credentials( self, - credentials_dict: Optional[Dict[str, str]], + credentials_dict: Optional[dict[str, str]], credentials_are_encrypted: bool, - ) -> Dict[str, Optional[str]]: + ) -> dict[str, Optional[str]]: """Extract and decrypt AWS SigV4 credential fields from credentials dict.""" if not credentials_dict: return {} @@ -2671,7 +3058,7 @@ class MCPServerManager: "aws_session_name": credentials_dict.get("aws_session_name"), } - def _extract_scopes(self, scopes_value: Any) -> Optional[List[str]]: + def _extract_scopes(self, scopes_value: Any) -> Optional[list[str]]: if isinstance(scopes_value, str): scopes = [s.strip() for s in scopes_value.split() if s.strip()] return scopes or None @@ -2684,46 +3071,34 @@ class MCPServerManager: self, client: MCPClient, server_name: str, - server: Optional[MCPServer] = None, - ) -> List[MCPTool]: + ) -> list[MCPTool]: """ Fetch tools from MCP client with timeout and error handling. Uses anyio.fail_after() instead of asyncio.wait_for() to avoid conflicts with the MCP SDK's anyio TaskGroup. See GitHub issue #20715 for details. - For OAuth pass-through and upstream-delegated OAuth2 MCP servers, an - upstream HTTP 401 is converted into :class:`MCPUpstreamAuthError` - instead of being swallowed to an empty tool list. That lets the - single-server HTTP routes surface a proper 401 + ``WWW-Authenticate`` - challenge so standards-compliant MCP clients trigger the upstream - OAuth flow. Other servers keep today's swallow-and-log behaviour so - the multi-server ``/mcp`` aggregator doesn't get tainted by a single - bad server. + An upstream HTTP 401 is converted into :class:`MCPUpstreamAuthError` + instead of being swallowed to an empty tool list, regardless of the + server's auth_type. Callers route it by surface: the single-server HTTP + routes turn it into a 401 + ``WWW-Authenticate`` challenge so standards- + compliant MCP clients trigger the upstream OAuth flow, while the + multi-server ``/mcp`` aggregator absorbs it to an empty list so one + unauthenticated server doesn't fail the whole listing. Only a 401 + (missing/invalid credential) drives the re-auth challenge; a 403 + (authenticated but forbidden, e.g. insufficient scope) is not a re-auth + signal and, like other non-auth errors, returns an empty list. Args: client: MCP client instance server_name: Name of the server for logging - server: Optional MCPServer; when upstream auth is delegated, auth - errors are re-raised as :class:`MCPUpstreamAuthError`. Returns: List of tools from the server """ - should_surface_upstream_auth = bool( - server is not None - and ( - server.is_oauth_passthrough - or ( - server.auth_type == MCPAuth.oauth2 - and getattr(server, "delegate_auth_to_upstream", False) is True - and not server.has_client_credentials - ) - ) - ) try: with anyio.fail_after(MCP_TOOL_LISTING_TIMEOUT): - tools = await client.list_tools(raise_on_error=should_surface_upstream_auth) + tools = await client.list_tools(raise_on_error=True) verbose_logger.debug(f"Tools from {server_name}: {tools}") return tools except TimeoutError: @@ -2736,16 +3111,15 @@ class MCPServerManager: verbose_logger.warning(f"Connection error while listing tools from {server_name}: {str(e)}") return [] except Exception as e: - if should_surface_upstream_auth: - auth_info = _extract_upstream_auth_failure(e) - if auth_info is not None: - status_code, www_authenticate = auth_info - verbose_logger.info(f"Upstream auth failure from MCP server {server_name}: HTTP {status_code}") - raise MCPUpstreamAuthError( - status_code=status_code, - www_authenticate=www_authenticate, - server_name=server_name, - ) from e + auth_info = _extract_upstream_auth_failure(e) + if auth_info is not None and auth_info[0] == 401: + _, www_authenticate = auth_info + verbose_logger.info(f"Upstream auth failure from MCP server {server_name}: HTTP 401") + raise MCPUpstreamAuthError( + status_code=401, + www_authenticate=www_authenticate, + server_name=server_name, + ) from e verbose_logger.warning(f"Error listing tools from {server_name}: {str(e)}") return [] @@ -2754,7 +3128,7 @@ class MCPServerManager: def _assign_unique_short_prefix( self, server: MCPServer, - registry: Optional[Dict[str, MCPServer]] = None, + registry: Optional[dict[str, MCPServer]] = None, ) -> None: """Resolve and cache a collision-free short tool prefix on ``server``. @@ -2778,7 +3152,7 @@ class MCPServerManager: if not server.server_id: return - used: Dict[str, str] = {} + used: dict[str, str] = {} registry_for_collision_check = registry or self.get_registry() for other in registry_for_collision_check.values(): if other.server_id == server.server_id: @@ -2811,7 +3185,7 @@ class MCPServerManager: "attempts; the 3-character prefix space is too crowded." ) - def _create_prefixed_tools(self, tools: List[MCPTool], server: MCPServer, add_prefix: bool = True) -> List[MCPTool]: + def _create_prefixed_tools(self, tools: list[MCPTool], server: MCPServer, add_prefix: bool = True) -> list[MCPTool]: """ Create prefixed tools and update tool mapping. @@ -2849,8 +3223,8 @@ class MCPServerManager: return prefixed_tools def _create_prefixed_prompts( - self, prompts: List[Prompt], server: MCPServer, add_prefix: bool = True - ) -> List[Prompt]: + self, prompts: list[Prompt], server: MCPServer, add_prefix: bool = True + ) -> list[Prompt]: """ Create prefixed prompts and update prompt mapping. @@ -2876,11 +3250,11 @@ class MCPServerManager: return prefixed_prompts def _create_prefixed_resources( - self, resources: List[Resource], server: MCPServer, add_prefix: bool = True - ) -> List[Resource]: + self, resources: list[Resource], server: MCPServer, add_prefix: bool = True + ) -> list[Resource]: """Prefix resource names and track origin server for read requests.""" - prefixed_resources: List[Resource] = [] + prefixed_resources: list[Resource] = [] prefix = get_server_prefix(server) for resource in resources: @@ -2893,13 +3267,13 @@ class MCPServerManager: def _create_prefixed_resource_templates( self, - resource_templates: List[ResourceTemplate], + resource_templates: list[ResourceTemplate], server: MCPServer, add_prefix: bool = True, - ) -> List[ResourceTemplate]: + ) -> list[ResourceTemplate]: """Prefix resource template names for multi-server scenarios.""" - prefixed_templates: List[ResourceTemplate] = [] + prefixed_templates: list[ResourceTemplate] = [] prefix = get_server_prefix(server) for resource_template in resource_templates: @@ -2932,7 +3306,7 @@ class MCPServerManager: ) return True - def validate_allowed_params(self, tool_name: str, arguments: Dict[str, Any], server: MCPServer) -> None: + def validate_allowed_params(self, tool_name: str, arguments: dict[str, Any], server: MCPServer) -> None: """ Filter arguments to only include allowed parameters for the given tool. @@ -3023,7 +3397,7 @@ class MCPServerManager: self, server: MCPServer, tool_name: str, - arguments: Dict[str, Any], + arguments: dict[str, Any], ) -> CallToolResult: """ Call an OpenAPI tool handler directly. @@ -3080,13 +3454,13 @@ class MCPServerManager: async def pre_call_tool_check( self, name: str, - arguments: Dict[str, Any], + arguments: dict[str, Any], server_name: str, user_api_key_auth: Optional[UserAPIKeyAuth], proxy_logging_obj: ProxyLogging, server: MCPServer, - raw_headers: Optional[Dict[str, str]] = None, - ) -> Dict[str, Any]: + raw_headers: Optional[dict[str, str]] = None, + ) -> dict[str, Any]: """ Run pre-call checks and guardrail hooks for an MCP tool call. @@ -3146,7 +3520,7 @@ class MCPServerManager: # Convert to LLM format for existing guardrail compatibility synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(mcp_request_obj, pre_hook_kwargs) - hook_result: Dict[str, Any] = {} + hook_result: dict[str, Any] = {} try: # Use standard pre_call_hook modified_data = await proxy_logging_obj.pre_call_hook( @@ -3176,7 +3550,7 @@ class MCPServerManager: def _create_during_hook_task( self, name: str, - arguments: Dict[str, Any], + arguments: dict[str, Any], server_name_from_prefix: Optional[str], user_api_key_auth: Optional[UserAPIKeyAuth], proxy_logging_obj: ProxyLogging, @@ -3211,19 +3585,78 @@ class MCPServerManager: ) ) + def _get_call_semaphore(self, mcp_server: MCPServer) -> Optional[asyncio.Semaphore]: + limit = mcp_server.max_concurrent_requests + if limit is None or limit <= 0: + return None + semaphore = self._server_call_semaphores.get(mcp_server.server_id) + if semaphore is None: + semaphore = asyncio.Semaphore(limit) + self._server_call_semaphores[mcp_server.server_id] = semaphore + return semaphore + + @asynccontextmanager + async def _limit_outbound_concurrency(self, mcp_server: MCPServer) -> AsyncIterator[None]: + semaphore = self._get_call_semaphore(mcp_server) + if semaphore is None: + yield + return + async with semaphore: + yield + + async def _obo_call_tool_with_retry( + self, + *, + client: MCPClient, + call_tool_params: MCPCallToolRequestParams, + host_progress_callback: Optional[Callable], + mcp_server: MCPServer, + server_auth_header: str | dict[str, str] | None, + extra_headers: Optional[dict[str, str]], + stdio_env: Optional[dict[str, str]], + subject_token: Optional[str], + user_api_key_auth: Optional[UserAPIKeyAuth], + ) -> CallToolResult: + """Call a token_exchange (OBO) tool; on an upstream 401/403 re-mint the token once and retry. + + The exchanged token is baked into the client at build time, so the retry invalidates the + cached exchange and rebuilds the client (which re-exchanges). One retry only: a non-auth + failure or a second auth failure degrades to the normal ``isError`` result, and a re-exchange + that now fails surfaces its own 401 challenge from ``_create_mcp_client``. + """ + try: + return await client.call_tool( + call_tool_params, host_progress_callback=host_progress_callback, raise_on_error=True + ) + except Exception as exc: + if _extract_upstream_auth_failure(exc) is None: + return MCPClient.error_tool_result(exc) + spec = to_server_spec(mcp_server) + if spec is not None: + await self._cred_provider.invalidate_credentials(to_subject(user_api_key_auth, subject_token), spec) + retry_client = await self._create_mcp_client( + server=mcp_server, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + stdio_env=stdio_env, + subject_token=subject_token, + user_api_key_auth=user_api_key_auth, + ) + return await retry_client.call_tool(call_tool_params, host_progress_callback=host_progress_callback) + async def _call_regular_mcp_tool( self, mcp_server: MCPServer, original_tool_name: str, - arguments: Dict[str, Any], - tasks: List, + arguments: dict[str, Any], + tasks: list, mcp_auth_header: Optional[str], - mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]], - oauth2_headers: Optional[Dict[str, str]], - raw_headers: Optional[Dict[str, str]], + mcp_server_auth_headers: Optional[dict[str, dict[str, str]]], + oauth2_headers: Optional[dict[str, str]], + raw_headers: Optional[dict[str, str]], proxy_logging_obj: Optional[ProxyLogging], host_progress_callback: Optional[Callable] = None, - hook_extra_headers: Optional[Dict[str, str]] = None, + hook_extra_headers: Optional[dict[str, str]] = None, user_api_key_auth: Optional[UserAPIKeyAuth] = None, ) -> CallToolResult: """ @@ -3254,7 +3687,7 @@ class MCPServerManager: # Get server-specific auth header if available (case-insensitive) # FIX: Added case-insensitive matching to handle auth header keys that may not match # the exact case of server alias/name (e.g., '1litellmagcgateway' vs '1LiteLLMAGCGateway') - server_auth_header: Optional[Union[Dict[str, str], str]] = None + server_auth_header: Optional[Union[dict[str, str], str]] = None if mcp_server_auth_headers: # Normalize keys for case-insensitive lookup from litellm.proxy._experimental.mcp_server.utils import ( @@ -3273,7 +3706,7 @@ class MCPServerManager: # Extract subject token for OAuth2 Token Exchange (OBO) flow subject_token: Optional[str] = None - extra_headers: Optional[Dict[str, str]] = None + extra_headers: Optional[dict[str, str]] = None if mcp_server.auth_type == MCPAuth.oauth2_token_exchange: subject_token = self._extract_bearer_token(oauth2_headers, raw_headers) elif mcp_server.auth_type == MCPAuth.oauth2: @@ -3369,10 +3802,30 @@ class MCPServerManager: arguments=arguments, ) - async def _call_tool_via_client(client, params): - return await client.call_tool(params, host_progress_callback=host_progress_callback) + if mcp_server.auth_type == MCPAuth.oauth2_token_exchange and subject_token: + # OBO: the exchanged token may have been revoked/rotated upstream since it was cached, so + # an upstream 401 gets one re-mint + retry. Gated to this mode; all others keep the plain + # single call below. + tool_call_coro = self._obo_call_tool_with_retry( + client=client, + call_tool_params=call_tool_params, + host_progress_callback=host_progress_callback, + mcp_server=mcp_server, + server_auth_header=server_auth_header, + extra_headers=extra_headers, + stdio_env=stdio_env, + subject_token=subject_token, + user_api_key_auth=user_api_key_auth, + ) + else: - tasks.append(asyncio.create_task(_call_tool_via_client(client, call_tool_params))) + async def _call_tool_via_client(client, params): + async with self._limit_outbound_concurrency(mcp_server): + return await client.call_tool(params, host_progress_callback=host_progress_callback) + + tool_call_coro = _call_tool_via_client(client, call_tool_params) + + tasks.append(asyncio.create_task(tool_call_coro)) _timeout = mcp_server.timeout if mcp_server.timeout is not None else MCP_CLIENT_TIMEOUT try: @@ -3460,9 +3913,9 @@ class MCPServerManager: async def _resolve_oauth2_headers_for_tool_call( self, mcp_server: MCPServer, - oauth2_headers: Optional[Dict[str, str]], + oauth2_headers: Optional[dict[str, str]], user_api_key_auth: Optional[UserAPIKeyAuth], - ) -> Optional[Dict[str, str]]: + ) -> Optional[dict[str, str]]: """Look up per-user OAuth headers when the client did not supply a token.""" if not mcp_server.needs_user_oauth_token or oauth2_headers or user_api_key_auth is None: return oauth2_headers @@ -3499,7 +3952,7 @@ class MCPServerManager: async def _gather_openapi_tool_tasks( self, - tasks: List[Any], + tasks: list[Any], proxy_logging_obj: Optional[ProxyLogging], ) -> CallToolResult: """Await OpenAPI tool tasks and return the tool call result.""" @@ -3519,13 +3972,13 @@ class MCPServerManager: self, server_name: str, name: str, - arguments: Dict[str, Any], + arguments: dict[str, Any], user_api_key_auth: Optional[UserAPIKeyAuth] = None, mcp_auth_header: Optional[str] = None, - mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + mcp_server_auth_headers: Optional[dict[str, dict[str, str]]] = None, proxy_logging_obj: Optional[ProxyLogging] = None, - oauth2_headers: Optional[Dict[str, str]] = None, - raw_headers: Optional[Dict[str, str]] = None, + oauth2_headers: Optional[dict[str, str]] = None, + raw_headers: Optional[dict[str, str]] = None, host_progress_callback: Optional[Callable] = None, ) -> CallToolResult: """ @@ -3547,12 +4000,21 @@ class MCPServerManager: start_time = datetime.datetime.now() mcp_server = self._resolve_mcp_server_for_tool_call(server_name, name) + # Resolved before any hook runs so a missing BYOK credential (401) never + # leaves during-hook side effects (audit logging, rate-limit bookkeeping) + # recorded against a call that ultimately fails. + mcp_auth_header = await _resolve_byok_mcp_auth_header( + mcp_server, + user_api_key_auth, + mcp_auth_header, + ) + ######################################################### # Pre MCP Tool Call Hook # Allow validation and modification of tool calls before execution # Using standard pre_call_hook ######################################################### - hook_result: Dict[str, Any] = {} + hook_result: dict[str, Any] = {} if proxy_logging_obj: hook_result = await self.pre_call_tool_check( name=name, @@ -3592,7 +4054,28 @@ class MCPServerManager: "transport to enable hook header injection.", server_name, ) - tasks.append(asyncio.create_task(self._call_openapi_tool_handler(mcp_server, name, arguments))) + + auth_header_value = ( + _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None + ) + forwarded_headers = _openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth) + + async def _call_openapi_via_handler(): + from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + _request_auth_header, + _request_extra_headers, + ) + + auth_token = _request_auth_header.set(auth_header_value) + extra_token = _request_extra_headers.set(forwarded_headers) + try: + async with self._limit_outbound_concurrency(mcp_server): + return await self._call_openapi_tool_handler(mcp_server, name, arguments) + finally: + _request_auth_header.reset(auth_token) + _request_extra_headers.reset(extra_token) + + tasks.append(asyncio.create_task(_call_openapi_via_handler())) else: return await self._call_regular_mcp_tool( mcp_server=mcp_server, @@ -3673,7 +4156,7 @@ class MCPServerManager: # Build prefix → server lookup covering every known form a tool name # may take (alias / server_name / server_id / short ID). This is what # makes the short-prefix mode work without breaking historical names. - prefix_to_server: Dict[str, MCPServer] = {} + prefix_to_server: dict[str, MCPServer] = {} for server in registry_servers: for known_prefix in iter_known_server_prefixes(server): normalised = normalize_server_name(known_prefix) @@ -3734,7 +4217,7 @@ class MCPServerManager: verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database") previous_registry = self.registry - new_registry: Dict[str, MCPServer] = {} + new_registry: dict[str, MCPServer] = {} # Stage one: build every server. Stage two assigns short prefixes # against the *full* set so dedup is deterministic regardless of @@ -3780,7 +4263,7 @@ class MCPServerManager: # Assign short prefixes against the full candidate set without # publishing the staged registry to concurrent callers. - registered_registry: Dict[str, MCPServer] = {} + registered_registry: dict[str, MCPServer] = {} registered_openapi_tools = False for server_id, new_server in new_registry.items(): try: @@ -3806,7 +4289,7 @@ class MCPServerManager: verbose_logger.debug("MCP registry refreshed (%s servers in registry)", len(registered_registry)) - def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]: + def get_mcp_servers_from_ids(self, server_ids: list[str]) -> list[MCPServer]: servers = [] registry = self.get_registry() for server in registry.values(): @@ -3814,7 +4297,7 @@ class MCPServerManager: servers.append(server) return servers - def _get_general_settings(self) -> Dict[str, Any]: + def _get_general_settings(self) -> dict[str, Any]: """Get general_settings, importing lazily to avoid circular imports.""" try: from litellm.proxy.proxy_server import ( @@ -3857,7 +4340,7 @@ class MCPServerManager: return server return None - def get_public_mcp_servers(self) -> List[MCPServer]: + def get_public_mcp_servers(self) -> list[MCPServer]: """ Return the MCP servers published to the AI Hub via /v1/mcp/make_public. @@ -3887,7 +4370,7 @@ class MCPServerManager: if server.available_on_public_internet or server.server_id in public_ids ] - def expand_permission_list(self, identifiers: List[str]) -> List[str]: + def expand_permission_list(self, identifiers: list[str]) -> list[str]: """ Expand a permission list of server_ids/names/aliases into concrete server_ids against the current region's config + DB registry union. @@ -3903,12 +4386,12 @@ class MCPServerManager: if not identifiers: return [] registry = self.get_registry() - expanded: Set[str] = set() + expanded: set[str] = set() for identifier in identifiers: if identifier in registry: expanded.add(identifier) continue - matches: List[str] = [ + matches: list[str] = [ server_id for server_id, server in registry.items() if server.alias == identifier or server.server_name == identifier or server.name == identifier @@ -3929,8 +4412,8 @@ class MCPServerManager: def expand_tool_permissions( self, - tool_permissions: Optional[Dict[str, List[str]]], - ) -> Dict[str, List[str]]: + tool_permissions: Optional[dict[str, list[str]]], + ) -> dict[str, list[str]]: """ Rewrite an ``mcp_tool_permissions`` dict keyed by id/name/alias so every key is a concrete server_id where possible. Tool lists from @@ -3945,7 +4428,7 @@ class MCPServerManager: """ if not tool_permissions: return {} - result: Dict[str, List[str]] = {} + result: dict[str, list[str]] = {} for key, tools in tool_permissions.items(): for server_id in self.expand_permission_list([key]): result.setdefault(server_id, []).extend(tools or []) @@ -3986,7 +4469,7 @@ class MCPServerManager: return server return None - def get_filtered_registry(self, client_ip: Optional[str] = None) -> Dict[str, MCPServer]: + def get_filtered_registry(self, client_ip: Optional[str] = None) -> dict[str, MCPServer]: """ Get registry filtered by client IP access control. @@ -4146,16 +4629,18 @@ class MCPServerManager: authorization_url=server.authorization_url, token_url=server.token_url, registration_url=server.registration_url, + oauth2_flow=server.oauth2_flow, allow_all_keys=server.allow_all_keys, instructions=server.instructions, timeout=server.timeout, + max_concurrent_requests=server.max_concurrent_requests, ) async def get_all_mcp_servers_with_health_and_teams( self, user_api_key_auth: Optional[UserAPIKeyAuth] = None, - server_ids: Optional[List[str]] = None, - ) -> List[LiteLLM_MCPServerTable]: + server_ids: Optional[list[str]] = None, + ) -> list[LiteLLM_MCPServerTable]: """ Get all MCP servers that the user has access to, with health status and team information. @@ -4184,7 +4669,7 @@ class MCPServerManager: async def get_all_allowed_mcp_servers( self, user_api_key_auth: Optional[UserAPIKeyAuth] = None, - ) -> List[LiteLLM_MCPServerTable]: + ) -> list[LiteLLM_MCPServerTable]: """ Get all MCP servers that the user has access to. @@ -4197,7 +4682,7 @@ class MCPServerManager: # Get allowed server IDs allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth) - list_mcp_servers: List[LiteLLM_MCPServerTable] = [] + list_mcp_servers: list[LiteLLM_MCPServerTable] = [] for server_id in allowed_server_ids: server = self.get_mcp_server_by_id(server_id) @@ -4212,8 +4697,8 @@ class MCPServerManager: @staticmethod def _env_vars_to_models( - env_vars: Optional[List[Dict[str, Any]]], - ) -> Optional[List[MCPEnvVar]]: + env_vars: Optional[list[dict[str, Any]]], + ) -> Optional[list[MCPEnvVar]]: if env_vars is None: return None return [MCPEnvVar.model_validate(env_var) for env_var in env_vars] @@ -4233,6 +4718,8 @@ class MCPServerManager: teams=[], mcp_access_groups=server.access_groups or [], allowed_tools=server.allowed_tools or [], + tool_name_to_display_name=server.tool_name_to_display_name, + tool_name_to_description=server.tool_name_to_description, extra_headers=server.extra_headers or [], mcp_info=server.mcp_info, static_headers=server.static_headers, @@ -4246,6 +4733,7 @@ class MCPServerManager: authorization_url=server.authorization_url, token_url=server.token_url, registration_url=server.registration_url, + oauth2_flow=server.oauth2_flow, allow_all_keys=server.allow_all_keys, available_on_public_internet=server.available_on_public_internet, delegate_auth_to_upstream=server.delegate_auth_to_upstream, @@ -4256,23 +4744,24 @@ class MCPServerManager: source_url=server.source_url, instructions=server.instructions, timeout=server.timeout, + max_concurrent_requests=server.max_concurrent_requests, ) - async def get_all_mcp_servers_unfiltered(self) -> List[LiteLLM_MCPServerTable]: + async def get_all_mcp_servers_unfiltered(self) -> list[LiteLLM_MCPServerTable]: """Return all MCP servers from registry without applying access controls.""" registry = self.get_registry() if not registry: return [] - servers: List[LiteLLM_MCPServerTable] = [] + servers: list[LiteLLM_MCPServerTable] = [] for server in registry.values(): servers.append(self._build_mcp_server_table(server)) return servers async def get_all_mcp_servers_with_health_unfiltered( - self, server_ids: Optional[List[str]] = None - ) -> List[LiteLLM_MCPServerTable]: + self, server_ids: Optional[list[str]] = None + ) -> list[LiteLLM_MCPServerTable]: """Return health info for all servers in registry regardless of user access.""" registry = self.get_registry() @@ -4289,7 +4778,7 @@ class MCPServerManager: return await self._run_health_checks(target_server_ids) - async def _run_health_checks(self, target_server_ids: List[str]) -> List[LiteLLM_MCPServerTable]: + async def _run_health_checks(self, target_server_ids: list[str]) -> list[LiteLLM_MCPServerTable]: if not target_server_ids: return [] diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_flow_backfill.py b/litellm/proxy/_experimental/mcp_server/oauth2_flow_backfill.py new file mode 100644 index 00000000000..02cec2475e2 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/oauth2_flow_backfill.py @@ -0,0 +1,155 @@ +"""Startup backfill for oauth2 MCP server rows persisted before oauth2_flow was written. + +Rows created before the write-side stamps (DCR persist, UI create, REST create) carry a +null ``oauth2_flow`` and rely on read-time field-shape inference, which cannot tell a +DCR-registered interactive server from an M2M server unless endpoint discovery succeeds +first. This backfill classifies each null row once, at rest, using signals inference +never had, and persists the result so the read path never has to infer again. + +Signal order, strongest first: + +1. Per-user OAuth token rows exist for the server: only the interactive flow mints + per-user tokens, so this is definitive and immune to the discovery trap. BYOK API + keys share the same table (``LiteLLM_MCPUserCredentials``), so only rows whose + payload decodes as a ``type: oauth2`` token count as proof; bare keys and + undecodable rows prove nothing about the flow. +2. ``authorization_url`` persisted: interactive needs a user-facing authorization + endpoint; M2M (RFC 6749 section 4.4) never has one. +3. ``registration_url`` persisted: dynamic client registration (RFC 7591) exists to mint + clients for the interactive flow; M2M servers are configured with static credentials. +4. ``token_url`` plus decryptable ``client_id`` and ``client_secret``: ambiguous, left + unstamped. The shape is shared by M2M servers and DCR-registered interactive servers + whose authorization endpoint lives only in discovery (registered but never signed + in), so stamping client_credentials here could permanently route per-user traffic + through the proxy's stored client credential. The row keeps working through the + request-time backstop and a warning names it with the one-line fix (set oauth2_flow + via the dashboard or ``PUT /v1/mcp/server``); a completed interactive sign-in also + heals it via rule 1 at the next boot. +5. Anything else is interactive: matching how ``needs_user_oauth_token`` treats a null + flow, so the stamp never changes runtime routing for rows no rule recognizes. + +The backfill never stamps client_credentials: M2M is asserted by a human (config +requires it, the API accepts it, the dashboard sets it), mirroring the config-level +validation error. Runs before the first registry load on every boot and is idempotent: +a healed fleet has no null rows and the backfill exits after one query. +""" + +import json +from collections import Counter +from typing import Any, Literal, Optional + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._experimental.mcp_server.db import _decode_oauth_payload, decrypt_credentials +from litellm.proxy.utils import PrismaClient +from litellm.types.mcp import MCPCredentials + +OAuth2Flow = Literal["client_credentials", "authorization_code"] +BackfillRule = Literal[ + "per_user_tokens", + "authorization_url", + "registration_url", + "ambiguous_m2m_shape", + "interactive_default", +] + +_BACKFILL_AUDIT_ACTOR = "oauth2_flow_backfill" + + +def _decrypted_credentials(raw_credentials: Any) -> Optional[MCPCredentials]: + if raw_credentials is None: + return None + if isinstance(raw_credentials, str): + try: + parsed = json.loads(raw_credentials) + except (ValueError, TypeError): + return None + else: + parsed = raw_credentials + if not isinstance(parsed, dict): + return None + return decrypt_credentials(credentials=dict(parsed)) + + +def classify_null_flow_row( + *, + has_per_user_tokens: bool, + authorization_url: Optional[str], + registration_url: Optional[str], + token_url: Optional[str], + credentials: Optional[MCPCredentials], +) -> tuple[Optional[OAuth2Flow], BackfillRule]: + if has_per_user_tokens: + return "authorization_code", "per_user_tokens" + if authorization_url: + return "authorization_code", "authorization_url" + if registration_url: + return "authorization_code", "registration_url" + if token_url and credentials and credentials.get("client_id") and credentials.get("client_secret"): + return None, "ambiguous_m2m_shape" + return "authorization_code", "interactive_default" + + +async def backfill_null_oauth2_flows(prisma_client: PrismaClient) -> dict[BackfillRule, int]: + """Classify every ``auth_type=oauth2`` row whose ``oauth2_flow`` is null; stamp the provable + ones, warn on the ambiguous ones, and return counts per rule.""" + null_rows: list[Any] = await prisma_client.db.litellm_mcpservertable.find_many( + where={"auth_type": "oauth2", "oauth2_flow": None}, + ) + if not null_rows: + return {} + + server_ids = [row.server_id for row in null_rows] + token_rows: list[Any] = await prisma_client.db.litellm_mcpusercredentials.find_many( + where={"server_id": {"in": server_ids}}, + ) + server_ids_with_oauth_tokens: set[str] = { + token_row.server_id for token_row in token_rows if _decode_oauth_payload(token_row.credential_b64) is not None + } + + classified = tuple( + ( + row, + classify_null_flow_row( + has_per_user_tokens=row.server_id in server_ids_with_oauth_tokens, + authorization_url=row.authorization_url, + registration_url=row.registration_url, + token_url=row.token_url, + credentials=_decrypted_credentials(row.credentials), + ), + ) + for row in null_rows + ) + + for row, (flow, rule) in classified: + if flow is None: + verbose_proxy_logger.warning( + "oauth2_flow backfill: server_id=%s is ambiguous (client credentials + token_url, " + "no interactive signal); left unstamped. Set oauth2_flow explicitly via the " + "dashboard or PUT /v1/mcp/server: client_credentials if this server is M2M, or " + "complete an interactive sign-in and it will be stamped authorization_code at the " + "next boot.", + row.server_id, + ) + else: + verbose_proxy_logger.info( + "oauth2_flow backfill: server_id=%s stamped %s (rule=%s)", + row.server_id, + flow, + rule, + ) + + stamped_flows = {flow for _, (flow, _) in classified if flow is not None} + for stamped_flow in stamped_flows: + server_ids_for_flow = [row.server_id for row, (row_flow, _) in classified if row_flow == stamped_flow] + await prisma_client.db.litellm_mcpservertable.update_many( + where={"server_id": {"in": server_ids_for_flow}, "oauth2_flow": None}, + data={"oauth2_flow": stamped_flow, "updated_by": _BACKFILL_AUDIT_ACTOR}, + ) + + counts: dict[BackfillRule, int] = dict(Counter(rule for _, (_, rule) in classified)) + verbose_proxy_logger.info( + "oauth2_flow backfill: processed %d oauth2 server row(s): %s", + len(null_rows), + counts, + ) + return counts diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py index 815fc2ba29d..169a1a5d707 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py @@ -12,7 +12,7 @@ every other mode so the caller defers to v1 (parity-safe); it grows one branch p from __future__ import annotations import base64 -from typing import TYPE_CHECKING, NoReturn, Optional +from typing import TYPE_CHECKING, Literal, NoReturn, Optional from fastapi import HTTPException from pydantic import SecretStr @@ -26,6 +26,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( ServerSpec, SharedKey, Subject, + TokenExchangeConfig, ) from litellm.types.mcp import MCPAuth @@ -61,8 +62,9 @@ def to_server_spec(server: MCPServer) -> Optional[ServerSpec]: an ``assert_never`` tail, so a newly added auth mode fails the type gate here until it is explicitly mapped or explicitly deferred, rather than silently falling through to v1. Live modes: ``none``, the static-header family (``api_key`` plus the Authorization schemes, - all shared-key), and ``oauth2`` per-user tokens (``authorization_code``); client_credentials - (M2M), delegated/passthrough oauth2, token exchange, and SigV4 return None and stay on v1. + all shared-key), ``oauth2`` per-user tokens (``authorization_code``), and + ``oauth2_token_exchange`` (OBO); client_credentials (M2M), delegated/passthrough + oauth2, and SigV4 return None and stay on v1. """ if server.is_byok: return None # per-user BYOK source not migrated yet -> defer to v1 (any auth_type) @@ -92,11 +94,47 @@ def to_server_spec(server: MCPServer) -> Optional[ServerSpec]: ) # client_credentials (M2M) and delegate/passthrough oauth2 stay on v1 return None - case MCPAuth.oauth2_token_exchange | MCPAuth.aws_sigv4: - return None # token exchange and SigV4 are not migrated yet -> defer to v1 + case MCPAuth.oauth2_token_exchange: + return _token_exchange_spec(server, resource) + case MCPAuth.aws_sigv4: + return None # SigV4 is not migrated yet -> defer to v1 assert_never(auth_type) +def _token_exchange_spec(server: MCPServer, resource: str) -> Optional[ServerSpec]: + """Build a token_exchange (OBO) spec, or defer (None) when it is not OBO-configured. + + An OBO server with ``client_id``/``client_secret`` is owned by the v2 arm even if the + ``token_exchange_endpoint``/``token_url`` is absent: a missing endpoint then fails closed (412) at + the exchanger rather than silently deferring to v1 and connecting unauthenticated, since the + gateway must not guess the IdP or fall back to a weaker source. Without client credentials there is + nothing to own, so the server stays on v1 (parity-safe). ``profile`` selects the wire dialect + (``rfc8693`` default, ``entra_obo`` for Microsoft Entra On-Behalf-Of); an unrecognized value + normalizes to ``rfc8693`` so a bad config value cannot crash spec-building. ``audience`` is + forwarded only when the operator set it; a missing one is omitted, not derived. + """ + endpoint = server.token_exchange_endpoint or server.token_url + if not server.client_id or not server.client_secret: + return None + profile: Literal["rfc8693", "entra_obo"] = ( + "entra_obo" if server.token_exchange_profile == "entra_obo" else "rfc8693" + ) + return ServerSpec( + server_id=server.server_id, + resource=resource, + config=TokenExchangeConfig( + profile=profile, + subject_token_type=server.subject_token_type or "urn:ietf:params:oauth:token-type:access_token", + token_exchange_endpoint=endpoint, + audience=server.audience, + client_id=server.client_id, + client_secret=SecretStr(server.client_secret), + token_endpoint_auth_method=server.token_endpoint_auth_method, + scopes=tuple(server.scopes or ()), + ), + ) + + def _shared_key_spec( server: MCPServer, resource: str, @@ -148,23 +186,75 @@ def raise_public(error: CredError) -> NoReturn: assert_never(error.tag) -def raise_user_oauth_challenge(server: MCPServer) -> NoReturn: +def oauth_protected_resource_path(root_path: str, server: MCPServer) -> str: + """The server's RFC 9728 Protected Resource Metadata path, the shared anchor of both challenges. + + ``root_path`` is the proxy's ``SERVER_ROOT_PATH``, resolved by the caller (the imperative shell) + so this stays a pure function of its inputs; ``"/"`` and ``""`` both mean no prefix. The path is + relative, so it resolves against the caller's own host (correct even behind a reverse proxy). + """ + prefix = "" if root_path == "/" else root_path + name = server.alias or server.server_name or server.name or server.server_id + return f"/.well-known/oauth-protected-resource{prefix}/mcp/{name}" + + +def raise_user_oauth_challenge(server: MCPServer, *, root_path: str) -> NoReturn: """Raise the 401 an ``authorization_code`` server returns at egress when the user has no token. - Points at the server's RFC 9728 Protected Resource Metadata (``resource_metadata``), which names - the upstream authorization server the client must complete OAuth with. The URL is per-server and - relative, so it resolves against the caller's own host (correct even behind a reverse proxy) - without needing request context. The listing-phase 401 still emits the RFC 8414 ``authorization_uri`` - form pending the format unification; both target the same server, so the difference is cosmetic. + Points at the server's RFC 9728 Protected Resource Metadata, which names the upstream + authorization server the client must complete OAuth with. The listing-phase 401 still emits the + RFC 8414 ``authorization_uri`` form pending the format unification; both target the same server, + so the difference is cosmetic. """ - from litellm.proxy.utils import get_server_root_path # noqa: PLC0415 - - root = get_server_root_path() - prefix = "" if root == "/" else root - name = server.alias or server.server_name or server.name or server.server_id - resource_metadata = f"/.well-known/oauth-protected-resource{prefix}/mcp/{name}" + resource_metadata = oauth_protected_resource_path(root_path, server) raise HTTPException( status_code=401, detail="Unauthorized", headers={"WWW-Authenticate": f'Bearer resource_metadata="{resource_metadata}"'}, ) + + +def raise_token_exchange_challenge( + server: MCPServer, + *, + root_path: str, + claims: str | None = None, +) -> NoReturn: + """Raise the RFC 9728 / RFC 6750 challenge an OBO (``token_exchange``) server returns when the + caller's subject token is missing or the IdP rejected it. + + Points at the server's Protected Resource Metadata, whose ``authorization_servers`` names the IdP + the client must SSO with to obtain a subject token; ``error="invalid_token"`` tells a + spec-compliant MCP client to discover that AS and retry with a fresh bearer. Mirrors + ``raise_user_oauth_challenge`` but for the exchange flow: there is no gateway-side browser OAuth — + the client re-authenticates directly with the IdP, and LiteLLM then exchanges the resulting token. + + An IdP step-up rejection (Entra Conditional Access / CAE) passes its ``claims`` blob. Per the + Microsoft claims-challenge format the challenge then uses ``error="insufficient_claims"`` (the + value MSAL-family clients key on) and carries the claims base64-encoded in a ``claims`` parameter + the client replays to the IdP to satisfy the step-up. Without a claims blob the challenge keeps + ``error="invalid_token"`` and is byte-identical to the static one. Both the error value (one of + two literals) and the base64 claims draw from a fixed alphabet, so nothing from the IdP body + reaches the header unescaped. + """ + resource_metadata = oauth_protected_resource_path(root_path, server) + encoded_claims = base64.b64encode(claims.encode()).decode() if claims else None + error = "insufficient_claims" if encoded_claims else "invalid_token" + error_description = ( + "Step-up authentication required; satisfy the returned claims challenge with the IdP and retry" + if encoded_claims + else "Missing or invalid subject token; authenticate with the IdP and retry" + ) + www_authenticate = ", ".join( + ( + f'Bearer resource_metadata="{resource_metadata}"', + f'error="{error}"', + f'error_description="{error_description}"', + *((f'claims="{encoded_claims}"',) if encoded_claims else ()), + ) + ) + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={"WWW-Authenticate": www_authenticate}, + ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py index f9a9fa00b23..c82ce1037d6 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py @@ -8,7 +8,8 @@ an arm fails the type gate (basedpyright `reportMatchNotExhaustive`); a bypassed at runtime instead of returning `None`. `none` and `api_key` (shared-key source) are live, as is `authorization_code`, which reads the -user's token from the injected `OAuthTokenStore`. The remaining arms are `not_implemented` stubs +user's token from the injected `OAuthTokenStore`, and `token_exchange`, which swaps the caller's +inbound token through the injected `TokenExchanger`. The remaining arms are `not_implemented` stubs that each land in a follow-up PR with their seam. Pure v2: no imports from v1. """ @@ -31,6 +32,9 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( Ok, Result, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import ( + TokenExchanger, +) from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( ApiKeyConfig, AuthorizationCodeConfig, @@ -55,16 +59,36 @@ class _NullOAuthTokenStore: return None +class _NullTokenExchanger: + """Fail-closed default: with no exchanger wired, token_exchange cannot produce a credential.""" + + async def exchange( + self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = "" + ) -> Result[OAuthToken, CredError]: + return Error(CredError.of_misconfigured("token exchange collaborator not wired")) + + async def invalidate( + self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = "" + ) -> None: + return None + + class UpstreamCredentialProvider: """Produces the one `httpx.Auth` for a `(subject, upstream)` pair, per declared mode. Collaborators (the per-mode credential stores and token fetchers) are injected as each arm is built; the live `none` and `api_key`-shared arms read from the config and need none, while - `authorization_code` reads the user's token from the injected `OAuthTokenStore`. + `authorization_code` reads the user's token from the injected `OAuthTokenStore` and + `token_exchange` swaps the caller's token through the injected `TokenExchanger`. """ - def __init__(self, oauth_token_store: OAuthTokenStore | None = None) -> None: + def __init__( + self, + oauth_token_store: OAuthTokenStore | None = None, + token_exchanger: TokenExchanger | None = None, + ) -> None: self._oauth_token_store: OAuthTokenStore = oauth_token_store or _NullOAuthTokenStore() + self._token_exchanger: TokenExchanger = token_exchanger or _NullTokenExchanger() async def resolve_credentials(self, subject: Subject, server: ServerSpec) -> Result[httpx.Auth, CredError]: match server.config: @@ -76,8 +100,8 @@ class UpstreamCredentialProvider: return _not_implemented(AuthSpecKind.passthrough) case ClientCredentialsConfig(): return _not_implemented(AuthSpecKind.client_credentials) - case TokenExchangeConfig(): - return _not_implemented(AuthSpecKind.token_exchange) + case TokenExchangeConfig() as config: + return await self._token_exchange(subject, server, config) case AuthorizationCodeConfig(): return await self._authorization_code(subject, server) case AwsSigV4Config(): @@ -110,6 +134,43 @@ class UpstreamCredentialProvider: return Error(CredError.of_unauthorized("Authorization required: complete the OAuth flow for this server.")) return Ok(StaticHeaderAuth(f"Bearer {token.access_token}", header_name="Authorization")) + async def _token_exchange( + self, subject: Subject, server: ServerSpec, config: TokenExchangeConfig + ) -> Result[StaticHeaderAuth, CredError]: + """RFC 8693 OBO: exchange the caller's inbound token for an upstream-bound bearer. + + No inbound token means there is nothing to exchange, so the arm fails closed with a 401 rather + than falling through to a weaker source (§1.5); the exchanger handles the IdP round-trip and + caching and returns the upstream token or a typed error. + """ + inbound = subject.inbound_token + if inbound is None: + return Error( + CredError.of_unauthorized( + "Token exchange requires a caller token to exchange (OBO).", + www_authenticate='Bearer error="invalid_request"', + ) + ) + match await self._token_exchanger.exchange( + inbound.get_secret_value(), server, config, tenant_id=subject.tenant_id + ): + case Ok(token): + return Ok(StaticHeaderAuth(f"Bearer {token.access_token}", header_name="Authorization")) + case Error(err): + return Error(err) + + async def invalidate_credentials(self, subject: Subject, server: ServerSpec) -> None: + """Drop any cached credential the resolver owns for this `(subject, server)`. + + Used after an upstream rejects the injected credential, so the next resolve re-mints rather + than serving the same rejected token until TTL. Only `token_exchange` holds a re-mintable + cached credential here; other modes are a no-op. + """ + if isinstance(server.config, TokenExchangeConfig) and subject.inbound_token is not None: + await self._token_exchanger.invalidate( + subject.inbound_token.get_secret_value(), server, server.config, tenant_id=subject.tenant_id + ) + async def _authz_token(self, subject: Subject, server: ServerSpec) -> OAuthToken | None: """The user's authorization_code token, or None when absent or the store is unreachable. diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py new file mode 100644 index 00000000000..e49de4559c6 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py @@ -0,0 +1,114 @@ +"""Composition root for the v2-native token_exchange (OBO) exchanger. + +Wires the pure ``OboTokenExchanger`` to its runtime edges: the real httpx POST against the IdP and +the configured cache sizing/TTL constants. ``build_token_exchanger`` is built once at egress +construction and reused, so the in-process exchanged-token cache survives across requests. Unlike the +per-user store, nothing here reads a runtime global at build time (the httpx client is acquired per +call), so it needs no lazy wrapper. +""" + +from __future__ import annotations + +import httpx + +from litellm._logging import verbose_logger +from litellm.constants import ( + MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL, + MCP_OAUTH2_TOKEN_CACHE_MIN_TTL, + MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS, + MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE, +) +from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( + InMemoryTokenCacheBackend, +) +from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import ( + OboTokenExchanger, + SubjectTokenRejected, + TokenExchangeClientError, +) + +# RFC 6749 5.2 error codes that mean the gateway's own request/credentials are wrong (not the +# caller's subject token), so they surface as a 500 the caller can't fix by re-authenticating. +_GATEWAY_FAULT_OAUTH_ERRORS = frozenset( + {"invalid_client", "unauthorized_client", "unsupported_grant_type", "invalid_target", "invalid_scope"} +) + + +def _oauth_error_fields(response: httpx.Response) -> tuple[str | None, str | None]: + """Read the RFC 6749 5.2 ``error`` code and the IdP's step-up ``claims`` blob from a + token-endpoint error body, as ``(error, claims)`` with None for whatever is absent. + + ``claims`` is the Entra Conditional Access / CAE challenge (a JSON string the client must + replay to the IdP to satisfy the step-up); it is the caller's own requirement, not an IdP + internal, so it may travel to the caller. The ``error_description`` is deliberately not read: + it can carry IdP internals and must never reach the caller. + """ + try: + body: object = response.json() + except Exception: # noqa: BLE001 + return None, None + if not isinstance(body, dict): + return None, None + code = body.get("error") + claims = body.get("claims") + return ( + code if isinstance(code, str) else None, + claims if isinstance(claims, str) and claims else None, + ) + + +async def _post_exchange_endpoint( + url: str, form: dict[str, str], client_auth_headers: dict[str, str] +) -> dict[str, object] | None: + from litellm.llms.custom_httpx.http_handler import ( # noqa: PLC0415 + get_async_httpx_client, # pyright: ignore + ) + from litellm.types.llms.custom_http import httpxSpecialProvider # noqa: PLC0415 + + # litellm's httpx handler and httpx.Response are only partially typed; the IdP returns a JSON + # object and the exchanger validates each field, so the untyped boundary is contained here. + # A 4xx is the IdP rejecting the subject (non-retryable -> 401 via SubjectTokenRejected); any + # other failure is a miss (-> None -> upstream_unavailable -> 503), matching v1's fail-closed. + headers = {"Accept": "application/json", **client_auth_headers} + try: + client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) # pyright: ignore + response = await client.post(url, headers=headers, data=form) # pyright: ignore + response.raise_for_status() # pyright: ignore + parsed: object = response.json() # pyright: ignore + except httpx.HTTPStatusError as status_err: + status_code = status_err.response.status_code + if 400 <= status_code < 500: + oauth_error, claims = _oauth_error_fields(status_err.response) + if oauth_error in _GATEWAY_FAULT_OAUTH_ERRORS: + verbose_logger.warning( + "MCP token exchange rejected as %s (HTTP %d); check the gateway client credentials, " + "audience, and scope for this server", + oauth_error, + status_code, + ) + raise TokenExchangeClientError(oauth_error) from status_err + raise SubjectTokenRejected( + f"IdP rejected the subject token (HTTP {status_code})", + claims=claims, + ) from status_err + verbose_logger.warning("MCP token exchange request failed: %s", status_err) + return None + except Exception as exc: # noqa: BLE001 + verbose_logger.warning("MCP token exchange request failed: %s", exc) + return None + if not isinstance(parsed, dict): + # A valid-but-non-object JSON body (list/string/number) would crash the field parsing; map it + # to a miss so it surfaces as a typed upstream_unavailable, not a 500. + verbose_logger.warning("MCP token exchange returned non-object JSON (%s)", type(parsed).__name__) + return None + return parsed # pyright: ignore + + +def build_token_exchanger() -> OboTokenExchanger: + return OboTokenExchanger( + _post_exchange_endpoint, + cache=InMemoryTokenCacheBackend(max_size=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE), + default_ttl_seconds=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL, + min_ttl_seconds=MCP_OAUTH2_TOKEN_CACHE_MIN_TTL, + expiry_buffer_seconds=MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS, + ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchanger.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchanger.py new file mode 100644 index 00000000000..02b6d4eafb1 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchanger.py @@ -0,0 +1,376 @@ +"""v2-native OBO token exchange: swap the caller's token for an upstream-bound one. + +The pure core of the ``token_exchange`` mode. Given the caller's inbound token and the server's +``TokenExchangeConfig``, ``OboTokenExchanger.exchange`` POSTs the grant selected by ``config.profile`` +to the configured endpoint and returns the upstream-bound ``access_token`` as a typed ``OAuthToken``, +or a typed ``CredError`` - never a raise (the HTTP edge is the injected ``ExchangeHttpPost``, whose +adapter contains the I/O). Two profiles share this one engine: ``rfc8693`` (the RFC 8693 token-exchange +grant) and ``entra_obo`` (Microsoft Entra On-Behalf-Of, which is the RFC 7523 ``jwt-bearer`` grant); +only the request form differs, so the cache, single-flight, and TTL machinery are dialect-agnostic. The +exchanged token is cached and single-flighted per ``(subject_token, tenant, config, server)`` so a +repeated caller token skips the IdP round-trip and concurrent calls collapse to one exchange, reusing +the shared in-process cache + coordinator foundation. A rotated caller token hashes to a new key and +re-exchanges. Pure v2 apart from the shared RFC 6749 client-auth helper, which carries no v1 state. + +A missing/expired exchange is an error, never a fall-through to a weaker source (§1.5): the caller +presenting no token is the resolver arm's 401, and an IdP that does not return a usable token is an +``upstream_unavailable`` here. +""" + +from __future__ import annotations + +import hashlib +import time +from collections.abc import Awaitable, Callable +from typing import Literal, Protocol + +from typing_extensions import assert_never + +from litellm._logging import verbose_logger +from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( + InMemoryTokenCacheBackend, + InProcessRefreshCoordinator, + OAuthToken, + RefreshCoordinator, + TokenCacheBackend, +) +from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( + Error, + Ok, + Result, +) +from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( + build_token_endpoint_client_auth, +) +from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( + CredError, + ServerSpec, + TokenExchangeConfig, +) + +# A token with no declared expiry is cached for this long; one with an expiry is cached until then +# minus the skew buffer, floored at the minimum. Values mirror v1's MCP_OAUTH2_* constants; the +# composition root injects the configured ones. +_DEFAULT_TTL_SECONDS = 3600.0 +_MIN_TTL_SECONDS = 10.0 +_EXPIRY_BUFFER_SECONDS = 60.0 + +_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange" +# Microsoft Entra On-Behalf-Of speaks the RFC 7523 jwt-bearer grant, not RFC 8693, and gates delegation +# behind ``requested_token_use=on_behalf_of`` (a Microsoft extension present in neither RFC). +_JWT_BEARER_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:jwt-bearer" +_REQUESTED_TOKEN_USE_OBO = "on_behalf_of" + +# RFC 8693 3 token-type URNs that are not usable as an upstream Bearer access token. token_type +# already rejects the common non-access case (N_A); this catches a malformed STS that mints one of +# these but still labels it Bearer. An access_token / jwt / absent / unknown type is accepted (lenient). +_NON_ACCESS_ISSUED_TOKEN_TYPES = frozenset( + { + "urn:ietf:params:oauth:token-type:refresh_token", + "urn:ietf:params:oauth:token-type:id_token", + "urn:ietf:params:oauth:token-type:saml1", + "urn:ietf:params:oauth:token-type:saml2", + } +) + +# The IdP returns an opaque JSON object; the post adapter hands it over untyped and the exchanger +# validates each field, so no Any leaks past this seam (None == any transport/HTTP failure). The +# second dict is the form body; the third is the client-auth headers (HTTP Basic for +# client_secret_basic, empty for client_secret_post). +ExchangeHttpPost = Callable[[str, "dict[str, str]", "dict[str, str]"], Awaitable["dict[str, object] | None"]] + + +class SubjectTokenRejected(Exception): + """The IdP refused to exchange the subject token (an RFC 8693 4xx, e.g. ``invalid_grant``). + + Distinct from a transport / IdP-availability failure, which the post adapter maps to ``None`` -> + ``upstream_unavailable`` -> 503 (retryable). A rejected subject is the caller's problem, not the + gateway's, so the arm surfaces it as a non-retryable 401 (the OBO challenge) instead. + ``claims`` is the IdP's step-up challenge blob (Entra Conditional Access / CAE) from the + rejection body; it threads into the 401 challenge so the client can satisfy the step-up and + retry. The ``error_description`` is never carried (it can leak IdP internals). + """ + + def __init__(self, detail: str, *, claims: str | None = None) -> None: + super().__init__(detail) + self.claims = claims + + +class TokenExchangeClientError(Exception): + """The IdP rejected the exchange for a reason that is the gateway's fault, not the caller's. + + RFC 6749 5.2 codes such as ``invalid_client`` (the gateway's own STS credentials are wrong), + ``unauthorized_client`` / ``unsupported_grant_type`` (the gateway is not permitted to exchange), + ``invalid_target`` / ``invalid_scope`` (the gateway's audience/scope config for this server is + wrong). The caller cannot fix these by re-authenticating, so the arm surfaces them as a 500 + (``misconfigured``), not the 401 OBO challenge. The IdP ``error_description`` is never carried. + """ + + +class TokenExchanger(Protocol): + """Exchanges a caller token for an upstream-bound one, per the server's token_exchange config.""" + + async def exchange( + self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = "" + ) -> Result[OAuthToken, CredError]: ... + + async def invalidate( + self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = "" + ) -> None: ... + + +def _cache_key(subject_token: str, tenant_id: str, config: TokenExchangeConfig) -> str: + """Bind the cache entry to the caller token, the tenant, AND the exchange config that minted it. + + A rotated caller token, a different tenant, profile, endpoint, audience, scope, client_id, secret, + auth method, or subject_token_type all change the key, so two tenants behind the same opaque token + never share an entry and a config change (including a profile flip that alters the wire form) + forces a fresh exchange instead of serving a token minted for the old config until TTL. Everything + is hashed, so no secret is held in the key. + """ + secret = config.client_secret.get_secret_value() if config.client_secret else "" + material = "\x00".join( + ( + subject_token, + tenant_id, + config.profile, + config.token_exchange_endpoint or "", + config.audience or "", + config.subject_token_type, + config.client_id or "", + secret, + config.token_endpoint_auth_method or "", + " ".join(config.scopes), + ) + ) + return hashlib.sha256(material.encode()).hexdigest() + + +def _parse_expires_in(raw: object) -> int | None: + if isinstance(raw, bool): + return None + if isinstance(raw, (int, float)): + return int(raw) + if isinstance(raw, str): + try: + return int(float(raw)) + except ValueError: + return None + return None + + +def _rfc8693_form( + *, + subject_token: str, + subject_token_type: str, + audience: str | None, + scopes: tuple[str, ...], +) -> dict[str, str]: + return { + "grant_type": _GRANT_TYPE, + "subject_token": subject_token, + "subject_token_type": subject_token_type, + **({"audience": audience} if audience else {}), + **({"scope": " ".join(scopes)} if scopes else {}), + } + + +def _entra_obo_form( + *, + subject_token: str, + scopes: tuple[str, ...], +) -> dict[str, str]: + # Microsoft Entra On-Behalf-Of (RFC 7523 jwt-bearer, not RFC 8693): the caller's inbound access + # token rides as ``assertion`` (its ``aud`` must be this gateway's ``client_id``); the target + # resource is carried in ``scope`` (e.g. api:///.default), since Entra has no audience + # parameter and ignores subject_token_type; ``requested_token_use=on_behalf_of`` is the Microsoft + # extension that turns the jwt-bearer grant into a delegation. ``scope`` is required, and the + # exchange precondition rejects an empty one, so it is always present here. Client authentication + # (client_id/client_secret via post, or Basic) is layered on by the caller through + # build_token_endpoint_client_auth, so it is not built into the form here. + return { + "grant_type": _JWT_BEARER_GRANT_TYPE, + "assertion": subject_token, + "scope": " ".join(scopes), + "requested_token_use": _REQUESTED_TOKEN_USE_OBO, + } + + +def _build_exchange_form( + *, + profile: Literal["rfc8693", "entra_obo"], + subject_token: str, + subject_token_type: str, + audience: str | None, + scopes: tuple[str, ...], +) -> dict[str, str]: + match profile: + case "rfc8693": + return _rfc8693_form( + subject_token=subject_token, + subject_token_type=subject_token_type, + audience=audience, + scopes=scopes, + ) + case "entra_obo": + return _entra_obo_form( + subject_token=subject_token, + scopes=scopes, + ) + assert_never(profile) + + +class OboTokenExchanger: + """``TokenExchanger`` that runs the profile's OBO grant once per caller token, then caches the result. + + The HTTP post is injected (``None`` on any IdP failure, mirroring v1: a failed exchange is a miss, + not a 500). The cache and single-flight coordinator default to the in-process foundation; a + deployment with no shared state needs nothing more (v1's exchanged-token cache is per-process too). + The clock is injected so TTL/expiry is deterministic in tests. + """ + + def __init__( + self, + http_post: ExchangeHttpPost, + *, + cache: TokenCacheBackend | None = None, + coordinator: RefreshCoordinator | None = None, + clock: Callable[[], float] = time.time, + default_ttl_seconds: float = _DEFAULT_TTL_SECONDS, + min_ttl_seconds: float = _MIN_TTL_SECONDS, + expiry_buffer_seconds: float = _EXPIRY_BUFFER_SECONDS, + ) -> None: + self._http_post = http_post + self._cache: TokenCacheBackend = cache or InMemoryTokenCacheBackend(clock=clock) + self._coordinator: RefreshCoordinator = coordinator or InProcessRefreshCoordinator() + self._clock = clock + self._default_ttl_seconds = default_ttl_seconds + self._min_ttl_seconds = min_ttl_seconds + self._expiry_buffer_seconds = expiry_buffer_seconds + + async def exchange( + self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = "" + ) -> Result[OAuthToken, CredError]: + endpoint = config.token_exchange_endpoint + client_id = config.client_id + client_secret = config.client_secret + if not endpoint: + # No endpoint configured and none discoverable: fail closed (412) rather than guess an IdP + # or fall back to a weaker source. The caller's token is never sent anywhere. + return Error( + CredError.of_precondition_required("token exchange endpoint is not configured for this server") + ) + if not client_id or client_secret is None: + return Error(CredError.of_misconfigured("token_exchange requires client_id and client_secret")) + if config.profile == "entra_obo" and not config.scopes: + # Entra carries the target resource in ``scope`` (api:///.default); with no scope the + # IdP cannot resolve an audience, so fail closed as misconfigured rather than POST a form the + # IdP will reject. + return Error( + CredError.of_misconfigured("entra_obo token exchange requires a scope (e.g. api:///.default)") + ) + + cache_key = _cache_key(subject_token, tenant_id, config) + server_id = server.server_id + cached = await self._cache.get(cache_key, server_id) + if cached is not None: + verbose_logger.debug("MCP token exchange cache hit for server %s", server_id) + return Ok(cached) + + client_auth = build_token_endpoint_client_auth( + auth_method=config.token_endpoint_auth_method, + client_id=client_id, + client_secret=client_secret.get_secret_value(), + ) + form = { + **_build_exchange_form( + profile=config.profile, + subject_token=subject_token, + subject_token_type=config.subject_token_type, + audience=config.audience, + scopes=config.scopes, + ), + **client_auth.body, + } + + async def run_exchange() -> OAuthToken | None: + fresh = await self._cache.get(cache_key, server_id) + if fresh is not None: + return fresh + verbose_logger.debug( + "Exchanging token for MCP server %s at %s (audience=%s)", server_id, endpoint, config.audience + ) + body = await self._http_post(endpoint, form, client_auth.headers) + if body is None: + return None + token = self._token_from_body(body) + if token is None: + return None + await self._cache.set(cache_key, server_id, token, self._ttl_seconds(token)) + verbose_logger.info("Token exchange succeeded for MCP server %s", server_id) + return token + + async def reread() -> OAuthToken | None: + return await self._cache.get(cache_key, server_id) + + try: + token = await self._coordinator.run(cache_key, server_id, refresh=run_exchange, reread=reread) + except SubjectTokenRejected as rejected: + # The IdP rejected the subject token (4xx). This is non-retryable: the caller must + # re-authenticate with the IdP, so it surfaces as a 401 (the OBO challenge), not a 503. + # A step-up rejection (Entra Conditional Access) carries the claims blob through so the + # edge's challenge tells the client how to satisfy it. + return Error( + CredError.of_unauthorized( + str(rejected) or "subject token rejected by the IdP", + claims=rejected.claims, + ) + ) + except TokenExchangeClientError: + # RFC 6749 5.2 gateway-fault code (invalid_client / invalid_target / ...): the caller can't + # fix it by re-authenticating, so surface a 500 rather than the OBO 401 challenge. The + # specific code is logged at the edge; the user-facing summary stays generic. + return Error( + CredError.of_misconfigured( + "token exchange configuration error: the gateway's credentials, audience, or scope " + "for this server were not accepted by the IdP" + ) + ) + if token is None: + return Error(CredError.of_upstream_unavailable("token exchange did not return a usable access token")) + return Ok(token) + + async def invalidate( + self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = "" + ) -> None: + """Drop the cached exchanged token so the next call re-exchanges (e.g. after an upstream 401).""" + await self._cache.delete(_cache_key(subject_token, tenant_id, config), server.server_id) + + def _token_from_body(self, body: dict[str, object]) -> OAuthToken | None: + access_token = body.get("access_token") + if not isinstance(access_token, str) or not access_token: + return None + # token_type is forwarded downstream as Bearer, so a present-but-non-Bearer type (e.g. N_A) + # must fail closed rather than be minted as a bogus Bearer; an absent type defaults to Bearer. + token_type = body.get("token_type") + if isinstance(token_type, str) and token_type.strip().lower() != "bearer": + verbose_logger.warning( + "MCP token exchange returned unusable token_type %r; refusing to forward it as Bearer", token_type + ) + return None + # issued_token_type says what representation was minted; reject a clearly-non-access type + # (refresh/id/saml) even if token_type claimed Bearer. access_token / jwt / absent / unknown pass. + issued_token_type = body.get("issued_token_type") + if isinstance(issued_token_type, str) and issued_token_type in _NON_ACCESS_ISSUED_TOKEN_TYPES: + return None + expires_in = _parse_expires_in(body.get("expires_in")) + expires_at = self._clock() + expires_in if expires_in is not None else None + return OAuthToken(access_token=access_token, expires_at=expires_at) + + def _ttl_seconds(self, token: OAuthToken) -> float: + if token.expires_at is None: + return self._default_ttl_seconds + lifetime = max(0.0, token.expires_at - self._clock()) + # Floor at min_ttl, but never cache past the token's own expiry: a token whose remaining + # lifetime is below the buffer (or even below min_ttl) must not be served stale upstream. + return min(max(lifetime - self._expiry_buffer_seconds, self._min_ttl_seconds), lifetime) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py index 671de63eabe..49e3973d363 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py @@ -67,11 +67,15 @@ class Unauthorized: ``detail`` is the human message; ``www_authenticate`` and ``body`` carry a scheme-specific challenge (e.g. BYOK's provisioning prompt) so the edge can reproduce it verbatim. + ``claims`` carries an IdP step-up challenge (e.g. Entra Conditional Access) so the edge can + fold it into the ``WWW-Authenticate`` it builds; the client replays the claims to the IdP to + satisfy the step-up, then retries with the fresh token. """ detail: str www_authenticate: str | None = None body: Mapping[str, str] | None = None + claims: str | None = None @tagged_union(frozen=True) @@ -104,8 +108,16 @@ class CredError: *, www_authenticate: str | None = None, body: Mapping[str, str] | None = None, + claims: str | None = None, ) -> CredError: - return CredError(unauthorized=Unauthorized(detail=detail, www_authenticate=www_authenticate, body=body)) + return CredError( + unauthorized=Unauthorized( + detail=detail, + www_authenticate=www_authenticate, + body=body, + claims=claims, + ) + ) @staticmethod def of_misconfigured(detail: str) -> CredError: @@ -182,18 +194,33 @@ class ClientCredentialsConfig(BaseModel): class TokenExchangeConfig(BaseModel): - """RFC 8693 OBO; swap the caller's live subject_token for a token bound to the upstream's - audience (`server.resource`, RFC 8707). The gateway authenticates to the exchange endpoint - as an OAuth client (`client_id`/`client_secret`); the inbound token is sent only to that - endpoint, never to the upstream. + """OBO: swap the caller's live inbound token for a token bound to the upstream's audience. The + gateway authenticates to the exchange endpoint as an OAuth client (`client_id`/`client_secret`); + the inbound token is sent only to that endpoint, never to the upstream. + + `profile` selects the wire dialect, since not every IdP speaks RFC 8693: + - `rfc8693` (default) is the standard token-exchange grant: the inbound token is the + `subject_token` (typed by `subject_token_type`), the target is the optional `audience`. + - `entra_obo` is Microsoft Entra On-Behalf-Of, which is the RFC 7523 `jwt-bearer` grant rather + than 8693: the inbound token rides as `assertion`, the target resource is carried in `scopes` + (`api:///.default`, since Entra has no audience parameter), and the Microsoft-only + `requested_token_use=on_behalf_of` extension makes the jwt-bearer grant a delegation. + `subject_token_type` and `audience` are unused in this profile. + + `audience` (rfc8693 only) is optional and sent only when the operator configured one, since both + `audience` and `resource` are optional in RFC 8693 and the authorization server applies its own + default when neither is sent (fabricating one risks `invalid_target`). """ model_config = ConfigDict(frozen=True) kind: Literal[AuthSpecKind.token_exchange] = AuthSpecKind.token_exchange + profile: Literal["rfc8693", "entra_obo"] = "rfc8693" subject_token_type: str = "urn:ietf:params:oauth:token-type:access_token" token_exchange_endpoint: str | None = None + audience: str | None = None client_id: str | None = None client_secret: SecretStr | None = None + token_endpoint_auth_method: Literal["client_secret_basic", "client_secret_post"] | None = None scopes: tuple[str, ...] = () diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index a6067a60105..d482e537c5d 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -77,6 +77,7 @@ if MCP_AVAILABLE: ListMCPToolsRestAPIResponseObject, MCPInfo, MCPServer, + _apply_toolset_scope, _fire_mcp_success_logging, _tool_name_matches, execute_mcp_tool, @@ -541,10 +542,37 @@ if MCP_AVAILABLE: "message": "Successfully retrieved tools", } + def _as_query_str(value: Any) -> Optional[str]: + """Coerce an Optional[str] Query param to str|None, dropping unresolved FastAPI defaults.""" + return value if isinstance(value, str) else None + + async def _resolve_toolset_scope( + toolset_name: Optional[str], + user_api_key_dict: UserAPIKeyAuth, + ) -> UserAPIKeyAuth: + """Resolve ``toolset_name`` to its scoped ``UserAPIKeyAuth``, or return unchanged.""" + if not toolset_name: + return user_api_key_dict + + from litellm.proxy.utils import get_prisma_client_or_throw + + prisma_client = get_prisma_client_or_throw("Database not available. Connect a database to your proxy") + toolset = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, toolset_name) + if toolset is None: + raise HTTPException( + status_code=404, + detail=f"Toolset '{toolset_name}' not found", + ) + return await _apply_toolset_scope(user_api_key_dict, toolset.toolset_id) + @router.get("/tools/list", dependencies=[Depends(user_api_key_auth)]) async def list_tool_rest_api( request: Request, server_id: Optional[str] = Query(None, description="The server id to list tools for"), + mcp_server_name: Optional[str] = Query( + None, description="Filter tools to a single MCP server by name or alias" + ), + toolset_name: Optional[str] = Query(None, description="Filter tools to a single toolset by name"), include_disabled_tools: bool = Query( False, description=( @@ -582,16 +610,29 @@ if MCP_AVAILABLE: ) try: + mcp_server_name = _as_query_str(mcp_server_name) + toolset_name = _as_query_str(toolset_name) + # The full catalog (allowlist filter skipped) is admin-only so the # REST endpoint can't be used to enumerate deliberately-disabled tools. apply_tool_filters = not ( include_disabled_tools and user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN ) - if apply_tool_filters and getattr( - getattr(user_api_key_dict, "object_permission", None), - "mcp_tool_search_enabled", - False, + user_api_key_dict = await _resolve_toolset_scope(toolset_name, user_api_key_dict) + + if server_id is None: + server_id = mcp_server_name + + if ( + apply_tool_filters + and server_id is None + and toolset_name is None + and getattr( + getattr(user_api_key_dict, "object_permission", None), + "mcp_tool_search_enabled", + False, + ) ): from litellm.proxy._experimental.mcp_server.tool_search import ( get_virtual_tool_definitions, @@ -719,6 +760,8 @@ if MCP_AVAILABLE: request_path=request.scope.get("_original_path") or request.url.path, ) except HTTPException as http_exc: + if http_exc.status_code == status.HTTP_404_NOT_FOUND: + raise # Internal access/IP 403s keep the legacy error-dict response shape # so the existing contract stays intact. verbose_logger.exception("HTTPException in list_tool_rest_api: %s", str(http_exc)) @@ -1079,11 +1122,6 @@ if MCP_AVAILABLE: forwarded_authorization = ( effective_oauth2_headers.get("Authorization") if effective_oauth2_headers else None ) - is_interactive_authz_code = ( - server_model.auth_type == MCPAuth.oauth2 - and forwarded_authorization is not None - and to_server_spec(server_model) is not None - ) preview_cred_provider = ( UpstreamCredentialProvider( oauth_token_store=PresentedOAuthTokenStore( @@ -1094,7 +1132,11 @@ if MCP_AVAILABLE: ) ) ) - if is_interactive_authz_code + if ( + server_model.auth_type == MCPAuth.oauth2 + and forwarded_authorization is not None + and to_server_spec(server_model) is not None + ) else None ) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 57404793269..e3812522ded 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -1427,18 +1427,8 @@ if MCP_AVAILABLE: for allowed_mcp_server_id in allowed_mcp_server_ids: mcp_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id) if mcp_server is not None: - # Apply oauth2_flow resolution for legacy DB rows where it may be NULL - resolved_flow = MCPServerManager._resolve_oauth2_flow( - auth_type=mcp_server.auth_type, - oauth2_flow=mcp_server.oauth2_flow, - token_url=mcp_server.token_url, - authorization_url=mcp_server.authorization_url, - client_id=mcp_server.client_id, - client_secret=mcp_server.client_secret, - ) - if resolved_flow and resolved_flow != mcp_server.oauth2_flow: - # Create a new instance with the resolved flow for this request - mcp_server = mcp_server.model_copy(update={"oauth2_flow": resolved_flow}) + # Apply the request-time oauth2_flow backstop for legacy null rows. + mcp_server = MCPServerManager.resolve_oauth2_flow_for_request(mcp_server) allowed_mcp_servers.append(mcp_server) if mcp_servers is not None: @@ -1838,6 +1828,7 @@ if MCP_AVAILABLE: add_prefix=True, # Always add server prefix raw_headers=raw_headers, user_api_key_auth=user_api_key_auth, + oauth2_headers=oauth2_headers, ) filtered_tools = filter_tools_by_allowed_tools(tools, server) @@ -1860,7 +1851,8 @@ if MCP_AVAILABLE: # tools. Surfacing the upstream 401 to the client as a re-auth challenge is # intentionally not done here: raising from this list handler cannot produce a # 401 + WWW-Authenticate (the MCP session manager serializes it as a JSON-RPC - # error), so that belongs in a request-scope preemptive check, tracked separately. + # error). Single-server routes surface it via the request-scope preemptive + # check in _raise_preemptive_401_for_unauthenticated_servers instead. verbose_logger.debug(f"MCP list_tools: omitting {server.name}; it needs upstream auth") return [] except Exception as e: @@ -2692,12 +2684,18 @@ if MCP_AVAILABLE: # Forward named client headers to OpenAPI tool upstream requests. # MCPServer.extra_headers lists header names to copy from raw_headers. - # OAuth2 M2M: never take Authorization from the caller (matches - # _prepare_mcp_server_headers for managed MCP). + # The strip decision is centralized in _should_strip_caller_authorization so this + # OpenAPI/local path agrees with the managed paths: M2M and the resolver-owned modes + # (token_exchange's raw subject token, authorization_code's stored token) must never + # have the caller's Authorization forwarded verbatim upstream. forwarded_headers: Optional[Dict[str, str]] = None if mcp_server and mcp_server.extra_headers and raw_headers: normalized_raw = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)} - skip_caller_authorization = bool(mcp_server.has_client_credentials) + skip_caller_authorization = _should_strip_caller_authorization( + mcp_server=mcp_server, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) for header_name in mcp_server.extra_headers: if not isinstance(header_name, str): continue @@ -2792,6 +2790,9 @@ if MCP_AVAILABLE: for allowed_mcp_server_id in allowed_mcp_server_ids: allowed_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id) if allowed_server is not None: + # Same request-time oauth2_flow backstop the listing path applies, + # so a null-flow M2M-shape row is treated as M2M on tool calls too. + allowed_server = MCPServerManager.resolve_oauth2_flow_for_request(allowed_server) allowed_mcp_servers.append(allowed_server) allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( @@ -3466,6 +3467,36 @@ if MCP_AVAILABLE: headers={"www-authenticate": authorization_uri}, ) + # token_exchange (OBO): the caller supplied no subject token. Challenge at connect + # (transport level, where WWW-Authenticate survives) with the RFC 9728 resource_metadata + # so the client discovers the IdP, SSOs, and retries with a subject token, which LiteLLM + # then exchanges. A tool-call-time 401 would be wrapped into a JSON-RPC error and the + # header lost, so the discovery flow needs this pre-emptive challenge. + if server and server.auth_type == MCPAuth.oauth2_token_exchange and not oauth2_headers: + from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 + raise_token_exchange_challenge, + ) + from litellm.proxy.utils import get_server_root_path # noqa: PLC0415 + + raise_token_exchange_challenge(server, root_path=get_server_root_path()) + + # token_exchange (OBO) with a subject present: run the exchange here at the transport + # edge, so a rejected subject raises the RFC 9728 challenge (and a gateway fault its + # public status) instead of the session opening and list_tools masking the failure as + # an empty tool list. Gated to single-server routes; the multi-server aggregate keeps + # absorbing per-server auth failures so one bad server cannot 401 the whole connect. + if ( + server + and server.auth_type == MCPAuth.oauth2_token_exchange + and oauth2_headers + and len(mcp_servers or []) == 1 + ): + await global_mcp_server_manager.preflight_token_exchange( + server=server, + oauth2_headers=oauth2_headers, + user_api_key_auth=user_api_key_auth, + ) + # Pass-through OAuth: when the admin has opted a server into # forwarding the client's bearer token (is_oauth_passthrough) and # the client hasn't supplied one, fail fast with 401 and point diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index 9cb6d404b01..c9c60030dbc 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -214,12 +214,14 @@ def server_applies_tool_allowlist(mcp_server: Any) -> bool: def validate_and_normalize_mcp_server_payload(payload: Any) -> None: """ - Validate and normalize MCP server payload fields (server_name and alias). + Validate and normalize MCP server payload fields (server_name, alias, and + tool_name_to_display_name). This function: 1. Validates that server_name and alias don't contain the MCP_TOOL_PREFIX_SEPARATOR - 2. Normalizes alias by replacing spaces with underscores - 3. Sets default alias if not provided (using server_name as base) + 2. Validates that tool_name_to_display_name values satisfy Bedrock's tool-name pattern + 3. Normalizes alias by replacing spaces with underscores + 4. Sets default alias if not provided (using server_name as base) Args: payload: The payload object containing server_name and alias fields @@ -235,6 +237,10 @@ def validate_and_normalize_mcp_server_payload(payload: Any) -> None: if hasattr(payload, "alias") and payload.alias: validate_mcp_server_name(payload.alias, raise_http_exception=True) + # Tool display name validation: must satisfy Bedrock's tool-name pattern + if hasattr(payload, "tool_name_to_display_name") and payload.tool_name_to_display_name: + validate_tool_display_names(payload.tool_name_to_display_name) + # Alias normalization and defaulting alias = getattr(payload, "alias", None) server_name = getattr(payload, "server_name", None) @@ -409,6 +415,42 @@ def validate_mcp_server_name(server_name: str, raise_http_exception: bool = Fals raise Exception(error_message) +TOOL_DISPLAY_NAME_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$") + + +def validate_tool_display_names(tool_name_to_display_name: Optional[Mapping[str, str]]) -> None: + """ + Validate tool display name overrides against Bedrock's tool-name constraint. + + A display name replaces the tool name sent to the LLM provider, so it must + satisfy the strictest provider requirement in use (Bedrock's + ``[a-zA-Z0-9_-]+``); a name with spaces or other characters saves + successfully but fails every subsequent Bedrock tool call. + + Raises: + HTTPException: If any display name fails the pattern. + """ + if not tool_name_to_display_name: + return + + for original_name, display_name in tool_name_to_display_name.items(): + if display_name and not TOOL_DISPLAY_NAME_PATTERN.match(display_name): + from fastapi import HTTPException + from starlette import status + + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": ( + f"Invalid display name '{display_name}' for tool '{original_name}'. " + "Display names may only contain letters, digits, underscores, and " + "hyphens (no spaces or other special characters), since they replace " + "the tool name sent to the LLM provider." + ) + }, + ) + + class MCPMissingUserEnvVarsError(Exception): """Raised when an MCP request can't be built because the calling user has not supplied one or more required per-user environment variables. diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html index dd2a01991de..4009a7f4b95 100644 --- a/litellm/proxy/_experimental/out/404.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/404/index.html b/litellm/proxy/_experimental/out/404/index.html index dd2a01991de..4009a7f4b95 100644 --- a/litellm/proxy/_experimental/out/404/index.html +++ b/litellm/proxy/_experimental/out/404/index.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt b/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt index 55b18876d5b..6aa34991087 100644 --- a/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt +++ b/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt @@ -1,9 +1,9 @@ 1:"$Sreact.fragment" -2:I[347257,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientPageRoot"] -3:I[871135,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js","/litellm-asset-prefix/_next/static/chunks/011mgw.-67gs_.js","/litellm-asset-prefix/_next/static/chunks/0~-ovi6c4wjt1.js","/litellm-asset-prefix/_next/static/chunks/0_y-b9_d9dsuv.js","/litellm-asset-prefix/_next/static/chunks/0c2apcdkbqq0o.js","/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0ngre0.s4-ej6.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/05t1k89l9tc3s.js","/litellm-asset-prefix/_next/static/chunks/17n.qg70cy9.9.js","/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0-3i_.uof35pm.js","/litellm-asset-prefix/_next/static/chunks/14566-_ogh-19.js","/litellm-asset-prefix/_next/static/chunks/0w39dn9x3dp9g.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0v1rxqc1hqmrl.js","/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js"],"default"] -6:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"OutletBoundary"] +2:I[347257,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientPageRoot"] +3:I[871135,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js","/litellm-asset-prefix/_next/static/chunks/0ae3np_qb52e-.js","/litellm-asset-prefix/_next/static/chunks/0zam.8alu6_vj.js","/litellm-asset-prefix/_next/static/chunks/0uu6lckpr0s15.js","/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0-s2am3eulbyd.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0.w8~sa9q0n_s.js","/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","/litellm-asset-prefix/_next/static/chunks/0efmbzvj03niy.js","/litellm-asset-prefix/_next/static/chunks/055egae-ggkjh.js","/litellm-asset-prefix/_next/static/chunks/0hwip5a7qsmis.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js"],"default"] +6:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"OutletBoundary"] 7:"$Sreact.suspense" -0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/011mgw.-67gs_.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0~-ovi6c4wjt1.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0_y-b9_d9dsuv.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0c2apcdkbqq0o.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0ngre0.s4-ej6.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/05t1k89l9tc3s.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/17n.qg70cy9.9.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0-3i_.uof35pm.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/14566-_ogh-19.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0w39dn9x3dp9g.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0v1rxqc1hqmrl.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"} +0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0ae3np_qb52e-.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0zam.8alu6_vj.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0uu6lckpr0s15.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0-s2am3eulbyd.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0.w8~sa9q0n_s.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0efmbzvj03niy.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/055egae-ggkjh.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0hwip5a7qsmis.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"} 4:{} 5:"$0:rsc:props:children:0:props:serverProvidedParams:params" 8:null diff --git a/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt b/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt index 55176f9118b..b52b61e168b 100644 --- a/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt +++ b/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt @@ -1,7 +1,7 @@ 1:"$Sreact.fragment" -2:I[92825,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientSegmentRoot"] -3:I[216370,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js"],"default"] -4:I[339756,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -5:I[837457,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"} +2:I[92825,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientSegmentRoot"] +3:I[216370,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js"],"default"] +4:I[339756,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +5:I[837457,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"} 6:"$0:rsc:props:children:1:props:serverProvidedParams:params" diff --git a/litellm/proxy/_experimental/out/__next._full.txt b/litellm/proxy/_experimental/out/__next._full.txt index f1ff1ff8411..bb67fb01bc2 100644 --- a/litellm/proxy/_experimental/out/__next._full.txt +++ b/litellm/proxy/_experimental/out/__next._full.txt @@ -1,30 +1,30 @@ 1:"$Sreact.fragment" -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -4:I[557951,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"AuthProvider"] -5:I[339756,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -6:I[837457,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -7:I[92825,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientSegmentRoot"] -8:I[216370,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js"],"default"] -c:I[168027,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default",1] +2:I[867271,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +3:I[71195,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +4:I[557951,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"AuthProvider"] +5:I[339756,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +6:I[837457,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +7:I[92825,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientSegmentRoot"] +8:I[216370,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js"],"default"] +c:I[168027,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default",1] :HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","style"] :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2","font",{"crossOrigin":"","type":"font/woff2"}] -0:{"P":null,"c":["",""],"q":"","i":false,"f":[[["",{"children":["(dashboard)",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",16],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js","async":true,"nonce":"$undefined"}]],["$","$L7",null,{"Component":"$8","slots":{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]},"serverProvidedParams":{"params":{},"promises":["$@9"]}}]]}],{"children":["$La",{},null,false,null]},null,false,null]},null,false,null],"$Lb",false]],"m":"$undefined","G":["$c",["$Ld","$Le"]],"S":true,"h":null,"s":"$undefined","l":"$undefined","p":"$undefined","d":"$undefined","b":"5rDiFx0t_mOGYmV_8kSkw"} -f:I[347257,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientPageRoot"] -10:I[871135,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js","/litellm-asset-prefix/_next/static/chunks/011mgw.-67gs_.js","/litellm-asset-prefix/_next/static/chunks/0~-ovi6c4wjt1.js","/litellm-asset-prefix/_next/static/chunks/0_y-b9_d9dsuv.js","/litellm-asset-prefix/_next/static/chunks/0c2apcdkbqq0o.js","/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0ngre0.s4-ej6.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/05t1k89l9tc3s.js","/litellm-asset-prefix/_next/static/chunks/17n.qg70cy9.9.js","/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0-3i_.uof35pm.js","/litellm-asset-prefix/_next/static/chunks/14566-_ogh-19.js","/litellm-asset-prefix/_next/static/chunks/0w39dn9x3dp9g.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0v1rxqc1hqmrl.js","/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js"],"default"] -13:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"OutletBoundary"] +0:{"P":null,"c":["",""],"q":"","i":false,"f":[[["",{"children":["(dashboard)",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",16],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js","async":true,"nonce":"$undefined"}]],["$","$L7",null,{"Component":"$8","slots":{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]},"serverProvidedParams":{"params":{},"promises":["$@9"]}}]]}],{"children":["$La",{},null,false,null]},null,false,null]},null,false,null],"$Lb",false]],"m":"$undefined","G":["$c",["$Ld","$Le"]],"S":true,"h":null,"s":"$undefined","l":"$undefined","p":"$undefined","d":"$undefined","b":"KYqiq5stbD-H4YcZ-6OuP"} +f:I[347257,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientPageRoot"] +10:I[871135,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js","/litellm-asset-prefix/_next/static/chunks/0ae3np_qb52e-.js","/litellm-asset-prefix/_next/static/chunks/0zam.8alu6_vj.js","/litellm-asset-prefix/_next/static/chunks/0uu6lckpr0s15.js","/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0-s2am3eulbyd.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0.w8~sa9q0n_s.js","/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","/litellm-asset-prefix/_next/static/chunks/0efmbzvj03niy.js","/litellm-asset-prefix/_next/static/chunks/055egae-ggkjh.js","/litellm-asset-prefix/_next/static/chunks/0hwip5a7qsmis.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js"],"default"] +13:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"OutletBoundary"] 14:"$Sreact.suspense" -16:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ViewportBoundary"] -18:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"MetadataBoundary"] -a:["$","$1","c",{"children":[["$","$Lf",null,{"Component":"$10","serverProvidedParams":{"searchParams":{},"params":"$0:f:0:1:1:children:0:props:children:1:props:serverProvidedParams:params","promises":["$@11","$@12"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/011mgw.-67gs_.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0~-ovi6c4wjt1.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0_y-b9_d9dsuv.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0c2apcdkbqq0o.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0ngre0.s4-ej6.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/05t1k89l9tc3s.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/17n.qg70cy9.9.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0-3i_.uof35pm.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/14566-_ogh-19.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0w39dn9x3dp9g.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true,"nonce":"$undefined"}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0v1rxqc1hqmrl.js","async":true,"nonce":"$undefined"}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","async":true,"nonce":"$undefined"}]],["$","$L13",null,{"children":["$","$14",null,{"name":"Next.MetadataOutlet","children":"$@15"}]}]]}] +16:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ViewportBoundary"] +18:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"MetadataBoundary"] +a:["$","$1","c",{"children":[["$","$Lf",null,{"Component":"$10","serverProvidedParams":{"searchParams":{},"params":"$0:f:0:1:1:children:0:props:children:1:props:serverProvidedParams:params","promises":["$@11","$@12"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0ae3np_qb52e-.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0zam.8alu6_vj.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0uu6lckpr0s15.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0-s2am3eulbyd.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0.w8~sa9q0n_s.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0efmbzvj03niy.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/055egae-ggkjh.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0hwip5a7qsmis.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true,"nonce":"$undefined"}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","async":true,"nonce":"$undefined"}]],["$","$L13",null,{"children":["$","$14",null,{"name":"Next.MetadataOutlet","children":"$@15"}]}]]}] b:["$","$1","h",{"children":[null,["$","$L16",null,{"children":"$L17"}],["$","div",null,{"hidden":true,"children":["$","$L18",null,{"children":["$","$14",null,{"name":"Next.Metadata","children":"$L19"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}] d:["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] -e:["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] +e:["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] 9:"$0:f:0:1:1:children:0:props:children:1:props:serverProvidedParams:params" 11:{} 12:"$0:f:0:1:1:children:0:props:children:1:props:serverProvidedParams:params" 17:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]] -1a:I[27201,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"IconMark"] +1a:I[27201,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"IconMark"] 15:null 19:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L1a","4",{}]] diff --git a/litellm/proxy/_experimental/out/__next._head.txt b/litellm/proxy/_experimental/out/__next._head.txt index 27acd699792..c896283665a 100644 --- a/litellm/proxy/_experimental/out/__next._head.txt +++ b/litellm/proxy/_experimental/out/__next._head.txt @@ -1,6 +1,6 @@ 1:"$Sreact.fragment" -2:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ViewportBoundary"] -3:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"MetadataBoundary"] +2:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ViewportBoundary"] +3:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"MetadataBoundary"] 4:"$Sreact.suspense" -5:I[27201,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"IconMark"] -0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"} +5:I[27201,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"IconMark"] +0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"} diff --git a/litellm/proxy/_experimental/out/__next._index.txt b/litellm/proxy/_experimental/out/__next._index.txt index 9714fd22643..e21c0fe74b8 100644 --- a/litellm/proxy/_experimental/out/__next._index.txt +++ b/litellm/proxy/_experimental/out/__next._index.txt @@ -1,9 +1,9 @@ 1:"$Sreact.fragment" -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -4:I[557951,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"AuthProvider"] -5:I[339756,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -6:I[837457,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +2:I[867271,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +3:I[71195,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +4:I[557951,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"AuthProvider"] +5:I[339756,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +6:I[837457,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] :HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","style"] -0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"} +:HL["/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","style"] +0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"} diff --git a/litellm/proxy/_experimental/out/__next._tree.txt b/litellm/proxy/_experimental/out/__next._tree.txt index c8aadb1d1e2..843f0806214 100644 --- a/litellm/proxy/_experimental/out/__next._tree.txt +++ b/litellm/proxy/_experimental/out/__next._tree.txt @@ -1,4 +1,4 @@ :HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","style"] :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2","font",{"crossOrigin":"","type":"font/woff2"}] -0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"5rDiFx0t_mOGYmV_8kSkw"} +0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"KYqiq5stbD-H4YcZ-6OuP"} diff --git a/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_clientMiddlewareManifest.js b/litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_clientMiddlewareManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_clientMiddlewareManifest.js rename to litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_clientMiddlewareManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0-.xiiczht-vh.js b/litellm/proxy/_experimental/out/_next/static/chunks/0-.xiiczht-vh.js new file mode 100644 index 00000000000..9d2f975ff66 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/0-.xiiczht-vh.js @@ -0,0 +1 @@ +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,728889,e=>{"use strict";var r=e.i(290571),t=e.i(271645),a=e.i(829087),o=e.i(480731),l=e.i(444755),d=e.i(673706),s=e.i(95779);let n={xs:{paddingX:"px-1.5",paddingY:"py-1.5"},sm:{paddingX:"px-1.5",paddingY:"py-1.5"},md:{paddingX:"px-2",paddingY:"py-2"},lg:{paddingX:"px-2",paddingY:"py-2"},xl:{paddingX:"px-2.5",paddingY:"py-2.5"}},i={xs:{height:"h-3",width:"w-3"},sm:{height:"h-5",width:"w-5"},md:{height:"h-5",width:"w-5"},lg:{height:"h-7",width:"w-7"},xl:{height:"h-9",width:"w-9"}},m={simple:{rounded:"",border:"",ring:"",shadow:""},light:{rounded:"rounded-tremor-default",border:"",ring:"",shadow:""},shadow:{rounded:"rounded-tremor-default",border:"border",ring:"",shadow:"shadow-tremor-card dark:shadow-dark-tremor-card"},solid:{rounded:"rounded-tremor-default",border:"border-2",ring:"ring-1",shadow:""},outlined:{rounded:"rounded-tremor-default",border:"border",ring:"ring-2",shadow:""}},c=(0,d.makeClassName)("Icon"),g=t.default.forwardRef((e,g)=>{let{icon:u,variant:b="simple",tooltip:f,size:h=o.Sizes.SM,color:w,className:C}=e,k=(0,r.__rest)(e,["icon","variant","tooltip","size","color","className"]),p=((e,r)=>{switch(e){case"simple":return{textColor:r?(0,d.getColorClassNames)(r,s.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:"",borderColor:"",ringColor:""};case"light":return{textColor:r?(0,d.getColorClassNames)(r,s.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:r?(0,l.tremorTwMerge)((0,d.getColorClassNames)(r,s.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-brand-muted dark:bg-dark-tremor-brand-muted",borderColor:"",ringColor:""};case"shadow":return{textColor:r?(0,d.getColorClassNames)(r,s.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:r?(0,l.tremorTwMerge)((0,d.getColorClassNames)(r,s.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:"border-tremor-border dark:border-dark-tremor-border",ringColor:""};case"solid":return{textColor:r?(0,d.getColorClassNames)(r,s.colorPalette.text).textColor:"text-tremor-brand-inverted dark:text-dark-tremor-brand-inverted",bgColor:r?(0,l.tremorTwMerge)((0,d.getColorClassNames)(r,s.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-brand dark:bg-dark-tremor-brand",borderColor:"border-tremor-brand-inverted dark:border-dark-tremor-brand-inverted",ringColor:"ring-tremor-ring dark:ring-dark-tremor-ring"};case"outlined":return{textColor:r?(0,d.getColorClassNames)(r,s.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:r?(0,l.tremorTwMerge)((0,d.getColorClassNames)(r,s.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:r?(0,d.getColorClassNames)(r,s.colorPalette.ring).borderColor:"border-tremor-brand-subtle dark:border-dark-tremor-brand-subtle",ringColor:r?(0,l.tremorTwMerge)((0,d.getColorClassNames)(r,s.colorPalette.ring).ringColor,"ring-opacity-40"):"ring-tremor-brand-muted dark:ring-dark-tremor-brand-muted"}}})(b,w),{tooltipProps:x,getReferenceProps:N}=(0,a.useTooltip)();return t.default.createElement("span",Object.assign({ref:(0,d.mergeRefs)([g,x.refs.setReference]),className:(0,l.tremorTwMerge)(c("root"),"inline-flex shrink-0 items-center justify-center",p.bgColor,p.textColor,p.borderColor,p.ringColor,m[b].rounded,m[b].border,m[b].shadow,m[b].ring,n[h].paddingX,n[h].paddingY,C)},N,k),t.default.createElement(a.default,Object.assign({text:f},x)),t.default.createElement(u,{className:(0,l.tremorTwMerge)(c("icon"),"shrink-0",i[h].height,i[h].width)}))});g.displayName="Icon",e.s(["default",0,g],728889)},752978,e=>{"use strict";var r=e.i(728889);e.s(["Icon",()=>r.default])},871943,e=>{"use strict";var r=e.i(271645);let t=r.forwardRef(function(e,t){return r.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:t},e),r.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M19 9l-7 7-7-7"}))});e.s(["ChevronDownIcon",0,t],871943)},360820,e=>{"use strict";var r=e.i(271645);let t=r.forwardRef(function(e,t){return r.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:t},e),r.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M5 15l7-7 7 7"}))});e.s(["ChevronUpIcon",0,t],360820)},269200,e=>{"use strict";var r=e.i(290571),t=e.i(271645),a=e.i(444755);let o=(0,e.i(673706).makeClassName)("Table"),l=t.default.forwardRef((e,l)=>{let{children:d,className:s}=e,n=(0,r.__rest)(e,["children","className"]);return t.default.createElement("div",{className:(0,a.tremorTwMerge)(o("root"),"overflow-auto",s)},t.default.createElement("table",Object.assign({ref:l,className:(0,a.tremorTwMerge)(o("table"),"w-full text-tremor-default","text-tremor-content","dark:text-dark-tremor-content")},n),d))});l.displayName="Table",e.s(["Table",0,l],269200)},427612,e=>{"use strict";var r=e.i(290571),t=e.i(271645),a=e.i(444755);let o=(0,e.i(673706).makeClassName)("TableHead"),l=t.default.forwardRef((e,l)=>{let{children:d,className:s}=e,n=(0,r.__rest)(e,["children","className"]);return t.default.createElement(t.default.Fragment,null,t.default.createElement("thead",Object.assign({ref:l,className:(0,a.tremorTwMerge)(o("root"),"text-left","text-tremor-content","dark:text-dark-tremor-content",s)},n),d))});l.displayName="TableHead",e.s(["TableHead",0,l],427612)},64848,e=>{"use strict";var r=e.i(290571),t=e.i(271645),a=e.i(444755);let o=(0,e.i(673706).makeClassName)("TableHeaderCell"),l=t.default.forwardRef((e,l)=>{let{children:d,className:s}=e,n=(0,r.__rest)(e,["children","className"]);return t.default.createElement(t.default.Fragment,null,t.default.createElement("th",Object.assign({ref:l,className:(0,a.tremorTwMerge)(o("root"),"whitespace-nowrap text-left font-semibold top-0 px-4 py-3.5","text-tremor-content-strong","dark:text-dark-tremor-content-strong",s)},n),d))});l.displayName="TableHeaderCell",e.s(["TableHeaderCell",0,l],64848)},942232,e=>{"use strict";var r=e.i(290571),t=e.i(271645),a=e.i(444755);let o=(0,e.i(673706).makeClassName)("TableBody"),l=t.default.forwardRef((e,l)=>{let{children:d,className:s}=e,n=(0,r.__rest)(e,["children","className"]);return t.default.createElement(t.default.Fragment,null,t.default.createElement("tbody",Object.assign({ref:l,className:(0,a.tremorTwMerge)(o("root"),"align-top divide-y","divide-tremor-border","dark:divide-dark-tremor-border",s)},n),d))});l.displayName="TableBody",e.s(["TableBody",0,l],942232)},496020,e=>{"use strict";var r=e.i(290571),t=e.i(271645),a=e.i(444755);let o=(0,e.i(673706).makeClassName)("TableRow"),l=t.default.forwardRef((e,l)=>{let{children:d,className:s}=e,n=(0,r.__rest)(e,["children","className"]);return t.default.createElement(t.default.Fragment,null,t.default.createElement("tr",Object.assign({ref:l,className:(0,a.tremorTwMerge)(o("row"),s)},n),d))});l.displayName="TableRow",e.s(["TableRow",0,l],496020)},977572,e=>{"use strict";var r=e.i(290571),t=e.i(271645),a=e.i(444755);let o=(0,e.i(673706).makeClassName)("TableCell"),l=t.default.forwardRef((e,l)=>{let{children:d,className:s}=e,n=(0,r.__rest)(e,["children","className"]);return t.default.createElement(t.default.Fragment,null,t.default.createElement("td",Object.assign({ref:l,className:(0,a.tremorTwMerge)(o("root"),"align-middle whitespace-nowrap text-left p-4",s)},n),d))});l.displayName="TableCell",e.s(["TableCell",0,l],977572)},68155,e=>{"use strict";var r=e.i(271645);let t=r.forwardRef(function(e,t){return r.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:t},e),r.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"}))});e.s(["TrashIcon",0,t],68155)},278587,e=>{"use strict";var r=e.i(271645);let t=r.forwardRef(function(e,t){return r.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:t},e),r.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15"}))});e.s(["RefreshIcon",0,t],278587)},973095,e=>{"use strict";var r=e.i(843476),t=e.i(502501),a=e.i(135214),o=e.i(936578),l=e.i(271645);function d(){let{isLoading:e,isAuthorized:l}=(0,a.default)();return e||!l?(0,r.jsx)(o.default,{}):(0,r.jsx)(t.default,{})}e.s(["default",0,function(){return(0,r.jsx)(l.Suspense,{fallback:(0,r.jsx)(o.default,{}),children:(0,r.jsx)(d,{})})}])}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0-0c4mv4-mc9n.js b/litellm/proxy/_experimental/out/_next/static/chunks/0-0c4mv4-mc9n.js new file mode 100644 index 00000000000..8f16e50edb1 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/0-0c4mv4-mc9n.js @@ -0,0 +1 @@ +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,186312,e=>{"use strict";var t=new WeakMap,r=new WeakMap,n={},s=0,o=function(e){return e&&(e.host||o(e.parentNode))},i=function(e,i,a,l){var u=(Array.isArray(e)?e:[e]).map(function(e){if(i.contains(e))return e;var t=o(e);return t&&i.contains(t)?t:(console.error("aria-hidden",e,"in not contained inside",i,". Doing nothing"),null)}).filter(function(e){return!!e});n[a]||(n[a]=new WeakMap);var c=n[a],d=[],h=new Set,p=new Set(u),f=function(e){!e||h.has(e)||(h.add(e),f(e.parentNode))};u.forEach(f);var m=function(e){!e||p.has(e)||Array.prototype.forEach.call(e.children,function(e){if(h.has(e))m(e);else try{var n=e.getAttribute(l),s=null!==n&&"false"!==n,o=(t.get(e)||0)+1,i=(c.get(e)||0)+1;t.set(e,o),c.set(e,i),d.push(e),1===o&&s&&r.set(e,!0),1===i&&e.setAttribute(a,"true"),s||e.setAttribute(l,"true")}catch(t){console.error("aria-hidden: cannot operate on ",e,t)}})};return m(i),h.clear(),s++,function(){d.forEach(function(e){var n=t.get(e)-1,s=c.get(e)-1;t.set(e,n),c.set(e,s),n||(r.has(e)||e.removeAttribute(l),r.delete(e)),s||e.removeAttribute(a)}),--s||(t=new WeakMap,t=new WeakMap,r=new WeakMap,n={})}};e.s(["hideOthers",0,function(e,t,r){void 0===r&&(r="data-aria-hidden");var n=Array.from(Array.isArray(e)?e:[e]),s=t||("u"{t.exports=e.r(976562)},266027,869230,469637,e=>{"use strict";let t;var r=e.i(175555),n=e.i(273911),s=e.i(540143),o=e.i(286491),i=e.i(915823),a=e.i(793803),l=e.i(619273),u=e.i(180166),c=class extends i.Subscribable{constructor(e,t){super(),this.options=t,this.#e=e,this.#t=null,this.#r=(0,a.pendingThenable)(),this.bindMethods(),this.setOptions(t)}#e;#n=void 0;#s=void 0;#o=void 0;#i;#a;#r;#t;#l;#u;#c;#d;#h;#p;#f=new Set;bindMethods(){this.refetch=this.refetch.bind(this)}onSubscribe(){1===this.listeners.size&&(this.#n.addObserver(this),d(this.#n,this.options)?this.#m():this.updateResult(),this.#g())}onUnsubscribe(){this.hasListeners()||this.destroy()}shouldFetchOnReconnect(){return h(this.#n,this.options,this.options.refetchOnReconnect)}shouldFetchOnWindowFocus(){return h(this.#n,this.options,this.options.refetchOnWindowFocus)}destroy(){this.listeners=new Set,this.#y(),this.#b(),this.#n.removeObserver(this)}setOptions(e){let t=this.options,r=this.#n;if(this.options=this.#e.defaultQueryOptions(e),void 0!==this.options.enabled&&"boolean"!=typeof this.options.enabled&&"function"!=typeof this.options.enabled&&"boolean"!=typeof(0,l.resolveQueryBoolean)(this.options.enabled,this.#n))throw Error("Expected enabled to be a boolean or a callback that returns a boolean");this.#v(),this.#n.setOptions(this.options),t._defaulted&&!(0,l.shallowEqualObjects)(this.options,t)&&this.#e.getQueryCache().notify({type:"observerOptionsUpdated",query:this.#n,observer:this});let n=this.hasListeners();n&&p(this.#n,r,this.options,t)&&this.#m(),this.updateResult(),n&&(this.#n!==r||(0,l.resolveQueryBoolean)(this.options.enabled,this.#n)!==(0,l.resolveQueryBoolean)(t.enabled,this.#n)||(0,l.resolveStaleTime)(this.options.staleTime,this.#n)!==(0,l.resolveStaleTime)(t.staleTime,this.#n))&&this.#C();let s=this.#R();n&&(this.#n!==r||(0,l.resolveQueryBoolean)(this.options.enabled,this.#n)!==(0,l.resolveQueryBoolean)(t.enabled,this.#n)||s!==this.#p)&&this.#S(s)}getOptimisticResult(e){var t,r;let n=this.#e.getQueryCache().build(this.#e,e),s=this.createResult(n,e);return t=this,r=s,(0,l.shallowEqualObjects)(t.getCurrentResult(),r)||(this.#o=s,this.#a=this.options,this.#i=this.#n.state),s}getCurrentResult(){return this.#o}trackResult(e,t){return new Proxy(e,{get:(e,r)=>(this.trackProp(r),t?.(r),"promise"===r&&(this.trackProp("data"),this.options.experimental_prefetchInRender||"pending"!==this.#r.status||this.#r.reject(Error("experimental_prefetchInRender feature flag is not enabled"))),Reflect.get(e,r))})}trackProp(e){this.#f.add(e)}getCurrentQuery(){return this.#n}refetch({...e}={}){return this.fetch({...e})}fetchOptimistic(e){let t=this.#e.defaultQueryOptions(e),r=this.#e.getQueryCache().build(this.#e,t);return r.fetch().then(()=>this.createResult(r,t))}fetch(e){return this.#m({...e,cancelRefetch:e.cancelRefetch??!0}).then(()=>(this.updateResult(),this.#o))}#m(e){this.#v();let t=this.#n.fetch(this.options,e);return e?.throwOnError||(t=t.catch(l.noop)),t}#C(){this.#y();let e=(0,l.resolveStaleTime)(this.options.staleTime,this.#n);if(n.environmentManager.isServer()||this.#o.isStale||!(0,l.isValidTimeout)(e))return;let t=(0,l.timeUntilStale)(this.#o.dataUpdatedAt,e);this.#d=u.timeoutManager.setTimeout(()=>{this.#o.isStale||this.updateResult()},t+1)}#R(){return("function"==typeof this.options.refetchInterval?this.options.refetchInterval(this.#n):this.options.refetchInterval)??!1}#S(e){this.#b(),this.#p=e,!n.environmentManager.isServer()&&!1!==(0,l.resolveQueryBoolean)(this.options.enabled,this.#n)&&(0,l.isValidTimeout)(this.#p)&&0!==this.#p&&(this.#h=u.timeoutManager.setInterval(()=>{(this.options.refetchIntervalInBackground||r.focusManager.isFocused())&&this.#m()},this.#p))}#g(){this.#C(),this.#S(this.#R())}#y(){void 0!==this.#d&&(u.timeoutManager.clearTimeout(this.#d),this.#d=void 0)}#b(){void 0!==this.#h&&(u.timeoutManager.clearInterval(this.#h),this.#h=void 0)}createResult(e,t){let r,n=this.#n,s=this.options,i=this.#o,u=this.#i,c=this.#a,h=e!==n?e.state:this.#s,{state:m}=e,g={...m},y=!1;if(t._optimisticResults){let r=this.hasListeners(),i=!r&&d(e,t),a=r&&p(e,n,t,s);(i||a)&&(g={...g,...(0,o.fetchState)(m.data,e.options)}),"isRestoring"===t._optimisticResults&&(g.fetchStatus="idle")}let{error:b,errorUpdatedAt:v,status:C}=g;r=g.data;let R=!1;if(void 0!==t.placeholderData&&void 0===r&&"pending"===C){let e;i?.isPlaceholderData&&t.placeholderData===c?.placeholderData?(e=i.data,R=!0):e="function"==typeof t.placeholderData?t.placeholderData(this.#c?.state.data,this.#c):t.placeholderData,void 0!==e&&(C="success",r=(0,l.replaceData)(i?.data,e,t),y=!0)}if(t.select&&void 0!==r&&!R)if(i&&r===u?.data&&t.select===this.#l)r=this.#u;else try{this.#l=t.select,r=t.select(r),r=(0,l.replaceData)(i?.data,r,t),this.#u=r,this.#t=null}catch(e){this.#t=e}this.#t&&(b=this.#t,r=this.#u,v=Date.now(),C="error");let S="fetching"===g.fetchStatus,O="pending"===C,w="error"===C,k=O&&S,I=void 0!==r,x={status:C,fetchStatus:g.fetchStatus,isPending:O,isSuccess:"success"===C,isError:w,isInitialLoading:k,isLoading:k,data:r,dataUpdatedAt:g.dataUpdatedAt,error:b,errorUpdatedAt:v,failureCount:g.fetchFailureCount,failureReason:g.fetchFailureReason,errorUpdateCount:g.errorUpdateCount,isFetched:e.isFetched(),isFetchedAfterMount:g.dataUpdateCount>h.dataUpdateCount||g.errorUpdateCount>h.errorUpdateCount,isFetching:S,isRefetching:S&&!O,isLoadingError:w&&!I,isPaused:"paused"===g.fetchStatus,isPlaceholderData:y,isRefetchError:w&&I,isStale:f(e,t),refetch:this.refetch,promise:this.#r,isEnabled:!1!==(0,l.resolveQueryBoolean)(t.enabled,e)};if(this.options.experimental_prefetchInRender){let t=void 0!==x.data,r="error"===x.status&&!t,s=e=>{r?e.reject(x.error):t&&e.resolve(x.data)},o=()=>{s(this.#r=x.promise=(0,a.pendingThenable)())},i=this.#r;switch(i.status){case"pending":e.queryHash===n.queryHash&&s(i);break;case"fulfilled":(r||x.data!==i.value)&&o();break;case"rejected":r&&x.error===i.reason||o()}}return x}updateResult(){let e=this.#o,t=this.createResult(this.#n,this.options);if(this.#i=this.#n.state,this.#a=this.options,void 0!==this.#i.data&&(this.#c=this.#n),(0,l.shallowEqualObjects)(t,e))return;this.#o=t;let r=()=>{if(!e)return!0;let{notifyOnChangeProps:t}=this.options,r="function"==typeof t?t():t;if("all"===r||!r&&!this.#f.size)return!0;let n=new Set(r??this.#f);return this.options.throwOnError&&n.add("error"),Object.keys(this.#o).some(t=>this.#o[t]!==e[t]&&n.has(t))};this.#O({listeners:r()})}#v(){let e=this.#e.getQueryCache().build(this.#e,this.options);if(e===this.#n)return;let t=this.#n;this.#n=e,this.#s=e.state,this.hasListeners()&&(t?.removeObserver(this),e.addObserver(this))}onQueryUpdate(){this.updateResult(),this.hasListeners()&&this.#g()}#O(e){s.notifyManager.batch(()=>{e.listeners&&this.listeners.forEach(e=>{e(this.#o)}),this.#e.getQueryCache().notify({query:this.#n,type:"observerResultsUpdated"})})}};function d(e,t){return!1!==(0,l.resolveQueryBoolean)(t.enabled,e)&&void 0===e.state.data&&("error"!==e.state.status||!1!==(0,l.resolveQueryBoolean)(t.retryOnMount,e))||void 0!==e.state.data&&h(e,t,t.refetchOnMount)}function h(e,t,r){if(!1!==(0,l.resolveQueryBoolean)(t.enabled,e)&&"static"!==(0,l.resolveStaleTime)(t.staleTime,e)){let n="function"==typeof r?r(e):r;return"always"===n||!1!==n&&f(e,t)}return!1}function p(e,t,r,n){return(e!==t||!1===(0,l.resolveQueryBoolean)(n.enabled,e))&&(!r.suspense||"error"!==e.state.status)&&f(e,r)}function f(e,t){return!1!==(0,l.resolveQueryBoolean)(t.enabled,e)&&e.isStaleByTime((0,l.resolveStaleTime)(t.staleTime,e))}e.s(["QueryObserver",0,c],869230),e.i(247167);var m=e.i(271645),g=e.i(912598);e.i(843476);var y=m.createContext((t=!1,{clearReset:()=>{t=!1},reset:()=>{t=!0},isReset:()=>t})),b=m.createContext(!1);b.Provider;var v=(e,t,r)=>t.fetchOptimistic(e).catch(()=>{r.clearReset()});function C(e,t,r){let o,i=m.useContext(b),a=m.useContext(y),u=(0,g.useQueryClient)(r),c=u.defaultQueryOptions(e);u.getDefaultOptions().queries?._experimental_beforeQuery?.(c);let d=u.getQueryCache().get(c.queryHash);if(c._optimisticResults=i?"isRestoring":"optimistic",c.suspense){let e=e=>"static"===e?e:Math.max(e??1e3,1e3),t=c.staleTime;c.staleTime="function"==typeof t?(...r)=>e(t(...r)):e(t),"number"==typeof c.gcTime&&(c.gcTime=Math.max(c.gcTime,1e3))}o=d?.state.error&&"function"==typeof c.throwOnError?(0,l.shouldThrowError)(c.throwOnError,[d.state.error,d]):c.throwOnError,(c.suspense||c.experimental_prefetchInRender||o)&&!a.isReset()&&(c.retryOnMount=!1),m.useEffect(()=>{a.clearReset()},[a]);let h=!u.getQueryCache().get(c.queryHash),[p]=m.useState(()=>new t(u,c)),f=p.getOptimisticResult(c),C=!i&&!1!==e.subscribed;if(m.useSyncExternalStore(m.useCallback(e=>{let t=C?p.subscribe(s.notifyManager.batchCalls(e)):l.noop;return p.updateResult(),t},[p,C]),()=>p.getCurrentResult(),()=>p.getCurrentResult()),m.useEffect(()=>{p.setOptions(c)},[c,p]),c?.suspense&&f.isPending)throw v(c,p,a);if((({result:e,errorResetBoundary:t,throwOnError:r,query:n,suspense:s})=>e.isError&&!t.isReset()&&!e.isFetching&&n&&(s&&void 0===e.data||(0,l.shouldThrowError)(r,[e.error,n])))({result:f,errorResetBoundary:a,throwOnError:c.throwOnError,query:d,suspense:c.suspense}))throw f.error;if(u.getDefaultOptions().queries?._experimental_afterQuery?.(c,f),c.experimental_prefetchInRender&&!n.environmentManager.isServer()&&f.isLoading&&f.isFetching&&!i){let e=h?v(c,p,a):d?.promise;e?.catch(l.noop).finally(()=>{p.updateResult()})}return c.notifyOnChangeProps?f:p.trackResult(f)}e.s(["useBaseQuery",0,C],469637),e.s(["useQuery",0,function(e,t){return C(e,c,t)}],266027)},612256,243652,e=>{"use strict";var t=e.i(602869),r=e.i(266027);function n(e){let t=[e];return{all:t,lists:()=>[...t,"list"],list:e=>[...t,"list",{params:e}],details:()=>[...t,"detail"],detail:e=>[...t,"detail",e]}}e.s(["createQueryKeys",0,n],243652);let s=n("uiConfig");e.s(["useUIConfig",0,()=>(0,r.useQuery)({queryKey:s.list({}),queryFn:async()=>await (0,t.getUiConfig)(),staleTime:864e5,gcTime:864e5})],612256)},321836,e=>{"use strict";let t="litellm_return_url",r="redirect_to";function n(){return window.location.href}function s(){if("u"typeof document&&(document.cookie=`${t}=; path=/; max-age=0`)}catch(e){console.error("Failed to clear return URL cookie:",e)}}function i(){return new URLSearchParams(window.location.search).get(r)}function a(){let e=window.location.hostname;return"localhost"===e||"127.0.0.1"===e||"::1"===e||e.startsWith("127.")||e.endsWith(".local")}function l(e){if(!e)return!1;if(e.startsWith("/")&&!e.startsWith("//"))return!0;try{let t=new URL(e),r=window.location.hostname;if(t.hostname!==r)return!1;if(a())return!0;return t.origin===window.location.origin}catch{return!1}}e.s(["buildLoginUrlWithReturn",0,function(e,t){let s=t||n();if(!s||s.includes("/login"))return e;let o=e.includes("?")?"&":"?";return`${e}${o}${r}=${encodeURIComponent(s)}`},"clearStoredReturnUrl",0,o,"consumeReturnUrl",0,function(){let e=i();if(e){if(l(e))return o(),e;a()&&console.warn("[returnUrlUtils] Invalid return URL in params rejected:",e)}let t=s();if(t){if(l(t))return o(),t;a()&&console.warn("[returnUrlUtils] Invalid return URL in cookie rejected:",t)}return null},"getReturnUrl",0,function(){let e=i();if(e)return e;let t=s();return t||null},"isValidReturnUrl",0,l,"normalizeUrlForCompare",0,function(e){try{let t=new URL(e,window.location.origin),r=t.pathname;r.length>1&&r.endsWith("/")&&(r=r.slice(0,-1));let n=new URLSearchParams(t.search),s=new URLSearchParams;Array.from(n.entries()).sort(([e],[t])=>e.localeCompare(t)).forEach(([e,t])=>{s.append(e,t)});let o=s.toString(),i=t.hash||"";return`${t.origin}${r}${o?`?${o}`:""}${i}`}catch{return e}},"storeReturnUrl",0,function(){let e=n();e&&function(e,t,r=300){if("u"{"use strict";var t=e.i(602869),r=e.i(268004),n=e.i(161281),s=e.i(321836),o=e.i(618566),i=e.i(271645),a=e.i(708347),l=e.i(612256);e.s(["default",0,()=>{let e=(0,o.useRouter)(),{data:u,isLoading:c}=(0,l.useUIConfig)(),d="u">typeof document?(0,r.getCookie)("token"):null,h=(0,i.useMemo)(()=>(0,n.decodeToken)(d),[d]),p=(0,i.useMemo)(()=>(0,n.checkTokenValidity)(d),[d])&&!u?.admin_ui_disabled,f=(0,i.useCallback)(()=>{(0,s.storeReturnUrl)();let r=`${(0,t.getProxyBaseUrl)()}/ui/login`,n=(0,s.buildLoginUrlWithReturn)(r);e.replace(n)},[e]);return(0,i.useEffect)(()=>{!c&&(p||(d&&(0,r.clearTokenCookies)(),f()))},[c,p,d,f]),{isLoading:c,isAuthorized:p,token:p?d:null,accessToken:h?.key??null,userId:h?.user_id??null,userEmail:h?.user_email??null,userRole:(0,a.formatUserRole)(h?.user_role),premiumUser:h?.premium_user??null,disabledPersonalKeyCreation:h?.disabled_non_admin_personal_key_creation??null,showSSOBanner:h?.login_method==="username_password"}}])},95779,e=>{"use strict";var t=e.i(480731);let r=[t.BaseColors.Blue,t.BaseColors.Cyan,t.BaseColors.Sky,t.BaseColors.Indigo,t.BaseColors.Violet,t.BaseColors.Purple,t.BaseColors.Fuchsia,t.BaseColors.Slate,t.BaseColors.Gray,t.BaseColors.Zinc,t.BaseColors.Neutral,t.BaseColors.Stone,t.BaseColors.Red,t.BaseColors.Orange,t.BaseColors.Amber,t.BaseColors.Yellow,t.BaseColors.Lime,t.BaseColors.Green,t.BaseColors.Emerald,t.BaseColors.Teal,t.BaseColors.Pink,t.BaseColors.Rose];e.s(["colorPalette",0,{canvasBackground:50,lightBackground:100,background:500,darkBackground:600,darkestBackground:800,lightBorder:200,border:500,darkBorder:700,lightRing:200,ring:300,iconRing:500,lightText:400,text:500,iconText:600,darkText:700,darkestText:900,icon:500},"themeColorRange",0,r])},563113,887719,e=>{"use strict";var t=e.i(271645),r=e.i(864517),n=e.i(244009),s=e.i(408850),o=e.i(87414);let i=function(...e){let t={};return e.forEach(e=>{e&&Object.keys(e).forEach(r=>{void 0!==e[r]&&(t[r]=e[r])})}),t};function a(e){let{closable:r,closeIcon:n}=e||{};return t.default.useMemo(()=>{if(!r&&(!1===r||!1===n||null===n))return!1;if(void 0===r&&void 0===n)return null;let e={closeIcon:"boolean"!=typeof n&&null!==n?n:void 0};return r&&"object"==typeof r&&(e=Object.assign(Object.assign({},e),r)),e},[r,n])}e.s(["default",0,i],887719);let l={};e.s(["pickClosable",0,function(e){if(!e)return;let{closable:t,closeIcon:r}=e;return{closable:t,closeIcon:r}},"useClosable",0,(e,u,c=l)=>{let d=a(e),h=a(u),[p]=(0,s.useLocale)("global",o.default.global),f="boolean"!=typeof d&&!!(null==d?void 0:d.disabled),m=t.default.useMemo(()=>Object.assign({closeIcon:t.default.createElement(r.default,null)},c),[c]),g=t.default.useMemo(()=>!1!==d&&(d?i(m,h,d):!1!==h&&(h?i(m,h):!!m.closable&&m)),[d,h,m]);return t.default.useMemo(()=>{var e,r;if(!1===g)return[!1,null,f,{}];let{closeIconRender:s}=m,{closeIcon:o}=g,i=o,a=(0,n.default)(g,!0);return null!=i&&(s&&(i=s(o)),i=t.default.isValidElement(i)?t.default.cloneElement(i,Object.assign(Object.assign(Object.assign({},i.props),{"aria-label":null!=(r=null==(e=i.props)?void 0:e["aria-label"])?r:p.close}),a)):t.default.createElement("span",Object.assign({"aria-label":p.close},a),i)),[!0,i,f,a]},[f,p.close,g,m])}],563113)},801312,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let n={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M724 218.3V141c0-6.7-7.7-10.4-12.9-6.3L260.3 486.8a31.86 31.86 0 000 50.3l450.8 352.1c5.3 4.1 12.9.4 12.9-6.3v-77.3c0-4.9-2.3-9.6-6.1-12.6l-360-281 360-281.1c3.8-3 6.1-7.7 6.1-12.6z"}}]},name:"left",theme:"outlined"};var s=e.i(9583),o=r.forwardRef(function(e,o){return r.createElement(s.default,(0,t.default)({},e,{ref:o,icon:n}))});e.s(["default",0,o],801312)},38243,908286,e=>{"use strict";e.i(247167);var t=e.i(271645),r=e.i(343794),n=e.i(876556);function s(e){return["small","middle","large"].includes(e)}function o(e){return!!e&&"number"==typeof e&&!Number.isNaN(e)}e.s(["isPresetSize",0,s,"isValidGapNumber",0,o],908286);var i=e.i(242064),a=e.i(249616),l=e.i(372409),u=e.i(246422);let c=(0,u.genStyleHooks)(["Space","Addon"],e=>[(e=>{let{componentCls:t,borderRadius:r,paddingSM:n,colorBorder:s,paddingXS:o,fontSizeLG:i,fontSizeSM:a,borderRadiusLG:u,borderRadiusSM:c,colorBgContainerDisabled:d,lineWidth:h}=e;return{[t]:[{display:"inline-flex",alignItems:"center",gap:0,paddingInline:n,margin:0,background:d,borderWidth:h,borderStyle:"solid",borderColor:s,borderRadius:r,"&-large":{fontSize:i,borderRadius:u},"&-small":{paddingInline:o,borderRadius:c,fontSize:a},"&-compact-last-item":{borderEndStartRadius:0,borderStartStartRadius:0},"&-compact-first-item":{borderEndEndRadius:0,borderStartEndRadius:0},"&-compact-item:not(:first-child):not(:last-child)":{borderRadius:0},"&-compact-item:not(:last-child)":{borderInlineEndWidth:0}},(0,l.genCompactItemStyle)(e,{focus:!1})]}})(e)]);var d=function(e,t){var r={};for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&0>t.indexOf(n)&&(r[n]=e[n]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var s=0,n=Object.getOwnPropertySymbols(e);st.indexOf(n[s])&&Object.prototype.propertyIsEnumerable.call(e,n[s])&&(r[n[s]]=e[n[s]]);return r};let h=t.default.forwardRef((e,n)=>{let{className:s,children:o,style:l,prefixCls:u}=e,h=d(e,["className","children","style","prefixCls"]),{getPrefixCls:p,direction:f}=t.default.useContext(i.ConfigContext),m=p("space-addon",u),[g,y,b]=c(m),{compactItemClassnames:v,compactSize:C}=(0,a.useCompactItemContext)(m,f),R=(0,r.default)(m,y,v,b,{[`${m}-${C}`]:C},s);return g(t.default.createElement("div",Object.assign({ref:n,className:R,style:l},h),o))}),p=t.default.createContext({latestIndex:0}),f=p.Provider,m=({className:e,index:r,children:n,split:s,style:o})=>{let{latestIndex:i}=t.useContext(p);return null==n?null:t.createElement(t.Fragment,null,t.createElement("div",{className:e,style:o},n),r{let t=(0,g.mergeToken)(e,{spaceGapSmallSize:e.paddingXS,spaceGapMiddleSize:e.padding,spaceGapLargeSize:e.paddingLG});return[(e=>{let{componentCls:t,antCls:r}=e;return{[t]:{display:"inline-flex","&-rtl":{direction:"rtl"},"&-vertical":{flexDirection:"column"},"&-align":{flexDirection:"column","&-center":{alignItems:"center"},"&-start":{alignItems:"flex-start"},"&-end":{alignItems:"flex-end"},"&-baseline":{alignItems:"baseline"}},[`${t}-item:empty`]:{display:"none"},[`${t}-item > ${r}-badge-not-a-wrapper:only-child`]:{display:"block"}}}})(t),(e=>{let{componentCls:t}=e;return{[t]:{"&-gap-row-small":{rowGap:e.spaceGapSmallSize},"&-gap-row-middle":{rowGap:e.spaceGapMiddleSize},"&-gap-row-large":{rowGap:e.spaceGapLargeSize},"&-gap-col-small":{columnGap:e.spaceGapSmallSize},"&-gap-col-middle":{columnGap:e.spaceGapMiddleSize},"&-gap-col-large":{columnGap:e.spaceGapLargeSize}}}})(t)]},()=>({}),{resetStyle:!1});var b=function(e,t){var r={};for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&0>t.indexOf(n)&&(r[n]=e[n]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var s=0,n=Object.getOwnPropertySymbols(e);st.indexOf(n[s])&&Object.prototype.propertyIsEnumerable.call(e,n[s])&&(r[n[s]]=e[n[s]]);return r};let v=t.forwardRef((e,a)=>{var l;let{getPrefixCls:u,direction:c,size:d,className:h,style:p,classNames:g,styles:v}=(0,i.useComponentConfig)("space"),{size:C=null!=d?d:"small",align:R,className:S,rootClassName:O,children:w,direction:k="horizontal",prefixCls:I,split:x,style:E,wrap:Q=!1,classNames:T,styles:$}=e,B=b(e,["size","align","className","rootClassName","children","direction","prefixCls","split","style","wrap","classNames","styles"]),[j,U]=Array.isArray(C)?C:[C,C],P=s(U),M=s(j),L=o(U),A=o(j),N=(0,n.default)(w,{keepEmpty:!0}),F=void 0===R&&"horizontal"===k?"center":R,_=u("space",I),[z,W,D]=y(_),G=(0,r.default)(_,h,W,`${_}-${k}`,{[`${_}-rtl`]:"rtl"===c,[`${_}-align-${F}`]:F,[`${_}-gap-row-${U}`]:P,[`${_}-gap-col-${j}`]:M},S,O,D),q=(0,r.default)(`${_}-item`,null!=(l=null==T?void 0:T.item)?l:g.item),H=Object.assign(Object.assign({},v.item),null==$?void 0:$.item),V=N.map((e,r)=>{let n=(null==e?void 0:e.key)||`${q}-${r}`;return t.createElement(m,{className:q,key:n,index:r,split:x,style:H},e)}),K=t.useMemo(()=>({latestIndex:N.reduce((e,t,r)=>null!=t?r:e,0)}),[N]);if(0===N.length)return null;let Z={};return Q&&(Z.flexWrap="wrap"),!M&&A&&(Z.columnGap=j),!P&&L&&(Z.rowGap=U),z(t.createElement("div",Object.assign({ref:a,className:G,style:Object.assign(Object.assign(Object.assign({},Z),p),E)},B),t.createElement(f,{value:K},V)))});v.Compact=a.default,v.Addon=h,e.s(["default",0,v],38243)},475254,e=>{"use strict";var t=e.i(271645);let r=e=>{let t=e.replace(/^([A-Z])|[\s-_]+(\w)/g,(e,t,r)=>r?r.toUpperCase():t.toLowerCase());return t.charAt(0).toUpperCase()+t.slice(1)},n=(...e)=>e.filter((e,t,r)=>!!e&&""!==e.trim()&&r.indexOf(e)===t).join(" ").trim();var s={xmlns:"http://www.w3.org/2000/svg",width:24,height:24,viewBox:"0 0 24 24",fill:"none",stroke:"currentColor",strokeWidth:2,strokeLinecap:"round",strokeLinejoin:"round"};let o=(0,t.forwardRef)(({color:e="currentColor",size:r=24,strokeWidth:o=2,absoluteStrokeWidth:i,className:a="",children:l,iconNode:u,...c},d)=>(0,t.createElement)("svg",{ref:d,...s,width:r,height:r,stroke:e,strokeWidth:i?24*Number(o)/Number(r):o,className:n("lucide",a),...!l&&!(e=>{for(let t in e)if(t.startsWith("aria-")||"role"===t||"title"===t)return!0})(c)&&{"aria-hidden":"true"},...c},[...u.map(([e,r])=>(0,t.createElement)(e,r)),...Array.isArray(l)?l:[l]]));e.s(["default",0,(e,s)=>{let i=(0,t.forwardRef)(({className:i,...a},l)=>(0,t.createElement)(o,{ref:l,iconNode:s,className:n(`lucide-${r(e).replace(/([a-z0-9])([A-Z])/g,"$1-$2").toLowerCase()}`,`lucide-${e}`,i),...a}));return i.displayName=r(e),i}],475254)},262218,e=>{"use strict";e.i(247167);var t=e.i(271645),r=e.i(343794),n=e.i(529681),s=e.i(702779),o=e.i(563113),i=e.i(763731),a=e.i(121872),l=e.i(242064);e.i(296059);var u=e.i(915654),c=e.i(135551),d=e.i(183293),h=e.i(246422),p=e.i(838378);let f=e=>{let{lineWidth:t,fontSizeIcon:r,calc:n}=e,s=e.fontSizeSM;return(0,p.mergeToken)(e,{tagFontSize:s,tagLineHeight:(0,u.unit)(n(e.lineHeightSM).mul(s).equal()),tagIconSize:n(r).sub(n(t).mul(2)).equal(),tagPaddingHorizontal:8,tagBorderlessBg:e.defaultBg})},m=e=>({defaultBg:new c.FastColor(e.colorFillQuaternary).onBackground(e.colorBgContainer).toHexString(),defaultColor:e.colorText}),g=(0,h.genStyleHooks)("Tag",e=>(e=>{let{paddingXXS:t,lineWidth:r,tagPaddingHorizontal:n,componentCls:s,calc:o}=e,i=o(n).sub(r).equal(),a=o(t).sub(r).equal();return{[s]:Object.assign(Object.assign({},(0,d.resetComponent)(e)),{display:"inline-block",height:"auto",marginInlineEnd:e.marginXS,paddingInline:i,fontSize:e.tagFontSize,lineHeight:e.tagLineHeight,whiteSpace:"nowrap",background:e.defaultBg,border:`${(0,u.unit)(e.lineWidth)} ${e.lineType} ${e.colorBorder}`,borderRadius:e.borderRadiusSM,opacity:1,transition:`all ${e.motionDurationMid}`,textAlign:"start",position:"relative",[`&${s}-rtl`]:{direction:"rtl"},"&, a, a:hover":{color:e.defaultColor},[`${s}-close-icon`]:{marginInlineStart:a,fontSize:e.tagIconSize,color:e.colorIcon,cursor:"pointer",transition:`all ${e.motionDurationMid}`,"&:hover":{color:e.colorTextHeading}},[`&${s}-has-color`]:{borderColor:"transparent",[`&, a, a:hover, ${e.iconCls}-close, ${e.iconCls}-close:hover`]:{color:e.colorTextLightSolid}},"&-checkable":{backgroundColor:"transparent",borderColor:"transparent",cursor:"pointer",[`&:not(${s}-checkable-checked):hover`]:{color:e.colorPrimary,backgroundColor:e.colorFillSecondary},"&:active, &-checked":{color:e.colorTextLightSolid},"&-checked":{backgroundColor:e.colorPrimary,"&:hover":{backgroundColor:e.colorPrimaryHover}},"&:active":{backgroundColor:e.colorPrimaryActive}},"&-hidden":{display:"none"},[`> ${e.iconCls} + span, > span + ${e.iconCls}`]:{marginInlineStart:i}}),[`${s}-borderless`]:{borderColor:"transparent",background:e.tagBorderlessBg}}})(f(e)),m);var y=function(e,t){var r={};for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&0>t.indexOf(n)&&(r[n]=e[n]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var s=0,n=Object.getOwnPropertySymbols(e);st.indexOf(n[s])&&Object.prototype.propertyIsEnumerable.call(e,n[s])&&(r[n[s]]=e[n[s]]);return r};let b=t.forwardRef((e,n)=>{let{prefixCls:s,style:o,className:i,checked:a,children:u,icon:c,onChange:d,onClick:h}=e,p=y(e,["prefixCls","style","className","checked","children","icon","onChange","onClick"]),{getPrefixCls:f,tag:m}=t.useContext(l.ConfigContext),b=f("tag",s),[v,C,R]=g(b),S=(0,r.default)(b,`${b}-checkable`,{[`${b}-checkable-checked`]:a},null==m?void 0:m.className,i,C,R);return v(t.createElement("span",Object.assign({},p,{ref:n,style:Object.assign(Object.assign({},o),null==m?void 0:m.style),className:S,onClick:e=>{null==d||d(!a),null==h||h(e)}}),c,t.createElement("span",null,u)))});var v=e.i(403541);let C=(0,h.genSubStyleComponent)(["Tag","preset"],e=>{let t;return t=f(e),(0,v.genPresetColor)(t,(e,{textColor:r,lightBorderColor:n,lightColor:s,darkColor:o})=>({[`${t.componentCls}${t.componentCls}-${e}`]:{color:r,background:s,borderColor:n,"&-inverse":{color:t.colorTextLightSolid,background:o,borderColor:o},[`&${t.componentCls}-borderless`]:{borderColor:"transparent"}}}))},m),R=(e,t,r)=>{let n="string"!=typeof r?r:r.charAt(0).toUpperCase()+r.slice(1);return{[`${e.componentCls}${e.componentCls}-${t}`]:{color:e[`color${r}`],background:e[`color${n}Bg`],borderColor:e[`color${n}Border`],[`&${e.componentCls}-borderless`]:{borderColor:"transparent"}}}},S=(0,h.genSubStyleComponent)(["Tag","status"],e=>{let t=f(e);return[R(t,"success","Success"),R(t,"processing","Info"),R(t,"error","Error"),R(t,"warning","Warning")]},m);var O=function(e,t){var r={};for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&0>t.indexOf(n)&&(r[n]=e[n]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var s=0,n=Object.getOwnPropertySymbols(e);st.indexOf(n[s])&&Object.prototype.propertyIsEnumerable.call(e,n[s])&&(r[n[s]]=e[n[s]]);return r};let w=t.forwardRef((e,u)=>{let{prefixCls:c,className:d,rootClassName:h,style:p,children:f,icon:m,color:y,onClose:b,bordered:v=!0,visible:R}=e,w=O(e,["prefixCls","className","rootClassName","style","children","icon","color","onClose","bordered","visible"]),{getPrefixCls:k,direction:I,tag:x}=t.useContext(l.ConfigContext),[E,Q]=t.useState(!0),T=(0,n.default)(w,["closeIcon","closable"]);t.useEffect(()=>{void 0!==R&&Q(R)},[R]);let $=(0,s.isPresetColor)(y),B=(0,s.isPresetStatusColor)(y),j=$||B,U=Object.assign(Object.assign({backgroundColor:y&&!j?y:void 0},null==x?void 0:x.style),p),P=k("tag",c),[M,L,A]=g(P),N=(0,r.default)(P,null==x?void 0:x.className,{[`${P}-${y}`]:j,[`${P}-has-color`]:y&&!j,[`${P}-hidden`]:!E,[`${P}-rtl`]:"rtl"===I,[`${P}-borderless`]:!v},d,h,L,A),F=e=>{e.stopPropagation(),null==b||b(e),e.defaultPrevented||Q(!1)},[,_]=(0,o.useClosable)((0,o.pickClosable)(e),(0,o.pickClosable)(x),{closable:!1,closeIconRender:e=>{let n=t.createElement("span",{className:`${P}-close-icon`,onClick:F},e);return(0,i.replaceElement)(e,n,e=>({onClick:t=>{var r;null==(r=null==e?void 0:e.onClick)||r.call(e,t),F(t)},className:(0,r.default)(null==e?void 0:e.className,`${P}-close-icon`)}))}}),z="function"==typeof w.onClick||f&&"a"===f.type,W=m||null,D=W?t.createElement(t.Fragment,null,W,f&&t.createElement("span",null,f)):f,G=t.createElement("span",Object.assign({},T,{ref:u,className:N,style:U}),D,_,$&&t.createElement(C,{key:"preset",prefixCls:P}),B&&t.createElement(S,{key:"status",prefixCls:P}));return M(z?t.createElement(a.default,{component:"Tag"},G):G)});w.CheckableTag=b,e.s(["Tag",0,w],262218)}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0-3i_.uof35pm.js b/litellm/proxy/_experimental/out/_next/static/chunks/0-3i_.uof35pm.js deleted file mode 100644 index aaaafac2d13..00000000000 --- a/litellm/proxy/_experimental/out/_next/static/chunks/0-3i_.uof35pm.js +++ /dev/null @@ -1,2 +0,0 @@ -(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,184163,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let n={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M505.7 661a8 8 0 0012.6 0l112-141.7c4.1-5.2.4-12.9-6.3-12.9h-74.1V168c0-4.4-3.6-8-8-8h-60c-4.4 0-8 3.6-8 8v338.3H400c-6.7 0-10.4 7.7-6.3 12.9l112 141.8zM878 626h-60c-4.4 0-8 3.6-8 8v154H214V634c0-4.4-3.6-8-8-8h-60c-4.4 0-8 3.6-8 8v198c0 17.7 14.3 32 32 32h684c17.7 0 32-14.3 32-32V634c0-4.4-3.6-8-8-8z"}}]},name:"download",theme:"outlined"};var i=e.i(9583),s=r.forwardRef(function(e,s){return r.createElement(i.default,(0,t.default)({},e,{ref:s,icon:n}))});e.s(["default",0,s],184163)},309821,e=>{"use strict";e.i(247167);var t=e.i(271645),r=e.i(135551),n=e.i(201072),i=e.i(121229),s=e.i(726289),a=e.i(864517),l=e.i(343794),o=e.i(529681),c=e.i(242064),u=e.i(931067),d=e.i(209428),m=e.i(703923),f={percent:0,prefixCls:"rc-progress",strokeColor:"#2db7f5",strokeLinecap:"round",strokeWidth:1,trailColor:"#D9D9D9",trailWidth:1,gapPosition:"bottom"},h=function(){var e=(0,t.useRef)([]),r=(0,t.useRef)(null);return(0,t.useEffect)(function(){var t=Date.now(),n=!1;e.current.forEach(function(e){if(e){n=!0;var i=e.style;i.transitionDuration=".3s, .3s, .3s, .06s",r.current&&t-r.current<100&&(i.transitionDuration="0s, 0s")}}),n&&(r.current=Date.now())}),e.current},p=e.i(410160),g=e.i(392221),x=e.i(654310),v=0,y=(0,x.default)();let b=function(e){var r=t.useState(),n=(0,g.default)(r,2),i=n[0],s=n[1];return t.useEffect(function(){var e;s("rc_progress_".concat((y?(e=v,v+=1):e="TEST_OR_SSR",e)))},[]),e||i};var _=function(e){var r=e.bg,n=e.children;return t.createElement("div",{style:{width:"100%",height:"100%",background:r}},n)};function j(e,t){return Object.keys(e).map(function(r){var n=parseFloat(r),i="".concat(Math.floor(n*t),"%");return"".concat(e[r]," ").concat(i)})}var w=t.forwardRef(function(e,r){var n=e.prefixCls,i=e.color,s=e.gradientId,a=e.radius,l=e.style,o=e.ptg,c=e.strokeLinecap,u=e.strokeWidth,d=e.size,m=e.gapDegree,f=i&&"object"===(0,p.default)(i),h=d/2,g=t.createElement("circle",{className:"".concat(n,"-circle-path"),r:a,cx:h,cy:h,stroke:f?"#FFF":void 0,strokeLinecap:c,strokeWidth:u,opacity:+(0!==o),style:l,ref:r});if(!f)return g;var x="".concat(s,"-conic"),v=j(i,(360-m)/360),y=j(i,1),b="conic-gradient(from ".concat(m?"".concat(180+m/2,"deg"):"0deg",", ").concat(v.join(", "),")"),w="linear-gradient(to ".concat(m?"bottom":"top",", ").concat(y.join(", "),")");return t.createElement(t.Fragment,null,t.createElement("mask",{id:x},g),t.createElement("foreignObject",{x:0,y:0,width:d,height:d,mask:"url(#".concat(x,")")},t.createElement(_,{bg:w},t.createElement(_,{bg:b}))))}),k=function(e,t,r,n,i,s,a,l,o,c){var u=arguments.length>10&&void 0!==arguments[10]?arguments[10]:0,d=(100-n)/100*t;return"round"===o&&100!==n&&(d+=c/2)>=t&&(d=t-.01),{stroke:"string"==typeof l?l:void 0,strokeDasharray:"".concat(t,"px ").concat(e),strokeDashoffset:d+u,transform:"rotate(".concat(i+r/100*360*((360-s)/360)+(0===s?0:({bottom:0,top:180,left:90,right:-90})[a]),"deg)"),transformOrigin:"".concat(50,"px ").concat(50,"px"),transition:"stroke-dashoffset .3s ease 0s, stroke-dasharray .3s ease 0s, stroke .3s, stroke-width .06s ease .3s, opacity .3s ease 0s",fillOpacity:0}},C=["id","prefixCls","steps","strokeWidth","trailWidth","gapDegree","gapPosition","trailColor","strokeLinecap","style","className","strokeColor","percent"];function S(e){var t=null!=e?e:[];return Array.isArray(t)?t:[t]}let E=function(e){var r,n,i,s,a=(0,d.default)((0,d.default)({},f),e),o=a.id,c=a.prefixCls,g=a.steps,x=a.strokeWidth,v=a.trailWidth,y=a.gapDegree,_=void 0===y?0:y,j=a.gapPosition,E=a.trailColor,O=a.strokeLinecap,N=a.style,I=a.className,T=a.strokeColor,R=a.percent,P=(0,m.default)(a,C),D=b(o),$="".concat(D,"-gradient"),A=50-x/2,F=2*Math.PI*A,L=_>0?90+_/2:-90,M=(360-_)/360*F,B="object"===(0,p.default)(g)?g:{count:g,gap:2},U=B.count,z=B.gap,V=S(R),H=S(T),W=H.find(function(e){return e&&"object"===(0,p.default)(e)}),K=W&&"object"===(0,p.default)(W)?"butt":O,q=k(F,M,0,100,L,_,j,E,K,x),X=h();return t.createElement("svg",(0,u.default)({className:(0,l.default)("".concat(c,"-circle"),I),viewBox:"0 0 ".concat(100," ").concat(100),style:N,id:o,role:"presentation"},P),!U&&t.createElement("circle",{className:"".concat(c,"-circle-trail"),r:A,cx:50,cy:50,stroke:E,strokeLinecap:K,strokeWidth:v||x,style:q}),U?(r=Math.round(U*(V[0]/100)),n=100/U,i=0,Array(U).fill(null).map(function(e,s){var a=s<=r-1?H[0]:E,l=a&&"object"===(0,p.default)(a)?"url(#".concat($,")"):void 0,o=k(F,M,i,n,L,_,j,a,"butt",x,z);return i+=(M-o.strokeDashoffset+z)*100/M,t.createElement("circle",{key:s,className:"".concat(c,"-circle-path"),r:A,cx:50,cy:50,stroke:l,strokeWidth:x,opacity:1,style:o,ref:function(e){X[s]=e}})})):(s=0,V.map(function(e,r){var n=H[r]||H[H.length-1],i=k(F,M,s,e,L,_,j,n,K,x);return s+=e,t.createElement(w,{key:r,color:n,ptg:e,radius:A,prefixCls:c,gradientId:$,style:i,strokeLinecap:K,strokeWidth:x,gapDegree:_,ref:function(e){X[r]=e},size:100})}).reverse()))};var O=e.i(491816);e.i(765846);var N=e.i(896091);function I(e){return!e||e<0?0:e>100?100:e}function T({success:e,successPercent:t}){let r=t;return e&&"progress"in e&&(r=e.progress),e&&"percent"in e&&(r=e.percent),r}let R=(e,t,r)=>{var n,i,s,a;let l=-1,o=-1;if("step"===t){let t=r.steps,n=r.strokeWidth;"string"==typeof e||void 0===e?(l="small"===e?2:14,o=null!=n?n:8):"number"==typeof e?[l,o]=[e,e]:[l=14,o=8]=Array.isArray(e)?e:[e.width,e.height],l*=t}else if("line"===t){let t=null==r?void 0:r.strokeWidth;"string"==typeof e||void 0===e?o=t||("small"===e?6:8):"number"==typeof e?[l,o]=[e,e]:[l=-1,o=8]=Array.isArray(e)?e:[e.width,e.height]}else("circle"===t||"dashboard"===t)&&("string"==typeof e||void 0===e?[l,o]="small"===e?[60,60]:[120,120]:"number"==typeof e?[l,o]=[e,e]:Array.isArray(e)&&(l=null!=(i=null!=(n=e[0])?n:e[1])?i:120,o=null!=(a=null!=(s=e[0])?s:e[1])?a:120));return[l,o]},P=e=>{let{prefixCls:r,trailColor:n=null,strokeLinecap:i="round",gapPosition:s,gapDegree:a,width:o=120,type:c,children:u,success:d,size:m=o,steps:f}=e,[h,p]=R(m,"circle"),{strokeWidth:g}=e;void 0===g&&(g=Math.max(3/h*100,6));let x=t.useMemo(()=>a||0===a?a:"dashboard"===c?75:void 0,[a,c]),v=(({percent:e,success:t,successPercent:r})=>{let n=I(T({success:t,successPercent:r}));return[n,I(I(e)-n)]})(e),y="[object Object]"===Object.prototype.toString.call(e.strokeColor),b=(({success:e={},strokeColor:t})=>{let{strokeColor:r}=e;return[r||N.presetPrimaryColors.green,t||null]})({success:d,strokeColor:e.strokeColor}),_=(0,l.default)(`${r}-inner`,{[`${r}-circle-gradient`]:y}),j=t.createElement(E,{steps:f,percent:f?v[1]:v,strokeWidth:g,trailWidth:g,strokeColor:f?b[1]:b,strokeLinecap:i,trailColor:n,prefixCls:r,gapDegree:x,gapPosition:s||"dashboard"===c&&"bottom"||void 0}),w=h<=20,k=t.createElement("div",{className:_,style:{width:h,height:p,fontSize:.15*h+6}},j,!w&&u);return w?t.createElement(O.default,{title:u},k):k};e.i(296059);var D=e.i(694758),$=e.i(915654),A=e.i(183293),F=e.i(246422),L=e.i(838378);let M="--progress-line-stroke-color",B="--progress-percent",U=e=>{let t=e?"100%":"-100%";return new D.Keyframes(`antProgress${e?"RTL":"LTR"}Active`,{"0%":{transform:`translateX(${t}) scaleX(0)`,opacity:.1},"20%":{transform:`translateX(${t}) scaleX(0)`,opacity:.5},to:{transform:"translateX(0) scaleX(1)",opacity:0}})},z=(0,F.genStyleHooks)("Progress",e=>{let t=e.calc(e.marginXXS).div(2).equal(),r=(0,L.mergeToken)(e,{progressStepMarginInlineEnd:t,progressStepMinWidth:t,progressActiveMotionDuration:"2.4s"});return[(e=>{let{componentCls:t,iconCls:r}=e;return{[t]:Object.assign(Object.assign({},(0,A.resetComponent)(e)),{display:"inline-block","&-rtl":{direction:"rtl"},"&-line":{position:"relative",width:"100%",fontSize:e.fontSize},[`${t}-outer`]:{display:"inline-flex",alignItems:"center",width:"100%"},[`${t}-inner`]:{position:"relative",display:"inline-block",width:"100%",flex:1,overflow:"hidden",verticalAlign:"middle",backgroundColor:e.remainingColor,borderRadius:e.lineBorderRadius},[`${t}-inner:not(${t}-circle-gradient)`]:{[`${t}-circle-path`]:{stroke:e.defaultColor}},[`${t}-success-bg, ${t}-bg`]:{position:"relative",background:e.defaultColor,borderRadius:e.lineBorderRadius,transition:`all ${e.motionDurationSlow} ${e.motionEaseInOutCirc}`},[`${t}-layout-bottom`]:{display:"flex",flexDirection:"column",alignItems:"center",justifyContent:"center",[`${t}-text`]:{width:"max-content",marginInlineStart:0,marginTop:e.marginXXS}},[`${t}-bg`]:{overflow:"hidden","&::after":{content:'""',background:{_multi_value_:!0,value:["inherit",`var(${M})`]},height:"100%",width:`calc(1 / var(${B}) * 100%)`,display:"block"},[`&${t}-bg-inner`]:{minWidth:"max-content","&::after":{content:"none"},[`${t}-text-inner`]:{color:e.colorWhite,[`&${t}-text-bright`]:{color:"rgba(0, 0, 0, 0.45)"}}}},[`${t}-success-bg`]:{position:"absolute",insetBlockStart:0,insetInlineStart:0,backgroundColor:e.colorSuccess},[`${t}-text`]:{display:"inline-block",marginInlineStart:e.marginXS,color:e.colorText,lineHeight:1,width:"2em",whiteSpace:"nowrap",textAlign:"start",verticalAlign:"middle",wordBreak:"normal",[r]:{fontSize:e.fontSize},[`&${t}-text-outer`]:{width:"max-content"},[`&${t}-text-outer${t}-text-start`]:{width:"max-content",marginInlineStart:0,marginInlineEnd:e.marginXS}},[`${t}-text-inner`]:{display:"flex",justifyContent:"center",alignItems:"center",width:"100%",height:"100%",marginInlineStart:0,padding:`0 ${(0,$.unit)(e.paddingXXS)}`,[`&${t}-text-start`]:{justifyContent:"start"},[`&${t}-text-end`]:{justifyContent:"end"}},[`&${t}-status-active`]:{[`${t}-bg::before`]:{position:"absolute",inset:0,backgroundColor:e.colorBgContainer,borderRadius:e.lineBorderRadius,opacity:0,animationName:U(),animationDuration:e.progressActiveMotionDuration,animationTimingFunction:e.motionEaseOutQuint,animationIterationCount:"infinite",content:'""'}},[`&${t}-rtl${t}-status-active`]:{[`${t}-bg::before`]:{animationName:U(!0)}},[`&${t}-status-exception`]:{[`${t}-bg`]:{backgroundColor:e.colorError},[`${t}-text`]:{color:e.colorError}},[`&${t}-status-exception ${t}-inner:not(${t}-circle-gradient)`]:{[`${t}-circle-path`]:{stroke:e.colorError}},[`&${t}-status-success`]:{[`${t}-bg`]:{backgroundColor:e.colorSuccess},[`${t}-text`]:{color:e.colorSuccess}},[`&${t}-status-success ${t}-inner:not(${t}-circle-gradient)`]:{[`${t}-circle-path`]:{stroke:e.colorSuccess}}})}})(r),(e=>{let{componentCls:t,iconCls:r}=e;return{[t]:{[`${t}-circle-trail`]:{stroke:e.remainingColor},[`&${t}-circle ${t}-inner`]:{position:"relative",lineHeight:1,backgroundColor:"transparent"},[`&${t}-circle ${t}-text`]:{position:"absolute",insetBlockStart:"50%",insetInlineStart:0,width:"100%",margin:0,padding:0,color:e.circleTextColor,fontSize:e.circleTextFontSize,lineHeight:1,whiteSpace:"normal",textAlign:"center",transform:"translateY(-50%)",[r]:{fontSize:e.circleIconFontSize}},[`${t}-circle&-status-exception`]:{[`${t}-text`]:{color:e.colorError}},[`${t}-circle&-status-success`]:{[`${t}-text`]:{color:e.colorSuccess}}},[`${t}-inline-circle`]:{lineHeight:1,[`${t}-inner`]:{verticalAlign:"bottom"}}}})(r),(e=>{let{componentCls:t}=e;return{[t]:{[`${t}-steps`]:{display:"inline-block","&-outer":{display:"flex",flexDirection:"row",alignItems:"center"},"&-item":{flexShrink:0,minWidth:e.progressStepMinWidth,marginInlineEnd:e.progressStepMarginInlineEnd,backgroundColor:e.remainingColor,transition:`all ${e.motionDurationSlow}`,"&-active":{backgroundColor:e.defaultColor}}}}}})(r),(e=>{let{componentCls:t,iconCls:r}=e;return{[t]:{[`${t}-small&-line, ${t}-small&-line ${t}-text ${r}`]:{fontSize:e.fontSizeSM}}}})(r)]},e=>({circleTextColor:e.colorText,defaultColor:e.colorInfo,remainingColor:e.colorFillSecondary,lineBorderRadius:100,circleTextFontSize:"1em",circleIconFontSize:`${e.fontSize/e.fontSizeSM}em`}));var V=function(e,t){var r={};for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&0>t.indexOf(n)&&(r[n]=e[n]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var i=0,n=Object.getOwnPropertySymbols(e);it.indexOf(n[i])&&Object.prototype.propertyIsEnumerable.call(e,n[i])&&(r[n[i]]=e[n[i]]);return r};let H=e=>{let{prefixCls:r,direction:n,percent:i,size:s,strokeWidth:a,strokeColor:o,strokeLinecap:c="round",children:u,trailColor:d=null,percentPosition:m,success:f}=e,{align:h,type:p}=m,g=o&&"string"!=typeof o?((e,t)=>{let{from:r=N.presetPrimaryColors.blue,to:n=N.presetPrimaryColors.blue,direction:i="rtl"===t?"to left":"to right"}=e,s=V(e,["from","to","direction"]);if(0!==Object.keys(s).length){let e,t=(e=[],Object.keys(s).forEach(t=>{let r=Number.parseFloat(t.replace(/%/g,""));Number.isNaN(r)||e.push({key:r,value:s[t]})}),(e=e.sort((e,t)=>e.key-t.key)).map(({key:e,value:t})=>`${t} ${e}%`).join(", ")),r=`linear-gradient(${i}, ${t})`;return{background:r,[M]:r}}let a=`linear-gradient(${i}, ${r}, ${n})`;return{background:a,[M]:a}})(o,n):{[M]:o,background:o},x="square"===c||"butt"===c?0:void 0,[v,y]=R(null!=s?s:[-1,a||("small"===s?6:8)],"line",{strokeWidth:a}),b=Object.assign(Object.assign({width:`${I(i)}%`,height:y,borderRadius:x},g),{[B]:I(i)/100}),_=T(e),j={width:`${I(_)}%`,height:y,borderRadius:x,backgroundColor:null==f?void 0:f.strokeColor},w=t.createElement("div",{className:`${r}-inner`,style:{backgroundColor:d||void 0,borderRadius:x}},t.createElement("div",{className:(0,l.default)(`${r}-bg`,`${r}-bg-${p}`),style:b},"inner"===p&&u),void 0!==_&&t.createElement("div",{className:`${r}-success-bg`,style:j})),k="outer"===p&&"start"===h,C="outer"===p&&"end"===h;return"outer"===p&&"center"===h?t.createElement("div",{className:`${r}-layout-bottom`},w,u):t.createElement("div",{className:`${r}-outer`,style:{width:v<0?"100%":v}},k&&u,w,C&&u)},W=e=>{let{size:r,steps:n,rounding:i=Math.round,percent:s=0,strokeWidth:a=8,strokeColor:o,trailColor:c=null,prefixCls:u,children:d}=e,m=i(s/100*n),[f,h]=R(null!=r?r:["small"===r?2:14,a],"step",{steps:n,strokeWidth:a}),p=f/n,g=Array.from({length:n});for(let e=0;et.indexOf(n)&&(r[n]=e[n]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var i=0,n=Object.getOwnPropertySymbols(e);it.indexOf(n[i])&&Object.prototype.propertyIsEnumerable.call(e,n[i])&&(r[n[i]]=e[n[i]]);return r};let q=["normal","exception","active","success"],X=t.forwardRef((e,u)=>{let d,{prefixCls:m,className:f,rootClassName:h,steps:p,strokeColor:g,percent:x=0,size:v="default",showInfo:y=!0,type:b="line",status:_,format:j,style:w,percentPosition:k={}}=e,C=K(e,["prefixCls","className","rootClassName","steps","strokeColor","percent","size","showInfo","type","status","format","style","percentPosition"]),{align:S="end",type:E="outer"}=k,O=Array.isArray(g)?g[0]:g,N="string"==typeof g||Array.isArray(g)?g:void 0,D=t.useMemo(()=>{if(O){let e="string"==typeof O?O:Object.values(O)[0];return new r.FastColor(e).isLight()}return!1},[g]),$=t.useMemo(()=>{var t,r;let n=T(e);return Number.parseInt(void 0!==n?null==(t=null!=n?n:0)?void 0:t.toString():null==(r=null!=x?x:0)?void 0:r.toString(),10)},[x,e.success,e.successPercent]),A=t.useMemo(()=>!q.includes(_)&&$>=100?"success":_||"normal",[_,$]),{getPrefixCls:F,direction:L,progress:M}=t.useContext(c.ConfigContext),B=F("progress",m),[U,V,X]=z(B),Q="line"===b,J=Q&&!p,Y=t.useMemo(()=>{let r;if(!y)return null;let o=T(e),c=j||(e=>`${e}%`),u=Q&&D&&"inner"===E;return"inner"===E||j||"exception"!==A&&"success"!==A?r=c(I(x),I(o)):"exception"===A?r=Q?t.createElement(s.default,null):t.createElement(a.default,null):"success"===A&&(r=Q?t.createElement(n.default,null):t.createElement(i.default,null)),t.createElement("span",{className:(0,l.default)(`${B}-text`,{[`${B}-text-bright`]:u,[`${B}-text-${S}`]:J,[`${B}-text-${E}`]:J}),title:"string"==typeof r?r:void 0},r)},[y,x,$,A,b,B,j]);"line"===b?d=p?t.createElement(W,Object.assign({},e,{strokeColor:N,prefixCls:B,steps:"object"==typeof p?p.count:p}),Y):t.createElement(H,Object.assign({},e,{strokeColor:O,prefixCls:B,direction:L,percentPosition:{align:S,type:E}}),Y):("circle"===b||"dashboard"===b)&&(d=t.createElement(P,Object.assign({},e,{strokeColor:O,prefixCls:B,progressStatus:A}),Y));let G=(0,l.default)(B,`${B}-status-${A}`,{[`${B}-${"dashboard"===b&&"circle"||b}`]:"line"!==b,[`${B}-inline-circle`]:"circle"===b&&R(v,"circle")[0]<=20,[`${B}-line`]:J,[`${B}-line-align-${S}`]:J,[`${B}-line-position-${E}`]:J,[`${B}-steps`]:p,[`${B}-show-info`]:y,[`${B}-${v}`]:"string"==typeof v,[`${B}-rtl`]:"rtl"===L},null==M?void 0:M.className,f,h,V,X);return U(t.createElement("div",Object.assign({ref:u,style:Object.assign(Object.assign({},null==M?void 0:M.style),w),className:G,role:"progressbar","aria-valuenow":$,"aria-valuemin":0,"aria-valuemax":100},(0,o.default)(C,["trailColor","strokeWidth","width","gapDegree","gapPosition","strokeLinecap","success","successPercent"])),d))});e.s(["default",0,X],309821)},519756,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let n={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M400 317.7h73.9V656c0 4.4 3.6 8 8 8h60c4.4 0 8-3.6 8-8V317.7H624c6.7 0 10.4-7.7 6.3-12.9L518.3 163a8 8 0 00-12.6 0l-112 141.7c-4.1 5.3-.4 13 6.3 13zM878 626h-60c-4.4 0-8 3.6-8 8v154H214V634c0-4.4-3.6-8-8-8h-60c-4.4 0-8 3.6-8 8v198c0 17.7 14.3 32 32 32h684c17.7 0 32-14.3 32-32V634c0-4.4-3.6-8-8-8z"}}]},name:"upload",theme:"outlined"};var i=e.i(9583),s=r.forwardRef(function(e,s){return r.createElement(i.default,(0,t.default)({},e,{ref:s,icon:n}))});e.s(["UploadOutlined",0,s],519756)},233538,e=>{"use strict";e.s(["isDisabledReactIssue7711",0,function(e){let t=e.parentElement,r=null;for(;t&&!(t instanceof HTMLFieldSetElement);)t instanceof HTMLLegendElement&&(r=t),t=t.parentElement;let n=(null==t?void 0:t.getAttribute("disabled"))==="";return!(n&&function(e){if(!e)return!1;let t=e.previousElementSibling;for(;null!==t;){if(t instanceof HTMLLegendElement)return!1;t=t.previousElementSibling}return!0}(r))&&n}])},83733,233137,e=>{"use strict";let t,r;var n,i,s=e.i(247167),a=e.i(271645),l=e.i(544508),o=e.i(746725),c=e.i(835696);void 0!==s.default&&"u">typeof globalThis&&"u">typeof Element&&(null==(n=null==s.default?void 0:s.default.env)?void 0:n.NODE_ENV)==="test"&&void 0===(null==(i=null==Element?void 0:Element.prototype)?void 0:i.getAnimations)&&(Element.prototype.getAnimations=function(){return console.warn(["Headless UI has polyfilled `Element.prototype.getAnimations` for your tests.","Please install a proper polyfill e.g. `jsdom-testing-mocks`, to silence these warnings.","","Example usage:","```js","import { mockAnimationsApi } from 'jsdom-testing-mocks'","mockAnimationsApi()","```"].join(` -`)),[]});var u=((t=u||{})[t.None=0]="None",t[t.Closed=1]="Closed",t[t.Enter=2]="Enter",t[t.Leave=4]="Leave",t);e.s(["transitionDataAttributes",0,function(e){let t={};for(let r in e)!0===e[r]&&(t[`data-${r}`]="");return t},"useTransition",0,function(e,t,r,n){let[i,s]=(0,a.useState)(r),{hasFlag:u,addFlag:d,removeFlag:m}=function(e=0){let[t,r]=(0,a.useState)(e),n=(0,a.useCallback)(e=>r(e),[t]),i=(0,a.useCallback)(e=>r(t=>t|e),[t]),s=(0,a.useCallback)(e=>(t&e)===e,[t]);return{flags:t,setFlag:n,addFlag:i,hasFlag:s,removeFlag:(0,a.useCallback)(e=>r(t=>t&~e),[r]),toggleFlag:(0,a.useCallback)(e=>r(t=>t^e),[r])}}(e&&i?3:0),f=(0,a.useRef)(!1),h=(0,a.useRef)(!1),p=(0,o.useDisposables)();return(0,c.useIsoMorphicEffect)(()=>{var i;if(e){if(r&&s(!0),!t){r&&d(3);return}return null==(i=null==n?void 0:n.start)||i.call(n,r),function(e,{prepare:t,run:r,done:n,inFlight:i}){let s=(0,l.disposables)();return function(e,{inFlight:t,prepare:r}){if(null!=t&&t.current)return r();let n=e.style.transition;e.style.transition="none",r(),e.offsetHeight,e.style.transition=n}(e,{prepare:t,inFlight:i}),s.nextFrame(()=>{r(),s.requestAnimationFrame(()=>{s.add(function(e,t){var r,n;let i=(0,l.disposables)();if(!e)return i.dispose;let s=!1;i.add(()=>{s=!0});let a=null!=(n=null==(r=e.getAnimations)?void 0:r.call(e).filter(e=>e instanceof CSSTransition))?n:[];return 0===a.length?t():Promise.allSettled(a.map(e=>e.finished)).then(()=>{s||t()}),i.dispose}(e,n))})}),s.dispose}(t,{inFlight:f,prepare(){h.current?h.current=!1:h.current=f.current,f.current=!0,h.current||(r?(d(3),m(4)):(d(4),m(2)))},run(){h.current?r?(m(3),d(4)):(m(4),d(3)):r?m(1):d(1)},done(){var e;h.current&&"function"==typeof t.getAnimations&&t.getAnimations().length>0||(f.current=!1,m(7),r||s(!1),null==(e=null==n?void 0:n.end)||e.call(n,r))}})}},[e,r,t,p]),e?[i,{closed:u(1),enter:u(2),leave:u(4),transition:u(2)||u(4)}]:[r,{closed:void 0,enter:void 0,leave:void 0,transition:void 0}]}],83733);let d=(0,a.createContext)(null);d.displayName="OpenClosedContext";var m=((r=m||{})[r.Open=1]="Open",r[r.Closed=2]="Closed",r[r.Closing=4]="Closing",r[r.Opening=8]="Opening",r);e.s(["OpenClosedProvider",0,function({value:e,children:t}){return a.default.createElement(d.Provider,{value:e},t)},"ResetOpenClosedProvider",0,function({children:e}){return a.default.createElement(d.Provider,{value:null},e)},"State",0,m,"useOpenClosed",0,function(){return(0,a.useContext)(d)}],233137)},677667,674175,886148,543086,e=>{"use strict";let t,r;var n,i=e.i(290571),s=e.i(783222),a=e.i(433336),l=e.i(271645),o=e.i(394487),c=e.i(914189),u=e.i(144279),d=e.i(294316),m=e.i(83733);let f=(0,l.createContext)(()=>{});function h({value:e,children:t}){return l.default.createElement(f.Provider,{value:e},t)}e.s(["CloseProvider",0,h],674175);var p=e.i(233137),g=e.i(233538),x=e.i(397701),v=e.i(402155),y=e.i(700020);let b=null!=(n=l.default.startTransition)?n:function(e){e()};var _=e.i(998348),j=((t=j||{})[t.Open=0]="Open",t[t.Closed=1]="Closed",t),w=((r=w||{})[r.ToggleDisclosure=0]="ToggleDisclosure",r[r.CloseDisclosure=1]="CloseDisclosure",r[r.SetButtonId=2]="SetButtonId",r[r.SetPanelId=3]="SetPanelId",r[r.SetButtonElement=4]="SetButtonElement",r[r.SetPanelElement=5]="SetPanelElement",r);let k={0:e=>({...e,disclosureState:(0,x.match)(e.disclosureState,{0:1,1:0})}),1:e=>1===e.disclosureState?e:{...e,disclosureState:1},2:(e,t)=>e.buttonId===t.buttonId?e:{...e,buttonId:t.buttonId},3:(e,t)=>e.panelId===t.panelId?e:{...e,panelId:t.panelId},4:(e,t)=>e.buttonElement===t.element?e:{...e,buttonElement:t.element},5:(e,t)=>e.panelElement===t.element?e:{...e,panelElement:t.element}},C=(0,l.createContext)(null);function S(e){let t=(0,l.useContext)(C);if(null===t){let t=Error(`<${e} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(t,S),t}return t}C.displayName="DisclosureContext";let E=(0,l.createContext)(null);E.displayName="DisclosureAPIContext";let O=(0,l.createContext)(null);function N(e,t){return(0,x.match)(t.type,k,e,t)}O.displayName="DisclosurePanelContext";let I=l.Fragment,T=y.RenderFeatures.RenderStrategy|y.RenderFeatures.Static,R=Object.assign((0,y.forwardRefWithAs)(function(e,t){let{defaultOpen:r=!1,...n}=e,i=(0,l.useRef)(null),s=(0,d.useSyncRefs)(t,(0,d.optionalRef)(e=>{i.current=e},void 0===e.as||e.as===l.Fragment)),a=(0,l.useReducer)(N,{disclosureState:+!r,buttonElement:null,panelElement:null,buttonId:null,panelId:null}),[{disclosureState:o,buttonId:u},m]=a,f=(0,c.useEvent)(e=>{m({type:1});let t=(0,v.getOwnerDocument)(i);if(!t||!u)return;let r=e?e instanceof HTMLElement?e:e.current instanceof HTMLElement?e.current:t.getElementById(u):t.getElementById(u);null==r||r.focus()}),g=(0,l.useMemo)(()=>({close:f}),[f]),b=(0,l.useMemo)(()=>({open:0===o,close:f}),[o,f]),_=(0,y.useRender)();return l.default.createElement(C.Provider,{value:a},l.default.createElement(E.Provider,{value:g},l.default.createElement(h,{value:f},l.default.createElement(p.OpenClosedProvider,{value:(0,x.match)(o,{0:p.State.Open,1:p.State.Closed})},_({ourProps:{ref:s},theirProps:n,slot:b,defaultTag:I,name:"Disclosure"})))))}),{Button:(0,y.forwardRefWithAs)(function(e,t){let r=(0,l.useId)(),{id:n=`headlessui-disclosure-button-${r}`,disabled:i=!1,autoFocus:m=!1,...f}=e,[h,p]=S("Disclosure.Button"),x=(0,l.useContext)(O),v=null!==x&&x===h.panelId,b=(0,l.useRef)(null),j=(0,d.useSyncRefs)(b,t,(0,c.useEvent)(e=>{if(!v)return p({type:4,element:e})}));(0,l.useEffect)(()=>{if(!v)return p({type:2,buttonId:n}),()=>{p({type:2,buttonId:null})}},[n,p,v]);let w=(0,c.useEvent)(e=>{var t;if(v){if(1===h.disclosureState)return;switch(e.key){case _.Keys.Space:case _.Keys.Enter:e.preventDefault(),e.stopPropagation(),p({type:0}),null==(t=h.buttonElement)||t.focus()}}else switch(e.key){case _.Keys.Space:case _.Keys.Enter:e.preventDefault(),e.stopPropagation(),p({type:0})}}),k=(0,c.useEvent)(e=>{e.key===_.Keys.Space&&e.preventDefault()}),C=(0,c.useEvent)(e=>{var t;(0,g.isDisabledReactIssue7711)(e.currentTarget)||i||(v?(p({type:0}),null==(t=h.buttonElement)||t.focus()):p({type:0}))}),{isFocusVisible:E,focusProps:N}=(0,s.useFocusRing)({autoFocus:m}),{isHovered:I,hoverProps:T}=(0,a.useHover)({isDisabled:i}),{pressed:R,pressProps:P}=(0,o.useActivePress)({disabled:i}),D=(0,l.useMemo)(()=>({open:0===h.disclosureState,hover:I,active:R,disabled:i,focus:E,autofocus:m}),[h,I,R,E,i,m]),$=(0,u.useResolveButtonType)(e,h.buttonElement),A=v?(0,y.mergeProps)({ref:j,type:$,disabled:i||void 0,autoFocus:m,onKeyDown:w,onClick:C},N,T,P):(0,y.mergeProps)({ref:j,id:n,type:$,"aria-expanded":0===h.disclosureState,"aria-controls":h.panelElement?h.panelId:void 0,disabled:i||void 0,autoFocus:m,onKeyDown:w,onKeyUp:k,onClick:C},N,T,P);return(0,y.useRender)()({ourProps:A,theirProps:f,slot:D,defaultTag:"button",name:"Disclosure.Button"})}),Panel:(0,y.forwardRefWithAs)(function(e,t){let r=(0,l.useId)(),{id:n=`headlessui-disclosure-panel-${r}`,transition:i=!1,...s}=e,[a,o]=S("Disclosure.Panel"),{close:u}=function e(t){let r=(0,l.useContext)(E);if(null===r){let r=Error(`<${t} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(r,e),r}return r}("Disclosure.Panel"),[f,h]=(0,l.useState)(null),g=(0,d.useSyncRefs)(t,(0,c.useEvent)(e=>{b(()=>o({type:5,element:e}))}),h);(0,l.useEffect)(()=>(o({type:3,panelId:n}),()=>{o({type:3,panelId:null})}),[n,o]);let x=(0,p.useOpenClosed)(),[v,_]=(0,m.useTransition)(i,f,null!==x?(x&p.State.Open)===p.State.Open:0===a.disclosureState),j=(0,l.useMemo)(()=>({open:0===a.disclosureState,close:u}),[a.disclosureState,u]),w={ref:g,id:n,...(0,m.transitionDataAttributes)(_)},k=(0,y.useRender)();return l.default.createElement(p.ResetOpenClosedProvider,null,l.default.createElement(O.Provider,{value:a.panelId},k({ourProps:w,theirProps:s,slot:j,defaultTag:"div",features:T,visible:v,name:"Disclosure.Panel"})))})});e.s(["Disclosure",0,R],886148);let P=(0,l.createContext)(void 0);var D=e.i(444755);let $=(0,e.i(673706).makeClassName)("Accordion"),A=(0,l.createContext)({isOpen:!1}),F=l.default.forwardRef((e,t)=>{var r;let{defaultOpen:n=!1,children:s,className:a}=e,o=(0,i.__rest)(e,["defaultOpen","children","className"]),c=null!=(r=(0,l.useContext)(P))?r:(0,D.tremorTwMerge)("rounded-tremor-default border");return l.default.createElement(R,Object.assign({as:"div",ref:t,className:(0,D.tremorTwMerge)($("root"),"overflow-hidden","bg-tremor-background border-tremor-border","dark:bg-dark-tremor-background dark:border-dark-tremor-border",c,a),defaultOpen:n},o),({open:e})=>l.default.createElement(A.Provider,{value:{isOpen:e}},s))});F.displayName="Accordion",e.s(["OpenContext",0,A,"default",0,F],543086),e.s(["Accordion",0,F],677667)},130643,e=>{"use strict";var t=e.i(290571),r=e.i(271645),n=e.i(886148),i=e.i(444755);let s=(0,e.i(673706).makeClassName)("AccordionBody"),a=r.default.forwardRef((e,a)=>{let{children:l,className:o}=e,c=(0,t.__rest)(e,["children","className"]);return r.default.createElement(n.Disclosure.Panel,Object.assign({ref:a,className:(0,i.tremorTwMerge)(s("root"),"w-full text-tremor-default px-4 pb-3","text-tremor-content","dark:text-dark-tremor-content",o)},c),l)});a.displayName="AccordionBody",e.s(["AccordionBody",0,a],130643)},898667,e=>{"use strict";var t=e.i(290571),r=e.i(271645),n=e.i(886148);let i=e=>{var n=(0,t.__rest)(e,[]);return r.default.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"},n),r.default.createElement("path",{d:"M11.9999 10.8284L7.0502 15.7782L5.63599 14.364L11.9999 8L18.3639 14.364L16.9497 15.7782L11.9999 10.8284Z"}))};var s=e.i(543086),a=e.i(444755);let l=(0,e.i(673706).makeClassName)("AccordionHeader"),o=r.default.forwardRef((e,o)=>{let{children:c,className:u}=e,d=(0,t.__rest)(e,["children","className"]),{isOpen:m}=(0,r.useContext)(s.OpenContext);return r.default.createElement(n.Disclosure.Button,Object.assign({ref:o,className:(0,a.tremorTwMerge)(l("root"),"w-full flex items-center justify-between px-4 py-3","text-tremor-content-emphasis","dark:text-dark-tremor-content-emphasis",u)},d),r.default.createElement("div",{className:(0,a.tremorTwMerge)(l("children"),"flex flex-1 text-inherit mr-4")},c),r.default.createElement("div",null,r.default.createElement(i,{className:(0,a.tremorTwMerge)(l("arrowIcon"),"h-5 w-5 -mr-1","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle",m?"transition-all":"transition-all -rotate-180")})))});o.displayName="AccordionHeader",e.s(["AccordionHeader",0,o],898667)},220508,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z"}))});e.s(["CheckCircleIcon",0,r],220508)},503269,214520,601893,694421,140721,942803,35889,722678,e=>{"use strict";var t=e.i(271645),r=e.i(914189);e.s(["useControllable",0,function(e,n,i){let[s,a]=(0,t.useState)(i),l=void 0!==e,o=(0,t.useRef)(l),c=(0,t.useRef)(!1),u=(0,t.useRef)(!1);return!l||o.current||c.current?l||!o.current||u.current||(u.current=!0,o.current=l,console.error("A component is changing from controlled to uncontrolled. This may be caused by the value changing from a defined value to undefined, which should not happen.")):(c.current=!0,o.current=l,console.error("A component is changing from uncontrolled to controlled. This may be caused by the value changing from undefined to a defined value, which should not happen.")),[l?e:s,(0,r.useEvent)(e=>(l||a(e),null==n?void 0:n(e)))]}],503269),e.s(["useDefaultValue",0,function(e){let[r]=(0,t.useState)(e);return r}],214520);let n=(0,t.createContext)(void 0);function i(){return(0,t.useContext)(n)}e.s(["useDisabled",0,i],601893);var s=e.i(174080),a=e.i(746725);function l(e={},t=null,r=[]){for(let[n,i]of Object.entries(e))!function e(t,r,n){if(Array.isArray(n))for(let[i,s]of n.entries())e(t,o(r,i.toString()),s);else n instanceof Date?t.push([r,n.toISOString()]):"boolean"==typeof n?t.push([r,n?"1":"0"]):"string"==typeof n?t.push([r,n]):"number"==typeof n?t.push([r,`${n}`]):null==n?t.push([r,""]):l(n,r,t)}(r,o(t,n),i);return r}function o(e,t){return e?e+"["+t+"]":t}e.s(["attemptSubmit",0,function(e){var t,r;let n=null!=(t=null==e?void 0:e.form)?t:e.closest("form");if(n){for(let t of n.elements)if(t!==e&&("INPUT"===t.tagName&&"submit"===t.type||"BUTTON"===t.tagName&&"submit"===t.type||"INPUT"===t.nodeName&&"image"===t.type))return void t.click();null==(r=n.requestSubmit)||r.call(n)}},"objectToFormEntries",0,l],694421);var c=e.i(700020),u=e.i(2788);let d=(0,t.createContext)(null);function m({children:e}){let r=(0,t.useContext)(d);if(!r)return t.default.createElement(t.default.Fragment,null,e);let{target:n}=r;return n?(0,s.createPortal)(t.default.createElement(t.default.Fragment,null,e),n):null}function f({setForm:e,formId:r}){return(0,t.useEffect)(()=>{if(r){let t=document.getElementById(r);t&&e(t)}},[e,r]),r?null:t.default.createElement(u.Hidden,{features:u.HiddenFeatures.Hidden,as:"input",type:"hidden",hidden:!0,readOnly:!0,ref:t=>{if(!t)return;let r=t.closest("form");r&&e(r)}})}e.s(["FormFields",0,function({data:e,form:r,disabled:n,onReset:i,overrides:s}){let[o,d]=(0,t.useState)(null),h=(0,a.useDisposables)();return(0,t.useEffect)(()=>{if(i&&o)return h.addEventListener(o,"reset",i)},[o,r,i]),t.default.createElement(m,null,t.default.createElement(f,{setForm:d,formId:r}),l(e).map(([e,i])=>t.default.createElement(u.Hidden,{features:u.HiddenFeatures.Hidden,...(0,c.compact)({key:e,as:"input",type:"hidden",hidden:!0,readOnly:!0,form:r,disabled:n,name:e,value:i,...s})})))}],140721);let h=(0,t.createContext)(void 0);function p(){return(0,t.useContext)(h)}e.s(["useProvidedId",0,p],942803);var g=e.i(835696),x=e.i(294316);let v=(0,t.createContext)(null);v.displayName="DescriptionContext";let y=Object.assign((0,c.forwardRefWithAs)(function(e,r){let n=(0,t.useId)(),s=i(),{id:a=`headlessui-description-${n}`,...l}=e,o=function e(){let r=(0,t.useContext)(v);if(null===r){let t=Error("You used a component, but it is not inside a relevant parent.");throw Error.captureStackTrace&&Error.captureStackTrace(t,e),t}return r}(),u=(0,x.useSyncRefs)(r);(0,g.useIsoMorphicEffect)(()=>o.register(a),[a,o.register]);let d=s||!1,m=(0,t.useMemo)(()=>({...o.slot,disabled:d}),[o.slot,d]),f={ref:u,...o.props,id:a};return(0,c.useRender)()({ourProps:f,theirProps:l,slot:m,defaultTag:"p",name:o.name||"Description"})}),{});e.s(["Description",0,y,"useDescribedBy",0,function(){var e,r;return null!=(r=null==(e=(0,t.useContext)(v))?void 0:e.value)?r:void 0},"useDescriptions",0,function(){let[e,n]=(0,t.useState)([]);return[e.length>0?e.join(" "):void 0,(0,t.useMemo)(()=>function(e){let i=(0,r.useEvent)(e=>(n(t=>[...t,e]),()=>n(t=>{let r=t.slice(),n=r.indexOf(e);return -1!==n&&r.splice(n,1),r}))),s=(0,t.useMemo)(()=>({register:i,slot:e.slot,name:e.name,props:e.props,value:e.value}),[i,e.slot,e.name,e.props,e.value]);return t.default.createElement(v.Provider,{value:s},e.children)},[n])]}],35889);let b=(0,t.createContext)(null);function _(e){var r,n,i;let s=null!=(n=null==(r=(0,t.useContext)(b))?void 0:r.value)?n:void 0;return(null!=(i=null==e?void 0:e.length)?i:0)>0?[s,...e].filter(Boolean).join(" "):s}b.displayName="LabelContext";let j=Object.assign((0,c.forwardRefWithAs)(function(e,n){var s;let a=(0,t.useId)(),l=function e(){let r=(0,t.useContext)(b);if(null===r){let t=Error("You used a