diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 8fbf1b3c5b4..39b46cba999 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -23,13 +23,15 @@ body: description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. render: shell - type: dropdown - id: ml-ops-team + id: component attributes: - label: Are you a ML Ops Team? - description: This helps us prioritize your requests correctly + label: What part of LiteLLM is this about? options: - - "No" - - "Yes" + - "SDK (litellm Python package)" + - "Proxy" + - "UI Dashboard" + - "Docs" + - "Other" validations: required: true - type: input diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 13a2132ec95..96b95cc7f02 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -22,6 +22,18 @@ body: description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too. validations: required: true + - type: dropdown + id: component + attributes: + label: What part of LiteLLM is this about? + options: + - "SDK (litellm Python package)" + - "Proxy" + - "UI Dashboard" + - "Docs" + - "Other" + validations: + required: true - type: dropdown id: hiring-interest attributes: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8977332ee01..b91b16c955c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,7 +1,3 @@ -## Title - - - ## Relevant issues @@ -11,7 +7,6 @@ **Please complete all items before asking a LiteLLM maintainer to review your PR** - [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code) -- [ ] I have added a screenshot of my new test passing locally - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code) - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem diff --git a/.github/workflows/create_daily_staging_branch.yml b/.github/workflows/create_daily_staging_branch.yml new file mode 100644 index 00000000000..a97cf6f9740 --- /dev/null +++ b/.github/workflows/create_daily_staging_branch.yml @@ -0,0 +1,43 @@ +name: Create Daily Staging Branch + +on: + schedule: + - cron: '0 0 * * *' # Runs daily at midnight UTC + workflow_dispatch: # Allow manual trigger + +jobs: + create-staging-branch: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Create daily staging branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Configure Git user + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Generate branch name with MM_DD_YYYY format + BRANCH_NAME="litellm_staging_$(date +'%m_%d_%Y')" + echo "Creating branch: $BRANCH_NAME" + + # Fetch all branches + git fetch --all + + # Check if the branch already exists + if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then + echo "Branch $BRANCH_NAME already exists. Skipping creation." + else + echo "Creating new branch: $BRANCH_NAME" + # Create the new branch from main + git checkout -b $BRANCH_NAME origin/main + # Push the new branch + git push origin $BRANCH_NAME + echo "Successfully created and pushed branch: $BRANCH_NAME" + fi diff --git a/.github/workflows/issue-keyword-labeler.yml b/.github/workflows/issue-keyword-labeler.yml index 60c18e3b9af..936f90f747f 100644 --- a/.github/workflows/issue-keyword-labeler.yml +++ b/.github/workflows/issue-keyword-labeler.yml @@ -19,7 +19,7 @@ jobs: id: scan env: PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }} - KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic + KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic,gemini,cohere,mistral,groq,ollama,deepseek run: python3 .github/scripts/scan_keywords.py - name: Ensure label exists diff --git a/.github/workflows/label-component.yml b/.github/workflows/label-component.yml new file mode 100644 index 00000000000..c0f9436288c --- /dev/null +++ b/.github/workflows/label-component.yml @@ -0,0 +1,144 @@ +name: Label Component Issues + +on: + issues: + types: + - opened + +jobs: + add-component-label: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Add SDK label + if: contains(github.event.issue.body, 'SDK (litellm Python package)') + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelName = 'sdk'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + color: '0E7C86', + description: 'Issues related to the litellm Python SDK' + }); + } else { + throw error; + } + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [labelName] + }); + + - name: Add Proxy label + if: contains(github.event.issue.body, 'Proxy') + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelName = 'proxy'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + color: '5319E7', + description: 'Issues related to the LiteLLM Proxy' + }); + } else { + throw error; + } + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [labelName] + }); + + - name: Add UI Dashboard label + if: contains(github.event.issue.body, 'UI Dashboard') + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelName = 'ui-dashboard'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + color: 'D876E3', + description: 'Issues related to the LiteLLM UI Dashboard' + }); + } else { + throw error; + } + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [labelName] + }); + + - name: Add Docs label + if: contains(github.event.issue.body, 'Docs') + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelName = 'docs'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + color: 'FBCA04', + description: 'Issues related to LiteLLM documentation' + }); + } else { + throw error; + } + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [labelName] + }); diff --git a/.github/workflows/label-mlops.yml b/.github/workflows/label-mlops.yml deleted file mode 100644 index 37789c1ea76..00000000000 --- a/.github/workflows/label-mlops.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Label ML Ops Team Issues - -on: - issues: - types: - - opened - -jobs: - add-mlops-label: - runs-on: ubuntu-latest - steps: - - name: Check if ML Ops Team is selected - uses: actions-ecosystem/action-add-labels@v1 - if: contains(github.event.issue.body, '### Are you a ML Ops Team?') && contains(github.event.issue.body, 'Yes') - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - labels: "mlops user request" diff --git a/deploy/charts/litellm-helm/README.md b/deploy/charts/litellm-helm/README.md index 6fdc423a177..2fa856843f3 100644 --- a/deploy/charts/litellm-helm/README.md +++ b/deploy/charts/litellm-helm/README.md @@ -29,7 +29,7 @@ If `db.useStackgresOperator` is used (not yet implemented): | `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A | | `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | | `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` | +| `image.repository` | LiteLLM Proxy image repository | `docker.litellm.ai/berriai/litellm` | | `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` | | `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` | | `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` | diff --git a/docker-compose.hardened.yml b/docker-compose.hardened.yml new file mode 100644 index 00000000000..31d0c2e9ef2 --- /dev/null +++ b/docker-compose.hardened.yml @@ -0,0 +1,46 @@ +services: + # Hardened stack: for testing the proxy under non-root, read-only, proxy-enforced constraints. + # Keep this file focused on hardening/QA scenarios; leave the main docker-compose.yml for default dev usage. + litellm: + build: + context: . + dockerfile: docker/Dockerfile.non_root + target: runtime + args: + PROXY_EXTRAS_SOURCE: "local" + depends_on: + - squid + user: "101:101" + group_add: + - "2345" + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + tmpfs: + - /app/cache:rw,noexec,nosuid,nodev,size=128m,uid=101,gid=101,mode=1777 + - /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777 + volumes: + - ./proxy_server_config.yaml:/app/config.yaml:ro + environment: + LITELLM_NON_ROOT: "true" + PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries" + XDG_CACHE_HOME: "/app/cache" + LITELLM_MIGRATION_DIR: "/app/migrations" + HTTP_PROXY: "http://squid:3128" + HTTPS_PROXY: "http://squid:3128" + NO_PROXY: "localhost,127.0.0.1,db" + command: + - "--port" + - "4000" + - "--config" + - "/app/config.yaml" + squid: + image: sameersbn/squid:3.5.27-2 + restart: unless-stopped + ports: + - "3128:3128" + tmpfs: + - /var/spool/squid:rw,noexec,nosuid,nodev,size=64m + - /var/log/squid:rw,noexec,nosuid,nodev,size=16m diff --git a/docker-compose.yml b/docker-compose.yml index 8898aff62da..988860a7877 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: context: . args: target: runtime - image: ghcr.io/berriai/litellm:main-stable + image: docker.litellm.ai/berriai/litellm:main-stable ######################################### ## Uncomment these lines to start proxy with a config.yaml file ## # volumes: diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index f036081549a..ce83cfe653c 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -34,8 +34,8 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt # Runtime stage FROM $LITELLM_RUNTIME_IMAGE AS runtime -# Update dependencies and clean up -RUN apk upgrade --no-cache +# Update dependencies and clean up, install libsndfile for audio processing +RUN apk upgrade --no-cache && apk add --no-cache libsndfile WORKDIR /app diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 9fc8acf2a18..d8a362680e4 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,154 +1,183 @@ # Base images ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base +ARG PROXY_EXTRAS_SOURCE=published # ----------------- # Builder Stage # ----------------- FROM $LITELLM_BUILD_IMAGE AS builder +ARG PROXY_EXTRAS_SOURCE WORKDIR /app - -# Install build dependencies including Node.js for UI build USER root + +# Install build dependencies with retry logic (includes node for UI build) RUN for i in 1 2 3; do \ - apk add --no-cache \ - python3 \ - py3-pip \ - clang \ - llvm \ - lld \ - gcc \ - linux-headers \ - build-base \ - bash \ - nodejs \ - npm && break || sleep 5; \ - done \ + apk add --no-cache \ + python3 \ + py3-pip \ + clang \ + llvm \ + lld \ + gcc \ + linux-headers \ + build-base \ + bash \ + nodejs \ + npm && break || sleep 5; \ + done \ && pip install --no-cache-dir --upgrade pip build -# Copy project files +# Cache Python dependencies +COPY requirements.txt . +RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \ + && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.9.0" + +# Copy source after dependency layers COPY . . -# Set LITELLM_NON_ROOT flag for build time +# Set non-root flag for build time consistency ENV LITELLM_NON_ROOT=true -# Build Admin UI -RUN mkdir -p /tmp/litellm_ui +# Build Admin UI using the upstream command order while keeping a single RUN layer +RUN mkdir -p /tmp/litellm_ui && \ + npm install -g npm@latest && npm cache clean --force && \ + cd /app/ui/litellm-dashboard && \ + if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ + cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ + fi && \ + rm -f package-lock.json && \ + npm install --legacy-peer-deps && \ + npm run build && \ + cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ && \ + mkdir -p /tmp/litellm_assets && \ + cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg && \ + ( cd /tmp/litellm_ui && \ + for html_file in *.html; do \ + if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ + folder_name="${html_file%.html}" && \ + mkdir -p "$folder_name" && \ + mv "$html_file" "$folder_name/index.html"; \ + fi; \ + done ) && \ + cd /app/ui/litellm-dashboard && rm -rf ./out -RUN npm install -g npm@latest && npm cache clean --force - -RUN cd /app/ui/litellm-dashboard && \ - if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ - cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ - fi - -RUN cd /app/ui/litellm-dashboard && rm -f package-lock.json - -RUN cd /app/ui/litellm-dashboard && npm install --legacy-peer-deps - -RUN cd /app/ui/litellm-dashboard && npm run build - -RUN cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ -RUN mkdir -p /tmp/litellm_assets && cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg - -RUN cd /tmp/litellm_ui && \ - for html_file in *.html; do \ - if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ - folder_name="${html_file%.html}" && \ - mkdir -p "$folder_name" && \ - mv "$html_file" "$folder_name/index.html"; \ - fi; \ - done - -RUN cd /app/ui/litellm-dashboard && rm -rf ./out - -# Build package and wheel dependencies +# Build litellm wheel and place it in wheels dir (replace any PyPI wheels) RUN rm -rf dist/* && python -m build && \ - pip install dist/*.whl && \ - pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt + rm -f /wheels/litellm-*.whl && \ + cp dist/*.whl /wheels/ + +# Optionally build local litellm-proxy-extras wheel +RUN if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \ + cd /app/litellm-proxy-extras && rm -rf dist && python -m build && \ + cp dist/*.whl /wheels/; \ + fi + +# Pre-cache Prisma binaries in the builder stage +ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \ + XDG_CACHE_HOME=/app/.cache \ + PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}" + +RUN pip install --no-cache-dir prisma==0.11.0 nodejs-bin==18.4.0a4 \ + && mkdir -p /app/.cache/npm + +RUN NPM_CONFIG_CACHE=/app/.cache/npm \ + python -c "import prisma.cli.prisma as p; p.ensure_cached()" + +RUN prisma generate && \ + prisma --version && \ + prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true # ----------------- # Runtime Stage # ----------------- FROM $LITELLM_RUNTIME_IMAGE AS runtime +ARG PROXY_EXTRAS_SOURCE WORKDIR /app - -# Install runtime dependencies USER root -RUN for i in 1 2 3; do \ - apk upgrade --no-cache && break || sleep 5; \ - done \ - && for i in 1 2 3; do \ - apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \ - done -# Copy only necessary artifacts from builder stage for runtime -COPY . . +# Install runtime dependencies with retry +RUN for i in 1 2 3; do \ + apk upgrade --no-cache && break || sleep 5; \ + done \ + && for i in 1 2 3; do \ + apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \ + done + +# Copy artifacts from builder +COPY --from=builder /app/requirements.txt /app/requirements.txt COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/ COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf -COPY --from=builder /app/schema.prisma /app/schema.prisma -COPY --from=builder /app/dist/*.whl . +COPY --from=builder /app/schema.prisma /app/ COPY --from=builder /wheels/ /wheels/ COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets +COPY --from=builder /app/.cache /app/.cache +COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras +COPY --from=builder \ + /usr/lib/python3.13/site-packages/nodejs* \ + /usr/lib/python3.13/site-packages/prisma* \ + /usr/lib/python3.13/site-packages/tomlkit* \ + /usr/lib/python3.13/site-packages/nodeenv* \ + /usr/lib/python3.13/site-packages/ +COPY --from=builder /usr/bin/prisma /usr/bin/prisma -# Install package from wheel and dependencies -RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \ - && rm -f *.whl \ - && rm -rf /wheels +# Final runtime environment configuration +ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \ + HOME=/app \ + LITELLM_NON_ROOT=true \ + XDG_CACHE_HOME=/app/.cache -# Remove test files and keys from dependencies -RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \ - find /usr/lib -type d -path "*/tornado/test" -delete +# Install packages from wheels and optional extras without network +RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \ + pip install --no-index --find-links=/wheels/ /wheels/litellm-*-py3-none-any.whl && \ + pip install --no-index --find-links=/wheels/ --no-deps semantic_router==0.1.11 && \ + pip install --no-index --find-links=/wheels/ aurelio-sdk==0.0.19 && \ + if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \ + if ls /wheels/litellm_proxy_extras-*.whl >/dev/null 2>&1; then \ + pip install --no-index --find-links=/wheels/ /wheels/litellm_proxy_extras-*.whl; \ + else \ + echo "litellm_proxy_extras wheel not found; skipping local install"; \ + fi; \ + fi -# Install semantic_router and aurelio-sdk using script -RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh +# Permissions, cleanup, and Prisma prep +RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ + mkdir -p /nonexistent /.npm /tmp/litellm_assets /tmp/litellm_ui && \ + chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \ + pip uninstall jwt -y || true && \ + pip uninstall PyJWT -y || true && \ + pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \ + rm -rf /wheels && \ + PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ + chown -R nobody:nogroup $PRISMA_PATH && \ + LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \ + [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \ + LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \ + chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ + [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \ + chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ + [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \ + chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ + [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \ + chmod -R g+rX $PRISMA_PATH && \ + chmod -R g+rX /app/.cache && \ + mkdir -p /tmp/.npm /nonexistent /.npm && \ + prisma generate -# Ensure correct JWT library is used (pyjwt not jwt) -RUN pip uninstall jwt -y && \ - pip uninstall PyJWT -y && \ - pip install PyJWT==2.9.0 --no-cache-dir - -# Set Prisma cache directories -ENV PRISMA_BINARY_CACHE_DIR=/nonexistent -ENV NPM_CONFIG_CACHE=/.npm - -# Install prisma and make entrypoints executable -RUN pip install --no-cache-dir prisma && \ - chmod +x docker/entrypoint.sh && \ - chmod +x docker/prod_entrypoint.sh - -# Create directories and set permissions for non-root user -RUN mkdir -p /nonexistent /.npm /tmp/litellm_assets && \ - chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \ - PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ - chown -R nobody:nogroup $PRISMA_PATH && \ - LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \ - [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH - -# OpenShift compatibility -RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ - LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \ - chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true - -# Switch to non-root user +# Switch to non-root user for runtime USER nobody -# Set HOME for prisma generate to have a writable directory -ENV HOME=/app - -# Set LITELLM_NON_ROOT flag for runtime -ENV LITELLM_NON_ROOT=true - -RUN prisma generate +# Prisma runtime knobs for offline containers +ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \ + PRISMA_HIDE_UPDATE_MESSAGE=1 \ + PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \ + NPM_CONFIG_CACHE=/app/.cache/npm \ + NPM_CONFIG_PREFER_OFFLINE=true \ + PRISMA_OFFLINE_MODE=true EXPOSE 4000/tcp - ENTRYPOINT ["/app/docker/prod_entrypoint.sh"] - -CMD ["--port", "4000"] \ No newline at end of file +CMD ["--port", "4000"] diff --git a/docker/README.md b/docker/README.md index ce478dfe0dd..6d81276bb4b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -59,6 +59,30 @@ To stop the running containers, use the following command: docker compose down ``` +## Hardened / Offline Testing + +To ensure changes are safe for non-root, read-only root filesystems and restricted egress, always validate with the hardened compose file: + +```bash +docker compose -f docker-compose.yml -f docker-compose.hardened.yml build --no-cache +docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d +``` + +This setup: +- Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image. +- Runs the proxy as a non-root user with a read-only rootfs and only two writable tmpfs mounts: + - `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`) + - `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`) +- Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines. + +You should also verify offline Prisma behaviour with: + +```bash +docker run --rm --network none --entrypoint prisma ghcr.io/berriai/litellm:main-stable --version +``` + +This command should succeed (showing engine versions) even with `--network none`, confirming that Prisma binaries are available without network access. + ## Troubleshooting - **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project. diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 1e5f968b2ca..7015918e924 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -6,7 +6,7 @@ authors: - name: Sameer Kankute title: SWE @ LiteLLM (LLM Translation) url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - name: Krrish Dholakia title: "CEO, LiteLLM" url: https://www.linkedin.com/in/krish-d/ diff --git a/docs/my-website/blog/gemini_3/index.md b/docs/my-website/blog/gemini_3/index.md index 1b9ff359f3a..26dbc2d02b5 100644 --- a/docs/my-website/blog/gemini_3/index.md +++ b/docs/my-website/blog/gemini_3/index.md @@ -6,7 +6,7 @@ authors: - name: Sameer Kankute title: SWE @ LiteLLM (LLM Translation) url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - name: Krrish Dholakia title: "CEO, LiteLLM" url: https://www.linkedin.com/in/krish-d/ diff --git a/docs/my-website/blog/gemini_3_flash/index.md b/docs/my-website/blog/gemini_3_flash/index.md new file mode 100644 index 00000000000..cc4298274a6 --- /dev/null +++ b/docs/my-website/blog/gemini_3_flash/index.md @@ -0,0 +1,222 @@ +--- +slug: gemini_3_flash +title: "DAY 0 Support: Gemini 3 Flash on LiteLLM" +date: 2025-12-17T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +tags: [gemini, day 0 support, llms] +hide_table_of_contents: false +--- + + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Gemini 3 Flash Day 0 Support + +LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along with it. + +## What's New + +### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM + +Gemini 3 Flash introduces granular thinking control with `thinkingLevel` instead of `thinkingBudget`. +- **MINIMAL**: Ultra-lightweight thinking for fast responses +- **MEDIUM**: Balanced thinking for complex reasoning +- **HIGH**: Maximum reasoning depth + +LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code! + +### 2. Thought Signatures + +Like `gemini-3-pro`, this model also includes thought signatures for tool calls. LiteLLM handles signature extraction and embedding internally. [Learn more about thought signatures](../gemini_3/index.md#thought-signatures). + +**Edge Case Handling**: If thought signatures are missing in the request, LiteLLM adds a dummy signature ensuring the API call doesn't break + +--- +## Supported Endpoints + +LiteLLM provides **full end-to-end support** for Gemini 3 Flash on: + +- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint +- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) +- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint +All endpoints support: +- Streaming and non-streaming responses +- Function calling with thought signatures +- Multi-turn conversations +- All Gemini 3-specific features +- Converstion of provider specific thinking related param to thinkingLevel + +## Quick Start + + + + +**Basic Usage with MEDIUM thinking (NEW)** + +```python +from litellm import completion + +# No need to make any changes to your code as we map openai reasoning param to thinkingLevel +response = completion( + model="gemini/gemini-3-flash-preview", + messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}], + reasoning_effort="medium", # NEW: MEDIUM thinking level +) + +print(response.choices[0].message.content) +``` + + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: gemini-3-flash + litellm_params: + model: gemini/gemini-3-flash-preview + api_key: os.environ/GEMINI_API_KEY +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Call with MEDIUM thinking** + +```bash +curl -X POST http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "model": "gemini-3-flash", + "messages": [{"role": "user", "content": "Complex reasoning task"}], + "reasoning_effort": "medium" + }' +``' + + + + +--- + +## All `reasoning_effort` Levels + + + + +**Ultra-fast, minimal reasoning** + +```python +from litellm import completion + +response = completion( + model="gemini/gemini-3-flash-preview", + messages=[{"role": "user", "content": "What's 2+2?"}], + reasoning_effort="minimal", +) +``` + + + + + +**Simple instruction following** + +```python +response = completion( + model="gemini/gemini-3-flash-preview", + messages=[{"role": "user", "content": "Write a haiku about coding"}], + reasoning_effort="low", +) +``` + + + + + +**Balanced reasoning for complex tasks** ✨ + +```python +response = completion( + model="gemini/gemini-3-flash-preview", + messages=[{"role": "user", "content": "Analyze this dataset and find patterns"}], + reasoning_effort="medium", # NEW! +) +``` + + + + + +**Maximum reasoning depth** + +```python +response = completion( + model="gemini/gemini-3-flash-preview", + messages=[{"role": "user", "content": "Prove this mathematical theorem"}], + reasoning_effort="high", +) +``` + + + + +--- + +## Key Features + +✅ **Thinking Levels**: MINIMAL, LOW, MEDIUM, HIGH +✅ **Thought Signatures**: Track reasoning with unique identifiers +✅ **Seamless Integration**: Works with existing OpenAI-compatible client +✅ **Backward Compatible**: Gemini 2.5 models continue using `thinkingBudget` + +--- + +## Installation + +```bash +pip install litellm --upgrade +``` + +```python +import litellm +from litellm import completion + +response = completion( + model="gemini/gemini-3-flash-preview", + messages=[{"role": "user", "content": "Your question here"}], + reasoning_effort="medium", # Use MEDIUM thinking +) +print(response) +``` + +## `reasoning_effort` Mapping for Gemini 3+ + +| reasoning_effort | thinking_level | +|------------------|----------------| +| `minimal` | `minimal` | +| `low` | `low` | +| `medium` | `medium` | +| `high` | `high` | +| `disable` | `minimal` | +| `none` | `minimal` | + diff --git a/docs/my-website/docs/a2a.md b/docs/my-website/docs/a2a.md index 9c94a2fbf29..d7145e4b83c 100644 --- a/docs/my-website/docs/a2a.md +++ b/docs/my-website/docs/a2a.md @@ -16,7 +16,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque | Feature | Supported | |---------|-----------| -| Supported Agent Providers | A2A, LangGraph, Azure AI Foundry, Bedrock AgentCore | +| Supported Agent Providers | A2A, Vertex AI Agent Engine, LangGraph, Azure AI Foundry, Bedrock AgentCore, Pydantic AI | | Logging | ✅ | | Load Balancing | ✅ | | Streaming | ✅ | @@ -45,17 +45,26 @@ You can add A2A-compatible agents through the LiteLLM Admin UI. The URL should be the invocation URL for your A2A agent (e.g., `http://localhost:10001`). + ### Add Azure AI Foundry Agents Follow [this guide, to add your azure ai foundry agent to LiteLLM Agent Gateway](./providers/azure_ai_agents#litellm-a2a-gateway) +### Add Vertex AI Agent Engine + +Follow [this guide, to add your Vertex AI Agent Engine to LiteLLM Agent Gateway](./providers/vertex_ai_agent_engine) + +### Add Bedrock AgentCore Agents + +Follow [this guide, to add your bedrock agentcore agent to LiteLLM Agent Gateway](./providers/bedrock_agentcore#litellm-a2a-gateway) + ### Add LangGraph Agents Follow [this guide, to add your langgraph agent to LiteLLM Agent Gateway](./providers/langgraph#litellm-a2a-gateway) -### Add Bedrock AgentCore Agents +### Add Pydantic AI Agents -Follow [this guide, to add your bedrock agentcore agent to LiteLLM Agent Gateway](./providers/bedrock_agentcore#litellm-a2a-gateway) +Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./providers/pydantic_ai_agent#litellm-a2a-gateway) ## Invoking your Agents diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md index 4e4234949f8..76b61d4c2bd 100644 --- a/docs/my-website/docs/benchmarks.md +++ b/docs/my-website/docs/benchmarks.md @@ -172,7 +172,7 @@ class MyUser(HttpUser): ## Logging Callbacks -### [GCS Bucket Logging](https://docs.litellm.ai/docs/proxy/bucket) +### [GCS Bucket Logging](https://docs.litellm.ai/docs/observability/gcs_bucket_integration) Using GCS Bucket has **no impact on latency, RPS compared to Basic Litellm Proxy** diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index f393b300f73..ba605e316d3 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -657,7 +657,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml --detailed_debug ``` diff --git a/docs/my-website/docs/interactions.md b/docs/my-website/docs/interactions.md new file mode 100644 index 00000000000..5458a4463f5 --- /dev/null +++ b/docs/my-website/docs/interactions.md @@ -0,0 +1,214 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# /interactions + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Logging | ✅ | Works across all integrations | +| Streaming | ✅ | | +| Loadbalancing | ✅ | Between supported models | +| Supported Providers | `gemini` | [Google Interactions API](https://ai.google.dev/gemini-api/docs/interactions) | + +## **LiteLLM Python SDK Usage** + +### Quick Start + +```python showLineNumbers title="Create Interaction" +from litellm import create_interaction +import os + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +response = create_interaction( + model="gemini/gemini-2.5-flash", + input="Tell me a short joke about programming." +) + +print(response.outputs[-1].text) +``` + +### Async Usage + +```python showLineNumbers title="Async Create Interaction" +from litellm import acreate_interaction +import os +import asyncio + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +async def main(): + response = await acreate_interaction( + model="gemini/gemini-2.5-flash", + input="Tell me a short joke about programming." + ) + print(response.outputs[-1].text) + +asyncio.run(main()) +``` + +### Streaming + +```python showLineNumbers title="Streaming Interaction" +from litellm import create_interaction +import os + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +response = create_interaction( + model="gemini/gemini-2.5-flash", + input="Write a 3 paragraph story about a robot.", + stream=True +) + +for chunk in response: + print(chunk) +``` + +## **LiteLLM AI Gateway (Proxy) Usage** + +### Setup + +Add this to your litellm proxy config.yaml: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: gemini-flash + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY +``` + +Start litellm: + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### Test Request + + + + +```bash showLineNumbers title="Create Interaction" +curl -X POST "http://localhost:4000/v1beta/interactions" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemini/gemini-2.5-flash", + "input": "Tell me a short joke about programming." + }' +``` + +**Streaming:** + +```bash showLineNumbers title="Streaming Interaction" +curl -N -X POST "http://localhost:4000/v1beta/interactions" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemini/gemini-2.5-flash", + "input": "Write a 3 paragraph story about a robot.", + "stream": true + }' +``` + +**Get Interaction:** + +```bash showLineNumbers title="Get Interaction by ID" +curl "http://localhost:4000/v1beta/interactions/{interaction_id}" \ + -H "Authorization: Bearer sk-1234" +``` + + + + + +Point the Google GenAI SDK to LiteLLM Proxy: + +```python showLineNumbers title="Google GenAI SDK with LiteLLM Proxy" +from google import genai +import os + +# Point SDK to LiteLLM Proxy +os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000" +os.environ["GEMINI_API_KEY"] = "sk-1234" # Your LiteLLM API key + +client = genai.Client() + +# Create an interaction +interaction = client.interactions.create( + model="gemini/gemini-2.5-flash", + input="Tell me a short joke about programming." +) + +print(interaction.outputs[-1].text) +``` + +**Streaming:** + +```python showLineNumbers title="Google GenAI SDK Streaming" +from google import genai +import os + +os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000" +os.environ["GEMINI_API_KEY"] = "sk-1234" + +client = genai.Client() + +for chunk in client.interactions.create_stream( + model="gemini/gemini-2.5-flash", + input="Write a story about space exploration.", +): + print(chunk) +``` + + + + +## **Request/Response Format** + +### Request Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `model` | string | Yes | Model to use (e.g., `gemini/gemini-2.5-flash`) | +| `input` | string | Yes | The input text for the interaction | +| `stream` | boolean | No | Enable streaming responses | +| `tools` | array | No | Tools available to the model | +| `system_instruction` | string | No | System instructions for the model | +| `generation_config` | object | No | Generation configuration | +| `previous_interaction_id` | string | No | ID of previous interaction for context | + +### Response Format + +```json +{ + "id": "interaction_abc123", + "object": "interaction", + "model": "gemini-2.5-flash", + "status": "completed", + "created": "2025-01-15T10:30:00Z", + "updated": "2025-01-15T10:30:05Z", + "role": "model", + "outputs": [ + { + "type": "text", + "text": "Why do programmers prefer dark mode? Because light attracts bugs!" + } + ], + "usage": { + "total_input_tokens": 10, + "total_output_tokens": 15, + "total_tokens": 25 + } +} +``` + +## **Supported Providers** + +| Provider | Link to Usage | +|----------|---------------| +| Google AI Studio | [Usage](#quick-start) | diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index b2901650ea6..7cf91ced34c 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -181,7 +181,7 @@ docker run \ -e USE_DDTRACE=true \ -e USE_DDPROFILER=true \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml --detailed_debug ``` diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index f78af51bd90..bcfb698a0f8 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -1936,3 +1936,87 @@ curl http://0.0.0.0:4000/v1/chat/completions \ + +## Usage - Agent Skills + +LiteLLM supports using Agent Skills with the API + + + + +```python +response = completion( + model="claude-sonnet-4-5-20250929", + messages=messages, + tools= [ + { + "type": "code_execution_20250825", + "name": "code_execution" + } + ], + container= { + "skills": [ + { + "type": "anthropic", + "skill_id": "pptx", + "version": "latest" + } + ] + } +) +``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-sonnet-4-5-20250929 + litellm_params: + model: anthropic/claude-sonnet-4-5-20250929 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start Proxy + +``` +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://localhost:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "model": "claude-sonnet-4-5-20250929", + "messages": [ + { + "role": "user", + "content": "Hi" + } + ], + "tools": [ + { + "type": "code_execution_20250825", + "name": "code_execution" + } + ], + "container": { + "skills": [ + { + "type": "anthropic", + "skill_id": "pptx", + "version": "latest" + } + ] + } +}' +``` + + + + +The container and its "id" will be present in "provider_specific_fields" in streaming/non-streaming response \ No newline at end of file diff --git a/docs/my-website/docs/providers/custom_llm_server.md b/docs/my-website/docs/providers/custom_llm_server.md index 61099d1a358..4fcbf8942ce 100644 --- a/docs/my-website/docs/providers/custom_llm_server.md +++ b/docs/my-website/docs/providers/custom_llm_server.md @@ -17,6 +17,7 @@ Supported Routes: - `/v1/completions` -> `litellm.atext_completion` - `/v1/embeddings` -> `litellm.aembedding` - `/v1/images/generations` -> `litellm.aimage_generation` +- `/v1/images/edits` -> `litellm.aimage_edit` - `/v1/messages` -> `litellm.acompletion` @@ -263,6 +264,83 @@ Expected Response } ``` +## Image Edit + +1. Setup your `custom_handler.py` file +```python +import litellm +from litellm import CustomLLM +from litellm.types.utils import ImageResponse, ImageObject +import time + +class MyCustomLLM(CustomLLM): + async def aimage_edit( + self, + model: str, + image: Any, + prompt: str, + model_response: ImageResponse, + api_key: Optional[str], + api_base: Optional[str], + optional_params: dict, + logging_obj: Any, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + ) -> ImageResponse: + # Your custom image edit logic here + # e.g., call Stability AI, Black Forest Labs, etc. + return ImageResponse( + created=int(time.time()), + data=[ImageObject(url="https://example.com/edited-image.png")], + ) + +my_custom_llm = MyCustomLLM() +``` + + +2. Add to `config.yaml` + +In the config below, we pass + +python_filename: `custom_handler.py` +custom_handler_instance_name: `my_custom_llm`. This is defined in Step 1 + +custom_handler: `custom_handler.my_custom_llm` + +```yaml +model_list: + - model_name: "my-custom-image-edit-model" + litellm_params: + model: "my-custom-llm/my-model" + +litellm_settings: + custom_provider_map: + - {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm} +``` + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl -X POST 'http://0.0.0.0:4000/v1/images/edits' \ +-H 'Authorization: Bearer sk-1234' \ +-F 'model=my-custom-image-edit-model' \ +-F 'image=@/path/to/image.png' \ +-F 'prompt=Make the sky blue' +``` + +Expected Response + +``` +{ + "created": 1721955063, + "data": [{"url": "https://example.com/edited-image.png"}], +} +``` + ## Anthropic `/v1/messages` - Write the integration for .acompletion @@ -517,4 +595,34 @@ class CustomLLM(BaseLLM): client: Optional[AsyncHTTPHandler] = None, ) -> ImageResponse: raise CustomLLMError(status_code=500, message="Not implemented yet!") + + def image_edit( + self, + model: str, + image: Any, + prompt: str, + model_response: ImageResponse, + api_key: Optional[str], + api_base: Optional[str], + optional_params: dict, + logging_obj: Any, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[HTTPHandler] = None, + ) -> ImageResponse: + raise CustomLLMError(status_code=500, message="Not implemented yet!") + + async def aimage_edit( + self, + model: str, + image: Any, + prompt: str, + model_response: ImageResponse, + api_key: Optional[str], + api_base: Optional[str], + optional_params: dict, + logging_obj: Any, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + ) -> ImageResponse: + raise CustomLLMError(status_code=500, message="Not implemented yet!") ``` diff --git a/docs/my-website/docs/providers/pydantic_ai_agent.md b/docs/my-website/docs/providers/pydantic_ai_agent.md new file mode 100644 index 00000000000..e96295faaf3 --- /dev/null +++ b/docs/my-website/docs/providers/pydantic_ai_agent.md @@ -0,0 +1,121 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Pydantic AI Agents + +Call Pydantic AI Agents via LiteLLM's A2A Gateway. + +| Property | Details | +|----------|---------| +| Description | Pydantic AI agents with native A2A support via the `to_a2a()` method. LiteLLM provides fake streaming support for agents that don't natively stream. | +| Provider Route on LiteLLM | A2A Gateway | +| Supported Endpoints | `/v1/a2a/message/send` | +| Provider Doc | [Pydantic AI Agents ↗](https://ai.pydantic.dev/agents/) | + +## LiteLLM A2A Gateway + +All Pydantic AI agents need to be exposed as A2A agents using the `to_a2a()` method. Once your agent server is running, you can add it to the LiteLLM Gateway. + +### 1. Setup Pydantic AI Agent Server + +LiteLLM requires Pydantic AI agents to follow the [A2A (Agent-to-Agent) protocol](https://github.com/google/A2A). Pydantic AI has native A2A support via the `to_a2a()` method, which exposes your agent as an A2A-compliant server. + +#### Install Dependencies + +```bash +pip install pydantic-ai fasta2a uvicorn +``` + +#### Create Agent + +```python title="agent.py" +from pydantic_ai import Agent + +agent = Agent('openai:gpt-4o-mini', instructions='Be helpful!') + +@agent.tool_plain +def get_weather(city: str) -> str: + """Get weather for a city.""" + return f"Weather in {city}: Sunny, 72°F" + +@agent.tool_plain +def calculator(expression: str) -> str: + """Evaluate a math expression.""" + return str(eval(expression)) + +# Native A2A server - Pydantic AI handles it automatically +app = agent.to_a2a() +``` + +#### Run Server + +```bash +uvicorn agent:app --host 0.0.0.0 --port 9999 +``` + +Server runs at `http://localhost:9999` + +### 2. Navigate to Agents + +From the sidebar, click "Agents" to open the agent management page, then click "+ Add New Agent". + +### 3. Select Pydantic AI Agent Type + +Click "A2A Standard" to see available agent types, then select "Pydantic AI". + +![Select A2A Standard](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/1055acb1-064b-4465-8e6a-8278291bc661/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=395,147) + +![Select Pydantic AI](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/0998e38c-8534-40f1-931a-be96c2cae0ad/ascreenshot.jpeg?tl_px=0,52&br_px=2201,1283&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=421,277) + +### 4. Configure the Agent + +Fill in the following fields: + +- **Agent Name** - A unique identifier for your agent (e.g., `test-pydantic-agent`) +- **Agent URL** - The URL where your Pydantic AI agent is running. We use `http://localhost:9999` because that's where we started our Pydantic AI agent server in the previous step. + +![Enter Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/8cf3fbde-05f3-48d1-81b6-6f857bd6d360/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=443,225) + +![Configure Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fb555808-4761-4c49-a415-200ac1bdb525/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1463&force_format=jpeg&q=100&width=1120.0) + +![Enter Agent URL](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/303eae61-4352-4fb0-a537-806839c234ba/ascreenshot.jpeg?tl_px=0,212&br_px=2201,1443&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=456,277) + +### 5. Create Agent + +Click "Create Agent" to save your configuration. + +![Create Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/914f3367-df7d-4244-bd4d-e99ce0a6193a/ascreenshot.jpeg?tl_px=416,438&br_px=2618,1669&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=690,277) + +### 6. Test in Playground + +Go to "Playground" in the sidebar to test your agent. + +![Go to Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/c73c9f3b-22af-4105-aafa-2d34c4986ef3/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=44,97) + +### 7. Select A2A Endpoint + +Click the endpoint dropdown and search for "a2a", then select `/v1/a2a/message/send`. + +![Click Endpoint Dropdown](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/196d97ac-bcba-47f0-9880-97b80250e00c/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=261,230) + +![Search for A2A](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/26b68f21-29f9-4c4c-b8b5-d2e11cbfd14a/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1463&force_format=jpeg&q=100&width=1120.0) + +![Select A2A Endpoint](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/41576fb1-d385-4fb2-84e9-142dd7fe5181/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=307,270) + +### 8. Select Your Agent and Send a Message + +Pick your Pydantic AI agent from the dropdown and send a test message. + +![Click Agent Dropdown](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a96d7967-3d54-4cbf-bd3e-b38f1be9df76/ascreenshot.jpeg?tl_px=0,54&br_px=2201,1285&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=274,277) + +![Select Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/e05a5a6e-d044-4480-b94e-7c03cfb92ac5/ascreenshot.jpeg?tl_px=0,113&br_px=2201,1344&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=290,277) + +![Send Message](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/29162702-968a-401a-aac1-c844bfc5f4a3/ascreenshot.jpeg?tl_px=91,653&br_px=2292,1883&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,436) + + +## Further Reading + +- [Pydantic AI Documentation](https://ai.pydantic.dev/) +- [Pydantic AI Agents](https://ai.pydantic.dev/agents/) +- [A2A Agent Gateway](../a2a.md) +- [A2A Cost Tracking](../a2a_cost_tracking.md) diff --git a/docs/my-website/docs/providers/sap.md b/docs/my-website/docs/providers/sap.md index a9183b9c0df..4bc72c27045 100644 --- a/docs/my-website/docs/providers/sap.md +++ b/docs/my-website/docs/providers/sap.md @@ -5,12 +5,12 @@ import TabItem from '@theme/TabItem'; LiteLLM supports SAP Generative AI Hub's Orchestration Service. -| Property | Details | -|-------|-------| -| Description | SAP's Generative AI Hub provides access to foundation models through the AI Core orchestration service. | -| Provider Route on LiteLLM | `sap/` | -| Supported Endpoints | `/chat/completions` | -| API Reference | [SAP AI Core Documentation](https://help.sap.com/docs/sap-ai-core) | +| Property | Details | +|-------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| Description | SAP's Generative AI Hub provides access to OpenAI, Anthropic, Gemini, Mistral, NVIDIA, Amazon, and SAP LLMs through the AI Core orchestration service. | +| Provider Route on LiteLLM | `sap/` | +| Supported Endpoints | `/chat/completions`, `/embeddings` | +| API Reference | [SAP AI Core Documentation](https://help.sap.com/docs/sap-ai-core) | ## Authentication @@ -23,7 +23,14 @@ SAP Generative AI Hub uses service key authentication. You can provide credentia import os os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}' ``` - +3. **Environment variables** - Set the following list of credentials in .env file +
+AICORE_AUTH_URL = "https://* * * .authentication.sap.hana.ondemand.com/oauth/token",
+AICORE_CLIENT_ID  = " *** ",
+AICORE_CLIENT_SECRET = " *** ",
+AICORE_RESOURCE_GROUP = " *** ",
+AICORE_BASE_URL = "https://api.ai.***.cfapps.sap.hana.ondemand.com/v2"
+
## Usage - LiteLLM Python SDK ```python showLineNumbers title="SAP Chat Completion" @@ -55,16 +62,33 @@ for chunk in response: print(chunk.choices[0].delta.content or "", end="") ``` +```python showLineNumbers title="SAP Embedding" +from litellm import embedding +import os + +os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}' + +result = embedding( + model="sap/text-embedding-3-small", + input="Answer to the ultimate question of life, the universe, and everything is 42") +print(result.data[0]) +``` + ## Usage - LiteLLM Proxy Add to your LiteLLM Proxy config: ```yaml showLineNumbers title="config.yaml" model_list: - - model_name: sap-gpt4 + - model_name: "sap/*" litellm_params: - model: sap/gpt-4 - api_key: os.environ/AICORE_SERVICE_KEY + model: "sap/*" + +general_settings: + master_key: your-proxy-api-key + +environment_variables: + AICORE_SERVICE_KEY: '{"clientid": "...", "clientsecret": "...", ...}' ``` Start the proxy: @@ -81,7 +105,7 @@ curl http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer your-proxy-api-key" \ -d '{ - "model": "sap-gpt4", + "model": "sap/gpt-4", "messages": [{"role": "user", "content": "Hello"}] }' ``` @@ -98,12 +122,29 @@ client = OpenAI( ) response = client.chat.completions.create( - model="sap-gpt4", + model="sap/gpt-4", messages=[{"role": "user", "content": "Hello"}] ) print(response.choices[0].message.content) ``` + + + +```python showLineNumbers title="LiteLLM SDK" +import os +import litellm +os.environ["LITELLM_PROXY_API_KEY"] = "your-proxy-api-key" +litellm.use_litellm_proxy = True # it is important to set this parameter +response = litellm.completion( + model="sap/gpt-4o", + messages=[{ "content": "Hello, how are you?","role": "user"}], + api_base="http://your-proxy-api-base" +) + +print(response) +``` + diff --git a/docs/my-website/docs/providers/vertex_ai_agent_engine.md b/docs/my-website/docs/providers/vertex_ai_agent_engine.md new file mode 100644 index 00000000000..3bd40e98684 --- /dev/null +++ b/docs/my-website/docs/providers/vertex_ai_agent_engine.md @@ -0,0 +1,216 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Vertex AI Agent Engine + +Call Vertex AI Agent Engine (Reasoning Engines) in the OpenAI Request/Response format. + +| Property | Details | +|----------|---------| +| Description | Vertex AI Agent Engine provides hosted agent runtimes that can execute agentic workflows with foundation models, tools, and custom logic. | +| Provider Route on LiteLLM | `vertex_ai/agent_engine/{RESOURCE_NAME}` | +| Supported Endpoints | `/chat/completions`, `/v1/messages`, `/v1/responses`, `/v1/a2a/message/send` | +| Provider Doc | [Vertex AI Agent Engine ↗](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/overview) | + +## Quick Start + +### Model Format + +```shell showLineNumbers title="Model Format" +vertex_ai/agent_engine/{RESOURCE_NAME} +``` + +**Example:** +- `vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888` + +### LiteLLM Python SDK + +```python showLineNumbers title="Basic Agent Completion" +import litellm + +response = litellm.completion( + model="vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888", + messages=[ + {"role": "user", "content": "Explain machine learning in simple terms"} + ], +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Streaming Agent Responses" +import litellm + +response = await litellm.acompletion( + model="vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888", + messages=[ + {"role": "user", "content": "What are the key principles of software architecture?"} + ], + stream=True, +) + +async for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +### LiteLLM Proxy + +#### 1. Configure your model in config.yaml + + + + +```yaml showLineNumbers title="LiteLLM Proxy Configuration" +model_list: + - model_name: vertex-agent-1 + litellm_params: + model: vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888 + vertex_project: your-project-id + vertex_location: us-central1 +``` + + + + +#### 2. Start the LiteLLM Proxy + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config config.yaml +``` + +#### 3. Make requests to your Vertex AI Agent Engine + + + + +```bash showLineNumbers title="Basic Agent Request" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "vertex-agent-1", + "messages": [ + {"role": "user", "content": "Summarize the main benefits of cloud computing"} + ] + }' +``` + + + + + +```python showLineNumbers title="Using OpenAI SDK with LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +response = client.chat.completions.create( + model="vertex-agent-1", + messages=[ + {"role": "user", "content": "What are best practices for API design?"} + ] +) + +print(response.choices[0].message.content) +``` + + + + +## LiteLLM A2A Gateway + +You can also connect to Vertex AI Agent Engine through LiteLLM's A2A (Agent-to-Agent) Gateway UI. This provides a visual way to register and test agents without writing code. + +### 1. Navigate to Agents + +From the sidebar, click "Agents" to open the agent management page, then click "+ Add New Agent". + +![Click Agents](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9a979927-ce6b-4168-9fba-e53e28f1c2c4/ascreenshot.jpeg?tl_px=0,14&br_px=1376,783&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=17,277) + +![Add New Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a311750c-2e85-4589-99cb-2ce7e4021e77/ascreenshot.jpeg?tl_px=0,0&br_px=1376,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=195,257) + +### 2. Select Vertex AI Agent Engine Type + +Click "A2A Standard" to see available agent types, then select "Vertex AI Agent Engine". + +![Select A2A Standard](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/5b1acc4c-dc3f-4639-b4a0-e64b35c228fd/ascreenshot.jpeg?tl_px=52,0&br_px=1428,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,271) + +![Select Vertex AI Agent Engine](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/2f3bab61-3e02-4db7-84f0-82200a0f4136/ascreenshot.jpeg?tl_px=0,244&br_px=1376,1013&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=477,277) + +### 3. Configure the Agent + +Fill in the following fields: + +- **Agent Name** - A friendly name for your agent (e.g., `my-vertex-agent`) +- **Reasoning Engine Resource ID** - The full resource path from Google Cloud Console (e.g., `projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888`) +- **Vertex Project** - Your Google Cloud project ID +- **Vertex Location** - The region where your agent is deployed (e.g., `us-central1`) + +![Enter Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/695b84c7-9511-4337-bf19-f4505ab2b72b/ascreenshot.jpeg?tl_px=0,90&br_px=1376,859&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=480,276) + +![Enter Resource ID](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/ddce64df-b3a3-4519-ab62-f137887bcea2/ascreenshot.jpeg?tl_px=0,294&br_px=1376,1063&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=440,277) + +You can find the Resource ID in Google Cloud Console under Vertex AI > Agent Engine: + +![Copy Resource ID from Google Cloud Console](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/185d7f17-cbaa-45de-948d-49d2091805ea/ascreenshot.jpeg?tl_px=0,165&br_px=1376,934&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=493,276) + +![Enter Vertex Project](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a64da441-3e61-4811-a1e3-9f0b12c949ff/ascreenshot.jpeg?tl_px=0,233&br_px=1376,1002&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=501,277) + +You can find the Project ID in Google Cloud Console: + +![Copy Project ID from Google Cloud Console](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9ecad3bb-a534-42d6-9604-33906014fad6/user_cropped_screenshot.webp?tl_px=0,0&br_px=1728,1028&force_format=jpeg&q=100&width=1120.0) + +![Enter Vertex Location](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/316d1f38-4fb7-4377-86b6-c0fe7ac24383/ascreenshot.jpeg?tl_px=0,330&br_px=1376,1099&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=423,277) + +### 4. Create Agent + +Click "Create Agent" to save your configuration. + +![Create Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fb04b95d-793f-4eed-acf4-d1b3b5fa65e9/ascreenshot.jpeg?tl_px=352,347&br_px=1728,1117&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=623,498) + +### 5. Test in Playground + +Go to "Playground" in the sidebar to test your agent. + +![Go to Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9e01369b-6102-4fe3-96a7-90082cadfd6e/ascreenshot.jpeg?tl_px=0,0&br_px=1376,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=55,226) + +### 6. Select A2A Endpoint + +Click the endpoint dropdown and select `/v1/a2a/message/send`. + +![Select Endpoint](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/d5aeac35-531b-4cf0-af2d-88f0a71fd736/ascreenshot.jpeg?tl_px=0,146&br_px=1376,915&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=299,277) + +### 7. Select Your Agent and Send a Message + +Pick your Vertex AI Agent Engine from the dropdown and send a test message. + +![Select Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/353431f3-a0ba-4436-865d-ae11595e9cc4/ascreenshot.jpeg?tl_px=0,263&br_px=1376,1032&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=270,277) + +![Send Message](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fbfce72e-f50b-43e1-b6e5-0d41192d8e2d/ascreenshot.jpeg?tl_px=95,347&br_px=1471,1117&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,474) + +![Agent Response](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/892dd826-fbf9-4530-8d82-95270889274a/ascreenshot.jpeg?tl_px=0,82&br_px=1376,851&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=485,277) + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `GOOGLE_APPLICATION_CREDENTIALS` | Path to service account JSON key file | +| `VERTEXAI_PROJECT` | Google Cloud project ID | +| `VERTEXAI_LOCATION` | Google Cloud region (default: `us-central1`) | + +```bash +export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json" +export VERTEXAI_PROJECT="your-project-id" +export VERTEXAI_LOCATION="us-central1" +``` + +## Further Reading + +- [Vertex AI Agent Engine Documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/overview) +- [Create a Reasoning Engine](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/create) +- [A2A Agent Gateway](../a2a.md) +- [Vertex AI Provider](./vertex.md) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 77ab3158f74..ba4ca190aa9 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -655,7 +655,7 @@ docker run --name litellm-proxy \ -e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \ -e LITELLM_CONFIG_BUCKET_TYPE="gcs" \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-latest --detailed_debug + docker.litellm.ai/berriai/litellm-database:main-latest --detailed_debug ``` @@ -676,7 +676,7 @@ docker run --name litellm-proxy \ -e LITELLM_CONFIG_BUCKET_NAME= \ -e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-latest + docker.litellm.ai/berriai/litellm-database:main-latest ``` diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 0f0e5f678d3..9b4bc6822c1 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -10,10 +10,38 @@ You can find the Dockerfile to build litellm proxy [here](https://github.com/Ber ## Quick Start +:::info +Facing issues with pulling the docker image? Email us at support@berri.ai. +::: + To start using Litellm, run the following commands in a shell: + + + + +``` +docker pull docker.litellm.ai/berriai/litellm:main-latest +``` + +[**See all docker images**](https://github.com/orgs/BerriAI/packages) + + + + + +```shell +$ pip install 'litellm[proxy]' +``` + + + + + +Use this docker compose to spin up the proxy with a postgres database running locally. + ```bash -# Get the code +# Get the docker compose file curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/prometheus.yml @@ -30,6 +58,8 @@ echo 'LITELLM_SALT_KEY="sk-1234"' >> .env docker compose up ``` + + ### Docker Run @@ -57,7 +87,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-stable \ + docker.litellm.ai/berriai/litellm:main-stable \ --config /app/config.yaml --detailed_debug ``` @@ -87,12 +117,12 @@ See all supported CLI args [here](https://docs.litellm.ai/docs/proxy/cli): Here's how you can run the docker image and pass your config to `litellm` ```shell -docker run ghcr.io/berriai/litellm:main-stable --config your_config.yaml +docker run docker.litellm.ai/berriai/litellm:main-stable --config your_config.yaml ``` Here's how you can run the docker image and start litellm on port 8002 with `num_workers=8` ```shell -docker run ghcr.io/berriai/litellm:main-stable --port 8002 --num_workers 8 +docker run docker.litellm.ai/berriai/litellm:main-stable --port 8002 --num_workers 8 ``` @@ -100,7 +130,7 @@ docker run ghcr.io/berriai/litellm:main-stable --port 8002 --num_workers 8 ```shell # Use the provided base image -FROM ghcr.io/berriai/litellm:main-stable +FROM docker.litellm.ai/berriai/litellm:main-stable # Set the working directory to /app WORKDIR /app @@ -242,7 +272,7 @@ spec: spec: containers: - name: litellm - image: ghcr.io/berriai/litellm:main-stable # it is recommended to fix a version generally + image: docker.litellm.ai/berriai/litellm:main-stable # it is recommended to fix a version generally args: - "--config" - "/app/proxy_server_config.yaml" @@ -279,9 +309,9 @@ Use this when you want to use litellm helm chart as a dependency for other chart #### Step 1. Pull the litellm helm chart ```bash -helm pull oci://ghcr.io/berriai/litellm-helm +helm pull oci://docker.litellm.ai/berriai/litellm-helm -# Pulled: ghcr.io/berriai/litellm-helm:0.1.2 +# Pulled: docker.litellm.ai/berriai/litellm-helm:0.1.2 # Digest: sha256:7d3ded1c99c1597f9ad4dc49d84327cf1db6e0faa0eeea0c614be5526ae94e2a ``` @@ -340,7 +370,7 @@ Requirements: We maintain a [separate Dockerfile](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) for reducing build time when running LiteLLM proxy with a connected Postgres Database ```shell -docker pull ghcr.io/berriai/litellm-database:main-stable +docker pull docker.litellm.ai/berriai/litellm-database:main-stable ``` ```shell @@ -351,7 +381,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-stable \ + docker.litellm.ai/berriai/litellm-database:main-stable \ --config /app/config.yaml --detailed_debug ``` @@ -379,7 +409,7 @@ spec: spec: containers: - name: litellm-container - image: ghcr.io/berriai/litellm:main-stable + image: docker.litellm.ai/berriai/litellm:main-stable imagePullPolicy: Always env: - name: AZURE_API_KEY @@ -516,9 +546,9 @@ Use this when you want to use litellm helm chart as a dependency for other chart #### Step 1. Pull the litellm helm chart ```bash -helm pull oci://ghcr.io/berriai/litellm-helm +helm pull oci://docker.litellm.ai/berriai/litellm-helm -# Pulled: ghcr.io/berriai/litellm-helm:0.1.2 +# Pulled: docker.litellm.ai/berriai/litellm-helm:0.1.2 # Digest: sha256:7d3ded1c99c1597f9ad4dc49d84327cf1db6e0faa0eeea0c614be5526ae94e2a ``` @@ -575,7 +605,7 @@ router_settings: Start docker container with config ```shell -docker run ghcr.io/berriai/litellm:main-stable --config your_config.yaml +docker run docker.litellm.ai/berriai/litellm:main-stable --config your_config.yaml ``` ### Deploy with Database + Redis @@ -610,7 +640,7 @@ Start `litellm-database`docker container with config docker run --name litellm-proxy \ -e DATABASE_URL=postgresql://:@:/ \ -p 4000:4000 \ -ghcr.io/berriai/litellm-database:main-stable --config your_config.yaml +docker.litellm.ai/berriai/litellm-database:main-stable --config your_config.yaml ``` ### (Non Root) - without Internet Connection @@ -620,7 +650,7 @@ By default `prisma generate` downloads [prisma's engine binaries](https://www.pr Use this docker image to deploy litellm with pre-generated prisma binaries. ```bash -docker pull ghcr.io/berriai/litellm-non_root:main-stable +docker pull docker.litellm.ai/berriai/litellm-non_root:main-stable ``` [Published Docker Image link](https://github.com/BerriAI/litellm/pkgs/container/litellm-non_root) @@ -639,7 +669,7 @@ Use this, If you need to set ssl certificates for your on prem litellm proxy Pass `ssl_keyfile_path` (Path to the SSL keyfile) and `ssl_certfile_path` (Path to the SSL certfile) when starting litellm proxy ```shell -docker run ghcr.io/berriai/litellm:main-stable \ +docker run docker.litellm.ai/berriai/litellm:main-stable \ --ssl_keyfile_path ssl_test/keyfile.key \ --ssl_certfile_path ssl_test/certfile.crt ``` @@ -654,7 +684,7 @@ Step 1. Build your custom docker image with hypercorn ```shell # Use the provided base image -FROM ghcr.io/berriai/litellm:main-stable +FROM docker.litellm.ai/berriai/litellm:main-stable # Set the working directory to /app WORKDIR /app @@ -702,7 +732,7 @@ Usage Example: In this example, we set the keepalive timeout to 75 seconds. ```shell showLineNumbers title="docker run" -docker run ghcr.io/berriai/litellm:main-stable \ +docker run docker.litellm.ai/berriai/litellm:main-stable \ --keepalive_timeout 75 ``` @@ -711,7 +741,7 @@ In this example, we set the keepalive timeout to 75 seconds. ```shell showLineNumbers title="Environment Variable" export KEEPALIVE_TIMEOUT=75 -docker run ghcr.io/berriai/litellm:main-stable +docker run docker.litellm.ai/berriai/litellm:main-stable ``` @@ -722,7 +752,7 @@ Use this to mitigate memory growth by recycling workers after a fixed number of Usage Examples: ```shell showLineNumbers title="docker run (CLI flag)" -docker run ghcr.io/berriai/litellm:main-stable \ +docker run docker.litellm.ai/berriai/litellm:main-stable \ --max_requests_before_restart 10000 ``` @@ -730,7 +760,7 @@ Or set via environment variable: ```shell showLineNumbers title="Environment Variable" export MAX_REQUESTS_BEFORE_RESTART=10000 -docker run ghcr.io/berriai/litellm:main-stable +docker run docker.litellm.ai/berriai/litellm:main-stable ``` @@ -759,7 +789,7 @@ docker run --name litellm-proxy \ -e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \ -e LITELLM_CONFIG_BUCKET_TYPE="gcs" \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-stable --detailed_debug + docker.litellm.ai/berriai/litellm-database:main-stable --detailed_debug ``` @@ -780,7 +810,7 @@ docker run --name litellm-proxy \ -e LITELLM_CONFIG_BUCKET_NAME= \ -e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-stable + docker.litellm.ai/berriai/litellm-database:main-stable ``` @@ -907,7 +937,7 @@ Run the following command, replacing `` with the value you copied docker run --name litellm-proxy \ -e DATABASE_URL= \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-stable + docker.litellm.ai/berriai/litellm-database:main-stable ``` #### 4. Access the Application: @@ -986,7 +1016,7 @@ services: context: . args: target: runtime - image: ghcr.io/berriai/litellm:main-stable + image: docker.litellm.ai/berriai/litellm:main-stable ports: - "4000:4000" # Map the container port to the host, change the host port if necessary volumes: diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index 35d9923e92c..efdc73de43e 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -20,7 +20,7 @@ End-to-End tutorial for LiteLLM Proxy to: ``` -docker pull ghcr.io/berriai/litellm:main-latest +docker pull docker.litellm.ai/berriai/litellm:main-latest ``` [**See all docker images**](https://github.com/orgs/BerriAI/packages) @@ -119,7 +119,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml --detailed_debug # RUNNING on http://0.0.0.0:4000 @@ -302,7 +302,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml --detailed_debug ``` diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index 3c6d77cc7a2..26d25873207 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -29,7 +29,7 @@ Features: - **Spend Tracking & Data Exports** - ✅ [Set USD Budgets Spend for Custom Tags](./provider_budget_routing#-tag-budgets) - ✅ [Set Model budgets for Virtual Keys](./users#-virtual-key-model-specific) - - ✅ [Exporting LLM Logs to GCS Bucket, Azure Blob Storage](./proxy/bucket#🪣-logging-gcs-s3-buckets) + - ✅ [Exporting LLM Logs to GCS Bucket, Azure Blob Storage](../observability/gcs_bucket_integration) - ✅ [`/spend/report` API endpoint](cost_tracking.md#✨-enterprise-api-endpoints-to-get-spend) - **Control Guardrails per API Key/Team** - **Custom Branding** diff --git a/docs/my-website/docs/proxy/guardrails/pangea.md b/docs/my-website/docs/proxy/guardrails/pangea.md index 180b9100d6b..3de5ddfa530 100644 --- a/docs/my-website/docs/proxy/guardrails/pangea.md +++ b/docs/my-website/docs/proxy/guardrails/pangea.md @@ -67,7 +67,7 @@ docker run --rm \ -e PANGEA_AI_GUARD_TOKEN=$PANGEA_AI_GUARD_TOKEN \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -v $(pwd)/config.yaml:/app/config.yaml \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml ``` diff --git a/docs/my-website/docs/proxy/guardrails/pillar_security.md b/docs/my-website/docs/proxy/guardrails/pillar_security.md index de0b0d53614..099919dc393 100644 --- a/docs/my-website/docs/proxy/guardrails/pillar_security.md +++ b/docs/my-website/docs/proxy/guardrails/pillar_security.md @@ -72,13 +72,15 @@ litellm --config config.yaml --port 4000 ### Overview -Pillar Security supports three execution modes for comprehensive protection: +Pillar Security supports five execution modes for comprehensive protection: | Mode | When It Runs | What It Protects | Use Case |------|-------------|------------------|---------- | **`pre_call`** | Before LLM call | User input only | Block malicious prompts, prevent prompt injection | **`during_call`** | Parallel with LLM call | User input only | Input monitoring with lower latency | **`post_call`** | After LLM response | Full conversation context | Output filtering, PII detection in responses +| **`pre_mcp_call`** | Before MCP tool call | MCP tool inputs | Validate and sanitize MCP tool call arguments +| **`during_mcp_call`** | During MCP tool call | MCP tool inputs | Real-time monitoring of MCP tool calls ### Why Dual Mode is Recommended @@ -198,6 +200,85 @@ litellm_settings: set_verbose: true # Enable detailed logging ``` + + + +**Best for:** +- 🔒 **PII Protection**: Automatically sanitize sensitive data before sending to LLM +- ✅ **Continue Workflows**: Allow requests to proceed with masked content +- 🛡️ **Zero Trust**: Never expose sensitive data to LLM models +- 📊 **Compliance**: Meet data privacy requirements without blocking legitimate requests + +```yaml +model_list: + - model_name: gpt-4.1-mini + litellm_params: + model: openai/gpt-4.1-mini + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: "pillar-masking" + litellm_params: + guardrail: pillar + mode: "pre_call" # Scan input before LLM call + api_key: os.environ/PILLAR_API_KEY # Your Pillar API key + api_base: os.environ/PILLAR_API_BASE # Pillar API endpoint + on_flagged_action: "mask" # Mask sensitive content instead of blocking + persist_session: true # Keep records for investigation + include_scanners: true # Understand which scanners triggered + include_evidence: true # Capture evidence for analysis + default_on: true # Enable for all requests + +general_settings: + master_key: "YOUR_LITELLM_PROXY_MASTER_KEY" + +litellm_settings: + set_verbose: true +``` + +**How it works:** +1. User sends request with sensitive data: `"My email is john@example.com"` +2. Pillar detects PII and returns masked version: `"My email is [MASKED_EMAIL]"` +3. LiteLLM replaces original messages with masked messages +4. Request proceeds to LLM with sanitized content +5. User receives response without exposing sensitive data + + + + +**Best for:** +- 🤖 **Agent Workflows**: Protect MCP (Model Context Protocol) tool calls +- 🔒 **Tool Input Validation**: Scan arguments passed to MCP tools +- 🛡️ **Comprehensive Coverage**: Extend security to all LLM endpoints + +```yaml +model_list: + - model_name: gpt-4.1-mini + litellm_params: + model: openai/gpt-4.1-mini + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: "pillar-mcp-guard" + litellm_params: + guardrail: pillar + mode: "pre_mcp_call" # Scan MCP tool call inputs + api_key: os.environ/PILLAR_API_KEY # Your Pillar API key + api_base: os.environ/PILLAR_API_BASE # Pillar API endpoint + on_flagged_action: "block" # Block malicious MCP calls + default_on: true # Enable for all MCP calls + +general_settings: + master_key: "YOUR_LITELLM_PROXY_MASTER_KEY" + +litellm_settings: + set_verbose: true +``` + +**MCP Modes:** +- `pre_mcp_call`: Scan MCP tool call inputs before execution +- `during_mcp_call`: Monitor MCP tool calls in real-time + @@ -251,6 +332,15 @@ Logs the violation but allows the request to proceed: on_flagged_action: "monitor" ``` +#### Mask +Automatically sanitizes sensitive content (PII, secrets, etc.) in your messages before sending them to the LLM: + +```yaml +on_flagged_action: "mask" +``` + +When masking is enabled, sensitive information is automatically replaced with masked versions, allowing requests to proceed safely without exposing sensitive data to the LLM. + **Response Headers:** You can opt in to receiving detection details in response headers by configuring `include_scanners: true` and/or `include_evidence: true`. When enabled, these headers are included for **every request**—not just flagged ones—enabling comprehensive metrics, false positive analysis, and threat investigation. @@ -383,7 +473,8 @@ export PILLAR_TIMEOUT="5.0" **Quick takeaways** - Every request still runs *all* Pillar scanners; these options only change what comes back. - Choose richer responses when you need audit trails, lighter responses when latency or cost matters. -- Blocking is controlled by LiteLLM’s `on_flagged_action` configuration—Pillar headers do not change block/monitor behaviour. +- Actions (block/monitor/mask) are controlled by LiteLLM's `on_flagged_action` configuration—Pillar headers are automatically set based on your config. +- When blocking (`on_flagged_action: "block"`), the `include_scanners` and `include_evidence` settings control what details are included in the exception response. Pillar Security executes the full scanner suite on each call. The settings below tune the Protect response headers LiteLLM sends, letting you balance fidelity, retention, and latency. @@ -415,9 +506,10 @@ include_evidence: true # → plr_evidence (default true in LiteLLM) ``` Use when you only care about whether Pillar detected a threat. - > **📝 Note:** `flagged: true` means Pillar’s scanners recommend blocking. Pillar only reports this verdict—LiteLLM enforces your policy via the `on_flagged_action` configuration (no Pillar header controls it): - > - `on_flagged_action: "block"` → LiteLLM raises a 400 guardrail error + > **📝 Note:** `flagged: true` means Pillar's scanners recommend blocking. Pillar only reports this verdict—LiteLLM enforces your policy via the `on_flagged_action` configuration: + > - `on_flagged_action: "block"` → LiteLLM raises a 400 guardrail error (exception includes scanners/evidence based on `include_scanners`/`include_evidence` settings) > - `on_flagged_action: "monitor"` → LiteLLM logs the threat but still returns the LLM response + > - `on_flagged_action: "mask"` → LiteLLM replaces messages with masked versions and allows the request to proceed - **Scanner breakdown** (`include_scanners=true`) ```json diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 54c917bbbca..4cff7e5d041 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -29,6 +29,10 @@ LiteLLM automatically distributes requests across multiple deployments of the sa | **latency-based-routing** | Routes to fastest responding deployment | Latency-critical applications | | **cost-based-routing** | Routes to deployment with lowest cost | Cost-sensitive applications | +:::tip Deployment Priority +Use the `order` parameter to prioritize specific deployments. [See Deployment Ordering](#deployment-ordering-priority) for details. +::: + ## Quick Start - Load Balancing #### Step 1 - Set deployments on config @@ -243,6 +247,27 @@ class RouterModelGroupAliasItem(TypedDict): hidden: bool # if 'True', don't return on `/v1/models`, `/v1/model/info`, `/v1/model_group/info` ``` +## Deployment Ordering (Priority) + +Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them. + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-primary + api_key: os.environ/AZURE_API_KEY + order: 1 # 👈 Highest priority - always tried first + + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-fallback + api_key: os.environ/AZURE_API_KEY_2 + order: 2 # 👈 Used when order=1 is unavailable +``` + +If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments. + ### When You'll See Load Balancing in Action **Immediate Effects:** diff --git a/docs/my-website/docs/proxy/shared_health_check.md b/docs/my-website/docs/proxy/shared_health_check.md index d4b70116309..c9c975c7911 100644 --- a/docs/my-website/docs/proxy/shared_health_check.md +++ b/docs/my-website/docs/proxy/shared_health_check.md @@ -269,7 +269,7 @@ spec: spec: containers: - name: litellm-proxy - image: ghcr.io/berriai/litellm:latest + image: docker.litellm.ai/berriai/litellm:latest env: - name: USE_SHARED_HEALTH_CHECK value: "true" diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 971427806ed..2539f70d5bc 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -832,6 +832,59 @@ asyncio.run(router_acompletion()) ## Basic Reliability +### Deployment Ordering (Priority) + +Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them. + + + + +```python +from litellm import Router + +model_list = [ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "azure/gpt-4-primary", + "api_key": os.getenv("AZURE_API_KEY"), + "order": 1, # 👈 Highest priority + }, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "azure/gpt-4-fallback", + "api_key": os.getenv("AZURE_API_KEY_2"), + "order": 2, # 👈 Used when order=1 is unavailable + }, + }, +] + +router = Router(model_list=model_list) +``` + + + + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-primary + api_key: os.environ/AZURE_API_KEY + order: 1 # 👈 Highest priority + + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-fallback + api_key: os.environ/AZURE_API_KEY_2 + order: 2 # 👈 Used when order=1 is unavailable +``` + + + + ### Weighted Deployments Set `weight` on a deployment to pick one deployment more often than others. diff --git a/docs/my-website/docs/secret_managers/custom_secret_manager.md b/docs/my-website/docs/secret_managers/custom_secret_manager.md index c51eeeb0727..a6a91a0336d 100644 --- a/docs/my-website/docs/secret_managers/custom_secret_manager.md +++ b/docs/my-website/docs/secret_managers/custom_secret_manager.md @@ -76,7 +76,7 @@ docker run -d \ --name litellm-proxy \ -v $(pwd)/config.yaml:/app/config.yaml \ -v $(pwd)/my_secret_manager.py:/app/my_secret_manager.py \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml \ --port 4000 \ --detailed_debug diff --git a/docs/my-website/docs/secret_managers/hashicorp_vault.md b/docs/my-website/docs/secret_managers/hashicorp_vault.md index 9e536270988..09619609cb7 100644 --- a/docs/my-website/docs/secret_managers/hashicorp_vault.md +++ b/docs/my-website/docs/secret_managers/hashicorp_vault.md @@ -47,6 +47,8 @@ HCP_VAULT_TOKEN="hvs.CAESIG52gL6ljBSdmq*****" # OPTIONAL HCP_VAULT_REFRESH_INTERVAL="86400" # defaults to 86400, frequency of cache refresh for Hashicorp Vault +HCP_VAULT_MOUNT_NAME="secret" # OPTIONAL. defaults to "secret", set this if your KV engine is mounted elsewhere +HCP_VAULT_PATH_PREFIX="litellm" # OPTIONAL. defaults to None, set this if your secrets live under a custom prefix like secret/data/litellm/OPENAI_API_KEY ``` **Step 2.** Add to proxy config.yaml @@ -151,18 +153,20 @@ export HCP_VAULT_TOKEN="hvs.CAESIG52gL6ljBSdmq*****" LiteLLM reads secrets from Hashicorp Vault's KV v2 engine using the following URL format: ``` -{VAULT_ADDR}/v1/{NAMESPACE}/secret/data/{SECRET_NAME} +{VAULT_ADDR}/v1/{NAMESPACE}/{MOUNT_NAME}/data/{PATH_PREFIX}/{SECRET_NAME} ``` For example, if you have: - `HCP_VAULT_ADDR="https://vault.example.com:8200"` - `HCP_VAULT_NAMESPACE="admin"` +- `HCP_VAULT_MOUNT_NAME="secret"` +- `HCP_VAULT_PATH_PREFIX="litellm"` - Secret name: `AZURE_API_KEY` LiteLLM will look up: ``` -https://vault.example.com:8200/v1/admin/secret/data/AZURE_API_KEY +https://vault.example.com:8200/v1/admin/secret/data/litellm/AZURE_API_KEY ``` ### Expected Secret Format @@ -193,4 +197,3 @@ When a Virtual Key is Created / Deleted on LiteLLM, LiteLLM will automatically c LiteLLM stores secret under the `prefix_for_stored_virtual_keys` path (default: `litellm/`) - diff --git a/docs/my-website/docs/tutorials/elasticsearch_logging.md b/docs/my-website/docs/tutorials/elasticsearch_logging.md index eabd47f095d..85a9f1452d7 100644 --- a/docs/my-website/docs/tutorials/elasticsearch_logging.md +++ b/docs/my-website/docs/tutorials/elasticsearch_logging.md @@ -221,7 +221,7 @@ services: - elasticsearch litellm: - image: ghcr.io/berriai/litellm:main-latest + image: docker.litellm.ai/berriai/litellm:main-latest ports: - "4000:4000" environment: diff --git a/docs/my-website/docs/tutorials/openai_codex.md b/docs/my-website/docs/tutorials/openai_codex.md index 41416f85159..563d6559ca5 100644 --- a/docs/my-website/docs/tutorials/openai_codex.md +++ b/docs/my-website/docs/tutorials/openai_codex.md @@ -53,7 +53,7 @@ yarn global add @openai/codex docker run \ -v $(pwd)/litellm_config.yaml:/app/config.yaml \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml ``` diff --git a/docs/my-website/release_notes/v1.55.8-stable/index.md b/docs/my-website/release_notes/v1.55.8-stable/index.md index 38c78eb5372..bf239e0889d 100644 --- a/docs/my-website/release_notes/v1.55.8-stable/index.md +++ b/docs/my-website/release_notes/v1.55.8-stable/index.md @@ -53,7 +53,7 @@ Send LLM usage (spend, tokens) data to [Azure Data Lake](https://learn.microsoft docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.55.8-stable +docker.litellm.ai/berriai/litellm:litellm_stable_release_branch-v1.55.8-stable ``` ## Get Daily Updates diff --git a/docs/my-website/release_notes/v1.57.3/index.md b/docs/my-website/release_notes/v1.57.3/index.md index ab1154a0a8c..bbffa990b32 100644 --- a/docs/my-website/release_notes/v1.57.3/index.md +++ b/docs/my-website/release_notes/v1.57.3/index.md @@ -39,7 +39,7 @@ Instead of `apt-get` use `apk`, the base litellm image will no longer have `apt- **You are only impacted if you use `apt-get` in your Dockerfile** ```shell # Use the provided base image -FROM ghcr.io/berriai/litellm:main-latest +FROM docker.litellm.ai/berriai/litellm:main-latest # Set the working directory WORKDIR /app diff --git a/docs/my-website/release_notes/v1.63.11-stable/index.md b/docs/my-website/release_notes/v1.63.11-stable/index.md index 882747a07b3..3273f9a8e06 100644 --- a/docs/my-website/release_notes/v1.63.11-stable/index.md +++ b/docs/my-website/release_notes/v1.63.11-stable/index.md @@ -36,7 +36,7 @@ This release is primarily focused on: docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.63.11-stable +docker.litellm.ai/berriai/litellm:main-v1.63.11-stable ``` ## Demo Instance diff --git a/docs/my-website/release_notes/v1.63.14/index.md b/docs/my-website/release_notes/v1.63.14/index.md index ff2630468c5..1ac713fc2d5 100644 --- a/docs/my-website/release_notes/v1.63.14/index.md +++ b/docs/my-website/release_notes/v1.63.14/index.md @@ -32,7 +32,7 @@ This release brings: docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.63.14-stable.patch1 +docker.litellm.ai/berriai/litellm:main-v1.63.14-stable.patch1 ``` ## Demo Instance diff --git a/docs/my-website/release_notes/v1.65.4-stable/index.md b/docs/my-website/release_notes/v1.65.4-stable/index.md index 872024a47ab..80d703e1116 100644 --- a/docs/my-website/release_notes/v1.65.4-stable/index.md +++ b/docs/my-website/release_notes/v1.65.4-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.65.4-stable +docker.litellm.ai/berriai/litellm:main-v1.65.4-stable ``` diff --git a/docs/my-website/release_notes/v1.66.0-stable/index.md b/docs/my-website/release_notes/v1.66.0-stable/index.md index 939322e0317..693cd7fc5ac 100644 --- a/docs/my-website/release_notes/v1.66.0-stable/index.md +++ b/docs/my-website/release_notes/v1.66.0-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.66.0-stable +docker.litellm.ai/berriai/litellm:main-v1.66.0-stable ``` diff --git a/docs/my-website/release_notes/v1.67.4-stable/index.md b/docs/my-website/release_notes/v1.67.4-stable/index.md index 93a27155d2b..f61c99f7d02 100644 --- a/docs/my-website/release_notes/v1.67.4-stable/index.md +++ b/docs/my-website/release_notes/v1.67.4-stable/index.md @@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.67.4-stable +docker.litellm.ai/berriai/litellm:main-v1.67.4-stable ``` diff --git a/docs/my-website/release_notes/v1.68.0-stable/index.md b/docs/my-website/release_notes/v1.68.0-stable/index.md index 4d456d9c853..f3e7fa27427 100644 --- a/docs/my-website/release_notes/v1.68.0-stable/index.md +++ b/docs/my-website/release_notes/v1.68.0-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.68.0-stable +docker.litellm.ai/berriai/litellm:main-v1.68.0-stable ``` diff --git a/docs/my-website/release_notes/v1.69.0-stable/index.md b/docs/my-website/release_notes/v1.69.0-stable/index.md index 3f8ce7a29c4..f3f094e5403 100644 --- a/docs/my-website/release_notes/v1.69.0-stable/index.md +++ b/docs/my-website/release_notes/v1.69.0-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.69.0-stable +docker.litellm.ai/berriai/litellm:main-v1.69.0-stable ``` diff --git a/docs/my-website/release_notes/v1.70.1-stable/index.md b/docs/my-website/release_notes/v1.70.1-stable/index.md index c55ac8b9c61..5d4bde0f6a0 100644 --- a/docs/my-website/release_notes/v1.70.1-stable/index.md +++ b/docs/my-website/release_notes/v1.70.1-stable/index.md @@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.70.1-stable +docker.litellm.ai/berriai/litellm:main-v1.70.1-stable ``` diff --git a/docs/my-website/release_notes/v1.71.1-stable/index.md b/docs/my-website/release_notes/v1.71.1-stable/index.md index 2d21d49171b..bd37183455d 100644 --- a/docs/my-website/release_notes/v1.71.1-stable/index.md +++ b/docs/my-website/release_notes/v1.71.1-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.71.1-stable +docker.litellm.ai/berriai/litellm:main-v1.71.1-stable ``` diff --git a/docs/my-website/release_notes/v1.72.0-stable/index.md b/docs/my-website/release_notes/v1.72.0-stable/index.md index 47bc19e8aa8..fe235cf07b1 100644 --- a/docs/my-website/release_notes/v1.72.0-stable/index.md +++ b/docs/my-website/release_notes/v1.72.0-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.72.0-stable +docker.litellm.ai/berriai/litellm:main-v1.72.0-stable ``` diff --git a/docs/my-website/release_notes/v1.72.2-stable/index.md b/docs/my-website/release_notes/v1.72.2-stable/index.md index 023180f9758..36d01c131c7 100644 --- a/docs/my-website/release_notes/v1.72.2-stable/index.md +++ b/docs/my-website/release_notes/v1.72.2-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.72.2-stable +docker.litellm.ai/berriai/litellm:main-v1.72.2-stable ``` diff --git a/docs/my-website/release_notes/v1.72.6-stable/index.md b/docs/my-website/release_notes/v1.72.6-stable/index.md index 5603548364f..a20488e2318 100644 --- a/docs/my-website/release_notes/v1.72.6-stable/index.md +++ b/docs/my-website/release_notes/v1.72.6-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.72.6-stable +docker.litellm.ai/berriai/litellm:main-v1.72.6-stable ``` diff --git a/docs/my-website/release_notes/v1.73.0-stable/index.md b/docs/my-website/release_notes/v1.73.0-stable/index.md index 307fecc36dd..802c5ac028b 100644 --- a/docs/my-website/release_notes/v1.73.0-stable/index.md +++ b/docs/my-website/release_notes/v1.73.0-stable/index.md @@ -37,7 +37,7 @@ The `non-root` docker image has a known issue around the UI not loading. If you docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.73.0-stable +docker.litellm.ai/berriai/litellm:v1.73.0-stable ``` diff --git a/docs/my-website/release_notes/v1.73.6-stable/index.md b/docs/my-website/release_notes/v1.73.6-stable/index.md index b03380f9b2b..da748c5c99f 100644 --- a/docs/my-website/release_notes/v1.73.6-stable/index.md +++ b/docs/my-website/release_notes/v1.73.6-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.73.6-stable.patch.1 +docker.litellm.ai/berriai/litellm:v1.73.6-stable.patch.1 ``` diff --git a/docs/my-website/release_notes/v1.74.0-stable/index.md b/docs/my-website/release_notes/v1.74.0-stable/index.md index e49c2b4f620..ee39c0a26a8 100644 --- a/docs/my-website/release_notes/v1.74.0-stable/index.md +++ b/docs/my-website/release_notes/v1.74.0-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.0-stable +docker.litellm.ai/berriai/litellm:v1.74.0-stable ``` diff --git a/docs/my-website/release_notes/v1.74.15-stable/index.md b/docs/my-website/release_notes/v1.74.15-stable/index.md index 9807a00b7e7..c0facf8afb0 100644 --- a/docs/my-website/release_notes/v1.74.15-stable/index.md +++ b/docs/my-website/release_notes/v1.74.15-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.15-stable +docker.litellm.ai/berriai/litellm:v1.74.15-stable ``` diff --git a/docs/my-website/release_notes/v1.74.3-stable/index.md b/docs/my-website/release_notes/v1.74.3-stable/index.md index 167d81e52af..05386172e71 100644 --- a/docs/my-website/release_notes/v1.74.3-stable/index.md +++ b/docs/my-website/release_notes/v1.74.3-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.3-stable +docker.litellm.ai/berriai/litellm:v1.74.3-stable ``` diff --git a/docs/my-website/release_notes/v1.74.7/index.md b/docs/my-website/release_notes/v1.74.7/index.md index 7d7a568e13f..10fbd21b498 100644 --- a/docs/my-website/release_notes/v1.74.7/index.md +++ b/docs/my-website/release_notes/v1.74.7/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.7-stable.patch.1 +docker.litellm.ai/berriai/litellm:v1.74.7-stable.patch.1 ``` diff --git a/docs/my-website/release_notes/v1.74.9-stable/index.md b/docs/my-website/release_notes/v1.74.9-stable/index.md index 3f100745dfe..9feed6d62e6 100644 --- a/docs/my-website/release_notes/v1.74.9-stable/index.md +++ b/docs/my-website/release_notes/v1.74.9-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.9-stable.patch.1 +docker.litellm.ai/berriai/litellm:v1.74.9-stable.patch.1 ``` diff --git a/docs/my-website/release_notes/v1.75.5-stable/index.md b/docs/my-website/release_notes/v1.75.5-stable/index.md index 7035d285057..043f1267fc8 100644 --- a/docs/my-website/release_notes/v1.75.5-stable/index.md +++ b/docs/my-website/release_notes/v1.75.5-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.75.5-stable +docker.litellm.ai/berriai/litellm:v1.75.5-stable ``` diff --git a/docs/my-website/release_notes/v1.75.8/index.md b/docs/my-website/release_notes/v1.75.8/index.md index d7d4f37c4ee..3db1fe4b2cd 100644 --- a/docs/my-website/release_notes/v1.75.8/index.md +++ b/docs/my-website/release_notes/v1.75.8/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.75.8-stable +docker.litellm.ai/berriai/litellm:v1.75.8-stable ``` diff --git a/docs/my-website/release_notes/v1.76.1-stable/index.md b/docs/my-website/release_notes/v1.76.1-stable/index.md index 4437b7f5799..f458dfde6d4 100644 --- a/docs/my-website/release_notes/v1.76.1-stable/index.md +++ b/docs/my-website/release_notes/v1.76.1-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.76.1 +docker.litellm.ai/berriai/litellm:v1.76.1 ``` diff --git a/docs/my-website/release_notes/v1.76.3-stable/index.md b/docs/my-website/release_notes/v1.76.3-stable/index.md index 6b40e4f5b35..9763a57975b 100644 --- a/docs/my-website/release_notes/v1.76.3-stable/index.md +++ b/docs/my-website/release_notes/v1.76.3-stable/index.md @@ -35,7 +35,7 @@ This release has a known issue where startup is leading to Out of Memory errors docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.76.3 +docker.litellm.ai/berriai/litellm:v1.76.3 ``` diff --git a/docs/my-website/release_notes/v1.77.2-stable/index.md b/docs/my-website/release_notes/v1.77.2-stable/index.md index fdd80693d05..4f732a1604d 100644 --- a/docs/my-website/release_notes/v1.77.2-stable/index.md +++ b/docs/my-website/release_notes/v1.77.2-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:main-v1.77.2-stable +docker.litellm.ai/berriai/litellm:main-v1.77.2-stable ``` diff --git a/docs/my-website/release_notes/v1.77.3-stable/index.md b/docs/my-website/release_notes/v1.77.3-stable/index.md index c7c17e5baee..11b82c4c834 100644 --- a/docs/my-website/release_notes/v1.77.3-stable/index.md +++ b/docs/my-website/release_notes/v1.77.3-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.77.3-stable +docker.litellm.ai/berriai/litellm:v1.77.3-stable ``` diff --git a/docs/my-website/release_notes/v1.77.5-stable/index.md b/docs/my-website/release_notes/v1.77.5-stable/index.md index 6843800ee6d..8e59ea92cc2 100644 --- a/docs/my-website/release_notes/v1.77.5-stable/index.md +++ b/docs/my-website/release_notes/v1.77.5-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.77.5-stable +docker.litellm.ai/berriai/litellm:v1.77.5-stable ``` diff --git a/docs/my-website/release_notes/v1.77.7-stable/index.md b/docs/my-website/release_notes/v1.77.7-stable/index.md index 62d9a2eee4f..b4df447f334 100644 --- a/docs/my-website/release_notes/v1.77.7-stable/index.md +++ b/docs/my-website/release_notes/v1.77.7-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.77.7.rc.1 +docker.litellm.ai/berriai/litellm:v1.77.7.rc.1 ``` diff --git a/docs/my-website/release_notes/v1.78.0-stable/index.md b/docs/my-website/release_notes/v1.78.0-stable/index.md index 7f6c5ba1e08..8322f0479c5 100644 --- a/docs/my-website/release_notes/v1.78.0-stable/index.md +++ b/docs/my-website/release_notes/v1.78.0-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.78.0-stable +docker.litellm.ai/berriai/litellm:v1.78.0-stable ``` diff --git a/docs/my-website/release_notes/v1.78.5-stable/index.md b/docs/my-website/release_notes/v1.78.5-stable/index.md index af1fd359fa2..2bcdfab472c 100644 --- a/docs/my-website/release_notes/v1.78.5-stable/index.md +++ b/docs/my-website/release_notes/v1.78.5-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.78.5-stable +docker.litellm.ai/berriai/litellm:v1.78.5-stable ``` diff --git a/docs/my-website/release_notes/v1.79.0-stable/index.md b/docs/my-website/release_notes/v1.79.0-stable/index.md index 8327f4b6178..4bb7094a3fc 100644 --- a/docs/my-website/release_notes/v1.79.0-stable/index.md +++ b/docs/my-website/release_notes/v1.79.0-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.79.0-stable +docker.litellm.ai/berriai/litellm:v1.79.0-stable ``` diff --git a/docs/my-website/release_notes/v1.79.1-stable/index.md b/docs/my-website/release_notes/v1.79.1-stable/index.md index ea8cfeae740..19fc7f9f3ff 100644 --- a/docs/my-website/release_notes/v1.79.1-stable/index.md +++ b/docs/my-website/release_notes/v1.79.1-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.79.1-stable +docker.litellm.ai/berriai/litellm:v1.79.1-stable ``` diff --git a/docs/my-website/release_notes/v1.79.3-stable/index.md b/docs/my-website/release_notes/v1.79.3-stable/index.md index c4f3ba1e017..542f88787e0 100644 --- a/docs/my-website/release_notes/v1.79.3-stable/index.md +++ b/docs/my-website/release_notes/v1.79.3-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.79.3-stable +docker.litellm.ai/berriai/litellm:v1.79.3-stable ``` diff --git a/docs/my-website/release_notes/v1.80.0-stable/index.md b/docs/my-website/release_notes/v1.80.0-stable/index.md index 17fcf6646ed..d0cf28a5c58 100644 --- a/docs/my-website/release_notes/v1.80.0-stable/index.md +++ b/docs/my-website/release_notes/v1.80.0-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.80.0-stable +docker.litellm.ai/berriai/litellm:v1.80.0-stable ``` diff --git a/docs/my-website/release_notes/v1.80.10-stable/index.md b/docs/my-website/release_notes/v1.80.10-stable/index.md index 1b0a9866fae..2290c06de53 100644 --- a/docs/my-website/release_notes/v1.80.10-stable/index.md +++ b/docs/my-website/release_notes/v1.80.10-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.80.10.rc.1 +docker.litellm.ai/berriai/litellm:v1.80.10.rc.1 ``` diff --git a/docs/my-website/release_notes/v1.80.5-stable/index.md b/docs/my-website/release_notes/v1.80.5-stable/index.md index 598fa47f223..9c769f8996f 100644 --- a/docs/my-website/release_notes/v1.80.5-stable/index.md +++ b/docs/my-website/release_notes/v1.80.5-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.80.5-stable +docker.litellm.ai/berriai/litellm:v1.80.5-stable ``` diff --git a/docs/my-website/release_notes/v1.80.8-stable/index.md b/docs/my-website/release_notes/v1.80.8-stable/index.md index 29075a9594f..106c594968f 100644 --- a/docs/my-website/release_notes/v1.80.8-stable/index.md +++ b/docs/my-website/release_notes/v1.80.8-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.80.8-stable +docker.litellm.ai/berriai/litellm:v1.80.8-stable ``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 954a27d3182..ead5d78e606 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -472,6 +472,7 @@ const sidebars = { "generateContent", "apply_guardrail", "bedrock_invoke", + "interactions", { type: "category", label: "/images", @@ -632,6 +633,7 @@ const sidebars = { "providers/vertex_speech", "providers/vertex_batch", "providers/vertex_ocr", + "providers/vertex_ai_agent_engine", ] }, { @@ -738,6 +740,7 @@ const sidebars = { "providers/petals", "providers/publicai", "providers/predibase", + "providers/pydantic_ai_agent", "providers/ragflow", "providers/recraft", "providers/replicate", diff --git a/docs/my-website/src/pages/index.md b/docs/my-website/src/pages/index.md index 1dc2995c5fe..91215b33c5d 100644 --- a/docs/my-website/src/pages/index.md +++ b/docs/my-website/src/pages/index.md @@ -604,7 +604,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml --detailed_debug ``` diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 6620db5ffa2..e12be6baf5d 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -750,9 +750,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_id=model_id, model_name=model_name, ) - await self.store_unified_file_id( # need to store otherwise any retrieve call will fail + + # Fetch the actual file object for the output file + file_object = None + try: + # Use litellm to retrieve the file object from the provider + from litellm import afile_retrieve + file_object = await afile_retrieve( + custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai", + file_id=original_output_file_id + ) + verbose_logger.debug( + f"Successfully retrieved file object for output_file_id={original_output_file_id}" + ) + except Exception as e: + verbose_logger.warning( + f"Failed to retrieve file object for output_file_id={original_output_file_id}: {str(e)}. Storing with None and will fetch on-demand." + ) + + await self.store_unified_file_id( file_id=response.output_file_id, - file_object=None, + file_object=file_object, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, model_mappings={model_id: original_output_file_id}, user_api_key_dict=user_api_key_dict, diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 96e1a5106ac..7ffbe95be13 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -18,6 +18,45 @@ def str_to_bool(value: Optional[str]) -> bool: return value.lower() in ("true", "1", "t", "y", "yes") + +def _get_prisma_env() -> dict: + """Get environment variables for Prisma, handling offline mode if configured.""" + prisma_env = os.environ.copy() + if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")): + # These env vars prevent Prisma from attempting downloads + prisma_env["NPM_CONFIG_PREFER_OFFLINE"] = "true" + prisma_env["NPM_CONFIG_CACHE"] = os.getenv("NPM_CONFIG_CACHE", "/app/.cache/npm") + return prisma_env + + +def _get_prisma_command() -> str: + """Get the Prisma command to use, bypassing Python wrapper in offline mode.""" + if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")): + # Primary location where Prisma Python package installs the CLI + default_cli_path = "/app/.cache/prisma-python/binaries/node_modules/.bin/prisma" + + # Check if custom path is provided (for flexibility) + custom_cli_path = os.getenv("PRISMA_CLI_PATH") + if custom_cli_path and os.path.exists(custom_cli_path): + logger.info(f"Using custom Prisma CLI at {custom_cli_path}") + return custom_cli_path + + # Check the default location + if os.path.exists(default_cli_path): + logger.info(f"Using cached Prisma CLI at {default_cli_path}") + return default_cli_path + + # If not found, log warning and fall back + logger.warning( + f"Prisma CLI not found at {default_cli_path}. " + "Falling back to Python wrapper (may attempt downloads)" + ) + + # Fall back to the Python wrapper (will work in online mode) + return "prisma" + + + class ProxyExtrasDBManager: @staticmethod def _get_prisma_dir() -> str: @@ -57,6 +96,11 @@ class ProxyExtrasDBManager: init_dir.mkdir(parents=True, exist_ok=True) database_url = os.getenv("DATABASE_URL") + if not database_url: + logger.error("DATABASE_URL not set") + return False + # Set up environment for offline mode if configured + prisma_env = _get_prisma_env() try: # 1. Generate migration SQL file by comparing empty state to current db state @@ -64,7 +108,7 @@ class ProxyExtrasDBManager: migration_file = init_dir / "migration.sql" subprocess.run( [ - "prisma", + _get_prisma_command(), "migrate", "diff", "--from-empty", @@ -75,13 +119,14 @@ class ProxyExtrasDBManager: stdout=open(migration_file, "w"), check=True, timeout=30, + env=prisma_env ) # 3. Mark the migration as applied since it represents current state logger.info("Marking baseline migration as applied...") subprocess.run( [ - "prisma", + _get_prisma_command(), "migrate", "resolve", "--applied", @@ -89,6 +134,7 @@ class ProxyExtrasDBManager: ], check=True, timeout=30, + env=prisma_env ) return True @@ -113,21 +159,26 @@ class ProxyExtrasDBManager: @staticmethod def _roll_back_migration(migration_name: str): """Mark a specific migration as rolled back""" + # Set up environment for offline mode if configured + prisma_env = _get_prisma_env() subprocess.run( - ["prisma", "migrate", "resolve", "--rolled-back", migration_name], + [_get_prisma_command(), "migrate", "resolve", "--rolled-back", migration_name], timeout=60, check=True, capture_output=True, + env=prisma_env ) @staticmethod def _resolve_specific_migration(migration_name: str): """Mark a specific migration as applied""" + prisma_env = _get_prisma_env() subprocess.run( - ["prisma", "migrate", "resolve", "--applied", migration_name], + [_get_prisma_command(), "migrate", "resolve", "--applied", migration_name], timeout=60, check=True, capture_output=True, + env=prisma_env ) @staticmethod @@ -194,6 +245,10 @@ class ProxyExtrasDBManager: 3. Mark all existing migrations as applied. """ database_url = os.getenv("DATABASE_URL") + if not database_url: + logger.error("DATABASE_URL not set") + return + diff_dir = ( Path(migrations_dir) / "migrations" @@ -216,7 +271,7 @@ class ProxyExtrasDBManager: with open(diff_sql_path, "w") as f: subprocess.run( [ - "prisma", + _get_prisma_command(), "migrate", "diff", "--from-url", @@ -228,6 +283,7 @@ class ProxyExtrasDBManager: check=True, timeout=60, stdout=f, + env=_get_prisma_env() ) except subprocess.CalledProcessError as e: logger.warning(f"Failed to generate migration diff: {e.stderr}") @@ -245,7 +301,7 @@ class ProxyExtrasDBManager: logger.info("Running prisma db execute to apply the migration diff...") result = subprocess.run( [ - "prisma", + _get_prisma_command(), "db", "execute", "--file", @@ -257,6 +313,7 @@ class ProxyExtrasDBManager: check=True, capture_output=True, text=True, + env=_get_prisma_env() ) logger.info(f"prisma db execute stdout: {result.stdout}") logger.info("✅ Migration diff applied successfully") @@ -274,11 +331,12 @@ class ProxyExtrasDBManager: try: logger.info(f"Resolving migration: {migration_name}") subprocess.run( - ["prisma", "migrate", "resolve", "--applied", migration_name], + [_get_prisma_command(), "migrate", "resolve", "--applied", migration_name], timeout=60, check=True, capture_output=True, text=True, + env=_get_prisma_env() ) logger.debug(f"Resolved migration: {migration_name}") except subprocess.CalledProcessError as e: @@ -312,11 +370,12 @@ class ProxyExtrasDBManager: try: # Set migrations directory for Prisma result = subprocess.run( - ["prisma", "migrate", "deploy"], + [_get_prisma_command(), "migrate", "deploy"], timeout=60, check=True, capture_output=True, text=True, + env=_get_prisma_env() ) logger.info(f"prisma migrate deploy stdout: {result.stdout}") @@ -344,7 +403,7 @@ class ProxyExtrasDBManager: # Mark the failed migration as rolled back subprocess.run( [ - "prisma", + _get_prisma_command(), "migrate", "resolve", "--rolled-back", @@ -354,6 +413,7 @@ class ProxyExtrasDBManager: check=True, capture_output=True, text=True, + env=_get_prisma_env() ) logger.info( f"✅ Migration {failed_migration} marked as rolled back... retrying" @@ -450,7 +510,7 @@ class ProxyExtrasDBManager: else: # Use prisma db push with increased timeout subprocess.run( - ["prisma", "db", "push", "--accept-data-loss"], + [_get_prisma_command(), "db", "push", "--accept-data-loss"], timeout=60, check=True, ) diff --git a/litellm/__init__.py b/litellm/__init__.py index ef44aa53a13..b71240777f9 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1,4 +1,6 @@ ### Hide pydantic namespace conflict warnings globally ### +from __future__ import annotations + import warnings warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*") @@ -26,18 +28,6 @@ from typing import ( ) from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams from litellm.types.integrations.datadog import DatadogInitParams -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.caching.caching import Cache, DualCache, RedisCache, InMemoryCache -from litellm.caching.llm_caching_handler import LLMClientCache -from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES -from litellm.types.utils import ( - ImageObject, - BudgetConfig, - all_litellm_params, - all_litellm_params as _litellm_completion_params, - CredentialItem, - PriorityReservationDict, -) # maintain backwards compatibility for root param. from litellm._logging import ( set_verbose, _turn_on_debug, @@ -84,12 +74,6 @@ from litellm.constants import ( DEFAULT_SOFT_BUDGET, DEFAULT_ALLOWED_FAILS, ) -from litellm.integrations.dotprompt import ( - global_prompt_manager, - global_prompt_directory, - set_global_prompt_directory, -) -from litellm.types.guardrails import GuardrailItem from litellm.types.secret_managers.main import ( KeyManagementSystem, KeyManagementSettings, @@ -98,11 +82,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( DefaultTeamSSOParams, LiteLLM_UpperboundKeyGenerateParams, ) -from litellm.types.utils import ( - StandardKeyGenerationConfig, - LlmProviders, - SearchProviders, -) +from litellm.types.utils import LlmProviders from litellm.types.utils import PriorityReservationSettings from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager @@ -287,7 +267,7 @@ disable_token_counter: bool = False disable_add_transform_inline_image_block: bool = False disable_add_user_agent_to_request_tags: bool = False extra_spend_tag_headers: Optional[List[str]] = None -in_memory_llm_clients_cache: LLMClientCache = LLMClientCache() +in_memory_llm_clients_cache: "LLMClientCache" safe_memory_mode: bool = False enable_azure_ad_token_refresh: Optional[bool] = False ### DEFAULT AZURE API VERSION ### @@ -295,9 +275,9 @@ AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the lates ### DEFAULT WATSONX API VERSION ### WATSONX_DEFAULT_API_VERSION = "2024-03-13" ### COHERE EMBEDDINGS DEFAULT TYPE ### -COHERE_DEFAULT_EMBEDDING_INPUT_TYPE: COHERE_EMBEDDING_INPUT_TYPES = "search_document" +COHERE_DEFAULT_EMBEDDING_INPUT_TYPE: "COHERE_EMBEDDING_INPUT_TYPES" = "search_document" ### CREDENTIALS ### -credential_list: List[CredentialItem] = [] +credential_list: List["CredentialItem"] = [] ### GUARDRAILS ### llamaguard_model_name: Optional[str] = None openai_moderations_model_name: Optional[str] = None @@ -333,7 +313,7 @@ caching: bool = ( caching_with_models: bool = ( False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 ) -cache: Optional[Cache] = ( +cache: Optional["Cache"] = ( None # cache object <- use this - https://docs.litellm.ai/docs/caching ) default_in_memory_ttl: Optional[float] = None @@ -372,7 +352,7 @@ aws_sqs_callback_params: Optional[Dict] = None generic_logger_headers: Optional[Dict] = None default_key_generate_params: Optional[Dict] = None upperbound_key_generate_params: Optional[LiteLLM_UpperboundKeyGenerateParams] = None -key_generation_settings: Optional[StandardKeyGenerationConfig] = None +key_generation_settings: Optional["StandardKeyGenerationConfig"] = None default_internal_user_params: Optional[Dict] = None default_team_params: Optional[Union[DefaultTeamSSOParams, Dict]] = None default_team_settings: Optional[List] = None @@ -381,7 +361,7 @@ default_max_internal_user_budget: Optional[float] = None max_internal_user_budget: Optional[float] = None max_ui_session_budget: Optional[float] = 10 # $10 USD budgets for UI Chat sessions internal_user_budget_duration: Optional[str] = None -tag_budget_config: Optional[Dict[str, BudgetConfig]] = None +tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None max_end_user_budget: Optional[float] = None max_end_user_budget_id: Optional[str] = None disable_end_user_cost_tracking: Optional[bool] = None @@ -404,7 +384,9 @@ public_agent_groups: Optional[List[str]] = None # Old format: { "displayName": "url" } (for backward compatibility) public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {} #### REQUEST PRIORITIZATION ####### -priority_reservation: Optional[Dict[str, Union[float, PriorityReservationDict]]] = None +priority_reservation: Optional[ + Dict[str, Union[float, "PriorityReservationDict"]] +] = None priority_reservation_settings: "PriorityReservationSettings" = ( PriorityReservationSettings() ) @@ -422,10 +404,6 @@ disable_aiohttp_trust_env: bool = ( force_ipv4: bool = ( False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. ) -module_level_aclient = AsyncHTTPHandler( - timeout=request_timeout, client_alias="module level aclient" -) -module_level_client = HTTPHandler(timeout=request_timeout) #### RETRIES #### num_retries: Optional[int] = None # per model endpoint @@ -1071,7 +1049,6 @@ openai_video_generation_models = ["sora-2"] from .timeout import timeout from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.core_helpers import remove_index_from_tool_calls -from litellm.litellm_core_utils.token_counter import get_modified_max_tokens # client must be imported immediately as it's used as a decorator at function definition time from .utils import client # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py @@ -1079,8 +1056,6 @@ from .utils import client from .llms.bytez.chat.transformation import BytezChatConfig from .llms.custom_llm import CustomLLM -from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig -from .llms.openai_like.chat.handler import OpenAILikeChatConfig from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig from .llms.galadriel.chat.transformation import GaladrielChatConfig from .llms.github.chat.transformation import GithubChatConfig @@ -1265,6 +1240,7 @@ from .llms.xai.responses.transformation import XAIResponsesAPIConfig from .llms.litellm_proxy.responses.transformation import ( LiteLLMProxyResponsesAPIConfig, ) +from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig from .llms.openai.chat.o_series_transformation import ( OpenAIOSeriesConfig as OpenAIO1Config, # maintain backwards compatibility OpenAIOSeriesConfig, @@ -1375,6 +1351,8 @@ from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig from .llms.lemonade.chat.transformation import LemonadeChatConfig from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig + +## Lazy loading this is not straightforward, will leave it here for now. from .main import * # type: ignore # Skills API @@ -1425,6 +1403,9 @@ from .batch_completion.main import * # type: ignore from .rerank_api.main import * from .llms.anthropic.experimental_pass_through.messages.handler import * from .responses.main import * +# Interactions API is available as litellm.interactions module +# Usage: litellm.interactions.create(), litellm.interactions.get(), etc. +from . import interactions from .skills.main import ( create_skill, acreate_skill, @@ -1478,7 +1459,6 @@ from . import rag ### CUSTOM LLMs ### from .types.llms.custom_llm import CustomLLMItem -from .types.utils import GenericStreamingChunk custom_provider_map: List[CustomLLMItem] = [] _custom_providers: List[str] = ( @@ -1520,6 +1500,17 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None: if TYPE_CHECKING: from litellm.types.utils import ModelInfo as _ModelInfoType + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.caching.caching import Cache + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES + from litellm.types.utils import ( + BudgetConfig, + CredentialItem, + PriorityReservationDict, + StandardKeyGenerationConfig, + ) + from litellm.types.guardrails import GuardrailItem # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] @@ -1560,47 +1551,104 @@ if TYPE_CHECKING: # Response types - truly lazy loaded only (not in main.py or elsewhere) ModelResponseListIterator: Type[Any] + # HTTP handler singletons (created lazily via __getattr__ at runtime) + module_level_aclient: AsyncHTTPHandler + module_level_client: HTTPHandler + + # LLM config classes - lazy loaded only + AmazonConverseConfig: Type[Any] + OpenAILikeChatConfig: Type[Any] + def __getattr__(name: str) -> Any: - """Lazy import handler for cost_calculator and litellm_logging functions.""" - # Lazy load cost_calculator functions - _cost_calculator_names = ( - "completion_cost", - "cost_per_token", - "response_cost_calculator", + """Lazy import handler""" + from ._lazy_imports import ( + COST_CALCULATOR_NAMES, + LITELLM_LOGGING_NAMES, + UTILS_NAMES, + TOKEN_COUNTER_NAMES, + LLM_CLIENT_CACHE_NAMES, + BEDROCK_TYPES_NAMES, + TYPES_UTILS_NAMES, + CACHING_NAMES, + HTTP_HANDLER_NAMES, + DOTPROMPT_NAMES, + LLM_CONFIG_NAMES, + TYPES_NAMES, ) - if name in _cost_calculator_names: + + # Lazy load cost_calculator functions + if name in COST_CALCULATOR_NAMES: from ._lazy_imports import _lazy_import_cost_calculator return _lazy_import_cost_calculator(name) # Lazy load litellm_logging functions - _litellm_logging_names = ( - "Logging", - "modify_integration", - ) - if name in _litellm_logging_names: + if name in LITELLM_LOGGING_NAMES: from ._lazy_imports import _lazy_import_litellm_logging return _lazy_import_litellm_logging(name) # Lazy load utils functions - _utils_names = ( - "exception_type", "get_optional_params", "get_response_string", "token_counter", - "create_pretrained_tokenizer", "create_tokenizer", "supports_function_calling", - "supports_web_search", "supports_url_context", "supports_response_schema", - "supports_parallel_function_calling", "supports_vision", "supports_audio_input", - "supports_audio_output", "supports_system_messages", "supports_reasoning", - "get_litellm_params", "acreate", "get_max_tokens", "get_model_info", - "register_prompt_template", "validate_environment", "check_valid_key", - "register_model", "encode", "decode", "_calculate_retry_after", "_should_retry", - "get_supported_openai_params", "get_api_base", "get_first_chars_messages", - "ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse", - "TranscriptionResponse", "TextCompletionResponse", "get_provider_fields", - "ModelResponseListIterator", "get_valid_models", - ) - if name in _utils_names: + if name in UTILS_NAMES: from ._lazy_imports import _lazy_import_utils return _lazy_import_utils(name) + # Lazy load token counter utilities + if name in TOKEN_COUNTER_NAMES: + from ._lazy_imports import _lazy_import_token_counter + return _lazy_import_token_counter(name) + + # Lazy load Bedrock type aliases + if name in BEDROCK_TYPES_NAMES: + from ._lazy_imports import _lazy_import_bedrock_types + return _lazy_import_bedrock_types(name) + + # Lazy load common types.utils symbols + if name in TYPES_UTILS_NAMES: + from ._lazy_imports import _lazy_import_types_utils + return _lazy_import_types_utils(name) + + # Lazy load LLM client cache and its singleton + if name in LLM_CLIENT_CACHE_NAMES: + from ._lazy_imports import _lazy_import_llm_client_cache + return _lazy_import_llm_client_cache(name) + + # Lazy load caching classes + if name in CACHING_NAMES: + from ._lazy_imports import _lazy_import_caching + return _lazy_import_caching(name) + + # Lazy-load HTTP handler singletons used across the codebase + if name in HTTP_HANDLER_NAMES: + from ._lazy_imports import _lazy_import_http_handlers + + return _lazy_import_http_handlers(name) + + # Lazy load dotprompt integration globals + if name in DOTPROMPT_NAMES: + from ._lazy_imports import _lazy_import_dotprompt + + return _lazy_import_dotprompt(name) + + # Lazy load LLM config classes + if name in LLM_CONFIG_NAMES: + from ._lazy_imports import _lazy_import_llm_configs + + return _lazy_import_llm_configs(name) + + # Lazy load types + if name in TYPES_NAMES: + from ._lazy_imports import _lazy_import_types + + return _lazy_import_types(name) + + # Lazy load encoding from main.py to avoid heavy tiktoken import + if name == "encoding": + from .main import encoding as _encoding + # Cache it in the module's __dict__ for subsequent accesses + import sys + sys.modules[__name__].__dict__["encoding"] = _encoding + return _encoding + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 91b16864de1..94b3e4a8da1 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -1,10 +1,170 @@ -from typing import Any +from typing import Any, Optional, cast import sys def _get_litellm_globals() -> dict: """Helper to get the globals dictionary of the litellm module.""" return sys.modules["litellm"].__dict__ +# Lazy loader for default encoding to avoid importing tiktoken at module import time +_default_encoding: Optional[Any] = None + + +def _get_default_encoding() -> Any: + """ + Lazily load and cache the default OpenAI encoding. + + This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken) + at `litellm` import time. The encoding is cached after the first import. + + This is used internally by utils.py functions that need the encoding but shouldn't + trigger its import during module load. + """ + global _default_encoding + if _default_encoding is None: + from litellm.litellm_core_utils.default_encoding import encoding + + _default_encoding = encoding + return _default_encoding + + +# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time +_get_modified_max_tokens_func: Optional[Any] = None + + +def _get_modified_max_tokens() -> Any: + """ + Lazily load and cache the get_modified_max_tokens function. + + This avoids importing `litellm.litellm_core_utils.token_counter` at `litellm` import time. + The function is cached after the first import. + + This is used internally by utils.py functions that need the token counter but shouldn't + trigger its import during module load. + """ + global _get_modified_max_tokens_func + if _get_modified_max_tokens_func is None: + from litellm.litellm_core_utils.token_counter import ( + get_modified_max_tokens as _get_modified_max_tokens_imported, + ) + + _get_modified_max_tokens_func = _get_modified_max_tokens_imported + return _get_modified_max_tokens_func + + +# Lazy loader for token_counter to avoid importing token_counter module at module import time +_token_counter_new_func: Optional[Any] = None + + +def _get_token_counter_new() -> Any: + """ + Lazily load and cache the token_counter function (aliased as token_counter_new). + + This avoids importing `litellm.litellm_core_utils.token_counter` at `litellm` import time. + The function is cached after the first import. + + This is used internally by utils.py functions that need the token counter but shouldn't + trigger its import during module load. + """ + global _token_counter_new_func + if _token_counter_new_func is None: + from litellm.litellm_core_utils.token_counter import ( + token_counter as _token_counter_imported, + ) + + _token_counter_new_func = _token_counter_imported + return _token_counter_new_func + +# Cost calculator names that support lazy loading via _lazy_import_cost_calculator +COST_CALCULATOR_NAMES = ( + "completion_cost", + "cost_per_token", + "response_cost_calculator", +) + +# Litellm logging names that support lazy loading via _lazy_import_litellm_logging +LITELLM_LOGGING_NAMES = ( + "Logging", + "modify_integration", +) + +# Utils names that support lazy loading via _lazy_import_utils +UTILS_NAMES = ( + "exception_type", "get_optional_params", "get_response_string", "token_counter", + "create_pretrained_tokenizer", "create_tokenizer", "supports_function_calling", + "supports_web_search", "supports_url_context", "supports_response_schema", + "supports_parallel_function_calling", "supports_vision", "supports_audio_input", + "supports_audio_output", "supports_system_messages", "supports_reasoning", + "get_litellm_params", "acreate", "get_max_tokens", "get_model_info", + "register_prompt_template", "validate_environment", "check_valid_key", + "register_model", "encode", "decode", "_calculate_retry_after", "_should_retry", + "get_supported_openai_params", "get_api_base", "get_first_chars_messages", + "ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse", + "TranscriptionResponse", "TextCompletionResponse", "get_provider_fields", + "ModelResponseListIterator", "get_valid_models", +) + +# Token counter names that support lazy loading via _lazy_import_token_counter +TOKEN_COUNTER_NAMES = ( + "get_modified_max_tokens", +) + +# LLM client cache names that support lazy loading via _lazy_import_llm_client_cache +LLM_CLIENT_CACHE_NAMES = ( + "LLMClientCache", + "in_memory_llm_clients_cache", +) + +# Bedrock type names that support lazy loading via _lazy_import_bedrock_types +BEDROCK_TYPES_NAMES = ( + "COHERE_EMBEDDING_INPUT_TYPES", +) + +# Common types from litellm.types.utils that support lazy loading via +# _lazy_import_types_utils +TYPES_UTILS_NAMES = ( + "ImageObject", + "BudgetConfig", + "all_litellm_params", + "_litellm_completion_params", + "CredentialItem", + "PriorityReservationDict", + "StandardKeyGenerationConfig", + "SearchProviders", + "GenericStreamingChunk", +) + +# Caching / cache classes that support lazy loading via _lazy_import_caching +CACHING_NAMES = ( + "Cache", + "DualCache", + "RedisCache", + "InMemoryCache", +) + +# HTTP handler names that support lazy loading via _lazy_import_http_handlers +HTTP_HANDLER_NAMES = ( + "module_level_aclient", + "module_level_client", +) + +# Dotprompt integration names that support lazy loading via _lazy_import_dotprompt +DOTPROMPT_NAMES = ( + "global_prompt_manager", + "global_prompt_directory", + "set_global_prompt_directory", +) + +# LLM config classes that support lazy loading via _lazy_import_llm_configs +LLM_CONFIG_NAMES = ( + "AmazonConverseConfig", + "OpenAILikeChatConfig", +) + +# Types that support lazy loading via _lazy_import_types +TYPES_NAMES = ( + "GuardrailItem", +) + # Lazy import for utils module - imports only the requested item by name. # Note: PLR0915 (too many statements) is suppressed because the many if statements # are intentional - each attribute is imported individually only when requested, @@ -218,42 +378,286 @@ def _lazy_import_utils(name: str) -> Any: # noqa: PLR0915 def _lazy_import_cost_calculator(name: str) -> Any: """Lazy import for cost_calculator functions.""" _globals = _get_litellm_globals() - from .cost_calculator import ( - completion_cost as _completion_cost, - cost_per_token as _cost_per_token, - response_cost_calculator as _response_cost_calculator, - ) + if name == "completion_cost": + from .cost_calculator import completion_cost as _completion_cost + _globals["completion_cost"] = _completion_cost + return _completion_cost - _cost_functions = { - "completion_cost": _completion_cost, - "cost_per_token": _cost_per_token, - "response_cost_calculator": _response_cost_calculator, - } + if name == "cost_per_token": + from .cost_calculator import cost_per_token as _cost_per_token + _globals["cost_per_token"] = _cost_per_token + return _cost_per_token - func = _cost_functions[name] - _globals[name] = func - return func + if name == "response_cost_calculator": + from .cost_calculator import response_cost_calculator as _response_cost_calculator + _globals["response_cost_calculator"] = _response_cost_calculator + return _response_cost_calculator + + raise AttributeError(f"Cost calculator lazy import: unknown attribute {name!r}") + + +def _lazy_import_token_counter(name: str) -> Any: + """Lazy import for token_counter utilities.""" + _globals = _get_litellm_globals() + + if name == "get_modified_max_tokens": + from litellm.litellm_core_utils.token_counter import ( + get_modified_max_tokens as _get_modified_max_tokens, + ) + + _globals["get_modified_max_tokens"] = _get_modified_max_tokens + return _get_modified_max_tokens + + raise AttributeError(f"Token counter lazy import: unknown attribute {name!r}") + + +def _lazy_import_bedrock_types(name: str) -> Any: + """Lazy import for Bedrock type aliases.""" + _globals = _get_litellm_globals() + + if name == "COHERE_EMBEDDING_INPUT_TYPES": + from litellm.types.llms.bedrock import ( + COHERE_EMBEDDING_INPUT_TYPES as _COHERE_EMBEDDING_INPUT_TYPES, + ) + + _globals["COHERE_EMBEDDING_INPUT_TYPES"] = _COHERE_EMBEDDING_INPUT_TYPES + return _COHERE_EMBEDDING_INPUT_TYPES + + raise AttributeError(f"Bedrock types lazy import: unknown attribute {name!r}") + + +def _lazy_import_types_utils(name: str) -> Any: + """Lazy import for common types and constants from litellm.types.utils.""" + _globals = _get_litellm_globals() + + if name == "ImageObject": + from .types.utils import ImageObject as _ImageObject + + _globals["ImageObject"] = _ImageObject + return _ImageObject + + if name == "BudgetConfig": + from .types.utils import BudgetConfig as _BudgetConfig + + _globals["BudgetConfig"] = _BudgetConfig + return _BudgetConfig + + if name == "all_litellm_params": + from .types.utils import all_litellm_params as _all_litellm_params + + _globals["all_litellm_params"] = _all_litellm_params + return _all_litellm_params + + if name == "_litellm_completion_params": + from .types.utils import all_litellm_params as _all_litellm_params + + _globals["_litellm_completion_params"] = _all_litellm_params + return _all_litellm_params + + if name == "CredentialItem": + from .types.utils import CredentialItem as _CredentialItem + + _globals["CredentialItem"] = _CredentialItem + return _CredentialItem + + if name == "PriorityReservationDict": + from .types.utils import ( + PriorityReservationDict as _PriorityReservationDict, + ) + + _globals["PriorityReservationDict"] = _PriorityReservationDict + return _PriorityReservationDict + + if name == "StandardKeyGenerationConfig": + from .types.utils import ( + StandardKeyGenerationConfig as _StandardKeyGenerationConfig, + ) + + _globals["StandardKeyGenerationConfig"] = _StandardKeyGenerationConfig + return _StandardKeyGenerationConfig + + if name == "SearchProviders": + from .types.utils import SearchProviders as _SearchProviders + + _globals["SearchProviders"] = _SearchProviders + return _SearchProviders + + if name == "GenericStreamingChunk": + from .types.utils import ( + GenericStreamingChunk as _GenericStreamingChunk, + ) + + _globals["GenericStreamingChunk"] = _GenericStreamingChunk + return _GenericStreamingChunk + + raise AttributeError(f"Types utils lazy import: unknown attribute {name!r}") + + +def _lazy_import_caching(name: str) -> Any: + """Lazy import for caching module classes.""" + _globals = _get_litellm_globals() + + if name == "Cache": + from litellm.caching.caching import Cache as _Cache + + _globals["Cache"] = _Cache + return _Cache + + if name == "DualCache": + from litellm.caching.caching import DualCache as _DualCache + + _globals["DualCache"] = _DualCache + return _DualCache + + if name == "RedisCache": + from litellm.caching.caching import RedisCache as _RedisCache + + _globals["RedisCache"] = _RedisCache + return _RedisCache + + if name == "InMemoryCache": + from litellm.caching.caching import InMemoryCache as _InMemoryCache + + _globals["InMemoryCache"] = _InMemoryCache + return _InMemoryCache + + raise AttributeError(f"Caching lazy import: unknown attribute {name!r}") + + +def _lazy_import_llm_client_cache(name: str) -> Any: + """Lazy import for LLM client cache class and singleton.""" + _globals = _get_litellm_globals() + + if name == "LLMClientCache": + from litellm.caching.llm_caching_handler import LLMClientCache as _LLMClientCache + + _globals["LLMClientCache"] = _LLMClientCache + return _LLMClientCache + + if name == "in_memory_llm_clients_cache": + from litellm.caching.llm_caching_handler import LLMClientCache as _LLMClientCache + + instance = _LLMClientCache() + # Only populate the requested singleton name to keep lazy-import + # semantics consistent with other helpers (no extra symbols). + _globals["in_memory_llm_clients_cache"] = instance + return instance + + raise AttributeError(f"LLM client cache lazy import: unknown attribute {name!r}") def _lazy_import_litellm_logging(name: str) -> Any: """Lazy import for litellm_logging module.""" _globals = _get_litellm_globals() - try: - from litellm.litellm_core_utils.litellm_logging import ( - Logging as _Logging, - modify_integration as _modify_integration, + if name == "Logging": + from litellm.litellm_core_utils.litellm_logging import Logging as _Logging + _globals["Logging"] = _Logging + return _Logging + + if name == "modify_integration": + from litellm.litellm_core_utils.litellm_logging import modify_integration as _modify_integration + _globals["modify_integration"] = _modify_integration + return _modify_integration + + raise AttributeError(f"Litellm logging lazy import: unknown attribute {name!r}") + + +def _lazy_import_http_handlers(name: str) -> Any: + """Lazy import and instantiate module-level HTTP handlers.""" + _globals = _get_litellm_globals() + + if name == "module_level_aclient": + # Use shared async client factory instead of directly instantiating AsyncHTTPHandler + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + + timeout = _globals.get("request_timeout") + params = {"timeout": timeout, "client_alias": "module level aclient"} + # llm_provider is only used for cache keying; use a string identifier but + # cast to Any so static type checkers don't complain about the literal. + provider_id = cast(Any, "litellm_module_level_client") + async_client = get_async_httpx_client( + llm_provider=provider_id, + params=params, ) - - _logging_objects = { - "Logging": _Logging, - "modify_integration": _modify_integration, - } - - obj = _logging_objects[name] - _globals[name] = obj - return obj - except Exception as e: - raise AttributeError( - f"module 'litellm' has no attribute {name!r}. " - f"Lazy import failed: {e}" - ) from e \ No newline at end of file + _globals["module_level_aclient"] = async_client + return async_client + + if name == "module_level_client": + # Import handler type locally to avoid heavy imports at module load time + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + timeout = _globals.get("request_timeout") + sync_client = HTTPHandler(timeout=timeout) + _globals["module_level_client"] = sync_client + return sync_client + + raise AttributeError(f"HTTP handlers lazy import: unknown attribute {name!r}") + + +def _lazy_import_dotprompt(name: str) -> Any: + """Lazy import for dotprompt integration globals.""" + _globals = _get_litellm_globals() + + if name == "global_prompt_manager": + from litellm.integrations.dotprompt import ( + global_prompt_manager as _global_prompt_manager, + ) + + _globals["global_prompt_manager"] = _global_prompt_manager + return _global_prompt_manager + + if name == "global_prompt_directory": + from litellm.integrations.dotprompt import ( + global_prompt_directory as _global_prompt_directory, + ) + + _globals["global_prompt_directory"] = _global_prompt_directory + return _global_prompt_directory + + if name == "set_global_prompt_directory": + from litellm.integrations.dotprompt import ( + set_global_prompt_directory as _set_global_prompt_directory, + ) + + _globals["set_global_prompt_directory"] = _set_global_prompt_directory + return _set_global_prompt_directory + + raise AttributeError(f"Dotprompt lazy import: unknown attribute {name!r}") + + +def _lazy_import_types(name: str) -> Any: + """Lazy import for type classes.""" + _globals = _get_litellm_globals() + + if name == "GuardrailItem": + from litellm.types.guardrails import ( + GuardrailItem as _GuardrailItem, + ) + + _globals["GuardrailItem"] = _GuardrailItem + return _GuardrailItem + + raise AttributeError(f"Types lazy import: unknown attribute {name!r}") + + +def _lazy_import_llm_configs(name: str) -> Any: + """Lazy import for LLM config classes.""" + _globals = _get_litellm_globals() + + if name == "AmazonConverseConfig": + from .llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig as _AmazonConverseConfig, + ) + + _globals["AmazonConverseConfig"] = _AmazonConverseConfig + return _AmazonConverseConfig + + if name == "OpenAILikeChatConfig": + from .llms.openai_like.chat.handler import ( + OpenAILikeChatConfig as _OpenAILikeChatConfig, + ) + + _globals["OpenAILikeChatConfig"] = _OpenAILikeChatConfig + return _OpenAILikeChatConfig + + raise AttributeError(f"LLM config lazy import: unknown attribute {name!r}") \ No newline at end of file diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index cc222d3ee18..1916b04454a 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -18,6 +18,7 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import ( A2ACompletionBridgeTransformation, A2AStreamingContext, ) +from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager class A2ACompletionBridgeHandler: @@ -44,6 +45,29 @@ class A2ACompletionBridgeHandler: Returns: A2A SendMessageResponse dict """ + # Get provider config for custom_llm_provider + custom_llm_provider = litellm_params.get("custom_llm_provider") + a2a_provider_config = A2AProviderConfigManager.get_provider_config( + custom_llm_provider=custom_llm_provider + ) + + # If provider config exists, use it + if a2a_provider_config is not None: + if api_base is None: + raise ValueError(f"api_base is required for {custom_llm_provider}") + + verbose_logger.info( + f"A2A: Using provider config for {custom_llm_provider}" + ) + + response_data = await a2a_provider_config.handle_non_streaming( + request_id=request_id, + params=params, + api_base=api_base, + ) + + return response_data + # Extract message from params message = params.get("message", {}) @@ -119,6 +143,30 @@ class A2ACompletionBridgeHandler: Yields: A2A streaming response events """ + # Get provider config for custom_llm_provider + custom_llm_provider = litellm_params.get("custom_llm_provider") + a2a_provider_config = A2AProviderConfigManager.get_provider_config( + custom_llm_provider=custom_llm_provider + ) + + # If provider config exists, use it + if a2a_provider_config is not None: + if api_base is None: + raise ValueError(f"api_base is required for {custom_llm_provider}") + + verbose_logger.info( + f"A2A: Using provider config for {custom_llm_provider} (streaming)" + ) + + async for chunk in a2a_provider_config.handle_streaming( + request_id=request_id, + params=params, + api_base=api_base, + ): + yield chunk + + return + # Extract message from params message = params.get("message", {}) diff --git a/litellm/a2a_protocol/providers/__init__.py b/litellm/a2a_protocol/providers/__init__.py new file mode 100644 index 00000000000..873a5a83749 --- /dev/null +++ b/litellm/a2a_protocol/providers/__init__.py @@ -0,0 +1,11 @@ +""" +A2A Protocol Providers. + +This module contains provider-specific implementations for the A2A protocol. +""" + +from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig +from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager + +__all__ = ["BaseA2AProviderConfig", "A2AProviderConfigManager"] + diff --git a/litellm/a2a_protocol/providers/base.py b/litellm/a2a_protocol/providers/base.py new file mode 100644 index 00000000000..9931076a948 --- /dev/null +++ b/litellm/a2a_protocol/providers/base.py @@ -0,0 +1,63 @@ +""" +Base configuration for A2A protocol providers. +""" + +from abc import ABC, abstractmethod +from typing import Any, AsyncIterator, Dict + + +class BaseA2AProviderConfig(ABC): + """ + Base configuration class for A2A protocol providers. + + Each provider should implement this interface to define how to handle + A2A requests for their specific agent type. + """ + + @abstractmethod + async def handle_non_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: str, + **kwargs, + ) -> Dict[str, Any]: + """ + Handle non-streaming A2A request. + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + api_base: Base URL of the agent + **kwargs: Additional provider-specific parameters + + Returns: + A2A SendMessageResponse dict + """ + pass + + @abstractmethod + async def handle_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: str, + **kwargs, + ) -> AsyncIterator[Dict[str, Any]]: + """ + Handle streaming A2A request. + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + api_base: Base URL of the agent + **kwargs: Additional provider-specific parameters + + Yields: + A2A streaming response events + """ + # This is an abstract method - subclasses must implement + # The yield is here to make this a generator function + if False: # pragma: no cover + yield {} + diff --git a/litellm/a2a_protocol/providers/config_manager.py b/litellm/a2a_protocol/providers/config_manager.py new file mode 100644 index 00000000000..e0703ec466b --- /dev/null +++ b/litellm/a2a_protocol/providers/config_manager.py @@ -0,0 +1,48 @@ +""" +A2A Provider Config Manager. + +Manages provider-specific configurations for A2A protocol. +""" + +from typing import Optional + +from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig + + +class A2AProviderConfigManager: + """ + Manager for A2A provider configurations. + + Similar to ProviderConfigManager in litellm.utils but specifically for A2A providers. + """ + + @staticmethod + def get_provider_config( + custom_llm_provider: Optional[str], + ) -> Optional[BaseA2AProviderConfig]: + """ + Get the provider configuration for a given custom_llm_provider. + + Args: + custom_llm_provider: The provider identifier (e.g., "pydantic_ai_agents") + + Returns: + Provider configuration instance or None if not found + """ + if custom_llm_provider is None: + return None + + if custom_llm_provider == "pydantic_ai_agents": + from litellm.a2a_protocol.providers.pydantic_ai_agents.config import ( + PydanticAIProviderConfig, + ) + + return PydanticAIProviderConfig() + + # Add more providers here as needed + # elif custom_llm_provider == "another_provider": + # from litellm.a2a_protocol.providers.another_provider.config import AnotherProviderConfig + # return AnotherProviderConfig() + + return None + diff --git a/litellm/a2a_protocol/providers/litellm_completion/README.md b/litellm/a2a_protocol/providers/litellm_completion/README.md new file mode 100644 index 00000000000..a809e9bf55e --- /dev/null +++ b/litellm/a2a_protocol/providers/litellm_completion/README.md @@ -0,0 +1,74 @@ +# A2A to LiteLLM Completion Bridge + +Routes A2A protocol requests through `litellm.acompletion`, enabling any LiteLLM-supported provider to be invoked via A2A. + +## Flow + +``` +A2A Request → Transform → litellm.acompletion → Transform → A2A Response +``` + +## SDK Usage + +Use the existing `asend_message` and `asend_message_streaming` functions with `litellm_params`: + +```python +from litellm.a2a_protocol import asend_message, asend_message_streaming +from a2a.types import SendMessageRequest, SendStreamingMessageRequest, MessageSendParams +from uuid import uuid4 + +# Non-streaming +request = SendMessageRequest( + id=str(uuid4()), + params=MessageSendParams( + message={"role": "user", "parts": [{"kind": "text", "text": "Hello!"}], "messageId": uuid4().hex} + ) +) +response = await asend_message( + request=request, + api_base="http://localhost:2024", + litellm_params={"custom_llm_provider": "langgraph", "model": "agent"}, +) + +# Streaming +stream_request = SendStreamingMessageRequest( + id=str(uuid4()), + params=MessageSendParams( + message={"role": "user", "parts": [{"kind": "text", "text": "Hello!"}], "messageId": uuid4().hex} + ) +) +async for chunk in asend_message_streaming( + request=stream_request, + api_base="http://localhost:2024", + litellm_params={"custom_llm_provider": "langgraph", "model": "agent"}, +): + print(chunk) +``` + +## Proxy Usage + +Configure an agent with `custom_llm_provider` in `litellm_params`: + +```yaml +agents: + - agent_name: my-langgraph-agent + agent_card_params: + name: "LangGraph Agent" + url: "http://localhost:2024" # Used as api_base + litellm_params: + custom_llm_provider: langgraph + model: agent +``` + +When an A2A request hits `/a2a/{agent_id}/message/send`, the bridge: + +1. Detects `custom_llm_provider` in agent's `litellm_params` +2. Transforms A2A message → OpenAI messages +3. Calls `litellm.acompletion(model="langgraph/agent", api_base="http://localhost:2024")` +4. Transforms response → A2A format + +## Classes + +- `A2ACompletionBridgeTransformation` - Static methods for message format conversion +- `A2ACompletionBridgeHandler` - Static methods for handling requests (streaming/non-streaming) + diff --git a/litellm/a2a_protocol/providers/litellm_completion/__init__.py b/litellm/a2a_protocol/providers/litellm_completion/__init__.py new file mode 100644 index 00000000000..3f2b88bfaa3 --- /dev/null +++ b/litellm/a2a_protocol/providers/litellm_completion/__init__.py @@ -0,0 +1,6 @@ +""" +LiteLLM Completion bridge provider for A2A protocol. + +Routes A2A requests through litellm.acompletion based on custom_llm_provider. +""" + diff --git a/litellm/a2a_protocol/providers/litellm_completion/handler.py b/litellm/a2a_protocol/providers/litellm_completion/handler.py new file mode 100644 index 00000000000..57388a5d0ed --- /dev/null +++ b/litellm/a2a_protocol/providers/litellm_completion/handler.py @@ -0,0 +1,295 @@ +""" +Handler for A2A to LiteLLM completion bridge. + +Routes A2A requests through litellm.acompletion based on custom_llm_provider. + +A2A Streaming Events (in order): +1. Task event (kind: "task") - Initial task creation with status "submitted" +2. Status update (kind: "status-update") - Status change to "working" +3. Artifact update (kind: "artifact-update") - Content/artifact delivery +4. Status update (kind: "status-update") - Final status "completed" with final=true +""" + +from typing import Any, AsyncIterator, Dict, Optional + +import litellm +from litellm._logging import verbose_logger +from litellm.a2a_protocol.litellm_completion_bridge.pydantic_ai_transformation import ( + PydanticAITransformation, +) +from litellm.a2a_protocol.litellm_completion_bridge.transformation import ( + A2ACompletionBridgeTransformation, + A2AStreamingContext, +) + + +class A2ACompletionBridgeHandler: + """ + Static methods for handling A2A requests via LiteLLM completion. + """ + + @staticmethod + async def handle_non_streaming( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + api_base: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Handle non-streaming A2A request via litellm.acompletion. + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.) + api_base: API base URL from agent_card_params + + Returns: + A2A SendMessageResponse dict + """ + # Check if this is a Pydantic AI agent request + custom_llm_provider = litellm_params.get("custom_llm_provider") + if custom_llm_provider == "pydantic_ai_agents": + if api_base is None: + raise ValueError("api_base is required for Pydantic AI agents") + + verbose_logger.info( + f"Pydantic AI: Routing to Pydantic AI agent at {api_base}" + ) + + # Send request directly to Pydantic AI agent + response_data = await PydanticAITransformation.send_non_streaming_request( + api_base=api_base, + request_id=request_id, + params=params, + ) + + return response_data + + # Extract message from params + message = params.get("message", {}) + + # Transform A2A message to OpenAI format + openai_messages = A2ACompletionBridgeTransformation.a2a_message_to_openai_messages( + message + ) + + # Get completion params + custom_llm_provider = litellm_params.get("custom_llm_provider") + model = litellm_params.get("model", "agent") + + # Build full model string if provider specified + # Skip prepending if model already starts with the provider prefix + if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"): + full_model = f"{custom_llm_provider}/{model}" + else: + full_model = model + + verbose_logger.info( + f"A2A completion bridge: model={full_model}, api_base={api_base}" + ) + + # Build completion params dict + completion_params = { + "model": full_model, + "messages": openai_messages, + "api_base": api_base, + "stream": False, + } + # Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.) + litellm_params_to_add = { + k: v for k, v in litellm_params.items() + if k not in ("model", "custom_llm_provider") + } + completion_params.update(litellm_params_to_add) + + # Call litellm.acompletion + response = await litellm.acompletion(**completion_params) + + # Transform response to A2A format + a2a_response = A2ACompletionBridgeTransformation.openai_response_to_a2a_response( + response=response, + request_id=request_id, + ) + + verbose_logger.info(f"A2A completion bridge completed: request_id={request_id}") + + return a2a_response + + @staticmethod + async def handle_streaming( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + api_base: Optional[str] = None, + ) -> AsyncIterator[Dict[str, Any]]: + """ + Handle streaming A2A request via litellm.acompletion with stream=True. + + Emits proper A2A streaming events: + 1. Task event (kind: "task") - Initial task with status "submitted" + 2. Status update (kind: "status-update") - Status "working" + 3. Artifact update (kind: "artifact-update") - Content delivery + 4. Status update (kind: "status-update") - Final "completed" status + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.) + api_base: API base URL from agent_card_params + + Yields: + A2A streaming response events + """ + # Check if this is a Pydantic AI agent request + custom_llm_provider = litellm_params.get("custom_llm_provider") + if custom_llm_provider == "pydantic_ai_agents": + if api_base is None: + raise ValueError("api_base is required for Pydantic AI agents") + + verbose_logger.info( + f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}" + ) + + # Get non-streaming response first + response_data = await PydanticAITransformation.send_non_streaming_request( + api_base=api_base, + request_id=request_id, + params=params, + ) + + # Convert to fake streaming + async for chunk in PydanticAITransformation.fake_streaming_from_response( + response_data=response_data, + request_id=request_id, + ): + yield chunk + + return + + # Extract message from params + message = params.get("message", {}) + + # Create streaming context + ctx = A2AStreamingContext( + request_id=request_id, + input_message=message, + ) + + # Transform A2A message to OpenAI format + openai_messages = A2ACompletionBridgeTransformation.a2a_message_to_openai_messages( + message + ) + + # Get completion params + custom_llm_provider = litellm_params.get("custom_llm_provider") + model = litellm_params.get("model", "agent") + + # Build full model string if provider specified + # Skip prepending if model already starts with the provider prefix + if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"): + full_model = f"{custom_llm_provider}/{model}" + else: + full_model = model + + verbose_logger.info( + f"A2A completion bridge streaming: model={full_model}, api_base={api_base}" + ) + + # Build completion params dict + completion_params = { + "model": full_model, + "messages": openai_messages, + "api_base": api_base, + "stream": True, + } + # Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.) + litellm_params_to_add = { + k: v for k, v in litellm_params.items() + if k not in ("model", "custom_llm_provider") + } + completion_params.update(litellm_params_to_add) + + # 1. Emit initial task event (kind: "task", status: "submitted") + task_event = A2ACompletionBridgeTransformation.create_task_event(ctx) + yield task_event + + # 2. Emit status update (kind: "status-update", status: "working") + working_event = A2ACompletionBridgeTransformation.create_status_update_event( + ctx=ctx, + state="working", + final=False, + message_text="Processing request...", + ) + yield working_event + + # Call litellm.acompletion with streaming + response = await litellm.acompletion(**completion_params) + + # 3. Accumulate content and emit artifact update + accumulated_text = "" + chunk_count = 0 + async for chunk in response: # type: ignore[union-attr] + chunk_count += 1 + + # Extract delta content + content = "" + if chunk is not None and hasattr(chunk, "choices") and chunk.choices: + choice = chunk.choices[0] + if hasattr(choice, "delta") and choice.delta: + content = choice.delta.content or "" + + if content: + accumulated_text += content + + # Emit artifact update with accumulated content + if accumulated_text: + artifact_event = A2ACompletionBridgeTransformation.create_artifact_update_event( + ctx=ctx, + text=accumulated_text, + ) + yield artifact_event + + # 4. Emit final status update (kind: "status-update", status: "completed", final: true) + completed_event = A2ACompletionBridgeTransformation.create_status_update_event( + ctx=ctx, + state="completed", + final=True, + ) + yield completed_event + + verbose_logger.info( + f"A2A completion bridge streaming completed: request_id={request_id}, chunks={chunk_count}" + ) + + +# Convenience functions that delegate to the class methods +async def handle_a2a_completion( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + api_base: Optional[str] = None, +) -> Dict[str, Any]: + """Convenience function for non-streaming A2A completion.""" + return await A2ACompletionBridgeHandler.handle_non_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + api_base=api_base, + ) + + +async def handle_a2a_completion_streaming( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + api_base: Optional[str] = None, +) -> AsyncIterator[Dict[str, Any]]: + """Convenience function for streaming A2A completion.""" + async for chunk in A2ACompletionBridgeHandler.handle_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + api_base=api_base, + ): + yield chunk diff --git a/litellm/a2a_protocol/providers/litellm_completion/transformation.py b/litellm/a2a_protocol/providers/litellm_completion/transformation.py new file mode 100644 index 00000000000..bbe7daa9fc4 --- /dev/null +++ b/litellm/a2a_protocol/providers/litellm_completion/transformation.py @@ -0,0 +1,286 @@ +""" +Transformation utilities for A2A <-> OpenAI message format conversion. + +A2A Message Format: +{ + "role": "user", + "parts": [{"kind": "text", "text": "Hello!"}], + "messageId": "abc123" +} + +OpenAI Message Format: +{"role": "user", "content": "Hello!"} + +A2A Streaming Events: +- Task event (kind: "task") - Initial task creation with status "submitted" +- Status update (kind: "status-update") - Status changes (working, completed) +- Artifact update (kind: "artifact-update") - Content/artifact delivery +""" + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional +from uuid import uuid4 + +from litellm._logging import verbose_logger + + +class A2AStreamingContext: + """ + Context holder for A2A streaming state. + Tracks task_id, context_id, and message accumulation. + """ + + def __init__(self, request_id: str, input_message: Dict[str, Any]): + self.request_id = request_id + self.task_id = str(uuid4()) + self.context_id = str(uuid4()) + self.input_message = input_message + self.accumulated_text = "" + self.has_emitted_task = False + self.has_emitted_working = False + + +class A2ACompletionBridgeTransformation: + """ + Static methods for transforming between A2A and OpenAI message formats. + """ + + @staticmethod + def a2a_message_to_openai_messages( + a2a_message: Dict[str, Any], + ) -> List[Dict[str, str]]: + """ + Transform an A2A message to OpenAI message format. + + Args: + a2a_message: A2A message with role, parts, and messageId + + Returns: + List of OpenAI-format messages + """ + role = a2a_message.get("role", "user") + parts = a2a_message.get("parts", []) + + # Map A2A roles to OpenAI roles + openai_role = role + if role == "user": + openai_role = "user" + elif role == "assistant": + openai_role = "assistant" + elif role == "system": + openai_role = "system" + + # Extract text content from parts + content_parts = [] + for part in parts: + kind = part.get("kind", "") + if kind == "text": + text = part.get("text", "") + content_parts.append(text) + + content = "\n".join(content_parts) if content_parts else "" + + verbose_logger.debug( + f"A2A -> OpenAI transform: role={role} -> {openai_role}, content_length={len(content)}" + ) + + return [{"role": openai_role, "content": content}] + + @staticmethod + def openai_response_to_a2a_response( + response: Any, + request_id: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Transform a LiteLLM ModelResponse to A2A SendMessageResponse format. + + Args: + response: LiteLLM ModelResponse object + request_id: Original A2A request ID + + Returns: + A2A SendMessageResponse dict + """ + # Extract content from response + content = "" + if hasattr(response, "choices") and response.choices: + choice = response.choices[0] + if hasattr(choice, "message") and choice.message: + content = choice.message.content or "" + + # Build A2A message + a2a_message = { + "role": "agent", + "parts": [{"kind": "text", "text": content}], + "messageId": uuid4().hex, + } + + # Build A2A response + a2a_response = { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "message": a2a_message, + }, + } + + verbose_logger.debug( + f"OpenAI -> A2A transform: content_length={len(content)}" + ) + + return a2a_response + + @staticmethod + def _get_timestamp() -> str: + """Get current timestamp in ISO format with timezone.""" + return datetime.now(timezone.utc).isoformat() + + @staticmethod + def create_task_event( + ctx: A2AStreamingContext, + ) -> Dict[str, Any]: + """ + Create the initial task event with status 'submitted'. + + This is the first event emitted in an A2A streaming response. + """ + return { + "id": ctx.request_id, + "jsonrpc": "2.0", + "result": { + "contextId": ctx.context_id, + "history": [ + { + "contextId": ctx.context_id, + "kind": "message", + "messageId": ctx.input_message.get("messageId", uuid4().hex), + "parts": ctx.input_message.get("parts", []), + "role": ctx.input_message.get("role", "user"), + "taskId": ctx.task_id, + } + ], + "id": ctx.task_id, + "kind": "task", + "status": { + "state": "submitted", + }, + }, + } + + @staticmethod + def create_status_update_event( + ctx: A2AStreamingContext, + state: str, + final: bool = False, + message_text: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Create a status update event. + + Args: + ctx: Streaming context + state: Status state ('working', 'completed') + final: Whether this is the final event + message_text: Optional message text for 'working' status + """ + status: Dict[str, Any] = { + "state": state, + "timestamp": A2ACompletionBridgeTransformation._get_timestamp(), + } + + # Add message for 'working' status + if state == "working" and message_text: + status["message"] = { + "contextId": ctx.context_id, + "kind": "message", + "messageId": str(uuid4()), + "parts": [{"kind": "text", "text": message_text}], + "role": "agent", + "taskId": ctx.task_id, + } + + return { + "id": ctx.request_id, + "jsonrpc": "2.0", + "result": { + "contextId": ctx.context_id, + "final": final, + "kind": "status-update", + "status": status, + "taskId": ctx.task_id, + }, + } + + @staticmethod + def create_artifact_update_event( + ctx: A2AStreamingContext, + text: str, + ) -> Dict[str, Any]: + """ + Create an artifact update event with content. + + Args: + ctx: Streaming context + text: The text content for the artifact + """ + return { + "id": ctx.request_id, + "jsonrpc": "2.0", + "result": { + "artifact": { + "artifactId": str(uuid4()), + "name": "response", + "parts": [{"kind": "text", "text": text}], + }, + "contextId": ctx.context_id, + "kind": "artifact-update", + "taskId": ctx.task_id, + }, + } + + @staticmethod + def openai_chunk_to_a2a_chunk( + chunk: Any, + request_id: Optional[str] = None, + is_final: bool = False, + ) -> Optional[Dict[str, Any]]: + """ + Transform a LiteLLM streaming chunk to A2A streaming format. + + NOTE: This method is deprecated for streaming. Use the event-based + methods (create_task_event, create_status_update_event, + create_artifact_update_event) instead for proper A2A streaming. + + Args: + chunk: LiteLLM ModelResponse chunk + request_id: Original A2A request ID + is_final: Whether this is the final chunk + + Returns: + A2A streaming chunk dict or None if no content + """ + # Extract delta content + content = "" + if chunk is not None and hasattr(chunk, "choices") and chunk.choices: + choice = chunk.choices[0] + if hasattr(choice, "delta") and choice.delta: + content = choice.delta.content or "" + + if not content and not is_final: + return None + + # Build A2A streaming chunk (legacy format) + a2a_chunk = { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "message": { + "role": "agent", + "parts": [{"kind": "text", "text": content}], + "messageId": uuid4().hex, + }, + "final": is_final, + }, + } + + return a2a_chunk diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/__init__.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/__init__.py new file mode 100644 index 00000000000..2187400b2d1 --- /dev/null +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/__init__.py @@ -0,0 +1,17 @@ +""" +Pydantic AI agent provider for A2A protocol. + +Pydantic AI agents follow A2A protocol but don't support streaming natively. +This provider handles fake streaming by converting non-streaming responses into streaming chunks. +""" + +from litellm.a2a_protocol.providers.pydantic_ai_agents.config import ( + PydanticAIProviderConfig, +) +from litellm.a2a_protocol.providers.pydantic_ai_agents.handler import PydanticAIHandler +from litellm.a2a_protocol.providers.pydantic_ai_agents.transformation import ( + PydanticAITransformation, +) + +__all__ = ["PydanticAIHandler", "PydanticAITransformation", "PydanticAIProviderConfig"] + diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py new file mode 100644 index 00000000000..acf09554e5e --- /dev/null +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py @@ -0,0 +1,51 @@ +""" +Pydantic AI provider configuration. +""" + +from typing import Any, AsyncIterator, Dict + +from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig +from litellm.a2a_protocol.providers.pydantic_ai_agents.handler import PydanticAIHandler + + +class PydanticAIProviderConfig(BaseA2AProviderConfig): + """ + Provider configuration for Pydantic AI agents. + + Pydantic AI agents follow A2A protocol but don't support streaming natively. + This config provides fake streaming by converting non-streaming responses into streaming chunks. + """ + + async def handle_non_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: str, + **kwargs, + ) -> Dict[str, Any]: + """Handle non-streaming request to Pydantic AI agent.""" + return await PydanticAIHandler.handle_non_streaming( + request_id=request_id, + params=params, + api_base=api_base, + timeout=kwargs.get("timeout", 60.0), + ) + + async def handle_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: str, + **kwargs, + ) -> AsyncIterator[Dict[str, Any]]: + """Handle streaming request with fake streaming.""" + async for chunk in PydanticAIHandler.handle_streaming( + request_id=request_id, + params=params, + api_base=api_base, + timeout=kwargs.get("timeout", 60.0), + chunk_size=kwargs.get("chunk_size", 50), + delay_ms=kwargs.get("delay_ms", 10), + ): + yield chunk + diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py new file mode 100644 index 00000000000..6680a9fe487 --- /dev/null +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py @@ -0,0 +1,106 @@ +""" +Handler for Pydantic AI agents. + +Pydantic AI agents follow A2A protocol but don't support streaming natively. +This handler provides fake streaming by converting non-streaming responses into streaming chunks. +""" + +from typing import Any, AsyncIterator, Dict + +from litellm._logging import verbose_logger +from litellm.a2a_protocol.providers.pydantic_ai_agents.transformation import ( + PydanticAITransformation, +) + + +class PydanticAIHandler: + """ + Handler for Pydantic AI agent requests. + + Provides: + - Direct non-streaming requests to Pydantic AI agents + - Fake streaming by converting non-streaming responses into streaming chunks + """ + + @staticmethod + async def handle_non_streaming( + request_id: str, + params: Dict[str, Any], + api_base: str, + timeout: float = 60.0, + ) -> Dict[str, Any]: + """ + Handle non-streaming request to Pydantic AI agent. + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + api_base: Base URL of the Pydantic AI agent + timeout: Request timeout in seconds + + Returns: + A2A SendMessageResponse dict + """ + verbose_logger.info( + f"Pydantic AI: Routing to Pydantic AI agent at {api_base}" + ) + + # Send request directly to Pydantic AI agent + response_data = await PydanticAITransformation.send_non_streaming_request( + api_base=api_base, + request_id=request_id, + params=params, + timeout=timeout, + ) + + return response_data + + @staticmethod + async def handle_streaming( + request_id: str, + params: Dict[str, Any], + api_base: str, + timeout: float = 60.0, + chunk_size: int = 50, + delay_ms: int = 10, + ) -> AsyncIterator[Dict[str, Any]]: + """ + Handle streaming request to Pydantic AI agent with fake streaming. + + Since Pydantic AI agents don't support streaming natively, this method: + 1. Makes a non-streaming request + 2. Converts the response into streaming chunks + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + api_base: Base URL of the Pydantic AI agent + timeout: Request timeout in seconds + chunk_size: Number of characters per chunk + delay_ms: Delay between chunks in milliseconds + + Yields: + A2A streaming response events + """ + verbose_logger.info( + f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}" + ) + + # Get raw task response first (not the transformed A2A format) + raw_response = await PydanticAITransformation.send_and_get_raw_response( + api_base=api_base, + request_id=request_id, + params=params, + timeout=timeout, + ) + + # Convert raw task response to fake streaming chunks + async for chunk in PydanticAITransformation.fake_streaming_from_response( + response_data=raw_response, + request_id=request_id, + chunk_size=chunk_size, + delay_ms=delay_ms, + ): + yield chunk + + diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py new file mode 100644 index 00000000000..9352eab6c8e --- /dev/null +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py @@ -0,0 +1,525 @@ +""" +Transformation layer for Pydantic AI agents. + +Pydantic AI agents follow A2A protocol but don't support streaming. +This module provides fake streaming by converting non-streaming responses into streaming chunks. +""" + +import asyncio +from typing import Any, AsyncIterator, Dict, cast +from uuid import uuid4 + +from litellm._logging import verbose_logger +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client + + +class PydanticAITransformation: + """ + Transformation layer for Pydantic AI agents. + + Handles: + - Direct A2A requests to Pydantic AI endpoints + - Polling for task completion (since Pydantic AI doesn't support streaming) + - Fake streaming by chunking non-streaming responses + """ + + @staticmethod + def _remove_none_values(obj: Any) -> Any: + """ + Recursively remove None values from a dict/list structure. + + FastA2A/Pydantic AI servers don't accept None values for optional fields - + they expect those fields to be omitted entirely. + + Args: + obj: Dict, list, or other value to clean + + Returns: + Cleaned object with None values removed + """ + if isinstance(obj, dict): + return { + k: PydanticAITransformation._remove_none_values(v) + for k, v in obj.items() + if v is not None + } + elif isinstance(obj, list): + return [ + PydanticAITransformation._remove_none_values(item) + for item in obj + if item is not None + ] + else: + return obj + + @staticmethod + def _params_to_dict(params: Any) -> Dict[str, Any]: + """ + Convert params to a dict, handling Pydantic models. + + Args: + params: Dict or Pydantic model + + Returns: + Dict representation of params + """ + if hasattr(params, "model_dump"): + # Pydantic v2 model + return params.model_dump(mode="python", exclude_none=True) + elif hasattr(params, "dict"): + # Pydantic v1 model + return params.dict(exclude_none=True) + elif isinstance(params, dict): + return params + else: + # Try to convert to dict + return dict(params) + + @staticmethod + async def _poll_for_completion( + client: AsyncHTTPHandler, + endpoint: str, + task_id: str, + request_id: str, + max_attempts: int = 30, + poll_interval: float = 0.5, + ) -> Dict[str, Any]: + """ + Poll for task completion using tasks/get method. + + Args: + client: HTTPX async client + endpoint: API endpoint URL + task_id: Task ID to poll for + request_id: JSON-RPC request ID + max_attempts: Maximum polling attempts + poll_interval: Seconds between poll attempts + + Returns: + Completed task response + """ + for attempt in range(max_attempts): + poll_request = { + "jsonrpc": "2.0", + "id": f"{request_id}-poll-{attempt}", + "method": "tasks/get", + "params": {"id": task_id}, + } + + response = await client.post( + endpoint, + json=poll_request, + headers={"Content-Type": "application/json"}, + ) + response.raise_for_status() + poll_data = response.json() + + result = poll_data.get("result", {}) + status = result.get("status", {}) + state = status.get("state", "") + + verbose_logger.debug( + f"Pydantic AI: Poll attempt {attempt + 1}/{max_attempts}, state={state}" + ) + + if state == "completed": + return poll_data + elif state in ("failed", "canceled"): + raise Exception(f"Task {task_id} ended with state: {state}") + + await asyncio.sleep(poll_interval) + + raise TimeoutError(f"Task {task_id} did not complete within {max_attempts * poll_interval} seconds") + + @staticmethod + async def _send_and_poll_raw( + api_base: str, + request_id: str, + params: Any, + timeout: float = 60.0, + ) -> Dict[str, Any]: + """ + Send a request to Pydantic AI agent and return the raw task response. + + This is an internal method used by both non-streaming and streaming handlers. + Returns the raw Pydantic AI task format with history/artifacts. + + Args: + api_base: Base URL of the Pydantic AI agent + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + timeout: Request timeout in seconds + + Returns: + Raw Pydantic AI task response (with history/artifacts) + """ + # Convert params to dict if it's a Pydantic model + params_dict = PydanticAITransformation._params_to_dict(params) + + # Remove None values - FastA2A doesn't accept null for optional fields + params_dict = PydanticAITransformation._remove_none_values(params_dict) + + # Ensure the message has 'kind': 'message' as required by FastA2A/Pydantic AI + if "message" in params_dict: + params_dict["message"]["kind"] = "message" + + # Build A2A JSON-RPC request using message/send method for FastA2A compatibility + a2a_request = { + "jsonrpc": "2.0", + "id": request_id, + "method": "message/send", + "params": params_dict, + } + + # FastA2A uses root endpoint (/) not /messages + endpoint = api_base.rstrip("/") + + verbose_logger.info( + f"Pydantic AI: Sending non-streaming request to {endpoint}" + ) + + # Send request to Pydantic AI agent using shared async HTTP client + client = get_async_httpx_client( + llm_provider=cast(Any, "pydantic_ai_agent"), + params={"timeout": timeout}, + ) + response = await client.post( + endpoint, + json=a2a_request, + headers={"Content-Type": "application/json"}, + ) + response.raise_for_status() + response_data = response.json() + + # Check if task is already completed + result = response_data.get("result", {}) + status = result.get("status", {}) + state = status.get("state", "") + + if state != "completed": + # Need to poll for completion + task_id = result.get("id") + if task_id: + verbose_logger.info( + f"Pydantic AI: Task {task_id} submitted, polling for completion..." + ) + response_data = await PydanticAITransformation._poll_for_completion( + client=client, + endpoint=endpoint, + task_id=task_id, + request_id=request_id, + ) + + verbose_logger.info(f"Pydantic AI: Received completed response for request_id={request_id}") + + return response_data + + @staticmethod + async def send_non_streaming_request( + api_base: str, + request_id: str, + params: Any, + timeout: float = 60.0, + ) -> Dict[str, Any]: + """ + Send a non-streaming A2A request to Pydantic AI agent and wait for completion. + + Args: + api_base: Base URL of the Pydantic AI agent (e.g., "http://localhost:9999") + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message (dict or Pydantic model) + timeout: Request timeout in seconds + + Returns: + Standard A2A non-streaming response format with message + """ + # Get raw task response + raw_response = await PydanticAITransformation._send_and_poll_raw( + api_base=api_base, + request_id=request_id, + params=params, + timeout=timeout, + ) + + # Transform to standard A2A non-streaming format + return PydanticAITransformation._transform_to_a2a_response( + response_data=raw_response, + request_id=request_id, + ) + + @staticmethod + async def send_and_get_raw_response( + api_base: str, + request_id: str, + params: Any, + timeout: float = 60.0, + ) -> Dict[str, Any]: + """ + Send a request to Pydantic AI agent and return the raw task response. + + Used by streaming handler to get raw response for fake streaming. + + Args: + api_base: Base URL of the Pydantic AI agent + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + timeout: Request timeout in seconds + + Returns: + Raw Pydantic AI task response (with history/artifacts) + """ + return await PydanticAITransformation._send_and_poll_raw( + api_base=api_base, + request_id=request_id, + params=params, + timeout=timeout, + ) + + @staticmethod + def _transform_to_a2a_response( + response_data: Dict[str, Any], + request_id: str, + ) -> Dict[str, Any]: + """ + Transform Pydantic AI task response to standard A2A non-streaming format. + + Pydantic AI returns a task with history/artifacts, but the standard A2A + non-streaming format expects: + { + "jsonrpc": "2.0", + "id": "...", + "result": { + "message": { + "role": "agent", + "parts": [{"kind": "text", "text": "..."}], + "messageId": "..." + } + } + } + + Args: + response_data: Pydantic AI task response + request_id: Original request ID + + Returns: + Standard A2A non-streaming response format + """ + # Extract the agent response text + full_text, message_id, parts = PydanticAITransformation._extract_response_text( + response_data + ) + + # Build standard A2A message + a2a_message = { + "role": "agent", + "parts": parts if parts else [{"kind": "text", "text": full_text}], + "messageId": message_id, + } + + # Return standard A2A non-streaming format + return { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "message": a2a_message, + }, + } + + @staticmethod + def _extract_response_text(response_data: Dict[str, Any]) -> tuple[str, str, list]: + """ + Extract response text from completed task response. + + Pydantic AI returns completed tasks with: + - history: list of messages (user and agent) + - artifacts: list of result artifacts + + Args: + response_data: Completed task response + + Returns: + Tuple of (full_text, message_id, parts) + """ + result = response_data.get("result", {}) + + # Try to extract from artifacts first (preferred for results) + artifacts = result.get("artifacts", []) + if artifacts: + for artifact in artifacts: + parts = artifact.get("parts", []) + for part in parts: + if part.get("kind") == "text": + text = part.get("text", "") + if text: + return text, str(uuid4()), parts + + # Fall back to history - get the last agent message + history = result.get("history", []) + for msg in reversed(history): + if msg.get("role") == "agent": + parts = msg.get("parts", []) + message_id = msg.get("messageId", str(uuid4())) + full_text = "" + for part in parts: + if part.get("kind") == "text": + full_text += part.get("text", "") + if full_text: + return full_text, message_id, parts + + # Fall back to message field (original format) + message = result.get("message", {}) + if message: + parts = message.get("parts", []) + message_id = message.get("messageId", str(uuid4())) + full_text = "" + for part in parts: + if part.get("kind") == "text": + full_text += part.get("text", "") + return full_text, message_id, parts + + return "", str(uuid4()), [] + + @staticmethod + async def fake_streaming_from_response( + response_data: Dict[str, Any], + request_id: str, + chunk_size: int = 50, + delay_ms: int = 10, + ) -> AsyncIterator[Dict[str, Any]]: + """ + Convert a non-streaming A2A response into fake streaming chunks. + + Emits proper A2A streaming events: + 1. Task event (kind: "task") - Initial task with status "submitted" + 2. Status update (kind: "status-update") - Status "working" + 3. Artifact update chunks (kind: "artifact-update") - Content delivery in chunks + 4. Status update (kind: "status-update") - Final "completed" status + + Args: + response_data: Non-streaming A2A response dict (completed task) + request_id: A2A JSON-RPC request ID + chunk_size: Number of characters per chunk (default: 50) + delay_ms: Delay between chunks in milliseconds (default: 10) + + Yields: + A2A streaming response events + """ + # Extract the response text from completed task + full_text, message_id, parts = PydanticAITransformation._extract_response_text( + response_data + ) + + # Extract input message from raw response for history + result = response_data.get("result", {}) + history = result.get("history", []) + input_message = {} + for msg in history: + if msg.get("role") == "user": + input_message = msg + break + + # Generate IDs for streaming events + task_id = str(uuid4()) + context_id = str(uuid4()) + artifact_id = str(uuid4()) + input_message_id = input_message.get("messageId", str(uuid4())) + + # 1. Emit initial task event (kind: "task", status: "submitted") + # Format matches A2ACompletionBridgeTransformation.create_task_event + task_event = { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "contextId": context_id, + "history": [ + { + "contextId": context_id, + "kind": "message", + "messageId": input_message_id, + "parts": input_message.get("parts", [{"kind": "text", "text": ""}]), + "role": "user", + "taskId": task_id, + } + ], + "id": task_id, + "kind": "task", + "status": { + "state": "submitted", + }, + }, + } + yield task_event + + # 2. Emit status update (kind: "status-update", status: "working") + # Format matches A2ACompletionBridgeTransformation.create_status_update_event + working_event = { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "contextId": context_id, + "final": False, + "kind": "status-update", + "status": { + "state": "working", + }, + "taskId": task_id, + }, + } + yield working_event + + # Small delay to simulate processing + await asyncio.sleep(delay_ms / 1000.0) + + # 3. Emit artifact update chunks (kind: "artifact-update") + # Format matches A2ACompletionBridgeTransformation.create_artifact_update_event + if full_text: + # Split text into chunks + for i in range(0, len(full_text), chunk_size): + chunk_text = full_text[i:i + chunk_size] + is_last_chunk = (i + chunk_size) >= len(full_text) + + artifact_event = { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "contextId": context_id, + "kind": "artifact-update", + "taskId": task_id, + "artifact": { + "artifactId": artifact_id, + "parts": [ + { + "kind": "text", + "text": chunk_text, + } + ], + }, + }, + } + yield artifact_event + + # Add delay between chunks (except for last chunk) + if not is_last_chunk: + await asyncio.sleep(delay_ms / 1000.0) + + # 4. Emit final status update (kind: "status-update", status: "completed", final: true) + completed_event = { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "contextId": context_id, + "final": True, + "kind": "status-update", + "status": { + "state": "completed", + }, + "taskId": task_id, + }, + } + yield completed_event + + verbose_logger.info( + f"Pydantic AI: Fake streaming completed for request_id={request_id}" + ) + + diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 56035dad68d..612bec239ba 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -457,6 +457,24 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): raw_response.usage ), ) + + # Preserve hidden params from the ResponsesAPIResponse, especially the headers + # which contain important provider information like x-request-id + raw_response_hidden_params = getattr(raw_response, "_hidden_params", {}) + if raw_response_hidden_params: + if not hasattr(model_response, "_hidden_params") or model_response._hidden_params is None: + model_response._hidden_params = {} + # Merge the raw_response hidden params with model_response hidden params + # Preserve existing keys in model_response but add/override with raw_response params + for key, value in raw_response_hidden_params.items(): + if key == "additional_headers" and key in model_response._hidden_params: + # Merge additional_headers to preserve both sets + existing_additional_headers = model_response._hidden_params.get("additional_headers", {}) + merged_headers = {**value, **existing_additional_headers} + model_response._hidden_params[key] = merged_headers + else: + model_response._hidden_params[key] = value + return model_response def get_model_response_iterator( diff --git a/litellm/images/main.py b/litellm/images/main.py index 4aae96bf715..b711aa31c05 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -1,7 +1,11 @@ import asyncio import contextvars +import importlib from functools import partial -from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, cast, overload +from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Literal, Optional, Union, cast, overload + +if TYPE_CHECKING: + from litellm.images.utils import ImageEditRequestUtils import httpx @@ -50,7 +54,20 @@ from litellm.utils import ( get_optional_params_image_gen, ) -from .utils import ImageEditRequestUtils +# Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls +_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None + + +def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils": + """Get ImageEditRequestUtils, loading it lazily if needed.""" + global _ImageEditRequestUtils_cache + if _ImageEditRequestUtils_cache is None: + # Access via module to trigger __getattr__ if not cached + module = importlib.import_module(__name__) + _ImageEditRequestUtils_cache = module.ImageEditRequestUtils + assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker + return _ImageEditRequestUtils_cache + ##### Image Generation ####################### @@ -702,6 +719,59 @@ def image_edit( custom_llm_provider=custom_llm_provider, ) + # Check for custom provider + if custom_llm_provider in litellm._custom_providers: + custom_handler: Optional[CustomLLM] = None + for item in litellm.custom_provider_map: + if item["provider"] == custom_llm_provider: + custom_handler = item["custom_handler"] + + if custom_handler is None: + raise LiteLLMUnknownProvider( + model=model, custom_llm_provider=custom_llm_provider + ) + + model_response = ImageResponse() + + if _is_async: + async_custom_client: Optional[AsyncHTTPHandler] = None + if kwargs.get("client") is not None and isinstance( + kwargs.get("client"), AsyncHTTPHandler + ): + async_custom_client = kwargs.get("client") + + return custom_handler.aimage_edit( + model=model, + image=images, + prompt=prompt, + model_response=model_response, + api_key=kwargs.get("api_key"), + api_base=kwargs.get("api_base"), + optional_params=kwargs, + logging_obj=litellm_logging_obj, + timeout=timeout, + client=async_custom_client, + ) + else: + custom_client: Optional[HTTPHandler] = None + if kwargs.get("client") is not None and isinstance( + kwargs.get("client"), HTTPHandler + ): + custom_client = kwargs.get("client") + + return custom_handler.image_edit( + model=model, + image=images, + prompt=prompt, + model_response=model_response, + api_key=kwargs.get("api_key"), + api_base=kwargs.get("api_base"), + optional_params=kwargs, + logging_obj=litellm_logging_obj, + timeout=timeout, + client=custom_client, + ) + # get provider config image_edit_provider_config: Optional[BaseImageEditConfig] = ( ProviderConfigManager.get_provider_image_edit_config( @@ -716,15 +786,17 @@ def image_edit( local_vars.update(kwargs) # Get ImageEditOptionalRequestParams with only valid parameters image_edit_optional_params: ImageEditOptionalRequestParams = ( - ImageEditRequestUtils.get_requested_image_edit_optional_param(local_vars) + _get_ImageEditRequestUtils().get_requested_image_edit_optional_param(local_vars) ) # Get optional parameters for the responses API image_edit_request_params: Dict = ( - ImageEditRequestUtils.get_optional_params_image_edit( + _get_ImageEditRequestUtils().get_optional_params_image_edit( model=model, image_edit_provider_config=image_edit_provider_config, image_edit_optional_params=image_edit_optional_params, + drop_params=kwargs.get("drop_params"), + additional_drop_params=kwargs.get("additional_drop_params"), ) ) @@ -845,3 +917,15 @@ async def aimage_edit( completion_kwargs=local_vars, extra_kwargs=kwargs, ) + + +def __getattr__(name: str) -> Any: + """Lazy import handler for images.main module""" + if name == "ImageEditRequestUtils": + # Lazy load ImageEditRequestUtils to avoid heavy import from images.utils at module load time + from .utils import ImageEditRequestUtils as _ImageEditRequestUtils + # Cache it in the module's __dict__ for subsequent accesses + module = importlib.import_module(__name__) + module.__dict__["ImageEditRequestUtils"] = _ImageEditRequestUtils + return _ImageEditRequestUtils + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/images/utils.py b/litellm/images/utils.py index 7b1875c4932..fdf240ba2af 100644 --- a/litellm/images/utils.py +++ b/litellm/images/utils.py @@ -1,5 +1,5 @@ from io import BufferedReader, BytesIO -from typing import Any, Dict, cast, get_type_hints +from typing import Any, Dict, List, Optional, cast, get_type_hints import litellm from litellm.litellm_core_utils.token_counter import get_image_type @@ -14,41 +14,53 @@ class ImageEditRequestUtils: model: str, image_edit_provider_config: BaseImageEditConfig, image_edit_optional_params: ImageEditOptionalRequestParams, + drop_params: Optional[bool] = None, + additional_drop_params: Optional[List[str]] = None, ) -> Dict: """ Get optional parameters for the image edit API. Args: - params: Dictionary of all parameters model: The model name image_edit_provider_config: The provider configuration for image edit API + image_edit_optional_params: The optional parameters for the image edit API + drop_params: If True, silently drop unsupported parameters instead of raising + additional_drop_params: List of additional parameter names to drop Returns: A dictionary of supported parameters for the image edit API """ - # Remove None values and internal parameters - - # Get supported parameters for the model supported_params = image_edit_provider_config.get_supported_openai_params(model) - # Check for unsupported parameters + should_drop = litellm.drop_params is True or drop_params is True + + filtered_optional_params = dict(image_edit_optional_params) + if additional_drop_params: + for param in additional_drop_params: + filtered_optional_params.pop(param, None) + unsupported_params = [ param - for param in image_edit_optional_params + for param in filtered_optional_params if param not in supported_params ] if unsupported_params: - raise litellm.UnsupportedParamsError( - model=model, - message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}", - ) + if should_drop: + for param in unsupported_params: + filtered_optional_params.pop(param, None) + else: + raise litellm.UnsupportedParamsError( + model=model, + message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}", + ) - # Map parameters to provider-specific format mapped_params = image_edit_provider_config.map_openai_params( - image_edit_optional_params=image_edit_optional_params, + image_edit_optional_params=cast( + ImageEditOptionalRequestParams, filtered_optional_params + ), model=model, - drop_params=litellm.drop_params, + drop_params=should_drop, ) return mapped_params diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 6488128b215..6771999cd35 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -16,7 +16,6 @@ from typing import ( from pydantic import BaseModel from litellm._logging import verbose_logger -from litellm.caching.caching import DualCache from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER from litellm.types.integrations.argilla import ArgillaItem from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest @@ -33,6 +32,7 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: + from litellm.caching.caching import DualCache from opentelemetry.trace import Span as _Span from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -334,7 +334,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, - cache: DualCache, + cache: "DualCache", data: dict, call_type: CallTypesLiteral, ) -> Optional[ diff --git a/litellm/integrations/gcs_bucket/Readme.md b/litellm/integrations/gcs_bucket/Readme.md index 2ab0b23353b..6808823c925 100644 --- a/litellm/integrations/gcs_bucket/Readme.md +++ b/litellm/integrations/gcs_bucket/Readme.md @@ -8,5 +8,5 @@ This folder contains the GCS Bucket Logging integration for LiteLLM Gateway. - `gcs_bucket_base.py`: This file contains the GCSBucketBase class which handles Authentication for GCS Buckets ## Further Reading -- [Doc setting up GCS Bucket Logging on LiteLLM Proxy (Gateway)](https://docs.litellm.ai/docs/proxy/bucket) +- [Doc setting up GCS Bucket Logging on LiteLLM Proxy (Gateway)](https://docs.litellm.ai/docs/observability/gcs_bucket_integration) - [Doc on Key / Team Based logging with GCS](https://docs.litellm.ai/docs/proxy/team_logging) \ No newline at end of file diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 4ce818f0cef..20f1357a1c8 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -815,7 +815,20 @@ class PrometheusLogger(CustomLogger): user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[ "metadata" ].get("user_api_key_auth_metadata") + + # Include top-level metadata fields (excluding nested dictionaries) + # This allows accessing fields like requester_ip_address from top-level metadata + top_level_metadata = standard_logging_payload.get("metadata", {}) + top_level_fields: Dict[str, Any] = {} + if isinstance(top_level_metadata, dict): + top_level_fields = { + k: v + for k, v in top_level_metadata.items() + if not isinstance(v, dict) # Exclude nested dicts to avoid conflicts + } + combined_metadata: Dict[str, Any] = { + **top_level_fields, # Include top-level fields first **(_requester_metadata if _requester_metadata else {}), **(user_api_key_auth_metadata if user_api_key_auth_metadata else {}), } diff --git a/litellm/interactions/__init__.py b/litellm/interactions/__init__.py new file mode 100644 index 00000000000..e1125b649a6 --- /dev/null +++ b/litellm/interactions/__init__.py @@ -0,0 +1,68 @@ +""" +LiteLLM Interactions API + +This module provides SDK methods for Google's Interactions API. + +Usage: + import litellm + + # Create an interaction with a model + response = litellm.interactions.create( + model="gemini-2.5-flash", + input="Hello, how are you?" + ) + + # Create an interaction with an agent + response = litellm.interactions.create( + agent="deep-research-pro-preview-12-2025", + input="Research the current state of cancer research" + ) + + # Async version + response = await litellm.interactions.acreate(...) + + # Get an interaction + response = litellm.interactions.get(interaction_id="...") + + # Delete an interaction + result = litellm.interactions.delete(interaction_id="...") + + # Cancel an interaction + result = litellm.interactions.cancel(interaction_id="...") + +Methods: +- create(): Sync create interaction +- acreate(): Async create interaction +- get(): Sync get interaction +- aget(): Async get interaction +- delete(): Sync delete interaction +- adelete(): Async delete interaction +- cancel(): Sync cancel interaction +- acancel(): Async cancel interaction +""" + +from litellm.interactions.main import ( + acancel, + acreate, + adelete, + aget, + cancel, + create, + delete, + get, +) + +__all__ = [ + # Create + "create", + "acreate", + # Get + "get", + "aget", + # Delete + "delete", + "adelete", + # Cancel + "cancel", + "acancel", +] diff --git a/litellm/interactions/http_handler.py b/litellm/interactions/http_handler.py new file mode 100644 index 00000000000..5555b1d7e32 --- /dev/null +++ b/litellm/interactions/http_handler.py @@ -0,0 +1,692 @@ +""" +HTTP Handler for Interactions API requests. + +This module handles the HTTP communication for the Google Interactions API. +""" + +import json +from typing import ( + Any, + AsyncIterator, + Coroutine, + Dict, + Iterator, + Optional, + Union, +) + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.constants import request_timeout +from litellm.interactions.streaming_iterator import ( + InteractionsAPIStreamingIterator, + SyncInteractionsAPIStreamingIterator, +) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, +) +from litellm.types.interactions import ( + CancelInteractionResult, + DeleteInteractionResult, + InteractionInput, + InteractionsAPIOptionalRequestParams, + InteractionsAPIResponse, + InteractionsAPIStreamingResponse, +) +from litellm.types.router import GenericLiteLLMParams + + +class InteractionsHTTPHandler: + """ + HTTP handler for Interactions API requests. + """ + + def _handle_error( + self, + e: Exception, + provider_config: BaseInteractionsAPIConfig, + ) -> Exception: + """Handle errors from HTTP requests.""" + if isinstance(e, httpx.HTTPStatusError): + error_message = e.response.text + status_code = e.response.status_code + headers = dict(e.response.headers) + return provider_config.get_error_class( + error_message=error_message, + status_code=status_code, + headers=headers, + ) + return e + + # ========================================================= + # CREATE INTERACTION + # ========================================================= + + def create_interaction( + self, + interactions_api_config: BaseInteractionsAPIConfig, + optional_params: InteractionsAPIOptionalRequestParams, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + model: Optional[str] = None, + agent: Optional[str] = None, + input: Optional[InteractionInput] = None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[HTTPHandler] = None, + _is_async: bool = False, + stream: Optional[bool] = None, + ) -> Union[ + InteractionsAPIResponse, + Iterator[InteractionsAPIStreamingResponse], + Coroutine[Any, Any, Union[InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]]], + ]: + """ + Create a new interaction (synchronous or async based on _is_async flag). + + Per Google's OpenAPI spec, the endpoint is POST /{api_version}/interactions + """ + if _is_async: + return self.async_create_interaction( + model=model, + agent=agent, + input=input, + interactions_api_config=interactions_api_config, + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + stream=stream, + ) + + if client is None: + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = interactions_api_config.validate_environment( + headers=extra_headers or {}, + model=model or "", + litellm_params=litellm_params, + ) + + api_base = interactions_api_config.get_complete_url( + api_base=litellm_params.api_base or "", + model=model, + agent=agent, + litellm_params=dict(litellm_params), + stream=stream, + ) + + data = interactions_api_config.transform_request( + model=model, + agent=agent, + input=input, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + if extra_body: + data.update(extra_body) + + # Logging + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + if stream: + response = sync_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout or request_timeout, + stream=True, + ) + return self._create_sync_streaming_iterator( + response=response, + model=model, + logging_obj=logging_obj, + interactions_api_config=interactions_api_config, + ) + else: + response = sync_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout or request_timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=interactions_api_config) + + return interactions_api_config.transform_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ) + + async def async_create_interaction( + self, + interactions_api_config: BaseInteractionsAPIConfig, + optional_params: InteractionsAPIOptionalRequestParams, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + model: Optional[str] = None, + agent: Optional[str] = None, + input: Optional[InteractionInput] = None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + stream: Optional[bool] = None, + ) -> Union[InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]]: + """ + Create a new interaction (async version). + """ + if client is None: + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = interactions_api_config.validate_environment( + headers=extra_headers or {}, + model=model or "", + litellm_params=litellm_params, + ) + + api_base = interactions_api_config.get_complete_url( + api_base=litellm_params.api_base or "", + model=model, + agent=agent, + litellm_params=dict(litellm_params), + stream=stream, + ) + + data = interactions_api_config.transform_request( + model=model, + agent=agent, + input=input, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + if extra_body: + data.update(extra_body) + + # Logging + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + if stream: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout or request_timeout, + stream=True, + ) + return self._create_async_streaming_iterator( + response=response, + model=model, + logging_obj=logging_obj, + interactions_api_config=interactions_api_config, + ) + else: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout or request_timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=interactions_api_config) + + return interactions_api_config.transform_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ) + + def _create_sync_streaming_iterator( + self, + response: httpx.Response, + model: Optional[str], + logging_obj: LiteLLMLoggingObj, + interactions_api_config: BaseInteractionsAPIConfig, + ) -> SyncInteractionsAPIStreamingIterator: + """Create a synchronous streaming iterator. + + Google AI's streaming format uses SSE (Server-Sent Events). + Returns a proper streaming iterator that yields chunks as they arrive. + """ + return SyncInteractionsAPIStreamingIterator( + response=response, + model=model, + interactions_api_config=interactions_api_config, + logging_obj=logging_obj, + ) + + def _create_async_streaming_iterator( + self, + response: httpx.Response, + model: Optional[str], + logging_obj: LiteLLMLoggingObj, + interactions_api_config: BaseInteractionsAPIConfig, + ) -> InteractionsAPIStreamingIterator: + """Create an asynchronous streaming iterator. + + Google AI's streaming format uses SSE (Server-Sent Events). + Returns a proper streaming iterator that yields chunks as they arrive. + """ + return InteractionsAPIStreamingIterator( + response=response, + model=model, + interactions_api_config=interactions_api_config, + logging_obj=logging_obj, + ) + + # ========================================================= + # GET INTERACTION + # ========================================================= + + def get_interaction( + self, + interaction_id: str, + interactions_api_config: BaseInteractionsAPIConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[HTTPHandler] = None, + _is_async: bool = False, + ) -> Union[InteractionsAPIResponse, Coroutine[Any, Any, InteractionsAPIResponse]]: + """Get an interaction by ID.""" + if _is_async: + return self.async_get_interaction( + interaction_id=interaction_id, + interactions_api_config=interactions_api_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + timeout=timeout, + ) + + if client is None: + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = interactions_api_config.validate_environment( + headers=extra_headers or {}, + model="", + litellm_params=litellm_params, + ) + + url, params = interactions_api_config.transform_get_interaction_request( + interaction_id=interaction_id, + api_base=litellm_params.api_base or "", + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=interaction_id, + api_key="", + additional_args={"api_base": url, "headers": headers}, + ) + + try: + response = sync_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=interactions_api_config) + + return interactions_api_config.transform_get_interaction_response( + raw_response=response, + logging_obj=logging_obj, + ) + + async def async_get_interaction( + self, + interaction_id: str, + interactions_api_config: BaseInteractionsAPIConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + ) -> InteractionsAPIResponse: + """Get an interaction by ID (async version).""" + if client is None: + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = interactions_api_config.validate_environment( + headers=extra_headers or {}, + model="", + litellm_params=litellm_params, + ) + + url, params = interactions_api_config.transform_get_interaction_request( + interaction_id=interaction_id, + api_base=litellm_params.api_base or "", + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=interaction_id, + api_key="", + additional_args={"api_base": url, "headers": headers}, + ) + + try: + response = await async_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=interactions_api_config) + + return interactions_api_config.transform_get_interaction_response( + raw_response=response, + logging_obj=logging_obj, + ) + + # ========================================================= + # DELETE INTERACTION + # ========================================================= + + def delete_interaction( + self, + interaction_id: str, + interactions_api_config: BaseInteractionsAPIConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[HTTPHandler] = None, + _is_async: bool = False, + ) -> Union[DeleteInteractionResult, Coroutine[Any, Any, DeleteInteractionResult]]: + """Delete an interaction by ID.""" + if _is_async: + return self.async_delete_interaction( + interaction_id=interaction_id, + interactions_api_config=interactions_api_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + timeout=timeout, + ) + + if client is None: + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = interactions_api_config.validate_environment( + headers=extra_headers or {}, + model="", + litellm_params=litellm_params, + ) + + url, data = interactions_api_config.transform_delete_interaction_request( + interaction_id=interaction_id, + api_base=litellm_params.api_base or "", + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=interaction_id, + api_key="", + additional_args={"api_base": url, "headers": headers}, + ) + + try: + response = sync_httpx_client.delete( + url=url, + headers=headers, + timeout=timeout or request_timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=interactions_api_config) + + return interactions_api_config.transform_delete_interaction_response( + raw_response=response, + logging_obj=logging_obj, + interaction_id=interaction_id, + ) + + async def async_delete_interaction( + self, + interaction_id: str, + interactions_api_config: BaseInteractionsAPIConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + ) -> DeleteInteractionResult: + """Delete an interaction by ID (async version).""" + if client is None: + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = interactions_api_config.validate_environment( + headers=extra_headers or {}, + model="", + litellm_params=litellm_params, + ) + + url, data = interactions_api_config.transform_delete_interaction_request( + interaction_id=interaction_id, + api_base=litellm_params.api_base or "", + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=interaction_id, + api_key="", + additional_args={"api_base": url, "headers": headers}, + ) + + try: + response = await async_httpx_client.delete( + url=url, + headers=headers, + timeout=timeout or request_timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=interactions_api_config) + + return interactions_api_config.transform_delete_interaction_response( + raw_response=response, + logging_obj=logging_obj, + interaction_id=interaction_id, + ) + + # ========================================================= + # CANCEL INTERACTION + # ========================================================= + + def cancel_interaction( + self, + interaction_id: str, + interactions_api_config: BaseInteractionsAPIConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[HTTPHandler] = None, + _is_async: bool = False, + ) -> Union[CancelInteractionResult, Coroutine[Any, Any, CancelInteractionResult]]: + """Cancel an interaction by ID.""" + if _is_async: + return self.async_cancel_interaction( + interaction_id=interaction_id, + interactions_api_config=interactions_api_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + timeout=timeout, + ) + + if client is None: + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = interactions_api_config.validate_environment( + headers=extra_headers or {}, + model="", + litellm_params=litellm_params, + ) + + url, data = interactions_api_config.transform_cancel_interaction_request( + interaction_id=interaction_id, + api_base=litellm_params.api_base or "", + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=interaction_id, + api_key="", + additional_args={"api_base": url, "headers": headers}, + ) + + try: + response = sync_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout or request_timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=interactions_api_config) + + return interactions_api_config.transform_cancel_interaction_response( + raw_response=response, + logging_obj=logging_obj, + ) + + async def async_cancel_interaction( + self, + interaction_id: str, + interactions_api_config: BaseInteractionsAPIConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + ) -> CancelInteractionResult: + """Cancel an interaction by ID (async version).""" + if client is None: + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = interactions_api_config.validate_environment( + headers=extra_headers or {}, + model="", + litellm_params=litellm_params, + ) + + url, data = interactions_api_config.transform_cancel_interaction_request( + interaction_id=interaction_id, + api_base=litellm_params.api_base or "", + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=interaction_id, + api_key="", + additional_args={"api_base": url, "headers": headers}, + ) + + try: + response = await async_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout or request_timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=interactions_api_config) + + return interactions_api_config.transform_cancel_interaction_response( + raw_response=response, + logging_obj=logging_obj, + ) + + +# Initialize the HTTP handler singleton +interactions_http_handler = InteractionsHTTPHandler() + diff --git a/litellm/interactions/main.py b/litellm/interactions/main.py new file mode 100644 index 00000000000..9fb58fc73d6 --- /dev/null +++ b/litellm/interactions/main.py @@ -0,0 +1,621 @@ +""" +LiteLLM Interactions API - Main Module + +Per OpenAPI spec (https://ai.google.dev/static/api/interactions.openapi.json): +- Create interaction: POST /{api_version}/interactions +- Get interaction: GET /{api_version}/interactions/{interaction_id} +- Delete interaction: DELETE /{api_version}/interactions/{interaction_id} + +Usage: + import litellm + + # Create an interaction with a model + response = litellm.interactions.create( + model="gemini-2.5-flash", + input="Hello, how are you?" + ) + + # Create an interaction with an agent + response = litellm.interactions.create( + agent="deep-research-pro-preview-12-2025", + input="Research the current state of cancer research" + ) + + # Async version + response = await litellm.interactions.acreate(...) + + # Get an interaction + response = litellm.interactions.get(interaction_id="...") + + # Delete an interaction + result = litellm.interactions.delete(interaction_id="...") +""" + +import asyncio +import contextvars +from functools import partial +from typing import ( + Any, + AsyncIterator, + Coroutine, + Dict, + Iterator, + List, + Optional, + Union, +) + +import httpx + +import litellm +from litellm.interactions.http_handler import interactions_http_handler +from litellm.interactions.utils import ( + InteractionsAPIRequestUtils, + get_provider_interactions_api_config, +) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.types.interactions import ( + CancelInteractionResult, + DeleteInteractionResult, + InteractionInput, + InteractionsAPIResponse, + InteractionsAPIStreamingResponse, + InteractionTool, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.utils import client + +# ============================================================ +# SDK Methods - CREATE INTERACTION +# ============================================================ + + +@client +async def acreate( + # Model or Agent (one required per OpenAPI spec) + model: Optional[str] = None, + agent: Optional[str] = None, + # Input (required) + input: Optional[InteractionInput] = None, + # Tools (for model interactions) + tools: Optional[List[InteractionTool]] = None, + # System instruction + system_instruction: Optional[str] = None, + # Generation config + generation_config: Optional[Dict[str, Any]] = None, + # Streaming + stream: Optional[bool] = None, + # Storage + store: Optional[bool] = None, + # Background execution + background: Optional[bool] = None, + # Response format + response_modalities: Optional[List[str]] = None, + response_format: Optional[Dict[str, Any]] = None, + response_mime_type: Optional[str] = None, + # Continuation + previous_interaction_id: Optional[str] = None, + # Extra params + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + # LiteLLM params + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> Union[InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]]: + """ + Async: Create a new interaction using Google's Interactions API. + + Per OpenAPI spec, provide either `model` or `agent`. + + Args: + model: The model to use (e.g., "gemini-2.5-flash") + agent: The agent to use (e.g., "deep-research-pro-preview-12-2025") + input: The input content (string, content object, or list) + tools: Tools available for the model + system_instruction: System instruction for the interaction + generation_config: Generation configuration + stream: Whether to stream the response + store: Whether to store the response for later retrieval + background: Whether to run in background + response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO) + response_format: JSON schema for response format + response_mime_type: MIME type of the response + previous_interaction_id: ID of previous interaction for continuation + extra_headers: Additional headers + extra_body: Additional body parameters + timeout: Request timeout + custom_llm_provider: Override the LLM provider + + Returns: + InteractionsAPIResponse or async iterator for streaming + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["acreate_interaction"] = True + + if custom_llm_provider is None and model: + _, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, api_base=kwargs.get("api_base", None) + ) + elif custom_llm_provider is None: + custom_llm_provider = "gemini" + + func = partial( + create, + model=model, + agent=agent, + input=input, + tools=tools, + system_instruction=system_instruction, + generation_config=generation_config, + stream=stream, + store=store, + background=background, + response_modalities=response_modalities, + response_format=response_format, + response_mime_type=response_mime_type, + previous_interaction_id=previous_interaction_id, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response # type: ignore + except Exception as e: + raise litellm.exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def create( + # Model or Agent (one required per OpenAPI spec) + model: Optional[str] = None, + agent: Optional[str] = None, + # Input (required) + input: Optional[InteractionInput] = None, + # Tools (for model interactions) + tools: Optional[List[InteractionTool]] = None, + # System instruction + system_instruction: Optional[str] = None, + # Generation config + generation_config: Optional[Dict[str, Any]] = None, + # Streaming + stream: Optional[bool] = None, + # Storage + store: Optional[bool] = None, + # Background execution + background: Optional[bool] = None, + # Response format + response_modalities: Optional[List[str]] = None, + response_format: Optional[Dict[str, Any]] = None, + response_mime_type: Optional[str] = None, + # Continuation + previous_interaction_id: Optional[str] = None, + # Extra params + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + # LiteLLM params + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> Union[ + InteractionsAPIResponse, + Iterator[InteractionsAPIStreamingResponse], + Coroutine[Any, Any, Union[InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]]], +]: + """ + Sync: Create a new interaction using Google's Interactions API. + + Per OpenAPI spec, provide either `model` or `agent`. + + Args: + model: The model to use (e.g., "gemini-2.5-flash") + agent: The agent to use (e.g., "deep-research-pro-preview-12-2025") + input: The input content (string, content object, or list) + tools: Tools available for the model + system_instruction: System instruction for the interaction + generation_config: Generation configuration + stream: Whether to stream the response + store: Whether to store the response for later retrieval + background: Whether to run in background + response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO) + response_format: JSON schema for response format + response_mime_type: MIME type of the response + previous_interaction_id: ID of previous interaction for continuation + extra_headers: Additional headers + extra_body: Additional body parameters + timeout: Request timeout + custom_llm_provider: Override the LLM provider + + Returns: + InteractionsAPIResponse or iterator for streaming + """ + local_vars = locals() + + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("acreate_interaction", False) is True + + litellm_params = GenericLiteLLMParams(**kwargs) + + if model: + model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + else: + custom_llm_provider = custom_llm_provider or "gemini" + + interactions_api_config = get_provider_interactions_api_config( + provider=custom_llm_provider, + model=model, + ) + + if interactions_api_config is None: + raise ValueError( + f"Interactions API is not supported for provider: {custom_llm_provider}. " + "Currently only 'gemini' is supported." + ) + + # Get optional params using utility (similar to responses API pattern) + local_vars.update(kwargs) + optional_params = InteractionsAPIRequestUtils.get_requested_interactions_api_optional_params( + local_vars + ) + + litellm_logging_obj.update_environment_variables( + model=model, + optional_params=dict(optional_params), + litellm_params={"litellm_call_id": litellm_call_id}, + custom_llm_provider=custom_llm_provider, + ) + + response = interactions_http_handler.create_interaction( + model=model, + agent=agent, + input=input, + interactions_api_config=interactions_api_config, + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + _is_async=_is_async, + stream=stream, + ) + + return response + except Exception as e: + raise litellm.exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +# ============================================================ +# SDK Methods - GET INTERACTION +# ============================================================ + + +@client +async def aget( + interaction_id: str, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> InteractionsAPIResponse: + """Async: Get an interaction by its ID.""" + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["aget_interaction"] = True + + func = partial( + get, + interaction_id=interaction_id, + extra_headers=extra_headers, + timeout=timeout, + custom_llm_provider=custom_llm_provider or "gemini", + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response # type: ignore + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=custom_llm_provider or "gemini", + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def get( + interaction_id: str, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> Union[InteractionsAPIResponse, Coroutine[Any, Any, InteractionsAPIResponse]]: + """Sync: Get an interaction by its ID.""" + local_vars = locals() + custom_llm_provider = custom_llm_provider or "gemini" + + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("aget_interaction", False) is True + + litellm_params = GenericLiteLLMParams(**kwargs) + + interactions_api_config = get_provider_interactions_api_config( + provider=custom_llm_provider, + ) + + if interactions_api_config is None: + raise ValueError(f"Interactions API not supported for: {custom_llm_provider}") + + litellm_logging_obj.update_environment_variables( + model=None, + optional_params={"interaction_id": interaction_id}, + litellm_params={"litellm_call_id": litellm_call_id}, + custom_llm_provider=custom_llm_provider, + ) + + return interactions_http_handler.get_interaction( + interaction_id=interaction_id, + interactions_api_config=interactions_api_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + timeout=timeout, + _is_async=_is_async, + ) + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +# ============================================================ +# SDK Methods - DELETE INTERACTION +# ============================================================ + + +@client +async def adelete( + interaction_id: str, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> DeleteInteractionResult: + """Async: Delete an interaction by its ID.""" + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["adelete_interaction"] = True + + func = partial( + delete, + interaction_id=interaction_id, + extra_headers=extra_headers, + timeout=timeout, + custom_llm_provider=custom_llm_provider or "gemini", + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response # type: ignore + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=custom_llm_provider or "gemini", + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def delete( + interaction_id: str, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> Union[DeleteInteractionResult, Coroutine[Any, Any, DeleteInteractionResult]]: + """Sync: Delete an interaction by its ID.""" + local_vars = locals() + custom_llm_provider = custom_llm_provider or "gemini" + + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("adelete_interaction", False) is True + + litellm_params = GenericLiteLLMParams(**kwargs) + + interactions_api_config = get_provider_interactions_api_config( + provider=custom_llm_provider, + ) + + if interactions_api_config is None: + raise ValueError(f"Interactions API not supported for: {custom_llm_provider}") + + litellm_logging_obj.update_environment_variables( + model=None, + optional_params={"interaction_id": interaction_id}, + litellm_params={"litellm_call_id": litellm_call_id}, + custom_llm_provider=custom_llm_provider, + ) + + return interactions_http_handler.delete_interaction( + interaction_id=interaction_id, + interactions_api_config=interactions_api_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + timeout=timeout, + _is_async=_is_async, + ) + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +# ============================================================ +# SDK Methods - CANCEL INTERACTION +# ============================================================ + + +@client +async def acancel( + interaction_id: str, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> CancelInteractionResult: + """Async: Cancel an interaction by its ID.""" + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["acancel_interaction"] = True + + func = partial( + cancel, + interaction_id=interaction_id, + extra_headers=extra_headers, + timeout=timeout, + custom_llm_provider=custom_llm_provider or "gemini", + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response # type: ignore + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=custom_llm_provider or "gemini", + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def cancel( + interaction_id: str, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> Union[CancelInteractionResult, Coroutine[Any, Any, CancelInteractionResult]]: + """Sync: Cancel an interaction by its ID.""" + local_vars = locals() + custom_llm_provider = custom_llm_provider or "gemini" + + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("acancel_interaction", False) is True + + litellm_params = GenericLiteLLMParams(**kwargs) + + interactions_api_config = get_provider_interactions_api_config( + provider=custom_llm_provider, + ) + + if interactions_api_config is None: + raise ValueError(f"Interactions API not supported for: {custom_llm_provider}") + + litellm_logging_obj.update_environment_variables( + model=None, + optional_params={"interaction_id": interaction_id}, + litellm_params={"litellm_call_id": litellm_call_id}, + custom_llm_provider=custom_llm_provider, + ) + + return interactions_http_handler.cancel_interaction( + interaction_id=interaction_id, + interactions_api_config=interactions_api_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + timeout=timeout, + _is_async=_is_async, + ) + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) diff --git a/litellm/interactions/streaming_iterator.py b/litellm/interactions/streaming_iterator.py new file mode 100644 index 00000000000..ad18477663c --- /dev/null +++ b/litellm/interactions/streaming_iterator.py @@ -0,0 +1,266 @@ +""" +Streaming iterators for the Interactions API. + +This module provides streaming iterators that properly stream SSE responses +from the Google Interactions API, similar to the responses API streaming iterator. +""" + +import asyncio +import json +from datetime import datetime +from typing import Any, Dict, Iterator, Optional + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.constants import STREAM_SSE_DONE_STRING +from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base +from litellm.litellm_core_utils.thread_pool_executor import executor +from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig +from litellm.types.interactions import ( + InteractionsAPIResponse, + InteractionsAPIStreamingResponse, +) +from litellm.utils import CustomStreamWrapper + + +class BaseInteractionsAPIStreamingIterator: + """ + Base class for streaming iterators that process responses from the Interactions API. + + This class contains shared logic for both synchronous and asynchronous iterators. + """ + + def __init__( + self, + response: httpx.Response, + model: Optional[str], + interactions_api_config: BaseInteractionsAPIConfig, + logging_obj: LiteLLMLoggingObj, + litellm_metadata: Optional[Dict[str, Any]] = None, + custom_llm_provider: Optional[str] = None, + ): + self.response = response + self.model = model + self.logging_obj = logging_obj + self.finished = False + self.interactions_api_config = interactions_api_config + self.completed_response: Optional[InteractionsAPIStreamingResponse] = None + self.start_time = datetime.now() + + # set request kwargs + self.litellm_metadata = litellm_metadata + self.custom_llm_provider = custom_llm_provider + + # set hidden params for response headers + _api_base = get_api_base( + model=model or "", + optional_params=self.logging_obj.model_call_details.get( + "litellm_params", {} + ), + ) + _model_info: Dict = litellm_metadata.get("model_info", {}) if litellm_metadata else {} + self._hidden_params = { + "model_id": _model_info.get("id", None), + "api_base": _api_base, + } + self._hidden_params["additional_headers"] = process_response_headers( + self.response.headers or {} + ) + + def _process_chunk(self, chunk: str) -> Optional[InteractionsAPIStreamingResponse]: + """Process a single chunk of data from the stream.""" + if not chunk: + return None + + # Handle SSE format (data: {...}) + stripped_chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk) + if stripped_chunk is None: + return None + + # Handle "[DONE]" marker + if stripped_chunk == STREAM_SSE_DONE_STRING: + self.finished = True + return None + + try: + # Parse the JSON chunk + parsed_chunk = json.loads(stripped_chunk) + + # Format as InteractionsAPIStreamingResponse + if isinstance(parsed_chunk, dict): + streaming_response = self.interactions_api_config.transform_streaming_response( + model=self.model, + parsed_chunk=parsed_chunk, + logging_obj=self.logging_obj, + ) + + # Store the completed response (check for status=completed) + if ( + streaming_response + and getattr(streaming_response, "status", None) == "completed" + ): + self.completed_response = streaming_response + self._handle_logging_completed_response() + + return streaming_response + + return None + except json.JSONDecodeError: + # If we can't parse the chunk, continue + verbose_logger.debug(f"Failed to parse streaming chunk: {stripped_chunk[:200]}...") + return None + + def _handle_logging_completed_response(self): + """Base implementation - should be overridden by subclasses.""" + pass + + +class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator): + """ + Async iterator for processing streaming responses from the Interactions API. + """ + + def __init__( + self, + response: httpx.Response, + model: Optional[str], + interactions_api_config: BaseInteractionsAPIConfig, + logging_obj: LiteLLMLoggingObj, + litellm_metadata: Optional[Dict[str, Any]] = None, + custom_llm_provider: Optional[str] = None, + ): + super().__init__( + response=response, + model=model, + interactions_api_config=interactions_api_config, + logging_obj=logging_obj, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + ) + self.stream_iterator = response.aiter_lines() + + def __aiter__(self): + return self + + async def __anext__(self) -> InteractionsAPIStreamingResponse: + try: + while True: + # Get the next chunk from the stream + try: + chunk = await self.stream_iterator.__anext__() + except StopAsyncIteration: + self.finished = True + raise StopAsyncIteration + + result = self._process_chunk(chunk) + + if self.finished: + raise StopAsyncIteration + elif result is not None: + return result + # If result is None, continue the loop to get the next chunk + + except httpx.HTTPError as e: + # Handle HTTP errors + self.finished = True + raise e + + def _handle_logging_completed_response(self): + """Handle logging for completed responses in async context.""" + import copy + logging_response = copy.deepcopy(self.completed_response) + + asyncio.create_task( + self.logging_obj.async_success_handler( + result=logging_response, + start_time=self.start_time, + end_time=datetime.now(), + cache_hit=None, + ) + ) + + executor.submit( + self.logging_obj.success_handler, + result=logging_response, + cache_hit=None, + start_time=self.start_time, + end_time=datetime.now(), + ) + + +class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator): + """ + Synchronous iterator for processing streaming responses from the Interactions API. + """ + + def __init__( + self, + response: httpx.Response, + model: Optional[str], + interactions_api_config: BaseInteractionsAPIConfig, + logging_obj: LiteLLMLoggingObj, + litellm_metadata: Optional[Dict[str, Any]] = None, + custom_llm_provider: Optional[str] = None, + ): + super().__init__( + response=response, + model=model, + interactions_api_config=interactions_api_config, + logging_obj=logging_obj, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + ) + self.stream_iterator = response.iter_lines() + + def __iter__(self): + return self + + def __next__(self) -> InteractionsAPIStreamingResponse: + try: + while True: + # Get the next chunk from the stream + try: + chunk = next(self.stream_iterator) + except StopIteration: + self.finished = True + raise StopIteration + + result = self._process_chunk(chunk) + + if self.finished: + raise StopIteration + elif result is not None: + return result + # If result is None, continue the loop to get the next chunk + + except httpx.HTTPError as e: + # Handle HTTP errors + self.finished = True + raise e + + def _handle_logging_completed_response(self): + """Handle logging for completed responses in sync context.""" + import copy + logging_response = copy.deepcopy(self.completed_response) + + run_async_function( + async_function=self.logging_obj.async_success_handler, + result=logging_response, + start_time=self.start_time, + end_time=datetime.now(), + cache_hit=None, + ) + + executor.submit( + self.logging_obj.success_handler, + result=logging_response, + cache_hit=None, + start_time=self.start_time, + end_time=datetime.now(), + ) + diff --git a/litellm/interactions/utils.py b/litellm/interactions/utils.py new file mode 100644 index 00000000000..4fc40916e52 --- /dev/null +++ b/litellm/interactions/utils.py @@ -0,0 +1,84 @@ +""" +Utility functions for Interactions API. +""" + +from typing import Any, Dict, Optional, cast + +from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig +from litellm.types.interactions import InteractionsAPIOptionalRequestParams + +# Valid optional parameter keys per OpenAPI spec +INTERACTIONS_API_OPTIONAL_PARAMS = { + "tools", + "system_instruction", + "generation_config", + "stream", + "store", + "background", + "response_modalities", + "response_format", + "response_mime_type", + "previous_interaction_id", + "agent_config", +} + + +def get_provider_interactions_api_config( + provider: str, + model: Optional[str] = None, +) -> Optional[BaseInteractionsAPIConfig]: + """ + Get the interactions API config for the given provider. + + Args: + provider: The LLM provider name + model: Optional model name + + Returns: + The provider-specific interactions API config, or None if not supported + """ + from litellm.types.utils import LlmProviders + + if provider == LlmProviders.GEMINI.value or provider == "gemini": + from litellm.llms.gemini.interactions.transformation import ( + GoogleAIStudioInteractionsConfig, + ) + return GoogleAIStudioInteractionsConfig() + + return None + + +class InteractionsAPIRequestUtils: + """Helper utils for constructing Interactions API requests.""" + + @staticmethod + def get_requested_interactions_api_optional_params( + params: Dict[str, Any], + ) -> InteractionsAPIOptionalRequestParams: + """ + Filter parameters to only include valid optional params per OpenAPI spec. + + Args: + params: Dictionary of parameters to filter (typically from locals()) + + Returns: + Dict with only the valid optional parameters + """ + from litellm.utils import PreProcessNonDefaultParams + + custom_llm_provider = params.pop("custom_llm_provider", None) + special_params = params.pop("kwargs", {}) + additional_drop_params = params.pop("additional_drop_params", None) + + non_default_params = ( + PreProcessNonDefaultParams.base_pre_process_non_default_params( + passed_params=params, + special_params=special_params, + custom_llm_provider=custom_llm_provider, + additional_drop_params=additional_drop_params, + default_param_values={k: None for k in INTERACTIONS_API_OPTIONAL_PARAMS}, + additional_endpoint_specific_params=["input", "model", "agent"], + ) + ) + + return cast(InteractionsAPIOptionalRequestParams, non_default_params) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f2f6a785969..ba516c1b78b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -917,9 +917,11 @@ class Logging(LiteLLMLoggingBaseClass): raw_request_body=self._get_raw_request_body( additional_args.get("complete_input_dict", {}) ), + # NOTE: setting ignore_sensitive_headers to True will cause + # the Authorization header to be leaked when calls to the health + # endpoint are made and fail. raw_request_headers=self._get_masked_headers( additional_args.get("headers", {}) or {}, - ignore_sensitive_headers=True, ), error=None, ) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 5a50806218f..59d2a8a8dd0 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -430,6 +430,18 @@ def convert_to_model_response_object( # noqa: PLR0915 if hidden_params is None: hidden_params = {} + + # Preserve existing additional_headers if they contain important provider headers + # For responses API, additional_headers may already be set with LLM provider headers + existing_additional_headers = hidden_params.get("additional_headers", {}) + if existing_additional_headers and _response_headers is None: + # Keep existing headers when _response_headers is None (responses API case) + additional_headers = existing_additional_headers + else: + # Merge new headers with existing ones + if existing_additional_headers: + additional_headers.update(existing_additional_headers) + hidden_params["additional_headers"] = additional_headers ### CHECK IF ERROR IN RESPONSE ### - openrouter returns these in the dictionary diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index d2c91f4a841..ca2a092dbc8 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -689,7 +689,14 @@ def _get_image_mime_type_from_url(url: str) -> Optional[str]: video/mpegps video/flv """ + from urllib.parse import urlparse + url = url.lower() + + # Parse URL to extract path without query parameters + # This handles URLs like: https://example.com/image.jpg?signature=... + parsed = urlparse(url) + path = parsed.path # Map file extensions to mime types mime_types = { @@ -717,7 +724,7 @@ def _get_image_mime_type_from_url(url: str) -> Optional[str]: # Check each extension group against the URL for extensions, mime_type in mime_types.items(): - if any(url.endswith(ext) for ext in extensions): + if any(path.endswith(ext) for ext in extensions): return mime_type return None diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 094b5842f07..6f53bf65714 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -253,20 +253,39 @@ class AnthropicMessagesHandler(BaseTranslation): task_mappings: List[Tuple[int, Optional[int]]] = [] # Track (content_index, None) for each text - response_content = response.get("content", []) + # Handle both dict and object responses + response_content: List[Any] = [] + if isinstance(response, dict): + response_content = response.get("content", []) or [] + elif hasattr(response, "content"): + content = getattr(response, "content", None) + response_content = content or [] + else: + response_content = [] + if not response_content: return response # Step 1: Extract all text content and tool calls from response for content_idx, content_block in enumerate(response_content): - # Check if this is a text or tool_use block by checking the 'type' field - if isinstance(content_block, dict) and content_block.get("type") in [ - "text", - "tool_use", - ]: - # Cast to dict to handle the union type properly + # Handle both dict and Pydantic object content blocks + block_dict: Dict[str, Any] = {} + if isinstance(content_block, dict): + block_type = content_block.get("type") + block_dict = cast(Dict[str, Any], content_block) + elif hasattr(content_block, "type"): + block_type = getattr(content_block, "type", None) + # Convert Pydantic object to dict for processing + if hasattr(content_block, "model_dump"): + block_dict = content_block.model_dump() + else: + block_dict = {"type": block_type, "text": getattr(content_block, "text", None)} + else: + continue + + if block_type in ["text", "tool_use"]: self._extract_output_text_and_images( - content_block=cast(Dict[str, Any], content_block), + content_block=block_dict, content_idx=content_idx, texts_to_check=texts_to_check, images_to_check=images_to_check, @@ -530,7 +549,11 @@ class AnthropicMessagesHandler(BaseTranslation): Override this method to customize text content detection. """ - response_content = response.get("content", []) + if isinstance(response, dict): + response_content = response.get("content", []) + else: + response_content = getattr(response, "content", None) or [] + if not response_content: return False for content_block in response_content: @@ -590,7 +613,16 @@ class AnthropicMessagesHandler(BaseTranslation): mapping = task_mappings[task_idx] content_idx = cast(int, mapping[0]) - response_content = response.get("content", []) + # Handle both dict and object responses + response_content: List[Any] = [] + if isinstance(response, dict): + response_content = response.get("content", []) or [] + elif hasattr(response, "content"): + content = getattr(response, "content", None) + response_content = content or [] + else: + continue + if not response_content: continue @@ -601,7 +633,11 @@ class AnthropicMessagesHandler(BaseTranslation): content_block = response_content[content_idx] # Verify it's a text block and update the text field - if isinstance(content_block, dict) and content_block.get("type") == "text": - # Cast to dict to handle the union type properly for assignment - content_block = cast("AnthropicResponseTextBlock", content_block) - content_block["text"] = guardrail_response + # Handle both dict and Pydantic object content blocks + if isinstance(content_block, dict): + if content_block.get("type") == "text": + cast(Dict[str, Any], content_block)["text"] = guardrail_response + elif hasattr(content_block, "type") and getattr(content_block, "type", None) == "text": + # Update Pydantic object's text attribute + if hasattr(content_block, "text"): + content_block.text = guardrail_response diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 90c8c30eed6..ffe8cb309f3 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -340,7 +340,7 @@ class AnthropicChatCompletion(BaseLLM): data = config.transform_request( model=model, messages=messages, - optional_params=optional_params, + optional_params={**optional_params, "is_vertex_request": is_vertex_request}, litellm_params=litellm_params, headers=headers, ) @@ -690,14 +690,14 @@ class ModelResponseIterator: self.current_content_block_type = content_block_start["content_block"]["type"] if content_block_start["content_block"]["type"] == "text": text = content_block_start["content_block"]["text"] - elif content_block_start["content_block"]["type"] == "tool_use": + elif content_block_start["content_block"]["type"] == "tool_use" or content_block_start["content_block"]["type"] == "server_tool_use": self.tool_index += 1 tool_use = ChatCompletionToolCallChunk( id=content_block_start["content_block"]["id"], type="function", function=ChatCompletionToolCallFunctionChunk( name=content_block_start["content_block"]["name"], - arguments="", + arguments=str(content_block_start["content_block"]["input"]), ), index=self.tool_index, ) @@ -706,18 +706,6 @@ class ModelResponseIterator: caller_data = content_block_start["content_block"]["caller"] if caller_data: tool_use["caller"] = cast(Dict[str, Any], caller_data) # type: ignore[typeddict-item] - elif content_block_start["content_block"]["type"] == "server_tool_use": - # Handle server tool use (for tool search) - self.tool_index += 1 - tool_use = ChatCompletionToolCallChunk( - id=content_block_start["content_block"]["id"], - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=content_block_start["content_block"]["name"], - arguments="", - ), - index=self.tool_index, - ) elif ( content_block_start["content_block"]["type"] == "redacted_thinking" ): @@ -765,7 +753,9 @@ class ModelResponseIterator: # These are automatically handled by Anthropic API, we just pass them through pass elif type_chunk == "message_delta": - finish_reason, usage = self._handle_message_delta(chunk) + finish_reason, usage, container = self._handle_message_delta(chunk) + if container: + provider_specific_fields["container"] = container elif type_chunk == "message_start": """ Anthropic @@ -881,15 +871,15 @@ class ModelResponseIterator: return text, tool_use - def _handle_message_delta(self, chunk: dict) -> Tuple[str, Optional[Usage]]: + def _handle_message_delta(self, chunk: dict) -> Tuple[str, Optional[Usage], Optional[Dict[str, Any]]]: """ - Handle message_delta event for finish_reason and usage. + Handle message_delta event for finish_reason, usage, and container. Args: chunk: The message_delta chunk Returns: - Tuple of (finish_reason, usage) + Tuple of (finish_reason, usage, container) """ message_delta = MessageBlockDelta(**chunk) # type: ignore finish_reason = map_finish_reason( @@ -900,7 +890,8 @@ class ModelResponseIterator: if self.converted_response_format_tool: finish_reason = "stop" usage = self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) - return finish_reason, usage + container = message_delta["delta"].get("container") + return finish_reason, usage, container def _handle_accumulated_json_chunk( self, data_str: str diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 628121ab11c..6bdc17f7979 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -942,6 +942,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self, headers: dict, optional_params: dict ) -> dict: """Update headers with optional anthropic beta.""" + + # Skip adding beta headers for Vertex requests + # Vertex AI handles these headers differently + is_vertex_request = optional_params.get("is_vertex_request", False) + if is_vertex_request: + return headers _tools = optional_params.get("tools", []) for tool in _tools: @@ -1067,6 +1073,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ): optional_params["metadata"] = {"user_id": _litellm_metadata["user_id"]} + # Remove internal LiteLLM parameters that should not be sent to Anthropic API + optional_params.pop("is_vertex_request", None) + data = { "model": model, "messages": anthropic_messages, @@ -1132,22 +1141,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if content["type"] == "text": text_content += content["text"] ## TOOL CALLING - elif content["type"] == "tool_use": + elif content["type"] == "tool_use" or content["type"] == "server_tool_use": tool_call = AnthropicConfig.convert_tool_use_to_openai_format( anthropic_tool_content=content, index=idx, ) tool_calls.append(tool_call) - ## SERVER TOOL USE (for tool search) - elif content["type"] == "server_tool_use": - # Server tool use blocks are for tool search - treat as tool calls - # Note: using .get("input", {}) for server_tool_use as input may not be present - content_with_input = {**content, "input": content.get("input", {})} - tool_call = AnthropicConfig.convert_tool_use_to_openai_format( - anthropic_tool_content=content_with_input, - index=idx, - ) - tool_calls.append(tool_call) ## TOOL SEARCH TOOL RESULT (skip - this is metadata about tool discovery) elif content["type"] == "tool_search_tool_result": # This block contains tool_references that were discovered @@ -1343,6 +1342,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "context_management" ) + container: Optional[Dict] = completion_response.get("container") + provider_specific_fields: Dict[str, Any] = { "citations": citations, "thinking_blocks": thinking_blocks, @@ -1351,7 +1352,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): provider_specific_fields["context_management"] = context_management if web_search_results is not None: provider_specific_fields["web_search_results"] = web_search_results - + if container is not None: + provider_specific_fields["container"] = container + _message = litellm.Message( tool_calls=tool_calls, content=text_content or None, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 7ca3c555542..098694f15ae 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -186,6 +186,37 @@ class AnthropicModelInfo(BaseLLMModelInfo): return False + def is_code_execution_tool_used(self, tools: Optional[List]) -> bool: + """ + Check if code execution tool is being used. + + Returns True if any tool has type "code_execution_20250825". + """ + if not tools: + return False + + for tool in tools: + tool_type = tool.get("type", "") + if tool_type == "code_execution_20250825": + return True + return False + + def is_container_with_skills_used(self, optional_params: Optional[dict]) -> bool: + """ + Check if container with skills is being used. + + Returns True if optional_params contains container with skills. + """ + if not optional_params: + return False + + container = optional_params.get("container") + if container and isinstance(container, dict): + skills = container.get("skills") + if skills and isinstance(skills, list) and len(skills) > 0: + return True + return False + def _get_user_anthropic_beta_headers( self, anthropic_beta_header: Optional[str] ) -> Optional[List[str]]: @@ -270,6 +301,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): effort_used: bool = False, is_vertex_request: bool = False, user_anthropic_beta_headers: Optional[List[str]] = None, + code_execution_tool_used: bool = False, + container_with_skills_used: bool = False, ) -> dict: betas = set() if prompt_caching_set: @@ -293,6 +326,14 @@ class AnthropicModelInfo(BaseLLMModelInfo): if effort_used: from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER betas.add(ANTHROPIC_EFFORT_BETA_HEADER) + + # Code execution tool uses a separate beta header + if code_execution_tool_used: + betas.add("code-execution-2025-08-25") + + # Container with skills uses a separate beta header + if container_with_skills_used: + betas.add("skills-2025-10-02") headers = { "anthropic-version": anthropic_version or "2023-06-01", @@ -345,6 +386,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools) input_examples_used = self.is_input_examples_used(tools=tools) effort_used = self.is_effort_used(optional_params=optional_params, model=model) + code_execution_tool_used = self.is_code_execution_tool_used(tools=tools) + container_with_skills_used = self.is_container_with_skills_used(optional_params=optional_params) user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( anthropic_beta_header=headers.get("anthropic-beta") ) @@ -362,6 +405,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): programmatic_tool_calling_used=programmatic_tool_calling_used, input_examples_used=input_examples_used, effort_used=effort_used, + code_execution_tool_used=code_execution_tool_used, + container_with_skills_used=container_with_skills_used, ) headers = {**headers, **anthropic_headers} diff --git a/litellm/llms/base_llm/image_edit/transformation.py b/litellm/llms/base_llm/image_edit/transformation.py index f3ae2d32eaa..d522675296f 100644 --- a/litellm/llms/base_llm/image_edit/transformation.py +++ b/litellm/llms/base_llm/image_edit/transformation.py @@ -109,6 +109,15 @@ class BaseImageEditConfig(ABC): ) -> ImageResponse: pass + def use_multipart_form_data(self) -> bool: + """ + Return True if the provider uses multipart/form-data for image edit requests. + Return False if the provider uses JSON requests. + + Default is True for backwards compatibility with OpenAI-style providers. + """ + return True + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: diff --git a/litellm/llms/base_llm/interactions/__init__.py b/litellm/llms/base_llm/interactions/__init__.py new file mode 100644 index 00000000000..2bec120f597 --- /dev/null +++ b/litellm/llms/base_llm/interactions/__init__.py @@ -0,0 +1,5 @@ +"""Base classes for Interactions API implementations.""" + +from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig + +__all__ = ["BaseInteractionsAPIConfig"] diff --git a/litellm/llms/base_llm/interactions/transformation.py b/litellm/llms/base_llm/interactions/transformation.py new file mode 100644 index 00000000000..4ceb3f5387b --- /dev/null +++ b/litellm/llms/base_llm/interactions/transformation.py @@ -0,0 +1,313 @@ +""" +Base transformation class for Interactions API implementations. + +This follows the same pattern as BaseResponsesAPIConfig for the Responses API. + +Per OpenAPI spec (https://ai.google.dev/static/api/interactions.openapi.json): +- Create: POST /{api_version}/interactions +- Get: GET /{api_version}/interactions/{interaction_id} +- Delete: DELETE /{api_version}/interactions/{interaction_id} +""" + +import types +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +import httpx + +from litellm.types.interactions import ( + CancelInteractionResult, + DeleteInteractionResult, + InteractionInput, + InteractionsAPIOptionalRequestParams, + InteractionsAPIResponse, + InteractionsAPIStreamingResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + from ..chat.transformation import BaseLLMException as _BaseLLMException + + LiteLLMLoggingObj = _LiteLLMLoggingObj + BaseLLMException = _BaseLLMException +else: + LiteLLMLoggingObj = Any + BaseLLMException = Any + + +class BaseInteractionsAPIConfig(ABC): + """ + Base configuration class for Google Interactions API implementations. + + Per OpenAPI spec, the Interactions API supports two types of interactions: + - Model interactions (with model parameter) + - Agent interactions (with agent parameter) + + Implementations should override the abstract methods to provide + provider-specific transformations for requests and responses. + """ + + def __init__(self): + pass + + @property + @abstractmethod + def custom_llm_provider(self) -> LlmProviders: + """Return the LLM provider identifier.""" + pass + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + @abstractmethod + def get_supported_params(self, model: str) -> List[str]: + """ + Return the list of supported parameters for the given model. + """ + pass + + @abstractmethod + def validate_environment( + self, + headers: dict, + model: str, + litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + """ + Validate and prepare environment settings including headers. + """ + return {} + + @abstractmethod + def get_complete_url( + self, + api_base: Optional[str], + model: Optional[str], + agent: Optional[str] = None, + litellm_params: Optional[dict] = None, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the interaction request. + + Per OpenAPI spec: POST /{api_version}/interactions + + Args: + api_base: Base URL for the API + model: The model name (for model interactions) + agent: The agent name (for agent interactions) + litellm_params: LiteLLM parameters + stream: Whether this is a streaming request + + Returns: + The complete URL for the request + """ + if api_base is None: + raise ValueError("api_base is required") + return api_base + + @abstractmethod + def transform_request( + self, + model: Optional[str], + agent: Optional[str], + input: Optional[InteractionInput], + optional_params: InteractionsAPIOptionalRequestParams, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + """ + Transform the input request into the provider's expected format. + + Per OpenAPI spec, the request body should be either: + - CreateModelInteractionParams (with model) + - CreateAgentInteractionParams (with agent) + + Args: + model: The model name (for model interactions) + agent: The agent name (for agent interactions) + input: The input content (string, content object, or list) + optional_params: Optional parameters for the request + litellm_params: LiteLLM-specific parameters + headers: Request headers + + Returns: + The transformed request body as a dictionary + """ + pass + + @abstractmethod + def transform_response( + self, + model: Optional[str], + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> InteractionsAPIResponse: + """ + Transform the raw HTTP response into an InteractionsAPIResponse. + + Per OpenAPI spec, the response is an Interaction object. + """ + pass + + @abstractmethod + def transform_streaming_response( + self, + model: Optional[str], + parsed_chunk: dict, + logging_obj: LiteLLMLoggingObj, + ) -> InteractionsAPIStreamingResponse: + """ + Transform a parsed streaming response chunk into an InteractionsAPIStreamingResponse. + + Per OpenAPI spec, streaming uses SSE with various event types. + """ + pass + + # ========================================================= + # GET INTERACTION TRANSFORMATION + # ========================================================= + + @abstractmethod + def transform_get_interaction_request( + self, + interaction_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the get interaction request into URL and query params. + + Per OpenAPI spec: GET /{api_version}/interactions/{interaction_id} + + Returns: + Tuple of (URL, query_params) + """ + pass + + @abstractmethod + def transform_get_interaction_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> InteractionsAPIResponse: + """ + Transform the get interaction response. + """ + pass + + # ========================================================= + # DELETE INTERACTION TRANSFORMATION + # ========================================================= + + @abstractmethod + def transform_delete_interaction_request( + self, + interaction_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the delete interaction request into URL and body. + + Per OpenAPI spec: DELETE /{api_version}/interactions/{interaction_id} + + Returns: + Tuple of (URL, request_body) + """ + pass + + @abstractmethod + def transform_delete_interaction_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + interaction_id: str, + ) -> DeleteInteractionResult: + """ + Transform the delete interaction response. + """ + pass + + # ========================================================= + # CANCEL INTERACTION TRANSFORMATION + # ========================================================= + + @abstractmethod + def transform_cancel_interaction_request( + self, + interaction_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the cancel interaction request into URL and body. + + Returns: + Tuple of (URL, request_body) + """ + pass + + @abstractmethod + def transform_cancel_interaction_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> CancelInteractionResult: + """ + Transform the cancel interaction response. + """ + pass + + # ========================================================= + # ERROR HANDLING + # ========================================================= + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + """ + Get the appropriate exception class for an error. + """ + from ..chat.transformation import BaseLLMException + + raise BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + ) -> bool: + """ + Returns True if litellm should fake a stream for the given model. + + Override in subclasses if the provider doesn't support native streaming. + """ + return False diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 816b93edd20..e53ac36a00d 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -357,6 +357,14 @@ class BaseAWSLLM: model_id = BaseAWSLLM._get_model_id_from_model_with_spec( model_id, spec="openai" ) + elif provider == "qwen2" and "qwen2/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="qwen2" + ) + elif provider == "qwen3" and "qwen3/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="qwen3" + ) return model_id @staticmethod diff --git a/litellm/llms/bedrock/image/image_handler.py b/litellm/llms/bedrock/image/image_handler.py index 89e37bbdd8d..2e76596eefe 100644 --- a/litellm/llms/bedrock/image/image_handler.py +++ b/litellm/llms/bedrock/image/image_handler.py @@ -170,6 +170,21 @@ class BedrockImageGeneration(BaseAWSLLM): ) return model_response + def _extract_headers_from_optional_params(self, optional_params: dict) -> dict: + """ + Extract guardrail parameters from optional_params and convert them to headers. + """ + headers = {} + guardrail_identifier = optional_params.pop("guardrailIdentifier", None) + guardrail_version = optional_params.pop("guardrailVersion", None) + + if guardrail_identifier is not None: + headers["x-amz-bedrock-guardrail-identifier"] = guardrail_identifier + if guardrail_version is not None: + headers["x-amz-bedrock-guardrail-version"] = guardrail_version + + return headers + def _prepare_request( self, model: str, @@ -228,6 +243,10 @@ class BedrockImageGeneration(BaseAWSLLM): if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} + # Extract guardrail parameters and add them as headers + guardrail_headers = self._extract_headers_from_optional_params(optional_params) + headers.update(guardrail_headers) + prepped = self.get_request_headers( credentials=boto3_credentials_info.credentials, aws_region_name=boto3_credentials_info.aws_region_name, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 32be1a780a3..81225159a7c 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -108,6 +108,27 @@ class AmazonAnthropicClaudeMessagesConfig( stream=stream, ) + def _remove_ttl_from_cache_control( + self, anthropic_messages_request: Dict + ) -> None: + """ + Remove `ttl` field from cache_control in messages. + Bedrock doesn't support the ttl field in cache_control. + + Args: + anthropic_messages_request: The request dictionary to modify in-place + """ + if "messages" in anthropic_messages_request: + for message in anthropic_messages_request["messages"]: + if isinstance(message, dict) and "content" in message: + content = message["content"] + if isinstance(content, list): + for item in content: + if isinstance(item, dict) and "cache_control" in item: + cache_control = item["cache_control"] + if isinstance(cache_control, dict) and "ttl" in cache_control: + cache_control.pop("ttl", None) + def transform_anthropic_messages_request( self, model: str, @@ -141,8 +162,11 @@ class AmazonAnthropicClaudeMessagesConfig( # 3. `model` is not allowed in request body for bedrock invoke if "model" in anthropic_messages_request: anthropic_messages_request.pop("model", None) + + # 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it) + self._remove_ttl_from_cache_control(anthropic_messages_request) - # 4. AUTO-INJECT beta headers based on features used + # 5. AUTO-INJECT beta headers based on features used anthropic_model_info = AnthropicModelInfo() tools = anthropic_messages_optional_request_params.get("tools") messages_typed = cast(List[AllMessageValues], messages) @@ -175,6 +199,7 @@ class AmazonAnthropicClaudeMessagesConfig( if beta_set: anthropic_messages_request["anthropic_beta"] = list(beta_set) + return anthropic_messages_request diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 5697700b46d..1b4d04af80a 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1153,7 +1153,17 @@ def get_async_httpx_client( pass _cache_key_name = "async_httpx_client" + _params_key_name + llm_provider - _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key_name) + + # Lazily initialize the global in-memory client cache to avoid relying on + # litellm globals being fully populated during import time. + cache = getattr(litellm, "in_memory_llm_clients_cache", None) + if cache is None: + from litellm.caching.llm_caching_handler import LLMClientCache + + cache = LLMClientCache() + setattr(litellm, "in_memory_llm_clients_cache", cache) + + _cached_client = cache.get_cache(_cache_key_name) if _cached_client: return _cached_client @@ -1166,7 +1176,7 @@ def get_async_httpx_client( shared_session=shared_session, ) - litellm.in_memory_llm_clients_cache.set_cache( + cache.set_cache( key=_cache_key_name, value=_new_client, ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, @@ -1191,7 +1201,16 @@ def _get_httpx_client(params: Optional[dict] = None) -> HTTPHandler: _cache_key_name = "httpx_client" + _params_key_name - _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key_name) + # Lazily initialize the global in-memory client cache to avoid relying on + # litellm globals being fully populated during import time. + cache = getattr(litellm, "in_memory_llm_clients_cache", None) + if cache is None: + from litellm.caching.llm_caching_handler import LLMClientCache + + cache = LLMClientCache() + setattr(litellm, "in_memory_llm_clients_cache", cache) + + _cached_client = cache.get_cache(_cache_key_name) if _cached_client: return _cached_client @@ -1200,7 +1219,7 @@ def _get_httpx_client(params: Optional[dict] = None) -> HTTPHandler: else: _new_client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) - litellm.in_memory_llm_clients_cache.set_cache( + cache.set_cache( key=_cache_key_name, value=_new_client, ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 4a7789a181f..4b38f542159 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -3768,13 +3768,24 @@ class BaseLLMHTTPHandler: ) try: - response = sync_httpx_client.post( - url=api_base, - headers=headers, - data=data, - files=files, - timeout=timeout, - ) + # Check if provider uses multipart/form-data or JSON + if image_edit_provider_config.use_multipart_form_data(): + # Use form-data (OpenAI style) + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=data, + files=files, + timeout=timeout, + ) + else: + # Use JSON (Gemini style) + response = sync_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout, + ) except Exception as e: raise self._handle_error( @@ -3853,13 +3864,24 @@ class BaseLLMHTTPHandler: ) try: - response = await async_httpx_client.post( - url=api_base, - headers=headers, - data=data, - files=files, - timeout=timeout, - ) + # Check if provider uses multipart/form-data or JSON + if image_edit_provider_config.use_multipart_form_data(): + # Use form-data (OpenAI style) + response = await async_httpx_client.post( + url=api_base, + headers=headers, + data=data, + files=files, + timeout=timeout, + ) + else: + # Use JSON (Gemini style) + response = await async_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout, + ) except Exception as e: raise self._handle_error( diff --git a/litellm/llms/custom_llm.py b/litellm/llms/custom_llm.py index e88e8d5f1e3..d235df30f25 100644 --- a/litellm/llms/custom_llm.py +++ b/litellm/llms/custom_llm.py @@ -197,6 +197,36 @@ class CustomLLM(BaseLLM): ) -> EmbeddingResponse: raise CustomLLMError(status_code=500, message="Not implemented yet!") + def image_edit( + self, + model: str, + image: Any, + prompt: str, + model_response: ImageResponse, + api_key: Optional[str], + api_base: Optional[str], + optional_params: dict, + logging_obj: Any, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[HTTPHandler] = None, + ) -> ImageResponse: + raise CustomLLMError(status_code=500, message="Not implemented yet!") + + async def aimage_edit( + self, + model: str, + image: Any, + prompt: str, + model_response: ImageResponse, + api_key: Optional[str], + api_base: Optional[str], + optional_params: dict, + logging_obj: Any, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + ) -> ImageResponse: + raise CustomLLMError(status_code=500, message="Not implemented yet!") + def custom_chat_llm_router( async_fn: bool, stream: Optional[bool], custom_llm: CustomLLM diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index bc32aca6554..d8692bb6a3a 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -75,6 +75,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): "seed", "response_mime_type", "response_schema", + "response_json_schema", "routing_config", "model_selection_config", "safety_settings", @@ -105,13 +106,37 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): Returns: Mapped parameters for the provider """ + from litellm.llms.vertex_ai.gemini.transformation import ( + _camel_to_snake, + _snake_to_camel, + ) + _generate_content_config_dict: Dict[str, Any] = {} supported_google_genai_params = ( self.get_supported_generate_content_optional_params(model) ) + # Create a set with both camelCase and snake_case versions for faster lookup + supported_params_set = set(supported_google_genai_params) + supported_params_set.update(_snake_to_camel(p) for p in supported_google_genai_params) + supported_params_set.update(_camel_to_snake(p) for p in supported_google_genai_params if "_" not in p) + for param, value in generate_content_config_dict.items(): - if param in supported_google_genai_params: - _generate_content_config_dict[param] = value + # Google GenAI API expects camelCase, so we'll always output in camelCase + # Check if param (or its variants) is supported + param_snake = _camel_to_snake(param) + param_camel = _snake_to_camel(param) + + # Check if param is supported in any format + is_supported = ( + param in supported_google_genai_params or + param_snake in supported_google_genai_params or + param_camel in supported_google_genai_params + ) + + if is_supported: + # Always output in camelCase for Google GenAI API + output_key = param_camel if param != param_camel else param + _generate_content_config_dict[output_key] = value return _generate_content_config_dict def validate_environment( diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py index 830c58a0062..78a7ff9546f 100644 --- a/litellm/llms/gemini/image_edit/transformation.py +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -63,6 +63,10 @@ class GeminiImageEditConfig(BaseImageEditConfig): headers["Content-Type"] = "application/json" return headers + def use_multipart_form_data(self) -> bool: + """Gemini uses JSON requests, not multipart/form-data.""" + return False + def get_complete_url( self, model: str, diff --git a/litellm/llms/gemini/interactions/__init__.py b/litellm/llms/gemini/interactions/__init__.py new file mode 100644 index 00000000000..1752d489a0c --- /dev/null +++ b/litellm/llms/gemini/interactions/__init__.py @@ -0,0 +1,7 @@ +"""Google AI Studio Interactions API implementation.""" + +from litellm.llms.gemini.interactions.transformation import ( + GoogleAIStudioInteractionsConfig, +) + +__all__ = ["GoogleAIStudioInteractionsConfig"] diff --git a/litellm/llms/gemini/interactions/transformation.py b/litellm/llms/gemini/interactions/transformation.py new file mode 100644 index 00000000000..d21775eb236 --- /dev/null +++ b/litellm/llms/gemini/interactions/transformation.py @@ -0,0 +1,262 @@ +""" +Google AI Studio Interactions API configuration. + +Per OpenAPI spec (https://ai.google.dev/static/api/interactions.openapi.json): +- Create: POST https://generativelanguage.googleapis.com/{api_version}/interactions +- Get: GET https://generativelanguage.googleapis.com/{api_version}/interactions/{interaction_id} +- Delete: DELETE https://generativelanguage.googleapis.com/{api_version}/interactions/{interaction_id} + +This is a thin wrapper - no transformation needed since we follow the spec directly. +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import httpx + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig +from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo +from litellm.types.interactions import ( + CancelInteractionResult, + DeleteInteractionResult, + InteractionInput, + InteractionsAPIOptionalRequestParams, + InteractionsAPIResponse, + InteractionsAPIStreamingResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): + """ + Configuration for Google AI Studio Interactions API. + + Minimal config - we follow the OpenAPI spec directly with no transformation. + """ + + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.GEMINI + + @property + def api_version(self) -> str: + return "v1beta" + + def get_supported_params(self, model: str) -> List[str]: + """Per OpenAPI spec CreateModelInteractionParams.""" + return [ + "model", "agent", "input", "tools", "system_instruction", + "generation_config", "stream", "store", "background", + "response_modalities", "response_format", "response_mime_type", + "previous_interaction_id", + ] + + def validate_environment( + self, + headers: dict, + model: str, + litellm_params: Optional[GenericLiteLLMParams], + ) -> dict: + """Google AI Studio uses API key in query params, not headers.""" + headers = headers or {} + headers["Content-Type"] = "application/json" + return headers + + def get_complete_url( + self, + api_base: Optional[str], + model: Optional[str], + agent: Optional[str] = None, + litellm_params: Optional[dict] = None, + stream: Optional[bool] = None, + ) -> str: + """POST /{api_version}/interactions""" + litellm_params = litellm_params or {} + api_base = GeminiModelInfo.get_api_base(api_base) + api_key = GeminiModelInfo.get_api_key(litellm_params.get("api_key")) + + if not api_key: + raise ValueError( + "Google API key is required. Set GOOGLE_API_KEY or GEMINI_API_KEY environment variable." + ) + + query_params = f"key={api_key}" + if stream: + query_params += "&alt=sse" + + return f"{api_base}/{self.api_version}/interactions?{query_params}" + + def transform_request( + self, + model: Optional[str], + agent: Optional[str], + input: Optional[InteractionInput], + optional_params: InteractionsAPIOptionalRequestParams, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + """ + Build request body per OpenAPI spec - minimal transformation. + """ + request_body: Dict[str, Any] = {} + + # Model or Agent (one required) + if model: + request_body["model"] = GeminiModelInfo.get_base_model(model) or model + elif agent: + request_body["agent"] = agent + else: + raise ValueError("Either 'model' or 'agent' must be provided") + + # Input + if input is not None: + request_body["input"] = input + + # Pass through optional params directly (they match the spec) + optional_keys = [ + "tools", "system_instruction", "generation_config", "stream", "store", + "background", "response_modalities", "response_format", + "response_mime_type", "previous_interaction_id", + ] + for key in optional_keys: + if optional_params.get(key) is not None: + request_body[key] = optional_params[key] + + return request_body + + def transform_response( + self, + model: Optional[str], + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> InteractionsAPIResponse: + """Parse response - it already matches our response type.""" + try: + logging_obj.post_call( + original_response=raw_response.text, + additional_args={"complete_input_dict": {}}, + ) + raw_json = raw_response.json() + except Exception: + raise GeminiError( + message=raw_response.text, + status_code=raw_response.status_code, + headers=dict(raw_response.headers), + ) + + verbose_logger.debug("Google AI Interactions response: %s", raw_json) + + response = InteractionsAPIResponse(**raw_json) + response._hidden_params["headers"] = dict(raw_response.headers) + response._hidden_params["additional_headers"] = process_response_headers(dict(raw_response.headers)) + + return response + + def transform_streaming_response( + self, + model: Optional[str], + parsed_chunk: dict, + logging_obj: LiteLLMLoggingObj, + ) -> InteractionsAPIStreamingResponse: + """Parse streaming chunk.""" + verbose_logger.debug("Google AI Interactions streaming chunk: %s", parsed_chunk) + return InteractionsAPIStreamingResponse(**parsed_chunk) + + # GET / DELETE / CANCEL - just build URLs, responses match spec directly + + def transform_get_interaction_request( + self, + interaction_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """GET /{api_version}/interactions/{interaction_id}""" + resolved_api_base = GeminiModelInfo.get_api_base(api_base) + api_key = GeminiModelInfo.get_api_key(litellm_params.api_key) + if not api_key: + raise ValueError("Google API key is required") + return f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}?key={api_key}", {} + + def transform_get_interaction_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> InteractionsAPIResponse: + try: + raw_json = raw_response.json() + except Exception: + raise GeminiError( + message=raw_response.text, + status_code=raw_response.status_code, + headers=dict(raw_response.headers), + ) + response = InteractionsAPIResponse(**raw_json) + response._hidden_params["headers"] = dict(raw_response.headers) + return response + + def transform_delete_interaction_request( + self, + interaction_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """DELETE /{api_version}/interactions/{interaction_id}""" + resolved_api_base = GeminiModelInfo.get_api_base(api_base) + api_key = GeminiModelInfo.get_api_key(litellm_params.api_key) + if not api_key: + raise ValueError("Google API key is required") + return f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}?key={api_key}", {} + + def transform_delete_interaction_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + interaction_id: str, + ) -> DeleteInteractionResult: + if 200 <= raw_response.status_code < 300: + return DeleteInteractionResult(success=True, id=interaction_id) + raise GeminiError( + message=raw_response.text, + status_code=raw_response.status_code, + headers=dict(raw_response.headers), + ) + + def transform_cancel_interaction_request( + self, + interaction_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """POST /{api_version}/interactions/{interaction_id}:cancel (if supported)""" + resolved_api_base = GeminiModelInfo.get_api_base(api_base) + api_key = GeminiModelInfo.get_api_key(litellm_params.api_key) + if not api_key: + raise ValueError("Google API key is required") + return f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}:cancel?key={api_key}", {} + + def transform_cancel_interaction_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> CancelInteractionResult: + try: + raw_json = raw_response.json() + except Exception: + raise GeminiError( + message=raw_response.text, + status_code=raw_response.status_code, + headers=dict(raw_response.headers), + ) + return CancelInteractionResult(**raw_json) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 4480ec497c7..9b8f15c7623 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -30,7 +30,7 @@ Output: response.output is List[GenericResponseOutputItem] where each has: from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast -from openai.types.responses import ResponseFunctionToolCall +from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from pydantic import BaseModel from litellm._logging import verbose_proxy_logger @@ -299,8 +299,25 @@ class OpenAIResponsesHandler(BaseTranslation): task_mappings: List[Tuple[int, int]] = [] # Track (output_item_index, content_index) for each text + # Handle both dict and Pydantic object responses + if isinstance(response, dict): + response_output = response.get("output", []) + elif hasattr(response, "output"): + response_output = response.output or [] + else: + verbose_proxy_logger.debug( + "OpenAI Responses API: No output found in response" + ) + return response + + if not response_output: + verbose_proxy_logger.debug( + "OpenAI Responses API: Empty output in response" + ) + return response + # Step 1: Extract all text content and tool calls from response output - for output_idx, output_item in enumerate(response.output): + for output_idx, output_item in enumerate(response_output): self._extract_output_text_and_images( output_item=output_item, output_idx=output_idx, @@ -538,13 +555,18 @@ class OpenAIResponsesHandler(BaseTranslation): content: Optional[Union[List[OutputText], List[dict]]] = None if isinstance(output_item, BaseModel): try: + output_item_dump = output_item.model_dump() generic_response_output_item = GenericResponseOutputItem.model_validate( - output_item.model_dump() + output_item_dump ) if generic_response_output_item.content: content = generic_response_output_item.content except Exception: - return + # Try to extract content directly from output_item if validation fails + if hasattr(output_item, "content") and output_item.content: + content = output_item.content + else: + return elif isinstance(output_item, dict): content = output_item.get("content", []) else: @@ -582,22 +604,53 @@ class OpenAIResponsesHandler(BaseTranslation): Override this method to customize how responses are applied. """ + # Handle both dict and Pydantic object responses + if isinstance(response, dict): + response_output = response.get("output", []) + elif hasattr(response, "output"): + response_output = response.output or [] + else: + return + for task_idx, guardrail_response in enumerate(responses): mapping = task_mappings[task_idx] output_idx = cast(int, mapping[0]) content_idx = cast(int, mapping[1]) - output_item = response.output[output_idx] + if output_idx >= len(response_output): + continue - # Handle both GenericResponseOutputItem and dict + output_item = response_output[output_idx] + + # Handle both GenericResponseOutputItem, BaseModel, and dict if isinstance(output_item, GenericResponseOutputItem): - content_item = output_item.content[content_idx] - if isinstance(content_item, OutputText): - content_item.text = guardrail_response - elif isinstance(content_item, dict): - content_item["text"] = guardrail_response + if output_item.content and content_idx < len(output_item.content): + content_item = output_item.content[content_idx] + if isinstance(content_item, OutputText): + content_item.text = guardrail_response + elif isinstance(content_item, dict): + content_item["text"] = guardrail_response + elif isinstance(output_item, BaseModel): + # Handle other Pydantic models by converting to GenericResponseOutputItem + try: + generic_item = GenericResponseOutputItem.model_validate( + output_item.model_dump() + ) + if generic_item.content and content_idx < len(generic_item.content): + content_item = generic_item.content[content_idx] + if isinstance(content_item, OutputText): + content_item.text = guardrail_response + # Update the original response output + if hasattr(output_item, "content") and output_item.content: + original_content = output_item.content[content_idx] + if hasattr(original_content, "text"): + original_content.text = guardrail_response + except Exception: + pass elif isinstance(output_item, dict): content = output_item.get("content", []) if content and content_idx < len(content): if isinstance(content[content_idx], dict): content[content_idx]["text"] = guardrail_response + elif hasattr(content[content_idx], "text"): + content[content_idx].text = guardrail_response diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 4c9d3828383..7ccec074703 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -6,6 +6,7 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( _safe_convert_created_field, ) @@ -15,7 +16,7 @@ from litellm.types.llms.openai import * from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from litellm.litellm_core_utils.core_helpers import process_response_headers + from ..common_utils import OpenAIError if TYPE_CHECKING: @@ -181,6 +182,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) response = ResponsesAPIResponse.model_construct(**raw_response_json) + # Store processed headers in additional_headers so they get returned to the client response._hidden_params["additional_headers"] = processed_headers response._hidden_params["headers"] = raw_response_headers return response diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index a6c19222619..2d801506d5f 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -14,5 +14,9 @@ "helicone": { "base_url": "https://ai-gateway.helicone.ai/", "api_key_env": "HELICONE_API_KEY" + }, + "veniceai": { + "base_url": "https://api.venice.ai/api/v1", + "api_key_env": "VENICE_AI_API_KEY" } } diff --git a/litellm/llms/vertex_ai/agent_engine/__init__.py b/litellm/llms/vertex_ai/agent_engine/__init__.py new file mode 100644 index 00000000000..de891f85602 --- /dev/null +++ b/litellm/llms/vertex_ai/agent_engine/__init__.py @@ -0,0 +1,13 @@ +""" +Vertex AI Agent Engine (Reasoning Engines) Provider + +Supports Vertex AI Reasoning Engines via the :query and :streamQuery endpoints. +""" + +from litellm.llms.vertex_ai.agent_engine.transformation import ( + VertexAgentEngineConfig, + VertexAgentEngineError, +) + +__all__ = ["VertexAgentEngineConfig", "VertexAgentEngineError"] + diff --git a/litellm/llms/vertex_ai/agent_engine/sse_iterator.py b/litellm/llms/vertex_ai/agent_engine/sse_iterator.py new file mode 100644 index 00000000000..06fb55e1848 --- /dev/null +++ b/litellm/llms/vertex_ai/agent_engine/sse_iterator.py @@ -0,0 +1,90 @@ +""" +SSE Stream Iterator for Vertex AI Agent Engine. + +Handles Server-Sent Events (SSE) streaming responses from Vertex AI Reasoning Engines. +""" + +from typing import Any, Union + +from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator +from litellm.types.llms.openai import ChatCompletionUsageBlock +from litellm.types.utils import ( + Delta, + GenericStreamingChunk, + ModelResponseStream, + StreamingChoices, +) + + +class VertexAgentEngineResponseIterator(BaseModelResponseIterator): + """ + Iterator for Vertex Agent Engine SSE streaming responses. + + Uses BaseModelResponseIterator which handles sync/async iteration. + We just need to implement chunk_parser to parse Vertex Agent Engine response format. + """ + + def __init__(self, streaming_response: Any, sync_stream: bool) -> None: + super().__init__(streaming_response=streaming_response, sync_stream=sync_stream) + + def chunk_parser( + self, chunk: dict + ) -> Union[GenericStreamingChunk, ModelResponseStream]: + """ + Parse a Vertex Agent Engine response chunk into ModelResponseStream. + + Vertex Agent Engine response format: + { + "content": { + "parts": [{"text": "..."}], + "role": "model" + }, + "finish_reason": "STOP", + "usage_metadata": { + "prompt_token_count": 100, + "candidates_token_count": 50, + "total_token_count": 150 + } + } + """ + # Extract text from content.parts + text = None + content = chunk.get("content", {}) + parts = content.get("parts", []) + for part in parts: + if isinstance(part, dict) and "text" in part: + text = part["text"] + break + + # Extract finish_reason + finish_reason = None + raw_finish_reason = chunk.get("finish_reason") + if raw_finish_reason == "STOP": + finish_reason = "stop" + elif raw_finish_reason: + finish_reason = raw_finish_reason.lower() + + # Extract usage from usage_metadata + usage = None + usage_metadata = chunk.get("usage_metadata", {}) + if usage_metadata: + usage = ChatCompletionUsageBlock( + prompt_tokens=usage_metadata.get("prompt_token_count", 0), + completion_tokens=usage_metadata.get("candidates_token_count", 0), + total_tokens=usage_metadata.get("total_token_count", 0), + ) + + # Return ModelResponseStream (OpenAI-compatible chunk) + return ModelResponseStream( + choices=[ + StreamingChoices( + finish_reason=finish_reason, + index=0, + delta=Delta( + content=text, + role="assistant" if text else None, + ), + ) + ], + usage=usage, + ) diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py new file mode 100644 index 00000000000..4c07e8455e3 --- /dev/null +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -0,0 +1,508 @@ +""" +Transformation for Vertex AI Agent Engine (Reasoning Engines) + +Handles the transformation between LiteLLM's OpenAI-compatible format and +Vertex AI Reasoning Engine's API format. + +API Reference: +- :query endpoint - for session management (create, get, list, delete) +- :streamQuery endpoint - for actual queries (stream_query method) +""" + +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast + +import httpx + +from litellm._logging import verbose_logger +from litellm._uuid import uuid +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_content_list_to_str, +) +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.vertex_ai.agent_engine.sse_iterator import ( + VertexAgentEngineResponseIterator, +) +from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, Message, ModelResponse, Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.utils import CustomStreamWrapper + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + HTTPHandler = Any + AsyncHTTPHandler = Any + CustomStreamWrapper = Any + + +class VertexAgentEngineError(BaseLLMException): + """Exception for Vertex Agent Engine errors.""" + + def __init__(self, status_code: int, message: str): + self.status_code = status_code + self.message = message + super().__init__(message=message, status_code=status_code) + + +class VertexAgentEngineConfig(BaseConfig, VertexBase): + """ + Configuration for Vertex AI Agent Engine (Reasoning Engines). + + Model format: vertex_ai/agent_engine/ + Where resource_id is the numeric ID of the reasoning engine. + """ + + def __init__(self, **kwargs): + BaseConfig.__init__(self, **kwargs) + VertexBase.__init__(self) + + def get_supported_openai_params(self, model: str) -> List[str]: + """Vertex Agent Engine has limited OpenAI compatible params.""" + return ["user"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """Map OpenAI params to Agent Engine params.""" + # Map 'user' to 'user_id' for session management + if "user" in non_default_params: + optional_params["user_id"] = non_default_params["user"] + return optional_params + + def _parse_model_string(self, model: str) -> Tuple[str, str]: + """ + Parse model string to extract resource ID. + + Model format: agent_engine/// + Or: agent_engine/ (uses default project/location) + + Returns: (resource_path, engine_id) + """ + # Remove 'agent_engine/' prefix if present + if model.startswith("agent_engine/"): + model = model[len("agent_engine/") :] + + # Check if it's a full resource path + if model.startswith("projects/"): + # Full path: projects/123/locations/us-central1/reasoningEngines/456 + return model, model.split("/")[-1] + + # Just the engine ID + return model, model + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the request. + + For Vertex Agent Engine: + - Non-streaming: :query endpoint (for session management) + - Streaming: :streamQuery endpoint (for actual queries) + """ + resource_path, engine_id = self._parse_model_string(model) + + # Get project and location from litellm_params or environment + vertex_project = self.safe_get_vertex_ai_project(litellm_params) + vertex_location = self.safe_get_vertex_ai_location(litellm_params) or "us-central1" + + # Build the full resource path if only engine_id was provided + if not resource_path.startswith("projects/"): + if not vertex_project: + raise ValueError( + "vertex_project is required for Vertex Agent Engine. " + "Set via litellm_params['vertex_project'] or VERTEXAI_PROJECT env var." + ) + resource_path = f"projects/{vertex_project}/locations/{vertex_location}/reasoningEngines/{engine_id}" + + # Build the base URL + base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + + # Always use :streamQuery endpoint for actual queries + # The :query endpoint only supports session management methods + # (create_session, get_session, list_sessions, delete_session, etc.) + endpoint = f"{base_url}/v1beta1/{resource_path}:streamQuery" + + verbose_logger.debug(f"Vertex Agent Engine URL: {endpoint}") + return endpoint + + def _get_auth_headers( + self, + optional_params: dict, + litellm_params: dict, + ) -> Dict[str, str]: + """Get authentication headers using Google Cloud credentials.""" + vertex_credentials = self.safe_get_vertex_ai_credentials(litellm_params) + vertex_project = self.safe_get_vertex_ai_project(litellm_params) + + # Get access token using VertexBase + access_token, project_id = self.get_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + ) + + verbose_logger.debug(f"Vertex Agent Engine: Authenticated for project {project_id}") + + return { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + } + + def _get_user_id(self, optional_params: dict) -> str: + """Get or generate user ID for session management.""" + user_id = optional_params.get("user_id") or optional_params.get("user") + if user_id: + return user_id + # Generate a user ID + return f"litellm-user-{str(uuid.uuid4())[:8]}" + + def _get_session_id(self, optional_params: dict) -> Optional[str]: + """Get session ID if provided.""" + return optional_params.get("session_id") + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request to Vertex Agent Engine format. + + The API expects: + { + "class_method": "stream_query", + "input": { + "message": "...", + "user_id": "...", + "session_id": "..." (optional) + } + } + """ + # Use the last message content as the prompt + prompt = convert_content_list_to_str(messages[-1]) + + # Get user_id and session_id + user_id = self._get_user_id(optional_params) + session_id = self._get_session_id(optional_params) + + # Build the input + input_data: Dict[str, Any] = { + "message": prompt, + "user_id": user_id, + } + + if session_id: + input_data["session_id"] = session_id + + # Build the request payload + # Note: stream_query is used for both streaming and non-streaming + # The difference is the endpoint (:streamQuery vs :query) + payload = { + "class_method": "stream_query", + "input": input_data, + } + + verbose_logger.debug(f"Vertex Agent Engine payload: {payload}") + return payload + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """Validate environment and set up authentication headers.""" + auth_headers = self._get_auth_headers(optional_params, litellm_params) + headers.update(auth_headers) + return headers + + def _extract_text_from_response(self, response_data: dict) -> str: + """Extract text content from the response.""" + # Try to get from content.parts + content = response_data.get("content", {}) + parts = content.get("parts", []) + for part in parts: + if "text" in part: + return part["text"] + + # Try actions.state_delta + actions = response_data.get("actions", {}) + state_delta = actions.get("state_delta", {}) + for key, value in state_delta.items(): + if isinstance(value, str) and value: + return value + + return "" + + def _calculate_usage( + self, model: str, messages: List[AllMessageValues], content: str + ) -> Optional[Usage]: + """Calculate token usage using LiteLLM's token counter.""" + try: + from litellm.utils import token_counter + + prompt_tokens = token_counter(model="gpt-3.5-turbo", messages=messages) + completion_tokens = token_counter( + model="gpt-3.5-turbo", text=content, count_response_tokens=True + ) + total_tokens = prompt_tokens + completion_tokens + + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + except Exception as e: + verbose_logger.warning(f"Failed to calculate token usage: {str(e)}") + return None + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """ + Transform Vertex Agent Engine response to LiteLLM ModelResponse format. + + The response is a streaming SSE format even for non-streaming requests. + We need to collect all the chunks and extract the final response. + """ + try: + content_type = raw_response.headers.get("content-type", "").lower() + verbose_logger.debug(f"Vertex Agent Engine response Content-Type: {content_type}") + + # Parse the SSE response + response_text = raw_response.text + verbose_logger.debug(f"Response (first 500 chars): {response_text[:500]}") + + # Extract content from SSE stream + content = "" + for line in response_text.strip().split("\n"): + line = line.strip() + if not line: + continue + + try: + data = json.loads(line) + if isinstance(data, dict): + text = self._extract_text_from_response(data) + if text: + content = text # Use the last non-empty text + except json.JSONDecodeError: + continue + + # Create the message + message = Message(content=content, role="assistant") + + # Create choices + choice = Choices(finish_reason="stop", index=0, message=message) + + # Update model response + model_response.choices = [choice] + model_response.model = model + + # Calculate usage + calculated_usage = self._calculate_usage(model, messages, content) + if calculated_usage: + setattr(model_response, "usage", calculated_usage) + + return model_response + + except Exception as e: + verbose_logger.error(f"Error processing Vertex Agent Engine response: {str(e)}") + raise VertexAgentEngineError( + message=f"Error processing response: {str(e)}", + status_code=raw_response.status_code, + ) + + def get_streaming_response( + self, + model: str, + raw_response: httpx.Response, + ) -> VertexAgentEngineResponseIterator: + """Return a streaming iterator for SSE responses.""" + return VertexAgentEngineResponseIterator( + streaming_response=raw_response.iter_lines(), + sync_stream=True, + ) + + def get_sync_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional[Union[HTTPHandler, "AsyncHTTPHandler"]] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> "CustomStreamWrapper": + """Get a CustomStreamWrapper for synchronous streaming.""" + from litellm.llms.custom_httpx.http_handler import ( + HTTPHandler, + _get_httpx_client, + ) + from litellm.utils import CustomStreamWrapper + + if client is None or not isinstance(client, HTTPHandler): + client = _get_httpx_client(params={}) + + # Avoid logging sensitive api_base directly + verbose_logger.debug("Making sync streaming request to Vertex AI endpoint.") + + # Make streaming request + response = client.post( + api_base, + headers=headers, + data=json.dumps(data), + stream=True, + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise VertexAgentEngineError( + status_code=response.status_code, message=str(response.read()) + ) + + # Create iterator for SSE stream + completion_stream = self.get_streaming_response(model=model, raw_response=response) + + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return streaming_response + + async def get_async_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional["AsyncHTTPHandler"] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> "CustomStreamWrapper": + """Get a CustomStreamWrapper for asynchronous streaming.""" + from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, + ) + from litellm.utils import CustomStreamWrapper + + if client is None or not isinstance(client, AsyncHTTPHandler): + client = get_async_httpx_client( + llm_provider=cast(Any, "vertex_ai"), params={} + ) + + # Avoid logging sensitive api_base directly + verbose_logger.debug("Making async streaming request to Vertex AI endpoint.") + + # Make async streaming request + response = await client.post( + api_base, + headers=headers, + data=json.dumps(data), + stream=True, + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise VertexAgentEngineError( + status_code=response.status_code, message=str(await response.aread()) + ) + + # Create iterator for SSE stream (async) + completion_stream = VertexAgentEngineResponseIterator( + streaming_response=response.aiter_lines(), + sync_stream=False, + ) + + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return streaming_response + + @property + def has_custom_stream_wrapper(self) -> bool: + """Indicates that this config has custom streaming support.""" + return True + + @property + def supports_stream_param_in_request_body(self) -> bool: + """Agent Engine does not allow passing `stream` in the request body.""" + return False + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return VertexAgentEngineError(status_code=status_code, message=error_message) + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + ) -> bool: + """Agent Engine always returns SSE streams, so we use real streaming.""" + return False + diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 3cfa55c0606..6bb11430f20 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -5,7 +5,6 @@ from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, get_ty import httpx import litellm -from litellm.utils import supports_response_schema, supports_system_messages from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs @@ -14,6 +13,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import AllMessageValues from litellm.types.llms.vertex_ai import PartType, Schema from litellm.types.utils import TokenCountResponse +from litellm.utils import supports_response_schema, supports_system_messages class VertexAIError(BaseLLMException): @@ -36,6 +36,7 @@ class VertexAIModelRoute(str, Enum): MODEL_GARDEN = "model_garden" NON_GEMINI = "non_gemini" OPENAI_COMPATIBLE = "openai" + AGENT_ENGINE = "agent_engine" VERTEX_AI_MODEL_ROUTES = [f"{route.value}/" for route in VertexAIModelRoute] @@ -76,6 +77,10 @@ def get_vertex_ai_model_route( if litellm_params and litellm_params.get("base_model") is not None: if "gemini" in litellm_params["base_model"]: return VertexAIModelRoute.GEMINI + + # Check for agent_engine models (Reasoning Engines) + if "agent_engine/" in model: + return VertexAIModelRoute.AGENT_ENGINE # Check if numeric endpoint ID with custom api_base (PSC endpoint) # Route to GEMINI (HTTP path) to support PSC endpoints properly diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index feae8395178..84a5958ee5e 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -228,12 +228,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Gemini 3 models include: - gemini-3-pro-preview + - gemini-3-flash + - gemini-3-flash-preview (Gemini 3 Flash) - Any future Gemini 3.x models """ # Check for Gemini 3 models if "gemini-3" in model: return True - return False def _supports_penalty_parameters(self, model: str) -> bool: @@ -685,22 +686,40 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Returns: GeminiThinkingConfig with thinkingLevel and includeThoughts """ + # Check if this is gemini-3-flash which supports MINIMAL thinking level + is_gemini3flash= model and ( + "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower() + ) if reasoning_effort == "minimal": - return {"thinkingLevel": "low", "includeThoughts": True} + if is_gemini3flash: + return {"thinkingLevel": "minimal", "includeThoughts": True} + else: + return {"thinkingLevel": "low", "includeThoughts": True} elif reasoning_effort == "low": return {"thinkingLevel": "low", "includeThoughts": True} elif reasoning_effort == "medium": - return { - "thinkingLevel": "high", - "includeThoughts": True, - } # medium is not out yet + # For gemini-3-flash-preview, medium maps to "medium", otherwise "high" + if is_gemini3flash: + return {"thinkingLevel": "medium", "includeThoughts": True} + else: + return { + "thinkingLevel": "high", + "includeThoughts": True, + } # medium is not out yet for other models elif reasoning_effort == "high": return {"thinkingLevel": "high", "includeThoughts": True} elif reasoning_effort == "disable": - # Gemini 3 cannot fully disable thinking, so we use "low" but hide thoughts - return {"thinkingLevel": "low", "includeThoughts": False} + # Gemini 3 cannot fully disable thinking, so we use "minimal" for gemini-3-flash-preview, "low" for others + if is_gemini3flash: + return {"thinkingLevel": "minimal", "includeThoughts": False} + else: + return {"thinkingLevel": "low", "includeThoughts": False} elif reasoning_effort == "none": - return {"thinkingLevel": "low", "includeThoughts": False} + # For gemini-3-flash-preview, use "minimal" instead of "low" + if is_gemini3flash: + return {"thinkingLevel": "minimal", "includeThoughts": False} + else: + return {"thinkingLevel": "low", "includeThoughts": False} else: raise ValueError(f"Invalid reasoning effort: {reasoning_effort}") @@ -751,17 +770,38 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @staticmethod def _map_thinking_param( thinking_param: AnthropicThinkingParam, + model: Optional[str] = None, ) -> GeminiThinkingConfig: thinking_enabled = thinking_param.get("type") == "enabled" thinking_budget = thinking_param.get("budget_tokens") params: GeminiThinkingConfig = {} - if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero( - thinking_budget - ): - params["includeThoughts"] = True - if thinking_budget is not None and isinstance(thinking_budget, int): - params["thinkingBudget"] = thinking_budget + + # For Gemini 3+ models, use thinkingLevel instead of thinkingBudget + if model and VertexGeminiConfig._is_gemini_3_or_newer(model): + if thinking_enabled: + if thinking_budget is None or thinking_budget == 0: + params["includeThoughts"] = False + else: + params["includeThoughts"] = True + if thinking_budget >= 10000: + is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower() + params["thinkingLevel"] = "minimal" if is_gemini3flash else "low" + else: + is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower() + params["thinkingLevel"] = "minimal" if is_gemini3flash else "low" + else: + # Thinking disabled + params["includeThoughts"] = False + else: + # For older Gemini models, use thinkingBudget + if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero( + thinking_budget + ): + params["includeThoughts"] = True + if thinking_budget is not None and isinstance(thinking_budget, int): + params["thinkingBudget"] = thinking_budget + return params def map_response_modalities(self, value: list) -> list: @@ -938,7 +978,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): optional_params[ "thinkingConfig" ] = VertexGeminiConfig._map_thinking_param( - cast(AnthropicThinkingParam, value) + cast(AnthropicThinkingParam, value), + model=model, ) elif param == "modalities" and isinstance(value, list): response_modalities = self.map_response_modalities(value) @@ -970,7 +1011,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "thinkingLevel" not in thinking_config and "thinkingBudget" not in thinking_config ): - thinking_config["thinkingLevel"] = "low" + # For gemini-3-flash-preview, default to "minimal" to match Gemini 2.5 Flash behavior + # For other Gemini 3 models, default to "low" + is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower() + thinking_config["thinkingLevel"] = "minimal" if is_gemini3flash else "low" optional_params["thinkingConfig"] = thinking_config return optional_params diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py index 859bb0a6984..07f57a4a7f6 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py @@ -46,6 +46,7 @@ class GoogleBatchEmbeddings(VertexLLM): aembedding: Optional[bool] = False, timeout=300, client=None, + extra_headers: Optional[dict] = None, ) -> EmbeddingResponse: _auth_header, vertex_project = self._ensure_access_token( credentials=vertex_credentials, @@ -90,6 +91,15 @@ class GoogleBatchEmbeddings(VertexLLM): headers = { "Content-Type": "application/json; charset=utf-8", } + if auth_header is not None: + if isinstance(auth_header, dict): + # For Gemini with custom api_base: auth_header is {"x-goog-api-key": "..."} + headers.update(auth_header) + else: + # For Vertex AI: auth_header is a Bearer token string + headers["Authorization"] = f"Bearer {auth_header}" + if extra_headers is not None: + headers.update(extra_headers) ## LOGGING logging_obj.pre_call( diff --git a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py index 469340f6bba..d575c5862e8 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py @@ -8,7 +8,6 @@ import httpx from httpx._types import RequestFiles import litellm - from litellm.images.utils import ImageEditRequestUtils from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM @@ -94,10 +93,22 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[dict] = None, + api_base: Optional[str] = None, ) -> dict: headers = headers or {} - vertex_project = self._resolve_vertex_project() - vertex_credentials = self._resolve_vertex_credentials() + litellm_params = litellm_params or {} + + # If a custom api_base is provided, skip credential validation + # This allows users to use proxies or mock endpoints without needing Vertex AI credentials + _api_base = litellm_params.get("api_base") or api_base + if _api_base is not None: + return headers + + # First check litellm_params (where vertex_ai_project/vertex_ai_credentials are passed) + # then fall back to environment variables and other sources + vertex_project = self.safe_get_vertex_ai_project(litellm_params) or self._resolve_vertex_project() + vertex_credentials = self.safe_get_vertex_ai_credentials(litellm_params) or self._resolve_vertex_credentials() access_token, _ = self._ensure_access_token( credentials=vertex_credentials, project_id=vertex_project, @@ -114,19 +125,27 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): """ Get the complete URL for Vertex AI Gemini generateContent API """ - vertex_project = self._resolve_vertex_project() - vertex_location = self._resolve_vertex_location() - - if not vertex_project or not vertex_location: - raise ValueError("vertex_project and vertex_location are required for Vertex AI") - # Use the model name as provided, handling vertex_ai prefix model_name = model if model.startswith("vertex_ai/"): model_name = model.replace("vertex_ai/", "") + # If a custom api_base is provided, use it directly + # This allows users to use proxies or mock endpoints if api_base: - base_url = api_base.rstrip("/") + return api_base.rstrip("/") + + # First check litellm_params (where vertex_ai_project/vertex_ai_location are passed) + # then fall back to environment variables and other sources + vertex_project = self.safe_get_vertex_ai_project(litellm_params) or self._resolve_vertex_project() + vertex_location = self.safe_get_vertex_ai_location(litellm_params) or self._resolve_vertex_location() + + if not vertex_project or not vertex_location: + raise ValueError("vertex_project and vertex_location are required for Vertex AI") + + # Handle global location differently (no region prefix in URL) + if vertex_location == "global": + base_url = "https://aiplatform.googleapis.com" else: base_url = f"https://{vertex_location}-aiplatform.googleapis.com" diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index b9747652362..619bd006300 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -13,7 +13,7 @@ from litellm.types.llms.openai import ( AllMessageValues, OpenAIImageGenerationOptionalParams, ) -from litellm.types.utils import ImageObject, ImageResponse +from litellm.types.utils import ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -234,6 +234,27 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): return request_body + def _transform_image_usage(self, usage: dict) -> ImageUsage: + input_tokens_details = ImageUsageInputTokensDetails( + image_tokens=0, + text_tokens=0, + ) + tokens_details = usage.get("promptTokensDetails", []) + for details in tokens_details: + if isinstance(details, dict) and (modality := details.get("modality")): + token_count = details.get("tokenCount", 0) + if modality == "TEXT": + input_tokens_details.text_tokens += token_count + elif modality == "IMAGE": + input_tokens_details.image_tokens += token_count + + return ImageUsage( + input_tokens=usage.get("promptTokenCount", 0), + input_tokens_details=input_tokens_details, + output_tokens=usage.get("candidatesTokenCount", 0), + total_tokens=usage.get("totalTokenCount", 0), + ) + def transform_image_generation_response( self, model: str, @@ -276,6 +297,9 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): b64_json=inline_data["data"], url=None, )) + + if usage_metadata := response_data.get("usageMetadata", None): + model_response.usage = self._transform_image_usage(usage_metadata) return model_response diff --git a/litellm/main.py b/litellm/main.py index b08ffd16e3d..b46208cc432 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -238,7 +238,6 @@ from .types.utils import ( all_litellm_params, ) -encoding = tiktoken.get_encoding("cl100k_base") from litellm.types.utils import ModelResponseStream from litellm.utils import ( Choices, @@ -1511,7 +1510,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, # pass AsyncOpenAI, OpenAI client custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, ) @@ -1734,7 +1733,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=provider_config, ) @@ -1813,7 +1812,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, headers=headers, @@ -1861,7 +1860,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, # pass AsyncOpenAI, OpenAI client custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, ) except Exception as e: @@ -1991,7 +1990,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=provider_config, ) @@ -2021,7 +2020,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=provider_config, ) @@ -2052,7 +2051,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=provider_config, ) @@ -2082,7 +2081,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=provider_config, ) @@ -2134,7 +2133,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider=custom_llm_provider, timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, @@ -2162,7 +2161,7 @@ def completion( # type: ignore # noqa: PLR0915 shared_session=shared_session, client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, api_base=api_base, stream=stream, @@ -2202,7 +2201,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, ) elif custom_llm_provider == "cometapi": @@ -2236,7 +2235,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=provider_config, ) @@ -2321,7 +2320,7 @@ def completion( # type: ignore # noqa: PLR0915 api_base=api_base, custom_llm_provider=custom_llm_provider, model_response=model_response, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, optional_params=optional_params, timeout=timeout, @@ -2389,7 +2388,7 @@ def completion( # type: ignore # noqa: PLR0915 api_base=api_base, custom_llm_provider=custom_llm_provider, model_response=model_response, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, optional_params=optional_params, timeout=timeout, @@ -2434,7 +2433,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, # for calculating input/output tokens + encoding=_get_encoding(), # for calculating input/output tokens api_key=replicate_key, logging_obj=logging, custom_prompt_dict=custom_prompt_dict, @@ -2499,7 +2498,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="anthropic_text", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements ) @@ -2545,7 +2544,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, # for calculating input/output tokens + encoding=_get_encoding(), # for calculating input/output tokens api_key=api_key, logging_obj=logging, headers=headers, @@ -2585,7 +2584,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), api_key=nlp_cloud_key, logging_obj=logging, ) @@ -2633,7 +2632,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), default_max_tokens_to_sample=litellm.max_tokens, api_key=aleph_alpha_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements @@ -2701,7 +2700,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="cohere_chat", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=cohere_key, provider_config=provider_config, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements @@ -2730,7 +2729,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), api_key=maritalk_key, logging_obj=logging, custom_llm_provider="maritalk", @@ -2760,7 +2759,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, timeout=timeout, @@ -2790,7 +2789,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, ) elif custom_llm_provider == "oci": @@ -2808,7 +2807,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, ) elif custom_llm_provider == "compactifai": @@ -2833,7 +2832,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=provider_config, ) @@ -2849,7 +2848,7 @@ def completion( # type: ignore # noqa: PLR0915 litellm_params=litellm_params, api_key=None, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, ) if "stream" in optional_params and optional_params["stream"] is True: @@ -2893,7 +2892,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="databricks", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, @@ -2932,7 +2931,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=provider_config, ) @@ -2994,7 +2993,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="openrouter", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, @@ -3057,7 +3056,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="vercel_ai_gateway", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, @@ -3115,7 +3114,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=new_params, litellm_params=litellm_params, # type: ignore logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), vertex_location=vertex_ai_location, vertex_project=vertex_ai_project, vertex_credentials=vertex_credentials, @@ -3164,7 +3163,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=new_params, litellm_params=litellm_params, # type: ignore logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), api_base=api_base, vertex_location=vertex_ai_location, vertex_project=vertex_ai_project, @@ -3185,7 +3184,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=new_params, litellm_params=litellm_params, # type: ignore logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), vertex_location=vertex_ai_location, vertex_project=vertex_ai_project, vertex_credentials=vertex_credentials, @@ -3208,7 +3207,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=new_params, litellm_params=litellm_params, # type: ignore logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), api_base=api_base, vertex_location=vertex_ai_location, vertex_project=vertex_ai_project, @@ -3230,7 +3229,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=new_params, litellm_params=litellm_params, # type: ignore logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), api_base=api_base, vertex_location=vertex_ai_location, vertex_project=vertex_ai_project, @@ -3242,6 +3241,37 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, client=client, ) + elif model_route == VertexAIModelRoute.AGENT_ENGINE: + # Vertex AI Agent Engine (Reasoning Engines) + from litellm.llms.vertex_ai.agent_engine.transformation import ( + VertexAgentEngineConfig, + ) + + vertex_agent_engine_config = VertexAgentEngineConfig() + + # Update litellm_params with vertex credentials + litellm_params["vertex_project"] = vertex_ai_project + litellm_params["vertex_location"] = vertex_ai_location + litellm_params["vertex_credentials"] = vertex_credentials + + model_response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + model_response=model_response, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + encoding=_get_encoding(), + api_key=None, + api_base=api_base, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, + custom_llm_provider="vertex_ai", + provider_config=vertex_agent_engine_config, + headers=headers or {}, + ) else: # VertexAIModelRoute.NON_GEMINI model_response = vertex_ai_non_gemini.completion( model=model, @@ -3251,7 +3281,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=new_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), vertex_location=vertex_ai_location, vertex_project=vertex_ai_project, vertex_credentials=vertex_credentials, @@ -3308,7 +3338,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, acompletion=acompletion, api_base=api_base, @@ -3348,7 +3378,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, acompletion=acompletion, api_base=api_base, @@ -3378,7 +3408,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="sagemaker_chat", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, @@ -3398,7 +3428,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_prompt_dict=custom_prompt_dict, hf_model_name=hf_model_name, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, acompletion=acompletion, ) @@ -3442,7 +3472,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, # type: ignore logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, extra_headers=headers, # Use merged headers instead of original extra_headers timeout=timeout, @@ -3465,7 +3495,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="bedrock", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, @@ -3483,7 +3513,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="bedrock", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, client=client, @@ -3505,7 +3535,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore custom_prompt_dict=custom_prompt_dict, client=client, # pass AsyncOpenAI, OpenAI client - encoding=encoding, + encoding=_get_encoding(), custom_llm_provider="watsonx", ) elif custom_llm_provider == "watsonx_text": @@ -3567,7 +3597,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="watsonx_text", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, @@ -3583,7 +3613,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, ) @@ -3624,7 +3654,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="ollama", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, @@ -3660,7 +3690,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="ollama_chat", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, @@ -3681,7 +3711,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider=custom_llm_provider, timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, ) @@ -3714,7 +3744,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="cloudflare", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements ) @@ -3733,7 +3763,7 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, client=client, ) @@ -3768,7 +3798,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, ) @@ -3796,7 +3826,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider="gradient_ai", timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, ) @@ -3823,7 +3853,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=bytez_transformation, ) @@ -3851,7 +3881,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=lemonade_transformation, ) @@ -3887,7 +3917,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore client=client, custom_llm_provider=custom_llm_provider, - encoding=encoding, + encoding=_get_encoding(), stream=stream, provider_config=ovhcloud_transformation, ) @@ -3993,7 +4023,7 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore custom_prompt_dict=custom_prompt_dict, client=client, # pass AsyncOpenAI, OpenAI client - encoding=encoding, + encoding=_get_encoding(), ) if stream is True: return CustomStreamWrapper( @@ -4030,7 +4060,7 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider=custom_llm_provider, timeout=timeout, headers=headers, - encoding=encoding, + encoding=_get_encoding(), api_key=api_key, logging_obj=logging, client=client, @@ -4476,6 +4506,12 @@ def embedding( # noqa: PLR0915 if extra_headers is not None: optional_params["extra_headers"] = extra_headers + + if encoding_format is not None: + optional_params["encoding_format"] = encoding_format + else: + # Omiting causes openai sdk to add default value of "float" + optional_params["encoding_format"] = None api_version = None @@ -4592,7 +4628,7 @@ def embedding( # noqa: PLR0915 response = huggingface_embed.embedding( model=model, input=input, - encoding=encoding, # type: ignore + encoding=_get_encoding(), # type: ignore api_key=api_key, api_base=api_base, logging_obj=logging, @@ -4610,7 +4646,7 @@ def embedding( # noqa: PLR0915 response = bedrock_embedding.embeddings( model=model, input=transformed_input, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, optional_params=optional_params, model_response=EmbeddingResponse(), @@ -4650,7 +4686,7 @@ def embedding( # noqa: PLR0915 response = google_batch_embeddings.batch_embeddings( # type: ignore model=model, input=input, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, optional_params=optional_params, model_response=EmbeddingResponse(), @@ -4663,6 +4699,7 @@ def embedding( # noqa: PLR0915 api_key=gemini_api_key, api_base=api_base, client=client, + extra_headers=headers, ) elif custom_llm_provider == "vertex_ai": @@ -4704,7 +4741,7 @@ def embedding( # noqa: PLR0915 response = vertex_multimodal_embedding.multimodal_embedding( model=model, input=input, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, optional_params=optional_params, litellm_params=litellm_params_dict, @@ -4722,7 +4759,7 @@ def embedding( # noqa: PLR0915 response = vertex_embedding.embedding( model=model, input=input, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, optional_params=optional_params, model_response=EmbeddingResponse(), @@ -4741,7 +4778,7 @@ def embedding( # noqa: PLR0915 response = oobabooga.embedding( model=model, input=input, - encoding=encoding, + encoding=_get_encoding(), api_base=api_base, logging_obj=logging, optional_params=optional_params, @@ -4773,7 +4810,7 @@ def embedding( # noqa: PLR0915 api_base=api_base, model=model, prompts=input, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, optional_params=optional_params, model_response=EmbeddingResponse(), @@ -4782,7 +4819,7 @@ def embedding( # noqa: PLR0915 response = sagemaker_llm.embedding( model=model, input=input, - encoding=encoding, + encoding=_get_encoding(), logging_obj=logging, optional_params=optional_params, model_response=EmbeddingResponse(), @@ -6853,3 +6890,31 @@ def stream_chunk_builder( # noqa: PLR0915 llm_provider="", model="", ) + + +# Cache for encoding to avoid repeated __getattr__ calls +_encoding_cache: Optional[Any] = None + + +def _get_encoding(): + """Get encoding, loading it lazily if needed.""" + global _encoding_cache + if _encoding_cache is None: + import sys + # Access via module to trigger __getattr__ if not cached + _encoding_cache = sys.modules[__name__].encoding + return _encoding_cache + + +def __getattr__(name: str) -> Any: + """Lazy import handler for main module""" + if name == "encoding": + # Lazy load encoding to avoid heavy tiktoken import at module load time + _encoding = tiktoken.get_encoding("cl100k_base") + # Cache it in the module's __dict__ for subsequent accesses + import sys + sys.modules[__name__].__dict__["encoding"] = _encoding + global _encoding_cache + _encoding_cache = _encoding + return _encoding + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8da7b93699e..024b89f5dba 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5166,6 +5166,34 @@ "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0 + }, + "azure_ai/deepseek-v3.2": { + "input_cost_per_token": 5.8e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.68e-06, + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "azure_ai/deepseek-v3.2-speciale": { + "input_cost_per_token": 5.8e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.68e-06, + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true }, "azure_ai/deepseek-r1": { "input_cost_per_token": 1.35e-06, @@ -6723,8 +6751,8 @@ "input_cost_per_token": 3e-06, "litellm_provider": "anthropic", "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -6752,8 +6780,8 @@ "input_cost_per_token": 3e-06, "litellm_provider": "anthropic", "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -14704,6 +14732,98 @@ "supports_web_search": true, "tpm": 800000 }, + "gemini/gemini-3-flash-preview": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 3e-06, + "output_cost_per_token": 3e-06, + "rpm": 2000, + "source": "https://ai.google.dev/pricing/gemini-3", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, + "gemini-3-flash-preview": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 3e-06, + "output_cost_per_token": 3e-06, + "source": "https://ai.google.dev/pricing/gemini-3", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true + }, "gemini/gemini-2.5-pro-exp-03-25": { "cache_read_input_token_cost": 0.0, "input_cost_per_token": 0.0, @@ -15185,6 +15305,301 @@ "video" ] }, + "github_copilot/claude-haiku-4.5": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/claude-opus-4.5": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/claude-opus-41": { + "litellm_provider": "github_copilot", + "max_input_tokens": 80000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions" + ], + "supports_vision": true + }, + "github_copilot/claude-sonnet-4": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/claude-sonnet-4.5": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gemini-2.5-pro": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gemini-3-pro-preview": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gpt-3.5-turbo": { + "litellm_provider": "github_copilot", + "max_input_tokens": 16384, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true + }, + "github_copilot/gpt-3.5-turbo-0613": { + "litellm_provider": "github_copilot", + "max_input_tokens": 16384, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true + }, + "github_copilot/gpt-4": { + "litellm_provider": "github_copilot", + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true + }, + "github_copilot/gpt-4-0613": { + "litellm_provider": "github_copilot", + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true + }, + "github_copilot/gpt-4-o-preview": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true + }, + "github_copilot/gpt-4.1": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-4.1-2025-04-14": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-41-copilot": { + "litellm_provider": "github_copilot", + "mode": "completion" + }, + "github_copilot/gpt-4o": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gpt-4o-2024-05-13": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gpt-4o-2024-08-06": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true + }, + "github_copilot/gpt-4o-2024-11-20": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gpt-4o-mini": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true + }, + "github_copilot/gpt-4o-mini-2024-07-18": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true + }, + "github_copilot/gpt-5": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions", + "/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-5-mini": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-5.1": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions", + "/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-5.1-codex-max": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-5.2": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions", + "/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/text-embedding-3-small": { + "litellm_provider": "github_copilot", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding" + }, + "github_copilot/text-embedding-3-small-inference": { + "litellm_provider": "github_copilot", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding" + }, + "github_copilot/text-embedding-ada-002": { + "litellm_provider": "github_copilot", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding" + }, "google.gemma-3-12b-it": { "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", @@ -16350,6 +16765,34 @@ "/v1/audio/transcriptions" ] }, + "gpt-image-1.5": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_token": 1e-05, + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3.2e-05, + "supported_endpoints": [ + "/v1/images/generations" + ], + "supports_vision": true + }, + "gpt-image-1.5-2025-12-16": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_token": 1e-05, + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3.2e-05, + "supported_endpoints": [ + "/v1/images/generations" + ], + "supports_vision": true + }, "gpt-5": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_flex": 6.25e-08, @@ -22282,7 +22725,7 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/openai/gpt-5.2": { + "openrouter/openai/gpt-5.2": { "input_cost_per_image": 0, "cache_read_input_token_cost": 1.75e-07, "input_cost_per_token": 1.75e-06, @@ -30628,11 +31071,11 @@ "litellm_provider": "fireworks_ai", "mode": "embedding" }, - "fireworks_ai/accounts/fireworks/models/qwen3-embedding-8b": { + "fireworks_ai/accounts/fireworks/models/": { "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, - "input_cost_per_token": 0.0, + "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, "litellm_provider": "fireworks_ai", "mode": "embedding" diff --git a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py b/litellm/proxy/_experimental/mcp_server/ui_session_utils.py index 6572b831a27..37a3228ebf0 100644 --- a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py +++ b/litellm/proxy/_experimental/mcp_server/ui_session_utils.py @@ -16,9 +16,9 @@ def clone_user_api_key_auth_with_team( """Return a deep copy of the auth context with a different team id.""" try: - cloned_auth = user_api_key_auth.model_copy(deep=True) + cloned_auth = user_api_key_auth.model_copy() except AttributeError: - cloned_auth = user_api_key_auth.copy(deep=True) # type: ignore[attr-defined] + cloned_auth = user_api_key_auth.copy() # type: ignore[attr-defined] cloned_auth.team_id = team_id return cloned_auth diff --git a/litellm/proxy/_experimental/out/assets/logos/pydantic.svg b/litellm/proxy/_experimental/out/assets/logos/pydantic.svg new file mode 100644 index 00000000000..0ff8e5c44c7 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/pydantic.svg @@ -0,0 +1,5 @@ + + + diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml index b12d5ba0fe1..b993b9cdfef 100644 --- a/litellm/proxy/_super_secret_config.yaml +++ b/litellm/proxy/_super_secret_config.yaml @@ -81,13 +81,13 @@ model_list: # # default_team_settings: # # - team_id: proj1 # # success_callback: ["langfuse"] -# # langfuse_public_key: pk-lf-a65841e9-5192-4397-a679-cfff029fd5b0 -# # langfuse_secret: sk-lf-d58c2891-3717-4f98-89dd-df44826215fd +# # langfuse_public_key: os.environ/LANGFUSE_PUBLIC_KEY +# # langfuse_secret: os.environ/LANGFUSE_SECRET # # langfuse_host: https://us.cloud.langfuse.com # # - team_id: proj2 # # success_callback: ["langfuse"] -# # langfuse_public_key: pk-lf-3d789fd1-f49f-4e73-a7d9-1b4e11acbf9a -# # langfuse_secret: sk-lf-11b13aca-b0d4-4cde-9d54-721479dace6d +# # langfuse_public_key: os.environ/LANGFUSE_PUBLIC_KEY +# # langfuse_secret: os.environ/LANGFUSE_SECRET # # langfuse_host: https://us.cloud.langfuse.com assistant_settings: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index fcc4097e452..6b646086d5e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -418,6 +418,13 @@ class LiteLLMRoutes(enum.Enum): "/models/{model_name}:countTokens", "/models/{model_name}:generateContent", "/models/{model_name}:streamGenerateContent", + # Google Interactions API + "/interactions", + "/v1beta/interactions", + "/interactions/{interaction_id}", + "/v1beta/interactions/{interaction_id}", + "/interactions/{interaction_id}/cancel", + "/v1beta/interactions/{interaction_id}/cancel", ] apply_guardrail_routes = [ @@ -2664,6 +2671,9 @@ class SpendLogsMetadata(TypedDict): cold_storage_object_key: Optional[ str ] # S3/GCS object key for cold storage retrieval + litellm_overhead_time_ms: Optional[ + float + ] # LiteLLM overhead time in milliseconds class SpendLogsPayload(TypedDict): diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index c4d0d2f8f1c..7a71af1da5c 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -616,6 +616,14 @@ def get_model_from_request( if match: model = match.group(1) + # If still not found, extract from Vertex AI passthrough route + # Pattern: /vertex_ai/.../models/{model_id}:* + # Example: /vertex_ai/v1/.../models/gemini-1.5-pro:generateContent + if model is None and "/vertex" in route.lower(): + vertex_match = re.search(r"/models/([^/:]+)", route) + if vertex_match: + model = vertex_match.group(1) + return model diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 03b9ac3deaa..086105042e8 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -573,6 +573,7 @@ async def list_batches( if target_model_names is None: raise ValueError("target_model_names is required for this routing scenario") model = target_model_names.split(",")[0] + data.pop("model", None) response = await llm_router.alist_batches( model=model, after=after, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 3f04ce39336..8637bc88c57 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -351,6 +351,10 @@ class ProxyBaseLLMRequestProcessing: "aget_skill", "adelete_skill", "anthropic_messages", + "acreate_interaction", + "aget_interaction", + "adelete_interaction", + "acancel_interaction", ], version: Optional[str] = None, user_model: Optional[str] = None, @@ -476,6 +480,10 @@ class ProxyBaseLLMRequestProcessing: "aget_skill", "adelete_skill", "anthropic_messages", + "acreate_interaction", + "aget_interaction", + "adelete_interaction", + "acancel_interaction", ], proxy_logging_obj: ProxyLogging, general_settings: dict, diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 647abb73648..9f228bb1184 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -21,11 +21,11 @@ router = APIRouter() class CredentialHelperUtils: @staticmethod - def encrypt_credential_values(credential: CredentialItem) -> CredentialItem: + def encrypt_credential_values(credential: CredentialItem, new_encryption_key: Optional[str] = None) -> CredentialItem: """Encrypt values in credential.credential_values and add to DB""" encrypted_credential_values = {} for key, value in (credential.credential_values or {}).items(): - encrypted_credential_values[key] = encrypt_value_helper(value) + encrypted_credential_values[key] = encrypt_value_helper(value, new_encryption_key) # Return a new object to avoid mutating the caller's credential, which # is kept in memory and should remain unencrypted. @@ -246,7 +246,7 @@ async def delete_credential( def update_db_credential( - db_credential: CredentialItem, updated_patch: CredentialItem + db_credential: CredentialItem, updated_patch: CredentialItem, new_encryption_key: Optional[str] = None ) -> CredentialItem: """ Update a credential in the DB. @@ -258,7 +258,8 @@ def update_db_credential( ) encrypted_credential = CredentialHelperUtils.encrypt_credential_values( - updated_patch + updated_patch, + new_encryption_key, ) # update model name if encrypted_credential.credential_name: diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 72620259b1a..cc0719235ac 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -1,9 +1,11 @@ -from fastapi import APIRouter, Depends, Request, Response, HTTPException -from fastapi.responses import StreamingResponse +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response +from fastapi.responses import ORJSONResponse, StreamingResponse from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth - +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.types.llms.vertex_ai import TokenCountDetailsResponse @@ -25,8 +27,13 @@ async def google_generate_content( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): - from litellm.proxy.proxy_server import llm_router, general_settings, proxy_config, version from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + version, + ) data = await _read_request_body(request=request) if "model" not in data: @@ -63,8 +70,13 @@ async def google_stream_generate_content( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): - from litellm.proxy.proxy_server import llm_router, general_settings, proxy_config, version from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + version, + ) data = await _read_request_body(request=request) @@ -89,8 +101,8 @@ async def google_stream_generate_content( response = await llm_router.agenerate_content_stream(**data) # Check if response is an async iterator (streaming response) - if hasattr(response, "__aiter__"): - return StreamingResponse(response, media_type="text/event-stream") + if response is not None and hasattr(response, "__aiter__"): + return StreamingResponse(content=response, media_type="text/event-stream") return response @@ -167,3 +179,299 @@ async def google_count_tokens(request: Request, model_name: str): totalTokens=0, promptTokensDetails=[], ) + + +# ============================================================ +# Google Interactions API Endpoints +# Per OpenAPI spec: https://ai.google.dev/static/api/interactions.openapi.json +# ============================================================ + + +@router.post( + "/v1beta/interactions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["interactions"], +) +@router.post( + "/interactions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["interactions"], +) +async def create_interaction( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Create a new interaction using Google's Interactions API. + + Per OpenAPI spec: POST /{api_version}/interactions + + Supports both model interactions and agent interactions: + - Model: Provide `model` parameter (e.g., "gemini-2.5-flash") + - Agent: Provide `agent` parameter (e.g., "deep-research-pro-preview-12-2025") + + Example: + ```bash + curl -X POST "http://localhost:4000/v1beta/interactions" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemini/gemini-2.5-flash", + "input": "Hello, how are you?" + }' + ``` + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + data = await _read_request_body(request=request) + + # Default to gemini provider for interactions + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = "gemini" + + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="acreate_interaction", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.get( + "/v1beta/interactions/{interaction_id}", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["interactions"], +) +@router.get( + "/interactions/{interaction_id}", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["interactions"], +) +async def get_interaction( + request: Request, + interaction_id: str, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get an interaction by ID. + + Per OpenAPI spec: GET /{api_version}/interactions/{interaction_id} + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + data = {"interaction_id": interaction_id, "custom_llm_provider": "gemini"} + + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="aget_interaction", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.delete( + "/v1beta/interactions/{interaction_id}", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["interactions"], +) +@router.delete( + "/interactions/{interaction_id}", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["interactions"], +) +async def delete_interaction( + request: Request, + interaction_id: str, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Delete an interaction by ID. + + Per OpenAPI spec: DELETE /{api_version}/interactions/{interaction_id} + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + data = {"interaction_id": interaction_id, "custom_llm_provider": "gemini"} + + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="adelete_interaction", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.post( + "/v1beta/interactions/{interaction_id}/cancel", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["interactions"], +) +@router.post( + "/interactions/{interaction_id}/cancel", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["interactions"], +) +async def cancel_interaction( + request: Request, + interaction_id: str, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Cancel an interaction by ID. + + Per OpenAPI spec: POST /{api_version}/interactions/{interaction_id}:cancel + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + data = {"interaction_id": interaction_id, "custom_llm_provider": "gemini"} + + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="acancel_interaction", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index fb14ccce50c..62c997659bd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -605,13 +605,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): """ Only raise exception for "BLOCKED" actions, not for "ANONYMIZED" actions. - If `self.mask_request_content` or `self.mask_response_content` is set to `True`, then use the output from the guardrail to mask the request or response content. + If `self.mask_request_content` or `self.mask_response_content` is set to `True`, + then use the output from the guardrail to mask the request or response content. + + However, even with masking enabled, content with action="BLOCKED" should still + raise an exception, only content with action="ANONYMIZED" should be masked. """ - # if user opted into masking, return False. since we'll use the masked output from the guardrail - if self.mask_request_content or self.mask_response_content: - return False - # if no intervention, return False if response.get("action") != "GUARDRAIL_INTERVENED": return False diff --git a/litellm/proxy/guardrails/guardrail_hooks/grayswan/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/grayswan/__init__.py index 389340014f8..99f58f654a7 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/grayswan/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/grayswan/__init__.py @@ -40,6 +40,12 @@ def initialize_guardrail( ), categories=_get_config_value(litellm_params, optional_params, "categories"), policy_id=_get_config_value(litellm_params, optional_params, "policy_id"), + streaming_end_of_stream_only=_get_config_value( + litellm_params, optional_params, "streaming_end_of_stream_only" + ) or False, + streaming_sampling_rate=_get_config_value( + litellm_params, optional_params, "streaming_sampling_rate" + ) or 5, event_hook=litellm_params.mode, default_on=litellm_params.default_on, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py index e1d91ee908d..38e75ebabbc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py +++ b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py @@ -1,26 +1,25 @@ """Gray Swan Cygnal guardrail integration.""" import os -from typing import Any, Dict, Literal, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional from fastapi import HTTPException from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, - log_guardrail_information, + ModifyResponseException, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.common_utils.callback_utils import ( - add_guardrail_to_applied_guardrails_header, -) from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.utils import Choices, LLMResponseTypes, ModelResponse +from litellm.types.utils import GenericGuardrailAPIInputs + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj class GraySwanGuardrailMissingSecrets(Exception): @@ -35,6 +34,15 @@ class GraySwanGuardrail(CustomGuardrail): """ Guardrail that calls Gray Swan's Cygnal monitoring endpoint. + Uses the unified guardrail system via `apply_guardrail` method, + which automatically works with all LiteLLM endpoints: + - OpenAI Chat Completions + - OpenAI Responses API + - OpenAI Text Completions + - Anthropic Messages + - Image Generation + - And more... + see: https://docs.grayswan.ai/cygnal/monitor-requests """ @@ -54,6 +62,8 @@ class GraySwanGuardrail(CustomGuardrail): reasoning_mode: Optional[str] = None, categories: Optional[Dict[str, str]] = None, policy_id: Optional[str] = None, + streaming_end_of_stream_only: bool = False, + streaming_sampling_rate: int = 5, **kwargs: Any, ) -> None: self.async_handler = get_async_httpx_client( @@ -88,6 +98,16 @@ class GraySwanGuardrail(CustomGuardrail): self.categories = categories self.policy_id = policy_id + # Streaming configuration + self.streaming_end_of_stream_only = streaming_end_of_stream_only + self.streaming_sampling_rate = streaming_sampling_rate + + verbose_proxy_logger.debug( + "GraySwan __init__: streaming_end_of_stream_only=%s, streaming_sampling_rate=%s", + streaming_end_of_stream_only, + streaming_sampling_rate, + ) + supported_event_hooks = [ GuardrailEventHooks.pre_call, GuardrailEventHooks.during_call, @@ -101,217 +121,227 @@ class GraySwanGuardrail(CustomGuardrail): ) # ------------------------------------------------------------------ - # Guardrail hook entry points + # Debug override to trace post_call issues # ------------------------------------------------------------------ - @log_guardrail_information - async def async_pre_call_hook( + def should_run_guardrail(self, data, event_type) -> bool: + """Override to add debug logging.""" + result = super().should_run_guardrail(data, event_type) + # Check if apply_guardrail is in __dict__ + has_apply_guardrail = "apply_guardrail" in type(self).__dict__ + verbose_proxy_logger.debug( + "GraySwan DEBUG: should_run_guardrail event_type=%s, result=%s, event_hook=%s, has_apply_guardrail=%s, class=%s", + event_type, + result, + self.event_hook, + has_apply_guardrail, + type(self).__name__, + ) + return result + + # ------------------------------------------------------------------ + # Unified Guardrail Interface (works with ALL endpoints automatically) + # ------------------------------------------------------------------ + + async def apply_guardrail( self, - user_api_key_dict: UserAPIKeyAuth, - cache, - data: dict, - call_type: Literal[ - "completion", - "text_completion", - "embeddings", - "image_generation", - "moderation", - "audio_transcription", - "pass_through_endpoint", - "rerank", - "mcp_call", - "anthropic_messages", - ], - ) -> Optional[Union[Exception, str, dict]]: - if ( - self.should_run_guardrail( - data=data, event_type=GuardrailEventHooks.pre_call - ) - is not True - ): - return data + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional["LiteLLMLoggingObj"] = None, + ) -> GenericGuardrailAPIInputs: + """ + Apply Gray Swan guardrail to extracted text content. - verbose_proxy_logger.debug("Gray Swan Guardrail: pre-call hook triggered") + This method is called by the unified guardrail system which handles + extracting text from any request format (OpenAI, Anthropic, etc.). - messages = data.get("messages") - if not messages: - verbose_proxy_logger.debug("Gray Swan Guardrail: No messages in data") - return data + Args: + inputs: Dictionary containing: + - texts: List of texts to scan + - images: Optional list of images (not currently used by GraySwan) + - tool_calls: Optional list of tool calls (not currently used) + request_data: The original request data + input_type: "request" for pre-call, "response" for post-call + logging_obj: Optional logging object - dynamic_body = self.get_guardrail_dynamic_request_body_params(data) or {} + Returns: + GenericGuardrailAPIInputs - texts may be replaced with violation message in passthrough mode + Raises: + HTTPException: If content is blocked (block mode) + Exception: If guardrail check fails + """ + # DEBUG: Log when apply_guardrail is called + verbose_proxy_logger.debug( + "GraySwan DEBUG: apply_guardrail called with input_type=%s, texts=%s", + input_type, + inputs.get("texts", [])[:100] if inputs.get("texts") else "NONE", + ) + + texts = inputs.get("texts", []) + if not texts: + verbose_proxy_logger.debug("Gray Swan Guardrail: No texts to scan") + return inputs + + verbose_proxy_logger.debug( + "Gray Swan Guardrail: Scanning %d text(s) for %s", + len(texts), + input_type, + ) + + # Convert texts to messages format for GraySwan API + # Use "user" role for request content, "assistant" for response content + role = "assistant" if input_type == "response" else "user" + messages = [{"role": role, "content": text} for text in texts] + + # Get dynamic params from request metadata + dynamic_body = self.get_guardrail_dynamic_request_body_params(request_data) or {} + + # Prepare and send payload payload = self._prepare_payload(messages, dynamic_body) if payload is None: - verbose_proxy_logger.debug( - "Gray Swan Guardrail: no content to scan; skipping request" - ) - return data + return inputs - await self.run_grayswan_guardrail(payload, data, GuardrailEventHooks.pre_call) - add_guardrail_to_applied_guardrails_header( - request_data=data, guardrail_name=self.guardrail_name + # Call GraySwan API + response_json = await self._call_grayswan_api(payload) + # Process response + is_output = input_type == "response" + result = self._process_response_internal( + response_json=response_json, + request_data=request_data, + inputs=inputs, + is_output=is_output, ) - return data - @log_guardrail_information - async def async_moderation_hook( - self, - data: dict, - user_api_key_dict: UserAPIKeyAuth, - call_type: Literal[ - "completion", - "embeddings", - "image_generation", - "moderation", - "audio_transcription", - "responses", - "mcp_call", - "anthropic_messages", - ], - ) -> Optional[Union[Exception, str, dict]]: - if ( - self.should_run_guardrail( - data=data, event_type=GuardrailEventHooks.during_call - ) - is not True - ): - return data - - verbose_proxy_logger.debug("GraySwan Guardrail: during-call hook triggered") - - messages = data.get("messages") - if not messages: - verbose_proxy_logger.debug("Gray Swan Guardrail: No messages in data") - return data - - dynamic_body = self.get_guardrail_dynamic_request_body_params(data) or {} - - payload = self._prepare_payload(messages, dynamic_body) - if payload is None: - verbose_proxy_logger.debug( - "Gray Swan Guardrail: no content to scan; skipping request" - ) - return data - - await self.run_grayswan_guardrail( - payload, data, GuardrailEventHooks.during_call - ) - add_guardrail_to_applied_guardrails_header( - request_data=data, guardrail_name=self.guardrail_name - ) - return data - - @log_guardrail_information - async def async_post_call_success_hook( - self, - data: dict, - user_api_key_dict: UserAPIKeyAuth, - response: LLMResponseTypes, - ) -> LLMResponseTypes: - if ( - self.should_run_guardrail( - data=data, event_type=GuardrailEventHooks.post_call - ) - is not True - ): - return response - - verbose_proxy_logger.debug("GraySwan Guardrail: post-call hook triggered") - - response_dict = response.model_dump() if hasattr(response, "model_dump") else {} # type: ignore[union-attr] - response_messages = [ - msg if isinstance(msg, dict) else msg.model_dump() - for choice in response_dict.get("choices", []) - if isinstance(choice, dict) - for msg in [choice.get("message")] - if msg is not None - ] - - if not response_messages: - verbose_proxy_logger.debug( - "Gray Swan Guardrail: no response messages detected; skipping post-call scan" - ) - return response - - dynamic_body = self.get_guardrail_dynamic_request_body_params(data) or {} - - payload = self._prepare_payload(response_messages, dynamic_body) - if payload is None: - verbose_proxy_logger.debug( - "Gray Swan Guardrail: no content to scan; skipping request" - ) - return response - - await self.run_grayswan_guardrail(payload, data, GuardrailEventHooks.post_call) - - # If passthrough mode and detection info exists, replace response content with violation message - if self.on_flagged_action == "passthrough" and "metadata" in data: - guardrail_detections = data.get("metadata", {}).get( - "guardrail_detections", [] - ) - if guardrail_detections: - # Replace the model response content with guardrail violation message - violation_message = self._format_violation_message( - guardrail_detections, is_output=True - ) - - # Handle ModelResponse (OpenAI-style chat/text completions) - # Use isinstance to narrow the type for mypy - if isinstance(response, ModelResponse) and response.choices: - verbose_proxy_logger.debug( - "Gray Swan Guardrail: Replacing response content in ModelResponse format" - ) - for choice in response.choices: - # Handle chat completion format (message.content) - # Choices has message attribute, StreamingChoices has delta - if isinstance(choice, Choices) and hasattr(choice, "message") and hasattr( - choice.message, "content" - ): - choice.message.content = violation_message - # Handle text completion format (text) - # Text attribute might be set dynamically, use setattr - elif hasattr(choice, "text"): - setattr(choice, "text", violation_message) - - # Update finish_reason to indicate content filtering - if hasattr(choice, "finish_reason"): - choice.finish_reason = "content_filter" - - # Handle AnthropicMessagesResponse format - elif hasattr(response, "content") and isinstance(response.content, list): # type: ignore - verbose_proxy_logger.debug( - "Gray Swan Guardrail: Replacing response content in Anthropic Messages format" - ) - # Replace content blocks with text block containing violation message - response.content = [ # type: ignore - {"type": "text", "text": violation_message} - ] - # Update stop_reason if present - if hasattr(response, "stop_reason"): - response.stop_reason = "end_turn" # type: ignore - - else: - verbose_proxy_logger.warning( - "Gray Swan Guardrail: Passthrough mode enabled but response format not recognized. " - "Cannot replace content. Response type: %s", - type(response).__name__, - ) - - add_guardrail_to_applied_guardrails_header( - request_data=data, guardrail_name=self.guardrail_name - ) - return response + return result # ------------------------------------------------------------------ - # Core GraySwan interaction + # Legacy Test Interface (for backward compatibility) # ------------------------------------------------------------------ - async def run_grayswan_guardrail( + async def run_grayswan_guardrail(self, payload: dict) -> Dict[str, Any]: + """ + Run the GraySwan guardrail on a payload. + + This is a legacy method for testing purposes. + + Args: + payload: The payload to scan + + Returns: + Dict containing the GraySwan API response + """ + response_json = await self._call_grayswan_api(payload) + # Call the legacy response processor (for test compatibility) + self._process_grayswan_response(response_json) + return response_json + + def _process_grayswan_response( self, - payload: dict, + response_json: dict, data: Optional[dict] = None, hook_type: Optional[GuardrailEventHooks] = None, - ): + ) -> None: + """ + Legacy method for processing GraySwan API responses. + + This method is maintained for backward compatibility with existing tests. + It handles the test scenarios where responses need to be processed with + knowledge of the request context (pre/during/post call hooks). + + Args: + response_json: Response from GraySwan API + data: Optional request data (for passthrough exceptions) + hook_type: Optional GuardrailEventHooks for determining behavior + """ + violation_score = float(response_json.get("violation", 0.0) or 0.0) + violated_rules = response_json.get("violated_rules", []) + mutation_detected = response_json.get("mutation") + ipi_detected = response_json.get("ipi") + + flagged = violation_score >= self.violation_threshold + if not flagged: + verbose_proxy_logger.debug( + "Gray Swan Guardrail: content passed (score=%s, threshold=%s)", + violation_score, + self.violation_threshold, + ) + return + + verbose_proxy_logger.warning( + "Gray Swan Guardrail: violation score %.3f exceeds threshold %.3f", + violation_score, + self.violation_threshold, + ) + + detection_info = { + "guardrail": "grayswan", + "flagged": True, + "violation_score": violation_score, + "violated_rules": violated_rules, + "mutation": mutation_detected, + "ipi": ipi_detected, + } + + # Determine if this is input (pre-call/during-call) or output (post-call) + if hook_type is not None: + is_input = hook_type in [ + GuardrailEventHooks.pre_call, + GuardrailEventHooks.during_call, + ] + else: + is_input = True + + if self.on_flagged_action == "block": + violation_location = "output" if (not is_input) else "input" + raise HTTPException( + status_code=400, + detail={ + "error": "Blocked by Gray Swan Guardrail", + "violation_location": violation_location, + "violation": violation_score, + "violated_rules": violated_rules, + "mutation": mutation_detected, + "ipi": ipi_detected, + }, + ) + elif self.on_flagged_action == "passthrough": + # For passthrough mode, we need to handle violations + detections = [detection_info] + violation_message = self._format_violation_message( + detections, is_output=not is_input + ) + verbose_proxy_logger.info( + "Gray Swan Guardrail: Passthrough mode - handling violation" + ) + + # If hook_type is provided and in pre/during call, raise exception + if hook_type in [GuardrailEventHooks.pre_call, GuardrailEventHooks.during_call]: + # Raise ModifyResponseException to short-circuit LLM call + if data is None: + data = {} + self.raise_passthrough_exception( + violation_message=violation_message, + request_data=data, + detection_info=detection_info, + ) + elif hook_type == GuardrailEventHooks.post_call: + # For post-call, store detection info in metadata + if data is None: + data = {} + if "metadata" not in data: + data["metadata"] = {} + if "guardrail_detections" not in data["metadata"]: + data["metadata"]["guardrail_detections"] = [] + data["metadata"]["guardrail_detections"].append(detection_info) + + # ------------------------------------------------------------------ + # Core GraySwan API interaction + # ------------------------------------------------------------------ + + async def _call_grayswan_api(self, payload: dict) -> Dict[str, Any]: + """Call the GraySwan monitoring API.""" headers = self._prepare_headers() try: @@ -326,15 +356,107 @@ class GraySwanGuardrail(CustomGuardrail): verbose_proxy_logger.debug( "Gray Swan Guardrail: monitor response %s", safe_dumps(result) ) + return result except HTTPException: raise - except Exception as exc: # pragma: no cover - depends on HTTP client behaviour + except Exception as exc: verbose_proxy_logger.exception( "Gray Swan Guardrail: API request failed: %s", exc ) raise GraySwanGuardrailAPIError(str(exc)) from exc - self._process_grayswan_response(result, data, hook_type) + def _process_response_internal( + self, + response_json: Dict[str, Any], + request_data: dict, + inputs: GenericGuardrailAPIInputs, + is_output: bool, + ) -> GenericGuardrailAPIInputs: + """ + Process GraySwan API response and handle violations. + + Args: + response_json: Response from GraySwan API + request_data: Original request data + inputs: The inputs being scanned + is_output: True if scanning model output, False for input + + Returns: + GenericGuardrailAPIInputs - possibly modified with violation message + + Raises: + HTTPException: If content is blocked (block mode) + """ + violation_score = float(response_json.get("violation", 0.0) or 0.0) + violated_rules = response_json.get("violated_rule_descriptions", []) + mutation_detected = response_json.get("mutation") + ipi_detected = response_json.get("ipi") + + flagged = violation_score >= self.violation_threshold + if not flagged: + verbose_proxy_logger.debug( + "Gray Swan Guardrail: content passed (score=%s, threshold=%s)", + violation_score, + self.violation_threshold, + ) + return inputs + + verbose_proxy_logger.warning( + "Gray Swan Guardrail: violation score %.3f exceeds threshold %.3f", + violation_score, + self.violation_threshold, + ) + + detection_info = { + "guardrail": "grayswan", + "flagged": True, + "violation_score": violation_score, + "violated_rules": violated_rules, + "mutation": mutation_detected, + "ipi": ipi_detected, + } + + if self.on_flagged_action == "block": + violation_location = "output" if is_output else "input" + raise HTTPException( + status_code=400, + detail={ + "error": "Blocked by Gray Swan Guardrail", + "violation_location": violation_location, + "violation": violation_score, + "violated_rules": violated_rules, + "mutation": mutation_detected, + "ipi": ipi_detected, + }, + ) + elif self.on_flagged_action == "monitor": + verbose_proxy_logger.info( + "Gray Swan Guardrail: Monitoring mode - allowing flagged content" + ) + return inputs + elif self.on_flagged_action == "passthrough": + # Replace content with violation message + violation_message = self._format_violation_message( + detection_info, is_output=is_output + ) + verbose_proxy_logger.info( + "Gray Swan Guardrail: Passthrough mode - replacing content with violation message" + ) + + if not is_output: + # For pre-call (request), raise exception to short-circuit LLM call + # and return synthetic response with violation message + self.raise_passthrough_exception( + violation_message=violation_message, + request_data=request_data, + detection_info=detection_info, + ) + + # For post-call (response), replace texts and let unified system apply them + inputs["texts"] = [violation_message] + return inputs + + return inputs # ------------------------------------------------------------------ # Helpers @@ -348,10 +470,9 @@ class GraySwanGuardrail(CustomGuardrail): } def _prepare_payload( - self, messages: list[dict], dynamic_body: dict + self, messages: List[Dict[str, str]], dynamic_body: dict ) -> Optional[Dict[str, Any]]: - payload: Dict[str, Any] = {} - payload["messages"] = messages + payload: Dict[str, Any] = {"messages": messages} categories = dynamic_body.get("categories") or self.categories if categories: @@ -367,128 +488,41 @@ class GraySwanGuardrail(CustomGuardrail): return payload - def _process_grayswan_response( - self, - response_json: Dict[str, Any], - data: Optional[dict] = None, - hook_type: Optional[GuardrailEventHooks] = None, - ) -> None: - violation_score = float(response_json.get("violation", 0.0) or 0.0) - violated_rules = response_json.get("violated_rules", []) - mutation_detected = response_json.get("mutation") - ipi_detected = response_json.get("ipi") - - flagged = violation_score >= self.violation_threshold - if not flagged: - verbose_proxy_logger.debug( - "Gray Swan Guardrail: request passed (score=%s, rules=%s)", - violation_score, - violated_rules, - ) - return - - verbose_proxy_logger.warning( - "Gray Swan Guardrail: violation score %.3f exceeds threshold %.3f", - violation_score, - self.violation_threshold, - ) - - if self.on_flagged_action == "block": - # Determine if violation was in input or output - violation_location = ( - "output" - if hook_type == GuardrailEventHooks.post_call - else "input" - ) - raise HTTPException( - status_code=400, - detail={ - "error": "Blocked by Gray Swan Guardrail", - "violation_location": violation_location, - "violation": violation_score, - "violated_rules": violated_rules, - "mutation": mutation_detected, - "ipi": ipi_detected, - }, - ) - elif self.on_flagged_action == "monitor": - verbose_proxy_logger.info( - "Gray Swan Guardrail: Monitoring mode - allowing flagged content to proceed" - ) - elif self.on_flagged_action == "passthrough": - # Store detection info - detection_info = { - "guardrail": "grayswan", - "flagged": True, - "violation_score": violation_score, - "violated_rules": violated_rules, - "mutation": mutation_detected, - "ipi": ipi_detected, - } - - # For pre_call and during_call, raise exception to short-circuit LLM call - if hook_type in ( - GuardrailEventHooks.pre_call, - GuardrailEventHooks.during_call, - ): - verbose_proxy_logger.info( - "Gray Swan Guardrail: Passthrough mode - raising exception to short-circuit LLM call" - ) - violation_message = self._format_violation_message( - [detection_info], is_output=False - ) - self.raise_passthrough_exception( - violation_message=violation_message, - request_data=data or {}, - detection_info=detection_info, - ) - - # For post_call, store in metadata to replace response later - verbose_proxy_logger.info( - "Gray Swan Guardrail: Passthrough mode - storing detection info in metadata" - ) - if data is not None: - if "metadata" not in data: - data["metadata"] = {} - if "guardrail_detections" not in data["metadata"]: - data["metadata"]["guardrail_detections"] = [] - data["metadata"]["guardrail_detections"].append(detection_info) - def _format_violation_message( - self, guardrail_detections: list, is_output: bool = False + self, detection_info: Any, is_output: bool = False ) -> str: """ - Format guardrail detections into a user-friendly violation message. + Format detection info into a user-friendly violation message. Args: - guardrail_detections: List of detection info dictionaries - is_output: True if violation is in model output (post_call), False if in input (pre_call/during_call) + detection_info: Can be either: + - A single dict with violation_score, violated_rules, mutation, ipi keys + - A list of such dicts (legacy format) + is_output: True if violation is in model output, False if in input Returns: Formatted violation message string """ - if not guardrail_detections: - return "Content was flagged by guardrail" + # Handle legacy format where detection_info is a list + if isinstance(detection_info, list) and len(detection_info) > 0: + detection_info = detection_info[0] + + violation_score = detection_info.get("violation_score", 0.0) + violated_rules = detection_info.get("violated_rules", []) + mutation = detection_info.get("mutation", False) + ipi = detection_info.get("ipi", False) - # Get the most recent detection (should be from this guardrail) - detection = guardrail_detections[-1] - - violation_score = detection.get("violation_score", 0.0) - violated_rules = detection.get("violated_rules", []) - mutation = detection.get("mutation", False) - ipi = detection.get("ipi", False) - - # Indicate whether violation was in input or output violation_location = "the model response" if is_output else "input query" message_parts = [ - f"Sorry I can't help with that. According to the Gray Swan Cygnal Guardrail, the {violation_location} has a violation score of {violation_score:.2f}.", + f"Sorry I can't help with that. According to the Gray Swan Cygnal Guardrail, " + f"the {violation_location} has a violation score of {violation_score:.2f}.", ] if violated_rules: - message_parts.append( - f"It was violating the rule(s): {', '.join(map(str, violated_rules))}." - ) + formatted_rules = self._format_violated_rules(violated_rules) + if formatted_rules: + message_parts.append(f"It was violating the rule(s): {formatted_rules}.") if mutation: message_parts.append( @@ -496,31 +530,51 @@ class GraySwanGuardrail(CustomGuardrail): ) if ipi: - message_parts.append("Indirect Prompt Injection was DETECTED.") + message_parts.append( + "Indirect Prompt Injection was DETECTED." + ) return "\n".join(message_parts) - def _resolve_threshold(self, threshold: Optional[float]) -> float: - if threshold is not None: - return min(max(threshold, 0.0), 1.0) + def _format_violated_rules(self, violated_rules: List) -> str: + """Format violated rules list into a readable string.""" + formatted: List[str] = [] + for rule in violated_rules: + if isinstance(rule, dict): + # New format: {'rule': 6, 'name': 'Illegal Activities...', 'description': '...'} + rule_num = rule.get("rule", "") + rule_name = rule.get("name", "") + rule_desc = rule.get("description", "") + if rule_num and rule_name: + if rule_desc: + formatted.append(f"#{rule_num} {rule_name}: {rule_desc}") + else: + formatted.append(f"#{rule_num} {rule_name}") + elif rule_name: + formatted.append(rule_name) + else: + formatted.append(str(rule)) + else: + # Legacy format: simple value + formatted.append(str(rule)) + + return ", ".join(formatted) + + def _resolve_threshold(self, value: Optional[float]) -> float: + if value is not None: + return float(value) + env_val = os.getenv("GRAYSWAN_VIOLATION_THRESHOLD") + if env_val: + try: + return float(env_val) + except ValueError: + pass return 0.5 - def _resolve_reasoning_mode(self, candidate: Optional[str]) -> Optional[str]: - if candidate is None: - return None - normalised = candidate.strip().lower() - if normalised in self.SUPPORTED_REASONING_MODES: - return normalised - verbose_proxy_logger.warning( - "Gray Swan Guardrail: ignoring unsupported reasoning_mode '%s'", - candidate, - ) + def _resolve_reasoning_mode(self, value: Optional[str]) -> Optional[str]: + if value and value.lower() in self.SUPPORTED_REASONING_MODES: + return value.lower() + env_val = os.getenv("GRAYSWAN_REASONING_MODE") + if env_val and env_val.lower() in self.SUPPORTED_REASONING_MODES: + return env_val.lower() return None - - @staticmethod - def get_config_model(): - from litellm.types.proxy.guardrails.guardrail_hooks.grayswan import ( - GraySwanGuardrailConfigModel, - ) - - return GraySwanGuardrailConfigModel diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json index 193d0868072..d8ec22f81a1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json @@ -317,6 +317,55 @@ "category": "PII Patterns", "action": "MASK", "description": "Detects Dutch BSN numbers with contextual keywords" + }, + { + "name": "br_cpf", + "display_name": "CPF - Brazilian Personal Tax ID (Formatted)", + "pattern": "\\d{3}\\.\\d{3}\\.\\d{3}(-|/)\\d{2}", + "category": "Brazilian PII Patterns", + "description": "Detects Brazilian CPF numbers (XXX.XXX.XXX-XX or XXX.XXX.XXX/XX format)" + }, + { + "name": "br_cpf_unformatted", + "display_name": "CPF - Brazilian Personal Tax ID (Unformatted)", + "pattern": "\\b\\d{11}\\b", + "category": "Brazilian PII Patterns", + "description": "Detects Brazilian CPF numbers without formatting (11 digits)" + }, + { + "name": "br_phone_landline", + "display_name": "Brazilian Phone Number (Landline)", + "pattern": "(?:\\(?\\d{2}\\)?\\s?)?(?:9\\d{4}|\\d{4})-?\\d{4}", + "category": "Brazilian PII Patterns", + "description": "Detects Brazilian landline phone numbers with optional area code" + }, + { + "name": "br_phone_mobile", + "display_name": "Brazilian Mobile Phone Number", + "pattern": "(?:\\+\\d{1,3}\\s?)?(?:\\(?\\d{2}\\)?\\s?)?9\\d{4}-?\\d{4}", + "category": "Brazilian PII Patterns", + "description": "Detects Brazilian mobile phone numbers (9 prefix for mobile)" + }, + { + "name": "br_cep", + "display_name": "CEP - Brazilian Zip / Postal Code", + "pattern": "\\b\\d{5}-?\\d{3}\\b", + "category": "Brazilian PII Patterns", + "description": "Detects Brazilian CEP postal codes (XXXXX-XXX or XXXXXXXX format)" + }, + { + "name": "br_cnpj", + "display_name": "CNPJ - Brazilian Company Tax ID", + "pattern": "\\d{2}\\.\\d{3}\\.\\d{3}/\\d{4}-\\d{2}", + "category": "Brazilian PII Patterns", + "description": "Detects Brazilian CNPJ company registration numbers (XX.XXX.XXX/XXXX-XX format)" + }, + { + "name": "br_rg", + "display_name": "RG - Brazilian National Identity Card (SP, RJ, MG)", + "pattern": "\\b\\d{1,2}\\.\\d{3}\\.\\d{3}-[\\dXx]\\b", + "category": "Brazilian PII Patterns", + "description": "Detects Brazilian RG identity card numbers (common pattern for SP, RJ, MG states)" } ] } diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py index 0df610177e5..ef22b099300 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py @@ -164,7 +164,7 @@ class PillarGuardrail(CustomGuardrail): using the Pillar Security API. """ - SUPPORTED_ON_FLAGGED_ACTIONS = ["block", "monitor"] + SUPPORTED_ON_FLAGGED_ACTIONS = ["block", "monitor", "mask"] DEFAULT_ON_FLAGGED_ACTION = "monitor" SUPPORTED_FALLBACK_ACTIONS = ["allow", "block"] DEFAULT_FALLBACK_ACTION = "allow" @@ -280,6 +280,8 @@ class PillarGuardrail(CustomGuardrail): GuardrailEventHooks.pre_call, GuardrailEventHooks.during_call, GuardrailEventHooks.post_call, + GuardrailEventHooks.pre_mcp_call, + GuardrailEventHooks.during_mcp_call, ] super().__init__( @@ -773,6 +775,15 @@ class PillarGuardrail(CustomGuardrail): verbose_proxy_logger.warning("Pillar Guardrail: Threat detected") if self.on_flagged_action == "block": self._raise_pillar_detection_exception(pillar_response) + elif self.on_flagged_action == "mask": + verbose_proxy_logger.info("Pillar Guardrail: Masking mode - masking flagged content") + masked_messages = pillar_response.get("masked_session_messages", []) + if masked_messages: + original_data["messages"] = masked_messages + else: + verbose_proxy_logger.warning( + "Pillar Guardrail: Masking requested but no masked_session_messages in response" + ) elif self.on_flagged_action == "monitor": verbose_proxy_logger.info("Pillar Guardrail: Monitoring mode - allowing flagged content to proceed") @@ -788,14 +799,20 @@ class PillarGuardrail(CustomGuardrail): Raises: HTTPException: Always raises with security detection details """ + pillar_response_dict = { + "session_id": pillar_response.get("session_id"), + } + + # Conditionally include scanners and evidence based on config + if self.include_scanners: + pillar_response_dict["scanners"] = pillar_response.get("scanners", {}) + if self.include_evidence: + pillar_response_dict["evidence"] = pillar_response.get("evidence", []) + error_detail = { "error": "Blocked by Pillar Security Guardrail", "detection_message": "Security threats detected", - "pillar_response": { - "session_id": pillar_response.get("session_id"), - "scanners": pillar_response.get("scanners", {}), - "evidence": pillar_response.get("evidence", []), - }, + "pillar_response": pillar_response_dict, } verbose_proxy_logger.warning("Pillar Guardrail: Request blocked - Security threats detected") diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index cece49e99cb..0dac30f72b2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -180,7 +180,7 @@ class UnifiedLLMGuardrails(CustomLogger): call_type: Optional[CallTypesLiteral] = None if user_api_key_dict.request_route is not None: call_types = get_call_types_for_route(user_api_key_dict.request_route) - if call_types is not None: + if call_types is not None and len(call_types) > 0: call_type = call_types[0] if call_type is None: call_type = _infer_call_type(call_type=None, completion_response=response) @@ -213,7 +213,7 @@ class UnifiedLLMGuardrails(CustomLogger): return response - async def async_post_call_streaming_iterator_hook( + async def async_post_call_streaming_iterator_hook( # noqa: PLR0915 self, user_api_key_dict: UserAPIKeyAuth, response: Any, @@ -238,19 +238,36 @@ class UnifiedLLMGuardrails(CustomLogger): "guardrail_to_apply", None ) - # Get sampling rate from guardrail config or optional_params, default to 5 + # Get streaming configuration from guardrail or optional_params sampling_rate = 5 + end_of_stream_only = False # If True, only apply guardrail at end of stream + if guardrail_to_apply is not None: - # Check guardrail config first - guardrail_config = getattr(guardrail_to_apply, "guardrail_config", {}) - sampling_rate = guardrail_config.get( - "streaming_sampling_rate", sampling_rate + # Check direct attributes on guardrail first + sampling_rate = getattr( + guardrail_to_apply, "streaming_sampling_rate", sampling_rate ) + end_of_stream_only = getattr( + guardrail_to_apply, "streaming_end_of_stream_only", end_of_stream_only + ) + + # Also check guardrail_config dict if present + guardrail_config = getattr(guardrail_to_apply, "guardrail_config", {}) + if isinstance(guardrail_config, dict): + sampling_rate = guardrail_config.get( + "streaming_sampling_rate", sampling_rate + ) + end_of_stream_only = guardrail_config.get( + "streaming_end_of_stream_only", end_of_stream_only + ) # Also check optional_params as fallback sampling_rate = self.optional_params.get( "streaming_sampling_rate", sampling_rate ) + end_of_stream_only = self.optional_params.get( + "streaming_end_of_stream_only", end_of_stream_only + ) if guardrail_to_apply is None: async for item in response: @@ -306,6 +323,11 @@ class UnifiedLLMGuardrails(CustomLogger): yield remaining_item return + # If end_of_stream_only mode, yield chunks without processing + if end_of_stream_only: + yield item + continue + # Process chunk based on sampling rate if chunk_counter % sampling_rate == 0: diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index f8e86334f83..2d5f07dbf6e 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -19,6 +19,10 @@ from litellm.types.guardrails import ( LitellmParams, SupportedGuardrailIntegrations, ) +from litellm.proxy.guardrails.guardrail_hooks.grayswan import ( + GraySwanGuardrail, + initialize_guardrail as initialize_grayswan, +) from .guardrail_initializers import ( initialize_bedrock, @@ -36,9 +40,12 @@ guardrail_initializer_registry = { SupportedGuardrailIntegrations.PRESIDIO.value: initialize_presidio, SupportedGuardrailIntegrations.HIDE_SECRETS.value: initialize_hide_secrets, SupportedGuardrailIntegrations.TOOL_PERMISSION.value: initialize_tool_permission, + SupportedGuardrailIntegrations.GRAYSWAN.value: initialize_grayswan, } -guardrail_class_registry: Dict[str, Type[CustomGuardrail]] = {} +guardrail_class_registry: Dict[str, Type[CustomGuardrail]] = { + SupportedGuardrailIntegrations.GRAYSWAN.value: GraySwanGuardrail +} def get_guardrail_initializer_from_hooks(): diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 79e9838d115..030843376bf 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -30,9 +30,48 @@ from litellm.proxy.health_check import ( perform_health_check, run_with_timeout, ) +from litellm.secret_managers.main import get_secret #### Health ENDPOINTS #### + +def _resolve_os_environ_variables(params: dict) -> dict: + """ + Resolve os.environ/ environment variables in litellm_params. + + This function recursively processes dictionary values that start with "os.environ/" + by replacing them with the actual environment variable values. + + Args: + params: Dictionary containing litellm_params that may have os.environ/ values + + Returns: + Dictionary with os.environ/ values resolved to actual environment variable values + """ + if not isinstance(params, dict): + return params + + resolved_params = {} + for key, value in params.items(): + if isinstance(value, str) and value.startswith("os.environ/"): + # Resolve the environment variable + resolved_value = get_secret(value) + resolved_params[key] = resolved_value + elif isinstance(value, dict): + # Recursively resolve nested dictionaries + resolved_params[key] = _resolve_os_environ_variables(value) + elif isinstance(value, list): + # Handle lists that might contain dictionaries with os.environ/ values + resolved_params[key] = [ + _resolve_os_environ_variables(item) if isinstance(item, dict) else item + for item in value + ] + else: + resolved_params[key] = value + + return resolved_params + + router = APIRouter() services = Union[ Literal[ @@ -1166,21 +1205,41 @@ async def test_model_connection( Example: ```bash + # If model is configured in proxy_config.yaml, you only need to specify the model name: curl -X POST 'http://localhost:4000/health/test_connection' \\ -H 'Authorization: Bearer sk-1234' \\ -H 'Content-Type: application/json' \\ -d '{ "litellm_params": { - "model": "gpt-4", - "custom_llm_provider": "azure_ai", - "litellm_credential_name": null, - "api_key": "6xxxxxxx", - "api_base": "https://litellm8397336933.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21", + "model": "gpt-4o" + }, + "mode": "chat" + }' + + # The endpoint will automatically use api_key, api_base, etc. from proxy_config.yaml + + # You can also override specific params or test with custom credentials: + curl -X POST 'http://localhost:4000/health/test_connection' \\ + -H 'Authorization: Bearer sk-1234' \\ + -H 'Content-Type: application/json' \\ + -d '{ + "litellm_params": { + "model": "azure/gpt-4o", + "api_key": "os.environ/AZURE_OPENAI_API_KEY", + "api_base": "os.environ/AZURE_OPENAI_ENDPOINT", + "api_version": "2024-10-21" }, "mode": "chat" }' ``` + Note: + - If the model is configured in proxy_config.yaml, credentials (api_key, api_base, etc.) + will be automatically loaded from the config (with resolved environment variables). + - You can override specific params by including them in the request. + - You can use `os.environ/VARIABLE_NAME` syntax to reference environment variables, + which will be resolved automatically (same as in proxy_config.yaml). + Returns: dict: A dictionary containing the health check result with either success information or error details. """ @@ -1188,7 +1247,7 @@ async def test_model_connection( from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelManagementAuthChecks, ) - from litellm.proxy.proxy_server import premium_user, prisma_client + from litellm.proxy.proxy_server import llm_router, premium_user, prisma_client from litellm.types.router import Deployment, LiteLLM_Params try: @@ -1197,6 +1256,46 @@ async def test_model_connection( status_code=500, detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) + + # Get model name from litellm_params + request_litellm_params = litellm_params or {} + model_name = request_litellm_params.get("model") + + # Look up model configuration from router if model name is provided + # This gets the litellm_params from proxy config (with resolved env vars) + config_litellm_params = {} + if model_name and llm_router is not None: + try: + # First try to find by proxy model_name (e.g., "gpt-4o") + deployments = llm_router.get_model_list(model_name=model_name) + + # If not found, try to find by litellm model name (e.g., "azure/gpt-4o") + if not deployments or len(deployments) == 0: + all_deployments = llm_router.get_model_list(model_name=None) + if all_deployments: + for deployment in all_deployments: + if deployment.get("litellm_params", {}).get("model") == model_name: + deployments = [deployment] + break + + if deployments and len(deployments) > 0: + # Use the first deployment's litellm_params as base config + # These already have resolved environment variables from proxy config + config_litellm_params = deployments[0].get("litellm_params", {}).copy() + except Exception as e: + verbose_proxy_logger.debug( + f"Could not find model {model_name} in router: {e}. " + "Proceeding with request params only." + ) + + # Merge: config params (from proxy config) as base, request params override + # This allows users to override specific params while using config for credentials + merged_litellm_params = {**config_litellm_params, **request_litellm_params} + + # Resolve os.environ/ environment variables in any remaining request params + # This handles cases where user explicitly passes os.environ/ values to override config + litellm_params = _resolve_os_environ_variables(merged_litellm_params) + ## Auth check await ModelManagementAuthChecks.can_user_make_model_call( model_params=Deployment( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 9dc255bd79a..5b5723efc3d 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -843,6 +843,11 @@ async def add_litellm_data_to_request( # noqa: PLR0915 ) ) + # Add headers to metadata for guardrails to access (fixes #17477) + # Guardrails use metadata["headers"] to access request headers (e.g., User-Agent) + if _metadata_variable_name in data and isinstance(data[_metadata_variable_name], dict): + data[_metadata_variable_name]["headers"] = _headers + # check for forwardable headers data = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( data=data, headers=_headers, user_api_key_dict=user_api_key_dict diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index da44bda791d..8ea3122ce01 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2539,6 +2539,40 @@ async def _rotate_master_key( new_master_key=new_master_key, ) + # 5. process credentials table + try: + credentials = await prisma_client.db.litellm_credentialstable.find_many() + except Exception: + credentials = None + if credentials: + from litellm.proxy.credential_endpoints.endpoints import update_db_credential + + for cred in credentials: + try: + decrypted_cred = proxy_config.decrypt_credentials(cred) + encrypted_cred = update_db_credential( + db_credential=cred, + updated_patch=decrypted_cred, + new_encryption_key=new_master_key, + ) + credential_object_jsonified = jsonify_object(encrypted_cred.model_dump()) + await prisma_client.db.litellm_credentialstable.update( + where={"credential_name": cred.credential_name}, + data={ + **credential_object_jsonified, + "updated_by": user_api_key_dict.user_id, + }, + ) + except Exception as e: + verbose_proxy_logger.error( + f"Failed to re-encrypt credential {cred.credential_name}: {str(e)}" + ) + # Continue with next credential instead of failing entire rotation + continue + verbose_proxy_logger.debug( + f"Successfully re-encrypted {len(credentials)} credentials with new master key" + ) + def get_new_token(data: Optional[RegenerateKeyRequest]) -> str: if data and data.new_key is not None: diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index d1db21a2706..d4dfd86744d 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -1126,6 +1126,91 @@ async def get_ui_settings(request: Request): } +@router.get( + "/sso/readiness", + tags=["experimental"], + dependencies=[Depends(user_api_key_auth)], +) +async def sso_readiness(): + """ + Health endpoint for checking SSO readiness. + Checks if the configured SSO provider has all required environment variables set in memory. + """ + microsoft_client_id = os.getenv("MICROSOFT_CLIENT_ID", None) + google_client_id = os.getenv("GOOGLE_CLIENT_ID", None) + generic_client_id = os.getenv("GENERIC_CLIENT_ID", None) + + # Determine which SSO provider is configured + configured_provider = None + if google_client_id is not None: + configured_provider = "google" + elif microsoft_client_id is not None: + configured_provider = "microsoft" + elif generic_client_id is not None: + configured_provider = "generic" + + # If no SSO is configured, return healthy (SSO is optional) + if configured_provider is None: + return { + "status": "healthy", + "sso_configured": False, + "message": "No SSO provider configured", + } + + # Check required environment variables for the configured provider + missing_vars = [] + + if configured_provider == "google": + google_client_secret = os.getenv("GOOGLE_CLIENT_SECRET", None) + if google_client_secret is None: + missing_vars.append("GOOGLE_CLIENT_SECRET") + + elif configured_provider == "microsoft": + microsoft_client_secret = os.getenv("MICROSOFT_CLIENT_SECRET", None) + microsoft_tenant = os.getenv("MICROSOFT_TENANT", None) + if microsoft_client_secret is None: + missing_vars.append("MICROSOFT_CLIENT_SECRET") + if microsoft_tenant is None: + missing_vars.append("MICROSOFT_TENANT") + + elif configured_provider == "generic": + generic_client_secret = os.getenv("GENERIC_CLIENT_SECRET", None) + generic_authorization_endpoint = os.getenv( + "GENERIC_AUTHORIZATION_ENDPOINT", None + ) + generic_token_endpoint = os.getenv("GENERIC_TOKEN_ENDPOINT", None) + generic_userinfo_endpoint = os.getenv("GENERIC_USERINFO_ENDPOINT", None) + if generic_client_secret is None: + missing_vars.append("GENERIC_CLIENT_SECRET") + if generic_authorization_endpoint is None: + missing_vars.append("GENERIC_AUTHORIZATION_ENDPOINT") + if generic_token_endpoint is None: + missing_vars.append("GENERIC_TOKEN_ENDPOINT") + if generic_userinfo_endpoint is None: + missing_vars.append("GENERIC_USERINFO_ENDPOINT") + + # If all required variables are present, return healthy + if len(missing_vars) == 0: + return { + "status": "healthy", + "sso_configured": True, + "provider": configured_provider, + "message": f"{configured_provider.capitalize()} SSO is properly configured", + } + + # If some variables are missing, return unhealthy + raise HTTPException( + status_code=503, + detail={ + "status": "unhealthy", + "sso_configured": True, + "provider": configured_provider, + "missing_environment_variables": missing_vars, + "message": f"{configured_provider.capitalize()} SSO is configured but missing required environment variables: {', '.join(missing_vars)}", + }, + ) + + class SSOAuthenticationHandler: """ Handler for SSO Authentication across all SSO providers @@ -1149,7 +1234,7 @@ class SSOAuthenticationHandler: generic_client_id (Optional[str], optional): The Generic Client ID. Defaults to None. Returns: - RedirectResponse: The redirect response from the SSO provider + RedirectResponse: The redirect response from the SSO provider. """ # Google SSO Auth if google_client_id is not None: diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 91bbed21291..a773e934ef1 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,19 +1,10 @@ model_list: - - model_name: openai/gpt-4o-mini + - model_name: gemini/* litellm_params: - model: openai/gpt-4o-mini - tpm: 1000 - - # LangGraph models - - model_name: langgraph/* - litellm_params: - model: langgraph/* + model: gemini/* litellm_settings: callbacks: ["dynamic_rate_limiter_v3"] priority_reservation: "prod": 0.9 # 90% reserved for production "dev": 0.1 # 10% reserved for development - -general_settings: - alerting: ["email"] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8eacf225eb6..3927380c8ae 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5,6 +5,7 @@ import io import os import random import secrets +import shutil import subprocess import sys import time @@ -939,31 +940,68 @@ origins = ["*"] # get current directory try: current_dir = os.path.dirname(os.path.abspath(__file__)) - ui_path = os.path.join(current_dir, "_experimental", "out") + packaged_ui_path = os.path.join(current_dir, "_experimental", "out") + ui_path = packaged_ui_path litellm_asset_prefix = "/litellm-asset-prefix" - # For non-root Docker, use the pre-built UI from /tmp/litellm_ui - # Support both "true" and "True" for case-insensitive comparison - if os.getenv("LITELLM_NON_ROOT", "").lower() == "true": - non_root_ui_path = "/tmp/litellm_ui" + def _dir_has_content(path: str) -> bool: + try: + return os.path.isdir(path) and any(os.scandir(path)) + except FileNotFoundError: + return False - # Check if the UI was built and exists at the expected location - if os.path.exists(non_root_ui_path) and os.listdir(non_root_ui_path): + # Use a writable runtime UI directory whenever possible. + # This prevents mutating the packaged UI directory (e.g. site-packages or the repo checkout) + # and ensures extensionless routes like /ui/login work via /index.html. + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + runtime_ui_path = "/tmp/litellm_ui" + + if _dir_has_content(runtime_ui_path): + if is_non_root: verbose_proxy_logger.info( - f"Using pre-built UI for non-root Docker: {non_root_ui_path}" + f"Using pre-built UI for non-root Docker: {runtime_ui_path}" ) - verbose_proxy_logger.info( - f"UI files found: {len(os.listdir(non_root_ui_path))} items" - ) - ui_path = non_root_ui_path else: + verbose_proxy_logger.info( + f"Using cached runtime UI directory: {runtime_ui_path}" + ) + ui_path = runtime_ui_path + else: + if is_non_root: verbose_proxy_logger.error( - f"UI not found at {non_root_ui_path}. UI will not be available." + f"UI not found at {runtime_ui_path}. Attempting to populate it from packaged UI." ) verbose_proxy_logger.error( - f"Path exists: {os.path.exists(non_root_ui_path)}, Has content: {os.path.exists(non_root_ui_path) and bool(os.listdir(non_root_ui_path))}" + f"Path exists: {os.path.exists(runtime_ui_path)}, Has content: {_dir_has_content(runtime_ui_path)}" ) + try: + os.makedirs(runtime_ui_path, exist_ok=True) + if not _dir_has_content(runtime_ui_path) and _dir_has_content( + packaged_ui_path + ): + shutil.copytree( + packaged_ui_path, + runtime_ui_path, + dirs_exist_ok=True, + ) + except Exception as e: + if is_non_root: + verbose_proxy_logger.exception( + f"Failed to populate runtime UI directory {runtime_ui_path} from {packaged_ui_path}: {e}" + ) + else: + if _dir_has_content(runtime_ui_path): + if is_non_root: + verbose_proxy_logger.info( + f"Using populated UI for non-root Docker: {runtime_ui_path}" + ) + else: + verbose_proxy_logger.info( + f"Using populated runtime UI directory: {runtime_ui_path}" + ) + ui_path = runtime_ui_path + # Only modify files if a custom server root path is set if server_root_path and server_root_path != "/": # Iterate through files in the UI directory @@ -1042,16 +1080,25 @@ try: target_path = os.path.join(target_dir, "index.html") os.makedirs(target_dir, exist_ok=True) - os.replace(file_path, target_path) + try: + os.replace(file_path, target_path) + except FileNotFoundError: + # Another process may have already moved this file. + continue # Handle HTML file restructuring - # Skip this for non-root Docker since it's done at build time - # Support both "true" and "True" for case-insensitive comparison - if os.getenv("LITELLM_NON_ROOT", "").lower() != "true": - _restructure_ui_html_files(ui_path) + # Always restructure the directory we actually serve, but avoid mutating the packaged UI. + # This is critical for extensionless routes like /ui/login (expects login/index.html). + if ui_path != packaged_ui_path: + try: + _restructure_ui_html_files(ui_path) + except PermissionError as e: + verbose_proxy_logger.exception( + f"Permission error while restructuring UI directory {ui_path}: {e}" + ) else: verbose_proxy_logger.info( - "Skipping runtime HTML restructuring for non-root Docker (already done at build time)" + f"Skipping runtime HTML restructuring for packaged UI directory: {ui_path}" ) except Exception: @@ -3375,8 +3422,17 @@ class ProxyConfig: decrypted_env_vars = self._decrypt_and_set_db_env_variables( db_param_value, return_original_value=True ) + # Normalize keys when loading from DB so services expecting uppercase + # (e.g. Datadog) can read them even if stored in lowercase. + merged_env_vars: dict = {} + for key, value in decrypted_env_vars.items(): + merged_env_vars[key] = value + upper_key = key.upper() + merged_env_vars[upper_key] = value + os.environ[upper_key] = value + current_config.setdefault("environment_variables", {}).update( - decrypted_env_vars + merged_env_vars ) return current_config elif param_name == "litellm_settings" and isinstance(db_param_value, dict): @@ -4263,7 +4319,7 @@ def get_litellm_model_info(model: dict = {}): model_info = model.get("model_info", {}) model_to_lookup = model.get("litellm_params", {}).get("model", None) try: - if "azure" in model_to_lookup: + if "azure" in model_to_lookup or model_info.get("base_model"): model_to_lookup = model_info.get("base_model", None) litellm_model_info = litellm.get_model_info(model_to_lookup) return litellm_model_info diff --git a/litellm/proxy/public_endpoints/agent_create_fields.json b/litellm/proxy/public_endpoints/agent_create_fields.json index 347a58a7675..931c9a43498 100644 --- a/litellm/proxy/public_endpoints/agent_create_fields.json +++ b/litellm/proxy/public_endpoints/agent_create_fields.json @@ -144,6 +144,51 @@ "litellm_params_template": { "custom_llm_provider": "azure_ai" } + }, + { + "agent_type": "pydantic_ai_agents", + "agent_type_display_name": "Pydantic AI", + "description": "Connect to Pydantic AI agents via A2A protocol (with fake streaming support)", + "logo_url": "/ui/assets/logos/pydantic.svg", + "use_a2a_form_fields": true, + "credential_fields": [ + { + "key": "api_base", + "label": "Agent URL", + "placeholder": "http://localhost:9999", + "tooltip": "The base URL for your Pydantic AI agent server", + "required": true, + "field_type": "text", + "default_value": "http://localhost:9999", + "include_in_litellm_params": true + } + ], + "litellm_params_template": { + "custom_llm_provider": "pydantic_ai_agents" + } + }, + { + "agent_type": "vertex_agent_engine", + "agent_type_display_name": "Vertex AI Agent Engine", + "description": "Connect to Google Cloud Vertex AI Reasoning Engines", + "logo_url": "/ui/assets/logos/google.svg", + "inherit_credentials_from_provider": "Vertex_AI", + "model_template": "vertex_ai/agent_engine/{reasoning_engine_id}", + "credential_fields": [ + { + "key": "reasoning_engine_id", + "label": "Reasoning Engine Resource ID", + "placeholder": "projects/123456789/locations/us-central1/reasoningEngines/987654321", + "tooltip": "The full resource ID of your Vertex AI Reasoning Engine. Find this in Google Cloud Console under Vertex AI > Agent Builder > Your Agent.", + "required": true, + "field_type": "text", + "default_value": null, + "include_in_litellm_params": false + } + ], + "litellm_params_template": { + "custom_llm_provider": "vertex_ai" + } } ] diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 629760a7dd2..68264a576fe 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2689,8 +2689,8 @@ "key": "vertex_credentials", "label": "Vertex Credentials", "placeholder": null, - "tooltip": null, - "required": true, + "tooltip": "Optional - Upload your GCP service account JSON file. If not provided, uses default GCP credentials (ADC).", + "required": false, "field_type": "upload", "options": null, "default_value": null diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 9d5bccecdf8..252b3a7d384 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1,12 +1,16 @@ import asyncio +import time from typing import Any, AsyncIterator, cast +from uuid import uuid4 from fastapi import APIRouter, Depends, HTTPException, Request, Response from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.responses.main import DeleteResponseResult router = APIRouter() @@ -169,6 +173,28 @@ async def responses_api( user_api_base=user_api_base, version=version, ) + except ModifyResponseException as e: + # Guardrail passthrough: return violation message in Responses API format (200) + _data = e.request_data + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=_data, + ) + + violation_text = e.message + response_obj = ResponsesAPIResponse( + id=f"resp_{uuid4()}", + object="response", + created_at=int(time.time()), + model=e.model or data.get("model"), + output=cast(Any, [{"content": [{"type": "text", "text": violation_text}]}]), + status="completed", + usage=ResponseAPIUsage( + input_tokens=0, output_tokens=0, total_tokens=0 + ), + ) + return response_obj except Exception as e: raise await processor._handle_llm_api_exception( e=e, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 6b86d722b2d..fd00cfc1c0a 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -46,6 +46,11 @@ ROUTE_ENDPOINT_MAPPING = { "aget_skill": "/skills/{skill_id}", "adelete_skill": "/skills/{skill_id}", "aingest": "/rag/ingest", + # Google Interactions API routes + "acreate_interaction": "/interactions", + "aget_interaction": "/interactions/{interaction_id}", + "adelete_interaction": "/interactions/{interaction_id}", + "acancel_interaction": "/interactions/{interaction_id}/cancel", } @@ -147,6 +152,10 @@ async def route_request( "adelete_skill", "aingest", "anthropic_messages", + "acreate_interaction", + "aget_interaction", + "adelete_interaction", + "acancel_interaction", ], ): """ @@ -199,6 +208,13 @@ async def route_request( "aretrieve_container_file_content", ]: return getattr(llm_router, f"{route_type}")(**data) + # Interactions API: get/delete/cancel don't need model routing + if route_type in [ + "aget_interaction", + "adelete_interaction", + "acancel_interaction", + ]: + return getattr(llm_router, f"{route_type}")(**data) if route_type in [ "avideo_list", "avideo_status", diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 090d870ba72..687af8a4514 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -55,6 +55,7 @@ def _get_spend_logs_metadata( usage_object: Optional[dict] = None, model_map_information: Optional[StandardLoggingModelInformation] = None, cold_storage_object_key: Optional[str] = None, + litellm_overhead_time_ms: Optional[float] = None, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -78,6 +79,7 @@ def _get_spend_logs_metadata( usage_object=None, guardrail_information=None, cold_storage_object_key=cold_storage_object_key, + litellm_overhead_time_ms=None, ) verbose_proxy_logger.debug( "getting payload for SpendLogs, available keys in metadata: " @@ -102,6 +104,7 @@ def _get_spend_logs_metadata( clean_metadata["usage_object"] = usage_object clean_metadata["model_map_information"] = model_map_information clean_metadata["cold_storage_object_key"] = cold_storage_object_key + clean_metadata["litellm_overhead_time_ms"] = litellm_overhead_time_ms return clean_metadata @@ -298,6 +301,12 @@ def get_logging_payload( # noqa: PLR0915 _model_id = metadata.get("model_info", {}).get("id", "") _model_group = metadata.get("model_group", "") + # Extract overhead from hidden_params if available + litellm_overhead_time_ms = None + if standard_logging_payload is not None: + hidden_params = standard_logging_payload.get("hidden_params", {}) + litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms") + # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( metadata, @@ -343,6 +352,7 @@ def get_logging_payload( # noqa: PLR0915 if standard_logging_payload is not None else None ), + litellm_overhead_time_ms=litellm_overhead_time_ms, ) special_usage_fields = ["completion_tokens", "prompt_tokens", "total_tokens"] diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 4f6af6e135a..5d96c389b61 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -90,6 +90,7 @@ class LiteLLMCompletionResponsesConfig: "metadata", "parallel_tool_calls", "previous_response_id", + "reasoning", "stream", "temperature", "text", @@ -178,6 +179,17 @@ class LiteLLMCompletionResponsesConfig: text_param ) + # Extract reasoning_effort from reasoning parameter + reasoning_effort = None + reasoning_param = responses_api_request.get("reasoning") + if reasoning_param: + if isinstance(reasoning_param, dict): + # reasoning can be {"effort": "low|medium|high"} + reasoning_effort = reasoning_param.get("effort") + elif isinstance(reasoning_param, str): + # reasoning could be a string directly + reasoning_effort = reasoning_param + litellm_completion_request: dict = { "messages": LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=input, @@ -198,6 +210,7 @@ class LiteLLMCompletionResponsesConfig: "service_tier": kwargs.get("service_tier"), "web_search_options": web_search_options, "response_format": response_format, + "reasoning_effort": reasoning_effort, # litellm specific params "custom_llm_provider": custom_llm_provider, "extra_headers": extra_headers, @@ -219,7 +232,6 @@ class LiteLLMCompletionResponsesConfig: litellm_completion_request = { k: v for k, v in litellm_completion_request.items() if v is not None } - return litellm_completion_request @staticmethod diff --git a/litellm/router.py b/litellm/router.py index 5e6027671b2..abb26456be3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1065,8 +1065,44 @@ class Router: litellm.adelete_skill, call_type="adelete_skill" ) + def _initialize_interactions_endpoints(self): + """Initialize Google Interactions API endpoints.""" + from litellm.interactions import acancel as acancel_interaction + from litellm.interactions import acreate as acreate_interaction + from litellm.interactions import adelete as adelete_interaction + from litellm.interactions import aget as aget_interaction + from litellm.interactions import cancel as cancel_interaction + from litellm.interactions import create as create_interaction + from litellm.interactions import delete as delete_interaction + from litellm.interactions import get as get_interaction + + self.acreate_interaction = self.factory_function( + acreate_interaction, call_type="acreate_interaction" + ) + self.create_interaction = self.factory_function( + create_interaction, call_type="create_interaction" + ) + self.aget_interaction = self.factory_function( + aget_interaction, call_type="aget_interaction" + ) + self.get_interaction = self.factory_function( + get_interaction, call_type="get_interaction" + ) + self.adelete_interaction = self.factory_function( + adelete_interaction, call_type="adelete_interaction" + ) + self.delete_interaction = self.factory_function( + delete_interaction, call_type="delete_interaction" + ) + self.acancel_interaction = self.factory_function( + acancel_interaction, call_type="acancel_interaction" + ) + self.cancel_interaction = self.factory_function( + cancel_interaction, call_type="cancel_interaction" + ) + def _initialize_specialized_endpoints(self): - """Helper to initialize specialized router endpoints (vector store, OCR, search, video, container, skills).""" + """Helper to initialize specialized router endpoints (vector store, OCR, search, video, container, skills, interactions).""" self._initialize_vector_store_endpoints() self._initialize_vector_store_file_endpoints() self._initialize_google_genai_endpoints() @@ -1074,6 +1110,7 @@ class Router: self._initialize_video_endpoints() self._initialize_container_endpoints() self._initialize_skills_endpoints() + self._initialize_interactions_endpoints() def initialize_router_endpoints(self): self._initialize_core_endpoints() @@ -3858,6 +3895,14 @@ class Router: "alist_skills", "aget_skill", "adelete_skill", + "acreate_interaction", + "create_interaction", + "aget_interaction", + "get_interaction", + "adelete_interaction", + "delete_interaction", + "acancel_interaction", + "cancel_interaction", ] = "assistants", ): """ @@ -3979,6 +4024,8 @@ class Router: "alist_skills", "aget_skill", "adelete_skill", + "acreate_interaction", + "create_interaction", ): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, @@ -4031,6 +4078,16 @@ class Router: client=client, **kwargs, ) + elif call_type in ( + "aget_interaction", + "adelete_interaction", + "acancel_interaction", + ): + return await self._init_interactions_api_endpoints( + original_function=original_function, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) return async_wrapper @@ -4085,6 +4142,25 @@ class Router: **kwargs, ) + async def _init_interactions_api_endpoints( + self, + original_function: Callable, + custom_llm_provider: Optional[str] = None, + **kwargs, + ): + """ + Initialize the Interactions API endpoints on the router. + + GET, DELETE, CANCEL Interactions API Requests don't need model-based routing, + so we call the original function directly with the custom_llm_provider. + """ + if custom_llm_provider and "custom_llm_provider" not in kwargs: + kwargs["custom_llm_provider"] = custom_llm_provider + # Default to gemini for interactions API + if "custom_llm_provider" not in kwargs: + kwargs["custom_llm_provider"] = "gemini" + return await original_function(**kwargs) + async def _pass_through_assistants_endpoint_factory( self, original_function: Callable, diff --git a/litellm/types/interactions/README.md b/litellm/types/interactions/README.md new file mode 100644 index 00000000000..a16744ce016 --- /dev/null +++ b/litellm/types/interactions/README.md @@ -0,0 +1,48 @@ +# Interactions API Types + +This directory contains type definitions for the Google Interactions API. + +## Generated Types + +The `generated.py` file is auto-generated from the official OpenAPI spec: +https://ai.google.dev/static/api/interactions.openapi.json + +### How to Regenerate + +When the API spec changes, regenerate the types with: + +```bash +pip install datamodel-code-generator + +datamodel-codegen \ + --url "https://ai.google.dev/static/api/interactions.openapi.json" \ + --output litellm/types/interactions/generated.py \ + --output-model-type pydantic_v2.BaseModel \ + --target-python-version 3.9 +``` + +Then add the LiteLLM-specific types at the bottom of the generated file: +- `InteractionsAPIResponse` +- `InteractionsAPIStreamingResponse` +- `DeleteInteractionResult` +- `CancelInteractionResult` + +### Key Types + +**Request Types:** +- `CreateModelInteractionParams` - For model interactions +- `CreateAgentInteractionParams` - For agent interactions + +**Content Types:** +- `Content` - Union of all content types (text, image, audio, etc.) +- `TextContent` - Text content with `type: "text"` +- `Turn` - A turn in multi-turn conversation with `role` and `content` + +**Tool Types:** +- `Tool` - Union of all tool types +- `Function` - Function tool declaration + +**Response Types:** +- `InteractionsAPIResponse` - LiteLLM response wrapper +- `InteractionsAPIStreamingResponse` - Streaming response chunk + diff --git a/litellm/types/interactions/__init__.py b/litellm/types/interactions/__init__.py new file mode 100644 index 00000000000..a3acdc4cb1f --- /dev/null +++ b/litellm/types/interactions/__init__.py @@ -0,0 +1,127 @@ +""" +Type definitions for Google Interactions API + +Auto-generated from OpenAPI spec: https://ai.google.dev/static/api/interactions.openapi.json +See README.md for regeneration instructions. +""" + +from litellm.types.interactions.generated import ( + AgentOption, + Annotation, + AudioContent, + CancelInteractionResult, + CodeExecution, + CodeExecutionCallContent, + CodeExecutionResultContent, + ComputerUse, + Content, + ContentDelta, + ContentStart, + ContentStop, + CreateAgentInteractionParams, + CreateModelInteractionParams, + DeepResearchAgentConfig, + DeleteInteractionResult, + DocumentContent, + DynamicAgentConfig, + ErrorEvent, + FileSearch, + FileSearchResultContent, + Function, + FunctionCallContent, + FunctionResultContent, + GenerationConfig, + GoogleSearch, + GoogleSearchCallContent, + GoogleSearchResultContent, + ImageContent, + Interaction, + InteractionEvent, + InteractionInput, + InteractionsAPIOptionalRequestParams, + InteractionsAPIResponse, + InteractionsAPIStreamingResponse, + InteractionSseEvent, + InteractionTool, + InteractionToolChoiceConfig, + McpServer, + McpServerToolCallContent, + McpServerToolResultContent, + ModelOption, + ResponseModality, +) +from litellm.types.interactions.generated import ( + Status3 as InteractionStatus, # Main request/response types; Content types; Turn for multi-turn conversations; Tool types; Config types; Usage; Status enum; Events for streaming; Agent configs; Model/Agent options; Response modality; Annotation; LiteLLM types; Backwards compat aliases +) +from litellm.types.interactions.generated import ( + TextContent, + ThoughtContent, + Tool, + ToolChoiceConfig, + Turn, + UrlContext, + UrlContextCallContent, + UrlContextResultContent, + Usage, + VideoContent, +) + +__all__ = [ + # Generated types + "CreateModelInteractionParams", + "CreateAgentInteractionParams", + "Interaction", + "Content", + "TextContent", + "ImageContent", + "AudioContent", + "DocumentContent", + "VideoContent", + "ThoughtContent", + "FunctionCallContent", + "FunctionResultContent", + "CodeExecutionCallContent", + "CodeExecutionResultContent", + "UrlContextCallContent", + "UrlContextResultContent", + "GoogleSearchCallContent", + "GoogleSearchResultContent", + "McpServerToolCallContent", + "McpServerToolResultContent", + "FileSearchResultContent", + "Turn", + "Tool", + "Function", + "GoogleSearch", + "CodeExecution", + "UrlContext", + "ComputerUse", + "McpServer", + "FileSearch", + "GenerationConfig", + "ToolChoiceConfig", + "Usage", + "InteractionStatus", + "InteractionEvent", + "InteractionSseEvent", + "ContentStart", + "ContentDelta", + "ContentStop", + "ErrorEvent", + "DynamicAgentConfig", + "DeepResearchAgentConfig", + "ModelOption", + "AgentOption", + "ResponseModality", + "Annotation", + # LiteLLM types + "InteractionInput", + "InteractionsAPIResponse", + "InteractionsAPIStreamingResponse", + "DeleteInteractionResult", + "CancelInteractionResult", + "InteractionsAPIOptionalRequestParams", + # Backwards compat + "InteractionTool", + "InteractionToolChoiceConfig", +] diff --git a/litellm/types/interactions/generated.py b/litellm/types/interactions/generated.py new file mode 100644 index 00000000000..72693e8f188 --- /dev/null +++ b/litellm/types/interactions/generated.py @@ -0,0 +1,1254 @@ +# generated by datamodel-codegen: +# filename: https://ai.google.dev/static/api/interactions.openapi.json +# timestamp: 2025-12-16T21:25:12+00:00 + +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import AwareDatetime, Base64Str, BaseModel, Field, RootModel + + +class Annotation(BaseModel): + start_index: Optional[int] = Field( + None, + description='Start of segment of the response that is attributed to this source.\n\nIndex indicates the start of the segment, measured in bytes.', + ) + end_index: Optional[int] = Field( + None, description='End of the attributed segment, exclusive.' + ) + source: Optional[str] = Field( + None, + description='Source attributed for a portion of the text. Could be a URL, title, or\nother identifier.', + ) + + +class DocumentContent(BaseModel): + data: Optional[Base64Str] = None + uri: Optional[str] = None + mime_type: Optional[str] = None + type: Literal['document'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + + +class FunctionCallContent(BaseModel): + name: str = Field(..., description='The name of the tool to call.') + arguments: Dict[str, Any] = Field( + ..., description='The arguments to pass to the function.' + ) + type: Literal['function_call'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + id: str = Field(..., description='A unique ID for this specific tool call.') + + +class Language(Enum): + python = 'python' + + +class CodeExecutionCallArguments(BaseModel): + language: Optional[Language] = Field( + None, description='Programming language of the `code`.' + ) + code: Optional[str] = Field(None, description='The code to be executed.') + + +class UrlContextCallArguments(BaseModel): + urls: Optional[List[str]] = Field(None, description='The URLs to fetch.') + + +class McpServerToolCallContent(BaseModel): + name: str = Field(..., description='The name of the tool which was called.') + server_name: str = Field(..., description='The name of the used MCP server.') + arguments: Dict[str, Any] = Field( + ..., description='The JSON object of arguments for the function.' + ) + type: Literal['mcp_server_tool_call'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + id: str = Field(..., description='A unique ID for this specific tool call.') + + +class GoogleSearchCallArguments(BaseModel): + queries: Optional[List[str]] = Field( + None, description='Web search queries for the following-up web search.' + ) + + +class CodeExecutionResultContent(BaseModel): + result: Optional[str] = Field(None, description='The output of the code execution.') + is_error: Optional[bool] = Field( + None, description='Whether the code execution resulted in an error.' + ) + signature: Optional[str] = Field( + None, description='A signature hash for backend validation.' + ) + type: Literal['code_execution_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + call_id: Optional[str] = Field( + None, description='ID to match the ID from the code execution call block.' + ) + + +class Status(Enum): + success = 'success' + error = 'error' + paywall = 'paywall' + unsafe = 'unsafe' + + +class UrlContextResult(BaseModel): + url: Optional[str] = Field(None, description='The URL that was fetched.') + status: Optional[Status] = Field( + None, description='The status of the URL retrieval.' + ) + + +class GoogleSearchResult(BaseModel): + url: Optional[str] = Field(None, description='URI reference of the search result.') + title: Optional[str] = Field(None, description='Title of the search result.') + rendered_content: Optional[str] = Field( + None, + description='Web content snippet that can be embedded in a web page or an app webview.', + ) + + +class FileSearchResult(BaseModel): + title: Optional[str] = Field(None, description='The title of the search result.') + text: Optional[str] = Field(None, description='The text of the search result.') + file_search_store: Optional[str] = Field( + None, description='The name of the file search store.' + ) + + +class SpeechConfig(BaseModel): + voice: Optional[str] = Field(None, description='The voice of the speaker.') + language: Optional[str] = Field(None, description='The language of the speech.') + speaker: Optional[str] = Field( + None, + description="The speaker's name, it should match the speaker name given in the prompt.", + ) + + +class DynamicAgentConfig(BaseModel): + type: Literal['dynamic'] = Field( + 'dynamic', + description='Used as the OpenAPI type discriminator for the content oneof.', + ) + + +class Function(BaseModel): + name: Optional[str] = Field(None, description='The name of the function.') + description: Optional[str] = Field( + None, description='A description of the function.' + ) + parameters: Optional[Any] = Field( + None, description="The JSON Schema for the function's parameters." + ) + type: Literal['function'] + + +class CodeExecution(BaseModel): + type: Literal['code_execution'] + + +class UrlContext(BaseModel): + type: Literal['url_context'] + + +class Environment(Enum): + browser = 'browser' + + +class ComputerUse(BaseModel): + type: Literal['computer_use'] + environment: Optional[Environment] = Field( + None, description='The environment being operated.' + ) + excludedPredefinedFunctions: Optional[List[str]] = Field( + None, + description='The list of predefined functions that are excluded from the model call.', + ) + + +class GoogleSearch(BaseModel): + type: Literal['google_search'] + + +class FileSearch(BaseModel): + file_search_store_names: Optional[List[str]] = Field( + None, description='The file search store names to search.' + ) + top_k: Optional[int] = Field( + None, description='The number of semantic retrieval chunks to retrieve.' + ) + metadata_filter: Optional[str] = Field( + None, + description='Metadata filter to apply to the semantic retrieval documents and chunks.', + ) + type: Literal['file_search'] + + +class EventType(Enum): + interaction_start = 'interaction.start' + interaction_complete = 'interaction.complete' + + +class Status1(Enum): + in_progress = 'in_progress' + requires_action = 'requires_action' + completed = 'completed' + failed = 'failed' + cancelled = 'cancelled' + + +class InteractionStatusUpdate(BaseModel): + interaction_id: Optional[str] = None + status: Optional[Status1] = None + event_type: Literal['interaction.status_update'] = 'interaction.status_update' + event_id: Optional[str] = Field( + None, + description='The event_id token to be used to resume the interaction stream, from\nthis event.', + ) + + +class TextDelta(BaseModel): + text: Optional[str] = None + type: Literal['text'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + annotations: Optional[List[Annotation]] = Field( + None, description='Citation information for model-generated content.' + ) + + +class DocumentDelta(BaseModel): + data: Optional[Base64Str] = None + uri: Optional[str] = None + mime_type: Optional[str] = None + type: Literal['document'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + + +class ThoughtSignatureDelta(BaseModel): + signature: Optional[Base64Str] = Field( + None, + description='Signature to match the backend source to be part of the generation.', + ) + type: Literal['thought_signature'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + + +class FunctionCallDelta(BaseModel): + name: Optional[str] = None + arguments: Optional[Dict[str, Any]] = None + type: Literal['function_call'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + id: Optional[str] = Field( + None, description='A unique ID for this specific tool call.' + ) + + +class CodeExecutionCallDelta(BaseModel): + arguments: Optional[CodeExecutionCallArguments] = None + type: Literal['code_execution_call'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + id: Optional[str] = Field( + None, description='A unique ID for this specific tool call.' + ) + + +class UrlContextCallDelta(BaseModel): + arguments: Optional[UrlContextCallArguments] = None + type: Literal['url_context_call'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + id: Optional[str] = Field( + None, description='A unique ID for this specific tool call.' + ) + + +class GoogleSearchCallDelta(BaseModel): + arguments: Optional[GoogleSearchCallArguments] = None + type: Literal['google_search_call'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + id: Optional[str] = Field( + None, description='A unique ID for this specific tool call.' + ) + + +class McpServerToolCallDelta(BaseModel): + name: Optional[str] = None + server_name: Optional[str] = None + arguments: Optional[Dict[str, Any]] = None + type: Literal['mcp_server_tool_call'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + id: Optional[str] = Field( + None, description='A unique ID for this specific tool call.' + ) + + +class CodeExecutionResultDelta(BaseModel): + result: Optional[str] = None + is_error: Optional[bool] = None + signature: Optional[str] = None + type: Literal['code_execution_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + call_id: Optional[str] = Field( + None, description='ID to match the ID from the function call block.' + ) + + +class UrlContextResultDelta(BaseModel): + signature: Optional[str] = None + result: Optional[List[UrlContextResult]] = None + is_error: Optional[bool] = None + type: Literal['url_context_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + call_id: Optional[str] = Field( + None, description='ID to match the ID from the function call block.' + ) + + +class GoogleSearchResultDelta(BaseModel): + signature: Optional[str] = None + result: Optional[List[GoogleSearchResult]] = None + is_error: Optional[bool] = None + type: Literal['google_search_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + call_id: Optional[str] = Field( + None, description='ID to match the ID from the function call block.' + ) + + +class FileSearchResultDelta(BaseModel): + result: Optional[List[FileSearchResult]] = None + type: Literal['file_search_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + + +class ContentStop(BaseModel): + index: Optional[int] = None + event_type: Literal['content.stop'] = 'content.stop' + event_id: Optional[str] = Field( + None, + description='The event_id token to be used to resume the interaction stream, from\nthis event.', + ) + + +class Error(BaseModel): + code: Optional[str] = Field( + None, description='A URI that identifies the error type.' + ) + message: Optional[str] = Field(None, description='A human-readable error message.') + + +class MediaResolution(Enum): + low = 'low' + medium = 'medium' + high = 'high' + + +class ToolChoiceType(Enum): + auto = 'auto' + any = 'any' + none = 'none' + validated = 'validated' + + +class ThinkingLevel(Enum): + low = 'low' + high = 'high' + + +class ThinkingSummaries(Enum): + auto = 'auto' + none = 'none' + + +class ResponseModality(Enum): + text = 'text' + image = 'image' + audio = 'audio' + + +class Status3(Enum): + UNSPECIFIED = 'UNSPECIFIED' + IN_PROGRESS = 'IN_PROGRESS' + REQUIRES_ACTION = 'REQUIRES_ACTION' + COMPLETED = 'COMPLETED' + FAILED = 'FAILED' + CANCELLED = 'CANCELLED' + + +class ModelOption(RootModel[str]): + root: str = Field( + ..., + description='The model that will complete your prompt.\\n\\nSee [models](https://ai.google.dev/gemini-api/docs/models) for additional details.', + title='Model', + ) + + +class AgentOption(RootModel[str]): + root: str = Field(..., description='The agent to interact with.', title='Agent') + + +class ImageMimeTypeOption(RootModel[str]): + root: str = Field( + ..., description='The mime type of the image.', title='ImageMimeType' + ) + + +class AudioMimeTypeOption(RootModel[str]): + root: str = Field( + ..., description='The mime type of the audio.', title='AudioMimeType' + ) + + +class VideoMimeTypeOption(RootModel[str]): + root: str = Field( + ..., description='The mime type of the video.', title='VideoMimeType' + ) + + +class TextContent(BaseModel): + text: Optional[str] = Field(None, description='The text content.') + type: Literal['text'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + annotations: Optional[List[Annotation]] = Field( + None, description='Citation information for model-generated content.' + ) + + +class ImageContent(BaseModel): + data: Optional[Base64Str] = None + uri: Optional[str] = None + mime_type: Optional[ImageMimeTypeOption] = None + type: Literal['image'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + resolution: Optional[MediaResolution] = Field( + None, description='The resolution of the media.' + ) + + +class AudioContent(BaseModel): + data: Optional[Base64Str] = None + uri: Optional[str] = None + mime_type: Optional[AudioMimeTypeOption] = None + type: Literal['audio'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + + +class VideoContent(BaseModel): + data: Optional[Base64Str] = None + uri: Optional[str] = None + mime_type: Optional[VideoMimeTypeOption] = None + type: Literal['video'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + resolution: Optional[MediaResolution] = Field( + None, description='The resolution of the media.' + ) + + +class ThoughtSummary1(RootModel[Union[TextContent, ImageContent]]): + root: Union[TextContent, ImageContent] = Field(..., discriminator='type') + + +class ThoughtSummary(RootModel[List[ThoughtSummary1]]): + root: List[ThoughtSummary1] = Field(..., description='A summary of the thought.') + + +class CodeExecutionCallContent(BaseModel): + arguments: Optional[CodeExecutionCallArguments] = Field( + None, description='The arguments to pass to the code execution.' + ) + type: Literal['code_execution_call'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + id: Optional[str] = Field( + None, description='A unique ID for this specific tool call.' + ) + + +class UrlContextCallContent(BaseModel): + arguments: Optional[UrlContextCallArguments] = Field( + None, description='The arguments to pass to the URL context.' + ) + type: Literal['url_context_call'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + id: Optional[str] = Field( + None, description='A unique ID for this specific tool call.' + ) + + +class GoogleSearchCallContent(BaseModel): + arguments: Optional[GoogleSearchCallArguments] = Field( + None, description='The arguments to pass to Google Search.' + ) + type: Literal['google_search_call'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + id: Optional[str] = Field( + None, description='A unique ID for this specific tool call.' + ) + + +class Result(BaseModel): + items: Optional[List[Union[str, ImageContent]]] = None + + +class FunctionResultContent(BaseModel): + name: Optional[str] = Field( + None, description='The name of the tool that was called.' + ) + is_error: Optional[bool] = Field( + None, description='Whether the tool call resulted in an error.' + ) + type: Literal['function_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + result: Union[Result, Dict[str, Any], str] = Field( + ..., description='The result of the tool call.' + ) + call_id: str = Field( + ..., description='ID to match the ID from the function call block.' + ) + + +class UrlContextResultContent(BaseModel): + signature: Optional[str] = Field( + None, description='The signature of the URL context result.' + ) + result: Optional[List[UrlContextResult]] = Field( + None, description='The results of the URL context.' + ) + is_error: Optional[bool] = Field( + None, description='Whether the URL context resulted in an error.' + ) + type: Literal['url_context_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + call_id: Optional[str] = Field( + None, description='ID to match the ID from the url context call block.' + ) + + +class GoogleSearchResultContent(BaseModel): + signature: Optional[str] = Field( + None, description='The signature of the Google Search result.' + ) + result: Optional[List[GoogleSearchResult]] = Field( + None, description='The results of the Google Search.' + ) + is_error: Optional[bool] = Field( + None, description='Whether the Google Search resulted in an error.' + ) + type: Literal['google_search_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + call_id: Optional[str] = Field( + None, description='ID to match the ID from the google search call block.' + ) + + +class McpServerToolResultContent(BaseModel): + name: Optional[str] = Field( + None, + description='Name of the tool which is called for this specific tool call.', + ) + server_name: Optional[str] = Field( + None, description='The name of the used MCP server.' + ) + type: Literal['mcp_server_tool_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + result: Union[Result, Dict[str, Any], str] = Field( + ..., description='The result of the tool call.' + ) + call_id: str = Field( + ..., description='ID to match the ID from the MCP server tool call block.' + ) + + +class FileSearchResultContent(BaseModel): + result: Optional[List[FileSearchResult]] = Field( + None, description='The results of the File Search.' + ) + type: Literal['file_search_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + + +class AllowedTools(BaseModel): + mode: Optional[ToolChoiceType] = Field( + None, description='The mode of the tool choice.' + ) + tools: Optional[List[str]] = Field( + None, description='The names of the allowed tools.' + ) + + +class DeepResearchAgentConfig(BaseModel): + type: Literal['deep-research'] = Field( + 'deep-research', + description='Used as the OpenAPI type discriminator for the content oneof.', + ) + thinking_summaries: Optional[ThinkingSummaries] = Field( + None, description='Whether to include thought summaries in the response.' + ) + + +class McpServer(BaseModel): + type: Literal['mcp_server'] + name: Optional[str] = Field(None, description='The name of the MCPServer.') + url: Optional[str] = Field( + None, + description='The full URL for the MCPServer endpoint.\nExample: "https://api.example.com/mcp"', + ) + headers: Optional[Dict[str, str]] = Field( + None, + description='Optional: Fields for authentication headers, timeouts, etc., if needed.', + ) + allowed_tools: Optional[List[AllowedTools]] = Field( + None, description='The allowed tools.' + ) + + +class ModalityTokens(BaseModel): + modality: Optional[ResponseModality] = Field( + None, description='The modality associated with the token count.' + ) + tokens: Optional[int] = Field( + None, description='Number of tokens for the modality.' + ) + + +class ImageDelta(BaseModel): + data: Optional[Base64Str] = None + uri: Optional[str] = None + mime_type: Optional[ImageMimeTypeOption] = None + type: Literal['image'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + resolution: Optional[MediaResolution] = Field( + None, description='The resolution of the media.' + ) + + +class AudioDelta(BaseModel): + data: Optional[Base64Str] = None + uri: Optional[str] = None + mime_type: Optional[AudioMimeTypeOption] = None + type: Literal['audio'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + + +class VideoDelta(BaseModel): + data: Optional[Base64Str] = None + uri: Optional[str] = None + mime_type: Optional[VideoMimeTypeOption] = None + type: Literal['video'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + resolution: Optional[MediaResolution] = Field( + None, description='The resolution of the media.' + ) + + +class ThoughtSummaryDelta(BaseModel): + type: Literal['thought_summary'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + content: Optional[Union[TextContent, ImageContent]] = Field( + None, discriminator='type' + ) + + +class FunctionResultDelta(BaseModel): + name: Optional[str] = None + is_error: Optional[bool] = None + type: Literal['function_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + result: Optional[Union[Result, str]] = Field( + None, description='Tool call result delta.' + ) + call_id: Optional[str] = Field( + None, description='ID to match the ID from the function call block.' + ) + + +class McpServerToolResultDelta(BaseModel): + name: Optional[str] = None + server_name: Optional[str] = None + type: Literal['mcp_server_tool_result'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + result: Optional[Union[Result, str]] = Field( + None, description='Tool call result delta.' + ) + call_id: Optional[str] = Field( + None, description='ID to match the ID from the function call block.' + ) + + +class ErrorEvent(BaseModel): + event_type: Literal['error'] = 'error' + error: Optional[Error] = None + event_id: Optional[str] = Field( + None, + description='The event_id token to be used to resume the interaction stream, from\nthis event.', + ) + + +class ToolChoiceConfig(BaseModel): + allowed_tools: Optional[AllowedTools] = None + + +class Tool( + RootModel[ + Union[ + Function, + GoogleSearch, + CodeExecution, + UrlContext, + ComputerUse, + McpServer, + FileSearch, + ] + ] +): + root: Union[ + Function, + GoogleSearch, + CodeExecution, + UrlContext, + ComputerUse, + McpServer, + FileSearch, + ] = Field(..., discriminator='type') + + +class ThoughtContent(BaseModel): + signature: Optional[Base64Str] = Field( + None, + description='Signature to match the backend source to be part of the generation.', + ) + type: Literal['thought'] = Field( + ..., description='Used as the OpenAPI type discriminator for the content oneof.' + ) + summary: Optional[ThoughtSummary] = Field( + None, description='A summary of the thought.' + ) + + +class ToolChoice(RootModel[Union[ToolChoiceType, ToolChoiceConfig]]): + root: Union[ToolChoiceType, ToolChoiceConfig] = Field( + ..., description='The configuration for tool choice.' + ) + + +class Usage(BaseModel): + total_input_tokens: Optional[int] = Field( + None, description='Number of tokens in the prompt (context).' + ) + input_tokens_by_modality: Optional[List[ModalityTokens]] = Field( + None, description='A breakdown of input token usage by modality.' + ) + total_cached_tokens: Optional[int] = Field( + None, + description='Number of tokens in the cached part of the prompt (the cached content).', + ) + cached_tokens_by_modality: Optional[List[ModalityTokens]] = Field( + None, description='A breakdown of cached token usage by modality.' + ) + total_output_tokens: Optional[int] = Field( + None, description='Total number of tokens across all the generated responses.' + ) + output_tokens_by_modality: Optional[List[ModalityTokens]] = Field( + None, description='A breakdown of output token usage by modality.' + ) + total_tool_use_tokens: Optional[int] = Field( + None, description='Number of tokens present in tool-use prompt(s).' + ) + tool_use_tokens_by_modality: Optional[List[ModalityTokens]] = Field( + None, description='A breakdown of tool-use token usage by modality.' + ) + total_reasoning_tokens: Optional[int] = Field( + None, description='Number of tokens of thoughts for thinking models.' + ) + total_tokens: Optional[int] = Field( + None, + description='Total token count for the interaction request (prompt + responses + other\ninternal tokens).', + ) + + +class ContentDelta(BaseModel): + index: Optional[int] = None + event_type: Literal['content.delta'] = 'content.delta' + event_id: Optional[str] = Field( + None, + description='The event_id token to be used to resume the interaction stream, from\nthis event.', + ) + delta: Optional[ + Union[ + TextDelta, + ImageDelta, + AudioDelta, + DocumentDelta, + VideoDelta, + ThoughtSummaryDelta, + ThoughtSignatureDelta, + FunctionCallDelta, + FunctionResultDelta, + CodeExecutionCallDelta, + CodeExecutionResultDelta, + UrlContextCallDelta, + UrlContextResultDelta, + GoogleSearchCallDelta, + GoogleSearchResultDelta, + McpServerToolCallDelta, + McpServerToolResultDelta, + FileSearchResultDelta, + ] + ] = Field(None, discriminator='type') + + +class Content( + RootModel[ + Union[ + TextContent, + ImageContent, + AudioContent, + DocumentContent, + VideoContent, + ThoughtContent, + FunctionCallContent, + FunctionResultContent, + CodeExecutionCallContent, + CodeExecutionResultContent, + UrlContextCallContent, + UrlContextResultContent, + GoogleSearchCallContent, + GoogleSearchResultContent, + McpServerToolCallContent, + McpServerToolResultContent, + FileSearchResultContent, + ] + ] +): + root: Union[ + TextContent, + ImageContent, + AudioContent, + DocumentContent, + VideoContent, + ThoughtContent, + FunctionCallContent, + FunctionResultContent, + CodeExecutionCallContent, + CodeExecutionResultContent, + UrlContextCallContent, + UrlContextResultContent, + GoogleSearchCallContent, + GoogleSearchResultContent, + McpServerToolCallContent, + McpServerToolResultContent, + FileSearchResultContent, + ] = Field(..., description='The content of the response.', discriminator='type') + + +class Turn(BaseModel): + role: Optional[str] = Field( + None, + description='The originator of this turn. Must be user for input or model for\nmodel output.', + ) + content: Optional[Union[str, List[Content]]] = Field( + None, description='The content of the turn.' + ) + + +class GenerationConfig(BaseModel): + temperature: Optional[float] = Field( + None, description='Controls the randomness of the output.' + ) + top_p: Optional[float] = Field( + None, + description='The maximum cumulative probability of tokens to consider when sampling.', + ) + seed: Optional[int] = Field( + None, description='Seed used in decoding for reproducibility.' + ) + stop_sequences: Optional[List[str]] = Field( + None, + description='A list of character sequences that will stop output interaction.', + ) + tool_choice: Optional[ToolChoice] = Field( + None, description='The tool choice for the interaction.' + ) + thinking_level: Optional[ThinkingLevel] = Field( + None, description='The level of thought tokens that the model should generate.' + ) + thinking_summaries: Optional[ThinkingSummaries] = Field( + None, description='Whether to include thought summaries in the response.' + ) + max_output_tokens: Optional[int] = Field( + None, description='The maximum number of tokens to include in the response.' + ) + speech_config: Optional[List[SpeechConfig]] = Field( + None, description='Configuration for speech interaction.' + ) + + +class ContentStart(BaseModel): + index: Optional[int] = None + content: Optional[Content] = None + event_type: Literal['content.start'] = 'content.start' + event_id: Optional[str] = Field( + None, + description='The event_id token to be used to resume the interaction stream, from\nthis event.', + ) + + +class Interaction(BaseModel): + model: Optional[ModelOption] = Field( + None, description='The name of the `Model` used for generating the interaction.' + ) + agent: Optional[AgentOption] = Field( + None, description='The name of the `Agent` used for generating the interaction.' + ) + id: str = Field( + ..., + description='Output only. A unique identifier for the interaction completion.', + ) + status: Status1 = Field( + ..., description='Output only. The status of the interaction.' + ) + created: Optional[AwareDatetime] = Field( + None, + description='Output only. The time at which the response was created in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).', + ) + updated: Optional[AwareDatetime] = Field( + None, + description='Output only. The time at which the response was last updated in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).', + ) + role: Optional[str] = Field( + None, description='Output only. The role of the interaction.' + ) + outputs: Optional[List[Content]] = Field( + None, description='Output only. Responses from the model.' + ) + system_instruction: Optional[str] = Field( + None, description='System instruction for the interaction.' + ) + tools: Optional[List[Tool]] = Field( + None, + description='A list of tool declarations the model may call during interaction.', + ) + background: Optional[bool] = Field( + None, description='Whether to run the model interaction in the background.' + ) + object: Literal['interaction'] = Field( + 'interaction', + description='Output only. The object type of the interaction. Always set to `interaction`.', + ) + usage: Optional[Usage] = Field( + None, + description="Output only. Statistics on the interaction request's token usage.", + ) + response_modalities: Optional[List[ResponseModality]] = Field( + None, + description='The requested modalities of the response (TEXT, IMAGE, AUDIO).', + ) + response_format: Optional[Any] = Field( + None, + description='Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.', + ) + response_mime_type: Optional[str] = Field( + None, + description='The mime type of the response. This is required if response_format is set.', + ) + previous_interaction_id: Optional[str] = Field( + None, description='The ID of the previous interaction, if any.' + ) + input: Optional[Union[str, List[Content], List[Turn], Content]] = Field( + None, description='The inputs for the interaction.' + ) + generation_config: Optional[GenerationConfig] = Field( + None, + description='Input only. Configuration parameters for the model interaction.', + ) + agent_config: Optional[Union[DynamicAgentConfig, DeepResearchAgentConfig]] = Field( + None, description='Configuration for the agent.', discriminator='type' + ) + + +class CreateModelInteractionParams(BaseModel): + model: ModelOption = Field( + ..., description='The name of the `Model` used for generating the interaction.' + ) + stream: Optional[bool] = Field( + None, description='Input only. Whether the interaction will be streamed.' + ) + store: Optional[bool] = Field( + None, + description='Input only. Whether to store the response and request for later retrieval.', + ) + id: Optional[str] = Field( + None, + description='Output only. A unique identifier for the interaction completion.', + ) + status: Optional[Status3] = Field( + None, description='Output only. The status of the interaction.' + ) + created: Optional[AwareDatetime] = Field( + None, + description='Output only. The time at which the response was created in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).', + ) + updated: Optional[AwareDatetime] = Field( + None, + description='Output only. The time at which the response was last updated in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).', + ) + role: Optional[str] = Field( + None, description='Output only. The role of the interaction.' + ) + outputs: Optional[List[Content]] = Field( + None, description='Output only. Responses from the model.' + ) + system_instruction: Optional[str] = Field( + None, description='System instruction for the interaction.' + ) + tools: Optional[List[Tool]] = Field( + None, + description='A list of tool declarations the model may call during interaction.', + ) + background: Optional[bool] = Field( + None, description='Whether to run the model interaction in the background.' + ) + usage: Optional[Usage] = Field( + None, + description="Output only. Statistics on the interaction request's token usage.", + ) + response_modalities: Optional[List[ResponseModality]] = Field( + None, + description='The requested modalities of the response (TEXT, IMAGE, AUDIO).', + ) + response_format: Optional[Any] = Field( + None, + description='Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.', + ) + response_mime_type: Optional[str] = Field( + None, + description='The mime type of the response. This is required if response_format is set.', + ) + previous_interaction_id: Optional[str] = Field( + None, description='The ID of the previous interaction, if any.' + ) + input: Union[str, List[Content], List[Turn], Content] = Field( + ..., description='The inputs for the interaction.' + ) + generation_config: Optional[GenerationConfig] = Field( + None, + description='Input only. Configuration parameters for the model interaction.', + ) + + +class CreateAgentInteractionParams(BaseModel): + agent: AgentOption = Field( + ..., description='The name of the `Agent` used for generating the interaction.' + ) + stream: Optional[bool] = Field( + None, description='Input only. Whether the interaction will be streamed.' + ) + store: Optional[bool] = Field( + None, + description='Input only. Whether to store the response and request for later retrieval.', + ) + id: Optional[str] = Field( + None, + description='Output only. A unique identifier for the interaction completion.', + ) + status: Optional[Status3] = Field( + None, description='Output only. The status of the interaction.' + ) + created: Optional[AwareDatetime] = Field( + None, + description='Output only. The time at which the response was created in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).', + ) + updated: Optional[AwareDatetime] = Field( + None, + description='Output only. The time at which the response was last updated in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).', + ) + role: Optional[str] = Field( + None, description='Output only. The role of the interaction.' + ) + outputs: Optional[List[Content]] = Field( + None, description='Output only. Responses from the model.' + ) + system_instruction: Optional[str] = Field( + None, description='System instruction for the interaction.' + ) + tools: Optional[List[Tool]] = Field( + None, + description='A list of tool declarations the model may call during interaction.', + ) + background: Optional[bool] = Field( + None, description='Whether to run the model interaction in the background.' + ) + usage: Optional[Usage] = Field( + None, + description="Output only. Statistics on the interaction request's token usage.", + ) + response_modalities: Optional[List[ResponseModality]] = Field( + None, + description='The requested modalities of the response (TEXT, IMAGE, AUDIO).', + ) + response_format: Optional[Any] = Field( + None, + description='Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.', + ) + response_mime_type: Optional[str] = Field( + None, + description='The mime type of the response. This is required if response_format is set.', + ) + previous_interaction_id: Optional[str] = Field( + None, description='The ID of the previous interaction, if any.' + ) + input: Union[str, List[Content], List[Turn], Content] = Field( + ..., description='The inputs for the interaction.' + ) + agent_config: Optional[Union[DynamicAgentConfig, DeepResearchAgentConfig]] = Field( + None, description='Configuration for the agent.', discriminator='type' + ) + + +class InteractionEvent(BaseModel): + event_type: Literal['interaction.start', 'interaction.complete'] + interaction: Optional[Interaction] = None + event_id: Optional[str] = Field( + None, + description='The event_id token to be used to resume the interaction stream, from\nthis event.', + ) + + +class InteractionSseEvent( + RootModel[ + Union[ + InteractionEvent, + InteractionStatusUpdate, + ContentStart, + ContentDelta, + ContentStop, + ErrorEvent, + ] + ] +): + root: Union[ + InteractionEvent, + InteractionStatusUpdate, + ContentStart, + ContentDelta, + ContentStop, + ErrorEvent, + ] = Field(..., discriminator='event_type') + + +# ============================================================ +# LiteLLM-specific types (added manually after generation) +# ============================================================ +# +# When regenerating this file, copy these types to the end. +# See README.md for regeneration instructions. + +from pydantic import PrivateAttr + +from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject + +# Type alias for input +InteractionInput = Union[str, Content, List[Content], List[Turn]] + + +class InteractionsAPIResponse(BaseLiteLLMOpenAIResponseObject): + """ + Response from the Interactions API. + + Wraps the API response with LiteLLM-specific hidden params. + """ + id: Optional[str] = None + object: Optional[str] = "interaction" + model: Optional[str] = None + agent: Optional[str] = None + status: Optional[str] = None + created: Optional[str] = None + updated: Optional[str] = None + role: Optional[str] = None + outputs: Optional[List[Dict[str, Any]]] = None + usage: Optional[Dict[str, Any]] = None + + _hidden_params: dict = PrivateAttr(default_factory=dict) + + +class InteractionsAPIStreamingResponse(BaseLiteLLMOpenAIResponseObject): + """ + Streaming response chunk from the Interactions API. + + Event types per OpenAPI spec: + - interaction.start, interaction.status_update, interaction.complete + - content.start, content.delta, content.stop + - error + """ + event_type: Optional[str] = None + id: Optional[str] = None + object: Optional[str] = "interaction" + model: Optional[str] = None + agent: Optional[str] = None + status: Optional[str] = None + created: Optional[str] = None + updated: Optional[str] = None + role: Optional[str] = None + outputs: Optional[List[Dict[str, Any]]] = None + usage: Optional[Dict[str, Any]] = None + delta: Optional[Dict[str, Any]] = None + + _hidden_params: dict = PrivateAttr(default_factory=dict) + + +class DeleteInteractionResult(BaseLiteLLMOpenAIResponseObject): + """Result of deleting an interaction.""" + success: bool = True + id: Optional[str] = None + + _hidden_params: dict = PrivateAttr(default_factory=dict) + + +class CancelInteractionResult(BaseLiteLLMOpenAIResponseObject): + """Result of cancelling an interaction.""" + id: Optional[str] = None + status: Optional[str] = None + + _hidden_params: dict = PrivateAttr(default_factory=dict) + + +# Backwards compatibility aliases +InteractionTool = Tool +InteractionToolChoiceConfig = ToolChoiceConfig +InteractionsAPIOptionalRequestParams = Dict[str, Any] diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 9bc4ca1703d..381d91de762 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -169,7 +169,7 @@ class SafetSettingsConfig(TypedDict, total=False): class GeminiThinkingConfig(TypedDict, total=False): includeThoughts: bool thinkingBudget: int - thinkingLevel: Literal["low", "medium", "high"] + thinkingLevel: Literal["minimal", "low", "medium", "high"] GeminiResponseModalities = Literal["TEXT", "IMAGE", "AUDIO", "VIDEO"] diff --git a/litellm/utils.py b/litellm/utils.py index 524e86cfbbe..49ab1744bfc 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1,3 +1,6 @@ +# from __future__ import annotations must be the first non-comment statement +from __future__ import annotations + # +-----------------------------------------------+ # | | # | Give Feedback / Get Help | @@ -96,11 +99,15 @@ from litellm.litellm_core_utils.core_helpers import ( process_response_headers, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor -from litellm.litellm_core_utils.default_encoding import encoding from litellm.litellm_core_utils.dot_notation_indexing import ( delete_nested_value, is_nested_path, ) +from litellm._lazy_imports import ( + _get_default_encoding, + _get_modified_max_tokens, + _get_token_counter_new, +) from litellm.litellm_core_utils.exception_mapping_utils import ( _get_response_headers, exception_type, @@ -144,7 +151,6 @@ from litellm.litellm_core_utils.redact_messages import ( ) from litellm.litellm_core_utils.rules import Rules from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -from litellm.litellm_core_utils.token_counter import get_modified_max_tokens from litellm.llms.base_llm.google_genai.transformation import ( BaseGoogleGenAIGenerateContentConfig, ) @@ -249,7 +255,6 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( update_response_metadata, ) from litellm.litellm_core_utils.thread_pool_executor import executor -from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -260,12 +265,17 @@ from litellm.llms.base_llm.base_utils import ( BaseLLMModelInfo, type_to_response_format_param, ) + +if TYPE_CHECKING: + # Heavy types that are only needed for type checking; avoid importing + # their modules at runtime during `litellm` import. + from litellm.llms.base_llm.files.transformation import BaseFilesConfig + from litellm.proxy._types import AllowedModelRegion from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig from litellm.llms.base_llm.containers.transformation import BaseContainerConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig -from litellm.llms.base_llm.files.transformation import BaseFilesConfig from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, @@ -293,6 +303,7 @@ from .caching.caching import ( RedisSemanticCache, S3Cache, ) + from .exceptions import ( APIConnectionError, APIError, @@ -310,7 +321,6 @@ from .exceptions import ( UnprocessableEntityError, UnsupportedParamsError, ) -from .proxy._types import AllowedModelRegion, KeyManagementSystem from .types.llms.openai import ( ChatCompletionDeltaToolCallChunk, ChatCompletionToolCallChunk, @@ -1244,7 +1254,7 @@ def client(original_function): # noqa: PLR0915 elif kwargs.get("messages", None): messages = kwargs["messages"] user_max_tokens = kwargs.get("max_tokens") - modified_max_tokens = get_modified_max_tokens( + modified_max_tokens = _get_modified_max_tokens()( model=model, base_model=base_model, messages=messages, @@ -1481,7 +1491,7 @@ def client(original_function): # noqa: PLR0915 elif kwargs.get("messages", None): messages = kwargs["messages"] user_max_tokens = kwargs.get("max_tokens") - modified_max_tokens = get_modified_max_tokens( + modified_max_tokens = _get_modified_max_tokens()( model=model, base_model=base_model, messages=messages, @@ -1752,7 +1762,7 @@ def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: def _return_openai_tokenizer(model: str) -> SelectTokenizerResponse: - return {"type": "openai_tokenizer", "tokenizer": encoding} + return {"type": "openai_tokenizer", "tokenizer": _get_default_encoding()} def _return_huggingface_tokenizer(model: str) -> Optional[SelectTokenizerResponse]: @@ -1872,7 +1882,7 @@ def token_counter( if litellm.disable_token_counter is True: return 0 - return token_counter_new( + return _get_token_counter_new()( model, custom_tokenizer, text, @@ -5842,7 +5852,7 @@ def prompt_token_calculator(model, messages): anthropic_obj = Anthropic() num_tokens = anthropic_obj.count_tokens(text) # type: ignore else: - num_tokens = len(encoding.encode(text)) + num_tokens = len(_get_default_encoding().encode(text)) return num_tokens @@ -7248,6 +7258,8 @@ class ProviderConfigManager: return litellm.AzureOpenAIGPT5Config() return litellm.AzureOpenAIConfig() elif litellm.LlmProviders.AZURE_AI == provider: + if "claude" in model.lower(): + return litellm.AzureAnthropicConfig() return litellm.AzureAIStudioConfig() elif litellm.LlmProviders.AZURE_TEXT == provider: return litellm.AzureOpenAITextConfig() @@ -8187,9 +8199,6 @@ def extract_duration_from_srt_or_vtt(srt_or_vtt_content: str) -> Optional[float] return max(durations) if durations else None -import httpx - - def _add_path_to_api_base(api_base: str, ending_path: str) -> str: """ Adds an ending path to an API base URL while preventing duplicate path segments. @@ -8386,3 +8395,17 @@ def should_run_mock_completion( if mock_response or mock_tool_calls or mock_timeout: return True return False + + +# Re-export encoding from main.py for backward compatibility +# This allows tests to import: from litellm.utils import encoding +# We use a lazy import to avoid loading main.py at utils.py import time +def __getattr__(name: str) -> Any: + """Lazy import handler for utils module""" + if name == "encoding": + from litellm.main import encoding as _encoding + # Cache it in the module's __dict__ for subsequent accesses + import sys + sys.modules[__name__].__dict__["encoding"] = _encoding + return _encoding + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4b016bc6ca6..024b89f5dba 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5166,6 +5166,34 @@ "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0 + }, + "azure_ai/deepseek-v3.2": { + "input_cost_per_token": 5.8e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.68e-06, + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "azure_ai/deepseek-v3.2-speciale": { + "input_cost_per_token": 5.8e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.68e-06, + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true }, "azure_ai/deepseek-r1": { "input_cost_per_token": 1.35e-06, @@ -6723,8 +6751,8 @@ "input_cost_per_token": 3e-06, "litellm_provider": "anthropic", "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -6752,8 +6780,8 @@ "input_cost_per_token": 3e-06, "litellm_provider": "anthropic", "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -14704,6 +14732,98 @@ "supports_web_search": true, "tpm": 800000 }, + "gemini/gemini-3-flash-preview": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 3e-06, + "output_cost_per_token": 3e-06, + "rpm": 2000, + "source": "https://ai.google.dev/pricing/gemini-3", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, + "gemini-3-flash-preview": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 3e-06, + "output_cost_per_token": 3e-06, + "source": "https://ai.google.dev/pricing/gemini-3", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true + }, "gemini/gemini-2.5-pro-exp-03-25": { "cache_read_input_token_cost": 0.0, "input_cost_per_token": 0.0, @@ -15185,6 +15305,301 @@ "video" ] }, + "github_copilot/claude-haiku-4.5": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/claude-opus-4.5": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/claude-opus-41": { + "litellm_provider": "github_copilot", + "max_input_tokens": 80000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions" + ], + "supports_vision": true + }, + "github_copilot/claude-sonnet-4": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/claude-sonnet-4.5": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gemini-2.5-pro": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gemini-3-pro-preview": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gpt-3.5-turbo": { + "litellm_provider": "github_copilot", + "max_input_tokens": 16384, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true + }, + "github_copilot/gpt-3.5-turbo-0613": { + "litellm_provider": "github_copilot", + "max_input_tokens": 16384, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true + }, + "github_copilot/gpt-4": { + "litellm_provider": "github_copilot", + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true + }, + "github_copilot/gpt-4-0613": { + "litellm_provider": "github_copilot", + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true + }, + "github_copilot/gpt-4-o-preview": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true + }, + "github_copilot/gpt-4.1": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-4.1-2025-04-14": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-41-copilot": { + "litellm_provider": "github_copilot", + "mode": "completion" + }, + "github_copilot/gpt-4o": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gpt-4o-2024-05-13": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gpt-4o-2024-08-06": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true + }, + "github_copilot/gpt-4o-2024-11-20": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "github_copilot/gpt-4o-mini": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true + }, + "github_copilot/gpt-4o-mini-2024-07-18": { + "litellm_provider": "github_copilot", + "max_input_tokens": 64000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true + }, + "github_copilot/gpt-5": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions", + "/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-5-mini": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-5.1": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions", + "/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-5.1-codex-max": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/gpt-5.2": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supported_endpoints": [ + "/chat/completions", + "/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "github_copilot/text-embedding-3-small": { + "litellm_provider": "github_copilot", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding" + }, + "github_copilot/text-embedding-3-small-inference": { + "litellm_provider": "github_copilot", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding" + }, + "github_copilot/text-embedding-ada-002": { + "litellm_provider": "github_copilot", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding" + }, "google.gemma-3-12b-it": { "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", @@ -16350,6 +16765,34 @@ "/v1/audio/transcriptions" ] }, + "gpt-image-1.5": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_token": 1e-05, + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3.2e-05, + "supported_endpoints": [ + "/v1/images/generations" + ], + "supports_vision": true + }, + "gpt-image-1.5-2025-12-16": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_token": 1e-05, + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3.2e-05, + "supported_endpoints": [ + "/v1/images/generations" + ], + "supports_vision": true + }, "gpt-5": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_flex": 6.25e-08, @@ -30628,11 +31071,11 @@ "litellm_provider": "fireworks_ai", "mode": "embedding" }, - "fireworks_ai/accounts/fireworks/models/qwen3-embedding-8b": { + "fireworks_ai/accounts/fireworks/models/": { "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, - "input_cost_per_token": 0.0, + "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, "litellm_provider": "fireworks_ai", "mode": "embedding" diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index cec9f9e37c0..c9f38b34f39 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -18,6 +18,7 @@ "ocr": "Supports /ocr endpoint", "search": "Supports /search endpoint", "skills": "Supports /skills endpoint", + "interactions": "Supports /interactions endpoint (Google AI Interactions API)", "a2a_(Agent Gateway)": "Supports /a2a/{agent}/message/send endpoint (A2A Protocol)", "create_container": "Supports POST /containers endpoint", "list_containers": "Supports GET /containers endpoint", @@ -831,6 +832,7 @@ "moderations": false, "batches": false, "rerank": false, + "interactions": true, "a2a": true } }, @@ -1946,6 +1948,40 @@ "rerank": false, "a2a": true } + }, + "vertex_ai/agent_engine": { + "display_name": "Vertex AI Agent Engine (`vertex_ai/agent_engine`)", + "url": "https://docs.litellm.ai/docs/providers/vertex_ai_agent_engine", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true + } + }, + "pydantic_ai_agents": { + "display_name": "Pydantic AI Agents (`pydantic_ai_agents`)", + "url": "https://docs.litellm.ai/docs/providers/pydantic_ai_agent", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true + } } } } \ No newline at end of file diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index df3a08a143b..85c26ed37e7 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -152,6 +152,7 @@ model_list: litellm_settings: # set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production drop_params: True + success_callback: ["prometheus"] # max_budget: 100 # budget_duration: 30d num_retries: 5 @@ -227,4 +228,4 @@ general_settings: # settings for using redis caching # REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com # REDIS_PORT: "16337" - # REDIS_PASSWORD: + # REDIS_PASSWORD: \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 54a965b6fbe..862bde939f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ uvloop==0.21.0 # uvicorn dep, gives us much better performance under load boto3==1.36.0 # aws bedrock/sagemaker calls redis==5.2.1 # redis caching prisma==0.11.0 # for db +nodejs-bin==18.4.0a4 ## required by prisma for migrations, prevents runtime download mangum==0.17.0 # for aws lambda functions pynacl==1.5.0 # for encrypting keys google-cloud-aiplatform==1.47.0 # for vertex ai calls diff --git a/tests/agent_tests/local_vertex_agent.py b/tests/agent_tests/local_vertex_agent.py new file mode 100644 index 00000000000..3cc9f868612 --- /dev/null +++ b/tests/agent_tests/local_vertex_agent.py @@ -0,0 +1,151 @@ +""" +Test script for Vertex AI Reasoning Engine. + +This script demonstrates how to: +1. Authenticate with Google Cloud +2. Send queries to a Vertex AI Reasoning Engine using the :query endpoint + +Usage: + python local_vertex_agent.py + +Requirements: + pip install httpx google-auth +""" + +import asyncio +import json +from uuid import uuid4 + +from google.auth import default +from google.auth.transport.requests import Request +import httpx + +# Configuration - update these for your agent +PROJECT_ID = "gen-lang-client-0682925754" # Your GCP project ID +LOCATION = "us-central1" # Your agent's location + +# For Reasoning Engines, use just the numeric ID at the end +REASONING_ENGINE_ID = "8263861224643493888" + +# The project number from the resource name +PROJECT_NUMBER = "1060139831167" + + +async def main(): + """Main function to test Vertex AI Reasoning Engine.""" + + # Step 1: Authenticate with Google Cloud + print("Step 1: Authenticating with Google Cloud...") + credentials, project = default(scopes=['https://www.googleapis.com/auth/cloud-platform']) + credentials.refresh(Request()) + print(f"Authenticated! Project: {project}") + print(f"Token (first 20 chars): {credentials.token[:20]}...") + + # Step 2: Build the endpoint URL + base_url = f"https://{LOCATION}-aiplatform.googleapis.com" + resource_path = f"projects/{PROJECT_NUMBER}/locations/{LOCATION}/reasoningEngines/{REASONING_ENGINE_ID}" + + # The Reasoning Engine uses :query endpoint with specific format + query_url = f"{base_url}/v1beta1/{resource_path}:query" + stream_url = f"{base_url}/v1beta1/{resource_path}:streamQuery" + + print(f"\nQuery URL: {query_url}") + print(f"Stream URL: {stream_url}") + + # Step 3: Create authenticated httpx client + print("\nStep 2: Creating authenticated HTTP client...") + client = httpx.AsyncClient( + headers={ + "Authorization": f"Bearer {credentials.token}", + "Content-Type": "application/json", + }, + timeout=120.0, + ) + + # Step 4: Build the query request (non-streaming) + # Note: For non-streaming, we need to: + # 1. Create a session + # 2. Use the streaming endpoint with stream_query method + # The :query endpoint only supports session management methods + + user_id = f"test-user-{uuid4().hex[:8]}" + + # First create a session + create_session_request = { + "class_method": "async_create_session", + "input": { + "user_id": user_id, + } + } + + print(f"\nStep 3: Creating session...") + print(f"User ID: {user_id}") + + async with client: + # Create session + print(f"\nSending to: {query_url}") + response = await client.post(query_url, json=create_session_request) + print(f"Create session status: {response.status_code}") + + if response.status_code == 200: + session_data = response.json() + print(f"Session created:\n{json.dumps(session_data, indent=2)}") + + # Extract session_id from response + session_id = session_data.get("output", {}).get("id") or session_data.get("output", {}).get("session_id") + print(f"\nSession ID: {session_id}") + + # Now send the actual query via streamQuery + query_request = { + "class_method": "stream_query", + "input": { + "message": "Hello! What can you do?", + "user_id": user_id, + "session_id": session_id, + } + } + + print(f"\nStep 4: Sending query via streamQuery...") + print(f"Request:\n{json.dumps(query_request, indent=2)}") + + # Use streaming endpoint but collect full response + async with client.stream("POST", stream_url, json=query_request) as stream_response: + print(f"Query status: {stream_response.status_code}") + + if stream_response.status_code == 200: + print("\nResponse:") + full_response = "" + async for line in stream_response.aiter_lines(): + if line: + full_response = line # Keep last line (full response) + + # Parse and display + try: + data = json.loads(full_response) + # Extract the text from the response + content = data.get("content", {}) + parts = content.get("parts", []) + for part in parts: + if "text" in part: + print(f"\nAgent response:\n{part['text']}") + except: + print(full_response) + else: + content = await stream_response.aread() + print(f"Error: {content.decode()}") + else: + print(f"Error creating session: {response.text}") + + +if __name__ == "__main__": + print("=" * 60) + print("Vertex AI Reasoning Engine Test Script") + print("=" * 60) + print(f"\nConfiguration:") + print(f" PROJECT_ID: {PROJECT_ID}") + print(f" PROJECT_NUMBER: {PROJECT_NUMBER}") + print(f" LOCATION: {LOCATION}") + print(f" REASONING_ENGINE_ID: {REASONING_ENGINE_ID}") + print() + + asyncio.run(main()) diff --git a/tests/agent_tests/test_a2a.py b/tests/agent_tests/test_a2a.py index eeab2680564..1550d61f7b0 100644 --- a/tests/agent_tests/test_a2a.py +++ b/tests/agent_tests/test_a2a.py @@ -21,10 +21,7 @@ from litellm.types.utils import StandardLoggingPayload sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path - from a2a.types import MessageSendParams, SendMessageRequest - - @pytest.mark.asyncio async def test_asend_message_with_client_decorator(): """ @@ -165,3 +162,163 @@ async def test_a2a_logging_payload(): # This confirms the A2A cost calculator is working assert response_cost is not None, "response_cost should not be None" assert response_cost == 0.0, f"response_cost should be 0.0 for A2A, got: {response_cost}" + + +@pytest.mark.asyncio +async def test_pydantic_ai_non_streaming(): + """ + Test non-streaming requests to Pydantic AI agents. + + Pydantic AI agents follow A2A protocol but don't support streaming. + This test validates non-streaming requests work correctly. + """ + litellm._turn_on_debug() + from litellm.a2a_protocol import asend_message + + # Build the request + send_message_payload = { + "message": { + "role": "user", + "parts": [ + { + "kind": "text", + "text": "Hello from Pydantic AI test!", + } + ], + "messageId": uuid4().hex, + }, + } + + request = SendMessageRequest( + id=str(uuid4()), + params=MessageSendParams(**send_message_payload), + ) + + # Send message using Pydantic AI provider + response = await asend_message( + request=request, + api_base="http://localhost:9999", + litellm_params={"custom_llm_provider": "pydantic_ai_agents"}, + ) + + # Print response for debugging + print("\n=== Pydantic AI Non-Streaming Response ===") + print(response.model_dump(mode="json", exclude_none=True)) + + # Basic assertions + assert response is not None + assert hasattr(response, "result") + + # Verify result structure + result = response.result + assert result is not None + + # Pydantic AI returns a task with history/artifacts, not a direct message + # Check for either format + result_dict = result if isinstance(result, dict) else result.model_dump(mode="python", exclude_none=True) + has_message = "message" in result_dict + has_history = "history" in result_dict + has_artifacts = "artifacts" in result_dict + + assert has_message or has_history or has_artifacts, ( + f"Result should contain 'message', 'history', or 'artifacts'. Got: {list(result_dict.keys())}" + ) + + # If it's a task response (Pydantic AI style), verify we got agent response + if has_history: + history = result_dict.get("history", []) + agent_messages = [m for m in history if m.get("role") == "agent"] + assert len(agent_messages) > 0, "Should have at least one agent message in history" + + # Verify agent message has text content + agent_msg = agent_messages[-1] + parts = agent_msg.get("parts", []) + text_parts = [p for p in parts if p.get("kind") == "text"] + assert len(text_parts) > 0, "Agent message should have text content" + print(f"\nAgent response: {text_parts[0].get('text')}") + + +@pytest.mark.asyncio +async def test_pydantic_ai_fake_streaming(): + """ + Test fake streaming for Pydantic AI agents. + + Pydantic AI agents don't support streaming natively. + This test validates that fake streaming works by converting + non-streaming responses into streaming chunks. + """ + litellm._turn_on_debug() + from litellm.a2a_protocol import asend_message_streaming + + # Build the request + from a2a.types import SendStreamingMessageRequest + + send_message_payload = { + "message": { + "role": "user", + "parts": [ + { + "kind": "text", + "text": "Hello from Pydantic AI streaming test!", + } + ], + "messageId": uuid4().hex, + }, + } + + request = SendStreamingMessageRequest( + id=str(uuid4()), + params=MessageSendParams(**send_message_payload), + ) + + # Send streaming message using Pydantic AI provider + print("\n=== Pydantic AI Fake Streaming Response ===") + chunks_received = 0 + task_event_received = False + working_event_received = False + artifact_event_received = False + completed_event_received = False + + async for chunk in asend_message_streaming( + request=request, + api_base="http://localhost:9999", + litellm_params={"custom_llm_provider": "pydantic_ai_agents"}, + ): + chunks_received += 1 + print(f"\nChunk {chunks_received}:") + + # Convert chunk to dict for inspection + chunk_dict = chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else chunk + print(json.dumps(chunk_dict, indent=2)) + + # Check event types + result = chunk_dict.get("result", {}) + kind = result.get("kind") + + if kind == "task": + task_event_received = True + elif kind == "status-update": + status = result.get("status", {}) + state = status.get("state") + if state == "working": + working_event_received = True + elif state == "completed": + completed_event_received = True + elif kind == "artifact-update": + artifact_event_received = True + + print(f"\n=== Streaming Summary ===") + print(f"Total chunks received: {chunks_received}") + print(f"Task event received: {task_event_received}") + print(f"Working event received: {working_event_received}") + print(f"Artifact event received: {artifact_event_received}") + print(f"Completed event received: {completed_event_received}") + + # Verify we received chunks + assert chunks_received > 0, "Should receive at least one chunk" + + # Verify all required event types were received + assert task_event_received, "Should receive task event" + assert working_event_received, "Should receive working status event" + assert artifact_event_received, "Should receive artifact update event" + assert completed_event_received, "Should receive completed status event" diff --git a/tests/agent_tests/test_a2a_completion_bridge.py b/tests/agent_tests/test_a2a_completion_bridge.py index 4191821f3de..224809dd7f5 100644 --- a/tests/agent_tests/test_a2a_completion_bridge.py +++ b/tests/agent_tests/test_a2a_completion_bridge.py @@ -201,3 +201,79 @@ async def test_a2a_completion_bridge_bedrock_agentcore(): print(f"Received {len(chunks)} chunks from Bedrock AgentCore") + +# ============================================================ +# Vertex AI Agent Engine Tests +# ============================================================ + +# Configuration - update these for your Vertex AI Reasoning Engine +VERTEX_AGENT_RESOURCE_NAME = "projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888" + + +@pytest.mark.asyncio +async def test_vertex_agent_engine_non_streaming(): + """ + Test non-streaming request to Vertex AI Agent Engine via litellm.acompletion. + + Uses the Reasoning Engine resource ID to call a hosted agent. + """ + + litellm._turn_on_debug() + + # Call via litellm.acompletion with vertex_ai/agent_engine/ prefix + response = await litellm.acompletion( + model=f"vertex_ai/agent_engine/{VERTEX_AGENT_RESOURCE_NAME}", + messages=[{"role": "user", "content": "Hello! What can you do?"}], + stream=False, + ) + + print(f"\n=== Vertex Agent Engine Non-Streaming Response ===") + print(f"Response: {response}") + + # Basic assertions + assert response is not None + assert hasattr(response, "choices") + assert len(response.choices) > 0 + assert response.choices[0].message is not None + assert response.choices[0].message.content is not None + assert len(response.choices[0].message.content) > 0 + + print(f"Agent response: {response.choices[0].message.content[:200]}...") + + +@pytest.mark.asyncio +async def test_vertex_agent_engine_streaming(): + """ + Test streaming request to Vertex AI Agent Engine via litellm.acompletion. + + Uses the Reasoning Engine resource ID to call a hosted agent with streaming. + """ + #litellm._turn_on_debug() + + # Call via litellm.acompletion with streaming + response = await litellm.acompletion( + model=f"vertex_ai/agent_engine/{VERTEX_AGENT_RESOURCE_NAME}", + messages=[{"role": "user", "content": "Hello! What can you do?"}], + stream=True, + ) + + print(f"\n=== Vertex Agent Engine Streaming Response ===") + + chunks = [] + full_content = "" + async for chunk in response: + print(f"Chunk: {chunk}") + # chunks.append(chunk) + # if hasattr(chunk, "choices") and len(chunk.choices) > 0: + # delta = chunk.choices[0].delta + # if hasattr(delta, "content") and delta.content: + # full_content += delta.content + # print(f"Chunk: {delta.content}", end="", flush=True) + + # # print(f"\n\nReceived {len(chunks)} chunks") + # print(f"Full content: {full_content[:200]}...") + + # # Basic assertions + # assert len(chunks) > 0 + # assert len(full_content) > 0 + diff --git a/tests/batches_tests/test_openai_batches_and_files.py b/tests/batches_tests/test_openai_batches_and_files.py index 2f4f9bbcda1..055af024949 100644 --- a/tests/batches_tests/test_openai_batches_and_files.py +++ b/tests/batches_tests/test_openai_batches_and_files.py @@ -577,3 +577,73 @@ async def test_vertex_list_batches(monkeypatch): assert len(list_response["data"]) == 2 assert list_response["data"][0].id == "test-batch-id-456" assert list_response["data"][1].id == "test-batch-id-789" + + +@pytest.mark.asyncio +async def test_delete_batch_output_file(): + """ + Test that deleting a batch output file works correctly. + + This test verifies the fix for: + - When a batch is retrieved and has an output_file_id, the file object is properly stored + - The output file can be deleted without validation errors + - The file_object is fetched and stored with proper metadata instead of None + """ + litellm._turn_on_debug() + print("Testing delete batch output file") + + file_name = "openai_batch_completions.jsonl" + _current_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(_current_dir, file_name) + + # Create file for batch + file_obj = await litellm.acreate_file( + file=open(file_path, "rb"), + purpose="batch", + custom_llm_provider="openai", + ) + print("Response from creating file=", file_obj) + batch_input_file_id = file_obj.id + + # Create batch + create_batch_response = await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id=batch_input_file_id, + custom_llm_provider="openai", + ) + print("Batch created with ID=", create_batch_response.id) + + # Retrieve batch to get output_file_id + retrieved_batch = await litellm.aretrieve_batch( + batch_id=create_batch_response.id, + custom_llm_provider="openai" + ) + print("Retrieved batch=", retrieved_batch) + + # If batch has completed and has output file, test deleting it + if retrieved_batch.output_file_id: + print(f"Testing deletion of output file: {retrieved_batch.output_file_id}") + + # This is the key test - deleting the output file should work + # without validation errors (file_object should not be None) + delete_output_file_response = await litellm.afile_delete( + file_id=retrieved_batch.output_file_id, + custom_llm_provider="openai" + ) + + print("Delete output file response=", delete_output_file_response) + assert delete_output_file_response.id == retrieved_batch.output_file_id + assert delete_output_file_response.deleted is True or hasattr(delete_output_file_response, 'id') + print("✓ Successfully deleted batch output file") + else: + print("⚠ Batch has not completed yet or no output file available, skipping output file deletion test") + + # Clean up - delete the input file + delete_input_file_response = await litellm.afile_delete( + file_id=batch_input_file_id, + custom_llm_provider="openai" + ) + print("Delete input file response=", delete_input_file_response) + assert delete_input_file_response.id == batch_input_file_id + print("✓ Successfully deleted batch input file") diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index e8fe4dd3393..2f92afb3824 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -1124,6 +1124,124 @@ def test_get_custom_labels_from_metadata_tags(monkeypatch): assert get_custom_labels_from_metadata(metadata) == {} +def test_get_custom_labels_from_top_level_metadata(monkeypatch): + """ + Test that get_custom_labels_from_metadata can extract fields from top-level metadata, + such as requester_ip_address, not just from nested dictionaries like requester_metadata. + """ + monkeypatch.setattr( + "litellm.custom_prometheus_metadata_labels", + ["requester_ip_address", "user_api_key_alias"], + ) + # Simulate metadata structure with top-level fields + metadata = { + "requester_ip_address": "10.48.203.20", # Top-level field + "user_api_key_alias": "TestAlias", # Top-level field + "requester_metadata": {"nested_field": "nested_value"}, # Nested dict (excluded) + "user_api_key_auth_metadata": {"another_nested": "value"}, # Nested dict (excluded) + } + result = get_custom_labels_from_metadata(metadata) + assert result == { + "requester_ip_address": "10.48.203.20", + "user_api_key_alias": "TestAlias", + } + + +def test_get_custom_labels_from_top_level_and_nested_metadata(monkeypatch): + """ + Test that get_custom_labels_from_metadata can extract fields from both top-level + and nested metadata (requester_metadata, user_api_key_auth_metadata). + """ + monkeypatch.setattr( + "litellm.custom_prometheus_metadata_labels", + [ + "requester_ip_address", # Top-level + "metadata.foo", # From requester_metadata + "metadata.bar", # From user_api_key_auth_metadata + ], + ) + # Simulate combined_metadata structure as it would appear after merging + # This is what gets passed to get_custom_labels_from_metadata + combined_metadata = { + "requester_ip_address": "10.48.203.20", # Top-level field + "foo": "bar_value", # From requester_metadata (spread) + "bar": "baz_value", # From user_api_key_auth_metadata (spread) + } + result = get_custom_labels_from_metadata(combined_metadata) + assert result == { + "requester_ip_address": "10.48.203.20", + "metadata_foo": "bar_value", + "metadata_bar": "baz_value", + } + + +async def test_async_log_success_event_with_top_level_metadata(prometheus_logger, monkeypatch): + """ + Test that async_log_success_event correctly extracts custom labels from top-level metadata + fields like requester_ip_address, not just from nested dictionaries. + """ + # Configure custom metadata labels to extract requester_ip_address + monkeypatch.setattr( + "litellm.custom_prometheus_metadata_labels", ["requester_ip_address"] + ) + + # Create standard logging payload with requester_ip_address at top-level metadata + standard_logging_object = create_standard_logging_payload() + standard_logging_object["metadata"]["requester_ip_address"] = "10.48.203.20" + standard_logging_object["metadata"]["requester_metadata"] = {} # Empty nested dict + standard_logging_object["metadata"]["user_api_key_auth_metadata"] = {} # Empty nested dict + + kwargs = { + "model": "gpt-3.5-turbo", + "stream": True, + "litellm_params": { + "metadata": { + "user_api_key": "test_key", + "user_api_key_user_id": "test_user", + "user_api_key_team_id": "test_team", + "user_api_key_end_user_id": "test_end_user", + } + }, + "start_time": datetime.now(), + "completion_start_time": datetime.now(), + "api_call_start_time": datetime.now(), + "end_time": datetime.now() + timedelta(seconds=1), + "standard_logging_object": standard_logging_object, + } + response_obj = MagicMock() + + # Mock the prometheus client methods + prometheus_logger.litellm_requests_metric = MagicMock() + prometheus_logger.litellm_spend_metric = MagicMock() + prometheus_logger.litellm_tokens_metric = MagicMock() + prometheus_logger.litellm_input_tokens_metric = MagicMock() + prometheus_logger.litellm_output_tokens_metric = MagicMock() + prometheus_logger.litellm_remaining_team_budget_metric = MagicMock() + prometheus_logger.litellm_remaining_api_key_budget_metric = MagicMock() + prometheus_logger.litellm_remaining_api_key_requests_for_model = MagicMock() + prometheus_logger.litellm_remaining_api_key_tokens_for_model = MagicMock() + prometheus_logger.litellm_llm_api_time_to_first_token_metric = MagicMock() + prometheus_logger.litellm_llm_api_latency_metric = MagicMock() + prometheus_logger.litellm_request_total_latency_metric = MagicMock() + + await prometheus_logger.async_log_success_event( + kwargs, response_obj, kwargs["start_time"], kwargs["end_time"] + ) + + # Verify that the metrics were called with labels including requester_ip_address + # Check that labels() was called - the actual labels dict should include requester_ip_address + assert prometheus_logger.litellm_requests_metric.labels.called + assert prometheus_logger.litellm_spend_metric.labels.called + + # Get the actual call arguments to verify requester_ip_address is included + # The custom labels should be extracted and included in the label factory + call_args = prometheus_logger.litellm_requests_metric.labels.call_args + assert call_args is not None + # The labels() method receives a dict with label names and values + # We can't easily assert the exact values without checking the internal implementation, + # but we've verified the function is called, which means the extraction happened + + def test_get_custom_labels_from_tags(monkeypatch): from litellm.integrations.prometheus import get_custom_labels_from_tags diff --git a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py index 5526f22cd5e..73331547772 100644 --- a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py +++ b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py @@ -541,3 +541,28 @@ def test_amazon_titan_image_gen(): print(f"response cost: {response._hidden_params['response_cost']}") assert response._hidden_params["response_cost"] > 0 + + +def test_extract_headers_from_optional_params_with_guardrails(): + """Test that guardrail parameters are correctly extracted from optional_params and converted to headers""" + handler = BedrockImageGeneration() + + # Test with both guardrail parameters + optional_params = { + "guardrailIdentifier": "4cf5knqaeq15", + "guardrailVersion": "1", + "someOtherParam": "value", + } + + headers = handler._extract_headers_from_optional_params(optional_params) + + # Verify headers are correctly set + assert headers["x-amz-bedrock-guardrail-identifier"] == "4cf5knqaeq15" + assert headers["x-amz-bedrock-guardrail-version"] == "1" + + # Verify guardrail params are removed from optional_params + assert "guardrailIdentifier" not in optional_params + assert "guardrailVersion" not in optional_params + + # Verify other params remain in optional_params + assert optional_params["someOtherParam"] == "value" diff --git a/tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py b/tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py new file mode 100644 index 00000000000..7b10c46c2fd --- /dev/null +++ b/tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py @@ -0,0 +1,99 @@ +""" +Tests for Pydantic AI agents transformation. + +Tests the helper functions and response transformation without making real API calls. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.a2a_protocol.providers.pydantic_ai_agents.transformation import ( + PydanticAITransformation, +) + + +class TestPydanticAITransformation: + """Tests for PydanticAITransformation helper methods.""" + + def test_remove_none_values(self): + """ + Test that _remove_none_values recursively removes None values from dicts. + FastA2A servers reject None values for optional fields. + """ + input_data = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Hello"}], + "contextId": None, + "taskId": None, + "metadata": None, + }, + "configuration": None, + "metadata": {"key": "value", "empty": None}, + } + + result = PydanticAITransformation._remove_none_values(input_data) + + # None values should be removed + assert "contextId" not in result["message"] + assert "taskId" not in result["message"] + assert "metadata" not in result["message"] + assert "configuration" not in result + assert "empty" not in result["metadata"] + + # Non-None values should be preserved + assert result["message"]["role"] == "user" + assert result["message"]["parts"] == [{"kind": "text", "text": "Hello"}] + assert result["metadata"]["key"] == "value" + + def test_transform_to_a2a_response(self): + """ + Test that _transform_to_a2a_response converts Pydantic AI task format + to standard A2A non-streaming response format. + """ + # Pydantic AI returns tasks with history/artifacts + pydantic_ai_response = { + "jsonrpc": "2.0", + "id": "req-123", + "result": { + "id": "task-456", + "kind": "task", + "status": {"state": "completed"}, + "history": [ + { + "role": "user", + "parts": [{"kind": "text", "text": "What is 2+2?"}], + "messageId": "msg-user-1", + }, + { + "role": "agent", + "parts": [{"kind": "text", "text": "The answer is 4."}], + "messageId": "msg-agent-1", + }, + ], + "artifacts": [ + { + "artifactId": "artifact-1", + "name": "response", + "parts": [{"kind": "text", "text": "The answer is 4."}], + } + ], + }, + } + + result = PydanticAITransformation._transform_to_a2a_response( + response_data=pydantic_ai_response, + request_id="req-123", + ) + + # Should return standard A2A format with message + assert result["jsonrpc"] == "2.0" + assert result["id"] == "req-123" + assert "message" in result["result"] + assert result["result"]["message"]["role"] == "agent" + assert result["result"]["message"]["parts"][0]["text"] == "The answer is 4." + diff --git a/tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py b/tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py new file mode 100644 index 00000000000..cb3a5807d8c --- /dev/null +++ b/tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py @@ -0,0 +1,128 @@ +""" +Tests for Vertex AI Agent Engine transformation. + +Tests the request transformation and streaming chunk parsing without making real API calls. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.vertex_ai.agent_engine.sse_iterator import ( + VertexAgentEngineResponseIterator, +) +from litellm.llms.vertex_ai.agent_engine.transformation import VertexAgentEngineConfig + + +class TestVertexAgentEngineTransformRequest: + """Tests for transform_request method.""" + + def test_transform_request_basic(self): + """ + Test that transform_request correctly formats messages into Vertex Agent Engine payload. + """ + config = VertexAgentEngineConfig() + + messages = [{"role": "user", "content": "Hello, what can you do?"}] + optional_params = {"user_id": "test-user-123"} + litellm_params = {} + + result = config.transform_request( + model="agent_engine/123456789", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + + assert result["class_method"] == "stream_query" + assert result["input"]["message"] == "Hello, what can you do?" + assert result["input"]["user_id"] == "test-user-123" + assert "session_id" not in result["input"] + + def test_transform_request_with_session_id(self): + """ + Test that transform_request includes session_id when provided. + """ + config = VertexAgentEngineConfig() + + messages = [{"role": "user", "content": "Follow up question"}] + optional_params = { + "user_id": "test-user-123", + "session_id": "session-abc-456", + } + litellm_params = {} + + result = config.transform_request( + model="agent_engine/123456789", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + + assert result["class_method"] == "stream_query" + assert result["input"]["message"] == "Follow up question" + assert result["input"]["user_id"] == "test-user-123" + assert result["input"]["session_id"] == "session-abc-456" + + +class TestVertexAgentEngineChunkParser: + """Tests for the streaming chunk parser.""" + + def test_chunk_parser_with_text_content(self): + """ + Test that chunk_parser correctly extracts text from Vertex Agent Engine response format. + """ + iterator = VertexAgentEngineResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + chunk = { + "content": { + "parts": [{"text": "Hello! I can help you with financial analysis."}], + "role": "model", + }, + "finish_reason": "STOP", + "usage_metadata": { + "prompt_token_count": 100, + "candidates_token_count": 50, + "total_token_count": 150, + }, + } + + result = iterator.chunk_parser(chunk) + + assert result.choices[0].delta.content == "Hello! I can help you with financial analysis." + assert result.choices[0].delta.role == "assistant" + assert result.choices[0].finish_reason == "stop" + assert result.usage["prompt_tokens"] == 100 + assert result.usage["completion_tokens"] == 50 + assert result.usage["total_tokens"] == 150 + + def test_chunk_parser_without_finish_reason(self): + """ + Test that chunk_parser handles chunks without finish_reason (intermediate chunks). + """ + iterator = VertexAgentEngineResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + chunk = { + "content": { + "parts": [{"text": "Partial response..."}], + "role": "model", + }, + } + + result = iterator.chunk_parser(chunk) + + assert result.choices[0].delta.content == "Partial response..." + assert result.choices[0].finish_reason is None + assert result.usage is None + diff --git a/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py b/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py new file mode 100644 index 00000000000..7047be4241b --- /dev/null +++ b/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py @@ -0,0 +1,145 @@ +""" +Test Gemini batch embeddings with custom api_base and extra_headers. + +This test ensures that: +1. Authentication headers are properly included when using custom api_base +2. The extra_headers parameter is correctly passed through +3. Both dict-based auth_header (Gemini) and Bearer token (Vertex AI) are handled +""" + +import json +import os +import sys +from unittest.mock import MagicMock, patch + +sys.path.insert(0, os.path.abspath("../../../..")) + +import pytest +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler + + +def test_gemini_batch_embeddings_with_custom_api_base_and_auth_header(): + """ + Test that Gemini batch embeddings include auth_header when using custom api_base. + + This test verifies that when using Gemini embeddings with a custom api_base + (e.g., Cloudflare AI Gateway), the x-goog-api-key header is properly included + in the HTTP request. + """ + client = HTTPHandler() + + def mock_auth_token(*args, **kwargs): + return None, "test-project" + + with patch.object(client, "post") as mock_post, patch( + "litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler.GoogleBatchEmbeddings._ensure_access_token", + side_effect=mock_auth_token + ), patch( + "litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler.GoogleBatchEmbeddings._get_token_and_url" + ) as mock_get_token: + # Mock the _get_token_and_url to return auth_header dict and URL + mock_get_token.return_value = ( + {"x-goog-api-key": "test-gemini-api-key"}, + "https://gateway.ai.cloudflare.com/v1/test/noauth/google-ai-studio/v1beta" + ) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "predictions": [ + { + "embeddings": { + "values": [0.1, 0.2, 0.3, 0.4, 0.5] + } + } + ] + } + mock_post.return_value = mock_response + + response = litellm.embedding( + model="gemini/text-embedding-004", + input=["Hello, world!"], + api_key="test-gemini-api-key", + api_base="https://gateway.ai.cloudflare.com/v1/test/noauth/google-ai-studio/v1beta", + client=client + ) + + # Verify the POST was called + mock_post.assert_called_once() + + # Get the headers that were passed to the POST request + call_args = mock_post.call_args + kwargs = call_args.kwargs if hasattr(call_args, 'kwargs') else call_args[1] + headers = kwargs.get("headers", {}) + + # Verify auth_header is included + assert "x-goog-api-key" in headers, f"x-goog-api-key not in headers: {headers}" + assert headers["x-goog-api-key"] == "test-gemini-api-key" + + # Verify Content-Type is still present + assert "Content-Type" in headers + assert headers["Content-Type"] == "application/json; charset=utf-8" + + +def test_gemini_batch_embeddings_with_extra_headers(): + """ + Test that extra_headers parameter is properly included in the request. + + This test verifies that custom headers passed via extra_headers are + properly merged into the request headers. + """ + client = HTTPHandler() + + def mock_auth_token(*args, **kwargs): + return None, "test-project" + + with patch.object(client, "post") as mock_post, patch( + "litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler.GoogleBatchEmbeddings._ensure_access_token", + side_effect=mock_auth_token + ), patch( + "litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler.GoogleBatchEmbeddings._get_token_and_url" + ) as mock_get_token: + # Mock the _get_token_and_url to return auth_header dict and URL + mock_get_token.return_value = ( + {"x-goog-api-key": "test-gemini-api-key"}, + "https://gateway.ai.cloudflare.com/v1/test/google-ai-studio/v1beta" + ) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "predictions": [ + { + "embeddings": { + "values": [0.1, 0.2, 0.3] + } + } + ] + } + mock_post.return_value = mock_response + + response = litellm.embedding( + model="gemini/text-embedding-004", + input=["Test"], + api_key="test-gemini-api-key", + api_base="https://gateway.ai.cloudflare.com/v1/test/google-ai-studio/v1beta", + headers={"Authorization": "Bearer test-token", "X-Custom": "custom-value"}, + client=client + ) + + # Verify the POST was called + mock_post.assert_called_once() + + # Get the headers that were passed to the POST request + call_args = mock_post.call_args + kwargs = call_args.kwargs if hasattr(call_args, 'kwargs') else call_args[1] + headers = kwargs.get("headers", {}) + + # Verify all headers are included + assert "x-goog-api-key" in headers + assert "Authorization" in headers + assert headers["Authorization"] == "Bearer test-token" + assert "X-Custom" in headers + assert headers["X-Custom"] == "custom-value" + diff --git a/tests/llm_responses_api_testing/test_azure_responses_api.py b/tests/llm_responses_api_testing/test_azure_responses_api.py index eeb7eb50151..86b490994d6 100644 --- a/tests/llm_responses_api_testing/test_azure_responses_api.py +++ b/tests/llm_responses_api_testing/test_azure_responses_api.py @@ -182,3 +182,94 @@ async def test_azure_responses_api_status_error(): f"Expected: {json.dumps(expected_input, indent=2)}\n" f"Got: {json.dumps(captured_request_body['input'], indent=2)}" ) + + +@pytest.mark.asyncio +async def test_azure_responses_api_headers_with_llm_provider_prefix(): + """ + Test that Azure-specific headers like 'x-request-id' and 'apim-request-id' + are properly forwarded with 'llm_provider-' prefix in response._hidden_params["headers"]. + + Issue: https://github.com/BerriAI/litellm/issues/16538 + + The fix ensures that processed headers (with llm_provider- prefix) are stored + in response._hidden_params["headers"] instead of additional_headers, making them + accessible via completion.headers in the same way as the completion API. + """ + import json + import httpx + + mock_response_data = { + "id": "resp_123", + "object": "response", + "created_at": 1234567890, + "model": "gpt-5-codex", + "status": "completed", + "output": [ + { + "id": "msg_123", + "role": "assistant", + "type": "message", + "content": [{"type": "output_text", "text": "Hello!"}], + } + ], + } + + # Mock headers that Azure returns - exactly like in the issue + mock_headers = { + "date": "Wed, 12 Nov 2025 15:31:28 GMT", + "server": "uvicorn", + "content-type": "application/json", + "x-ratelimit-remaining-tokens": "5010000", + "x-ratelimit-limit-tokens": "5010000", + # These are the Azure-specific headers that should be forwarded with llm_provider- prefix + "x-request-id": "12086715-aca3-4006-a29f-2f1e1d552043", + "apim-request-id": "25664b0d-cf4b-4e10-8d27-c7272e7efd49", + "x-ms-region": "Sweden Central", + } + + async def mock_post(*args, **kwargs): + response_content = json.dumps(mock_response_data).encode("utf-8") + response = httpx.Response( + status_code=200, + headers=mock_headers, + content=response_content, + request=httpx.Request(method="POST", url="https://test.openai.azure.com"), + ) + return response + + with patch.object(AsyncHTTPHandler, "post", new=mock_post): + response = await litellm.aresponses( + model="azure/gpt-5-codex", + api_version="2025-03-01-preview", + api_base="https://test.openai.azure.com", + api_key="test-key", + input="Hello, can you tell me a short joke?", + ) + + # Check that the response has the expected headers structure + assert hasattr(response, "_hidden_params"), "Response should have _hidden_params" + assert "additional_headers" in response._hidden_params, ( + "Response _hidden_params should contain 'additional_headers' with the LLM provider headers" + ) + + headers = response._hidden_params["additional_headers"] + + # Verify that Azure-specific headers are present with llm_provider- prefix + assert "llm_provider-x-request-id" in headers, ( + f"Response should contain 'llm_provider-x-request-id' header. " + f"Headers: {list(headers.keys())}" + ) + assert "llm_provider-apim-request-id" in headers, ( + f"Response should contain 'llm_provider-apim-request-id' header. " + f"Headers: {list(headers.keys())}" + ) + + # Verify the header values match + assert headers["llm_provider-x-request-id"] == "12086715-aca3-4006-a29f-2f1e1d552043" + assert headers["llm_provider-apim-request-id"] == "25664b0d-cf4b-4e10-8d27-c7272e7efd49" + assert headers["llm_provider-x-ms-region"] == "Sweden Central" + + # Also verify openai-compatible headers are included + assert "x-ratelimit-limit-tokens" in headers + assert "x-ratelimit-remaining-tokens" in headers diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index dbbf0d31f1f..ac895f415a8 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -1229,3 +1229,175 @@ def test_gemini_function_args_preserve_unicode(): assert parsed_args["recipient"] == "José" assert "\\u" not in arguments_str assert "José" in arguments_str + + +def test_anthropic_thinking_param_to_gemini_3_thinkingLevel(): + """ + Test that Anthropic thinking parameters are correctly transformed to Gemini 3 thinkingLevel + instead of thinkingBudget. + + For Gemini 3+ models (gemini-3-flash, gemini-3-pro, gemini-3-flash-preview): + - Should use thinkingLevel instead of thinkingBudget + - budget_tokens should map to thinkingLevel + + Related issue: https://github.com/BerriAI/litellm/issues/XXXX + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + from litellm.types.llms.anthropic import AnthropicThinkingParam + + # Test 1: Anthropic thinking enabled with budget_tokens for Gemini 3 model + thinking_param: AnthropicThinkingParam = { + "type": "enabled", + "budget_tokens": 10000, + } + + result = VertexGeminiConfig._map_thinking_param( + thinking_param=thinking_param, + model="gemini-3-flash", + ) + + # For Gemini 3, should use thinkingLevel, not thinkingBudget + assert "thinkingLevel" in result, "Should have thinkingLevel for Gemini 3" + assert "thinkingBudget" not in result, "Should NOT have thinkingBudget for Gemini 3" + assert result["includeThoughts"] is True + assert result["thinkingLevel"] in ["minimal", "low"], "thinkingLevel should be 'minimal' or 'low'" + + # Test 2: Anthropic thinking disabled for Gemini 3 + thinking_param_disabled: AnthropicThinkingParam = { + "type": "disabled", + "budget_tokens": None, + } + + result_disabled = VertexGeminiConfig._map_thinking_param( + thinking_param=thinking_param_disabled, + model="gemini-3-pro-preview", + ) + + assert result_disabled.get("includeThoughts") is False + assert "thinkingLevel" not in result_disabled or result_disabled.get("thinkingLevel") is None + + # Test 3: Budget tokens = 0 for Gemini 3 + thinking_param_zero: AnthropicThinkingParam = { + "type": "enabled", + "budget_tokens": 0, + } + + result_zero = VertexGeminiConfig._map_thinking_param( + thinking_param=thinking_param_zero, + model="gemini-3-flash", + ) + + assert result_zero["includeThoughts"] is False + assert "thinkingLevel" not in result_zero or result_zero.get("thinkingLevel") is None + + # Test 4: Fiercefalcon model (Gemini 3 Flash checkpoint) should use thinkingLevel + result_gemini3flashpreview = VertexGeminiConfig._map_thinking_param( + thinking_param=thinking_param, + model="gemini-3-flash-preview", + ) + + assert "thinkingLevel" in result_gemini3flashpreview, "Should have thinkingLevel for gemini-3-flash-preview" + assert "thinkingBudget" not in result_gemini3flashpreview, "Should NOT have thinkingBudget for gemini-3-flash-preview" + assert result_gemini3flashpreview["includeThoughts"] is True + + +def test_anthropic_thinking_param_to_gemini_2_thinkingBudget(): + """ + Test that Anthropic thinking parameters are correctly transformed to Gemini 2 thinkingBudget + (not thinkingLevel). + + For Gemini 2.x models (gemini-2.5-flash, gemini-2.0-flash): + - Should continue using thinkingBudget + - thinkingLevel should NOT be used + + Related issue: https://github.com/BerriAI/litellm/issues/XXXX + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + from litellm.types.llms.anthropic import AnthropicThinkingParam + + # Test 1: Anthropic thinking enabled with budget_tokens for Gemini 2 model + thinking_param: AnthropicThinkingParam = { + "type": "enabled", + "budget_tokens": 10000, + } + + result = VertexGeminiConfig._map_thinking_param( + thinking_param=thinking_param, + model="gemini-2.5-flash", + ) + + # For Gemini 2, should use thinkingBudget, not thinkingLevel + assert "thinkingBudget" in result, "Should have thinkingBudget for Gemini 2" + assert "thinkingLevel" not in result, "Should NOT have thinkingLevel for Gemini 2" + assert result["includeThoughts"] is True + assert result["thinkingBudget"] == 10000 + + # Test 2: Anthropic thinking enabled for gemini-2.0-flash model + result_gemini2 = VertexGeminiConfig._map_thinking_param( + thinking_param=thinking_param, + model="gemini-2.0-flash-thinking-exp-01-21", + ) + + assert "thinkingBudget" in result_gemini2, "Should have thinkingBudget for Gemini 2" + assert "thinkingLevel" not in result_gemini2, "Should NOT have thinkingLevel for Gemini 2" + assert result_gemini2["includeThoughts"] is True + assert result_gemini2["thinkingBudget"] == 10000 + + +def test_anthropic_thinking_param_via_map_openai_params(): + """ + Test that the thinking parameter is correctly transformed through the full map_openai_params flow + for Gemini 3 models, resulting in thinkingConfig with thinkingLevel. + + This tests the full integration from Anthropic API format to Gemini format. + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + from litellm.types.llms.anthropic import AnthropicThinkingParam + + config = VertexGeminiConfig() + + # Test with Gemini 3 model + non_default_params = { + "thinking": { + "type": "enabled", + "budget_tokens": 10000, + } + } + optional_params: dict = {} + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gemini-3-flash", + drop_params=False, + ) + + # Check that thinkingConfig was created with thinkingLevel + assert "thinkingConfig" in result, "Should have thinkingConfig in optional_params" + thinking_config = result["thinkingConfig"] + assert "thinkingLevel" in thinking_config, "Should have thinkingLevel for Gemini 3" + assert "thinkingBudget" not in thinking_config, "Should NOT have thinkingBudget for Gemini 3" + assert thinking_config["includeThoughts"] is True + + # Test with Gemini 2 model + optional_params_2 = {} + result_2 = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params_2, + model="gemini-2.5-flash", + drop_params=False, + ) + + # Check that thinkingConfig was created with thinkingBudget + assert "thinkingConfig" in result_2, "Should have thinkingConfig in optional_params" + thinking_config_2 = result_2["thinkingConfig"] + assert "thinkingBudget" in thinking_config_2, "Should have thinkingBudget for Gemini 2" + assert "thinkingLevel" not in thinking_config_2, "Should NOT have thinkingLevel for Gemini 2" + assert thinking_config_2["includeThoughts"] is True + assert thinking_config_2["thinkingBudget"] == 10000 diff --git a/tests/local_testing/test_auth_utils.py b/tests/local_testing/test_auth_utils.py index 11261592c32..72f799a6cf0 100644 --- a/tests/local_testing/test_auth_utils.py +++ b/tests/local_testing/test_auth_utils.py @@ -311,3 +311,56 @@ def test_get_internal_user_header_from_mapping_no_internal_returns_none(): single_mapping = {"header_name": "X-Only-Customer", "litellm_user_role": "customer"} result = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(single_mapping) assert result is None + + +@pytest.mark.parametrize( + "request_data, route, expected_model", + [ + # Vertex AI passthrough URL patterns + ( + {}, + "/vertex_ai/v1/projects/my-project/locations/us-central1/publishers/google/models/gemini-1.5-pro:generateContent", + "gemini-1.5-pro" + ), + ( + {}, + "/vertex_ai/v1beta1/projects/my-project/locations/us-central1/publishers/google/models/gemini-1.0-pro:streamGenerateContent", + "gemini-1.0-pro" + ), + ( + {}, + "/vertex_ai/v1/projects/my-project/locations/asia-southeast1/publishers/google/models/gemini-2.0-flash:generateContent", + "gemini-2.0-flash" + ), + # Model without method suffix (no colon) - should still extract + ( + {}, + "/vertex_ai/v1/projects/my-project/locations/us-central1/publishers/google/models/gemini-pro", + "gemini-pro" # Should match even without colon + ), + # Request body model takes precedence over URL + ( + {"model": "gpt-4o"}, + "/vertex_ai/v1/projects/my-project/locations/us-central1/publishers/google/models/gemini-1.5-pro:generateContent", + "gpt-4o" + ), + # Non-vertex route should not extract from vertex pattern + ( + {}, + "/openai/v1/chat/completions", + None + ), + # Azure deployment pattern should still work + ( + {}, + "/openai/deployments/my-deployment/chat/completions", + "my-deployment" + ), + ], +) +def test_get_model_from_request_vertex_ai_passthrough(request_data, route, expected_model): + """Test that get_model_from_request correctly extracts Vertex AI model from URL""" + from litellm.proxy.auth.auth_utils import get_model_from_request + + model = get_model_from_request(request_data, route) + assert model == expected_model diff --git a/tests/local_testing/test_custom_llm.py b/tests/local_testing/test_custom_llm.py index e61ede755e6..d0f32926551 100644 --- a/tests/local_testing/test_custom_llm.py +++ b/tests/local_testing/test_custom_llm.py @@ -309,6 +309,44 @@ class MyCustomLLM(CustomLLM): return model_response + def image_edit( + self, + model: str, + image: Any, + prompt: str, + model_response: ImageResponse, + api_key: Optional[str], + api_base: Optional[str], + optional_params: dict, + logging_obj: Any, + timeout=None, + client: Optional[HTTPHandler] = None, + ) -> ImageResponse: + return ImageResponse( + created=int(time.time()), + data=[ImageObject(url="https://example.com/edited-image.png")], + response_ms=1000, + ) + + async def aimage_edit( + self, + model: str, + image: Any, + prompt: str, + model_response: ImageResponse, + api_key: Optional[str], + api_base: Optional[str], + optional_params: dict, + logging_obj: Any, + timeout=None, + client: Optional[AsyncHTTPHandler] = None, + ) -> ImageResponse: + return ImageResponse( + created=int(time.time()), + data=[ImageObject(url="https://example.com/edited-image.png")], + response_ms=1000, + ) + def test_get_llm_provider(): """""" @@ -451,6 +489,69 @@ async def test_image_generation_async_additional_params(): } +def test_simple_image_edit(): + """Test sync image_edit with custom handler""" + my_custom_llm = MyCustomLLM() + litellm.custom_provider_map = [ + {"provider": "custom_llm", "custom_handler": my_custom_llm} + ] + resp = litellm.image_edit( + model="custom_llm/my-fake-model", + image=b"fake_image_bytes", + prompt="Edit this image", + ) + + print(resp) + assert resp.data[0].url == "https://example.com/edited-image.png" + + +@pytest.mark.asyncio +async def test_simple_image_edit_async(): + """Test async image_edit with custom handler""" + my_custom_llm = MyCustomLLM() + litellm.custom_provider_map = [ + {"provider": "custom_llm", "custom_handler": my_custom_llm} + ] + resp = await litellm.aimage_edit( + model="custom_llm/my-fake-model", + image=b"fake_image_bytes", + prompt="Edit this image", + ) + + print(resp) + assert resp.data[0].url == "https://example.com/edited-image.png" + + +@pytest.mark.asyncio +async def test_image_edit_async_additional_params(): + """Test that additional params are passed to custom handler""" + my_custom_llm = MyCustomLLM() + litellm.custom_provider_map = [ + {"provider": "custom_llm", "custom_handler": my_custom_llm} + ] + + with patch.object( + my_custom_llm, "aimage_edit", new=AsyncMock(return_value=ImageResponse( + created=int(time.time()), + data=[ImageObject(url="https://example.com/edited-image.png")], + )) + ) as mock_client: + resp = await litellm.aimage_edit( + model="custom_llm/my-fake-model", + image=b"fake_image_bytes", + prompt="Edit this image", + api_key="my-api-key", + api_base="my-api-base", + my_custom_param="my-custom-param", + ) + + print(resp) + + mock_client.assert_awaited_once() + assert mock_client.call_args.kwargs["api_key"] == "my-api-key" + assert mock_client.call_args.kwargs["api_base"] == "my-api-base" + + def test_get_supported_openai_params(): class MyCustomLLM(CustomLLM): diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index 13ff81bc695..4855932ca9f 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -1308,3 +1308,110 @@ def test_jina_ai_img_embeddings(input_data, expected_payload_input): # Assert that the 'input' field in the payload matches our expectation. assert "input" in sent_data assert sent_data["input"] == expected_payload_input + + +def test_encoding_format_none_not_omitted_from_openai_sdk(): + """ + Test that encoding_format=None is explicitly sent to OpenAI SDK. + + This test verifies that when encoding_format is not provided by the user, + liteLLM explicitly sets it to None rather than omitting it. This prevents + the OpenAI SDK from adding its default value of 'base64'. + + Without this fix: + - OpenAI SDK adds encoding_format='base64' as default when parameter is missing + - This causes issues with providers that don't support encoding_format (like Gemini) + + With this fix: + - encoding_format=None is explicitly passed + - OpenAI SDK respects the explicit None and doesn't add defaults + """ + with patch("litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client") as mock_get_client: + # Create a mock client instance + mock_client_instance = MagicMock() + mock_get_client.return_value = mock_client_instance + + # Mock the embeddings.with_raw_response.create method + mock_response = MagicMock() + mock_response.parse.return_value = MagicMock( + model_dump=lambda: { + 'data': [{'embedding': [0.1, 0.2, 0.3], 'index': 0}], + 'model': 'text-embedding-ada-002', + 'object': 'list', + 'usage': {'prompt_tokens': 1, 'total_tokens': 1} + } + ) + mock_response.headers = {} + + mock_client_instance.embeddings.with_raw_response.create.return_value = mock_response + + # Call the embedding function without encoding_format + response = embedding( + model="text-embedding-ada-002", + input="Hello world", + ) + + # Get the call arguments to verify what was sent to OpenAI SDK + call_args = mock_client_instance.embeddings.with_raw_response.create.call_args + assert call_args is not None, "OpenAI SDK embeddings.create should have been called" + + call_kwargs = call_args[1] # Get kwargs + + # The key assertion: encoding_format should be in the request with value None + # This prevents OpenAI SDK from adding its default 'base64' value + assert 'encoding_format' in call_kwargs, ( + "encoding_format should be explicitly passed to OpenAI SDK " + "(even if None) to prevent SDK from adding default value" + ) + assert call_kwargs['encoding_format'] is None, ( + "encoding_format should be None when not provided by user" + ) + + print("✅ PASS: encoding_format=None is correctly passed to OpenAI SDK") + + +def test_encoding_format_explicit_value_preserved(): + """ + Test that explicitly provided encoding_format values are preserved. + + When user provides encoding_format='float' or 'base64', it should be + sent as-is to the OpenAI SDK. + """ + with patch("litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client") as mock_get_client: + # Create a mock client instance + mock_client_instance = MagicMock() + mock_get_client.return_value = mock_client_instance + + # Mock the embeddings.with_raw_response.create method + mock_response = MagicMock() + mock_response.parse.return_value = MagicMock( + model_dump=lambda: { + 'data': [{'embedding': [0.1, 0.2, 0.3], 'index': 0}], + 'model': 'text-embedding-ada-002', + 'object': 'list', + 'usage': {'prompt_tokens': 1, 'total_tokens': 1} + } + ) + mock_response.headers = {} + + mock_client_instance.embeddings.with_raw_response.create.return_value = mock_response + + # Test with explicit encoding_format='float' + response = embedding( + model="text-embedding-ada-002", + input="Hello world", + encoding_format="float" + ) + + # Verify the encoding_format was passed correctly + call_args = mock_client_instance.embeddings.with_raw_response.create.call_args + call_kwargs = call_args[1] + + assert 'encoding_format' in call_kwargs, ( + "encoding_format should be in the request" + ) + assert call_kwargs['encoding_format'] == 'float', ( + "encoding_format should be 'float' when explicitly provided" + ) + + print("✅ PASS: encoding_format='float' is correctly preserved") diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index 4172659e659..d45110b3277 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -39,6 +39,7 @@ ignored_keys = [ "metadata.model_map_information", "metadata.usage_object", "metadata.cold_storage_object_key", + "metadata.litellm_overhead_time_ms", ] diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index d7e338d657b..8d3b0fee48f 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2671,3 +2671,61 @@ async def test_update_config_success_callback_normalization(): assert "sQs" not in callbacks # Existing callback should still be present assert "langfuse" in callbacks + + +@pytest.mark.parametrize( + "data", + [ + { + "model": { + "model_name": "azure/gpt-4.1-mini", + "litellm_params": {"model": "azure/gpt-4.1-mini"}, + "model_info": {"base_model": "gpt-4.1-mini"}, + }, + "expected": "gpt-4.1-mini", + }, + { + "model": { + "model_name": "openai/gpt-4.1-mini", + "litellm_params": {"model": "openai/gpt-4.1-mini"}, + }, + "expected": "openai/gpt-4.1-mini", + }, + { + "model": { + "model_name": "openai/gpt-4.1-mini", + "litellm_params": {"model": "openai/gpt-4.1-mini"}, + "model_info": {"base_model": "gpt-4.1-mini"}, + }, + "expected": "gpt-4.1-mini", + }, + { + "model": { + "model_name": "claude-sonnet-4-5-20250929", + "litellm_params": {"model": "anthropic/claude-sonnet-4-5@20250929"}, + "model_info": {"base_model": "anthropic/claude-sonnet-4-5-20250929"}, + }, + "expected": "anthropic/claude-sonnet-4-5-20250929", + }, + { + "model": { + "model_name": "gemini-2.5-flash-001", + "litellm_params": {"model": "gemini/gemini-2.5-flash@001"}, + "model_info": {"base_model": "gemini-2.5-flash-001"}, + }, + "expected": "gemini-2.5-flash-001", + }, + ], +) +def test_get_litellm_model_info(data): + from litellm.proxy.proxy_server import get_litellm_model_info + + model = data["model"] + get_info_mock = MagicMock() + + with mock.patch( + "litellm.get_model_info", + new=get_info_mock, + ): + get_litellm_model_info(model=model) + get_info_mock.assert_called_once_with(data["expected"]) diff --git a/tests/test_litellm/google_genai/test_google_genai_transformation.py b/tests/test_litellm/google_genai/test_google_genai_transformation.py new file mode 100644 index 00000000000..c953a504a38 --- /dev/null +++ b/tests/test_litellm/google_genai/test_google_genai_transformation.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +""" +Test to verify the Google GenAI transformation logic for generateContent parameters +""" +import os +import sys + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +import pytest + +from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) + + +def test_map_generate_content_optional_params_response_json_schema_camelcase(): + """Test that responseJsonSchema (camelCase) is passed through correctly""" + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "responseJsonSchema": { + "type": "object", + "properties": { + "recipe_name": {"type": "string"} + } + }, + "temperature": 1.0 + } + + result = config.map_generate_content_optional_params( + generate_content_config_dict=generate_content_config_dict, + model="gemini/gemini-3-flash-preview" + ) + + # responseJsonSchema should be in the result (camelCase format for Google GenAI API) + assert "responseJsonSchema" in result + assert result["responseJsonSchema"] == generate_content_config_dict["responseJsonSchema"] + assert "temperature" in result + assert result["temperature"] == 1.0 + + +def test_map_generate_content_optional_params_response_schema_snakecase(): + """Test that response_schema (snake_case) is converted to responseJsonSchema (camelCase)""" + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "response_json_schema": { + "type": "object", + "properties": { + "recipe_name": {"type": "string"} + } + }, + "temperature": 1.0 + } + + result = config.map_generate_content_optional_params( + generate_content_config_dict=generate_content_config_dict, + model="gemini/gemini-3-flash-preview" + ) + + # response_schema should be converted to responseJsonSchema (camelCase) + assert "responseJsonSchema" in result + assert result["responseJsonSchema"] == generate_content_config_dict["response_json_schema"] + assert "temperature" in result + + +def test_map_generate_content_optional_params_thinking_config_camelcase(): + """Test that thinkingConfig (camelCase) is passed through correctly""" + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "thinkingConfig": { + "thinkingLevel": "minimal", + "includeThoughts": True + }, + "temperature": 1.0 + } + + result = config.map_generate_content_optional_params( + generate_content_config_dict=generate_content_config_dict, + model="gemini/gemini-3-flash-preview" + ) + + # thinkingConfig should be in the result (camelCase format for Google GenAI API) + assert "thinkingConfig" in result + assert result["thinkingConfig"]["thinkingLevel"] == "minimal" + assert result["thinkingConfig"]["includeThoughts"] is True + assert "temperature" in result + + +def test_map_generate_content_optional_params_thinking_config_snakecase(): + """Test that thinking_config (snake_case) is converted to thinkingConfig (camelCase)""" + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "thinking_config": { + "thinkingLevel": "medium", + "includeThoughts": True + }, + "temperature": 1.0 + } + + result = config.map_generate_content_optional_params( + generate_content_config_dict=generate_content_config_dict, + model="gemini/gemini-3-flash-preview" + ) + + # thinking_config should be converted to thinkingConfig (camelCase) + assert "thinkingConfig" in result + assert result["thinkingConfig"]["thinkingLevel"] == "medium" + assert result["thinkingConfig"]["includeThoughts"] is True + assert "thinking_config" not in result # Should not be in snake_case format + assert "temperature" in result + + +def test_map_generate_content_optional_params_mixed_formats(): + """Test that both camelCase and snake_case parameters work together""" + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "responseJsonSchema": { + "type": "object", + "properties": { + "recipe_name": {"type": "string"} + } + }, + "thinking_config": { + "thinkingLevel": "low", + "includeThoughts": True + }, + "temperature": 1.0, + "max_output_tokens": 100 + } + + result = config.map_generate_content_optional_params( + generate_content_config_dict=generate_content_config_dict, + model="gemini/gemini-3-flash-preview" + ) + + # All parameters should be converted to camelCase + assert "responseJsonSchema" in result + assert "thinkingConfig" in result + assert result["thinkingConfig"]["thinkingLevel"] == "low" + assert "temperature" in result + assert "maxOutputTokens" in result # This one stays as-is if it's in supported list + + +def test_map_generate_content_optional_params_response_mime_type(): + """Test that responseMimeType is handled correctly""" + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "responseMimeType": "application/json", + "responseJsonSchema": { + "type": "object", + "properties": { + "recipe_name": {"type": "string"} + } + } + } + + result = config.map_generate_content_optional_params( + generate_content_config_dict=generate_content_config_dict, + model="gemini/gemini-3-flash-preview" + ) + + # responseMimeType should be passed through (it's already camelCase) + assert "responseMimeType" in result or "response_mime_type" in result + assert "responseJsonSchema" in result + + +def test_responses_api_reasoning_dict_format(): + """Test that reasoning parameter with dict format is mapped to reasoning_effort""" + from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams + + responses_api_request: ResponsesAPIOptionalRequestParams = { + "reasoning": {"effort": "high"}, + "temperature": 1.0, + } + + result = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini/2.5-pro", + input="Hello, what is the capital of France?", + responses_api_request=responses_api_request, + ) + + # reasoning_effort should be extracted from reasoning dict + assert "reasoning_effort" in result + assert result["reasoning_effort"] == "high" + + +def test_responses_api_reasoning_string_format(): + """Test that reasoning parameter with string format is mapped to reasoning_effort""" + from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams + + responses_api_request: ResponsesAPIOptionalRequestParams = { + "reasoning": "medium", # Could be a string directly + "temperature": 1.0, + } + + result = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini/2.5-pro", + input="Hello, what is the capital of France?", + responses_api_request=responses_api_request, + ) + + # reasoning_effort should be extracted from reasoning string + assert "reasoning_effort" in result + assert result["reasoning_effort"] == "medium" + + +def test_responses_api_reasoning_low_effort(): + """Test that low reasoning effort is correctly mapped""" + from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams + + responses_api_request: ResponsesAPIOptionalRequestParams = { + "reasoning": {"effort": "low"}, + } + + result = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini/2.5-pro", + input="Test", + responses_api_request=responses_api_request, + ) + + assert "reasoning_effort" in result + assert result["reasoning_effort"] == "low" + + +def test_responses_api_no_reasoning(): + """Test that no reasoning_effort is included when reasoning is not provided""" + from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams + + responses_api_request: ResponsesAPIOptionalRequestParams = { + "temperature": 1.0, + } + + result = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini/2.5-pro", + input="Test", + responses_api_request=responses_api_request, + ) + + # reasoning_effort should not be in result if not provided (filtered out as None) + assert "reasoning_effort" not in result or result.get("reasoning_effort") is None diff --git a/tests/test_litellm/images/test_image_edit_utils.py b/tests/test_litellm/images/test_image_edit_utils.py new file mode 100644 index 00000000000..56d8e48405b --- /dev/null +++ b/tests/test_litellm/images/test_image_edit_utils.py @@ -0,0 +1,170 @@ +from typing import Any, Dict, List +from unittest.mock import MagicMock, patch + +import pytest + +import litellm +from litellm.images.utils import ImageEditRequestUtils +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.types.images.main import ImageEditOptionalRequestParams + + +class MockImageEditConfig(BaseImageEditConfig): + def get_supported_openai_params(self, model: str) -> List[str]: + return ["size", "quality"] + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + return dict(image_edit_optional_params) + + def get_complete_url( + self, model: str, api_base: str, litellm_params: dict + ) -> str: + return "https://example.com/api" + + def validate_environment( + self, headers: dict, model: str, api_key: str = None + ) -> dict: + return headers + + def transform_image_edit_request(self, *args, **kwargs): + return {}, [] + + def transform_image_edit_response(self, *args, **kwargs): + return MagicMock() + + +class TestImageEditRequestUtilsDropParams: + def setup_method(self): + self.config = MockImageEditConfig() + self.model = "test-model" + self._original_drop_params = getattr(litellm, "drop_params", None) + + def teardown_method(self): + if self._original_drop_params is None: + if hasattr(litellm, "drop_params"): + delattr(litellm, "drop_params") + else: + litellm.drop_params = self._original_drop_params + + def test_unsupported_params_raises_without_drop(self): + litellm.drop_params = False + optional_params: ImageEditOptionalRequestParams = { + "size": "1024x1024", + "unsupported_param": "value", + } + + with pytest.raises(litellm.UnsupportedParamsError) as exc_info: + ImageEditRequestUtils.get_optional_params_image_edit( + model=self.model, + image_edit_provider_config=self.config, + image_edit_optional_params=optional_params, + ) + + assert "unsupported_param" in str(exc_info.value) + + def test_drop_params_global_setting(self): + litellm.drop_params = True + optional_params: ImageEditOptionalRequestParams = { + "size": "1024x1024", + "unsupported_param": "value", + } + + result = ImageEditRequestUtils.get_optional_params_image_edit( + model=self.model, + image_edit_provider_config=self.config, + image_edit_optional_params=optional_params, + ) + + assert "size" in result + assert "unsupported_param" not in result + + def test_drop_params_explicit_parameter(self): + litellm.drop_params = False + optional_params: ImageEditOptionalRequestParams = { + "size": "1024x1024", + "unsupported_param": "value", + } + + result = ImageEditRequestUtils.get_optional_params_image_edit( + model=self.model, + image_edit_provider_config=self.config, + image_edit_optional_params=optional_params, + drop_params=True, + ) + + assert "size" in result + assert "unsupported_param" not in result + + def test_additional_drop_params(self): + litellm.drop_params = False + optional_params: ImageEditOptionalRequestParams = { + "size": "1024x1024", + "quality": "high", + } + + result = ImageEditRequestUtils.get_optional_params_image_edit( + model=self.model, + image_edit_provider_config=self.config, + image_edit_optional_params=optional_params, + additional_drop_params=["quality"], + ) + + assert "size" in result + assert "quality" not in result + + def test_drop_params_false_with_global_true(self): + litellm.drop_params = True + optional_params: ImageEditOptionalRequestParams = { + "size": "1024x1024", + "unsupported_param": "value", + } + + result = ImageEditRequestUtils.get_optional_params_image_edit( + model=self.model, + image_edit_provider_config=self.config, + image_edit_optional_params=optional_params, + drop_params=False, + ) + + assert "size" in result + assert "unsupported_param" not in result + + def test_supported_params_pass_through(self): + litellm.drop_params = False + optional_params: ImageEditOptionalRequestParams = { + "size": "1024x1024", + "quality": "high", + } + + result = ImageEditRequestUtils.get_optional_params_image_edit( + model=self.model, + image_edit_provider_config=self.config, + image_edit_optional_params=optional_params, + ) + + assert result["size"] == "1024x1024" + assert result["quality"] == "high" + + def test_additional_drop_params_with_unsupported_and_drop_true(self): + litellm.drop_params = True + optional_params: ImageEditOptionalRequestParams = { + "size": "1024x1024", + "quality": "high", + "unsupported_param": "value", + } + + result = ImageEditRequestUtils.get_optional_params_image_edit( + model=self.model, + image_edit_provider_config=self.config, + image_edit_optional_params=optional_params, + additional_drop_params=["quality"], + ) + + assert "size" in result + assert "quality" not in result + assert "unsupported_param" not in result diff --git a/tests/test_litellm/interactions/test_google_interactions_integration.py b/tests/test_litellm/interactions/test_google_interactions_integration.py new file mode 100644 index 00000000000..a2b255f315d --- /dev/null +++ b/tests/test_litellm/interactions/test_google_interactions_integration.py @@ -0,0 +1,338 @@ +""" +Integration tests for Google Interactions API. + +Tests the litellm.interactions.create() and related methods against the Google AI Studio API. + +Per OpenAPI spec: https://ai.google.dev/static/api/interactions.openapi.json + +Run with: pytest tests/test_litellm/interactions/test_google_interactions_integration.py -v +""" + +import asyncio +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +import litellm +import litellm.interactions as interactions + +# Test API key - should be set in environment +GEMINI_API_KEY = os.getenv("GEMINI_API_KEY") + + +@pytest.fixture +def api_key(): + """Fixture to provide the API key.""" + if not GEMINI_API_KEY: + pytest.skip("GEMINI_API_KEY not set") + return GEMINI_API_KEY + + +class TestGoogleInteractionsCreate: + """Tests for creating interactions via litellm.interactions.create().""" + + def test_create_simple_string_input(self, api_key): + """Test creating an interaction with a simple string input.""" + response = interactions.create( + model="gemini/gemini-2.5-flash", + input="Hello, what is 2 + 2?", + api_key=api_key, + ) + print("SIMPLE RESPONSE: ", response) + assert response is not None + assert response.id is not None or response.status is not None + + # Check outputs per OpenAPI spec + if response.outputs: + assert len(response.outputs) > 0 + print(f"Response outputs: {response.outputs}") + + # Check usage per OpenAPI spec + if response.usage: + print(f"Usage: {response.usage}") + + def test_create_with_content_list(self, api_key): + """Test creating an interaction with a structured content list (Turn format).""" + response = interactions.create( + model="gemini/gemini-2.5-flash", + input=[ + { + "role": "user", + "content": [{"type": "text", "text": "What is the capital of France?"}] + } + ], + api_key=api_key, + ) + + assert response is not None + print(f"Response: {response}") + + def test_create_with_system_instruction(self, api_key): + """Test creating an interaction with system_instruction (per OpenAPI spec).""" + response = interactions.create( + model="gemini/gemini-2.5-flash", + input="What are you?", + system_instruction="You are a helpful pirate assistant. Always respond like a pirate.", + api_key=api_key, + ) + + assert response is not None + print(f"Response with system_instruction: {response}") + + def test_create_with_tools(self, api_key): + """Test creating an interaction with tools (per OpenAPI spec).""" + response = interactions.create( + model="gemini/gemini-2.5-flash", + input="What's the weather in Boston?", + tools=[ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name"} + }, + "required": ["location"] + } + } + ], + api_key=api_key, + ) + + assert response is not None + # Check if status is requires_action (function call) + print(f"Response status: {response.status}") + print(f"Response outputs: {response.outputs}") + + @pytest.mark.asyncio + async def test_acreate_simple(self, api_key): + """Test async interaction creation.""" + response = await interactions.acreate( + model="gemini/gemini-2.5-flash", + input="What is the speed of light?", + api_key=api_key, + ) + + assert response is not None + print(f"Async response: {response}") + + +class TestGoogleInteractionsStreaming: + """Tests for streaming interactions.""" + + def test_create_streaming(self, api_key): + """Test creating a streaming interaction.""" + response_stream = interactions.create( + model="gemini/gemini-2.5-flash", + input="Count from 1 to 5 slowly.", + stream=True, + api_key=api_key, + ) + + # Collect all chunks + chunks = [] + for chunk in response_stream: + chunks.append(chunk) + print(f"Streaming chunk: {chunk}") + + assert len(chunks) > 0 + print(f"Total chunks received: {len(chunks)}") + + @pytest.mark.asyncio + async def test_acreate_streaming(self, api_key): + """Test async streaming interaction.""" + response_stream = await interactions.acreate( + model="gemini/gemini-2.5-flash", + input="Count from 1 to 3.", + stream=True, + api_key=api_key, + ) + + # Collect all chunks + chunks = [] + async for chunk in response_stream: + chunks.append(chunk) + print(f"Async streaming chunk: {chunk}") + + assert len(chunks) > 0 + print(f"Total async chunks received: {len(chunks)}") + + +class TestGoogleInteractionsMultiTurn: + """Tests for multi-turn conversations using Turn[] input.""" + + def test_multi_turn_conversation(self, api_key): + """Test a multi-turn conversation per OpenAPI spec (Turn[] format).""" + response = interactions.create( + model="gemini/gemini-2.5-flash", + input=[ + { + "role": "user", + "content": [{"type": "text", "text": "My name is Alice."}] + }, + { + "role": "model", + "content": [{"type": "text", "text": "Hello Alice! Nice to meet you."}] + }, + { + "role": "user", + "content": [{"type": "text", "text": "What is my name?"}] + } + ], + api_key=api_key, + ) + + assert response is not None + print(f"Multi-turn response: {response}") + + +class TestGoogleInteractionsAgent: + """Tests for agent interactions (per OpenAPI spec).""" + + @pytest.mark.skip(reason="Deep research agent may not be available in all accounts") + def test_create_agent_interaction(self, api_key): + """Test creating an agent interaction per OpenAPI spec.""" + response = interactions.create( + agent="deep-research-pro-preview-12-2025", + input="Research the current state of quantum computing", + api_key=api_key, + ) + + assert response is not None + print(f"Agent response: {response}") + + +class TestGoogleInteractionsGetDelete: + """Tests for get and delete operations.""" + + @pytest.mark.skip(reason="Get/Delete require valid interaction IDs from previous calls") + def test_get_interaction(self, api_key): + """Test getting an interaction by ID.""" + # First create an interaction + create_response = interactions.create( + model="gemini/gemini-2.5-flash", + input="Hello", + api_key=api_key, + ) + + if create_response.id: + # Then get it + get_response = interactions.get( + interaction_id=create_response.id, + api_key=api_key, + ) + assert get_response is not None + print(f"Get response: {get_response}") + + @pytest.mark.skip(reason="Get/Delete require valid interaction IDs from previous calls") + def test_delete_interaction(self, api_key): + """Test deleting an interaction by ID.""" + # First create an interaction + create_response = interactions.create( + model="gemini/gemini-2.5-flash", + input="Hello", + api_key=api_key, + ) + + if create_response.id: + # Then delete it + delete_result = interactions.delete( + interaction_id=create_response.id, + api_key=api_key, + ) + assert delete_result.success is True + print(f"Delete result: {delete_result}") + + +class TestGoogleInteractionsErrorHandling: + """Tests for error handling.""" + + def test_invalid_model(self, api_key): + """Test error handling for invalid model.""" + with pytest.raises(Exception): + interactions.create( + model="gemini/invalid-model-name-xyz", + input="Hello", + api_key=api_key, + ) + + def test_missing_model_and_agent(self, api_key): + """Test error when neither model nor agent is provided.""" + with pytest.raises(Exception): # Can be ValueError or APIConnectionError + interactions.create( + input="Hello", + api_key=api_key, + ) + + +class TestGoogleInteractionsResponseStructure: + """Tests to verify the response structure matches OpenAPI spec.""" + + def test_response_has_expected_fields(self, api_key): + """Test that the response has fields per OpenAPI spec.""" + response = interactions.create( + model="gemini/gemini-2.5-flash", + input="Hello", + api_key=api_key, + ) + + # Check fields per OpenAPI spec + assert hasattr(response, 'id') + assert hasattr(response, 'object') + assert hasattr(response, 'status') + assert hasattr(response, 'outputs') + assert hasattr(response, 'usage') + assert hasattr(response, 'model') or hasattr(response, 'agent') + assert hasattr(response, 'role') + assert hasattr(response, 'created') + assert hasattr(response, 'updated') + + print(f"Response structure: id={response.id}, status={response.status}, object={response.object}") + + +if __name__ == "__main__": + # Run a quick smoke test + print("Running Google Interactions API smoke test...") + + api_key = GEMINI_API_KEY + if not api_key: + print("GEMINI_API_KEY not set, skipping smoke test") + exit(1) + + print("\n1. Testing basic interaction...") + response = interactions.create( + model="gemini/gemini-2.5-flash", + input="What is 2 + 2?", + api_key=api_key, + ) + print(f"Response: {response}") + + print("\n2. Testing streaming interaction...") + stream = interactions.create( + model="gemini/gemini-2.5-flash", + input="Count to 3.", + stream=True, + api_key=api_key, + ) + print("Streaming response chunks:") + for chunk in stream: + print(f" {chunk}") + + print("\n3. Testing async interaction...") + async def test_async(): + response = await interactions.acreate( + model="gemini/gemini-2.5-flash", + input="Say hello!", + api_key=api_key, + ) + return response + + async_response = asyncio.run(test_async()) + print(f"Async response: {async_response}") + + print("\nSmoke test complete!") diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py new file mode 100644 index 00000000000..d18d52f96be --- /dev/null +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -0,0 +1,237 @@ +""" +OpenAPI compliance tests for Google Interactions API. + +Validates that our SDK requests/responses match the OpenAPI spec at: +https://ai.google.dev/static/api/interactions.openapi.json + +Run with: pytest tests/test_litellm/interactions/test_openapi_compliance.py -v +""" + +import json +import os +from typing import Any, Dict +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from openapi_core import OpenAPI +from openapi_core.testing.mock import MockRequest, MockResponse + +OPENAPI_SPEC_URL = "https://ai.google.dev/static/api/interactions.openapi.json" + + +@pytest.fixture(scope="module") +def openapi_spec(): + """Load the OpenAPI spec.""" + response = httpx.get(OPENAPI_SPEC_URL) + response.raise_for_status() + spec_dict = response.json() + return OpenAPI.from_dict(spec_dict) + + +@pytest.fixture(scope="module") +def spec_dict(): + """Load raw spec dict for manual validation.""" + response = httpx.get(OPENAPI_SPEC_URL) + response.raise_for_status() + return response.json() + + +class TestRequestCompliance: + """Tests that our request bodies match the OpenAPI spec.""" + + def test_create_model_interaction_request_schema(self, spec_dict): + """Verify CreateModelInteractionParams schema fields.""" + schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] + + # Required fields per spec + assert "model" in schema["required"] + assert "input" in schema["required"] + + # Check our supported optional fields exist in spec + our_optional_fields = [ + "tools", "system_instruction", "generation_config", + "stream", "store", "background", "response_modalities", + "response_format", "response_mime_type", "previous_interaction_id" + ] + + spec_properties = schema["properties"] + for field in our_optional_fields: + assert field in spec_properties, f"Field '{field}' not in OpenAPI spec" + print(f"✓ Field '{field}' exists in spec") + + def test_input_types_match_spec(self, spec_dict): + """Verify input field supports string, Content, Content[], Turn[].""" + schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] + input_schema = schema["properties"]["input"] + + # Should be oneOf with multiple types + assert "oneOf" in input_schema + + input_types = [] + for option in input_schema["oneOf"]: + if option.get("type") == "string": + input_types.append("string") + elif option.get("type") == "array": + input_types.append("array") + elif "$ref" in option: + input_types.append(option["$ref"]) + + print(f"Input supports types: {input_types}") + assert "string" in input_types, "Input should support string" + assert "array" in input_types, "Input should support array" + + def test_content_schema_uses_discriminator(self, spec_dict): + """Verify Content uses type discriminator.""" + content_schema = spec_dict["components"]["schemas"]["Content"] + + assert "discriminator" in content_schema + assert content_schema["discriminator"]["propertyName"] == "type" + + # Check TextContent is an option + mapping = content_schema["discriminator"]["mapping"] + assert "text" in mapping + print(f"Content type discriminator mapping: {list(mapping.keys())}") + + def test_text_content_schema(self, spec_dict): + """Verify TextContent schema.""" + text_schema = spec_dict["components"]["schemas"]["TextContent"] + + assert "type" in text_schema["required"] + assert "text" in text_schema["properties"] + assert text_schema["properties"]["type"].get("const") == "text" + print("✓ TextContent schema is correct") + + def test_turn_schema(self, spec_dict): + """Verify Turn schema for multi-turn conversations.""" + turn_schema = spec_dict["components"]["schemas"]["Turn"] + + assert "role" in turn_schema["properties"] + assert "content" in turn_schema["properties"] + + # Content can be string or Content[] + content_prop = turn_schema["properties"]["content"] + assert "oneOf" in content_prop + print("✓ Turn schema supports role + content") + + +class TestResponseCompliance: + """Tests that our response types match the OpenAPI spec.""" + + def test_interaction_response_fields(self, spec_dict): + """Verify our InteractionsAPIResponse has correct fields.""" + # The response is the Interaction schema + # Check CreateModelInteractionParams which includes output fields + schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] + + # Output fields (readOnly) + output_fields = ["id", "status", "created", "updated", "role", "outputs", "usage"] + + for field in output_fields: + assert field in schema["properties"], f"Output field '{field}' not in spec" + print(f"✓ Output field '{field}' exists in spec") + + def test_status_enum_values(self, spec_dict): + """Verify status enum values match spec.""" + schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] + status_prop = schema["properties"]["status"] + + expected_statuses = ["UNSPECIFIED", "IN_PROGRESS", "REQUIRES_ACTION", "COMPLETED", "FAILED", "CANCELLED"] + assert status_prop["enum"] == expected_statuses + print(f"✓ Status enum values: {expected_statuses}") + + def test_usage_schema(self, spec_dict): + """Verify Usage schema fields.""" + usage_schema = spec_dict["components"]["schemas"]["Usage"] + + # Key usage fields + expected_fields = ["total_input_tokens", "total_output_tokens", "total_tokens"] + + for field in expected_fields: + assert field in usage_schema["properties"], f"Usage field '{field}' not in spec" + print(f"✓ Usage field '{field}' exists") + + +class TestToolsCompliance: + """Tests that our tool types match the OpenAPI spec.""" + + def test_tool_schema(self, spec_dict): + """Verify Tool schema.""" + tool_schema = spec_dict["components"]["schemas"]["Tool"] + + # Tool should be oneOf multiple tool types + assert "oneOf" in tool_schema or "properties" in tool_schema + print(f"✓ Tool schema found") + + def test_function_declaration_schema(self, spec_dict): + """Verify FunctionDeclaration schema for function tools.""" + if "FunctionDeclaration" in spec_dict["components"]["schemas"]: + func_schema = spec_dict["components"]["schemas"]["FunctionDeclaration"] + assert "name" in func_schema.get("properties", {}) or "name" in func_schema.get("required", []) + print("✓ FunctionDeclaration schema found") + else: + print("⚠ FunctionDeclaration schema not found (may be nested)") + + +class TestEndpointCompliance: + """Tests that our endpoints match the OpenAPI spec.""" + + def test_create_endpoint_exists(self, spec_dict): + """Verify POST /interactions endpoint exists.""" + paths = spec_dict["paths"] + + # Find the create interactions endpoint + create_path = None + for path, methods in paths.items(): + if "interactions" in path and "post" in methods: + create_path = path + break + + assert create_path is not None, "POST /interactions endpoint not found" + print(f"✓ Create endpoint: POST {create_path}") + + def test_get_endpoint_exists(self, spec_dict): + """Verify GET /interactions/{id} endpoint exists.""" + paths = spec_dict["paths"] + + get_path = None + for path, methods in paths.items(): + if "{id}" in path and "interactions" in path and "get" in methods: + get_path = path + break + + assert get_path is not None, "GET /interactions/{id} endpoint not found" + print(f"✓ Get endpoint: GET {get_path}") + + def test_delete_endpoint_exists(self, spec_dict): + """Verify DELETE /interactions/{id} endpoint exists.""" + paths = spec_dict["paths"] + + delete_path = None + for path, methods in paths.items(): + if "{id}" in path and "interactions" in path and "delete" in methods: + delete_path = path + break + + assert delete_path is not None, "DELETE /interactions/{id} endpoint not found" + print(f"✓ Delete endpoint: DELETE {delete_path}") + + +if __name__ == "__main__": + # Quick manual test + import httpx + + print("Loading OpenAPI spec...") + response = httpx.get(OPENAPI_SPEC_URL) + spec = response.json() + + print(f"\nSpec version: {spec.get('openapi')}") + print(f"API title: {spec.get('info', {}).get('title')}") + print(f"\nEndpoints:") + for path, methods in spec.get("paths", {}).items(): + for method in methods: + if method in ["get", "post", "delete", "put", "patch"]: + print(f" {method.upper()} {path}") + + print(f"\nSchemas: {list(spec.get('components', {}).get('schemas', {}).keys())[:10]}...") + diff --git a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py index 49be7f39a18..867ab675943 100644 --- a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py @@ -11,6 +11,7 @@ sys.path.insert( from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers +from litellm.main import ahealth_check from litellm.proxy._types import UserAPIKeyAuth @@ -78,4 +79,59 @@ def test_get_litellm_internal_health_check_user_api_key_auth(): assert result.api_key == LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME assert result.team_id == LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME assert result.key_alias == LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME - assert result.team_alias == LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME \ No newline at end of file + assert result.team_alias == LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME + + +@pytest.mark.asyncio +async def test_ahealth_check_failure_masks_raw_request_headers(): + """ + Security test: Verify that when ahealth_check() fails, the raw_request_headers + in raw_request_typed_dict are properly masked to prevent API key leaks. + + This tests the fix for the security vulnerability where Authorization headers + were being exposed in health check error responses. + """ + # Use a model configuration that will fail (invalid endpoint) + test_api_key = "dapi-test-key-1234567890abcdef" + test_headers = { + "Authorization": f"Bearer {test_api_key}", + "Content-Type": "application/json", + } + + response = await ahealth_check( + model_params={ + "model": "databricks/dbrx-instruct", + "api_base": "https://invalid-endpoint-that-will-fail.com/", + "api_key": test_api_key, + "headers": test_headers, + }, + mode="chat", + ) + + # Should have error and raw_request_typed_dict + assert "error" in response + assert "raw_request_typed_dict" in response + + raw_request_dict = response["raw_request_typed_dict"] + assert raw_request_dict is not None + assert isinstance(raw_request_dict, dict) + assert "raw_request_headers" in raw_request_dict + + headers = raw_request_dict["raw_request_headers"] + assert headers is not None + + # Security check: Authorization header should be masked, not show full key + if "Authorization" in headers: + auth_header = headers["Authorization"] + # Should be masked (e.g., "Be****90" or similar) + assert auth_header != f"Bearer {test_api_key}", "Authorization header must be masked" + assert auth_header != test_api_key, "API key must not appear in Authorization header" + # Masked headers typically have asterisks or are truncated + assert "*" in auth_header or len(auth_header) < len(f"Bearer {test_api_key}"), \ + f"Authorization header should be masked but got: {auth_header}" + + # Content-Type should remain unmasked (not sensitive) + if "Content-Type" in headers: + assert headers["Content-Type"] == "application/json" + + print(f"Masked Authorization header: {headers.get('Authorization', 'NOT FOUND')}") \ No newline at end of file diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index e96d6cc61a9..41febd4920a 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1,11 +1,11 @@ from unittest.mock import MagicMock +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.llms.anthropic.chat.handler import ModelResponseIterator from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ) -from litellm.constants import RESPONSE_FORMAT_TOOL_NAME def test_redacted_thinking_content_block_delta(): @@ -779,3 +779,195 @@ def test_web_search_tool_result_captured_in_provider_specific_fields(): assert ( web_search_results[0]["content"][0]["title"] == "Fun Otter Facts" ), "First result title should match" + + +def test_container_in_provider_specific_fields_streaming(): + """ + Test that container is captured in provider_specific_fields for streaming responses. + + When container with skills is used, the container field should be present in + the provider_specific_fields of the message_delta chunk. + """ + iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=False + ) + + # Simulate streaming chunks + chunks = [ + # 1. message_start + { + "type": "message_start", + "message": { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 98976, "output_tokens": 1}, + }, + }, + # 2. content_block_start for text + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "text", + "text": "", + }, + }, + # 3. content_block_delta with text + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Hello, this is a response"}, + }, + # 4. content_block_stop for text + {"type": "content_block_stop", "index": 0}, + # 5. message_delta with container - THIS IS WHAT WE'RE TESTING + { + "type": "message_delta", + "delta": { + "stop_reason": "end_turn", + "stop_sequence": None, + "container": { + "id": "container_011CW9hA9zpZ8xD3bjjShy4p", + "expires_at": "2025-12-16T04:57:16.913181Z", + "skills": [ + { + "type": "anthropic", + "skill_id": "pptx", + "version": "20251013", + } + ], + }, + }, + "usage": { + "input_tokens": 98976, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 931, + "server_tool_use": {"web_search_requests": 0}, + }, + }, + ] + + container_field = None + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + if ( + parsed.choices + and parsed.choices[0].delta.provider_specific_fields + and "container" in parsed.choices[0].delta.provider_specific_fields + ): + container_field = parsed.choices[0].delta.provider_specific_fields[ + "container" + ] + + # Verify container was captured + assert container_field is not None, "container should be captured in provider_specific_fields" + assert ( + container_field["id"] == "container_011CW9hA9zpZ8xD3bjjShy4p" + ), "container id should match" + assert ( + container_field["expires_at"] == "2025-12-16T04:57:16.913181Z" + ), "expires_at should match" + assert len(container_field["skills"]) == 1, "Should have 1 skill" + assert ( + container_field["skills"][0]["skill_id"] == "pptx" + ), "skill_id should be pptx" + assert ( + container_field["skills"][0]["version"] == "20251013" + ), "version should match" + + +def test_container_in_provider_specific_fields_non_streaming(): + """ + Test that container is captured in provider_specific_fields for non-streaming responses. + + When container with skills is used in non-streaming, the container field should be + present in the provider_specific_fields of the response. + """ + iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=False, json_mode=False + ) + + # Simulate a message_delta chunk with container (as it would appear in non-streaming) + message_delta_chunk = { + "type": "message_delta", + "delta": { + "stop_reason": "end_turn", + "stop_sequence": None, + "container": { + "id": "container_abc123xyz", + "expires_at": "2025-12-20T10:30:00.000000Z", + "skills": [ + { + "type": "anthropic", + "skill_id": "code_execution", + "version": "latest", + }, + { + "type": "anthropic", + "skill_id": "pptx", + "version": "20251013", + }, + ], + }, + }, + "usage": { + "input_tokens": 1000, + "output_tokens": 200, + }, + } + + model_response = iterator.chunk_parser(message_delta_chunk) + + # Verify container is in provider_specific_fields + assert model_response.choices[0].delta.provider_specific_fields is not None + assert "container" in model_response.choices[0].delta.provider_specific_fields + container_field = model_response.choices[0].delta.provider_specific_fields[ + "container" + ] + + assert container_field["id"] == "container_abc123xyz", "container id should match" + assert ( + container_field["expires_at"] == "2025-12-20T10:30:00.000000Z" + ), "expires_at should match" + assert len(container_field["skills"]) == 2, "Should have 2 skills" + assert ( + container_field["skills"][0]["skill_id"] == "code_execution" + ), "First skill_id should be code_execution" + assert ( + container_field["skills"][1]["skill_id"] == "pptx" + ), "Second skill_id should be pptx" + + +def test_container_absent_when_not_provided(): + """ + Test that container is not added to provider_specific_fields when not provided. + + This ensures we don't add empty or None container fields. + """ + iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=False, json_mode=False + ) + + # message_delta without container + message_delta_chunk = { + "type": "message_delta", + "delta": { + "stop_reason": "end_turn", + "stop_sequence": None, + }, + "usage": { + "input_tokens": 1000, + "output_tokens": 200, + }, + } + + model_response = iterator.chunk_parser(message_delta_chunk) + + # Verify container is NOT in provider_specific_fields when not provided + if model_response.choices[0].delta.provider_specific_fields: + assert ( + "container" not in model_response.choices[0].delta.provider_specific_fields + ), "container should not be present when not provided in delta" diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index ec612109d9c..9b6d1c6e178 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1651,15 +1651,16 @@ def test_get_max_tokens_for_model_claude_35(): def test_get_max_tokens_for_model_claude_37(): """ Test that get_max_tokens_for_model returns correct value for Claude 3.7 models. - Claude 3.7 Sonnet has max_output_tokens of 128000 (128K with extended thinking). + Claude 3.7 Sonnet has max_output_tokens of 64000 by default. + 128K output requires the beta header 'output-128k-2025-02-19'. Fixes: https://github.com/BerriAI/litellm/issues/8835 """ config = AnthropicConfig() - # Claude 3.7 Sonnet should return 128000 (128K) + # Claude 3.7 Sonnet should return 64000 (64K default, 128K requires beta header) max_tokens = config.get_max_tokens_for_model("claude-3-7-sonnet-20250219") - assert max_tokens == 128000 + assert max_tokens == 64000 def test_get_max_tokens_for_model_unknown(): @@ -1698,9 +1699,9 @@ def test_get_config_with_model_uses_dynamic_max_tokens(): config_claude35 = AnthropicConfig.get_config(model="claude-3-5-sonnet-20241022") assert config_claude35["max_tokens"] == 8192 - # Claude 3.7 model should get 128000 (128K with extended thinking) + # Claude 3.7 model should get 64000 (64K default, 128K requires beta header) config_claude37 = AnthropicConfig.get_config(model="claude-3-7-sonnet-20250219") - assert config_claude37["max_tokens"] == 128000 + assert config_claude37["max_tokens"] == 64000 def test_get_config_without_model_uses_fallback(): diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen2_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen2_transformation.py index 737e1279e65..eb963ec4263 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen2_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen2_transformation.py @@ -290,3 +290,72 @@ def test_qwen2_provider_detection(): assert config is not None assert isinstance(config, AmazonQwen2Config) + +def test_qwen2_model_id_extraction_with_arn(): + """Test that model ID is correctly extracted from bedrock/qwen2/arn... paths""" + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + # Test case: bedrock/qwen2/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-qwen2 + # The qwen2/ prefix should be stripped, leaving only the ARN for encoding + model = "qwen2/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-qwen2" + provider = "qwen2" + + result = BaseAWSLLM.get_bedrock_model_id( + optional_params={}, + provider=provider, + model=model + ) + + # The result should NOT contain "qwen2/" - it should be stripped + assert "qwen2/" not in result + # The result should be URL-encoded ARN + assert "arn%3Aaws%3Abedrock" in result or "arn:aws:bedrock" in result + + +def test_qwen2_model_id_extraction_without_qwen2_prefix(): + """Test that model ID extraction doesn't strip qwen2/ when provider is not qwen2""" + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + # Test case: just a model name without qwen2/ prefix + model = "arn:aws:bedrock:us-east-1:123456789012:imported-model/test-qwen2" + provider = "qwen2" + + result = BaseAWSLLM.get_bedrock_model_id( + optional_params={}, + provider=provider, + model=model + ) + + # Result should be encoded ARN + assert "arn" in result.lower() or "aws" in result.lower() + + +def test_qwen2_get_bedrock_model_id_with_various_formats(): + """Test get_bedrock_model_id with various Qwen2 model path formats""" + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + test_cases = [ + { + "model": "qwen2/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-qwen2", + "provider": "qwen2", + "should_not_contain": "qwen2/", + "description": "Qwen2 imported model ARN" + }, + { + "model": "bedrock/qwen2/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-qwen2", + "provider": "qwen2", + "should_not_contain": "qwen2/", + "description": "Bedrock prefixed Qwen2 ARN" + } + ] + + for test_case in test_cases: + result = BaseAWSLLM.get_bedrock_model_id( + optional_params={}, + provider=test_case["provider"], + model=test_case["model"] + ) + + assert test_case["should_not_contain"] not in result, \ + f"Failed for {test_case['description']}: {test_case['should_not_contain']} found in {result}" + diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 0d21c163761..a4da4ebb683 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -79,3 +79,102 @@ def test_chunk_parser_usage_transformation(): assert "usage" in parsed assert parsed["usage"]["input_tokens"] == 10 assert parsed["usage"]["output_tokens"] == 5 + + +def test_remove_ttl_from_cache_control(): + """Ensure ttl field is removed from cache_control in messages.""" + + cfg = AmazonAnthropicClaudeMessagesConfig() + + # Test case 1: Message with cache_control containing ttl + request = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + } + ] + } + + cfg._remove_ttl_from_cache_control(request) + + # Verify ttl is removed but cache_control remains + assert "cache_control" in request["messages"][0]["content"][0] + assert "ttl" not in request["messages"][0]["content"][0]["cache_control"] + assert request["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" + + # Test case 2: Message with multiple content items + request2 = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + }, + { + "type": "text", + "text": "World", + "cache_control": { + "type": "ephemeral", + "ttl": "2h" + } + } + ] + } + ] + } + + cfg._remove_ttl_from_cache_control(request2) + + # Verify ttl is removed from all items + for item in request2["messages"][0]["content"]: + if "cache_control" in item: + assert "ttl" not in item["cache_control"] + + # Test case 3: Message without ttl (should remain unchanged) + request3 = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello", + "cache_control": { + "type": "ephemeral" + } + } + ] + } + ] + } + + cfg._remove_ttl_from_cache_control(request3) + + # Verify cache_control is unchanged + assert request3["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" + + # Test case 4: Empty messages (should not raise error) + request4 = {"messages": []} + cfg._remove_ttl_from_cache_control(request4) + assert request4 == {"messages": []} + + # Test case 5: Request without messages key (should not raise error) + request5 = {} + cfg._remove_ttl_from_cache_control(request5) + assert request5 == {} diff --git a/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py b/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py index 2732bf1595a..021cfaeff5e 100644 --- a/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py +++ b/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py @@ -147,3 +147,14 @@ class TestGeminiImageEditTransformation: headers={}, ) + def test_use_multipart_form_data_returns_false(self) -> None: + """ + Gemini uses JSON requests, not multipart/form-data. + This is critical because httpx sends data differently: + - data=dict sends form-encoded + - json=dict sends JSON + + Without this, Gemini returns: "Invalid JSON payload received. Unexpected token." + """ + assert self.config.use_multipart_form_data() is False + diff --git a/tests/test_litellm/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py b/tests/test_litellm/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py index af07534eb57..c231904e710 100644 --- a/tests/test_litellm/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py @@ -140,6 +140,79 @@ class TestVertexAIGeminiImageEditTransformation: headers={}, ) + def test_validate_environment_with_litellm_params(self) -> None: + """Test validate_environment uses credentials from litellm_params""" + with patch.object( + self.config, "_ensure_access_token", return_value=("test-token", "test-expiry") + ) as mock_token: + with patch.object(self.config, "set_headers", return_value={"Authorization": "Bearer test-token"}) as mock_headers: + litellm_params = { + "vertex_ai_project": "custom-project", + "vertex_ai_credentials": "/path/to/custom/credentials.json", + } + + result = self.config.validate_environment( + headers={"X-Custom": "header"}, + model=self.model, + litellm_params=litellm_params, + api_base=None, + ) + + # Verify that safe_get_vertex_ai_project and safe_get_vertex_ai_credentials were used + mock_token.assert_called_once() + call_kwargs = mock_token.call_args[1] + assert call_kwargs["credentials"] == "/path/to/custom/credentials.json" + assert call_kwargs["project_id"] == "custom-project" + assert result == {"Authorization": "Bearer test-token"} + def test_get_complete_url_from_litellm_params(self) -> None: + """Test vertex_project/vertex_location read from litellm_params first""" + url = self.config.get_complete_url( + model="gemini-2.5-flash", + api_base=None, + litellm_params={ + "vertex_project": "params-project", + "vertex_location": "us-east1", + }, + ) + assert "params-project" in url + assert "us-east1" in url + + def test_get_complete_url_global_location(self) -> None: + """Test global location uses correct base URL without region prefix""" + url = self.config.get_complete_url( + model="gemini-2.5-flash", + api_base=None, + litellm_params={ + "vertex_project": "test-project", + "vertex_location": "global", + }, + ) + assert "aiplatform.googleapis.com" in url + assert "global-aiplatform.googleapis.com" not in url + assert "/locations/global/" in url + + def test_get_complete_url_litellm_params_overrides_env(self) -> None: + """Test litellm_params takes precedence over environment variables""" + with patch.dict( + os.environ, + { + "VERTEXAI_PROJECT": "env-project", + "VERTEXAI_LOCATION": "us-central1", + }, + ): + url = self.config.get_complete_url( + model="gemini-2.5-flash", + api_base=None, + litellm_params={ + "vertex_project": "params-project", + "vertex_location": "eu-west1", + }, + ) + assert "params-project" in url + assert "eu-west1" in url + assert "env-project" not in url + assert "us-central1" not in url + class TestVertexAIImagenImageEditTransformation: def setup_method(self) -> None: diff --git a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py index 7cba03c38c8..b91438b3cac 100644 --- a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py @@ -141,7 +141,22 @@ class TestVertexAIGeminiImageGenerationConfig: ] } } - ] + ], + "usageMetadata": { + "promptTokenCount": 93, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 54, + }, + { + "modality": "IMAGE", + "tokenCount": 39, + } + ], + "candidatesTokenCount": 17, + "totalTokenCount": 110, + } } mock_response.headers = {} @@ -162,6 +177,12 @@ class TestVertexAIGeminiImageGenerationConfig: assert len(result.data) == 1 assert result.data[0].b64_json == "base64_encoded_image_data" assert result.data[0].url is None + assert result.usage.input_tokens == 93 + assert result.usage.input_tokens_details.text_tokens == 54 + assert result.usage.input_tokens_details.image_tokens == 39 + assert result.usage.output_tokens == 17 + assert result.usage.total_tokens == 110 + def test_transform_image_generation_response_multiple_images(self): """Test response transformation with multiple images""" diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index b129b7bab7f..5f2dd387b95 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -72,3 +72,46 @@ def test_vertex_ai_anthropic_web_search_header_in_completion(): # because Anthropic doesn't require it assert "anthropic-beta" not in headers_non_vertex or "web-search" not in headers_non_vertex.get("anthropic-beta", ""), \ "anthropic-beta with web-search should not be present for non-Vertex requests" + + +def test_vertex_ai_anthropic_structured_output_header_not_added(): + """Test that structured output beta headers are NOT added for Vertex AI requests""" + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + config = AnthropicConfig() + + # Test case 1: Vertex request with output_format should NOT add beta header + headers_vertex = {} + optional_params_vertex = { + 'output_format': { + 'type': 'json_schema', + 'json_schema': { + 'name': 'MathResult', + 'schema': {'properties': {'result': {'type': 'integer'}}} + } + }, + 'is_vertex_request': True + } + result_vertex = config.update_headers_with_optional_anthropic_beta(headers_vertex, optional_params_vertex) + + assert "anthropic-beta" not in result_vertex, \ + f"Vertex request should NOT have anthropic-beta header for structured output, got: {result_vertex.get('anthropic-beta')}" + + # Test case 2: Non-Vertex request with output_format SHOULD add beta header + headers_non_vertex = {} + optional_params_non_vertex = { + 'output_format': { + 'type': 'json_schema', + 'json_schema': { + 'name': 'MathResult', + 'schema': {'properties': {'result': {'type': 'integer'}}} + } + }, + 'is_vertex_request': False + } + result_non_vertex = config.update_headers_with_optional_anthropic_beta(headers_non_vertex, optional_params_non_vertex) + + assert "anthropic-beta" in result_non_vertex, \ + "Non-Vertex request SHOULD have anthropic-beta header for structured output" + assert result_non_vertex["anthropic-beta"] == "structured-outputs-2025-11-13", \ + f"Expected 'structured-outputs-2025-11-13', got: {result_non_vertex.get('anthropic-beta')}" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_ui_session_utils.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_ui_session_utils.py index f372f7b181c..35cfbee0d54 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_ui_session_utils.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_ui_session_utils.py @@ -1,7 +1,9 @@ -import pytest +import threading from types import SimpleNamespace from unittest.mock import AsyncMock +import pytest + from litellm.constants import UI_SESSION_TOKEN_TEAM_ID from litellm.proxy._types import UserAPIKeyAuth @@ -90,3 +92,27 @@ async def test_build_effective_auth_contexts_returns_original_when_no_resolution assert contexts == [user_auth] mock_resolve.assert_awaited_once_with(user_auth) + +@pytest.mark.asyncio +async def test_build_effective_auth_contexts_handles_unpicklable_parent_span(monkeypatch): + class DummySpan: + def __init__(self) -> None: + self._lock = threading.RLock() + + parent_span = DummySpan() + user_auth = UserAPIKeyAuth( + team_id=UI_SESSION_TOKEN_TEAM_ID, + user_id="user-span", + parent_otel_span=parent_span, + ) + + mock_resolve = AsyncMock(return_value=["team-span"]) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.ui_session_utils.resolve_ui_session_team_ids", + mock_resolve, + ) + + contexts = await build_effective_auth_contexts(user_auth) + + assert contexts[0].team_id == "team-span" + assert contexts[0].parent_otel_span is parent_span diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 69b0bb27b4b..84d320a0a27 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -1101,3 +1101,91 @@ async def test_bedrock_apply_guardrail_with_only_tool_calls_response(): # Verify that the Bedrock API was NOT called since there's no text to process mock_api_request.assert_not_called() print("✅ apply_guardrail with tool_calls test passed - no API call made") + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_blocked_content_with_masking_enabled(): + """Test that BLOCKED content raises exception even when masking is enabled + + This test verifies the bug fix where previously mask_request_content=True or + mask_response_content=True would bypass all BLOCKED content checks. Now it + properly distinguishes between BLOCKED (raise exception) and ANONYMIZED (apply masking). + """ + + # Create guardrail with masking enabled + guardrail = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + mask_request_content=True, # Masking enabled + mask_response_content=True, # Masking enabled + ) + + # Mock Bedrock response with BLOCKED content (hate speech) + blocked_response = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [ + { + "contentPolicy": { + "filters": [ + { + "type": "HATE", + "confidence": "HIGH", + "action": "BLOCKED", # Should raise exception + } + ] + }, + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "type": "NAME", + "match": "John Doe", + "action": "ANONYMIZED", # Should be masked + } + ] + }, + } + ], + "outputs": [{"text": "Content blocked due to policy violation"}], + } + + mock_bedrock_response = MagicMock() + mock_bedrock_response.status_code = 200 + mock_bedrock_response.json.return_value = blocked_response + + # Mock credentials + mock_credentials = MagicMock() + mock_credentials.access_key = "test-access-key" + mock_credentials.secret_key = "test-secret-key" + mock_credentials.token = None + + request_data = { + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Test message with PII and hate speech"}, + ], + } + + # Mock AWS-related methods + with patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock + ) as mock_post, patch.object( + guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ), patch.object( + guardrail, "_prepare_request", return_value=MagicMock() + ): + mock_post.return_value = mock_bedrock_response + + # Should raise HTTPException for BLOCKED content + with pytest.raises(HTTPException) as exc_info: + await guardrail.make_bedrock_api_request( + source="INPUT", + messages=request_data.get("messages"), + request_data=request_data, + ) + + # Verify exception details + assert exc_info.value.status_code == 400 + assert "Violated guardrail policy" in str(exc_info.value.detail) + + print("✅ BLOCKED content with masking enabled raises exception correctly") + diff --git a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py index 2e7443e889f..0607b0de981 100644 --- a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py @@ -1142,6 +1142,305 @@ def test_get_config_model(): assert hasattr(config_model, "ui_friendly_name") +# ============================================================================ +# MASKING TESTS +# ============================================================================ + + +@pytest.fixture +def pillar_masked_response(): + """Fixture providing a Pillar API response with masked messages.""" + return Response( + json={ + "session_id": "test-session-123", + "flagged": True, + "masked_session_messages": [ + {"role": "user", "content": "My email is [MASKED_EMAIL]"} + ], + "evidence": [ + { + "category": "pii", + "type": "email", + "evidence": "test@example.com", + } + ], + "scanners": { + "jailbreak": False, + "prompt_injection": False, + "pii": True, + "toxic_language": False, + }, + }, + status_code=200, + request=Request( + method="POST", url="https://api.pillar.security/api/v1/protect" + ), + ) + + +@pytest.fixture +def pillar_mask_guardrail(env_setup): + """Fixture providing a PillarGuardrail instance in mask mode.""" + return PillarGuardrail( + guardrail_name="pillar-mask", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + on_flagged_action="mask", + ) + + +@pytest.mark.asyncio +async def test_pre_call_hook_masking_mode( + pillar_mask_guardrail, + sample_request_data, + user_api_key_dict, + dual_cache, + pillar_masked_response, +): + """Test pre-call hook masks content when action is 'mask'.""" + original_messages = sample_request_data["messages"].copy() + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=pillar_masked_response, + ): + result = await pillar_mask_guardrail.async_pre_call_hook( + data=sample_request_data, + cache=dual_cache, + user_api_key_dict=user_api_key_dict, + call_type="completion", + ) + + # Messages should be replaced with masked messages + assert result["messages"] == pillar_masked_response.json()["masked_session_messages"] + assert result["messages"] != original_messages + + +@pytest.mark.asyncio +async def test_pre_call_hook_masking_no_masked_messages( + pillar_mask_guardrail, + sample_request_data, + user_api_key_dict, + dual_cache, +): + """Test masking mode when API doesn't return masked_session_messages.""" + response_no_mask = Response( + json={ + "session_id": "test-session-123", + "flagged": True, + # No masked_session_messages + }, + status_code=200, + request=Request( + method="POST", url="https://api.pillar.security/api/v1/protect" + ), + ) + + original_messages = sample_request_data["messages"].copy() + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=response_no_mask, + ): + result = await pillar_mask_guardrail.async_pre_call_hook( + data=sample_request_data, + cache=dual_cache, + user_api_key_dict=user_api_key_dict, + call_type="completion", + ) + + # Messages should remain unchanged if no masked messages provided + assert result["messages"] == original_messages + + +# ============================================================================ +# CONDITIONAL EXCEPTION DETAILS TESTS +# ============================================================================ + + +@pytest.mark.asyncio +async def test_exception_without_scanners( + sample_request_data, + user_api_key_dict, + dual_cache, + pillar_flagged_response, +): + """Test exception excludes scanners when include_scanners is False.""" + guardrail = PillarGuardrail( + guardrail_name="pillar-no-scanners", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + on_flagged_action="block", + include_scanners=False, + include_evidence=True, + ) + + with pytest.raises(HTTPException) as excinfo: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=pillar_flagged_response, + ): + await guardrail.async_pre_call_hook( + data=sample_request_data, + cache=dual_cache, + user_api_key_dict=user_api_key_dict, + call_type="completion", + ) + + error_detail = excinfo.value.detail + assert "pillar_response" in error_detail + assert "scanners" not in error_detail["pillar_response"] + assert "evidence" in error_detail["pillar_response"] + + +@pytest.mark.asyncio +async def test_exception_without_evidence( + sample_request_data, + user_api_key_dict, + dual_cache, + pillar_flagged_response, +): + """Test exception excludes evidence when include_evidence is False.""" + guardrail = PillarGuardrail( + guardrail_name="pillar-no-evidence", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + on_flagged_action="block", + include_scanners=True, + include_evidence=False, + ) + + with pytest.raises(HTTPException) as excinfo: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=pillar_flagged_response, + ): + await guardrail.async_pre_call_hook( + data=sample_request_data, + cache=dual_cache, + user_api_key_dict=user_api_key_dict, + call_type="completion", + ) + + error_detail = excinfo.value.detail + assert "pillar_response" in error_detail + assert "scanners" in error_detail["pillar_response"] + assert "evidence" not in error_detail["pillar_response"] + + +@pytest.mark.asyncio +async def test_exception_without_scanners_or_evidence( + sample_request_data, + user_api_key_dict, + dual_cache, + pillar_flagged_response, +): + """Test exception excludes both scanners and evidence when both are False.""" + guardrail = PillarGuardrail( + guardrail_name="pillar-minimal", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + on_flagged_action="block", + include_scanners=False, + include_evidence=False, + ) + + with pytest.raises(HTTPException) as excinfo: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=pillar_flagged_response, + ): + await guardrail.async_pre_call_hook( + data=sample_request_data, + cache=dual_cache, + user_api_key_dict=user_api_key_dict, + call_type="completion", + ) + + error_detail = excinfo.value.detail + assert "pillar_response" in error_detail + pillar_response = error_detail["pillar_response"] + assert "scanners" not in pillar_response + assert "evidence" not in pillar_response + assert "session_id" in pillar_response # session_id should always be present + + +# ============================================================================ +# MCP CALL SUPPORT TESTS +# ============================================================================ + + +@pytest.mark.asyncio +async def test_pre_call_hook_mcp_call( + pillar_guardrail_instance, + sample_request_data, + user_api_key_dict, + dual_cache, + pillar_clean_response, +): + """Test pre-call hook works with MCP call type.""" + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=pillar_clean_response, + ): + result = await pillar_guardrail_instance.async_pre_call_hook( + data=sample_request_data, + cache=dual_cache, + user_api_key_dict=user_api_key_dict, + call_type="mcp_call", + ) + + assert result == sample_request_data + + +@pytest.mark.asyncio +async def test_moderation_hook_mcp_call( + pillar_guardrail_instance, + sample_request_data, + user_api_key_dict, + pillar_clean_response, +): + """Test moderation hook works with MCP call type.""" + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=pillar_clean_response, + ): + result = await pillar_guardrail_instance.async_moderation_hook( + data=sample_request_data, + user_api_key_dict=user_api_key_dict, + call_type="mcp_call", + ) + + assert result == sample_request_data + + +@pytest.mark.asyncio +async def test_mcp_call_masking( + pillar_mask_guardrail, + sample_request_data, + user_api_key_dict, + dual_cache, + pillar_masked_response, +): + """Test masking works with MCP call type.""" + original_messages = sample_request_data["messages"].copy() + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=pillar_masked_response, + ): + result = await pillar_mask_guardrail.async_pre_call_hook( + data=sample_request_data, + cache=dual_cache, + user_api_key_dict=user_api_key_dict, + call_type="mcp_call", + ) + + # Messages should be replaced with masked messages + assert result["messages"] == pillar_masked_response.json()["masked_session_messages"] + assert result["messages"] != original_messages + + if __name__ == "__main__": # Run the tests pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 0127be8e7a7..23b3b0287ee 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -14,6 +14,7 @@ from litellm.proxy.health_endpoints._health_endpoints import ( _db_health_readiness_check, db_health_cache, health_services_endpoint, + test_model_connection as health_test_model_connection, ) # Import shared proxy test helpers from conftest @@ -127,6 +128,123 @@ async def test_health_services_endpoint_sqs(status, error_message): mock_instance.async_health_check.assert_awaited_once() +@pytest.mark.asyncio +async def test_test_model_connection_loads_config_from_router(): + """ + Test that /health/test_connection automatically loads model configuration + (including resolved environment variables) from the router when model name is provided. + """ + # Mock request + mock_request = MagicMock() + + # Mock user_api_key_dict + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.token = "test-token" + + # Mock prisma_client + mock_prisma_client = MagicMock() + + # Mock router with model configuration + mock_router = MagicMock() + mock_deployment = { + "model_name": "gpt-4o", + "litellm_params": { + "model": "azure/gpt-4o", + "api_key": "resolved-api-key-from-env", + "api_base": "https://resolved-endpoint.openai.azure.com/", + "api_version": "2024-10-21", + }, + "model_info": {}, + } + mock_router.get_model_list.return_value = [mock_deployment] + + # Mock ModelManagementAuthChecks - patch at the source module since it's imported inside the function + mock_can_user_make_model_call = AsyncMock() + + # Mock litellm.ahealth_check + mock_health_check_result = { + "status": "healthy", + "response_time_ms": 100, + } + mock_ahealth_check = AsyncMock(return_value=mock_health_check_result) + + # Mock run_with_timeout + mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) + + # Mock _update_litellm_params_for_health_check + def mock_update_params(model_info, litellm_params): + # Just return params with messages added + params = litellm_params.copy() + params["messages"] = [{"role": "user", "content": "test"}] + return params + + # Mock _resolve_os_environ_variables + def mock_resolve_os_environ(params): + return params + + with patch( + "litellm.proxy.proxy_server.prisma_client", + mock_prisma_client, + ), patch( + "litellm.proxy.proxy_server.llm_router", + mock_router, + ), patch( + "litellm.proxy.proxy_server.premium_user", + False, + ), patch( + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + mock_can_user_make_model_call, + ), patch( + "litellm.proxy.health_endpoints._health_endpoints.litellm.ahealth_check", + mock_ahealth_check, + ), patch( + "litellm.proxy.health_endpoints._health_endpoints.run_with_timeout", + mock_run_with_timeout, + ), patch( + "litellm.proxy.health_endpoints._health_endpoints._update_litellm_params_for_health_check", + mock_update_params, + ), patch( + "litellm.proxy.health_endpoints._health_endpoints._resolve_os_environ_variables", + mock_resolve_os_environ, + ): + # Call the endpoint with only model name (no credentials) + result = await health_test_model_connection( + request=mock_request, + mode="chat", + litellm_params={"model": "gpt-4o"}, + model_info={}, + user_api_key_dict=mock_user_api_key_dict, + ) + + # Verify router.get_model_list was called with the model name + mock_router.get_model_list.assert_called_once_with(model_name="gpt-4o") + + # Verify that run_with_timeout was called (which wraps ahealth_check) + assert mock_run_with_timeout.called + + # Get the call args to verify merged params + call_args = mock_run_with_timeout.call_args + assert call_args is not None + + # The first arg should be the coroutine from ahealth_check + # We need to check what was passed to ahealth_check + ahealth_check_call_args = mock_ahealth_check.call_args + assert ahealth_check_call_args is not None + model_params = ahealth_check_call_args.kwargs.get("model_params", {}) + + # Verify that config params were loaded and merged + # Note: request params override config params, so model from request is used + assert model_params.get("api_key") == "resolved-api-key-from-env" + assert model_params.get("api_base") == "https://resolved-endpoint.openai.azure.com/" + assert model_params.get("api_version") == "2024-10-21" + assert model_params.get("model") == "gpt-4o" # Request param overrides config param + + # Verify result + assert result["status"] == "success" + assert "result" in result + + @pytest.fixture(scope="function") def proxy_client(monkeypatch): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 500fc67de89..a08fc2cba67 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -3043,3 +3043,236 @@ class TestAddMissingTeamMember: assert set(added_teams) == set( expected_teams_added ), f"Expected teams {expected_teams_added}, but got {added_teams}" + + +class TestSSOReadinessEndpoint: + """Test the /sso/readiness endpoint""" + + @pytest.mark.asyncio + async def test_sso_readiness_no_sso_configured(self): + """Test that readiness returns healthy when no SSO is configured""" + from fastapi.testclient import TestClient + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + client = TestClient(app) + + with patch.dict(os.environ, {}, clear=True): + response = client.get("/sso/readiness") + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + assert data["sso_configured"] is False + assert data["message"] == "No SSO provider configured" + finally: + app.dependency_overrides.clear() + + @pytest.mark.asyncio + async def test_sso_readiness_google_fully_configured(self): + """Test that readiness returns healthy when Google SSO is fully configured""" + from fastapi.testclient import TestClient + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + client = TestClient(app) + + with patch.dict( + os.environ, + { + "GOOGLE_CLIENT_ID": "test-google-client-id", + "GOOGLE_CLIENT_SECRET": "test-google-secret", + }, + clear=True, + ): + response = client.get("/sso/readiness") + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + assert data["sso_configured"] is True + assert data["provider"] == "google" + assert "Google SSO is properly configured" in data["message"] + finally: + app.dependency_overrides.clear() + + @pytest.mark.asyncio + async def test_sso_readiness_google_missing_secret(self): + """Test that readiness returns unhealthy when Google SSO is missing GOOGLE_CLIENT_SECRET""" + from fastapi.testclient import TestClient + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + client = TestClient(app) + + with patch.dict( + os.environ, + {"GOOGLE_CLIENT_ID": "test-google-client-id"}, + clear=True, + ): + response = client.get("/sso/readiness") + + assert response.status_code == 503 + data = response.json()["detail"] + assert data["status"] == "unhealthy" + assert data["sso_configured"] is True + assert data["provider"] == "google" + assert "GOOGLE_CLIENT_SECRET" in data["missing_environment_variables"] + assert "Google SSO is configured but missing required environment variables" in data["message"] + finally: + app.dependency_overrides.clear() + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "env_vars,expected_status,expected_provider,expected_missing_vars", + [ + ( + { + "MICROSOFT_CLIENT_ID": "test-microsoft-client-id", + "MICROSOFT_CLIENT_SECRET": "test-microsoft-secret", + "MICROSOFT_TENANT": "test-tenant", + }, + 200, + "microsoft", + [], + ), + ( + {"MICROSOFT_CLIENT_ID": "test-microsoft-client-id"}, + 503, + "microsoft", + ["MICROSOFT_CLIENT_SECRET", "MICROSOFT_TENANT"], + ), + ], + ) + async def test_sso_readiness_microsoft_configurations( + self, env_vars, expected_status, expected_provider, expected_missing_vars + ): + """Test Microsoft SSO readiness with both fully configured and missing variables""" + from fastapi.testclient import TestClient + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + client = TestClient(app) + + with patch.dict(os.environ, env_vars, clear=True): + response = client.get("/sso/readiness") + + assert response.status_code == expected_status + + if expected_status == 200: + data = response.json() + assert data["sso_configured"] is True + assert data["provider"] == expected_provider + assert data["status"] == "healthy" + assert "Microsoft SSO is properly configured" in data["message"] + else: + data = response.json()["detail"] + assert data["sso_configured"] is True + assert data["provider"] == expected_provider + assert data["status"] == "unhealthy" + assert set(data["missing_environment_variables"]) == set( + expected_missing_vars + ) + finally: + app.dependency_overrides.clear() + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "env_vars,expected_status,expected_provider,expected_missing_vars", + [ + ( + { + "GENERIC_CLIENT_ID": "test-generic-client-id", + "GENERIC_CLIENT_SECRET": "test-generic-secret", + "GENERIC_AUTHORIZATION_ENDPOINT": "https://auth.example.com/authorize", + "GENERIC_TOKEN_ENDPOINT": "https://auth.example.com/token", + "GENERIC_USERINFO_ENDPOINT": "https://auth.example.com/userinfo", + }, + 200, + "generic", + [], + ), + ( + {"GENERIC_CLIENT_ID": "test-generic-client-id"}, + 503, + "generic", + [ + "GENERIC_CLIENT_SECRET", + "GENERIC_AUTHORIZATION_ENDPOINT", + "GENERIC_TOKEN_ENDPOINT", + "GENERIC_USERINFO_ENDPOINT", + ], + ), + ], + ) + async def test_sso_readiness_generic_configurations( + self, env_vars, expected_status, expected_provider, expected_missing_vars + ): + """Test Generic SSO readiness with both fully configured and missing variables""" + from fastapi.testclient import TestClient + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + client = TestClient(app) + + with patch.dict(os.environ, env_vars, clear=True): + response = client.get("/sso/readiness") + + assert response.status_code == expected_status + + if expected_status == 200: + data = response.json() + assert data["sso_configured"] is True + assert data["provider"] == expected_provider + assert data["status"] == "healthy" + assert "Generic SSO is properly configured" in data["message"] + else: + data = response.json()["detail"] + assert data["sso_configured"] is True + assert data["provider"] == expected_provider + assert data["status"] == "unhealthy" + assert set(data["missing_environment_variables"]) == set( + expected_missing_vars + ) + finally: + app.dependency_overrides.clear() diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index ab0faa615b9..c585089c7be 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1884,3 +1884,73 @@ async def test_bedrock_router_passthrough_metadata_initialization(): # Verify response was returned assert result == mock_response + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_adds_headers_to_metadata(): + """ + Test that add_litellm_data_to_request adds headers to metadata for guardrails. + + This test verifies the fix for issue #17477 where guardrails couldn't access + request headers (like User-Agent) on Bedrock pass-through endpoints. + + The fix ensures headers are available in data["metadata"]["headers"] so + guardrails can validate User-Agent, API keys, and other header-based checks. + """ + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy._types import UserAPIKeyAuth + + # Create mock request with headers including User-Agent + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = MagicMock() + mock_request.url.path = "/bedrock/model/my-model/converse" + mock_request.headers = Headers( + { + "content-type": "application/json", + "user-agent": "claude-cli/2.0.69 (external, cli)", + "authorization": "Bearer sk-test-key", + "x-custom-header": "test-value", + } + ) + mock_request.query_params = QueryParams({}) + + # Create mock user API key dict + mock_user_api_key_dict = UserAPIKeyAuth() + + # Create mock proxy config + mock_proxy_config = MagicMock() + mock_proxy_config.pass_through_endpoints = [] + + # Initial data dict (simulating Bedrock pass-through) + data = { + "model": "my-bedrock-model", + "messages": [{"role": "user", "content": "Hello"}], + } + + # Call add_litellm_data_to_request + result = await add_litellm_data_to_request( + data=data, + request=mock_request, + user_api_key_dict=mock_user_api_key_dict, + proxy_config=mock_proxy_config, + general_settings={}, + version="1.0", + ) + + # Verify headers are added to metadata for guardrails + assert "metadata" in result, "metadata should be present in result" + assert "headers" in result["metadata"], "headers should be present in metadata" + assert isinstance( + result["metadata"]["headers"], dict + ), "headers should be a dictionary" + + # Verify specific headers are accessible (important for guardrails) + headers = result["metadata"]["headers"] + assert ( + "user-agent" in headers or "User-Agent" in headers + ), "User-Agent header should be accessible in metadata" + + # Also verify proxy_server_request has headers (original location) + assert "proxy_server_request" in result + assert "headers" in result["proxy_server_request"] diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index b64706e5ac2..e08f2ad98dd 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -201,6 +201,7 @@ ignored_keys = [ "metadata.usage_object", "metadata.cold_storage_object_key", "metadata.additional_usage_values.prompt_tokens_details.cache_creation_tokens", + "metadata.litellm_overhead_time_ms", ] MODEL_LIST = [ diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 5adf0bb1a3d..69b7e504184 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -24,7 +24,12 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _sanitize_request_body_for_spend_logs_payload, get_logging_payload, ) -from litellm.types.utils import StandardLoggingPayload +from litellm.types.utils import ( + StandardLoggingHiddenParams, + StandardLoggingMetadata, + StandardLoggingModelInformation, + StandardLoggingPayload, +) def test_sanitize_request_body_for_spend_logs_payload_basic(): @@ -632,3 +637,216 @@ def test_get_logging_payload_includes_agent_id_from_kwargs(): assert payload["agent_id"] == test_agent_id, f"Expected agent_id '{test_agent_id}', got '{payload.get('agent_id')}'" + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_includes_overhead_in_spend_logs_metadata(): + """ + Test that get_logging_payload extracts litellm_overhead_time_ms from hidden_params + and stores it in spend_logs_metadata within the metadata JSON. + """ + test_overhead_ms = 123.45 + + # Create StandardLoggingPayload with hidden_params containing overhead + standard_logging_payload = StandardLoggingPayload( + id="test-id-123", + call_type="completion", + stream=False, + response_cost=0.001, + status="success", + total_tokens=100, + prompt_tokens=50, + completion_tokens=50, + startTime=1234567890.0, + endTime=1234567891.0, + completionStartTime=None, + model_map_information=StandardLoggingModelInformation( + model_map_key="gpt-3.5-turbo", model_map_value=None + ), + model="gpt-3.5-turbo", + model_id="model-123", + model_group="openai", + custom_llm_provider="openai", + api_base="https://api.openai.com", + metadata=StandardLoggingMetadata( + user_api_key_hash="test_hash", + user_api_key_alias=None, + user_api_key_team_id=None, + user_api_key_org_id=None, + user_api_key_user_id=None, + user_api_key_team_alias=None, + spend_logs_metadata=None, + requester_ip_address=None, + requester_metadata=None, + user_api_key_end_user_id=None, + ), + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address=None, + messages=[], + response={}, + error_str=None, + model_parameters={}, + hidden_params=StandardLoggingHiddenParams( + model_id="model-123", + cache_key=None, + api_base="https://api.openai.com", + response_cost="0.001", + litellm_overhead_time_ms=test_overhead_ms, + additional_headers=None, + batch_models=None, + litellm_model_name=None, + usage_object=None, + ), + ) + + kwargs = { + "model": "gpt-3.5-turbo", + "litellm_params": { + "metadata": { + "user_api_key": "sk-test-key", + } + }, + "standard_logging_object": standard_logging_payload, + } + + response_obj = { + "id": "test-response-123", + "choices": [{"message": {"content": "Hello!"}}], + "usage": { + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + }, + } + + start_time = datetime.datetime.now(timezone.utc) + end_time = datetime.datetime.now(timezone.utc) + + payload = get_logging_payload( + kwargs=kwargs, + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + ) + + # Parse the metadata JSON string + metadata_json = payload.get("metadata") + assert metadata_json is not None, "metadata should not be None" + + metadata = json.loads(metadata_json) + + # Verify overhead is stored directly in metadata + assert ( + metadata.get("litellm_overhead_time_ms") == test_overhead_ms + ), f"Expected overhead '{test_overhead_ms}', got '{metadata.get('litellm_overhead_time_ms')}'" + + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_handles_missing_overhead_gracefully(): + """ + Test that get_logging_payload handles missing overhead gracefully + (backward compatibility - when overhead is not present, it should not break). + """ + # Create StandardLoggingPayload WITHOUT overhead in hidden_params + standard_logging_payload = StandardLoggingPayload( + id="test-id-456", + call_type="completion", + stream=False, + response_cost=0.001, + status="success", + total_tokens=100, + prompt_tokens=50, + completion_tokens=50, + startTime=1234567890.0, + endTime=1234567891.0, + completionStartTime=None, + model_map_information=StandardLoggingModelInformation( + model_map_key="gpt-3.5-turbo", model_map_value=None + ), + model="gpt-3.5-turbo", + model_id="model-123", + model_group="openai", + custom_llm_provider="openai", + api_base="https://api.openai.com", + metadata=StandardLoggingMetadata( + user_api_key_hash="test_hash", + user_api_key_alias=None, + user_api_key_team_id=None, + user_api_key_org_id=None, + user_api_key_user_id=None, + user_api_key_team_alias=None, + spend_logs_metadata=None, + requester_ip_address=None, + requester_metadata=None, + user_api_key_end_user_id=None, + ), + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address=None, + messages=[], + response={}, + error_str=None, + model_parameters={}, + hidden_params=StandardLoggingHiddenParams( + model_id="model-123", + cache_key=None, + api_base="https://api.openai.com", + response_cost="0.001", + litellm_overhead_time_ms=None, # No overhead + additional_headers=None, + batch_models=None, + litellm_model_name=None, + usage_object=None, + ), + ) + + kwargs = { + "model": "gpt-3.5-turbo", + "litellm_params": { + "metadata": { + "user_api_key": "sk-test-key", + } + }, + "standard_logging_object": standard_logging_payload, + } + + response_obj = { + "id": "test-response-456", + "choices": [{"message": {"content": "Hello!"}}], + "usage": { + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + }, + } + + start_time = datetime.datetime.now(timezone.utc) + end_time = datetime.datetime.now(timezone.utc) + + # Should not raise an exception + payload = get_logging_payload( + kwargs=kwargs, + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + ) + + # Parse the metadata JSON string + metadata_json = payload.get("metadata") + assert metadata_json is not None, "metadata should not be None" + + metadata = json.loads(metadata_json) + + # When overhead is None, litellm_overhead_time_ms should be None or not present + assert ( + metadata.get("litellm_overhead_time_ms") is None + ), "litellm_overhead_time_ms should be None when overhead is not provided" + diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 22a9d5e647b..05362e50d6a 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -15,6 +15,7 @@ import httpx import pytest import yaml from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient sys.path.insert( @@ -196,6 +197,32 @@ def test_restructure_ui_html_files_handles_nested_routes(tmp_path): ) +def test_ui_extensionless_route_requires_restructure(tmp_path): + """Regression for non-root fallback: /ui/login expects login/index.html.""" + + from litellm.proxy import proxy_server + + ui_root = tmp_path / "ui" + ui_root.mkdir() + (ui_root / "index.html").write_text("index") + (ui_root / "login.html").write_text("login") + + fastapi_app = FastAPI() + fastapi_app.mount( + "/ui", StaticFiles(directory=str(ui_root), html=True), name="ui" + ) + client = TestClient(fastapi_app) + + assert client.get("/ui/login.html").status_code == 200 + assert client.get("/ui/login").status_code == 404 + + proxy_server._restructure_ui_html_files(str(ui_root)) + + response = client.get("/ui/login") + assert response.status_code == 200 + assert "login" in response.text + + @pytest.mark.asyncio async def test_initialize_scheduled_jobs_credentials(monkeypatch): """ @@ -2609,6 +2636,30 @@ async def test_init_sso_settings_in_db_empty_settings(): assert uppercased_settings == {} +def test_update_config_fields_uppercases_env_vars(monkeypatch): + """ + Ensure environment variables pulled from DB are uppercased when applied so + integrations like Datadog that expect uppercase env keys can read them. + """ + from litellm.proxy.proxy_server import ProxyConfig + + for key in ["DD_API_KEY", "DD_SITE", "dd_api_key", "dd_site"]: + monkeypatch.delenv(key, raising=False) + + proxy_config = ProxyConfig() + updated_config = proxy_config._update_config_fields( + current_config={}, + param_name="environment_variables", + db_param_value={"dd_api_key": "test-api-key", "dd_site": "us5.datadoghq.com"}, + ) + + env_vars = updated_config.get("environment_variables", {}) + assert env_vars["DD_API_KEY"] == "test-api-key" + assert env_vars["DD_SITE"] == "us5.datadoghq.com" + assert os.environ.get("DD_API_KEY") == "test-api-key" + assert os.environ.get("DD_SITE") == "us5.datadoghq.com" + + def test_get_prompt_spec_for_db_prompt_with_versions(): """ Test that _get_prompt_spec_for_db_prompt correctly converts database prompts diff --git a/tests/test_litellm/test_lazy_imports.py b/tests/test_litellm/test_lazy_imports.py new file mode 100644 index 00000000000..0eaedaab601 --- /dev/null +++ b/tests/test_litellm/test_lazy_imports.py @@ -0,0 +1,248 @@ +"""Simple tests for lazy import functionality.""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm._lazy_imports import ( + COST_CALCULATOR_NAMES, + LITELLM_LOGGING_NAMES, + UTILS_NAMES, + TOKEN_COUNTER_NAMES, + CACHING_NAMES, + BEDROCK_TYPES_NAMES, + TYPES_UTILS_NAMES, + LLM_CLIENT_CACHE_NAMES, + HTTP_HANDLER_NAMES, + _lazy_import_cost_calculator, + _lazy_import_litellm_logging, + _lazy_import_utils, + _lazy_import_token_counter, + _lazy_import_bedrock_types, + _lazy_import_types_utils, + _lazy_import_caching, + _lazy_import_llm_client_cache, + _lazy_import_http_handlers, + DOTPROMPT_NAMES, + _lazy_import_dotprompt, + LLM_CONFIG_NAMES, + _lazy_import_llm_configs, + TYPES_NAMES, + _lazy_import_types, +) + + +def _clear_names_from_globals(names: tuple): + """Clear all names from litellm globals.""" + for name in names: + if name in litellm.__dict__: + del litellm.__dict__[name] + + +def _verify_only_requested_name_imported(name: str, all_names: tuple): + """Verify that only the requested name is in globals, not the others.""" + for other_name in all_names: + if other_name != name: + assert other_name not in litellm.__dict__, f"{other_name} should not be imported when importing {name}" + + +def test_cost_calculator_lazy_imports(): + """Test that all cost calculator functions can be lazy imported.""" + # Test each name individually - only that name should be imported + for name in COST_CALCULATOR_NAMES: + # Clear all names before importing just one + _clear_names_from_globals(COST_CALCULATOR_NAMES) + + func = _lazy_import_cost_calculator(name) + assert func is not None + assert callable(func) + assert name in litellm.__dict__ + + # Verify only the requested name is in globals, not the others + _verify_only_requested_name_imported(name, COST_CALCULATOR_NAMES) + + +def test_litellm_logging_lazy_imports(): + """Test that all litellm_logging items can be lazy imported.""" + # Test each name individually - only that name should be imported + for name in LITELLM_LOGGING_NAMES: + # Clear all names before importing just one + _clear_names_from_globals(LITELLM_LOGGING_NAMES) + + item = _lazy_import_litellm_logging(name) + assert item is not None + assert name in litellm.__dict__ + + # Verify only the requested name is in globals, not the others + _verify_only_requested_name_imported(name, LITELLM_LOGGING_NAMES) + + +def test_utils_lazy_imports(): + """Test that all utils functions can be lazy imported.""" + # Test each name individually - only that name should be imported + for name in UTILS_NAMES: + # Clear all names before importing just one + _clear_names_from_globals(UTILS_NAMES) + + attr = _lazy_import_utils(name) + assert attr is not None + assert name in litellm.__dict__ + + # Verify only the requested name is in globals, not the others + _verify_only_requested_name_imported(name, UTILS_NAMES) + + +def test_caching_lazy_imports(): + """Test that all caching classes can be lazy imported.""" + # Test each name individually - only that name should be imported + for name in CACHING_NAMES: + # Clear all names before importing just one + _clear_names_from_globals(CACHING_NAMES) + + cls = _lazy_import_caching(name) + assert cls is not None + assert name in litellm.__dict__ + + # Verify only the requested name is in globals, not the others + _verify_only_requested_name_imported(name, CACHING_NAMES) + + +def test_token_counter_lazy_imports(): + """Test that token counter utilities can be lazy imported.""" + for name in TOKEN_COUNTER_NAMES: + _clear_names_from_globals(TOKEN_COUNTER_NAMES) + + func = _lazy_import_token_counter(name) + assert func is not None + assert name in litellm.__dict__ + + _verify_only_requested_name_imported(name, TOKEN_COUNTER_NAMES) + + +def test_bedrock_types_lazy_imports(): + """Test that Bedrock type aliases can be lazy imported.""" + for name in BEDROCK_TYPES_NAMES: + _clear_names_from_globals(BEDROCK_TYPES_NAMES) + + alias = _lazy_import_bedrock_types(name) + assert alias is not None + assert name in litellm.__dict__ + + _verify_only_requested_name_imported(name, BEDROCK_TYPES_NAMES) + + +def test_types_utils_lazy_imports(): + """Test that common types.utils symbols can be lazy imported.""" + for name in TYPES_UTILS_NAMES: + _clear_names_from_globals(TYPES_UTILS_NAMES) + + obj = _lazy_import_types_utils(name) + assert obj is not None + assert name in litellm.__dict__ + + _verify_only_requested_name_imported(name, TYPES_UTILS_NAMES) + + +def test_llm_client_cache_lazy_imports(): + """Test that LLM client cache class and singleton can be lazy imported.""" + for name in LLM_CLIENT_CACHE_NAMES: + _clear_names_from_globals(LLM_CLIENT_CACHE_NAMES) + + obj = _lazy_import_llm_client_cache(name) + assert obj is not None + assert name in litellm.__dict__ + + _verify_only_requested_name_imported(name, LLM_CLIENT_CACHE_NAMES) + + +def test_http_handler_lazy_imports(): + """Test that HTTP handler singletons can be lazy imported.""" + for name in HTTP_HANDLER_NAMES: + _clear_names_from_globals(HTTP_HANDLER_NAMES) + + handler = _lazy_import_http_handlers(name) + assert handler is not None + assert name in litellm.__dict__ + + _verify_only_requested_name_imported(name, HTTP_HANDLER_NAMES) + + +def test_dotprompt_lazy_imports(): + """Test that dotprompt globals can be lazy imported.""" + for name in DOTPROMPT_NAMES: + _clear_names_from_globals(DOTPROMPT_NAMES) + + obj = _lazy_import_dotprompt(name) + assert name in litellm.__dict__ + + # Only the setter must be callable; others may be None by default + if name == "set_global_prompt_directory": + assert callable(obj), f"{name} should be callable" + + _verify_only_requested_name_imported(name, DOTPROMPT_NAMES) + + +def test_unknown_attribute_raises_error(): + """Test that unknown attributes raise AttributeError.""" + with pytest.raises(AttributeError): + _lazy_import_cost_calculator("unknown") + + with pytest.raises(AttributeError): + _lazy_import_litellm_logging("unknown") + + with pytest.raises(AttributeError): + _lazy_import_utils("unknown") + + with pytest.raises(AttributeError): + _lazy_import_caching("unknown") + + with pytest.raises(AttributeError): + _lazy_import_token_counter("unknown") + + with pytest.raises(AttributeError): + _lazy_import_llm_client_cache("unknown") + + with pytest.raises(AttributeError): + _lazy_import_bedrock_types("unknown") + + with pytest.raises(AttributeError): + _lazy_import_types_utils("unknown") + + with pytest.raises(AttributeError): + _lazy_import_llm_configs("unknown") + + with pytest.raises(AttributeError): + _lazy_import_types("unknown") + + +def test_llm_config_lazy_imports(): + """Test that LLM config classes can be lazy imported.""" + for name in LLM_CONFIG_NAMES: + _clear_names_from_globals(LLM_CONFIG_NAMES) + + obj = _lazy_import_llm_configs(name) + assert obj is not None + assert name in litellm.__dict__ + # Config classes should be classes/types + assert isinstance(obj, type), f"{name} should be a class" + + _verify_only_requested_name_imported(name, LLM_CONFIG_NAMES) + + +def test_types_lazy_imports(): + """Test that type classes can be lazy imported.""" + for name in TYPES_NAMES: + _clear_names_from_globals(TYPES_NAMES) + + obj = _lazy_import_types(name) + assert obj is not None + assert name in litellm.__dict__ + # Type classes should be classes/types + assert isinstance(obj, type), f"{name} should be a class" + + _verify_only_requested_name_imported(name, TYPES_NAMES) + diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index b9485f6a317..7dba7c99916 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2603,6 +2603,7 @@ class TestIsCachedMessage: message = {"role": "user", "content": []} assert is_cached_message(message) is False + @pytest.mark.asyncio class TestProxyLoggingBudgetAlerts: """Test budget_alerts method in ProxyLogging class.""" @@ -2727,3 +2728,30 @@ class TestProxyLoggingBudgetAlerts: proxy_logging.email_logging_instance.budget_alerts.assert_called_once_with( type=alert_type, user_info=user_info ) + + +def test_azure_ai_claude_provider_config(): + """Test that Azure AI Claude models return AzureAnthropicConfig for proper tool transformation.""" + from litellm import AzureAnthropicConfig, AzureAIStudioConfig + from litellm.utils import ProviderConfigManager + + # Claude models should return AzureAnthropicConfig + config = ProviderConfigManager.get_provider_chat_config( + model="claude-sonnet-4-5", + provider=LlmProviders.AZURE_AI, + ) + assert isinstance(config, AzureAnthropicConfig) + + # Test case-insensitive matching + config = ProviderConfigManager.get_provider_chat_config( + model="Claude-Opus-4", + provider=LlmProviders.AZURE_AI, + ) + assert isinstance(config, AzureAnthropicConfig) + + # Non-Claude models should return AzureAIStudioConfig + config = ProviderConfigManager.get_provider_chat_config( + model="mistral-large", + provider=LlmProviders.AZURE_AI, + ) + assert isinstance(config, AzureAIStudioConfig) diff --git a/ui/litellm-dashboard/public/assets/logos/milvus.svg b/ui/litellm-dashboard/public/assets/logos/milvus.svg new file mode 100644 index 00000000000..76154467b4b --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/milvus.svg @@ -0,0 +1 @@ +milvus-horizontal-color \ No newline at end of file diff --git a/ui/litellm-dashboard/public/assets/logos/pydantic.svg b/ui/litellm-dashboard/public/assets/logos/pydantic.svg new file mode 100644 index 00000000000..0ff8e5c44c7 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/pydantic.svg @@ -0,0 +1,5 @@ + + + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.test.tsx index b165b71be7e..428f52dd98c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.test.tsx @@ -1,7 +1,7 @@ /* @vitest-environment jsdom */ -import { render } from "@testing-library/react"; -import { describe, it, expect, vi } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import ModelsAndEndpointsView from "./ModelsAndEndpointsView"; // Minimal stubs to avoid Next.js router and network usage during render @@ -57,21 +57,40 @@ vi.mock("@/app/(dashboard)/hooks/useTeams", () => ({ }), })); +const mockUseModelsInfo = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ + useModelsInfo: () => mockUseModelsInfo(), +})); + +const mockUseUISettings = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/uiSettings/useUISettings", () => ({ + useUISettings: () => mockUseUISettings(), +})); + const createQueryClient = () => new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } }, }); describe("ModelsAndEndpointsView", () => { - it("should render the models and endpoints view", async () => { - // JSDOM polyfill for libraries expecting ResizeObserver (e.g., recharts) - // Note: ResizeObserver is now globally mocked in setupTests.ts, but keeping this for backwards compatibility + beforeEach(() => { + mockUseModelsInfo.mockReturnValue({ + data: { data: [] }, + isLoading: false, + refetch: vi.fn(), + }); + mockUseUISettings.mockReturnValue({ + data: { values: {} }, + }); // eslint-disable-next-line @typescript-eslint/no-explicit-any (global as any).ResizeObserver = class { observe() {} unobserve() {} disconnect() {} }; + }); + + it("should render the models and endpoints view", async () => { const queryClient = createQueryClient(); const { findByText } = render( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx index 7b6199bc88d..ffb92d1897f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx @@ -36,7 +36,7 @@ import ModelAnalyticsTab from "@/app/(dashboard)/models-and-endpoints/components import ModelRetrySettingsTab from "@/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab"; import PriceDataManagementTab from "@/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab"; import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; -import { all_admin_roles, internalUserRoles } from "@/utils/roles"; +import { all_admin_roles, internalUserRoles, isProxyAdminRole, isUserTeamAdminForAnyTeam } from "@/utils/roles"; import HealthCheckComponent from "../../../components/model_dashboard/HealthCheckComponent"; import ModelGroupAliasSettings from "../../../components/model_group_alias_settings"; import NotificationsManager from "../../../components/molecules/notifications_manager"; @@ -161,8 +161,13 @@ const ModelsAndEndpointsView: React.FC = ({ const credentialsList = credentialsResponse?.credentials || []; const { data: uiSettings } = useUISettings(accessToken || ""); + const isProxyAdmin = userRole && isProxyAdminRole(userRole); const isInternalUser = userRole && internalUserRoles.includes(userRole); - const shouldHideAddModelTab = isInternalUser && uiSettings?.values?.disable_model_add_for_internal_users === true; + const isUserTeamAdmin = userID && isUserTeamAdminForAnyTeam(teams, userID); + const addModelDisabledForInternalUsers = + isInternalUser && uiSettings?.values?.disable_model_add_for_internal_users === true; + // Hide tab if user is NOT a proxy admin AND (internal user with setting enabled OR not a team admin) + const shouldHideAddModelTab = !isProxyAdmin && (addModelDisabledForInternalUsers || !isUserTeamAdmin); const setProviderModelsFn = (provider: Providers) => { const _providerModels = getProviderModels(provider, modelMap); diff --git a/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx b/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx index f44d71cbada..f4e0137bd06 100644 --- a/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx +++ b/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect } from "react"; -import { Modal, Form, message, Select } from "antd"; +import { Modal, Form, message, Select, Input } from "antd"; import { Button } from "@tremor/react"; import { createAgentCall, getAgentCreateMetadata, AgentCreateInfo } from "../networking"; import AgentFormFields from "./agent_form_fields"; @@ -57,6 +57,26 @@ const AddAgentForm: React.FC = ({ if (agentType === "a2a") { agentData = buildAgentDataFromForm(values); + } else if (selectedAgentTypeInfo?.use_a2a_form_fields) { + // A2A-compatible agents use the standard A2A form builder + // but need to add litellm_params from the agent type config + agentData = buildAgentDataFromForm(values); + + // Merge litellm_params_template + if (selectedAgentTypeInfo.litellm_params_template) { + agentData.litellm_params = { + ...agentData.litellm_params, + ...selectedAgentTypeInfo.litellm_params_template, + }; + } + + // Add credential fields to litellm_params + for (const field of selectedAgentTypeInfo.credential_fields) { + const value = values[field.key]; + if (value && field.include_in_litellm_params !== false) { + agentData.litellm_params[field.key] = value; + } + } } else if (selectedAgentTypeInfo) { agentData = buildDynamicAgentData(values, selectedAgentTypeInfo); } @@ -167,6 +187,35 @@ const AddAgentForm: React.FC = ({
{agentType === "a2a" ? ( + ) : selectedAgentTypeInfo?.use_a2a_form_fields ? ( + // A2A-compatible agents (like Pydantic AI) use full A2A form fields + // plus any additional credential fields + <> + + {selectedAgentTypeInfo.credential_fields.length > 0 && ( +
+

+ {selectedAgentTypeInfo.agent_type_display_name} Settings +

+ {selectedAgentTypeInfo.credential_fields.map((field) => ( + + {field.field_type === "password" ? ( + + ) : ( + + )} + + ))} +
+ )} + ) : selectedAgentTypeInfo ? ( ) : null} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 65c41c5aab4..ddabc6f5212 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -215,6 +215,7 @@ export interface AgentCreateInfo { credential_fields: AgentCredentialFieldMetadata[]; litellm_params_template?: Record | null; model_template?: string | null; + use_a2a_form_fields?: boolean; } export interface PublicModelHubInfo { diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index b245119cfee..e23bc88dda8 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -20,8 +20,8 @@ Object.defineProperty(window, "matchMedia", { describe("KeyEditView", () => { const MOCK_KEY_DATA: KeyResponse = { - token: "40b7608ea43423400d5b82bb5ee11042bfb2ed4655f05b5992b5abbc2f294931", - token_id: "40b7608ea43423400d5b82bb5ee11042bfb2ed4655f05b5992b5abbc2f294931", + token: "test-token-123", + token_id: "test-token-123", key_name: "sk-...TUuw", key_alias: "asdasdas", spend: 0, diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx index bd6e560a1c9..8137a70124f 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx @@ -5,8 +5,8 @@ import KeyInfoView from "./key_info_view"; describe("KeyInfoView", () => { const MOCK_KEY_DATA: KeyResponse = { - token: "40b7608ea43423400d5b82bb5ee11042bfb2ed4655f05b5992b5abbc2f294931", - token_id: "40b7608ea43423400d5b82bb5ee11042bfb2ed4655f05b5992b5abbc2f294931", + token: "test-token-123", + token_id: "test-token-123", key_name: "sk-...TUuw", key_alias: "asdasdas", spend: 0, diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx new file mode 100644 index 00000000000..97d18e33640 --- /dev/null +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx @@ -0,0 +1,27 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import { CredentialItem } from "../networking"; +import VectorStoreForm from "./VectorStoreForm"; + +vi.mock("../networking"); + +describe("VectorStoreForm", () => { + it("should render the form when visible", () => { + const mockOnCancel = vi.fn(); + const mockOnSuccess = vi.fn(); + const mockAccessToken = "test-token"; + const mockCredentials: CredentialItem[] = []; + + render( + , + ); + + expect(screen.getByText("Add New Vector Store")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.tsx index d1cd3c5e443..8be879b2239 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.tsx @@ -1,4 +1,4 @@ -import React, { useState } from "react"; +import React, { useState, useEffect } from "react"; import { TextInput, Button as TremorButton } from "@tremor/react"; import { Modal, Form, Select, Tooltip, Input, Alert } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; @@ -10,6 +10,7 @@ import { getProviderSpecificFields, VectorStoreFieldConfig, } from "../vector_store_providers"; +import { fetchAvailableModels, ModelGroup } from "../playground/llm_calls/fetch_models"; import NotificationsManager from "../molecules/notifications_manager"; interface VectorStoreFormProps { @@ -30,6 +31,24 @@ const VectorStoreForm: React.FC = ({ const [form] = Form.useForm(); const [metadataJson, setMetadataJson] = useState("{}"); const [selectedProvider, setSelectedProvider] = useState("bedrock"); + const [modelInfo, setModelInfo] = useState([]); + + useEffect(() => { + if (!accessToken) return; + + const loadModels = async () => { + try { + const uniqueModels = await fetchAvailableModels(accessToken); + if (uniqueModels.length > 0) { + setModelInfo(uniqueModels); + } + } catch (error) { + console.error("Error fetching model info:", error); + } + }; + + loadModels(); + }, [accessToken]); const handleCreate = async (formValues: any) => { if (!accessToken) return; @@ -207,23 +226,62 @@ const VectorStoreForm: React.FC = ({ {/* Provider-specific fields */} - {getProviderSpecificFields(selectedProvider).map((field: VectorStoreFieldConfig) => ( - - {field.label}{" "} - - - - - } - name={field.name} - rules={field.required ? [{ required: true, message: `Please input the ${field.label.toLowerCase()}` }] : []} - > - - - ))} + {getProviderSpecificFields(selectedProvider).map((field: VectorStoreFieldConfig) => { + if (field.type === "select") { + const embeddingModels = modelInfo + .filter((option: ModelGroup) => option.mode === "embedding") + .map((option: ModelGroup) => ({ + value: option.model_group, + label: option.model_group, + })); + + return ( + + {field.label}{" "} + + + + + } + name={field.name} + rules={ + field.required ? [{ required: true, message: `Please select the ${field.label.toLowerCase()}` }] : [] + } + > +